From 48f27655a6751723ed44737838d9820a02b500c4 Mon Sep 17 00:00:00 2001 From: Ralph Castain Date: Sun, 10 Jun 2018 15:27:42 -0700 Subject: [PATCH] Sync to PMIx v3.0rc and add ext4x Sync to the draft rc for PMIx v3.0. Add an external component for PMIx master, which is at v4.0 Signed-off-by: Ralph Castain --- config/opal_check_pmi.m4 | 15 +- opal/mca/pmix/ext4x/Makefile.am | 54 + opal/mca/pmix/ext4x/configure.m4 | 64 + opal/mca/pmix/ext4x/ext4x.c | 1829 +++++++++++++++++ opal/mca/pmix/ext4x/ext4x.h | 351 ++++ opal/mca/pmix/ext4x/ext4x_client.c | 1664 +++++++++++++++ opal/mca/pmix/ext4x/ext4x_component.c | 153 ++ opal/mca/pmix/ext4x/ext4x_local.c | 27 + opal/mca/pmix/ext4x/ext4x_server_north.c | 1312 ++++++++++++ opal/mca/pmix/ext4x/ext4x_server_south.c | 790 +++++++ opal/mca/pmix/ext4x/help-pmix-ext4x.txt | 32 + opal/mca/pmix/pmix3x/pmix/Makefile.am | 4 +- opal/mca/pmix/pmix3x/pmix/NEWS | 55 +- opal/mca/pmix/pmix3x/pmix/VERSION | 10 +- .../mca/pmix/pmix3x/pmix/bindings/Makefile.am | 22 + opal/mca/pmix/pmix3x/pmix/bindings/README | 19 + .../pmix3x/pmix/bindings/python/Makefile.am | 39 + .../pmix/pmix3x/pmix/bindings/python/README | 49 + opal/mca/pmix/pmix3x/pmix/config/pmix.m4 | 50 +- .../pmix3x/pmix/config/pmix_setup_hwloc.m4 | 106 + .../pmix3x/pmix/contrib/perf_tools/Makefile | 28 +- .../pmix/pmix3x/pmix/contrib/perf_tools/pmi.h | 1 + .../pmix3x/pmix/contrib/perf_tools/pmi2.c | 7 +- .../pmix/contrib/perf_tools/pmi_intra_perf.c | 25 +- .../pmix3x/pmix/contrib/perf_tools/pmix.c | 97 +- opal/mca/pmix/pmix3x/pmix/contrib/pmix.spec | 2 +- .../mca/pmix/pmix3x/pmix/examples/Makefile.am | 8 +- .../mca/pmix/pmix3x/pmix/examples/debuggerd.c | 2 +- opal/mca/pmix/pmix3x/pmix/examples/dynamic.c | 12 +- opal/mca/pmix/pmix3x/pmix/examples/pub.c | 6 +- opal/mca/pmix/pmix3x/pmix/examples/pubi.c | 173 ++ opal/mca/pmix/pmix3x/pmix/examples/server.c | 20 +- opal/mca/pmix/pmix3x/pmix/include/pmix.h | 81 +- .../pmix/pmix3x/pmix/include/pmix_common.h.in | 542 +++-- .../pmix/pmix3x/pmix/include/pmix_server.h | 127 +- opal/mca/pmix/pmix3x/pmix/include/pmix_tool.h | 32 +- opal/mca/pmix/pmix3x/pmix/src/Makefile.am | 3 +- .../pmix3x/pmix/src/class/Makefile.include | 4 +- .../pmix/pmix3x/pmix/src/class/pmix_bitmap.c | 412 ++++ .../pmix/pmix3x/pmix/src/class/pmix_bitmap.h | 259 +++ .../pmix/pmix3x/pmix/src/class/pmix_hotel.c | 2 +- .../pmix/pmix3x/pmix/src/class/pmix_hotel.h | 2 +- opal/mca/pmix/pmix3x/pmix/src/client/pmi2.c | 15 +- .../pmix/pmix3x/pmix/src/client/pmix_client.c | 227 +- .../pmix/src/client/pmix_client_connect.c | 164 +- .../pmix3x/pmix/src/client/pmix_client_get.c | 3 +- .../pmix3x/pmix/src/common/pmix_control.c | 135 +- .../pmix/pmix3x/pmix/src/common/pmix_data.c | 7 +- .../pmix/pmix3x/pmix/src/common/pmix_iof.c | 43 +- .../pmix/pmix3x/pmix/src/common/pmix_iof.h | 11 +- .../pmix/pmix3x/pmix/src/common/pmix_log.c | 166 +- .../pmix/pmix3x/pmix/src/common/pmix_query.c | 162 +- .../pmix3x/pmix/src/common/pmix_security.c | 8 +- .../pmix3x/pmix/src/common/pmix_strings.c | 8 + .../pmix/pmix3x/pmix/src/event/pmix_event.h | 45 +- .../pmix/src/event/pmix_event_notification.c | 475 +++-- .../pmix/src/event/pmix_event_registration.c | 163 +- .../pmix3x/pmix/src/hwloc/Makefile.include | 15 + .../pmix3x/pmix/src/hwloc/hwloc-internal.h | 60 + opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc.c | 741 +++++++ .../pmix3x/pmix/src/include/pmix_globals.c | 11 +- .../pmix3x/pmix/src/include/pmix_globals.h | 39 +- opal/mca/pmix/pmix3x/pmix/src/include/types.h | 4 +- .../pmix/pmix3x/pmix/src/mca/base/Makefile.am | 4 +- ...lp-mca-base.txt => help-pmix-mca-base.txt} | 1 + ...help-mca-var.txt => help-pmix-mca-var.txt} | 1 + .../mca/base/pmix_mca_base_component_find.c | 6 +- .../pmix/src/mca/base/pmix_mca_base_var.c | 36 +- .../pmix3x/pmix/src/mca/bfrops/base/base.h | 2 +- .../src/mca/bfrops/base/bfrop_base_copy.c | 29 +- .../pmix/src/mca/bfrops/base/bfrop_base_fns.c | 465 +---- .../src/mca/bfrops/base/bfrop_base_frame.c | 4 +- .../src/mca/bfrops/base/bfrop_base_pack.c | 3 +- .../src/mca/bfrops/base/bfrop_base_print.c | 2 +- .../src/mca/bfrops/base/bfrop_base_unpack.c | 9 +- .../pmix/pmix3x/pmix/src/mca/bfrops/bfrops.h | 9 +- .../pmix3x/pmix/src/mca/gds/ds12/gds_dstore.c | 6 + .../src/mca/gds/ds12/gds_dstore_component.c | 11 +- .../pmix3x/pmix/src/mca/gds/hash/gds_hash.c | 69 +- .../pmix/src/mca/pif/posix_ipv4/pif_posix.c | 5 +- .../pmix/pmix3x/pmix/src/mca/plog/Makefile.am | 44 + .../pmix/src/mca/plog/base/Makefile.include | 34 + .../pmix/pmix3x/pmix/src/mca/plog/base/base.h | 93 + .../pmix/src/mca/plog/base/help-pmix-plog.txt | 56 + .../pmix/src/mca/plog/base/plog_base_frame.c | 105 + .../pmix/src/mca/plog/base/plog_base_select.c | 199 ++ .../pmix/src/mca/plog/base/plog_base_stubs.c | 252 +++ .../pmix/src/mca/plog/default/Makefile.am | 46 + .../pmix/src/mca/plog/default/plog_default.c | 150 ++ .../pmix/src/mca/plog/default/plog_default.h | 40 + .../mca/plog/default/plog_default_component.c | 47 + opal/mca/pmix/pmix3x/pmix/src/mca/plog/plog.h | 102 + .../pmix/src/mca/plog/stdfd/Makefile.am | 46 + .../pmix/src/mca/plog/stdfd/plog_stdfd.c | 122 ++ .../pmix/src/mca/plog/stdfd/plog_stdfd.h | 40 + .../src/mca/plog/stdfd/plog_stdfd_component.c | 47 + .../pmix/src/mca/plog/syslog/Makefile.am | 46 + .../pmix/src/mca/plog/syslog/configure.m4 | 30 + .../pmix/src/mca/plog/syslog/plog_syslog.c | 227 ++ .../pmix/src/mca/plog/syslog/plog_syslog.h | 47 + .../mca/plog/syslog/plog_syslog_component.c | 124 ++ .../pmix/pmix3x/pmix/src/mca/pnet/base/base.h | 58 +- .../pmix/src/mca/pnet/base/pnet_base_fns.c | 576 +++++- .../pmix/src/mca/pnet/base/pnet_base_frame.c | 99 +- .../pmix3x/pmix/src/mca/pnet/opa/configure.m4 | 63 +- .../pmix3x/pmix/src/mca/pnet/opa/pnet_opa.c | 334 ++- .../pmix3x/pmix/src/mca/pnet/opa/pnet_opa.h | 13 +- .../src/mca/pnet/opa/pnet_opa_component.c | 85 +- opal/mca/pmix/pmix3x/pmix/src/mca/pnet/pnet.h | 127 +- .../pmix3x/pmix/src/mca/pnet/tcp/Makefile.am | 56 + .../pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.c | 1137 ++++++++++ .../pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.h | 35 + .../src/mca/pnet/tcp/pnet_tcp_component.c | 133 ++ .../pmix3x/pmix/src/mca/pnet/test/Makefile.am | 52 + .../pmix3x/pmix/src/mca/pnet/test/pnet_test.c | 485 +++++ .../pmix3x/pmix/src/mca/pnet/test/pnet_test.h | 36 + .../src/mca/pnet/test/pnet_test_component.c | 101 + .../pmix/src/mca/preg/native/preg_native.c | 250 +-- .../pmix/pmix3x/pmix/src/mca/ptl/base/base.h | 4 +- .../pmix/src/mca/ptl/base/ptl_base_sendrecv.c | 7 +- .../pmix/src/mca/ptl/base/ptl_base_stubs.c | 22 +- opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl.h | 8 +- .../pmix/pmix3x/pmix/src/mca/ptl/ptl_types.h | 27 +- .../pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.c | 310 ++- .../pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.h | 3 +- .../pmix/src/mca/ptl/tcp/ptl_tcp_component.c | 250 ++- .../pmix/src/runtime/help-pmix-runtime.txt | 22 +- .../pmix3x/pmix/src/runtime/pmix_finalize.c | 26 +- .../pmix/pmix3x/pmix/src/runtime/pmix_init.c | 36 +- .../pmix/pmix3x/pmix/src/server/pmix_server.c | 710 +++++-- .../pmix3x/pmix/src/server/pmix_server_get.c | 8 +- .../pmix3x/pmix/src/server/pmix_server_ops.c | 590 +++--- .../pmix3x/pmix/src/server/pmix_server_ops.h | 32 +- .../pmix/pmix3x/pmix/src/threads/threads.h | 3 +- .../mca/pmix/pmix3x/pmix/src/tool/pmix_tool.c | 1170 +++++++---- .../pmix3x/pmix/src/tools/Makefile.include | 6 +- .../tools/{pinfo => pmix_info}/Makefile.am | 12 +- .../help-pmix-info.txt} | 2 +- .../src/tools/{pinfo => pmix_info}/pinfo.h | 2 +- .../{pinfo/pinfo.c => pmix_info/pmix_info.c} | 2 +- .../src/tools/{pinfo => pmix_info}/support.c | 2 +- .../src/tools/{pinfo => pmix_info}/support.h | 2 +- opal/mca/pmix/pmix3x/pmix/src/util/error.c | 303 +-- opal/mca/pmix/pmix3x/pmix/src/util/error.h | 3 +- opal/mca/pmix/pmix3x/pmix/src/util/hash.c | 5 +- opal/mca/pmix/pmix3x/pmix/src/util/name_fns.c | 1 - .../pmix/pmix3x/pmix/test/server_callbacks.c | 14 +- .../pmix/pmix3x/pmix/test/server_callbacks.h | 6 +- .../pmix/pmix3x/pmix/test/simple/Makefile.am | 30 +- .../pmix/pmix3x/pmix/test/simple/gwclient.c | 100 + .../mca/pmix/pmix3x/pmix/test/simple/gwtest.c | 999 +++++++++ .../pmix3x/pmix/test/simple/quietclient.c | 288 +++ .../pmix/pmix3x/pmix/test/simple/simpclient.c | 17 +- .../pmix/pmix3x/pmix/test/simple/simpdyn.c | 13 +- .../pmix/pmix3x/pmix/test/simple/simptest.c | 194 +- .../pmix3x/pmix/test/simple/simptimeout.c | 4 +- .../pmix/pmix3x/pmix/test/simple/stability.c | 886 ++++++++ opal/mca/pmix/pmix3x/pmix/test/test_cd.c | 20 +- opal/mca/pmix/pmix3x/pmix/test/test_common.h | 12 +- opal/mca/pmix/pmix3x/pmix/test/test_fence.c | 8 +- .../pmix3x/pmix/test/test_resolve_peers.c | 13 +- opal/mca/pmix/pmix3x/pmix3x.c | 1 - opal/mca/pmix/pmix3x/pmix3x.h | 1 - opal/mca/pmix/pmix3x/pmix3x_client.c | 68 +- opal/mca/pmix/pmix3x/pmix3x_server_north.c | 39 +- 165 files changed, 20493 insertions(+), 2979 deletions(-) create mode 100644 opal/mca/pmix/ext4x/Makefile.am create mode 100644 opal/mca/pmix/ext4x/configure.m4 create mode 100644 opal/mca/pmix/ext4x/ext4x.c create mode 100644 opal/mca/pmix/ext4x/ext4x.h create mode 100644 opal/mca/pmix/ext4x/ext4x_client.c create mode 100644 opal/mca/pmix/ext4x/ext4x_component.c create mode 100644 opal/mca/pmix/ext4x/ext4x_local.c create mode 100644 opal/mca/pmix/ext4x/ext4x_server_north.c create mode 100644 opal/mca/pmix/ext4x/ext4x_server_south.c create mode 100644 opal/mca/pmix/ext4x/help-pmix-ext4x.txt create mode 100644 opal/mca/pmix/pmix3x/pmix/bindings/Makefile.am create mode 100644 opal/mca/pmix/pmix3x/pmix/bindings/README create mode 100644 opal/mca/pmix/pmix3x/pmix/bindings/python/Makefile.am create mode 100644 opal/mca/pmix/pmix3x/pmix/bindings/python/README create mode 100644 opal/mca/pmix/pmix3x/pmix/config/pmix_setup_hwloc.m4 mode change 100755 => 100644 opal/mca/pmix/pmix3x/pmix/contrib/pmix.spec create mode 100644 opal/mca/pmix/pmix3x/pmix/examples/pubi.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/class/pmix_bitmap.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/class/pmix_bitmap.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/hwloc/Makefile.include create mode 100644 opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc-internal.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc.c rename opal/mca/pmix/pmix3x/pmix/src/mca/base/{help-mca-base.txt => help-pmix-mca-base.txt} (97%) rename opal/mca/pmix/pmix3x/pmix/src/mca/base/{help-mca-var.txt => help-pmix-mca-var.txt} (98%) create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/Makefile.am create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/Makefile.include create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/base.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/help-pmix-plog.txt create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_frame.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_select.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_stubs.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/Makefile.am create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default_component.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/plog.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/Makefile.am create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd_component.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/Makefile.am create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/configure.m4 create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog_component.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/Makefile.am create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp_component.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/Makefile.am create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test_component.c rename opal/mca/pmix/pmix3x/pmix/src/tools/{pinfo => pmix_info}/Makefile.am (90%) rename opal/mca/pmix/pmix3x/pmix/src/tools/{pinfo/help-pinfo.txt => pmix_info/help-pmix-info.txt} (96%) rename opal/mca/pmix/pmix3x/pmix/src/tools/{pinfo => pmix_info}/pinfo.h (97%) rename opal/mca/pmix/pmix3x/pmix/src/tools/{pinfo/pinfo.c => pmix_info/pmix_info.c} (99%) rename opal/mca/pmix/pmix3x/pmix/src/tools/{pinfo => pmix_info}/support.c (99%) rename opal/mca/pmix/pmix3x/pmix/src/tools/{pinfo => pmix_info}/support.h (98%) create mode 100644 opal/mca/pmix/pmix3x/pmix/test/simple/gwclient.c create mode 100644 opal/mca/pmix/pmix3x/pmix/test/simple/gwtest.c create mode 100644 opal/mca/pmix/pmix3x/pmix/test/simple/quietclient.c create mode 100644 opal/mca/pmix/pmix3x/pmix/test/simple/stability.c diff --git a/config/opal_check_pmi.m4 b/config/opal_check_pmi.m4 index 8877d672f53..8a23a1a981a 100644 --- a/config/opal_check_pmi.m4 +++ b/config/opal_check_pmi.m4 @@ -13,7 +13,7 @@ # Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights # reserved. -# Copyright (c) 2014-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2014-2018 Intel, Inc. All rights reserved. # Copyright (c) 2014-2018 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -311,6 +311,19 @@ AC_DEFUN([OPAL_CHECK_PMIX],[ # if it does exist, then we need to parse it to find # the actual release series + AS_IF([test "$opal_external_pmix_version_found" = "0"], + [AC_MSG_CHECKING([version 4x]) + AC_PREPROC_IFELSE([AC_LANG_PROGRAM([ + #include + #if (PMIX_VERSION_MAJOR != 4L) + #error "not version 4" + #endif + ], [])], + [AC_MSG_RESULT([found]) + opal_external_pmix_version=4x + opal_external_pmix_version_found=1], + [AC_MSG_RESULT([not found])])]) + AS_IF([test "$opal_external_pmix_version_found" = "0"], [AC_MSG_CHECKING([version 3x]) AC_PREPROC_IFELSE([AC_LANG_PROGRAM([ diff --git a/opal/mca/pmix/ext4x/Makefile.am b/opal/mca/pmix/ext4x/Makefile.am new file mode 100644 index 00000000000..32ac0435e8c --- /dev/null +++ b/opal/mca/pmix/ext4x/Makefile.am @@ -0,0 +1,54 @@ +# +# Copyright (c) 2014-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2015-2018 Research Organization for Information Science +# and Technology (RIST). All rights reserved. +# Copyright (c) 2017 IBM Corporation. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +dist_opaldata_DATA = help-pmix-ext4x.txt + +sources = \ + ext4x.c \ + ext4x_client.c \ + ext4x_component.c \ + ext4x_server_north.c \ + ext4x_server_south.c \ + ext4x_local.c + +headers = \ + ext4x.h + + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_opal_pmix_ext4x_DSO +component_noinst = +component_install = mca_pmix_ext4x.la +else +component_noinst = libmca_pmix_ext4x.la +component_install = +endif + +mcacomponentdir = $(opallibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_pmix_ext4x_la_SOURCES = $(sources) +mca_pmix_ext4x_la_CFLAGS = $(opal_pmix_ext4x_CFLAGS) +mca_pmix_ext4x_la_CPPFLAGS =$(opal_pmix_ext4x_CPPFLAGS) +mca_pmix_ext4x_la_LDFLAGS = -module -avoid-version $(opal_pmix_ext4x_LDFLAGS) +mca_pmix_ext4x_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \ + $(opal_pmix_ext4x_LIBS) + +noinst_LTLIBRARIES = $(component_noinst) +libmca_pmix_ext4x_la_SOURCES =$(sources) +libmca_pmix_ext4x_la_CFLAGS = $(opal_pmix_ext4x_CFLAGS) +libmca_pmix_ext4x_la_CPPFLAGS = $(opal_pmix_ext4x_CPPFLAGS) +libmca_pmix_ext4x_la_LDFLAGS = -module -avoid-version $(opal_pmix_ext4x_LDFLAGS) +libmca_pmix_ext4x_la_LIBADD = $(opal_pmix_ext4x_LIBS) diff --git a/opal/mca/pmix/ext4x/configure.m4 b/opal/mca/pmix/ext4x/configure.m4 new file mode 100644 index 00000000000..639731d7da0 --- /dev/null +++ b/opal/mca/pmix/ext4x/configure.m4 @@ -0,0 +1,64 @@ +# -*- shell-script -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2011-2013 Los Alamos National Security, LLC. +# All rights reserved. +# Copyright (c) 2010-2015 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2015-2017 Research Organization for Information Science +# and Technology (RIST). All rights reserved. +# Copyright (c) 2014-2015 Mellanox Technologies, Inc. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# MCA_pmix_ext4x_CONFIG([action-if-found], [action-if-not-found]) +# ----------------------------------------------------------- +AC_DEFUN([MCA_opal_pmix_ext4x_CONFIG],[ + AC_CONFIG_FILES([opal/mca/pmix/ext4x/Makefile]) + + AS_IF([test "$opal_external_pmix_happy" = "yes"], + [ # check for the 4.x version + AC_MSG_CHECKING([if external component is version 4.x]) + AS_IF([test "$opal_external_pmix_version" = "4x"], + [AC_MSG_RESULT([yes]) + AS_IF([test "$opal_event_external_support" != "yes"], + [AC_MSG_WARN([EXTERNAL PMIX SUPPORT REQUIRES USE OF EXTERNAL LIBEVENT]) + AC_MSG_WARN([AND HWLOC LIBRARIES. THESE LIBRARIES MUST POINT TO THE]) + AC_MSG_WARN([SAME ONES USED TO BUILD PMIX OR ELSE UNPREDICTABLE]) + AC_MSG_WARN([BEHAVIOR MAY RESULT]) + AC_MSG_ERROR([PLEASE CORRECT THE CONFIGURE COMMAND LINE AND REBUILD])]) + opal_pmix_external_4x_happy=yes], + [AC_MSG_RESULT([no]) + opal_pmix_external_4x_happy=no]) + + AS_IF([test "$opal_pmix_external_4x_happy" = "yes"], + [$1 + # need to set the wrapper flags for static builds + pmix_ext4x_WRAPPER_EXTRA_LDFLAGS=$opal_external_pmix_LDFLAGS + pmix_ext4x_WRAPPER_EXTRA_LIBS=$opal_external_pmix_LIBS], + [$2])], + [$2]) + + opal_pmix_ext4x_CPPFLAGS=$opal_external_pmix_CPPFLAGS + opal_pmix_ext4x_LDFLAGS=$opal_external_pmix_LDFLAGS + opal_pmix_ext4x_LIBS=$opal_external_pmix_LIBS + + AC_SUBST([opal_pmix_ext4x_CPPFLAGS]) + AC_SUBST([opal_pmix_ext4x_LDFLAGS]) + AC_SUBST([opal_pmix_ext4x_LIBS]) + +])dnl diff --git a/opal/mca/pmix/ext4x/ext4x.c b/opal/mca/pmix/ext4x/ext4x.c new file mode 100644 index 00000000000..6d7195c1477 --- /dev/null +++ b/opal/mca/pmix/ext4x/ext4x.c @@ -0,0 +1,1829 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2017 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2015 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "opal_config.h" +#include "opal/constants.h" +#include "opal/types.h" + +#ifdef HAVE_STRING_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif + +#include "opal/dss/dss.h" +#include "opal/mca/event/event.h" +#include "opal/mca/hwloc/base/base.h" +#include "opal/runtime/opal.h" +#include "opal/runtime/opal_progress_threads.h" +#include "opal/threads/threads.h" +#include "opal/util/argv.h" +#include "opal/util/error.h" +#include "opal/util/opal_environ.h" +#include "opal/util/output.h" +#include "opal/util/proc.h" +#include "opal/util/show_help.h" + +#include "ext4x.h" +#include "opal/mca/pmix/base/base.h" +#include "opal/mca/pmix/pmix_types.h" + +#include +#include + +/**** C.O.M.M.O.N I.N.T.E.R.F.A.C.E.S ****/ + +/* These are functions used by both client and server to + * access common functions in the embedded PMIx library */ +static bool legacy_get(void); +static const char *ext4x_get_nspace(opal_jobid_t jobid); +static void ext4x_register_jobid(opal_jobid_t jobid, const char *nspace); +static void register_handler(opal_list_t *event_codes, + opal_list_t *info, + opal_pmix_notification_fn_t evhandler, + opal_pmix_evhandler_reg_cbfunc_t cbfunc, + void *cbdata); +static void deregister_handler(size_t evhandler, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +static int notify_event(int status, + const opal_process_name_t *source, + opal_pmix_data_range_t range, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); +static void ext4x_query(opal_list_t *queries, + opal_pmix_info_cbfunc_t cbfunc, void *cbdata); +static void ext4x_log(opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); + +static int ext4x_register_cleanup(char *path, bool directory, bool ignore, bool jobscope); + +const opal_pmix_base_module_t opal_pmix_ext4x_module = { + .legacy_get = legacy_get, + /* client APIs */ + .init = ext4x_client_init, + .finalize = ext4x_client_finalize, + .initialized = ext4x_initialized, + .abort = ext4x_abort, + .commit = ext4x_commit, + .fence = ext4x_fence, + .fence_nb = ext4x_fencenb, + .put = ext4x_put, + .get = ext4x_get, + .get_nb = ext4x_getnb, + .publish = ext4x_publish, + .publish_nb = ext4x_publishnb, + .lookup = ext4x_lookup, + .lookup_nb = ext4x_lookupnb, + .unpublish = ext4x_unpublish, + .unpublish_nb = ext4x_unpublishnb, + .spawn = ext4x_spawn, + .spawn_nb = ext4x_spawnnb, + .connect = ext4x_connect, + .connect_nb = ext4x_connectnb, + .disconnect = ext4x_disconnect, + .disconnect_nb = ext4x_disconnectnb, + .resolve_peers = ext4x_resolve_peers, + .resolve_nodes = ext4x_resolve_nodes, + .query = ext4x_query, + .log = ext4x_log, + .allocate = ext4x_allocate, + .job_control = ext4x_job_control, + .register_cleanup = ext4x_register_cleanup, + /* server APIs */ + .server_init = ext4x_server_init, + .server_finalize = ext4x_server_finalize, + .generate_regex = ext4x_server_gen_regex, + .generate_ppn = ext4x_server_gen_ppn, + .server_register_nspace = ext4x_server_register_nspace, + .server_deregister_nspace = ext4x_server_deregister_nspace, + .server_register_client = ext4x_server_register_client, + .server_deregister_client = ext4x_server_deregister_client, + .server_setup_fork = ext4x_server_setup_fork, + .server_dmodex_request = ext4x_server_dmodex, + .server_notify_event = ext4x_server_notify_event, + .server_iof_push = ext4x_server_iof_push, + .server_setup_application = ext4x_server_setup_application, + .server_setup_local_support = ext4x_server_setup_local_support, + /* tool APIs */ + .tool_init = ext4x_tool_init, + .tool_finalize = ext4x_tool_fini, + /* utility APIs */ + .get_version = PMIx_Get_version, + .register_evhandler = register_handler, + .deregister_evhandler = deregister_handler, + .notify_event = notify_event, + .store_local = ext4x_store_local, + .get_nspace = ext4x_get_nspace, + .register_jobid = ext4x_register_jobid +}; + +static bool legacy_get(void) +{ + return false; +} + +static void opcbfunc(pmix_status_t status, void *cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + + OPAL_ACQUIRE_OBJECT(op); + + if (NULL != op->opcbfunc) { + op->opcbfunc(ext4x_convert_rc(status), op->cbdata); + } + OBJ_RELEASE(op); +} + + +static const char *ext4x_get_nspace(opal_jobid_t jobid) +{ + opal_ext4x_jobid_trkr_t *jptr; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) { + if (jptr->jobid == jobid) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return jptr->nspace; + } + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return NULL; +} + +static void ext4x_register_jobid(opal_jobid_t jobid, const char *nspace) +{ + opal_ext4x_jobid_trkr_t *jptr; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + + /* if we don't already have it, add this to our jobid tracker */ + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) { + if (jptr->jobid == jobid) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return; + } + } + jptr = OBJ_NEW(opal_ext4x_jobid_trkr_t); + (void)strncpy(jptr->nspace, nspace, PMIX_MAX_NSLEN); + jptr->jobid = jobid; + opal_list_append(&mca_pmix_ext4x_component.jobids, &jptr->super); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); +} + +static void event_hdlr_complete(pmix_status_t status, void *cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + + OBJ_RELEASE(op); +} + +static void return_local_event_hdlr(int status, opal_list_t *results, + opal_pmix_op_cbfunc_t cbfunc, void *thiscbdata, + void *notification_cbdata) +{ + ext4x_threadshift_t *cd = (ext4x_threadshift_t*)notification_cbdata; + ext4x_opcaddy_t *op; + opal_value_t *kv; + pmix_status_t pstatus; + size_t n; + + OPAL_ACQUIRE_OBJECT(cd); + if (NULL != cd->pmixcbfunc) { + op = OBJ_NEW(ext4x_opcaddy_t); + + if (NULL != results && 0 < (op->ninfo = opal_list_get_size(results))) { + /* convert the list of results to an array of info */ + PMIX_INFO_CREATE(op->info, op->ninfo); + n=0; + OPAL_LIST_FOREACH(kv, cd->info, opal_value_t) { + (void)strncpy(op->info[n].key, kv->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&op->info[n].value, kv); + ++n; + } + } + /* convert the status */ + pstatus = ext4x_convert_opalrc(status); + /* call the library's callback function */ + cd->pmixcbfunc(pstatus, op->info, op->ninfo, event_hdlr_complete, op, cd->cbdata); + } + + /* release the threadshift object */ + if (NULL != cd->info) { + OPAL_LIST_RELEASE(cd->info); + } + OBJ_RELEASE(cd); + + /* release the caller */ + if (NULL != cbfunc) { + cbfunc(OPAL_SUCCESS, thiscbdata); + } +} + +/* process the notification */ +static void process_event(int sd, short args, void *cbdata) +{ + ext4x_threadshift_t *cd = (ext4x_threadshift_t*)cbdata; + opal_ext4x_event_t *event; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + + /* cycle thru the registrations */ + OPAL_LIST_FOREACH(event, &mca_pmix_ext4x_component.events, opal_ext4x_event_t) { + if (cd->id == event->index) { + /* found it - invoke the handler, pointing its + * callback function to our callback function */ + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s _EVENT_HDLR CALLING EVHDLR", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + if (NULL != event->handler) { + OBJ_RETAIN(event); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + event->handler(cd->status, &cd->pname, + cd->info, &cd->results, + return_local_event_hdlr, cd); + OBJ_RELEASE(event); + return; + } + } + } + + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* if we didn't find a match, we still have to call their final callback */ + if (NULL != cd->pmixcbfunc) { + cd->pmixcbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cd->cbdata); + } + OPAL_LIST_RELEASE(cd->info); + OBJ_RELEASE(cd); + return; + +} +/* this function will be called by the PMIx client library + * whenever it receives notification of an event. The + * notification can come from an ORTE daemon (when launched + * by mpirun), directly from a RM (when direct launched), or + * from another process (via the local daemon). + * The call will occur in the PMIx event base */ +void ext4x_event_hdlr(size_t evhdlr_registration_id, + pmix_status_t status, const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) +{ + ext4x_threadshift_t *cd; + int rc; + opal_value_t *iptr; + size_t n; + + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s RECEIVED NOTIFICATION OF STATUS %d ON HDLR %lu", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), status, + (unsigned long)evhdlr_registration_id); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + + cd = OBJ_NEW(ext4x_threadshift_t); + cd->id = evhdlr_registration_id; + cd->pmixcbfunc = cbfunc; + cd->cbdata = cbdata; + + /* convert the incoming status */ + cd->status = ext4x_convert_rc(status); + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s CONVERTED STATUS %d TO STATUS %d", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), status, cd->status); + + /* convert the nspace/rank to an opal_process_name_t */ + if (NULL == source) { + cd->pname.jobid = OPAL_NAME_INVALID->jobid; + cd->pname.vpid = OPAL_NAME_INVALID->vpid; + } else { + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&cd->pname.jobid, source->nspace))) { + OPAL_ERROR_LOG(rc); + cd->pname.jobid = OPAL_NAME_INVALID->jobid; + } + cd->pname.vpid = ext4x_convert_rank(source->rank); + } + + /* convert the array of info */ + if (NULL != info) { + cd->info = OBJ_NEW(opal_list_t); + for (n=0; n < ninfo; n++) { + iptr = OBJ_NEW(opal_value_t); + iptr->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(iptr, &info[n].value))) { + OPAL_ERROR_LOG(rc); + OBJ_RELEASE(iptr); + continue; + } + opal_list_append(cd->info, &iptr->super); + } + } + + /* convert the array of prior results */ + if (NULL != results) { + for (n=0; n < nresults; n++) { + iptr = OBJ_NEW(opal_value_t); + iptr->key = strdup(results[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(iptr, &results[n].value))) { + OPAL_ERROR_LOG(rc); + OBJ_RELEASE(iptr); + continue; + } + opal_list_append(&cd->results, &iptr->super); + } + } + + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* do NOT directly call the event handler as this + * may lead to a deadlock condition should the + * handler invoke a PMIx function */ + OPAL_PMIX2X_THREADSHIFT(cd, process_event); + return; +} + +static void cleanup_cbfunc(pmix_status_t status, + pmix_info_t *info, size_t ninfo, + void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + opal_pmix_lock_t *lk = (opal_pmix_lock_t*)cbdata; + + OPAL_POST_OBJECT(lk); + + /* let the library release the data and cleanup from + * the operation */ + if (NULL != release_fn) { + release_fn(release_cbdata); + } + + /* release the block */ + lk->status = ext4x_convert_rc(status); + OPAL_PMIX_WAKEUP_THREAD(lk); +} + +static int ext4x_register_cleanup(char *path, bool directory, bool ignore, bool jobscope) +{ + opal_pmix_lock_t lk; + pmix_info_t pinfo[3]; + size_t n, ninfo=0; + pmix_status_t rc; + int ret; + + OPAL_PMIX_CONSTRUCT_LOCK(&lk); + + if (ignore) { + /* they want this path ignored */ + PMIX_INFO_LOAD(&pinfo[ninfo], PMIX_CLEANUP_IGNORE, path, PMIX_STRING); + ++ninfo; + } else { + if (directory) { + PMIX_INFO_LOAD(&pinfo[ninfo], PMIX_REGISTER_CLEANUP_DIR, path, PMIX_STRING); + ++ninfo; + /* recursively cleanup directories */ + PMIX_INFO_LOAD(&pinfo[ninfo], PMIX_CLEANUP_RECURSIVE, NULL, PMIX_BOOL); + ++ninfo; + } else { + /* order cleanup of the provided path */ + PMIX_INFO_LOAD(&pinfo[ninfo], PMIX_REGISTER_CLEANUP, path, PMIX_STRING); + ++ninfo; + } + } + + /* if they want this applied to the job, then indicate so */ + if (jobscope) { + rc = PMIx_Job_control_nb(NULL, 0, pinfo, ninfo, cleanup_cbfunc, (void*)&lk); + } else { + /* only applies to us */ + rc = PMIx_Job_control_nb(&mca_pmix_ext4x_component.myproc, 1, pinfo, ninfo, cleanup_cbfunc, (void*)&lk); + } + if (PMIX_SUCCESS != rc) { + ret = ext4x_convert_rc(rc); + } else { + OPAL_PMIX_WAIT_THREAD(&lk); + ret = lk.status; + } + OPAL_PMIX_DESTRUCT_LOCK(&lk); + for (n=0; n < ninfo; n++) { + PMIX_INFO_DESTRUCT(&pinfo[n]); + } + return ret; +} + +opal_vpid_t ext4x_convert_rank(pmix_rank_t rank) +{ + switch(rank) { + case PMIX_RANK_UNDEF: + return OPAL_VPID_INVALID; + case PMIX_RANK_WILDCARD: + return OPAL_VPID_WILDCARD; + default: + return (opal_vpid_t)rank; + } +} + +pmix_rank_t ext4x_convert_opalrank(opal_vpid_t vpid) +{ + switch(vpid) { + case OPAL_VPID_WILDCARD: + return PMIX_RANK_WILDCARD; + case OPAL_VPID_INVALID: + return PMIX_RANK_UNDEF; + default: + return (pmix_rank_t)vpid; + } +} + +pmix_status_t ext4x_convert_opalrc(int rc) +{ + switch (rc) { + case OPAL_ERR_DEBUGGER_RELEASE: + return PMIX_ERR_DEBUGGER_RELEASE; + + case OPAL_ERR_HANDLERS_COMPLETE: + return PMIX_EVENT_ACTION_COMPLETE; + + case OPAL_ERR_PROC_ABORTED: + return PMIX_ERR_PROC_ABORTED; + + case OPAL_ERR_PROC_REQUESTED_ABORT: + return PMIX_ERR_PROC_REQUESTED_ABORT; + + case OPAL_ERR_PROC_ABORTING: + return PMIX_ERR_PROC_ABORTING; + + case OPAL_ERR_NODE_DOWN: + return PMIX_ERR_NODE_DOWN; + + case OPAL_ERR_NODE_OFFLINE: + return PMIX_ERR_NODE_OFFLINE; + + case OPAL_ERR_JOB_TERMINATED: + return PMIX_ERR_JOB_TERMINATED; + + case OPAL_ERR_PROC_RESTART: + return PMIX_ERR_PROC_RESTART; + + case OPAL_ERR_PROC_CHECKPOINT: + return PMIX_ERR_PROC_CHECKPOINT; + + case OPAL_ERR_PROC_MIGRATE: + return PMIX_ERR_PROC_MIGRATE; + + case OPAL_ERR_EVENT_REGISTRATION: + return PMIX_ERR_EVENT_REGISTRATION; + + case OPAL_ERR_NOT_IMPLEMENTED: + case OPAL_ERR_NOT_SUPPORTED: + return PMIX_ERR_NOT_SUPPORTED; + + case OPAL_ERR_NOT_FOUND: + return PMIX_ERR_NOT_FOUND; + + case OPAL_ERR_PERM: + case OPAL_ERR_UNREACH: + case OPAL_ERR_SERVER_NOT_AVAIL: + return PMIX_ERR_UNREACH; + + case OPAL_ERR_BAD_PARAM: + return PMIX_ERR_BAD_PARAM; + + case OPAL_ERR_OUT_OF_RESOURCE: + return PMIX_ERR_OUT_OF_RESOURCE; + + case OPAL_ERR_DATA_VALUE_NOT_FOUND: + return PMIX_ERR_DATA_VALUE_NOT_FOUND; + + case OPAL_ERR_TIMEOUT: + return PMIX_ERR_TIMEOUT; + + case OPAL_ERR_WOULD_BLOCK: + return PMIX_ERR_WOULD_BLOCK; + + case OPAL_EXISTS: + return PMIX_EXISTS; + + case OPAL_ERR_PARTIAL_SUCCESS: + return PMIX_QUERY_PARTIAL_SUCCESS; + + case OPAL_ERR_MODEL_DECLARED: + return PMIX_MODEL_DECLARED; + + case OPAL_ERROR: + return PMIX_ERROR; + case OPAL_SUCCESS: + return PMIX_SUCCESS; + default: + return rc; + } +} + +int ext4x_convert_rc(pmix_status_t rc) +{ + switch (rc) { + case PMIX_ERR_DEBUGGER_RELEASE: + return OPAL_ERR_DEBUGGER_RELEASE; + + case PMIX_EVENT_ACTION_COMPLETE: + return OPAL_ERR_HANDLERS_COMPLETE; + + case PMIX_ERR_PROC_ABORTED: + return OPAL_ERR_PROC_ABORTED; + + case PMIX_ERR_PROC_REQUESTED_ABORT: + return OPAL_ERR_PROC_REQUESTED_ABORT; + + case PMIX_ERR_PROC_ABORTING: + return OPAL_ERR_PROC_ABORTING; + + case PMIX_ERR_NODE_DOWN: + return OPAL_ERR_NODE_DOWN; + + case PMIX_ERR_NODE_OFFLINE: + return OPAL_ERR_NODE_OFFLINE; + + case PMIX_ERR_JOB_TERMINATED: + return OPAL_ERR_JOB_TERMINATED; + + case PMIX_ERR_PROC_RESTART: + return OPAL_ERR_PROC_RESTART; + + case PMIX_ERR_PROC_CHECKPOINT: + return OPAL_ERR_PROC_CHECKPOINT; + + case PMIX_ERR_PROC_MIGRATE: + return OPAL_ERR_PROC_MIGRATE; + + case PMIX_ERR_EVENT_REGISTRATION: + return OPAL_ERR_EVENT_REGISTRATION; + + case PMIX_ERR_NOT_SUPPORTED: + return OPAL_ERR_NOT_SUPPORTED; + + case PMIX_ERR_NOT_FOUND: + return OPAL_ERR_NOT_FOUND; + + case PMIX_ERR_OUT_OF_RESOURCE: + return OPAL_ERR_OUT_OF_RESOURCE; + + case PMIX_ERR_INIT: + return OPAL_ERROR; + + case PMIX_ERR_BAD_PARAM: + return OPAL_ERR_BAD_PARAM; + + case PMIX_ERR_UNREACH: + case PMIX_ERR_NO_PERMISSIONS: + return OPAL_ERR_UNREACH; + + case PMIX_ERR_TIMEOUT: + return OPAL_ERR_TIMEOUT; + + case PMIX_ERR_WOULD_BLOCK: + return OPAL_ERR_WOULD_BLOCK; + + case PMIX_ERR_LOST_CONNECTION_TO_SERVER: + case PMIX_ERR_LOST_PEER_CONNECTION: + case PMIX_ERR_LOST_CONNECTION_TO_CLIENT: + return OPAL_ERR_COMM_FAILURE; + + case PMIX_EXISTS: + return OPAL_EXISTS; + + case PMIX_QUERY_PARTIAL_SUCCESS: + return OPAL_ERR_PARTIAL_SUCCESS; + + case PMIX_MONITOR_HEARTBEAT_ALERT: + return OPAL_ERR_HEARTBEAT_ALERT; + + case PMIX_MONITOR_FILE_ALERT: + return OPAL_ERR_FILE_ALERT; + + case PMIX_MODEL_DECLARED: + return OPAL_ERR_MODEL_DECLARED; + + case PMIX_ERROR: + return OPAL_ERROR; + case PMIX_SUCCESS: + return OPAL_SUCCESS; + default: + return rc; + } +} + +opal_pmix_scope_t ext4x_convert_scope(pmix_scope_t scope) +{ + switch(scope) { + case PMIX_SCOPE_UNDEF: + return OPAL_PMIX_SCOPE_UNDEF; + case PMIX_LOCAL: + return OPAL_PMIX_LOCAL; + case PMIX_REMOTE: + return OPAL_PMIX_REMOTE; + case PMIX_GLOBAL: + return OPAL_PMIX_GLOBAL; + default: + return OPAL_PMIX_SCOPE_UNDEF; + } +} + +pmix_scope_t ext4x_convert_opalscope(opal_pmix_scope_t scope) { + switch(scope) { + case OPAL_PMIX_LOCAL: + return PMIX_LOCAL; + case OPAL_PMIX_REMOTE: + return PMIX_REMOTE; + case OPAL_PMIX_GLOBAL: + return PMIX_GLOBAL; + default: + return PMIX_SCOPE_UNDEF; + } +} + +pmix_data_range_t ext4x_convert_opalrange(opal_pmix_data_range_t range) { + switch(range) { + case OPAL_PMIX_RANGE_UNDEF: + return PMIX_RANGE_UNDEF; + case OPAL_PMIX_RANGE_LOCAL: + return PMIX_RANGE_LOCAL; + case OPAL_PMIX_RANGE_NAMESPACE: + return PMIX_RANGE_NAMESPACE; + case OPAL_PMIX_RANGE_SESSION: + return PMIX_RANGE_SESSION; + case OPAL_PMIX_RANGE_GLOBAL: + return PMIX_RANGE_GLOBAL; + case OPAL_PMIX_RANGE_CUSTOM: + return PMIX_RANGE_CUSTOM; + case OPAL_PMIX_RANGE_PROC_LOCAL: + return PMIX_RANGE_PROC_LOCAL; + default: + return PMIX_SCOPE_UNDEF; + } +} + +opal_pmix_data_range_t ext4x_convert_range(pmix_data_range_t range) { + switch(range) { + case PMIX_RANGE_UNDEF: + return OPAL_PMIX_RANGE_UNDEF; + case PMIX_RANGE_LOCAL: + return OPAL_PMIX_RANGE_LOCAL; + case PMIX_RANGE_NAMESPACE: + return OPAL_PMIX_RANGE_NAMESPACE; + case PMIX_RANGE_SESSION: + return OPAL_PMIX_RANGE_SESSION; + case PMIX_RANGE_GLOBAL: + return OPAL_PMIX_RANGE_GLOBAL; + case PMIX_RANGE_CUSTOM: + return OPAL_PMIX_RANGE_CUSTOM; + default: + return OPAL_PMIX_RANGE_UNDEF; + } +} + +opal_pmix_persistence_t ext4x_convert_persist(pmix_persistence_t persist) +{ + switch(persist) { + case PMIX_PERSIST_INDEF: + return OPAL_PMIX_PERSIST_INDEF; + case PMIX_PERSIST_FIRST_READ: + return OPAL_PMIX_PERSIST_FIRST_READ; + case PMIX_PERSIST_PROC: + return OPAL_PMIX_PERSIST_PROC; + case PMIX_PERSIST_APP: + return OPAL_PMIX_PERSIST_APP; + case PMIX_PERSIST_SESSION: + return OPAL_PMIX_PERSIST_SESSION; + default: + return OPAL_PMIX_PERSIST_INDEF; + } +} + +pmix_persistence_t ext4x_convert_opalpersist(opal_pmix_persistence_t persist) +{ + switch(persist) { + case OPAL_PMIX_PERSIST_INDEF: + return PMIX_PERSIST_INDEF; + case OPAL_PMIX_PERSIST_FIRST_READ: + return PMIX_PERSIST_FIRST_READ; + case OPAL_PMIX_PERSIST_PROC: + return PMIX_PERSIST_PROC; + case OPAL_PMIX_PERSIST_APP: + return PMIX_PERSIST_APP; + case OPAL_PMIX_PERSIST_SESSION: + return PMIX_PERSIST_SESSION; + default: + return PMIX_PERSIST_INDEF; + } +} + +char* ext4x_convert_jobid(opal_jobid_t jobid) +{ + opal_ext4x_jobid_trkr_t *jptr; + + /* look thru our list of jobids and find the + * corresponding nspace */ + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) { + if (jptr->jobid == jobid) { + return jptr->nspace; + } + } + return NULL; +} + +/**** RHC: NEED TO ADD SUPPORT FOR NEW PMIX DATA TYPES, INCLUDING + **** CONVERSION OF PROC STATES ****/ + +void ext4x_value_load(pmix_value_t *v, + opal_value_t *kv) +{ + opal_ext4x_jobid_trkr_t *job; + bool found; + opal_list_t *list; + opal_value_t *val; + pmix_info_t *info; + size_t n; + + switch(kv->type) { + case OPAL_UNDEF: + v->type = PMIX_UNDEF; + break; + case OPAL_BOOL: + v->type = PMIX_BOOL; + memcpy(&(v->data.flag), &kv->data.flag, 1); + break; + case OPAL_BYTE: + v->type = PMIX_BYTE; + memcpy(&(v->data.byte), &kv->data.byte, 1); + break; + case OPAL_STRING: + v->type = PMIX_STRING; + if (NULL != kv->data.string) { + v->data.string = strdup(kv->data.string); + } else { + v->data.string = NULL; + } + break; + case OPAL_SIZE: + v->type = PMIX_SIZE; + memcpy(&(v->data.size), &kv->data.size, sizeof(size_t)); + break; + case OPAL_PID: + v->type = PMIX_PID; + memcpy(&(v->data.pid), &kv->data.pid, sizeof(pid_t)); + break; + case OPAL_INT: + v->type = PMIX_INT; + memcpy(&(v->data.integer), &kv->data.integer, sizeof(int)); + break; + case OPAL_INT8: + v->type = PMIX_INT8; + memcpy(&(v->data.int8), &kv->data.int8, 1); + break; + case OPAL_INT16: + v->type = PMIX_INT16; + memcpy(&(v->data.int16), &kv->data.int16, 2); + break; + case OPAL_INT32: + v->type = PMIX_INT32; + memcpy(&(v->data.int32), &kv->data.int32, 4); + break; + case OPAL_INT64: + v->type = PMIX_INT64; + memcpy(&(v->data.int64), &kv->data.int64, 8); + break; + case OPAL_UINT: + v->type = PMIX_UINT; + memcpy(&(v->data.uint), &kv->data.uint, sizeof(int)); + break; + case OPAL_UINT8: + v->type = PMIX_UINT8; + memcpy(&(v->data.uint8), &kv->data.uint8, 1); + break; + case OPAL_UINT16: + v->type = PMIX_UINT16; + memcpy(&(v->data.uint16), &kv->data.uint16, 2); + break; + case OPAL_UINT32: + v->type = PMIX_UINT32; + memcpy(&(v->data.uint32), &kv->data.uint32, 4); + break; + case OPAL_UINT64: + v->type = PMIX_UINT64; + memcpy(&(v->data.uint64), &kv->data.uint64, 8); + break; + case OPAL_FLOAT: + v->type = PMIX_FLOAT; + memcpy(&(v->data.fval), &kv->data.fval, sizeof(float)); + break; + case OPAL_DOUBLE: + v->type = PMIX_DOUBLE; + memcpy(&(v->data.dval), &kv->data.dval, sizeof(double)); + break; + case OPAL_TIMEVAL: + v->type = PMIX_TIMEVAL; + memcpy(&(v->data.tv), &kv->data.tv, sizeof(struct timeval)); + break; + case OPAL_TIME: + v->type = PMIX_TIME; + memcpy(&(v->data.time), &kv->data.time, sizeof(time_t)); + break; + case OPAL_STATUS: + v->type = PMIX_STATUS; + v->data.status = ext4x_convert_opalrc(kv->data.status); + break; + case OPAL_VPID: + v->type = PMIX_PROC_RANK; + v->data.rank = ext4x_convert_opalrank(kv->data.name.vpid); + break; + case OPAL_NAME: + v->type = PMIX_PROC; + /* have to stringify the jobid */ + PMIX_PROC_CREATE(v->data.proc, 1); + /* see if this job is in our list of known nspaces */ + found = false; + OPAL_LIST_FOREACH(job, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) { + if (job->jobid == kv->data.name.jobid) { + (void)strncpy(v->data.proc->nspace, job->nspace, PMIX_MAX_NSLEN); + found = true; + break; + } + } + if (!found) { + (void)opal_snprintf_jobid(v->data.proc->nspace, PMIX_MAX_NSLEN, kv->data.name.jobid); + } + v->data.proc->rank = ext4x_convert_opalrank(kv->data.name.vpid); + break; + case OPAL_BYTE_OBJECT: + v->type = PMIX_BYTE_OBJECT; + if (NULL != kv->data.bo.bytes) { + v->data.bo.bytes = (char*)malloc(kv->data.bo.size); + memcpy(v->data.bo.bytes, kv->data.bo.bytes, kv->data.bo.size); + v->data.bo.size = (size_t)kv->data.bo.size; + } else { + v->data.bo.bytes = NULL; + v->data.bo.size = 0; + } + break; + case OPAL_PERSIST: + v->type = PMIX_PERSIST; + v->data.persist = ext4x_convert_opalpersist((opal_pmix_persistence_t)kv->data.uint8); + break; + case OPAL_SCOPE: + v->type = PMIX_SCOPE; + v->data.scope = ext4x_convert_opalscope((opal_pmix_scope_t)kv->data.uint8); + break; + case OPAL_DATA_RANGE: + v->type = PMIX_DATA_RANGE; + v->data.range = ext4x_convert_opalrange((opal_pmix_data_range_t)kv->data.uint8); + break; + case OPAL_PROC_STATE: + v->type = PMIX_PROC_STATE; + /* the OPAL layer doesn't have any concept of proc state, + * so the ORTE layer is responsible for converting it */ + memcpy(&v->data.state, &kv->data.uint8, sizeof(uint8_t)); + break; + case OPAL_PTR: + /* if the opal_value_t is passing a true pointer, then + * respect that request and pass it along */ + if (0 == strcmp(kv->key, OPAL_PMIX_EVENT_RETURN_OBJECT)) { + v->type = PMIX_POINTER; + v->data.ptr = kv->data.ptr; + break; + } + /* otherwise, it must be to a list of + * opal_value_t's that we need to convert to a pmix_data_array + * of pmix_info_t structures */ + list = (opal_list_t*)kv->data.ptr; + v->type = PMIX_DATA_ARRAY; + v->data.darray = (pmix_data_array_t*)malloc(sizeof(pmix_data_array_t)); + v->data.darray->type = PMIX_INFO; + v->data.darray->size = opal_list_get_size(list); + if (0 < v->data.darray->size) { + PMIX_INFO_CREATE(info, v->data.darray->size); + v->data.darray->array = info; + n=0; + OPAL_LIST_FOREACH(val, list, opal_value_t) { + if (NULL != val->key) { + (void)strncpy(info[n].key, val->key, PMIX_MAX_KEYLEN); + } + ext4x_value_load(&info[n].value, val); + ++n; + } + } else { + v->data.darray->array = NULL; + } + break; + case OPAL_PROC_INFO: + v->type = PMIX_PROC_INFO; + PMIX_PROC_INFO_CREATE(v->data.pinfo, 1); + /* see if this job is in our list of known nspaces */ + found = false; + OPAL_LIST_FOREACH(job, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) { + if (job->jobid == kv->data.pinfo.name.jobid) { + (void)strncpy(v->data.pinfo->proc.nspace, job->nspace, PMIX_MAX_NSLEN); + found = true; + break; + } + } + if (!found) { + (void)opal_snprintf_jobid(v->data.pinfo->proc.nspace, PMIX_MAX_NSLEN, kv->data.pinfo.name.jobid); + } + v->data.pinfo->proc.rank = ext4x_convert_opalrank(kv->data.pinfo.name.vpid); + if (NULL != kv->data.pinfo.hostname) { + v->data.pinfo->hostname = strdup(kv->data.pinfo.hostname); + } + if (NULL != kv->data.pinfo.executable_name) { + v->data.pinfo->executable_name = strdup(kv->data.pinfo.executable_name); + } + v->data.pinfo->pid = kv->data.pinfo.pid; + v->data.pinfo->exit_code = kv->data.pinfo.exit_code; + v->data.pinfo->state = ext4x_convert_opalstate(kv->data.pinfo.state); + break; + case OPAL_ENVAR: + v->type = PMIX_ENVAR; + PMIX_ENVAR_CONSTRUCT(&v->data.envar); + if (NULL != kv->data.envar.envar) { + v->data.envar.envar = strdup(kv->data.envar.envar); + } + if (NULL != kv->data.envar.value) { + v->data.envar.value = strdup(kv->data.envar.value); + } + v->data.envar.separator = kv->data.envar.separator; + break; + default: + /* silence warnings */ + break; + } +} + +int ext4x_value_unload(opal_value_t *kv, + const pmix_value_t *v) +{ + int rc=OPAL_SUCCESS; + bool found; + opal_ext4x_jobid_trkr_t *job; + opal_list_t *lt; + opal_value_t *ival; + size_t n; + + switch(v->type) { + case PMIX_UNDEF: + kv->type = OPAL_UNDEF; + break; + case PMIX_BOOL: + kv->type = OPAL_BOOL; + memcpy(&kv->data.flag, &(v->data.flag), 1); + break; + case PMIX_BYTE: + kv->type = OPAL_BYTE; + memcpy(&kv->data.byte, &(v->data.byte), 1); + break; + case PMIX_STRING: + kv->type = OPAL_STRING; + if (NULL != v->data.string) { + kv->data.string = strdup(v->data.string); + } + break; + case PMIX_SIZE: + kv->type = OPAL_SIZE; + memcpy(&kv->data.size, &(v->data.size), sizeof(size_t)); + break; + case PMIX_PID: + kv->type = OPAL_PID; + memcpy(&kv->data.pid, &(v->data.pid), sizeof(pid_t)); + break; + case PMIX_INT: + kv->type = OPAL_INT; + memcpy(&kv->data.integer, &(v->data.integer), sizeof(int)); + break; + case PMIX_INT8: + kv->type = OPAL_INT8; + memcpy(&kv->data.int8, &(v->data.int8), 1); + break; + case PMIX_INT16: + kv->type = OPAL_INT16; + memcpy(&kv->data.int16, &(v->data.int16), 2); + break; + case PMIX_INT32: + kv->type = OPAL_INT32; + memcpy(&kv->data.int32, &(v->data.int32), 4); + break; + case PMIX_INT64: + kv->type = OPAL_INT64; + memcpy(&kv->data.int64, &(v->data.int64), 8); + break; + case PMIX_UINT: + kv->type = OPAL_UINT; + memcpy(&kv->data.uint, &(v->data.uint), sizeof(int)); + break; + case PMIX_UINT8: + kv->type = OPAL_UINT8; + memcpy(&kv->data.uint8, &(v->data.uint8), 1); + break; + case PMIX_UINT16: + kv->type = OPAL_UINT16; + memcpy(&kv->data.uint16, &(v->data.uint16), 2); + break; + case PMIX_UINT32: + kv->type = OPAL_UINT32; + memcpy(&kv->data.uint32, &(v->data.uint32), 4); + break; + case PMIX_UINT64: + kv->type = OPAL_UINT64; + memcpy(&kv->data.uint64, &(v->data.uint64), 8); + break; + case PMIX_FLOAT: + kv->type = OPAL_FLOAT; + memcpy(&kv->data.fval, &(v->data.fval), sizeof(float)); + break; + case PMIX_DOUBLE: + kv->type = OPAL_DOUBLE; + memcpy(&kv->data.dval, &(v->data.dval), sizeof(double)); + break; + case PMIX_TIMEVAL: + kv->type = OPAL_TIMEVAL; + memcpy(&kv->data.tv, &(v->data.tv), sizeof(struct timeval)); + break; + case PMIX_TIME: + kv->type = OPAL_TIME; + memcpy(&kv->data.time, &(v->data.time), sizeof(time_t)); + break; + case PMIX_STATUS: + kv->type = OPAL_STATUS; + kv->data.status = ext4x_convert_rc(v->data.status); + break; + case PMIX_PROC_RANK: + kv->type = OPAL_VPID; + kv->data.name.vpid = ext4x_convert_rank(v->data.rank); + break; + case PMIX_PROC: + kv->type = OPAL_NAME; + /* see if this job is in our list of known nspaces */ + found = false; + OPAL_LIST_FOREACH(job, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) { + if (0 == strncmp(job->nspace, v->data.proc->nspace, PMIX_MAX_NSLEN)) { + kv->data.name.jobid = job->jobid; + found = true; + break; + } + } + if (!found) { + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&kv->data.name.jobid, v->data.proc->nspace))) { + return ext4x_convert_opalrc(rc); + } + } + kv->data.name.vpid = ext4x_convert_rank(v->data.proc->rank); + break; + case PMIX_BYTE_OBJECT: + kv->type = OPAL_BYTE_OBJECT; + if (NULL != v->data.bo.bytes && 0 < v->data.bo.size) { + kv->data.bo.bytes = (uint8_t*)malloc(v->data.bo.size); + memcpy(kv->data.bo.bytes, v->data.bo.bytes, v->data.bo.size); + kv->data.bo.size = (int)v->data.bo.size; + } else { + kv->data.bo.bytes = NULL; + kv->data.bo.size = 0; + } + break; + case PMIX_PERSIST: + kv->type = OPAL_PERSIST; + kv->data.uint8 = ext4x_convert_persist(v->data.persist); + break; + case PMIX_SCOPE: + kv->type = OPAL_SCOPE; + kv->data.uint8 = ext4x_convert_scope(v->data.scope); + break; + case PMIX_DATA_RANGE: + kv->type = OPAL_DATA_RANGE; + kv->data.uint8 = ext4x_convert_range(v->data.range); + break; + case PMIX_PROC_STATE: + kv->type = OPAL_PROC_STATE; + /* the OPAL layer doesn't have any concept of proc state, + * so the ORTE layer is responsible for converting it */ + memcpy(&kv->data.uint8, &v->data.state, sizeof(uint8_t)); + break; + case PMIX_POINTER: + kv->type = OPAL_PTR; + kv->data.ptr = v->data.ptr; + break; + case PMIX_DATA_ARRAY: + if (NULL == v->data.darray || NULL == v->data.darray->array) { + kv->data.ptr = NULL; + break; + } + lt = OBJ_NEW(opal_list_t); + kv->type = OPAL_PTR; + kv->data.ptr = (void*)lt; + for (n=0; n < v->data.darray->size; n++) { + ival = OBJ_NEW(opal_value_t); + opal_list_append(lt, &ival->super); + /* handle the various types */ + if (PMIX_INFO == v->data.darray->type) { + pmix_info_t *iptr = (pmix_info_t*)v->data.darray->array; + if (NULL != iptr[n].key) { + ival->key = strdup(iptr[n].key); + } + rc = ext4x_value_unload(ival, &iptr[n].value); + if (OPAL_SUCCESS != rc) { + OPAL_LIST_RELEASE(lt); + kv->type = OPAL_UNDEF; + kv->data.ptr = NULL; + break; + } + } + } + break; + case PMIX_PROC_INFO: + kv->type = OPAL_PROC_INFO; + if (NULL == v->data.pinfo) { + rc = OPAL_ERR_BAD_PARAM; + break; + } + /* see if this job is in our list of known nspaces */ + found = false; + OPAL_LIST_FOREACH(job, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) { + if (0 == strncmp(job->nspace, v->data.pinfo->proc.nspace, PMIX_MAX_NSLEN)) { + kv->data.pinfo.name.jobid = job->jobid; + found = true; + break; + } + } + if (!found) { + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&kv->data.pinfo.name.jobid, v->data.pinfo->proc.nspace))) { + return ext4x_convert_opalrc(rc); + } + } + kv->data.pinfo.name.vpid = ext4x_convert_rank(v->data.pinfo->proc.rank); + if (NULL != v->data.pinfo->hostname) { + kv->data.pinfo.hostname = strdup(v->data.pinfo->hostname); + } + if (NULL != v->data.pinfo->executable_name) { + kv->data.pinfo.executable_name = strdup(v->data.pinfo->executable_name); + } + kv->data.pinfo.pid = v->data.pinfo->pid; + kv->data.pinfo.exit_code = v->data.pinfo->exit_code; + kv->data.pinfo.state = ext4x_convert_state(v->data.pinfo->state); + break; + case PMIX_ENVAR: + kv->type = OPAL_ENVAR; + OBJ_CONSTRUCT(&kv->data.envar, opal_envar_t); + if (NULL != v->data.envar.envar) { + kv->data.envar.envar = strdup(v->data.envar.envar); + } + if (NULL != v->data.envar.value) { + kv->data.envar.value = strdup(v->data.envar.value); + } + kv->data.envar.separator = v->data.envar.separator; + break; + default: + /* silence warnings */ + rc = OPAL_ERROR; + break; + } + return rc; +} + +static void errreg_cbfunc (pmix_status_t status, + size_t errhandler_ref, + void *cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + + OPAL_ACQUIRE_OBJECT(op); + op->event->index = errhandler_ref; + opal_output_verbose(5, opal_pmix_base_framework.framework_output, + "PMIX2x errreg_cbfunc - error handler registered status=%d, reference=%lu", + status, (unsigned long)errhandler_ref); + if (NULL != op->evregcbfunc) { + op->evregcbfunc(ext4x_convert_rc(status), errhandler_ref, op->cbdata); + } + OBJ_RELEASE(op); +} + +static void register_handler(opal_list_t *event_codes, + opal_list_t *info, + opal_pmix_notification_fn_t evhandler, + opal_pmix_evhandler_reg_cbfunc_t cbfunc, + void *cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + size_t n; + opal_value_t *kv; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + if (NULL != cbfunc) { + cbfunc(OPAL_ERR_NOT_INITIALIZED, 0, cbdata); + } + return; + } + + op = OBJ_NEW(ext4x_opcaddy_t); + op->evregcbfunc = cbfunc; + op->cbdata = cbdata; + + /* convert the event codes */ + if (NULL != event_codes) { + op->ncodes = opal_list_get_size(event_codes); + op->pcodes = (pmix_status_t*)malloc(op->ncodes * sizeof(pmix_status_t)); + n=0; + OPAL_LIST_FOREACH(kv, event_codes, opal_value_t) { + op->pcodes[n] = ext4x_convert_opalrc(kv->data.integer); + ++n; + } + } + + /* convert the list of info to an array of pmix_info_t */ + if (NULL != info && 0 < (op->ninfo = opal_list_get_size(info))) { + PMIX_INFO_CREATE(op->info, op->ninfo); + n=0; + OPAL_LIST_FOREACH(kv, info, opal_value_t) { + (void)strncpy(op->info[n].key, kv->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&op->info[n].value, kv); + ++n; + } + } + + /* register the event */ + op->event = OBJ_NEW(opal_ext4x_event_t); + op->event->handler = evhandler; + opal_list_append(&mca_pmix_ext4x_component.events, &op->event->super); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + PMIx_Register_event_handler(op->pcodes, op->ncodes, + op->info, op->ninfo, + ext4x_event_hdlr, errreg_cbfunc, op); + return; +} + +static void deregister_handler(size_t evhandler, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + ext4x_opcaddy_t *op; + opal_ext4x_event_t *event; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + if (NULL != cbfunc) { + cbfunc(OPAL_ERR_NOT_INITIALIZED, cbdata); + } + return; + } + + /* look for this event */ + OPAL_LIST_FOREACH(event, &mca_pmix_ext4x_component.events, opal_ext4x_event_t) { + if (evhandler == event->index) { + opal_list_remove_item(&mca_pmix_ext4x_component.events, &event->super); + OBJ_RELEASE(event); + break; + } + } + + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + op = OBJ_NEW(ext4x_opcaddy_t); + op->opcbfunc = cbfunc; + op->cbdata = cbdata; + + /* tell the library to deregister this handler */ + PMIx_Deregister_event_handler(evhandler, opcbfunc, op); + return; +} + +static void notify_complete(pmix_status_t status, void *cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + if (NULL != op->opcbfunc) { + op->opcbfunc(ext4x_convert_rc(status), op->cbdata); + } + OBJ_RELEASE(op); +} + +static int notify_event(int status, + const opal_process_name_t *source, + opal_pmix_data_range_t range, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + ext4x_opcaddy_t *op; + opal_value_t *kv; + pmix_proc_t p, *pptr; + pmix_status_t pstatus; + size_t n; + pmix_data_range_t prange; + char *nsptr; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + op = OBJ_NEW(ext4x_opcaddy_t); + op->opcbfunc = cbfunc; + op->cbdata = cbdata; + + /* convert the status */ + pstatus = ext4x_convert_opalrc(status); + + /* convert the source */ + if (NULL == source) { + pptr = NULL; + } else { + if (NULL == (nsptr = ext4x_convert_jobid(source->jobid))) { + OBJ_RELEASE(op); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_FOUND; + } + (void)strncpy(p.nspace, nsptr, PMIX_MAX_NSLEN); + p.rank = ext4x_convert_opalrank(source->vpid); + pptr = &p; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* convert the range */ + prange = ext4x_convert_opalrange(range); + + /* convert the list of info */ + if (NULL != info && 0 < (op->ninfo = opal_list_get_size(info))) { + PMIX_INFO_CREATE(op->info, op->ninfo); + n=0; + OPAL_LIST_FOREACH(kv, info, opal_value_t) { + (void)strncpy(op->info[n].key, kv->key, PMIX_MAX_KEYLEN); + /* little dicey here as we need to convert a status, if + * provided, and it will be an int coming down to us */ + if (0 == strcmp(kv->key, OPAL_PMIX_JOB_TERM_STATUS)) { + op->info[n].value.type = PMIX_STATUS; + op->info[n].value.data.status = ext4x_convert_opalrc(kv->data.integer); + } else { + ext4x_value_load(&op->info[n].value, kv); + } + ++n; + } + } + + /* ask the library to notify our clients */ + pstatus = PMIx_Notify_event(pstatus, pptr, prange, op->info, op->ninfo, notify_complete, op); + + return ext4x_convert_rc(pstatus); +} + +static void relcbfunc(void *cbdata) +{ + opal_list_t *results = (opal_list_t*)cbdata; + if (NULL != results) { + OPAL_LIST_RELEASE(results); + } +} + +static void infocbfunc(pmix_status_t status, + pmix_info_t *info, size_t ninfo, + void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + ext4x_opcaddy_t *cd = (ext4x_opcaddy_t*)cbdata; + int rc = OPAL_SUCCESS; + opal_list_t *results = NULL; + opal_value_t *iptr; + size_t n; + + OPAL_ACQUIRE_OBJECT(cd); + + /* convert the array of pmix_info_t to the list of info */ + if (NULL != info) { + results = OBJ_NEW(opal_list_t); + for (n=0; n < ninfo; n++) { + iptr = OBJ_NEW(opal_value_t); + opal_list_append(results, &iptr->super); + iptr->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(iptr, &info[n].value))) { + OPAL_ERROR_LOG(rc); + OPAL_LIST_RELEASE(results); + results = NULL; + break; + } + } + } + + if (NULL != release_fn) { + release_fn(release_cbdata); + } + + /* return the values to the original requestor */ + if (NULL != cd->qcbfunc) { + cd->qcbfunc(rc, results, cd->cbdata, relcbfunc, results); + } + OBJ_RELEASE(cd); +} + +static void ext4x_query(opal_list_t *queries, + opal_pmix_info_cbfunc_t cbfunc, void *cbdata) +{ + int rc; + opal_value_t *ival; + size_t n, nqueries, nq; + ext4x_opcaddy_t *cd; + pmix_status_t prc; + opal_pmix_query_t *q; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + if (NULL != cbfunc) { + cbfunc(OPAL_ERR_NOT_INITIALIZED, NULL, cbdata, NULL, NULL); + } + return; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* create the caddy */ + cd = OBJ_NEW(ext4x_opcaddy_t); + + /* bozo check */ + if (NULL == queries || 0 == (nqueries = opal_list_get_size(queries))) { + rc = OPAL_ERR_BAD_PARAM; + goto CLEANUP; + } + + /* setup the operation */ + cd->qcbfunc = cbfunc; + cd->cbdata = cbdata; + cd->nqueries = nqueries; + + /* convert the list to an array of query objects */ + PMIX_QUERY_CREATE(cd->queries, cd->nqueries); + n=0; + OPAL_LIST_FOREACH(q, queries, opal_pmix_query_t) { + cd->queries[n].keys = opal_argv_copy(q->keys); + cd->queries[n].nqual = opal_list_get_size(&q->qualifiers); + if (0 < cd->queries[n].nqual) { + PMIX_INFO_CREATE(cd->queries[n].qualifiers, cd->queries[n].nqual); + nq = 0; + OPAL_LIST_FOREACH(ival, &q->qualifiers, opal_value_t) { + (void)strncpy(cd->queries[n].qualifiers[nq].key, ival->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&cd->queries[n].qualifiers[nq].value, ival); + ++nq; + } + } + ++n; + } + + /* pass it down */ + if (PMIX_SUCCESS != (prc = PMIx_Query_info_nb(cd->queries, cd->nqueries, + infocbfunc, cd))) { + /* do not hang! */ + rc = ext4x_convert_rc(prc); + goto CLEANUP; + } + + return; + + CLEANUP: + if (NULL != cbfunc) { + cbfunc(rc, NULL, cbdata, NULL, NULL); + } + OBJ_RELEASE(cd); + return; +} + +static void ext4x_log(opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + int rc; + opal_value_t *ival; + size_t n, ninfo; + ext4x_opcaddy_t *cd; + pmix_status_t prc; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + if (NULL != cbfunc) { + cbfunc(OPAL_ERR_NOT_INITIALIZED, cbdata); + } + return; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* create the caddy */ + cd = OBJ_NEW(ext4x_opcaddy_t); + + /* bozo check */ + if (NULL == info || 0 == (ninfo = opal_list_get_size(info))) { + rc = OPAL_ERR_BAD_PARAM; + goto CLEANUP; + } + + /* setup the operation */ + cd->opcbfunc = cbfunc; + cd->cbdata = cbdata; + cd->ninfo = ninfo; + + /* convert the list to an array of info objects */ + PMIX_INFO_CREATE(cd->info, cd->ninfo); + n=0; + OPAL_LIST_FOREACH(ival, info, opal_value_t) { + (void)strncpy(cd->info[n].key, ival->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&cd->info[n].value, ival); + ++n; + } + + /* pass it down */ + if (PMIX_SUCCESS != (prc = PMIx_Log_nb(cd->info, cd->ninfo, NULL, 0, + opcbfunc, cd))) { + /* do not hang! */ + rc = ext4x_convert_rc(prc); + goto CLEANUP; + } + + return; + + CLEANUP: + if (NULL != cbfunc) { + cbfunc(rc, cbdata); + } + OBJ_RELEASE(cd); +} + +opal_pmix_alloc_directive_t ext4x_convert_allocdir(pmix_alloc_directive_t dir) +{ + switch (dir) { + case PMIX_ALLOC_NEW: + return OPAL_PMIX_ALLOC_NEW; + case PMIX_ALLOC_EXTEND: + return OPAL_PMIX_ALLOC_EXTEND; + case PMIX_ALLOC_RELEASE: + return OPAL_PMIX_ALLOC_RELEASE; + case PMIX_ALLOC_REAQUIRE: + return OPAL_PMIX_ALLOC_REAQCUIRE; + default: + return OPAL_PMIX_ALLOC_UNDEF; + } +} + +int ext4x_convert_state(pmix_proc_state_t state) +{ + switch(state) { + case PMIX_PROC_STATE_UNDEF: + return 0; + case PMIX_PROC_STATE_PREPPED: + case PMIX_PROC_STATE_LAUNCH_UNDERWAY: + return 1; + case PMIX_PROC_STATE_RESTART: + return 2; + case PMIX_PROC_STATE_TERMINATE: + return 3; + case PMIX_PROC_STATE_RUNNING: + return 4; + case PMIX_PROC_STATE_CONNECTED: + return 5; + case PMIX_PROC_STATE_UNTERMINATED: + return 15; + case PMIX_PROC_STATE_TERMINATED: + return 20; + case PMIX_PROC_STATE_KILLED_BY_CMD: + return 51; + case PMIX_PROC_STATE_ABORTED: + return 52; + case PMIX_PROC_STATE_FAILED_TO_START: + return 53; + case PMIX_PROC_STATE_ABORTED_BY_SIG: + return 54; + case PMIX_PROC_STATE_TERM_WO_SYNC: + return 55; + case PMIX_PROC_STATE_COMM_FAILED: + return 56; + case PMIX_PROC_STATE_CALLED_ABORT: + return 58; + case PMIX_PROC_STATE_MIGRATING: + return 60; + case PMIX_PROC_STATE_CANNOT_RESTART: + return 61; + case PMIX_PROC_STATE_TERM_NON_ZERO: + return 62; + case PMIX_PROC_STATE_FAILED_TO_LAUNCH: + return 63; + default: + return 0; // undef + } +} + +pmix_proc_state_t ext4x_convert_opalstate(int state) +{ + switch(state) { + case 0: + return PMIX_PROC_STATE_UNDEF; + case 1: + return PMIX_PROC_STATE_LAUNCH_UNDERWAY; + case 2: + return PMIX_PROC_STATE_RESTART; + case 3: + return PMIX_PROC_STATE_TERMINATE; + case 4: + return PMIX_PROC_STATE_RUNNING; + case 5: + return PMIX_PROC_STATE_CONNECTED; + case 51: + return PMIX_PROC_STATE_KILLED_BY_CMD; + case 52: + return PMIX_PROC_STATE_ABORTED; + case 53: + return PMIX_PROC_STATE_FAILED_TO_START; + case 54: + return PMIX_PROC_STATE_ABORTED_BY_SIG; + case 55: + return PMIX_PROC_STATE_TERM_WO_SYNC; + case 56: + return PMIX_PROC_STATE_COMM_FAILED; + case 58: + return PMIX_PROC_STATE_CALLED_ABORT; + case 59: + return PMIX_PROC_STATE_MIGRATING; + case 61: + return PMIX_PROC_STATE_CANNOT_RESTART; + case 62: + return PMIX_PROC_STATE_TERM_NON_ZERO; + case 63: + return PMIX_PROC_STATE_FAILED_TO_LAUNCH; + default: + return PMIX_PROC_STATE_UNDEF; + } +} + +/**** INSTANTIATE INTERNAL CLASSES ****/ +OBJ_CLASS_INSTANCE(opal_ext4x_jobid_trkr_t, + opal_list_item_t, + NULL, NULL); + +static void evcon(opal_ext4x_event_t *p) +{ + OPAL_PMIX_CONSTRUCT_LOCK(&p->lock); + p->handler = NULL; + p->cbdata = NULL; +} +static void evdes(opal_ext4x_event_t *p) +{ + OPAL_PMIX_DESTRUCT_LOCK(&p->lock); +} +OBJ_CLASS_INSTANCE(opal_ext4x_event_t, + opal_list_item_t, + evcon, evdes); + +static void opcon(ext4x_opcaddy_t *p) +{ + memset(&p->p, 0, sizeof(pmix_proc_t)); + p->nspace = NULL; + p->procs = NULL; + p->nprocs = 0; + p->pdata = NULL; + p->npdata = 0; + p->error_procs = NULL; + p->nerror_procs = 0; + p->info = NULL; + p->ninfo = 0; + p->apps = NULL; + p->sz = 0; + OPAL_PMIX_CONSTRUCT_LOCK(&p->lock); + p->codes = NULL; + p->pcodes = NULL; + p->ncodes = 0; + p->queries = NULL; + p->nqueries = 0; + p->event = NULL; + p->opcbfunc = NULL; + p->mdxcbfunc = NULL; + p->valcbfunc = NULL; + p->lkcbfunc = NULL; + p->spcbfunc = NULL; + p->evregcbfunc = NULL; + p->qcbfunc = NULL; + p->cbdata = NULL; +} +static void opdes(ext4x_opcaddy_t *p) +{ + OPAL_PMIX_DESTRUCT_LOCK(&p->lock); + if (NULL != p->nspace) { + free(p->nspace); + } + if (NULL != p->procs) { + PMIX_PROC_FREE(p->procs, p->nprocs); + } + if (NULL != p->pdata) { + PMIX_PDATA_FREE(p->pdata, p->npdata); + } + if (NULL != p->error_procs) { + PMIX_PROC_FREE(p->error_procs, p->nerror_procs); + } + if (NULL != p->info) { + PMIX_INFO_FREE(p->info, p->ninfo); + } + if (NULL != p->apps) { + PMIX_APP_FREE(p->apps, p->sz); + } + if (NULL != p->pcodes) { + free(p->pcodes); + } + if (NULL != p->queries) { + PMIX_QUERY_FREE(p->queries, p->nqueries); + } +} +OBJ_CLASS_INSTANCE(ext4x_opcaddy_t, + opal_object_t, + opcon, opdes); + +static void ocadcon(ext4x_opalcaddy_t *p) +{ + OBJ_CONSTRUCT(&p->procs, opal_list_t); + OBJ_CONSTRUCT(&p->info, opal_list_t); + OBJ_CONSTRUCT(&p->apps, opal_list_t); + p->opcbfunc = NULL; + p->dmdxfunc = NULL; + p->mdxcbfunc = NULL; + p->lkupcbfunc = NULL; + p->spwncbfunc = NULL; + p->cbdata = NULL; + p->odmdxfunc = NULL; + p->infocbfunc = NULL; + p->toolcbfunc = NULL; + p->ocbdata = NULL; +} +static void ocaddes(ext4x_opalcaddy_t *p) +{ + OPAL_LIST_DESTRUCT(&p->procs); + OPAL_LIST_DESTRUCT(&p->info); + OPAL_LIST_DESTRUCT(&p->apps); +} +OBJ_CLASS_INSTANCE(ext4x_opalcaddy_t, + opal_object_t, + ocadcon, ocaddes); + +static void tscon(ext4x_threadshift_t *p) +{ + OPAL_PMIX_CONSTRUCT_LOCK(&p->lock); + p->msg = NULL; + p->strings = NULL; + p->source = NULL; + p->event_codes = NULL; + p->info = NULL; + OBJ_CONSTRUCT(&p->results, opal_list_t); + p->evhandler = NULL; + p->nondefault = false; + p->cbfunc = NULL; + p->opcbfunc = NULL; + p->cbdata = NULL; +} +static void tsdes(ext4x_threadshift_t *p) +{ + OPAL_PMIX_DESTRUCT_LOCK(&p->lock); + if (NULL != p->strings) { + free(p->strings); + } + OPAL_LIST_DESTRUCT(&p->results); +} +OBJ_CLASS_INSTANCE(ext4x_threadshift_t, + opal_object_t, + tscon, tsdes); + +static void dmcon(opal_ext4x_dmx_trkr_t *p) +{ + p->nspace = NULL; + p->cbfunc = NULL; + p->cbdata = NULL; +} +static void dmdes(opal_ext4x_dmx_trkr_t *p) +{ + if (NULL != p->nspace) { + free(p->nspace); + } +} +OBJ_CLASS_INSTANCE(opal_ext4x_dmx_trkr_t, + opal_list_item_t, + dmcon, dmdes); diff --git a/opal/mca/pmix/ext4x/ext4x.h b/opal/mca/pmix/ext4x/ext4x.h new file mode 100644 index 00000000000..c75d58d8354 --- /dev/null +++ b/opal/mca/pmix/ext4x/ext4x.h @@ -0,0 +1,351 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2015 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2016 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_PMIX_PMIX2X_H +#define MCA_PMIX_PMIX2X_H + +#include "opal_config.h" + +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_SYS_UN_H +#include +#endif + +#include "opal/class/opal_list.h" +#include "opal/mca/mca.h" +#include "opal/mca/event/event.h" +#include "opal/util/proc.h" + +#include "opal/mca/pmix/base/base.h" +#include "pmix_server.h" +#include "pmix_common.h" + +BEGIN_C_DECLS + +typedef struct { + opal_pmix_base_component_t super; + pmix_proc_t myproc; + opal_list_t jobids; + bool native_launch; + size_t evindex; + opal_list_t events; + int cache_size; + opal_list_t cache; + opal_list_t dmdx; + bool silence_warning; +} mca_pmix_ext4x_component_t; + +OPAL_DECLSPEC extern mca_pmix_ext4x_component_t mca_pmix_ext4x_component; + +OPAL_DECLSPEC extern const opal_pmix_base_module_t opal_pmix_ext4x_module; + +/**** INTERNAL OBJECTS ****/ +typedef struct { + opal_list_item_t super; + opal_jobid_t jobid; + char nspace[PMIX_MAX_NSLEN + 1]; +} opal_ext4x_jobid_trkr_t; +OBJ_CLASS_DECLARATION(opal_ext4x_jobid_trkr_t); + +typedef struct { + opal_list_item_t super; + opal_pmix_lock_t lock; + size_t index; + opal_pmix_notification_fn_t handler; + void *cbdata; +} opal_ext4x_event_t; +OBJ_CLASS_DECLARATION(opal_ext4x_event_t); + +typedef struct { + opal_list_item_t super; + char *nspace; + pmix_modex_cbfunc_t cbfunc; + void *cbdata; +} opal_ext4x_dmx_trkr_t; +OBJ_CLASS_DECLARATION(opal_ext4x_dmx_trkr_t); + +typedef struct { + opal_object_t super; + opal_event_t ev; + pmix_status_t status; + char *nspace; + pmix_proc_t p; + pmix_proc_t *procs; + size_t nprocs; + pmix_pdata_t *pdata; + size_t npdata; + pmix_proc_t *error_procs; + size_t nerror_procs; + pmix_info_t *info; + size_t ninfo; + pmix_app_t *apps; + size_t sz; + opal_pmix_lock_t lock; + opal_list_t *codes; + pmix_status_t *pcodes; + size_t ncodes; + pmix_query_t *queries; + size_t nqueries; + opal_ext4x_event_t *event; + opal_pmix_op_cbfunc_t opcbfunc; + opal_pmix_modex_cbfunc_t mdxcbfunc; + opal_pmix_value_cbfunc_t valcbfunc; + opal_pmix_lookup_cbfunc_t lkcbfunc; + opal_pmix_spawn_cbfunc_t spcbfunc; + opal_pmix_evhandler_reg_cbfunc_t evregcbfunc; + opal_pmix_info_cbfunc_t qcbfunc; + opal_pmix_setup_application_cbfunc_t setupcbfunc; + void *cbdata; +} ext4x_opcaddy_t; +OBJ_CLASS_DECLARATION(ext4x_opcaddy_t); + +typedef struct { + opal_object_t super; + opal_list_t procs; + opal_list_t info; + opal_list_t apps; + pmix_op_cbfunc_t opcbfunc; + pmix_dmodex_response_fn_t dmdxfunc; + pmix_modex_cbfunc_t mdxcbfunc; + pmix_lookup_cbfunc_t lkupcbfunc; + pmix_spawn_cbfunc_t spwncbfunc; + pmix_info_cbfunc_t infocbfunc; + pmix_tool_connection_cbfunc_t toolcbfunc; + void *cbdata; + opal_pmix_release_cbfunc_t odmdxfunc; + void *ocbdata; +} ext4x_opalcaddy_t; +OBJ_CLASS_DECLARATION(ext4x_opalcaddy_t); + +typedef struct { + opal_object_t super; + opal_event_t ev; + opal_pmix_lock_t lock; + const char *msg; + char *strings; + size_t id; + int status; + opal_process_name_t pname; + opal_jobid_t jobid; + const opal_process_name_t *source; + opal_pmix_data_range_t range; + bool nondefault; + size_t handler; + opal_value_t *val; + opal_list_t *event_codes; + opal_list_t *info; + opal_list_t results; + opal_pmix_notification_fn_t evhandler; + opal_pmix_evhandler_reg_cbfunc_t cbfunc; + opal_pmix_op_cbfunc_t opcbfunc; + pmix_event_notification_cbfunc_fn_t pmixcbfunc; + opal_pmix_value_cbfunc_t valcbfunc; + opal_pmix_lookup_cbfunc_t lkcbfunc; + void *cbdata; +} ext4x_threadshift_t; +OBJ_CLASS_DECLARATION(ext4x_threadshift_t); + +#define OPAL_PMIX_OP_THREADSHIFT(e, fn, cb, cd) \ + do { \ + ext4x_threadshift_t *_cd; \ + _cd = OBJ_NEW(ext4x_threadshift_t); \ + _cd->handler = (e); \ + _cd->opcbfunc = (cb); \ + _cd->cbdata = (cd); \ + opal_event_assign(&((_cd)->ev), opal_pmix_base.evbase, \ + -1, EV_WRITE, (fn), (_cd)); \ + OPAL_POST_OBJECT(_cd); \ + opal_event_active(&((_cd)->ev), EV_WRITE, 1); \ + } while(0) + +#define OPAL_PMIX_THREADSHIFT(e, i, eh, fn, cb, cd) \ + do { \ + ext4x_threadshift_t *_cd; \ + _cd = OBJ_NEW(ext4x_threadshift_t); \ + _cd->event_codes = (e); \ + _cd->info = (i); \ + _cd->evhandler = (eh); \ + _cd->cbfunc = (cb); \ + _cd->cbdata = (cd); \ + opal_event_assign(&((_cd)->ev), opal_pmix_base.evbase, \ + -1, EV_WRITE, (fn), (_cd)); \ + OPAL_POST_OBJECT(_cd); \ + opal_event_active(&((_cd)->ev), EV_WRITE, 1); \ + } while(0) + +#define OPAL_PMIX_NOTIFY_THREADSHIFT(s, sr, r, i, fn, cb, cd) \ + do { \ + ext4x_threadshift_t *_cd; \ + _cd = OBJ_NEW(ext4x_threadshift_t); \ + _cd->status = (s); \ + _cd->source = (sr); \ + _cd->range = (r); \ + _cd->info = (i); \ + _cd->opcbfunc = (cb); \ + _cd->cbdata = (cd); \ + opal_event_assign(&((_cd)->ev), opal_pmix_base.evbase, \ + -1, EV_WRITE, (fn), (_cd)); \ + OPAL_POST_OBJECT(_cd); \ + opal_event_active(&((_cd)->ev), EV_WRITE, 1); \ + } while(0) + +#define OPAL_PMIX2X_THREADSHIFT(p, cb) \ + do { \ + opal_event_assign(&((p)->ev), opal_pmix_base.evbase, \ + -1, EV_WRITE, (cb), (p)); \ + OPAL_POST_OBJECT(p); \ + opal_event_active(&((p)->ev), EV_WRITE, 1); \ + } while(0) + +/**** CLIENT FUNCTIONS ****/ +OPAL_MODULE_DECLSPEC int ext4x_client_init(opal_list_t *ilist); +OPAL_MODULE_DECLSPEC int ext4x_client_finalize(void); +OPAL_MODULE_DECLSPEC int ext4x_initialized(void); +OPAL_MODULE_DECLSPEC int ext4x_abort(int flag, const char *msg, + opal_list_t *procs); +OPAL_MODULE_DECLSPEC int ext4x_commit(void); +OPAL_MODULE_DECLSPEC int ext4x_fence(opal_list_t *procs, int collect_data); +OPAL_MODULE_DECLSPEC int ext4x_fencenb(opal_list_t *procs, int collect_data, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_put(opal_pmix_scope_t scope, + opal_value_t *val); +OPAL_MODULE_DECLSPEC int ext4x_get(const opal_process_name_t *proc, const char *key, + opal_list_t *info, opal_value_t **val); +OPAL_MODULE_DECLSPEC int ext4x_getnb(const opal_process_name_t *proc, const char *key, + opal_list_t *info, + opal_pmix_value_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_publish(opal_list_t *info); +OPAL_MODULE_DECLSPEC int ext4x_publishnb(opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_lookup(opal_list_t *data, opal_list_t *info); +OPAL_MODULE_DECLSPEC int ext4x_lookupnb(char **keys, opal_list_t *info, + opal_pmix_lookup_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_unpublish(char **keys, opal_list_t *info); +OPAL_MODULE_DECLSPEC int ext4x_unpublishnb(char **keys, opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid); +OPAL_MODULE_DECLSPEC int ext4x_spawnnb(opal_list_t *job_info, opal_list_t *apps, + opal_pmix_spawn_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_connect(opal_list_t *procs); +OPAL_MODULE_DECLSPEC int ext4x_connectnb(opal_list_t *procs, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_disconnect(opal_list_t *procs); +OPAL_MODULE_DECLSPEC int ext4x_disconnectnb(opal_list_t *procs, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_resolve_peers(const char *nodename, opal_jobid_t jobid, + opal_list_t *procs); +OPAL_MODULE_DECLSPEC int ext4x_resolve_nodes(opal_jobid_t jobid, char **nodelist); +OPAL_MODULE_DECLSPEC int ext4x_allocate(opal_pmix_alloc_directive_t directive, + opal_list_t *info, + opal_pmix_info_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_job_control(opal_list_t *targets, + opal_list_t *directives, + opal_pmix_info_cbfunc_t cbfunc, void *cbdata); + +/**** TOOL FUNCTIONS ****/ +OPAL_MODULE_DECLSPEC int ext4x_tool_init(opal_list_t *info); +OPAL_MODULE_DECLSPEC int ext4x_tool_fini(void); + +/**** COMMON FUNCTIONS ****/ +OPAL_MODULE_DECLSPEC int ext4x_store_local(const opal_process_name_t *proc, + opal_value_t *val); + +/**** SERVER SOUTHBOUND FUNCTIONS ****/ +OPAL_MODULE_DECLSPEC int ext4x_server_init(opal_pmix_server_module_t *module, + opal_list_t *info); +OPAL_MODULE_DECLSPEC int ext4x_server_finalize(void); +OPAL_MODULE_DECLSPEC int ext4x_server_gen_regex(const char *input, char **regex); +OPAL_MODULE_DECLSPEC int ext4x_server_gen_ppn(const char *input, char **ppn); +OPAL_MODULE_DECLSPEC int ext4x_server_register_nspace(opal_jobid_t jobid, + int nlocalprocs, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC void ext4x_server_deregister_nspace(opal_jobid_t jobid, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_server_register_client(const opal_process_name_t *proc, + uid_t uid, gid_t gid, + void *server_object, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC void ext4x_server_deregister_client(const opal_process_name_t *proc, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_server_setup_fork(const opal_process_name_t *proc, char ***env); +OPAL_MODULE_DECLSPEC int ext4x_server_dmodex(const opal_process_name_t *proc, + opal_pmix_modex_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_server_notify_event(int status, + const opal_process_name_t *source, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); + +OPAL_MODULE_DECLSPEC int ext4x_server_iof_push(const opal_process_name_t *source, + opal_pmix_iof_channel_t channel, + unsigned char *data, size_t nbytes); + +OPAL_MODULE_DECLSPEC int ext4x_server_setup_application(opal_jobid_t jobid, + opal_list_t *info, + opal_pmix_setup_application_cbfunc_t cbfunc, void *cbdata); +OPAL_MODULE_DECLSPEC int ext4x_server_setup_local_support(opal_jobid_t jobid, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata); + +/**** COMPONENT UTILITY FUNCTIONS ****/ +OPAL_MODULE_DECLSPEC int opal_pmix_ext4x_check_evars(void); + +OPAL_MODULE_DECLSPEC void ext4x_event_hdlr(size_t evhdlr_registration_id, + pmix_status_t status, const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata); +OPAL_MODULE_DECLSPEC pmix_status_t ext4x_convert_opalrc(int rc); +OPAL_MODULE_DECLSPEC int ext4x_convert_rc(pmix_status_t rc); + +OPAL_MODULE_DECLSPEC opal_vpid_t ext4x_convert_rank(pmix_rank_t rank); +OPAL_MODULE_DECLSPEC pmix_rank_t ext4x_convert_opalrank(opal_vpid_t vpid); + +OPAL_MODULE_DECLSPEC opal_pmix_scope_t ext4x_convert_scope(pmix_scope_t scope); +OPAL_MODULE_DECLSPEC pmix_scope_t ext4x_convert_opalscope(opal_pmix_scope_t scope); + +OPAL_MODULE_DECLSPEC pmix_data_range_t ext4x_convert_opalrange(opal_pmix_data_range_t range); +OPAL_MODULE_DECLSPEC opal_pmix_data_range_t ext4x_convert_range(pmix_data_range_t range); + +OPAL_MODULE_DECLSPEC opal_pmix_persistence_t ext4x_convert_persist(pmix_persistence_t scope); +OPAL_MODULE_DECLSPEC pmix_persistence_t ext4x_convert_opalpersist(opal_pmix_persistence_t scope); + +OPAL_MODULE_DECLSPEC void ext4x_value_load(pmix_value_t *v, + opal_value_t *kv); +OPAL_MODULE_DECLSPEC int ext4x_value_unload(opal_value_t *kv, + const pmix_value_t *v); + +OPAL_MODULE_DECLSPEC opal_pmix_alloc_directive_t ext4x_convert_allocdir(pmix_alloc_directive_t dir); + +OPAL_MODULE_DECLSPEC char* ext4x_convert_jobid(opal_jobid_t jobid); + +OPAL_MODULE_DECLSPEC int ext4x_convert_state(pmix_proc_state_t state); + +OPAL_MODULE_DECLSPEC pmix_proc_state_t ext4x_convert_opalstate(int state); + + +END_C_DECLS + +#endif /* MCA_PMIX_EXTERNAL_H */ diff --git a/opal/mca/pmix/ext4x/ext4x_client.c b/opal/mca/pmix/ext4x/ext4x_client.c new file mode 100644 index 00000000000..931bd8df046 --- /dev/null +++ b/opal/mca/pmix/ext4x/ext4x_client.c @@ -0,0 +1,1664 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2017 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2017 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2016 Los Alamos National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "opal_config.h" +#include "opal/constants.h" +#include "opal/types.h" + +#ifdef HAVE_STRING_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif + +#include "opal/hash_string.h" +#include "opal/threads/threads.h" +#include "opal/util/argv.h" +#include "opal/util/opal_environ.h" +#include "opal/util/proc.h" +#include "opal/util/show_help.h" + +#include "opal/mca/pmix/base/base.h" +#include "ext4x.h" +#include "pmix.h" +#include "pmix_tool.h" + +static char *dbgvalue=NULL; + +static void errreg_cbfunc (pmix_status_t status, + size_t errhandler_ref, + void *cbdata) +{ + opal_ext4x_event_t *event = (opal_ext4x_event_t*)cbdata; + + OPAL_ACQUIRE_OBJECT(event); + + event->index = errhandler_ref; + opal_output_verbose(5, opal_pmix_base_framework.framework_output, + "PMIX client errreg_cbfunc - error handler registered status=%d, reference=%lu", + status, (unsigned long)errhandler_ref); + OPAL_POST_OBJECT(event); + OPAL_PMIX_WAKEUP_THREAD(&event->lock); +} + +int ext4x_client_init(opal_list_t *ilist) +{ + opal_process_name_t pname; + pmix_status_t rc; + int dbg; + opal_ext4x_jobid_trkr_t *job; + opal_ext4x_event_t *event; + pmix_info_t *pinfo; + size_t ninfo, n; + opal_value_t *ival; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "PMIx_client init"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + + if (0 == opal_pmix_base.initialized) { + if (0 < (dbg = opal_output_get_verbosity(opal_pmix_base_framework.framework_output))) { + asprintf(&dbgvalue, "PMIX_DEBUG=%d", dbg); + putenv(dbgvalue); + } + /* check the evars for a mismatch */ + if (OPAL_SUCCESS != (dbg = opal_pmix_ext4x_check_evars())) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return dbg; + } + } + + /* convert the incoming list to info structs */ + if (NULL != ilist && 0 < (ninfo = opal_list_get_size(ilist))) { + PMIX_INFO_CREATE(pinfo, ninfo); + n=0; + OPAL_LIST_FOREACH(ival, ilist, opal_value_t) { + (void)strncpy(pinfo[n].key, ival->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&pinfo[n].value, ival); + ++n; + } + } else { + pinfo = NULL; + ninfo = 0; + } + + /* check for direct modex use-case */ + if (opal_pmix_base_async_modex && !opal_pmix_collect_all_data) { + opal_setenv("PMIX_MCA_gds", "hash", true, &environ); + } + + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + rc = PMIx_Init(&mca_pmix_ext4x_component.myproc, pinfo, ninfo); + if (NULL != pinfo) { + PMIX_INFO_FREE(pinfo, ninfo); + } + if (PMIX_SUCCESS != rc) { + dbg = ext4x_convert_rc(rc); + OPAL_ERROR_LOG(dbg); + return dbg; + } + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + + ++opal_pmix_base.initialized; + if (1 < opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_SUCCESS; + } + + /* store our jobid and rank */ + if (NULL != getenv(OPAL_MCA_PREFIX"orte_launch")) { + /* if we were launched by the OMPI RTE, then + * the jobid is in a special format - so get it */ + mca_pmix_ext4x_component.native_launch = true; + opal_convert_string_to_jobid(&pname.jobid, mca_pmix_ext4x_component.myproc.nspace); + } else { + /* we were launched by someone else, so make the + * jobid just be the hash of the nspace */ + OPAL_HASH_JOBID(mca_pmix_ext4x_component.myproc.nspace, pname.jobid); + } + /* insert this into our list of jobids - it will be the + * first, and so we'll check it first */ + job = OBJ_NEW(opal_ext4x_jobid_trkr_t); + (void)strncpy(job->nspace, mca_pmix_ext4x_component.myproc.nspace, PMIX_MAX_NSLEN); + job->jobid = pname.jobid; + opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super); + + pname.vpid = ext4x_convert_rank(mca_pmix_ext4x_component.myproc.rank); + opal_proc_set_name(&pname); + + /* release the thread in case the event handler fires when + * registered */ + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* register the default event handler */ + event = OBJ_NEW(opal_ext4x_event_t); + opal_list_append(&mca_pmix_ext4x_component.events, &event->super); + PMIX_INFO_CREATE(pinfo, 1); + PMIX_INFO_LOAD(&pinfo[0], PMIX_EVENT_HDLR_NAME, "OPAL-PMIX-2X-DEFAULT", PMIX_STRING); + PMIx_Register_event_handler(NULL, 0, NULL, 0, ext4x_event_hdlr, errreg_cbfunc, event); + OPAL_PMIX_WAIT_THREAD(&event->lock); + PMIX_INFO_FREE(pinfo, 1); + + return OPAL_SUCCESS; + +} + +static void dereg_cbfunc(pmix_status_t st, void *cbdata) +{ + opal_ext4x_event_t *ev = (opal_ext4x_event_t*)cbdata; + OPAL_PMIX_WAKEUP_THREAD(&ev->lock); +} + +int ext4x_client_finalize(void) +{ + pmix_status_t rc; + opal_ext4x_event_t *event, *ev2; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "PMIx_client finalize"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + --opal_pmix_base.initialized; + + if (0 == opal_pmix_base.initialized) { + /* deregister all event handlers */ + OPAL_LIST_FOREACH_SAFE(event, ev2, &mca_pmix_ext4x_component.events, opal_ext4x_event_t) { + OPAL_PMIX_DESTRUCT_LOCK(&event->lock); + OPAL_PMIX_CONSTRUCT_LOCK(&event->lock); + PMIx_Deregister_event_handler(event->index, dereg_cbfunc, (void*)event); + OPAL_PMIX_WAIT_THREAD(&event->lock); + opal_list_remove_item(&mca_pmix_ext4x_component.events, &event->super); + OBJ_RELEASE(event); + } + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + rc = PMIx_Finalize(NULL, 0); + + return ext4x_convert_rc(rc); +} + +int ext4x_tool_init(opal_list_t *info) +{ + pmix_info_t *pinfo; + size_t ninfo, n; + opal_ext4x_jobid_trkr_t *job; + opal_value_t *val; + pmix_status_t rc; + int ret; + opal_process_name_t pname = {OPAL_JOBID_INVALID, OPAL_VPID_INVALID}; + opal_ext4x_event_t *event; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "PMIx_tool init"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + + /* convert the incoming list to info structs */ + if (NULL != info && 0 < (ninfo = opal_list_get_size(info))) { + PMIX_INFO_CREATE(pinfo, ninfo); + n=0; + OPAL_LIST_FOREACH(val, info, opal_value_t) { + (void)strncpy(pinfo[n].key, val->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&pinfo[n].value, val); + ++n; + /* check to see if our name is being given from above */ + if (0 == strcmp(val->key, OPAL_PMIX_TOOL_NSPACE)) { + opal_convert_string_to_jobid(&pname.jobid, val->data.string); + (void)strncpy(mca_pmix_ext4x_component.myproc.nspace, val->data.string, PMIX_MAX_NSLEN); + } else if (0 == strcmp(val->key, OPAL_PMIX_TOOL_RANK)) { + pname.vpid = val->data.name.vpid; + mca_pmix_ext4x_component.myproc.rank = pname.vpid; + } + } + } else { + pinfo = NULL; + ninfo = 0; + } + /* we are going to get our name from the server, or we were given it by the tool, + * so mark as native launch so we don't convert back/forth */ + mca_pmix_ext4x_component.native_launch = true; + + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + rc = PMIx_tool_init(&mca_pmix_ext4x_component.myproc, pinfo, ninfo); + if (NULL != pinfo) { + PMIX_INFO_FREE(pinfo, ninfo); + } + if (PMIX_SUCCESS != rc) { + ret = ext4x_convert_rc(rc); + OPAL_ERROR_LOG(ret); + return ret; + } + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + + ++opal_pmix_base.initialized; + if (1 < opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_SUCCESS; + } + + /* store our jobid and rank */ + opal_convert_string_to_jobid(&pname.jobid, mca_pmix_ext4x_component.myproc.nspace); + pname.vpid = ext4x_convert_rank(mca_pmix_ext4x_component.myproc.rank); + + /* insert this into our list of jobids - it will be the + * first, and so we'll check it first */ + job = OBJ_NEW(opal_ext4x_jobid_trkr_t); + (void)strncpy(job->nspace, mca_pmix_ext4x_component.myproc.nspace, PMIX_MAX_NSLEN); + job->jobid = pname.jobid; + opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super); + + opal_proc_set_name(&pname); + + /* release the thread in case the event handler fires when + * registered */ + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* register the default event handler */ + event = OBJ_NEW(opal_ext4x_event_t); + opal_list_append(&mca_pmix_ext4x_component.events, &event->super); + PMIX_INFO_CREATE(pinfo, 1); + PMIX_INFO_LOAD(&pinfo[0], PMIX_EVENT_HDLR_NAME, "OPAL-PMIX-2X-DEFAULT", PMIX_STRING); + PMIx_Register_event_handler(NULL, 0, NULL, 0, ext4x_event_hdlr, errreg_cbfunc, event); + OPAL_PMIX_WAIT_THREAD(&event->lock); + PMIX_INFO_FREE(pinfo, 1); + + return OPAL_SUCCESS; +} + +int ext4x_tool_fini(void) +{ + pmix_status_t rc; + opal_ext4x_event_t *event, *ev2; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "PMIx_tool finalize"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + --opal_pmix_base.initialized; + + if (0 == opal_pmix_base.initialized) { + /* deregister all event handlers */ + OPAL_LIST_FOREACH_SAFE(event, ev2, &mca_pmix_ext4x_component.events, opal_ext4x_event_t) { + OPAL_PMIX_DESTRUCT_LOCK(&event->lock); + OPAL_PMIX_CONSTRUCT_LOCK(&event->lock); + PMIx_Deregister_event_handler(event->index, dereg_cbfunc, (void*)event); + OPAL_PMIX_WAIT_THREAD(&event->lock); + opal_list_remove_item(&mca_pmix_ext4x_component.events, &event->super); + OBJ_RELEASE(event); + } + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + rc = PMIx_tool_finalize(); + + return ext4x_convert_rc(rc); +} + + +int ext4x_initialized(void) +{ + int init; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "PMIx_client initialized"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + init = opal_pmix_base.initialized; + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + return init; +} + +int ext4x_abort(int flag, const char *msg, + opal_list_t *procs) +{ + pmix_status_t rc; + pmix_proc_t *parray=NULL; + size_t n, cnt=0; + opal_namelist_t *ptr; + char *nsptr; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "PMIx_client abort"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* convert the list of procs to an array + * of pmix_proc_t */ + if (NULL != procs && 0 < (cnt = opal_list_get_size(procs))) { + PMIX_PROC_CREATE(parray, cnt); + n=0; + OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { + if (NULL == (nsptr = ext4x_convert_jobid(ptr->name.jobid))) { + PMIX_PROC_FREE(parray, cnt); + return OPAL_ERR_NOT_FOUND; + } + (void)strncpy(parray[n].nspace, nsptr, PMIX_MAX_NSLEN); + parray[n].rank = ext4x_convert_opalrank(ptr->name.vpid); + ++n; + } + } + + /* call the library abort - this is a blocking call */ + rc = PMIx_Abort(flag, msg, parray, cnt); + + /* release the array */ + PMIX_PROC_FREE(parray, cnt); + + return ext4x_convert_rc(rc); +} + +int ext4x_store_local(const opal_process_name_t *proc, opal_value_t *val) +{ + pmix_value_t kv; + pmix_status_t rc; + pmix_proc_t p; + char *nsptr; + opal_ext4x_jobid_trkr_t *job; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + if (NULL != proc) { + if (NULL == (nsptr = ext4x_convert_jobid(proc->jobid))) { + job = OBJ_NEW(opal_ext4x_jobid_trkr_t); + (void)opal_snprintf_jobid(job->nspace, PMIX_MAX_NSLEN, proc->jobid); + job->jobid = proc->jobid; + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + nsptr = job->nspace; + } + (void)strncpy(p.nspace, nsptr, PMIX_MAX_NSLEN); + p.rank = ext4x_convert_opalrank(proc->vpid); + } else { + /* use our name */ + (void)strncpy(p.nspace, mca_pmix_ext4x_component.myproc.nspace, PMIX_MAX_NSLEN); + p.rank = ext4x_convert_opalrank(OPAL_PROC_MY_NAME.vpid); + } + + PMIX_VALUE_CONSTRUCT(&kv); + ext4x_value_load(&kv, val); + + /* call the library - this is a blocking call */ + rc = PMIx_Store_internal(&p, val->key, &kv); + PMIX_VALUE_DESTRUCT(&kv); + + return ext4x_convert_rc(rc); +} + +int ext4x_commit(void) +{ + pmix_status_t rc; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + rc = PMIx_Commit(); + return ext4x_convert_rc(rc); +} + +static void opcbfunc(pmix_status_t status, void *cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + + OPAL_ACQUIRE_OBJECT(op); + if (NULL != op->opcbfunc) { + op->opcbfunc(ext4x_convert_rc(status), op->cbdata); + } + OBJ_RELEASE(op); +} + +int ext4x_fence(opal_list_t *procs, int collect_data) +{ + pmix_status_t rc; + opal_namelist_t *ptr; + char *nsptr; + size_t cnt = 0, n; + pmix_proc_t *parray = NULL; + pmix_info_t info, *iptr; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "PMIx_client fence"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + /* convert the list of procs to an array + * of pmix_proc_t */ + if (NULL != procs && 0 < (cnt = opal_list_get_size(procs))) { + PMIX_PROC_CREATE(parray, cnt); + n=0; + OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { + if (NULL == (nsptr = ext4x_convert_jobid(ptr->name.jobid))) { + PMIX_PROC_FREE(parray, cnt); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_FOUND; + } + (void)strncpy(parray[n].nspace, nsptr, PMIX_MAX_NSLEN); + parray[n].rank = ext4x_convert_opalrank(ptr->name.vpid); + ++n; + } + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + if (collect_data) { + PMIX_INFO_CONSTRUCT(&info); + (void)strncpy(info.key, PMIX_COLLECT_DATA, PMIX_MAX_KEYLEN); + info.value.type = PMIX_BOOL; + info.value.data.flag = true; + iptr = &info; + n = 1; + } else { + iptr = NULL; + n = 0; + } + + rc = PMIx_Fence(parray, cnt, iptr, n); + if (collect_data) { + PMIX_INFO_DESTRUCT(&info); + } + if (NULL != parray) { + PMIX_PROC_FREE(parray, cnt); + } + + return ext4x_convert_rc(rc); +} + +int ext4x_fencenb(opal_list_t *procs, int collect_data, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_status_t rc; + pmix_proc_t *parray=NULL; + size_t n, cnt=0; + opal_namelist_t *ptr; + ext4x_opcaddy_t *op; + char *nsptr; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "PMIx_client fencenb"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + /* convert the list of procs to an array + * of pmix_proc_t */ + if (NULL != procs && 0 < (cnt = opal_list_get_size(procs))) { + PMIX_PROC_CREATE(parray, cnt); + n=0; + OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { + if (NULL == (nsptr = ext4x_convert_jobid(ptr->name.jobid))) { + PMIX_PROC_FREE(parray, cnt); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_FOUND; + } + (void)strncpy(parray[n].nspace, nsptr, PMIX_MAX_NSLEN); + parray[n].rank = ext4x_convert_opalrank(ptr->name.vpid); + ++n; + } + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* create the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->opcbfunc = cbfunc; + op->cbdata = cbdata; + op->procs = parray; + op->nprocs = cnt; + + if (collect_data) { + op->ninfo = 1; + PMIX_INFO_CREATE(op->info, op->ninfo); + PMIX_INFO_LOAD(&op->info[0], PMIX_COLLECT_DATA, NULL, PMIX_BOOL); + } + + /* call the library function */ + rc = PMIx_Fence_nb(op->procs, op->nprocs, op->info, op->ninfo, opcbfunc, op); + return ext4x_convert_rc(rc); +} + +int ext4x_put(opal_pmix_scope_t opal_scope, + opal_value_t *val) +{ + pmix_value_t kv; + pmix_scope_t pmix_scope = ext4x_convert_opalscope(opal_scope); + pmix_status_t rc; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "PMIx_client put"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + PMIX_VALUE_CONSTRUCT(&kv); + ext4x_value_load(&kv, val); + + rc = PMIx_Put(pmix_scope, val->key, &kv); + PMIX_VALUE_DESTRUCT(&kv); + return ext4x_convert_rc(rc); +} + +int ext4x_get(const opal_process_name_t *proc, const char *key, + opal_list_t *info, opal_value_t **val) +{ + pmix_status_t rc; + pmix_proc_t p; + char *nsptr; + pmix_info_t *pinfo = NULL; + size_t sz = 0, n; + opal_value_t *ival; + pmix_value_t *pval = NULL; + int ret; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "%s ext4x:client get on proc %s key %s", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + (NULL == proc) ? "NULL" : OPAL_NAME_PRINT(*proc), key); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + if (NULL == proc) { + /* if they are asking for our jobid, then return it */ + if (0 == strcmp(key, OPAL_PMIX_JOBID)) { + (*val) = OBJ_NEW(opal_value_t); + (*val)->key = strdup(key); + (*val)->type = OPAL_UINT32; + (*val)->data.uint32 = OPAL_PROC_MY_NAME.jobid; + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_SUCCESS; + } + /* if they are asking for our rank, return it */ + if (0 == strcmp(key, OPAL_PMIX_RANK)) { + (*val) = OBJ_NEW(opal_value_t); + (*val)->key = strdup(key); + (*val)->type = OPAL_INT; + (*val)->data.integer = ext4x_convert_rank(mca_pmix_ext4x_component.myproc.rank); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_SUCCESS; + } + } + *val = NULL; + + if (NULL == proc) { + (void)strncpy(p.nspace, mca_pmix_ext4x_component.myproc.nspace, PMIX_MAX_NSLEN); + p.rank = ext4x_convert_rank(PMIX_RANK_WILDCARD); + } else { + if (NULL == (nsptr = ext4x_convert_jobid(proc->jobid))) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_FOUND; + } + (void)strncpy(p.nspace, nsptr, PMIX_MAX_NSLEN); + p.rank = ext4x_convert_opalrank(proc->vpid); + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + if (NULL != info && 0 < (sz = opal_list_get_size(info))) { + PMIX_INFO_CREATE(pinfo, sz); + n=0; + OPAL_LIST_FOREACH(ival, info, opal_value_t) { + (void)strncpy(pinfo[n].key, ival->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&pinfo[n].value, ival); + ++n; + } + } + + rc = PMIx_Get(&p, key, pinfo, sz, &pval); + if (PMIX_SUCCESS == rc) { + ival = OBJ_NEW(opal_value_t); + if (NULL != key) { + ival->key = strdup(key); + } + if (OPAL_SUCCESS != (ret = ext4x_value_unload(ival, pval))) { + rc = ext4x_convert_opalrc(ret); + } else { + *val = ival; + } + PMIX_VALUE_FREE(pval, 1); + } + PMIX_INFO_FREE(pinfo, sz); + + return ext4x_convert_rc(rc); +} + +static void val_cbfunc(pmix_status_t status, + pmix_value_t *kv, void *cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + int rc; + opal_value_t val, *v=NULL; + + OPAL_ACQUIRE_OBJECT(op); + OBJ_CONSTRUCT(&val, opal_value_t); + if (NULL != op->nspace) { + val.key = strdup(op->nspace); + } + rc = ext4x_convert_opalrc(status); + if (PMIX_SUCCESS == status && NULL != kv) { + rc = ext4x_value_unload(&val, kv); + v = &val; + } + + if (NULL != op->valcbfunc) { + op->valcbfunc(rc, v, op->cbdata); + } + OBJ_DESTRUCT(&val); + OBJ_RELEASE(op); +} + +int ext4x_getnb(const opal_process_name_t *proc, const char *key, + opal_list_t *info, + opal_pmix_value_cbfunc_t cbfunc, void *cbdata) +{ + ext4x_opcaddy_t *op; + opal_value_t *val; + pmix_status_t rc; + char *nsptr; + size_t n; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "%s PMIx_client get_nb on proc %s key %s", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + (NULL == proc) ? "NULL" : OPAL_NAME_PRINT(*proc), key); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + if (NULL == proc) { + /* if they are asking for our jobid, then return it */ + if (0 == strcmp(key, OPAL_PMIX_JOBID)) { + if (NULL != cbfunc) { + val = OBJ_NEW(opal_value_t); + val->key = strdup(key); + val->type = OPAL_UINT32; + val->data.uint32 = OPAL_PROC_MY_NAME.jobid; + cbfunc(OPAL_SUCCESS, val, cbdata); + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_SUCCESS; + } + /* if they are asking for our rank, return it */ + if (0 == strcmp(key, OPAL_PMIX_RANK)) { + if (NULL != cbfunc) { + val = OBJ_NEW(opal_value_t); + val->key = strdup(key); + val->type = OPAL_INT; + val->data.integer = ext4x_convert_rank(mca_pmix_ext4x_component.myproc.rank); + cbfunc(OPAL_SUCCESS, val, cbdata); + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_SUCCESS; + } + } + + /* create the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->valcbfunc = cbfunc; + op->cbdata = cbdata; + if (NULL != key) { + op->nspace = strdup(key); + } + if (NULL == proc) { + (void)strncpy(op->p.nspace, mca_pmix_ext4x_component.myproc.nspace, PMIX_MAX_NSLEN); + op->p.rank = ext4x_convert_rank(PMIX_RANK_WILDCARD); + } else { + if (NULL == (nsptr = ext4x_convert_jobid(proc->jobid))) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_FOUND; + } + (void)strncpy(op->p.nspace, nsptr, PMIX_MAX_NSLEN); + op->p.rank = ext4x_convert_opalrank(proc->vpid); + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + if (NULL != info && 0 < (op->sz = opal_list_get_size(info))) { + PMIX_INFO_CREATE(op->info, op->sz); + n=0; + OPAL_LIST_FOREACH(val, info, opal_value_t) { + (void)strncpy(op->info[n].key, val->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&op->info[n].value, val); + ++n; + } + } + + /* call the library function */ + rc = PMIx_Get_nb(&op->p, key, op->info, op->sz, val_cbfunc, op); + if (PMIX_SUCCESS != rc) { + OBJ_RELEASE(op); + } + + return ext4x_convert_rc(rc); +} + +int ext4x_publish(opal_list_t *info) +{ + pmix_info_t *pinfo; + pmix_status_t ret; + opal_value_t *iptr; + size_t sz, n; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "PMIx_client publish"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + if (NULL == info) { + return OPAL_ERR_BAD_PARAM; + } + + sz = opal_list_get_size(info); + if (0 < sz) { + PMIX_INFO_CREATE(pinfo, sz); + n=0; + OPAL_LIST_FOREACH(iptr, info, opal_value_t) { + (void)strncpy(pinfo[n].key, iptr->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&pinfo[n].value, iptr); + ++n; + } + } else { + pinfo = NULL; + } + + ret = PMIx_Publish(pinfo, sz); + if (0 < sz) { + PMIX_INFO_FREE(pinfo, sz); + } + + return ext4x_convert_rc(ret); +} + +int ext4x_publishnb(opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_status_t ret; + opal_value_t *iptr; + size_t n; + ext4x_opcaddy_t *op; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "PMIx_client publish_nb"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + if (NULL == info) { + return OPAL_ERR_BAD_PARAM; + } + + /* create the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->opcbfunc = cbfunc; + op->cbdata = cbdata; + + op->sz = opal_list_get_size(info); + if (0 < op->sz) { + PMIX_INFO_CREATE(op->info, op->sz); + n=0; + OPAL_LIST_FOREACH(iptr, info, opal_value_t) { + (void)strncpy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&op->info[n].value, iptr); + ++n; + } + } + + ret = PMIx_Publish_nb(op->info, op->sz, opcbfunc, op); + + return ext4x_convert_rc(ret); +} + +int ext4x_lookup(opal_list_t *data, opal_list_t *info) +{ + opal_pmix_pdata_t *d; + pmix_pdata_t *pdata; + pmix_info_t *pinfo = NULL; + pmix_status_t rc; + size_t cnt, n, sz = 0; + opal_value_t *iptr; + opal_ext4x_jobid_trkr_t *jptr, *job; + int ret; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "ext4x:client lookup"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + if (NULL == data || 0 == (cnt = opal_list_get_size(data))) { + return OPAL_ERR_BAD_PARAM; + } + PMIX_PDATA_CREATE(pdata, cnt); + n = 0; + OPAL_LIST_FOREACH(d, data, opal_pmix_pdata_t) { + (void)strncpy(pdata[n].key, d->value.key, PMIX_MAX_KEYLEN); + ++n; + } + + if (NULL != info && 0 < (sz = opal_list_get_size(info))) { + PMIX_INFO_CREATE(pinfo, sz); + n=0; + OPAL_LIST_FOREACH(iptr, info, opal_value_t) { + (void)strncpy(pinfo[n].key, iptr->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&pinfo[n].value, iptr); + ++n; + } + } + + rc = PMIx_Lookup(pdata, cnt, pinfo, sz); + if (PMIX_SUCCESS == rc) { + /* load the answers back into the list */ + n=0; + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + OPAL_LIST_FOREACH(d, data, opal_pmix_pdata_t) { + if (mca_pmix_ext4x_component.native_launch) { + /* if we were launched by the OMPI RTE, then + * the jobid is in a special format - so get it */ + opal_convert_string_to_jobid(&d->proc.jobid, pdata[n].proc.nspace); + } else { + /* we were launched by someone else, so make the + * jobid just be the hash of the nspace */ + OPAL_HASH_JOBID(pdata[n].proc.nspace, d->proc.jobid); + } + /* if we don't already have it, add this to our jobid tracker */ + job = NULL; + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) { + if (jptr->jobid == d->proc.jobid) { + job = jptr; + break; + } + } + if (NULL == job) { + job = OBJ_NEW(opal_ext4x_jobid_trkr_t); + (void)strncpy(job->nspace, pdata[n].proc.nspace, PMIX_MAX_NSLEN); + job->jobid = d->proc.jobid; + opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super); + } + d->proc.vpid = ext4x_convert_rank(pdata[n].proc.rank); + if (OPAL_SUCCESS != (ret = ext4x_value_unload(&d->value, &pdata[n].value))) { + OPAL_ERROR_LOG(ret); + } + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + } + PMIX_PDATA_FREE(pdata, cnt); + if (NULL != pinfo) { + PMIX_INFO_FREE(pinfo, sz); + } + return ext4x_convert_rc(rc); +} + +static void lk_cbfunc(pmix_status_t status, + pmix_pdata_t data[], size_t ndata, + void *cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + opal_pmix_pdata_t *d; + opal_list_t results, *r = NULL; + int rc; + size_t n; + opal_ext4x_jobid_trkr_t *job, *jptr; + + OPAL_ACQUIRE_OBJECT(op); + + if (NULL == op->lkcbfunc) { + OBJ_RELEASE(op); + return; + } + + rc = ext4x_convert_rc(op->status); + if (OPAL_SUCCESS == rc) { + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + OBJ_CONSTRUCT(&results, opal_list_t); + for (n=0; n < ndata; n++) { + d = OBJ_NEW(opal_pmix_pdata_t); + opal_list_append(&results, &d->super); + if (mca_pmix_ext4x_component.native_launch) { + /* if we were launched by the OMPI RTE, then + * the jobid is in a special format - so get it */ + opal_convert_string_to_jobid(&d->proc.jobid, data[n].proc.nspace); + } else { + /* we were launched by someone else, so make the + * jobid just be the hash of the nspace */ + OPAL_HASH_JOBID(data[n].proc.nspace, d->proc.jobid); + } + /* if we don't already have it, add this to our jobid tracker */ + job = NULL; + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) { + if (jptr->jobid == d->proc.jobid) { + job = jptr; + break; + } + } + if (NULL == job) { + job = OBJ_NEW(opal_ext4x_jobid_trkr_t); + (void)strncpy(job->nspace, data[n].proc.nspace, PMIX_MAX_NSLEN); + job->jobid = d->proc.jobid; + opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super); + } + d->proc.vpid = ext4x_convert_rank(data[n].proc.rank); + d->value.key = strdup(data[n].key); + rc = ext4x_value_unload(&d->value, &data[n].value); + if (OPAL_SUCCESS != rc) { + rc = OPAL_ERR_BAD_PARAM; + OPAL_ERROR_LOG(rc); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + goto release; + } + } + r = &results; + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + } + + release: + /* execute the callback */ + op->lkcbfunc(rc, r, op->cbdata); + + if (NULL != r) { + OPAL_LIST_DESTRUCT(&results); + } + OBJ_RELEASE(op); +} + +int ext4x_lookupnb(char **keys, opal_list_t *info, + opal_pmix_lookup_cbfunc_t cbfunc, void *cbdata) +{ + pmix_status_t ret; + ext4x_opcaddy_t *op; + opal_value_t *iptr; + size_t n; + + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "ext4x:client lookup_nb"); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* create the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->lkcbfunc = cbfunc; + op->cbdata = cbdata; + + if (NULL != info && 0 < (op->sz = opal_list_get_size(info))) { + PMIX_INFO_CREATE(op->info, op->sz); + n=0; + OPAL_LIST_FOREACH(iptr, info, opal_value_t) { + (void)strncpy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&op->info[n].value, iptr); + ++n; + } + } + ret = PMIx_Lookup_nb(keys, op->info, op->sz, lk_cbfunc, op); + + return ext4x_convert_rc(ret); +} + +int ext4x_unpublish(char **keys, opal_list_t *info) +{ + pmix_status_t ret; + size_t ninfo, n; + pmix_info_t *pinfo; + opal_value_t *iptr; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + if (NULL != info && 0 < (ninfo = opal_list_get_size(info))) { + PMIX_INFO_CREATE(pinfo, ninfo); + n=0; + OPAL_LIST_FOREACH(iptr, info, opal_value_t) { + (void)strncpy(pinfo[n].key, iptr->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&pinfo[n].value, iptr); + ++n; + } + } else { + pinfo = NULL; + ninfo = 0; + } + + ret = PMIx_Unpublish(keys, pinfo, ninfo); + PMIX_INFO_FREE(pinfo, ninfo); + + return ext4x_convert_rc(ret); +} + +int ext4x_unpublishnb(char **keys, opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_status_t ret; + ext4x_opcaddy_t *op; + opal_value_t *iptr; + size_t n; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* create the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->opcbfunc = cbfunc; + op->cbdata = cbdata; + + if (NULL != info && 0 < (op->sz = opal_list_get_size(info))) { + PMIX_INFO_CREATE(op->info, op->sz); + n=0; + OPAL_LIST_FOREACH(iptr, info, opal_value_t) { + (void)strncpy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&op->info[n].value, iptr); + ++n; + } + } + + ret = PMIx_Unpublish_nb(keys, op->info, op->sz, opcbfunc, op); + + return ext4x_convert_rc(ret); +} + +int ext4x_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid) +{ + pmix_status_t rc; + pmix_info_t *info = NULL; + pmix_app_t *papps; + size_t ninfo = 0, napps, n, m; + opal_value_t *ival; + opal_pmix_app_t *app; + char nspace[PMIX_MAX_NSLEN+1]; + opal_ext4x_jobid_trkr_t *job; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + *jobid = OPAL_JOBID_INVALID; + + if (NULL != job_info && 0 < (ninfo = opal_list_get_size(job_info))) { + PMIX_INFO_CREATE(info, ninfo); + n=0; + OPAL_LIST_FOREACH(ival, job_info, opal_value_t) { + (void)strncpy(info[n].key, ival->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&info[n].value, ival); + ++n; + } + } + + napps = opal_list_get_size(apps); + PMIX_APP_CREATE(papps, napps); + n=0; + OPAL_LIST_FOREACH(app, apps, opal_pmix_app_t) { + papps[n].cmd = strdup(app->cmd); + if (NULL != app->argv) { + papps[n].argv = opal_argv_copy(app->argv); + } + if (NULL != app->env) { + papps[n].env = opal_argv_copy(app->env); + } + if (NULL != app->cwd) { + papps[n].cwd = strdup(app->cwd); + } + papps[n].maxprocs = app->maxprocs; + if (0 < (papps[n].ninfo = opal_list_get_size(&app->info))) { + PMIX_INFO_CREATE(papps[n].info, papps[n].ninfo); + m=0; + OPAL_LIST_FOREACH(ival, &app->info, opal_value_t) { + (void)strncpy(papps[n].info[m].key, ival->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&papps[n].info[m].value, ival); + ++m; + } + } + ++n; + } + + rc = PMIx_Spawn(info, ninfo, papps, napps, nspace); + if (PMIX_SUCCESS == rc) { + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (mca_pmix_ext4x_component.native_launch) { + /* if we were launched by the OMPI RTE, then + * the jobid is in a special format - so get it */ + opal_convert_string_to_jobid(jobid, nspace); + } else { + /* we were launched by someone else, so make the + * jobid just be the hash of the nspace */ + OPAL_HASH_JOBID(nspace, *jobid); + } + /* add this to our jobid tracker */ + job = OBJ_NEW(opal_ext4x_jobid_trkr_t); + (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN); + job->jobid = *jobid; + opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + } + return rc; +} + +static void spcbfunc(pmix_status_t status, + char *nspace, void *cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + opal_ext4x_jobid_trkr_t *job; + opal_jobid_t jobid = OPAL_JOBID_INVALID; + int rc; + + OPAL_ACQUIRE_OBJECT(op); + + rc = ext4x_convert_rc(status); + if (PMIX_SUCCESS == status) { + /* this is in the PMIx local thread - need to protect + * the framework-level data */ + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (mca_pmix_ext4x_component.native_launch) { + /* if we were launched by the OMPI RTE, then + * the jobid is in a special format - so get it */ + opal_convert_string_to_jobid(&jobid, nspace); + } else { + /* we were launched by someone else, so make the + * jobid just be the hash of the nspace */ + OPAL_HASH_JOBID(nspace, jobid); + } + /* add this to our jobid tracker */ + job = OBJ_NEW(opal_ext4x_jobid_trkr_t); + (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN); + job->jobid = jobid; + opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + } + + op->spcbfunc(rc, jobid, op->cbdata); + OBJ_RELEASE(op); +} + +int ext4x_spawnnb(opal_list_t *job_info, opal_list_t *apps, + opal_pmix_spawn_cbfunc_t cbfunc, void *cbdata) +{ + pmix_status_t ret; + ext4x_opcaddy_t *op; + size_t n, m; + opal_value_t *info; + opal_pmix_app_t *app; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* create the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->spcbfunc = cbfunc; + op->cbdata = cbdata; + + if (NULL != job_info && 0 < (op->ninfo = opal_list_get_size(job_info))) { + PMIX_INFO_CREATE(op->info, op->ninfo); + n=0; + OPAL_LIST_FOREACH(info, job_info, opal_value_t) { + (void)strncpy(op->info[n].key, info->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&op->info[n].value, info); + ++n; + } + } + + op->sz = opal_list_get_size(apps); + PMIX_APP_CREATE(op->apps, op->sz); + n=0; + OPAL_LIST_FOREACH(app, apps, opal_pmix_app_t) { + op->apps[n].cmd = strdup(app->cmd); + if (NULL != app->argv) { + op->apps[n].argv = opal_argv_copy(app->argv); + } + if (NULL != app->env) { + op->apps[n].env = opal_argv_copy(app->env); + } + op->apps[n].maxprocs = app->maxprocs; + if (0 < (op->apps[n].ninfo = opal_list_get_size(&app->info))) { + PMIX_INFO_CREATE(op->apps[n].info, op->apps[n].ninfo); + m=0; + OPAL_LIST_FOREACH(info, &app->info, opal_value_t) { + (void)strncpy(op->apps[n].info[m].key, info->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&op->apps[n].info[m].value, info); + ++m; + } + } + ++n; + } + + ret = PMIx_Spawn_nb(op->info, op->ninfo, op->apps, op->sz, spcbfunc, op); + + return ext4x_convert_rc(ret); +} + +int ext4x_connect(opal_list_t *procs) +{ + pmix_proc_t *p; + size_t nprocs; + opal_namelist_t *ptr; + pmix_status_t ret; + char *nsptr; + size_t n; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "ext4x:client connect"); + + /* protect against bozo error */ + if (NULL == procs || 0 == (nprocs = opal_list_get_size(procs))) { + return OPAL_ERR_BAD_PARAM; + } + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + /* convert the list of procs to an array + * of pmix_proc_t */ + PMIX_PROC_CREATE(p, nprocs); + n=0; + OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { + if (NULL == (nsptr = ext4x_convert_jobid(ptr->name.jobid))) { + PMIX_PROC_FREE(p, nprocs); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_FOUND; + } + (void)strncpy(p[n].nspace, nsptr, PMIX_MAX_NSLEN); + p[n].rank = ext4x_convert_opalrank(ptr->name.vpid); + ++n; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + ret = PMIx_Connect(p, nprocs, NULL, 0); + PMIX_PROC_FREE(p, nprocs); + + return ext4x_convert_rc(ret); +} + +int ext4x_connectnb(opal_list_t *procs, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + ext4x_opcaddy_t *op; + opal_namelist_t *ptr; + pmix_status_t ret; + char *nsptr; + size_t n; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "ext4x:client connect NB"); + + /* protect against bozo error */ + if (NULL == procs || 0 == opal_list_get_size(procs)) { + return OPAL_ERR_BAD_PARAM; + } + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + /* create the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->opcbfunc = cbfunc; + op->cbdata = cbdata; + op->nprocs = opal_list_get_size(procs); + + /* convert the list of procs to an array + * of pmix_proc_t */ + PMIX_PROC_CREATE(op->procs, op->nprocs); + n=0; + OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { + if (NULL == (nsptr = ext4x_convert_jobid(ptr->name.jobid))) { + OBJ_RELEASE(op); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_FOUND; + } + (void)strncpy(op->procs[n].nspace, nsptr, PMIX_MAX_NSLEN); + op->procs[n].rank = ext4x_convert_opalrank(ptr->name.vpid); + ++n; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + ret = PMIx_Connect_nb(op->procs, op->nprocs, NULL, 0, opcbfunc, op); + if (PMIX_SUCCESS != ret) { + OBJ_RELEASE(op); + } + return ext4x_convert_rc(ret); +} + +int ext4x_disconnect(opal_list_t *procs) +{ + size_t nprocs, n; + opal_namelist_t *ptr; + pmix_status_t ret=PMIX_SUCCESS; + pmix_proc_t *p; + char *nsptr; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "ext4x:client disconnect"); + + /* protect against bozo error */ + if (NULL == procs || 0 == (nprocs = opal_list_get_size(procs))) { + return OPAL_ERR_BAD_PARAM; + } + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + /* convert the list of procs to an array + * of pmix_proc_t */ + PMIX_PROC_CREATE(p, nprocs); + n=0; + OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { + if (NULL == (nsptr = ext4x_convert_jobid(ptr->name.jobid))) { + PMIX_PROC_FREE(p, nprocs); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_FOUND; + } + (void)strncpy(p[n].nspace, nsptr, PMIX_MAX_NSLEN); + p[n].rank = ext4x_convert_opalrank(ptr->name.vpid); + ++n; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + ret = PMIx_Disconnect(p, nprocs, NULL, 0); + PMIX_PROC_FREE(p, nprocs); + + return ext4x_convert_rc(ret); +} + +int ext4x_disconnectnb(opal_list_t *procs, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + ext4x_opcaddy_t *op; + opal_namelist_t *ptr; + pmix_status_t ret; + char *nsptr; + size_t n; + + opal_output_verbose(1, opal_pmix_base_framework.framework_output, + "ext4x:client disconnect NB"); + + /* protect against bozo error */ + if (NULL == procs || 0 == opal_list_get_size(procs)) { + return OPAL_ERR_BAD_PARAM; + } + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + /* create the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->opcbfunc = cbfunc; + op->cbdata = cbdata; + op->nprocs = opal_list_get_size(procs); + + /* convert the list of procs to an array + * of pmix_proc_t */ + PMIX_PROC_CREATE(op->procs, op->nprocs); + n=0; + OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { + if (NULL == (nsptr = ext4x_convert_jobid(ptr->name.jobid))) { + OBJ_RELEASE(op); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_FOUND; + } + (void)strncpy(op->procs[n].nspace, nsptr, PMIX_MAX_NSLEN); + op->procs[n].rank = ext4x_convert_opalrank(ptr->name.vpid); + ++n; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + ret = PMIx_Disconnect_nb(op->procs, op->nprocs, NULL, 0, opcbfunc, op); + if (PMIX_SUCCESS != ret) { + OBJ_RELEASE(op); + } + return ext4x_convert_rc(ret); +} + +int ext4x_resolve_peers(const char *nodename, + opal_jobid_t jobid, + opal_list_t *procs) +{ + pmix_status_t ret; + char *nspace; + pmix_proc_t *array=NULL; + size_t nprocs, n; + opal_namelist_t *nm; + opal_ext4x_jobid_trkr_t *job; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + if (OPAL_JOBID_WILDCARD != jobid) { + if (NULL == (nspace = ext4x_convert_jobid(jobid))) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_FOUND; + } + } else { + nspace = NULL; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + ret = PMIx_Resolve_peers(nodename, nspace, &array, &nprocs); + + if (NULL != array && 0 < nprocs) { + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + for (n=0; n < nprocs; n++) { + nm = OBJ_NEW(opal_namelist_t); + opal_list_append(procs, &nm->super); + if (mca_pmix_ext4x_component.native_launch) { + /* if we were launched by the OMPI RTE, then + * the jobid is in a special format - so get it */ + opal_convert_string_to_jobid(&nm->name.jobid, array[n].nspace); + } else { + /* we were launched by someone else, so make the + * jobid just be the hash of the nspace */ + OPAL_HASH_JOBID(array[n].nspace, nm->name.jobid); + } + /* if we don't already have it, add this to our jobid tracker */ + if (NULL == ext4x_convert_jobid(nm->name.jobid)) { + job = OBJ_NEW(opal_ext4x_jobid_trkr_t); + (void)strncpy(job->nspace, array[n].nspace, PMIX_MAX_NSLEN); + job->jobid = nm->name.jobid; + opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super); + } + nm->name.vpid = ext4x_convert_rank(array[n].rank); + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + } + PMIX_PROC_FREE(array, nprocs); + return ext4x_convert_rc(ret); +} + +int ext4x_resolve_nodes(opal_jobid_t jobid, char **nodelist) +{ + pmix_status_t ret; + char *nsptr; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + if (NULL == (nsptr = ext4x_convert_jobid(jobid))) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_FOUND; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + ret = PMIx_Resolve_nodes(nsptr, nodelist); + + return ext4x_convert_rc(ret); +} + +static void relcbfunc(void *cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + OBJ_RELEASE(op); +} + +static void infocbfunc(pmix_status_t status, + pmix_info_t *info, size_t ninfo, + void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + int rc; + + if (NULL != release_fn) { + release_fn(release_cbdata); + } + rc = ext4x_convert_rc(status); + if (NULL != op->qcbfunc) { + op->qcbfunc(rc, NULL, op->cbdata, relcbfunc, op); + } else { + OBJ_RELEASE(op); + } +} + +int ext4x_allocate(opal_pmix_alloc_directive_t directive, + opal_list_t *info, + opal_pmix_info_cbfunc_t cbfunc, void *cbdata) +{ + return OPAL_ERR_NOT_SUPPORTED; +} + +int ext4x_job_control(opal_list_t *targets, + opal_list_t *directives, + opal_pmix_info_cbfunc_t cbfunc, void *cbdata) +{ + ext4x_opcaddy_t *op; + size_t n; + opal_namelist_t *ptr; + opal_value_t *iptr; + pmix_status_t rc; + char *nsptr; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + /* create the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->qcbfunc = cbfunc; + op->cbdata = cbdata; + if (NULL != targets) { + op->nprocs = opal_list_get_size(targets); + + /* convert the list of procs to an array + * of pmix_proc_t */ + PMIX_PROC_CREATE(op->procs, op->nprocs); + n=0; + OPAL_LIST_FOREACH(ptr, targets, opal_namelist_t) { + if (NULL == (nsptr = ext4x_convert_jobid(ptr->name.jobid))) { + OBJ_RELEASE(op); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_FOUND; + } + (void)strncpy(op->procs[n].nspace, nsptr, PMIX_MAX_NSLEN); + op->procs[n].rank = ext4x_convert_opalrank(ptr->name.vpid); + ++n; + } + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + if (NULL != directives && 0 < (op->ninfo = opal_list_get_size(directives))) { + PMIX_INFO_CREATE(op->info, op->ninfo); + n=0; + OPAL_LIST_FOREACH(iptr, directives, opal_value_t) { + (void)strncpy(op->info[n].key, iptr->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&op->info[n].value, iptr); + ++n; + } + } + + rc = PMIx_Job_control_nb(op->procs,op->nprocs, op->info, op->ninfo, infocbfunc, op); + if (PMIX_SUCCESS != rc) { + OBJ_RELEASE(op); + } + return ext4x_convert_rc(rc); +} diff --git a/opal/mca/pmix/ext4x/ext4x_component.c b/opal/mca/pmix/ext4x/ext4x_component.c new file mode 100644 index 00000000000..aa834195b50 --- /dev/null +++ b/opal/mca/pmix/ext4x/ext4x_component.c @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2016-2018 Cisco Systems, Inc. All rights reserved + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include "opal_config.h" + +#include "opal/constants.h" +#include "opal/class/opal_list.h" +#include "opal/util/proc.h" +#include "opal/util/show_help.h" +#include "opal/mca/pmix/pmix.h" +#include "ext4x.h" + +/* + * Public string showing the pmix external component version number + */ +const char *opal_pmix_ext4x_component_version_string = + "OPAL ext4x MCA component version " OPAL_VERSION; + +/* + * Local function + */ +static int external_register(void); +static int external_open(void); +static int external_close(void); +static int external_component_query(mca_base_module_t **module, int *priority); + +/* + * Local variable + */ +static char *pmix_library_version = NULL; + + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ + +mca_pmix_ext4x_component_t mca_pmix_ext4x_component = { + { + /* First, the mca_component_t struct containing meta information + about the component itself */ + + .base_version = { + /* Indicate that we are a pmix v1.1.0 component (which also + implies a specific MCA version) */ + + OPAL_PMIX_BASE_VERSION_2_0_0, + + /* Component name and version */ + + .mca_component_name = "ext4x", + MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION, + OPAL_RELEASE_VERSION), + + /* Component open and close functions */ + + .mca_open_component = external_open, + .mca_close_component = external_close, + .mca_query_component = external_component_query, + .mca_register_component_params = external_register + }, + /* Next the MCA v1.0.0 component meta data */ + .base_data = { + /* The component is checkpoint ready */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + } + }, + .native_launch = false +}; + +static int external_register(void) +{ + mca_base_component_t *component = &mca_pmix_ext4x_component.super.base_version; + + mca_pmix_ext4x_component.silence_warning = false; + (void) mca_base_component_var_register (component, "silence_warning", + "Silence warning about PMIX_INSTALL_PREFIX", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, + OPAL_INFO_LVL_4, + MCA_BASE_VAR_SCOPE_READONLY, + &mca_pmix_ext4x_component.silence_warning); + + asprintf(&pmix_library_version, + "PMIx library version %s (embedded in Open MPI)", PMIx_Get_version()); + (void) mca_base_component_var_register(component, "library_version", + "Version of the underlying PMIx library", + MCA_BASE_VAR_TYPE_STRING, + NULL, 0, 0, + OPAL_INFO_LVL_4, + MCA_BASE_VAR_SCOPE_CONSTANT, + &pmix_library_version); + + return OPAL_SUCCESS; +} + +static int external_open(void) +{ + const char *version; + + mca_pmix_ext4x_component.evindex = 0; + OBJ_CONSTRUCT(&mca_pmix_ext4x_component.jobids, opal_list_t); + OBJ_CONSTRUCT(&mca_pmix_ext4x_component.events, opal_list_t); + OBJ_CONSTRUCT(&mca_pmix_ext4x_component.dmdx, opal_list_t); + + version = PMIx_Get_version(); + if ('3' != version[0]) { + opal_show_help("help-pmix-base.txt", + "incorrect-pmix", true, version, "v3.x"); + return OPAL_ERROR; + } + return OPAL_SUCCESS; +} + +static int external_close(void) +{ + OPAL_LIST_DESTRUCT(&mca_pmix_ext4x_component.jobids); + OPAL_LIST_DESTRUCT(&mca_pmix_ext4x_component.events); + OPAL_LIST_DESTRUCT(&mca_pmix_ext4x_component.dmdx); + return OPAL_SUCCESS; +} + + +static int external_component_query(mca_base_module_t **module, int *priority) +{ + char *t, *id; + + /* see if a PMIx server is present */ + if (NULL != (t = getenv("PMIX_SERVER_URI")) || + NULL != (id = getenv("PMIX_ID"))) { + /* if PMIx is present, then we are a client and need to use it */ + *priority = 100; + } else { + /* we could be a server, so we still need to be considered */ + *priority = 5; + } + *module = (mca_base_module_t *)&opal_pmix_ext4x_module; + return OPAL_SUCCESS; +} diff --git a/opal/mca/pmix/ext4x/ext4x_local.c b/opal/mca/pmix/ext4x/ext4x_local.c new file mode 100644 index 00000000000..7f5e7d62736 --- /dev/null +++ b/opal/mca/pmix/ext4x/ext4x_local.c @@ -0,0 +1,27 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2017 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2015 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "opal_config.h" +#include "opal/constants.h" + +#include "ext4x.h" + +int opal_pmix_ext4x_check_evars(void) +{ + /* a dummy function */ + return OPAL_SUCCESS; +} diff --git a/opal/mca/pmix/ext4x/ext4x_server_north.c b/opal/mca/pmix/ext4x/ext4x_server_north.c new file mode 100644 index 00000000000..a8102f7c4c8 --- /dev/null +++ b/opal/mca/pmix/ext4x/ext4x_server_north.c @@ -0,0 +1,1312 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2017 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2015 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "opal_config.h" +#include "opal/constants.h" +#include "opal/types.h" + +#ifdef HAVE_STRING_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif + +#include "opal/dss/dss.h" +#include "opal/mca/event/event.h" +#include "opal/mca/hwloc/base/base.h" +#include "opal/runtime/opal.h" +#include "opal/runtime/opal_progress_threads.h" +#include "opal/threads/threads.h" +#include "opal/util/argv.h" +#include "opal/util/error.h" +#include "opal/util/output.h" +#include "opal/util/proc.h" +#include "opal/util/show_help.h" +#include "opal/mca/pmix/base/base.h" +#include "ext4x.h" + +#include "pmix.h" +#include "pmix_server.h" + +/**** N.O.R.T.H.B.O.U.N.D I.N.T.E.R.F.A.C.E.S ****/ + +/* These are the interfaces used by the embedded PMIx server + * to call up into ORTE for service requests */ + +static pmix_status_t server_client_connected_fn(const pmix_proc_t *proc, void* server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_client_finalized_fn(const pmix_proc_t *proc, void* server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_abort_fn(const pmix_proc_t *proc, void *server_object, + int status, const char msg[], + pmix_proc_t procs[], size_t nprocs, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + char *data, size_t ndata, + pmix_modex_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *proc, + const pmix_info_t info[], size_t ninfo, + pmix_modex_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_publish_fn(const pmix_proc_t *proc, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_lookup_fn(const pmix_proc_t *proc, char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_lookup_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_unpublish_fn(const pmix_proc_t *proc, char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_spawn_fn(const pmix_proc_t *proc, + const pmix_info_t job_info[], size_t ninfo, + const pmix_app_t apps[], size_t napps, + pmix_spawn_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_register_events(pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_deregister_events(pmix_status_t *codes, size_t ncodes, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_notify_event(pmix_status_t code, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_query(pmix_proc_t *proct, + pmix_query_t *queryies, size_t nqueries, + pmix_info_cbfunc_t cbfunc, + void *cbdata); +static void server_tool_connection(pmix_info_t *info, size_t ninfo, + pmix_tool_connection_cbfunc_t cbfunc, + void *cbdata); +static void server_log(const pmix_proc_t *client, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +static pmix_status_t server_allocate(const pmix_proc_t *client, + pmix_alloc_directive_t directive, + const pmix_info_t data[], size_t ndata, + pmix_info_cbfunc_t cbfunc, void *cbdata); + +static pmix_status_t server_job_control(const pmix_proc_t *requestor, + const pmix_proc_t targets[], size_t ntargets, + const pmix_info_t directives[], size_t ndirs, + pmix_info_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_iof_pull(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t directives[], size_t ndirs, + pmix_iof_channel_t channels, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_stdin(const pmix_proc_t *source, + const pmix_proc_t targets[], size_t ntargets, + const pmix_info_t directives[], size_t ndirs, + const pmix_byte_object_t *bo, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +pmix_server_module_t mymodule = { + .client_connected = server_client_connected_fn, + .client_finalized = server_client_finalized_fn, + .abort = server_abort_fn, + .fence_nb = server_fencenb_fn, + .direct_modex = server_dmodex_req_fn, + .publish = server_publish_fn, + .lookup = server_lookup_fn, + .unpublish = server_unpublish_fn, + .spawn = server_spawn_fn, + .connect = server_connect_fn, + .disconnect = server_disconnect_fn, + .register_events = server_register_events, + .deregister_events = server_deregister_events, + .notify_event = server_notify_event, + .query = server_query, + .tool_connected = server_tool_connection, + .log = server_log, + .allocate = server_allocate, + .job_control = server_job_control, + /* we do not support monitoring, but use the + * PMIx internal monitoring capability */ + .iof_pull = server_iof_pull, + .push_stdin = server_stdin +}; + +opal_pmix_server_module_t *host_module = NULL; + + +static void opal_opcbfunc(int status, void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy = (ext4x_opalcaddy_t*)cbdata; + + OPAL_ACQUIRE_OBJECT(opalcaddy); + if (NULL != opalcaddy->opcbfunc) { + opalcaddy->opcbfunc(ext4x_convert_opalrc(status), opalcaddy->cbdata); + } + OBJ_RELEASE(opalcaddy); +} + +static pmix_status_t server_client_connected_fn(const pmix_proc_t *p, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + int rc; + opal_process_name_t proc; + ext4x_opalcaddy_t *opalcaddy; + + if (NULL == host_module || NULL == host_module->client_connected) { + return PMIX_SUCCESS; + } + + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->opcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the nspace/rank to an opal_process_name_t */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { + return ext4x_convert_opalrc(rc); + } + proc.vpid = ext4x_convert_rank(p->rank); + + /* pass it up */ + rc = host_module->client_connected(&proc, server_object, + opal_opcbfunc, opalcaddy); + return ext4x_convert_opalrc(rc); +} + +static pmix_status_t server_client_finalized_fn(const pmix_proc_t *p, void* server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + int rc; + ext4x_opalcaddy_t *opalcaddy; + opal_process_name_t proc; + + if (NULL == host_module || NULL == host_module->client_finalized) { + return PMIX_SUCCESS; + } + + /* convert the nspace/rank to an opal_process_name_t */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { + return ext4x_convert_opalrc(rc); + } + proc.vpid = ext4x_convert_rank(p->rank); + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->opcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* pass it up */ + opal_output_verbose(3, opal_pmix_base_framework.framework_output, + "%s CLIENT %s FINALIZED", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + OPAL_NAME_PRINT(proc)); + rc = host_module->client_finalized(&proc, server_object, opal_opcbfunc, opalcaddy); + if (OPAL_SUCCESS != rc) { + OBJ_RELEASE(opalcaddy); + } + return ext4x_convert_opalrc(rc); +} + +static pmix_status_t server_abort_fn(const pmix_proc_t *p, void *server_object, + int status, const char msg[], + pmix_proc_t procs[], size_t nprocs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + size_t n; + opal_namelist_t *nm; + opal_process_name_t proc; + int rc; + ext4x_opalcaddy_t *opalcaddy; + + if (NULL == host_module || NULL == host_module->abort) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* convert the nspace/rank to an opal_process_name_t */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { + return ext4x_convert_opalrc(rc); + } + proc.vpid = ext4x_convert_rank(p->rank); + + opal_output_verbose(3, opal_pmix_base_framework.framework_output, + "%s CLIENT %s CALLED ABORT", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + OPAL_NAME_PRINT(proc)); + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->opcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the array of pmix_proc_t to the list of procs */ + for (n=0; n < nprocs; n++) { + nm = OBJ_NEW(opal_namelist_t); + opal_list_append(&opalcaddy->procs, &nm->super); + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + nm->name.vpid = ext4x_convert_rank(procs[n].rank); + } + + /* pass it up */ + rc = host_module->abort(&proc, server_object, status, msg, + &opalcaddy->procs, opal_opcbfunc, opalcaddy); + if (OPAL_SUCCESS != rc) { + OBJ_RELEASE(opalcaddy); + } + return ext4x_convert_opalrc(rc); +} + +static void _data_release(void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy = (ext4x_opalcaddy_t*)cbdata; + + if (NULL != opalcaddy->odmdxfunc) { + opalcaddy->odmdxfunc(opalcaddy->ocbdata); + } + OBJ_RELEASE(opalcaddy); +} + +static void opmdx_response(int status, const char *data, size_t sz, void *cbdata, + opal_pmix_release_cbfunc_t relcbfunc, void *relcbdata) +{ + pmix_status_t rc; + ext4x_opalcaddy_t *opalcaddy = (ext4x_opalcaddy_t*)cbdata; + opal_ext4x_dmx_trkr_t *dmdx; + + rc = ext4x_convert_rc(status); + if (NULL != opalcaddy->mdxcbfunc) { + opalcaddy->odmdxfunc = relcbfunc; + opalcaddy->ocbdata = relcbdata; + opalcaddy->mdxcbfunc(rc, data, sz, opalcaddy->cbdata, + _data_release, opalcaddy); + } else { + OBJ_RELEASE(opalcaddy); + } + if (opal_pmix_collect_all_data) { + /* if we were collecting all data, then check for any pending + * dmodx requests that we cached and notify them that the + * data has arrived */ + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + while (NULL != (dmdx = (opal_ext4x_dmx_trkr_t*)opal_list_remove_first(&mca_pmix_ext4x_component.dmdx))) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + dmdx->cbfunc(PMIX_SUCCESS, NULL, 0, dmdx->cbdata, NULL, NULL); + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + OBJ_RELEASE(dmdx); + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + } +} + +static pmix_status_t server_fencenb_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + char *data, size_t ndata, + pmix_modex_cbfunc_t cbfunc, void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy; + size_t n; + opal_namelist_t *nm; + opal_value_t *iptr; + int rc; + + opal_output_verbose(3, opal_pmix_base_framework.framework_output, + "%s FENCE CALLED", OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + + if (NULL == host_module || NULL == host_module->fence_nb) { + return PMIX_ERR_NOT_SUPPORTED; + } + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->mdxcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the array of pmix_proc_t to the list of procs */ + for (n=0; n < nprocs; n++) { + nm = OBJ_NEW(opal_namelist_t); + opal_list_append(&opalcaddy->procs, &nm->super); + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + nm->name.vpid = ext4x_convert_rank(procs[n].rank); + } + + /* convert the array of pmix_info_t to the list of info */ + for (n=0; n < ninfo; n++) { + iptr = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &iptr->super); + iptr->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(iptr, &info[n].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + + /* pass it up */ + rc = host_module->fence_nb(&opalcaddy->procs, &opalcaddy->info, + data, ndata, opmdx_response, opalcaddy); + if (OPAL_SUCCESS != rc) { + OBJ_RELEASE(opalcaddy); + } + return ext4x_convert_opalrc(rc); +} + +static pmix_status_t server_dmodex_req_fn(const pmix_proc_t *p, + const pmix_info_t info[], size_t ninfo, + pmix_modex_cbfunc_t cbfunc, void *cbdata) +{ + int rc; + ext4x_opalcaddy_t *opalcaddy; + opal_process_name_t proc; + opal_value_t *iptr; + size_t n; + opal_ext4x_dmx_trkr_t *dmdx; + + if (NULL == host_module || NULL == host_module->direct_modex) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* convert the nspace/rank to an opal_process_name_t */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { + return ext4x_convert_opalrc(rc); + } + proc.vpid = ext4x_convert_rank(p->rank); + + opal_output_verbose(3, opal_pmix_base_framework.framework_output, + "%s CLIENT %s CALLED DMODX", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + OPAL_NAME_PRINT(proc)); + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->mdxcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* this function should only get called if we are in an async modex. + * If we are also collecting data, then the fence_nb will eventually + * complete and return all the required data down to the pmix + * server beneath us. Thus, we only need to track the dmodex_req + * and ensure that the release gets called once the data has + * arrived - this will trigger the pmix server to tell the + * client that the data is available */ + if (opal_pmix_base_async_modex && opal_pmix_collect_all_data) { + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + dmdx = OBJ_NEW(opal_ext4x_dmx_trkr_t); + dmdx->cbfunc = cbfunc; + dmdx->cbdata = cbdata; + opal_list_append(&mca_pmix_ext4x_component.dmdx, &dmdx->super); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return PMIX_SUCCESS; + } + + /* convert the array of pmix_info_t to the list of info */ + for (n=0; n < ninfo; n++) { + iptr = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &iptr->super); + iptr->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(iptr, &info[n].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + + /* pass it up */ + rc = host_module->direct_modex(&proc, &opalcaddy->info, opmdx_response, opalcaddy); + if (OPAL_SUCCESS != rc && OPAL_ERR_IN_PROCESS != rc) { + OBJ_RELEASE(opalcaddy); + } + if (OPAL_ERR_IN_PROCESS == rc) { + rc = OPAL_SUCCESS; + } + return ext4x_convert_opalrc(rc); +} + +static pmix_status_t server_publish_fn(const pmix_proc_t *p, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + int rc; + size_t n; + ext4x_opalcaddy_t *opalcaddy; + opal_process_name_t proc; + opal_value_t *oinfo; + + if (NULL == host_module || NULL == host_module->publish) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* convert the nspace/rank to an opal_process_name_t */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { + return ext4x_convert_opalrc(rc); + } + proc.vpid = ext4x_convert_rank(p->rank); + + opal_output_verbose(3, opal_pmix_base_framework.framework_output, + "%s CLIENT %s CALLED PUBLISH", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + OPAL_NAME_PRINT(proc)); + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->opcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the info array */ + for (n=0; n < ninfo; n++) { + oinfo = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &oinfo->super); + oinfo->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &info[n].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + + /* pass it up */ + rc = host_module->publish(&proc, &opalcaddy->info, opal_opcbfunc, opalcaddy); + if (OPAL_SUCCESS != rc) { + OBJ_RELEASE(opalcaddy); + } + + return ext4x_convert_opalrc(rc); +} + +static void opal_lkupcbfunc(int status, + opal_list_t *data, + void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy = (ext4x_opalcaddy_t*)cbdata; + pmix_status_t rc; + pmix_pdata_t *d=NULL; + size_t nd=0, n; + opal_pmix_pdata_t *p; + + if (NULL != opalcaddy->lkupcbfunc) { + rc = ext4x_convert_opalrc(status); + /* convert any returned data */ + if (NULL != data) { + nd = opal_list_get_size(data); + PMIX_PDATA_CREATE(d, nd); + n=0; + OPAL_LIST_FOREACH(p, data, opal_pmix_pdata_t) { + /* convert the jobid */ + (void)opal_snprintf_jobid(d[n].proc.nspace, PMIX_MAX_NSLEN, p->proc.jobid); + d[n].proc.rank = ext4x_convert_opalrank(p->proc.vpid); + (void)strncpy(d[n].key, p->value.key, PMIX_MAX_KEYLEN); + ext4x_value_load(&d[n].value, &p->value); + } + } + opalcaddy->lkupcbfunc(rc, d, nd, opalcaddy->cbdata); + PMIX_PDATA_FREE(d, nd); + } + OBJ_RELEASE(opalcaddy); +} + +static pmix_status_t server_lookup_fn(const pmix_proc_t *p, char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_lookup_cbfunc_t cbfunc, void *cbdata) +{ + int rc; + ext4x_opalcaddy_t *opalcaddy; + opal_process_name_t proc; + opal_value_t *iptr; + size_t n; + + if (NULL == host_module || NULL == host_module->lookup) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* convert the nspace/rank to an opal_process_name_t */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { + return ext4x_convert_opalrc(rc); + } + proc.vpid = ext4x_convert_rank(p->rank); + + opal_output_verbose(3, opal_pmix_base_framework.framework_output, + "%s CLIENT %s CALLED LOOKUP", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + OPAL_NAME_PRINT(proc)); + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->lkupcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the array of pmix_info_t to the list of info */ + for (n=0; n < ninfo; n++) { + iptr = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &iptr->super); + iptr->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(iptr, &info[n].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + + /* pass it up */ + rc = host_module->lookup(&proc, keys, &opalcaddy->info, opal_lkupcbfunc, opalcaddy); + if (OPAL_SUCCESS != rc) { + OBJ_RELEASE(opalcaddy); + } + + return ext4x_convert_opalrc(rc); +} + + +static pmix_status_t server_unpublish_fn(const pmix_proc_t *p, char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + int rc; + ext4x_opalcaddy_t *opalcaddy; + opal_process_name_t proc; + opal_value_t *iptr; + size_t n; + + if (NULL == host_module || NULL == host_module->unpublish) { + return PMIX_SUCCESS; + } + + /* convert the nspace/rank to an opal_process_name_t */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { + return ext4x_convert_opalrc(rc); + } + proc.vpid = ext4x_convert_rank(p->rank); + + opal_output_verbose(3, opal_pmix_base_framework.framework_output, + "%s CLIENT %s CALLED UNPUBLISH", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + OPAL_NAME_PRINT(proc)); + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->opcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the array of pmix_info_t to the list of info */ + for (n=0; n < ninfo; n++) { + iptr = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &iptr->super); + iptr->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(iptr, &info[n].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + + /* pass it up */ + rc = host_module->unpublish(&proc, keys, &opalcaddy->info, opal_opcbfunc, opalcaddy); + if (OPAL_SUCCESS != rc) { + OBJ_RELEASE(opalcaddy); + } + + return ext4x_convert_opalrc(rc); +} + +static void opal_spncbfunc(int status, opal_jobid_t jobid, void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy = (ext4x_opalcaddy_t*)cbdata; + pmix_status_t rc; + char nspace[PMIX_MAX_NSLEN]; + + if (NULL != opalcaddy->spwncbfunc) { + rc = ext4x_convert_opalrc(status); + /* convert the jobid */ + (void)opal_snprintf_jobid(nspace, PMIX_MAX_NSLEN, jobid); + opalcaddy->spwncbfunc(rc, nspace, opalcaddy->cbdata); + } + OBJ_RELEASE(opalcaddy); +} + +static pmix_status_t server_spawn_fn(const pmix_proc_t *p, + const pmix_info_t job_info[], size_t ninfo, + const pmix_app_t apps[], size_t napps, + pmix_spawn_cbfunc_t cbfunc, void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy; + opal_process_name_t proc; + opal_pmix_app_t *app; + opal_value_t *oinfo; + size_t k, n; + int rc; + + if (NULL == host_module || NULL == host_module->spawn) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* convert the nspace/rank to an opal_process_name_t */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&proc.jobid, p->nspace))) { + return ext4x_convert_opalrc(rc); + } + proc.vpid = ext4x_convert_rank(p->rank); + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->spwncbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the job info */ + for (k=0; k < ninfo; k++) { + oinfo = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &oinfo->super); + oinfo->key = strdup(job_info[k].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &job_info[k].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + + /* convert the apps */ + for (n=0; n < napps; n++) { + app = OBJ_NEW(opal_pmix_app_t); + opal_list_append(&opalcaddy->apps, &app->super); + if (NULL != apps[n].cmd) { + app->cmd = strdup(apps[n].cmd); + } + if (NULL != apps[n].argv) { + app->argv = opal_argv_copy(apps[n].argv); + } + if (NULL != apps[n].env) { + app->env = opal_argv_copy(apps[n].env); + } + if (NULL != apps[n].cwd) { + app->cwd = strdup(apps[n].cwd); + } + app->maxprocs = apps[n].maxprocs; + for (k=0; k < apps[n].ninfo; k++) { + oinfo = OBJ_NEW(opal_value_t); + opal_list_append(&app->info, &oinfo->super); + oinfo->key = strdup(apps[n].info[k].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &apps[n].info[k].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + } + + /* pass it up */ + rc = host_module->spawn(&proc, &opalcaddy->info, &opalcaddy->apps, opal_spncbfunc, opalcaddy); + if (OPAL_SUCCESS != rc) { + OPAL_ERROR_LOG(rc); + OBJ_RELEASE(opalcaddy); + } + + return ext4x_convert_opalrc(rc); +} + + +static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + int rc; + ext4x_opalcaddy_t *opalcaddy; + opal_namelist_t *nm; + size_t n; + opal_value_t *oinfo; + + if (NULL == host_module || NULL == host_module->connect) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->opcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the array of pmix_proc_t to the list of procs */ + for (n=0; n < nprocs; n++) { + nm = OBJ_NEW(opal_namelist_t); + opal_list_append(&opalcaddy->procs, &nm->super); + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + nm->name.vpid = ext4x_convert_rank(procs[n].rank); + } + + /* convert the info */ + for (n=0; n < ninfo; n++) { + oinfo = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &oinfo->super); + oinfo->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &info[n].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + + /* pass it up */ + rc = host_module->connect(&opalcaddy->procs, &opalcaddy->info, opal_opcbfunc, opalcaddy); + if (OPAL_SUCCESS != rc) { + OBJ_RELEASE(opalcaddy); + } + + return ext4x_convert_opalrc(rc); +} + + +static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + int rc; + ext4x_opalcaddy_t *opalcaddy; + opal_namelist_t *nm; + size_t n; + opal_value_t *oinfo; + + if (NULL == host_module || NULL == host_module->disconnect) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->opcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the array of pmix_proc_t to the list of procs */ + for (n=0; n < nprocs; n++) { + nm = OBJ_NEW(opal_namelist_t); + opal_list_append(&opalcaddy->procs, &nm->super); + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + nm->name.vpid = ext4x_convert_rank(procs[n].rank); + } + + /* convert the info */ + for (n=0; n < ninfo; n++) { + oinfo = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &oinfo->super); + oinfo->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &info[n].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + + /* pass it up */ + rc = host_module->disconnect(&opalcaddy->procs, &opalcaddy->info, opal_opcbfunc, opalcaddy); + if (OPAL_SUCCESS != rc) { + OBJ_RELEASE(opalcaddy); + } + + return ext4x_convert_opalrc(rc); +} + +static pmix_status_t server_register_events(pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy; + size_t n; + opal_value_t *oinfo; + int rc; + + opal_output_verbose(3, opal_pmix_base_framework.framework_output, + "%s REGISTER EVENTS", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->opcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the info */ + for (n=0; n < ninfo; n++) { + oinfo = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &oinfo->super); + oinfo->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &info[n].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + + /* pass it up */ + rc = host_module->register_events(&opalcaddy->info, opal_opcbfunc, opalcaddy); + if (OPAL_SUCCESS != rc) { + OBJ_RELEASE(opalcaddy); + } + + return ext4x_convert_opalrc(rc); +} + +static pmix_status_t server_deregister_events(pmix_status_t *codes, size_t ncodes, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + opal_output_verbose(3, opal_pmix_base_framework.framework_output, + "%s DEREGISTER EVENTS", OPAL_NAME_PRINT(OPAL_PROC_MY_NAME)); + + return PMIX_ERR_NOT_SUPPORTED; +} + +static pmix_status_t server_notify_event(pmix_status_t code, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy; + opal_process_name_t src; + size_t n; + opal_value_t *oinfo; + int rc, status; + + if (NULL == host_module || NULL == host_module->notify_event) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->opcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the code */ + status = ext4x_convert_rc(code); + + /* convert the source */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&src.jobid, source->nspace))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + src.vpid = ext4x_convert_rank(source->rank); + + opal_output_verbose(3, opal_pmix_base_framework.framework_output, + "%s CLIENT %s CALLED NOTIFY", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + OPAL_NAME_PRINT(src)); + + /* ignore the range for now */ + + /* convert the info */ + for (n=0; n < ninfo; n++) { + oinfo = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &oinfo->super); + oinfo->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &info[n].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + + /* send it upstairs */ + if (OPAL_SUCCESS != (rc = host_module->notify_event(status, &src, &opalcaddy->info, + opal_opcbfunc, opalcaddy))) { + OBJ_RELEASE(opalcaddy); + } + return ext4x_convert_opalrc(rc); +} + +static void _info_rel(void *cbdata) +{ + ext4x_opcaddy_t *pcaddy = (ext4x_opcaddy_t*)cbdata; + + OBJ_RELEASE(pcaddy); +} +static void info_cbfunc(int status, + opal_list_t *info, + void *cbdata, + opal_pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + ext4x_opalcaddy_t *opalcaddy = (ext4x_opalcaddy_t*)cbdata; + ext4x_opcaddy_t *pcaddy; + opal_value_t *kv; + size_t n; + + pcaddy = OBJ_NEW(ext4x_opcaddy_t); + + /* convert the status */ + pcaddy->status = ext4x_convert_opalrc(status); + + /* convert the list to a pmix_info_t array */ + if (NULL != info && 0 < (pcaddy->ninfo = opal_list_get_size(info))) { + PMIX_INFO_CREATE(pcaddy->info, pcaddy->ninfo); + n = 0; + OPAL_LIST_FOREACH(kv, info, opal_value_t) { + (void)strncpy(pcaddy->info[n].key, kv->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&pcaddy->info[n].value, kv); + ++n; + } + } + /* we are done with the incoming data */ + if (NULL != release_fn) { + release_fn(release_cbdata); + } + + /* provide the answer downward */ + if (NULL != opalcaddy->infocbfunc) { + opalcaddy->infocbfunc(pcaddy->status, pcaddy->info, pcaddy->ninfo, + opalcaddy->cbdata, _info_rel, pcaddy); + } + OBJ_RELEASE(opalcaddy); +} + +static pmix_status_t server_query(pmix_proc_t *proct, + pmix_query_t *queries, size_t nqueries, + pmix_info_cbfunc_t cbfunc, + void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy; + opal_process_name_t requestor; + int rc; + size_t n, m; + opal_pmix_query_t *q; + opal_value_t *oinfo; + + if (NULL == host_module || NULL == host_module->query) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->infocbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the requestor */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&requestor.jobid, proct->nspace))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + requestor.vpid = ext4x_convert_rank(proct->rank); + + opal_output_verbose(3, opal_pmix_base_framework.framework_output, + "%s CLIENT %s CALLED QUERY", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + OPAL_NAME_PRINT(requestor)); + + /* convert the queries */ + for (n=0; n < nqueries; n++) { + q = OBJ_NEW(opal_pmix_query_t); + /* we "borrow" the info field of the caddy as we and the + * server function both agree on what will be there */ + opal_list_append(&opalcaddy->info, &q->super); + q->keys = opal_argv_copy(queries[n].keys); + for (m=0; m < queries[n].nqual; m++) { + oinfo = OBJ_NEW(opal_value_t); + opal_list_append(&q->qualifiers, &oinfo->super); + + if (0 == strcmp(queries[n].qualifiers[m].key, PMIX_NSPACE)) { + /* must convert this to jobid */ + oinfo->key = strdup(OPAL_PMIX_PROCID); + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&oinfo->data.name.jobid, queries[n].qualifiers[m].value.data.string))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } else { + oinfo->key = strdup(queries[n].qualifiers[m].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &queries[n].qualifiers[m].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + } + } + + /* pass the call upwards */ + if (OPAL_SUCCESS != (rc = host_module->query(&requestor, + &opalcaddy->info, + info_cbfunc, opalcaddy))) { + OBJ_RELEASE(opalcaddy); + } + + return ext4x_convert_opalrc(rc); +} + +static void toolcbfunc(int status, + opal_process_name_t proc, + void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy = (ext4x_opalcaddy_t*)cbdata; + pmix_status_t rc; + pmix_proc_t p; + opal_ext4x_jobid_trkr_t *job; + + /* convert the status */ + rc = ext4x_convert_opalrc(status); + + memset(&p, 0, sizeof(pmix_proc_t)); + if (OPAL_SUCCESS == status) { + /* convert the process name */ + (void)opal_snprintf_jobid(p.nspace, PMIX_MAX_NSLEN, proc.jobid); + p.rank = ext4x_convert_opalrank(proc.vpid); + /* store this job in our list of known nspaces */ + job = OBJ_NEW(opal_ext4x_jobid_trkr_t); + (void)strncpy(job->nspace, p.nspace, PMIX_MAX_NSLEN); + job->jobid = proc.jobid; + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + } + + /* pass it down */ + if (NULL != opalcaddy->toolcbfunc) { + opalcaddy->toolcbfunc(rc, &p, opalcaddy->cbdata); + } + OBJ_RELEASE(opalcaddy); +} + +static void server_tool_connection(pmix_info_t *info, size_t ninfo, + pmix_tool_connection_cbfunc_t cbfunc, + void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy; + size_t n; + opal_value_t *oinfo; + int rc; + pmix_status_t err; + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->toolcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the info */ + for (n=0; n < ninfo; n++) { + oinfo = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &oinfo->super); + oinfo->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &info[n].value))) { + OBJ_RELEASE(opalcaddy); + err = ext4x_convert_opalrc(rc); + if (NULL != cbfunc) { + cbfunc(err, NULL, cbdata); + } + } + } + + /* pass it up */ + host_module->tool_connected(&opalcaddy->info, toolcbfunc, opalcaddy); +} + +static void server_log(const pmix_proc_t *proct, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy; + opal_process_name_t requestor; + int rc; + size_t n; + opal_value_t *oinfo; + pmix_status_t ret; + + if (NULL == host_module || NULL == host_module->log) { + if (NULL != cbfunc) { + cbfunc(PMIX_ERR_NOT_SUPPORTED, cbdata); + } + return; + } + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->opcbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the requestor */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&requestor.jobid, proct->nspace))) { + OBJ_RELEASE(opalcaddy); + ret = ext4x_convert_opalrc(rc); + if (NULL != cbfunc) { + cbfunc(ret, cbdata); + } + return; + } + requestor.vpid = ext4x_convert_rank(proct->rank); + + /* convert the data */ + for (n=0; n < ndata; n++) { + oinfo = OBJ_NEW(opal_value_t); + oinfo->key = strdup(data[n].key); + /* we "borrow" the info field of the caddy as we and the + * server function both agree on what will be there */ + opal_list_append(&opalcaddy->info, &oinfo->super); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &data[n].value))) { + OBJ_RELEASE(opalcaddy); + ret = ext4x_convert_opalrc(rc); + if (NULL != cbfunc) { + cbfunc(ret, cbdata); + } + return; + } + } + + /* convert the directives */ + for (n=0; n < ndirs; n++) { + oinfo = OBJ_NEW(opal_value_t); + /* we "borrow" the apps field of the caddy as we and the + * server function both agree on what will be there */ + opal_list_append(&opalcaddy->apps, &oinfo->super); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &directives[n].value))) { + OBJ_RELEASE(opalcaddy); + ret = ext4x_convert_opalrc(rc); + if (NULL != cbfunc) { + cbfunc(ret, cbdata); + } + return; + } + } + + /* pass the call upwards */ + host_module->log(&requestor, + &opalcaddy->info, + &opalcaddy->apps, + opal_opcbfunc, opalcaddy); +} + +static pmix_status_t server_allocate(const pmix_proc_t *proct, + pmix_alloc_directive_t directive, + const pmix_info_t data[], size_t ndata, + pmix_info_cbfunc_t cbfunc, void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy; + opal_process_name_t requestor; + int rc; + size_t n; + opal_value_t *oinfo; + opal_pmix_alloc_directive_t odir; + + if (NULL == host_module || NULL == host_module->allocate) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->infocbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the requestor */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&requestor.jobid, proct->nspace))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + requestor.vpid = ext4x_convert_rank(proct->rank); + + /* convert the directive */ + odir = ext4x_convert_allocdir(directive); + + /* convert the data */ + for (n=0; n < ndata; n++) { + oinfo = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &oinfo->super); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &data[n].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + + /* pass the call upwards */ + if (OPAL_SUCCESS != (rc = host_module->allocate(&requestor, odir, + &opalcaddy->info, + info_cbfunc, opalcaddy))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + + return PMIX_SUCCESS; + +} + +static pmix_status_t server_job_control(const pmix_proc_t *proct, + const pmix_proc_t targets[], size_t ntargets, + const pmix_info_t directives[], size_t ndirs, + pmix_info_cbfunc_t cbfunc, void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy; + opal_process_name_t requestor; + int rc; + size_t n; + opal_value_t *oinfo; + opal_namelist_t *nm; + + if (NULL == host_module || NULL == host_module->job_control) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + opalcaddy->infocbfunc = cbfunc; + opalcaddy->cbdata = cbdata; + + /* convert the requestor */ + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&requestor.jobid, proct->nspace))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + requestor.vpid = ext4x_convert_rank(proct->rank); + + /* convert the targets */ + for (n=0; n < ntargets; n++) { + nm = OBJ_NEW(opal_namelist_t); + opal_list_append(&opalcaddy->procs, &nm->super); + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, targets[n].nspace))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + nm->name.vpid = ext4x_convert_rank(targets[n].rank); + } + + /* convert the directives */ + for (n=0; n < ndirs; n++) { + oinfo = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &oinfo->super); + oinfo->key = strdup(directives[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(oinfo, &directives[n].value))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + } + + /* pass the call upwards */ + if (OPAL_SUCCESS != (rc = host_module->job_control(&requestor, + &opalcaddy->procs, + &opalcaddy->info, + info_cbfunc, opalcaddy))) { + OBJ_RELEASE(opalcaddy); + return ext4x_convert_opalrc(rc); + } + + return PMIX_SUCCESS; +} + +static pmix_status_t server_iof_pull(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t directives[], size_t ndirs, + pmix_iof_channel_t channels, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + if (NULL == host_module || NULL == host_module->iof_pull) { + return PMIX_ERR_NOT_SUPPORTED; + } + return PMIX_ERR_NOT_SUPPORTED; +} + +static pmix_status_t server_stdin(const pmix_proc_t *source, + const pmix_proc_t targets[], size_t ntargets, + const pmix_info_t directives[], size_t ndirs, + const pmix_byte_object_t *bo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + if (NULL == host_module || NULL == host_module->iof_push) { + return PMIX_ERR_NOT_SUPPORTED; + } + return PMIX_ERR_NOT_SUPPORTED; +} diff --git a/opal/mca/pmix/ext4x/ext4x_server_south.c b/opal/mca/pmix/ext4x/ext4x_server_south.c new file mode 100644 index 00000000000..1633f759424 --- /dev/null +++ b/opal/mca/pmix/ext4x/ext4x_server_south.c @@ -0,0 +1,790 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2017 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2017 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "opal_config.h" +#include "opal/constants.h" +#include "opal/types.h" + +#ifdef HAVE_STRING_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif + +#include "opal/dss/dss.h" +#include "opal/mca/event/event.h" +#include "opal/mca/hwloc/base/base.h" +#include "opal/runtime/opal.h" +#include "opal/runtime/opal_progress_threads.h" +#include "opal/threads/threads.h" +#include "opal/util/argv.h" +#include "opal/util/error.h" +#include "opal/util/output.h" +#include "opal/util/opal_environ.h" +#include "opal/util/proc.h" +#include "opal/util/show_help.h" +#include "opal/mca/pmix/base/base.h" +#include "ext4x.h" + +#include "pmix.h" +#include "pmix_server.h" + +/**** S.O.U.T.H.B.O.U.N.D I.N.T.E.R.F.A.C.E.S ****/ + +/* These are the interfaces used by the OMPI/ORTE/OPAL layer to call + * down into the embedded PMIx server. */ + +extern pmix_server_module_t mymodule; +extern opal_pmix_server_module_t *host_module; +static char *dbgvalue=NULL; + +static void errreg_cbfunc (pmix_status_t status, + size_t errhandler_ref, + void *cbdata) +{ + opal_ext4x_event_t *ev = (opal_ext4x_event_t*)cbdata; + + OPAL_ACQUIRE_OBJECT(ev); + ev->index = errhandler_ref; + opal_output_verbose(5, opal_pmix_base_framework.framework_output, + "PMIX server errreg_cbfunc - error handler registered status=%d, reference=%lu", + status, (unsigned long)errhandler_ref); + OPAL_POST_OBJECT(ev); + OPAL_PMIX_WAKEUP_THREAD(&ev->lock); +} + +static void opcbfunc(pmix_status_t status, void *cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + + OPAL_ACQUIRE_OBJECT(op); + + if (NULL != op->opcbfunc) { + op->opcbfunc(ext4x_convert_rc(status), op->cbdata); + } + OBJ_RELEASE(op); +} + +static void lkcbfunc(pmix_status_t status, void *cbdata) +{ + opal_pmix_lock_t *lk = (opal_pmix_lock_t*)cbdata; + + OPAL_POST_OBJECT(lk); + lk->status = ext4x_convert_rc(status); + OPAL_PMIX_WAKEUP_THREAD(lk); +} + +int ext4x_server_init(opal_pmix_server_module_t *module, + opal_list_t *info) +{ + pmix_status_t rc; + int dbg; + opal_value_t *kv; + pmix_info_t *pinfo; + size_t sz, n; + opal_ext4x_event_t *event; + opal_ext4x_jobid_trkr_t *job; + opal_pmix_lock_t lk; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + + if (0 == opal_pmix_base.initialized) { + if (0 < (dbg = opal_output_get_verbosity(opal_pmix_base_framework.framework_output))) { + asprintf(&dbgvalue, "PMIX_DEBUG=%d", dbg); + putenv(dbgvalue); + } + /* check the evars for a mismatch */ + if (OPAL_SUCCESS != (dbg = opal_pmix_ext4x_check_evars())) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return dbg; + } + } + ++opal_pmix_base.initialized; + + /* convert the list to an array of pmix_info_t */ + sz = 2 + ((NULL==info)?0:opal_list_get_size(info)); + PMIX_INFO_CREATE(pinfo, sz); + n = 0; + if (NULL != info) { + OPAL_LIST_FOREACH(kv, info, opal_value_t) { + (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&pinfo[n].value, kv); + ++n; + } + } + + /* check for direct modex use-case */ + if (opal_pmix_base_async_modex && !opal_pmix_collect_all_data) { + opal_setenv("PMIX_MCA_gds", "hash", true, &environ); + } + + /* insert ourselves into our list of jobids - it will be the + * first, and so we'll check it first */ + job = OBJ_NEW(opal_ext4x_jobid_trkr_t); + (void)opal_snprintf_jobid(job->nspace, PMIX_MAX_NSLEN, OPAL_PROC_MY_NAME.jobid); + job->jobid = OPAL_PROC_MY_NAME.jobid; + opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* add our nspace and rank to the array going down to the PMIx server */ + PMIX_INFO_LOAD(&pinfo[sz-2], PMIX_SERVER_NSPACE, job->nspace, PMIX_STRING); + PMIX_INFO_LOAD(&pinfo[sz-1], PMIX_SERVER_RANK, &OPAL_PROC_MY_NAME.vpid, PMIX_PROC_RANK); + if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, pinfo, sz))) { + PMIX_INFO_FREE(pinfo, sz); + return ext4x_convert_rc(rc); + } + PMIX_INFO_FREE(pinfo, sz); + + /* record the host module */ + host_module = module; + + /* register the default event handler */ + event = OBJ_NEW(opal_ext4x_event_t); + opal_list_append(&mca_pmix_ext4x_component.events, &event->super); + PMIX_INFO_CREATE(pinfo, 1); + PMIX_INFO_LOAD(&pinfo[0], PMIX_EVENT_HDLR_NAME, "OPAL-PMIX-2X-SERVER-DEFAULT", PMIX_STRING); + PMIx_Register_event_handler(NULL, 0, pinfo, 1, ext4x_event_hdlr, errreg_cbfunc, (void*)event); + OPAL_PMIX_WAIT_THREAD(&event->lock); + PMIX_INFO_FREE(pinfo, 1); + + /* as we might want to use some client-side functions, be sure + * to register our own nspace */ + OPAL_PMIX_CONSTRUCT_LOCK(&lk); + PMIX_INFO_CREATE(pinfo, 1); + PMIX_INFO_LOAD(&pinfo[0], PMIX_REGISTER_NODATA, NULL, PMIX_BOOL); + PMIx_server_register_nspace(job->nspace, 1, pinfo, 1, lkcbfunc, (void*)&lk); + OPAL_PMIX_WAIT_THREAD(&lk); + OPAL_PMIX_DESTRUCT_LOCK(&lk); + PMIX_INFO_FREE(pinfo, 1); + + return OPAL_SUCCESS; +} + +static void dereg_cbfunc(pmix_status_t st, void *cbdata) +{ + opal_ext4x_event_t *ev = (opal_ext4x_event_t*)cbdata; + OPAL_PMIX_WAKEUP_THREAD(&ev->lock); +} + +int ext4x_server_finalize(void) +{ + pmix_status_t rc; + opal_ext4x_event_t *event, *ev2; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + --opal_pmix_base.initialized; + + if (0 < opal_pmix_base.initialized) { + /* deregister all event handlers */ + OPAL_LIST_FOREACH_SAFE(event, ev2, &mca_pmix_ext4x_component.events, opal_ext4x_event_t) { + OPAL_PMIX_DESTRUCT_LOCK(&event->lock); + OPAL_PMIX_CONSTRUCT_LOCK(&event->lock); + PMIx_Deregister_event_handler(event->index, dereg_cbfunc, (void*)event); + OPAL_PMIX_WAIT_THREAD(&event->lock); + opal_list_remove_item(&mca_pmix_ext4x_component.events, &event->super); + OBJ_RELEASE(event); + } + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + rc = PMIx_server_finalize(); + return ext4x_convert_rc(rc); +} + +int ext4x_server_gen_regex(const char *input, char **regex) +{ + pmix_status_t rc; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + rc = PMIx_generate_regex(input, regex); + return ext4x_convert_rc(rc); +} + + +int ext4x_server_gen_ppn(const char *input, char **ppn) +{ + pmix_status_t rc; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + rc = PMIx_generate_ppn(input, ppn); + return ext4x_convert_rc(rc); +} + +int ext4x_server_register_nspace(opal_jobid_t jobid, + int nlocalprocs, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + opal_value_t *kv, *k2; + pmix_info_t *pinfo = NULL, *pmap; + size_t sz, szmap, m, n; + char nspace[PMIX_MAX_NSLEN]; + pmix_status_t rc; + opal_list_t *pmapinfo; + opal_ext4x_jobid_trkr_t *job; + opal_pmix_lock_t lock; + int ret; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + + /* convert the jobid */ + (void)opal_snprintf_jobid(nspace, PMIX_MAX_NSLEN, jobid); + + /* store this job in our list of known nspaces */ + job = OBJ_NEW(opal_ext4x_jobid_trkr_t); + (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN); + job->jobid = jobid; + opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* convert the list to an array of pmix_info_t */ + if (NULL != info && 0 < (sz = opal_list_get_size(info))) { + PMIX_INFO_CREATE(pinfo, sz); + n = 0; + OPAL_LIST_FOREACH(kv, info, opal_value_t) { + (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); + if (0 == strcmp(kv->key, OPAL_PMIX_PROC_DATA)) { + pinfo[n].value.type = PMIX_DATA_ARRAY; + /* the value contains a list of values - convert + * that list to another array */ + pmapinfo = (opal_list_t*)kv->data.ptr; + szmap = opal_list_get_size(pmapinfo); + if (0 < szmap) { + PMIX_INFO_CREATE(pmap, szmap); + pinfo[n].value.data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); + pinfo[n].value.data.darray->type = PMIX_INFO; + pinfo[n].value.data.darray->array = (struct pmix_info_t*)pmap; + pinfo[n].value.data.darray->size = szmap; + m = 0; + OPAL_LIST_FOREACH(k2, pmapinfo, opal_value_t) { + (void)strncpy(pmap[m].key, k2->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&pmap[m].value, k2); + ++m; + } + } + OPAL_LIST_RELEASE(pmapinfo); + } else { + ext4x_value_load(&pinfo[n].value, kv); + } + ++n; + } + } else { + sz = 0; + pinfo = NULL; + } + + OPAL_PMIX_CONSTRUCT_LOCK(&lock); + rc = PMIx_server_register_nspace(nspace, nlocalprocs, pinfo, sz, + lkcbfunc, (void*)&lock); + if (PMIX_SUCCESS == rc) { + OPAL_PMIX_WAIT_THREAD(&lock); + } + OPAL_PMIX_DESTRUCT_LOCK(&lock); + + if (NULL != pinfo) { + PMIX_INFO_FREE(pinfo, sz); + } + + ret = ext4x_convert_rc(rc); + + /* release the caller */ + if (NULL != cbfunc) { + cbfunc(ret, cbdata); + } + return ret; +} + +void ext4x_server_deregister_nspace(opal_jobid_t jobid, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + opal_ext4x_jobid_trkr_t *jptr; + opal_pmix_lock_t lock; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + /* release the caller */ + if (NULL != cbfunc) { + cbfunc(OPAL_ERR_NOT_INITIALIZED, cbdata); + } + return; + } + + /* if we don't already have it, we can ignore this */ + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) { + if (jptr->jobid == jobid) { + /* found it - tell the server to deregister */ + OPAL_PMIX_CONSTRUCT_LOCK(&lock); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + PMIx_server_deregister_nspace(jptr->nspace, lkcbfunc, (void*)&lock); + OPAL_PMIX_WAIT_THREAD(&lock); + OPAL_PMIX_DESTRUCT_LOCK(&lock); + /* now get rid of it from our list */ + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + opal_list_remove_item(&mca_pmix_ext4x_component.jobids, &jptr->super); + OBJ_RELEASE(jptr); + break; + } + } + + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + /* release the caller */ + if (NULL != cbfunc) { + cbfunc(OPAL_SUCCESS, cbdata); + } +} + +int ext4x_server_register_client(const opal_process_name_t *proc, + uid_t uid, gid_t gid, + void *server_object, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + pmix_status_t rc; + pmix_proc_t p; + opal_pmix_lock_t lock; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* convert the jobid */ + (void)opal_snprintf_jobid(p.nspace, PMIX_MAX_NSLEN, proc->jobid); + p.rank = ext4x_convert_opalrank(proc->vpid); + + OPAL_PMIX_CONSTRUCT_LOCK(&lock); + rc = PMIx_server_register_client(&p, uid, gid, server_object, + lkcbfunc, (void*)&lock); + if (PMIX_SUCCESS == rc) { + OPAL_PMIX_WAIT_THREAD(&lock); + } + OPAL_PMIX_DESTRUCT_LOCK(&lock); + return ext4x_convert_rc(rc); +} + +/* tell the local PMIx server to cleanup this client as it is + * done executing */ +void ext4x_server_deregister_client(const opal_process_name_t *proc, + opal_pmix_op_cbfunc_t cbfunc, + void *cbdata) +{ + opal_ext4x_jobid_trkr_t *jptr; + pmix_proc_t p; + opal_pmix_lock_t lock; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + if (NULL != cbfunc) { + cbfunc(OPAL_ERR_NOT_INITIALIZED, cbdata); + } + return; + } + + /* if we don't already have it, we can ignore this */ + OPAL_LIST_FOREACH(jptr, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) { + if (jptr->jobid == proc->jobid) { + /* found it - tell the server to deregister */ + (void)strncpy(p.nspace, jptr->nspace, PMIX_MAX_NSLEN); + p.rank = ext4x_convert_opalrank(proc->vpid); + OPAL_PMIX_CONSTRUCT_LOCK(&lock); + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + PMIx_server_deregister_client(&p, lkcbfunc, (void*)&lock); + OPAL_PMIX_WAIT_THREAD(&lock); + OPAL_PMIX_DESTRUCT_LOCK(&lock); + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + break; + } + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + if (NULL != cbfunc) { + cbfunc(OPAL_SUCCESS, cbdata); + } +} + +/* have the local PMIx server setup the environment for this client */ +int ext4x_server_setup_fork(const opal_process_name_t *proc, char ***env) +{ + pmix_status_t rc; + pmix_proc_t p; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* convert the jobid */ + (void)opal_snprintf_jobid(p.nspace, PMIX_MAX_NSLEN, proc->jobid); + p.rank = ext4x_convert_opalrank(proc->vpid); + + rc = PMIx_server_setup_fork(&p, env); + return ext4x_convert_rc(rc); +} + +/* this is the call back up from the embedded PMIx server that + * will contain the returned data. Note that the embedded server + * "owns" the data and will free it upon return from this function */ +static void dmdx_response(pmix_status_t status, char *data, size_t sz, void *cbdata) +{ + int rc; + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata; + + rc = ext4x_convert_rc(status); + if (NULL != op->mdxcbfunc) { + op->mdxcbfunc(rc, data, sz, op->cbdata, NULL, NULL); + } + OBJ_RELEASE(op); +} + +/* request modex data for a local proc from the PMIx server */ +int ext4x_server_dmodex(const opal_process_name_t *proc, + opal_pmix_modex_cbfunc_t cbfunc, void *cbdata) +{ + ext4x_opcaddy_t *op; + pmix_status_t rc; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* setup the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->mdxcbfunc = cbfunc; + op->cbdata = cbdata; + + /* convert the jobid */ + (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, proc->jobid); + op->p.rank = ext4x_convert_opalrank(proc->vpid); + + /* find the internally-cached data for this proc */ + rc = PMIx_server_dmodex_request(&op->p, dmdx_response, op); + if (PMIX_SUCCESS != rc) { + OBJ_RELEASE(op); + } + return ext4x_convert_rc(rc); +} + +/* tell the PMIx server to notify its local clients of an event */ +int ext4x_server_notify_event(int status, + const opal_process_name_t *source, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + opal_value_t *kv; + pmix_info_t *pinfo; + size_t sz, n; + pmix_status_t rc; + ext4x_opcaddy_t *op; + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* convert the list to an array of pmix_info_t */ + if (NULL != info && 0 < (sz = opal_list_get_size(info))) { + PMIX_INFO_CREATE(pinfo, sz); + n = 0; + OPAL_LIST_FOREACH(kv, info, opal_value_t) { + (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); + if (0 == strcmp(kv->key, OPAL_PMIX_JOB_TERM_STATUS)) { + pinfo[n].value.type = PMIX_STATUS; + pinfo[n].value.data.status = ext4x_convert_opalrc(kv->data.integer); + } else { + ext4x_value_load(&pinfo[n].value, kv); + } + ++n; + } + } else { + sz = 0; + pinfo = NULL; + } + /* setup the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->info = pinfo; + op->sz = sz; + op->opcbfunc = cbfunc; + op->cbdata = cbdata; + /* convert the jobid */ + if (NULL == source) { + (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, OPAL_JOBID_INVALID); + op->p.rank = ext4x_convert_opalrank(OPAL_VPID_INVALID); + } else { + (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, source->jobid); + op->p.rank = ext4x_convert_opalrank(source->vpid); + } + + + rc = ext4x_convert_opalrc(status); + /* the range must be nonlocal so the server will pass + * the event down to its local clients */ + rc = PMIx_Notify_event(rc, &op->p, PMIX_RANGE_SESSION, + pinfo, sz, opcbfunc, op); + if (PMIX_SUCCESS != rc) { + OBJ_RELEASE(op); + } + return ext4x_convert_rc(rc); +} + +int ext4x_server_iof_push(const opal_process_name_t *source, + opal_pmix_iof_channel_t channel, + unsigned char *data, size_t nbytes) +{ + ext4x_opcaddy_t *op; + pmix_byte_object_t bo; + pmix_iof_channel_t pchan; + opal_pmix_lock_t lock; + pmix_status_t rc; + int ret; + + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s IOF push from %s with %d bytes", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + OPAL_NAME_PRINT(*source), (int)nbytes); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* setup the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + /* convert the source */ + (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, source->jobid); + op->p.rank = ext4x_convert_opalrank(source->vpid); + /* convert the channel */ + pchan = 0; + if (OPAL_PMIX_FWD_STDIN_CHANNEL & channel) { + pchan |= PMIX_FWD_STDIN_CHANNEL; + } + if (OPAL_PMIX_FWD_STDOUT_CHANNEL & channel) { + pchan |= PMIX_FWD_STDOUT_CHANNEL; + } + if (OPAL_PMIX_FWD_STDERR_CHANNEL & channel) { + pchan |= PMIX_FWD_STDERR_CHANNEL; + } + if (OPAL_PMIX_FWD_STDDIAG_CHANNEL & channel) { + pchan |= PMIX_FWD_STDDIAG_CHANNEL; + } + + /* setup the byte object */ + PMIX_BYTE_OBJECT_CONSTRUCT(&bo); + if (0 < nbytes) { + bo.bytes = (char*)data; + } + bo.size = nbytes; + + /* push the IO */ + OPAL_PMIX_CONSTRUCT_LOCK(&lock); + rc = PMIx_server_IOF_deliver(&op->p, pchan, &bo, NULL, 0, lkcbfunc, (void*)&lock); + if (PMIX_SUCCESS != rc) { + ret = ext4x_convert_rc(rc); + } else { + /* wait for completion */ + OPAL_PMIX_WAIT_THREAD(&lock); + ret = lock.status; + OPAL_PMIX_DESTRUCT_LOCK(&lock); + } + /* cleanup */ + OBJ_RELEASE(op); + + return ret; +} + +static void final_cleanup(int status, void *cbdata) +{ + ext4x_opalcaddy_t *opalcaddy = (ext4x_opalcaddy_t*)cbdata; + OBJ_RELEASE(opalcaddy); +} + +static void setup_cbfunc(pmix_status_t status, + pmix_info_t info[], size_t ninfo, + void *provided_cbdata, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)provided_cbdata; + ext4x_opalcaddy_t *opalcaddy; + size_t n; + opal_value_t *iptr; + int rc; + pmix_status_t ret = PMIX_SUCCESS; + + /* setup the caddy */ + opalcaddy = OBJ_NEW(ext4x_opalcaddy_t); + + rc = ext4x_convert_rc(status); + if (OPAL_SUCCESS == rc && NULL != info) { + /* need to convert the info array to a list */ + for (n=0; n < ninfo; n++) { + iptr = OBJ_NEW(opal_value_t); + opal_list_append(&opalcaddy->info, &iptr->super); + iptr->key = strdup(info[n].key); + if (OPAL_SUCCESS != (rc = ext4x_value_unload(iptr, &info[n].value))) { + OBJ_RELEASE(opalcaddy); + ret = ext4x_convert_opalrc(rc); + goto done; + } + } + } + + done: + /* release our caller */ + if (NULL != cbfunc) { + cbfunc(ret, cbdata); + } + /* pass what we have upstairs */ + if (NULL != op->setupcbfunc) { + op->setupcbfunc(rc, &opalcaddy->info, op->cbdata, + final_cleanup, opalcaddy); + } + OBJ_RELEASE(op); +} + +int ext4x_server_setup_application(opal_jobid_t jobid, + opal_list_t *info, + opal_pmix_setup_application_cbfunc_t cbfunc, void *cbdata) +{ + opal_value_t *kv; + pmix_info_t *pinfo; + size_t sz, n; + pmix_status_t rc; + ext4x_opcaddy_t *op; + + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s setup application for job %s", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + OPAL_JOBID_PRINT(jobid)); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* convert the list to an array of pmix_info_t */ + if (NULL != info && 0 < (sz = opal_list_get_size(info))) { + PMIX_INFO_CREATE(pinfo, sz); + n = 0; + OPAL_LIST_FOREACH(kv, info, opal_value_t) { + (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&pinfo[n].value, kv); + ++n; + } + } else { + sz = 0; + pinfo = NULL; + } + /* setup the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->info = pinfo; + op->sz = sz; + op->setupcbfunc = cbfunc; + op->cbdata = cbdata; + /* convert the jobid */ + (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, jobid); + + rc = PMIx_server_setup_application(op->p.nspace, op->info, op->sz, + setup_cbfunc, op); + if (PMIX_SUCCESS != rc) { + OBJ_RELEASE(op); + } + return ext4x_convert_rc(rc); +} + +int ext4x_server_setup_local_support(opal_jobid_t jobid, + opal_list_t *info, + opal_pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + opal_value_t *kv; + pmix_info_t *pinfo; + size_t sz, n; + pmix_status_t rc; + ext4x_opcaddy_t *op; + + opal_output_verbose(2, opal_pmix_base_framework.framework_output, + "%s setup local support for job %s", + OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), + OPAL_JOBID_PRINT(jobid)); + + OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + if (0 >= opal_pmix_base.initialized) { + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + return OPAL_ERR_NOT_INITIALIZED; + } + OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + + /* convert the list to an array of pmix_info_t */ + if (NULL != info && 0 < (sz = opal_list_get_size(info))) { + PMIX_INFO_CREATE(pinfo, sz); + n = 0; + OPAL_LIST_FOREACH(kv, info, opal_value_t) { + (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN); + ext4x_value_load(&pinfo[n].value, kv); + ++n; + } + } else { + sz = 0; + pinfo = NULL; + } + /* setup the caddy */ + op = OBJ_NEW(ext4x_opcaddy_t); + op->info = pinfo; + op->sz = sz; + op->opcbfunc = cbfunc; + op->cbdata = cbdata; + /* convert the jobid */ + (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, jobid); + + rc = PMIx_server_setup_local_support(op->p.nspace, op->info, op->sz, + opcbfunc, op); + if (PMIX_SUCCESS != rc) { + OBJ_RELEASE(op); + } + return ext4x_convert_rc(rc); +} diff --git a/opal/mca/pmix/ext4x/help-pmix-ext4x.txt b/opal/mca/pmix/ext4x/help-pmix-ext4x.txt new file mode 100644 index 00000000000..0967326db8e --- /dev/null +++ b/opal/mca/pmix/ext4x/help-pmix-ext4x.txt @@ -0,0 +1,32 @@ +# -*- text -*- +# +# Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2017-2018 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# +# This is the US/English help file for Open MPI MCA error messages. +# +[evars] +We found conflicting directives regarding the location of OPAL vs PMIx +installation directories: + +%s + +This usually indicates that OMPI was configured to use its internal copy +of PMIx, but another installation of PMIx is also in use on this system +and could potentially cause confusion between the two sets of plugins. +Please either unset the indicated environment variables, or configure +OMPI to use the external PMIx installation. diff --git a/opal/mca/pmix/pmix3x/pmix/Makefile.am b/opal/mca/pmix/pmix3x/pmix/Makefile.am index 1c70624e913..4007e0d9b08 100644 --- a/opal/mca/pmix/pmix3x/pmix/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2006-2016 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -23,7 +23,7 @@ # via AC_CONFIG_MACRO_DIR in configure.ac. ACLOCAL_AMFLAGS = -I ./config -SUBDIRS = config contrib include src etc +SUBDIRS = config contrib include src etc bindings headers = diff --git a/opal/mca/pmix/pmix3x/pmix/NEWS b/opal/mca/pmix/pmix3x/pmix/NEWS index c52c0b0de67..0a53c436bc0 100644 --- a/opal/mca/pmix/pmix3x/pmix/NEWS +++ b/opal/mca/pmix/pmix3x/pmix/NEWS @@ -21,8 +21,59 @@ example, a bug might be fixed in the master, and then moved to the current release as well as the "stable" bug fix release branch. -Master (not on release branches yet) +3.0.0 -- TBD ------------------------------------ +**** NOTE: This release implements the complete PMIX v3.0 Standard +**** and therefore includes a number of new APIs and features. These +**** can be tracked by their RFC's on the community website: +**** https://pmix.org/pmix-standard. +- Added blocking forms of several existing APIs: + - PMIx_Log + - PMIx_Allocation_request + - PMIx_Job_control + - PMIx_Process_monitor +- Added support for getting/validating security credentials + - PMIx_Get_credential, PMIx_Validate_credential +- Extended support for debuggers/tools + - Added IO forwarding support allowing tools to request + forwarding of output from specific application procs, + and to forward their input to specified target procs + - Extended tool attributes to support synchronization + during startup of applications. This includes the + ability to modify an application's environment + (including support for LD_PRELOAD) and define an + alternate fork/exec agent + - Added ability for a tool to switch server connections + so it can first connect to a system-level server to + launch a starter program, and then reconnect to that + starter for debugging purposes +- Extended network support to collect network inventory by + either rolling it up from individual nodes or by direct + query of fabric managers. Added an API by which the + host can inject any rolled up inventory into the local + PMIx server. Applications and/or the host RM can access + the inventory via the PMIx_Query function. +- Added the ability for applications and/or tools to register + files and directories for cleanup upon their termination +- Added support for inter-library coordination within a process +- Extended PMIx_Log support by adding plugin support for new + channels, including local/remote syslog and email. Added + attributes to query available channels and to tag and + format output. + + + + +2.1.1 -- 23 Feb 2018 +---------------------- +- Fix direct modex when receiving new nspace +- Resolve direct modex of job-level info +- Fix a bug in attribute configuration checks +- Fix a couple of bugs in unpacking of direct modex job-level data +- Correcly handle application setup data during "instant on" launch +- add a PMIX_BYTE_OBJECT_LOAD convenience macro +- Fix two early "free" bugs +- Add an example PMI-1 client program 2.1.0 -- 1 Feb 2018 @@ -40,7 +91,7 @@ Master (not on release branches yet) sets of numbers -2.0.3 -- TBD +2.0.3 -- 1 Feb 2018 ---------------------- - Fix event notification so all sides of multi-library get notified of other library's existence diff --git a/opal/mca/pmix/pmix3x/pmix/VERSION b/opal/mca/pmix/pmix3x/pmix/VERSION index 27c205e0df0..52531c09510 100644 --- a/opal/mca/pmix/pmix3x/pmix/VERSION +++ b/opal/mca/pmix/pmix3x/pmix/VERSION @@ -30,7 +30,7 @@ greek= # command, or with the date (if "git describe" fails) in the form of # "date". -repo_rev=gitf0b8151 +repo_rev=gita493add # If tarball_version is not empty, it is used as the version string in # the tarball filename, regardless of all other versions listed in @@ -44,7 +44,7 @@ tarball_version= # The date when this release was created -date="Mar 24, 2018" +date="Jun 10, 2018" # The shared library version of each of PMIx's public libraries. # These versions are maintained in accordance with the "Library @@ -75,6 +75,6 @@ date="Mar 24, 2018" # Version numbers are described in the Libtool current:revision:age # format. -libpmix_so_version=0:0:0 -libpmi_so_version=0:0:0 -libpmi2_so_version=0:0:0 +libpmix_so_version=4:0:2 +libpmi_so_version=1:0:0 +libpmi2_so_version=1:0:0 diff --git a/opal/mca/pmix/pmix3x/pmix/bindings/Makefile.am b/opal/mca/pmix/pmix3x/pmix/bindings/Makefile.am new file mode 100644 index 00000000000..e1c8d72b977 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/bindings/Makefile.am @@ -0,0 +1,22 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +SUBDIRS = python diff --git a/opal/mca/pmix/pmix3x/pmix/bindings/README b/opal/mca/pmix/pmix3x/pmix/bindings/README new file mode 100644 index 00000000000..5e7fafbf3fb --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/bindings/README @@ -0,0 +1,19 @@ +Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + +$COPYRIGHT$ + +Additional copyrights may follow + +$HEADER$ + +=========================================================================== + +This is where bindings of PMIx functions to alternative programming languages +such as Python reside. All functions defined in the public headers have been +provided with a wrapper. Note that there is no restriction on the number of +wrappers that can exist, nor on what type of function is wrapped. + +There is only one rule to observe: you can wrap a framework, but you cannot wrap a +specific plugin within that framework. This constraint flows from the fact that +plugins are only accessed via the framework interface - thus, there is no way to +guarantee that a particular plugin will be the active selection. diff --git a/opal/mca/pmix/pmix3x/pmix/bindings/python/Makefile.am b/opal/mca/pmix/pmix3x/pmix/bindings/python/Makefile.am new file mode 100644 index 00000000000..47dde8f34ff --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/bindings/python/Makefile.am @@ -0,0 +1,39 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +helpers = setup.py client.py server.py cpmix.pxd pmix.pyx + +if WANT_PYTHON_BINDINGS + +install-exec-local: $(helpers) + $(PYTHON) setup.py build_ext --include-dirs="$(top_builddir)/include" --library-dirs="$(DESTDIR)$(libdir)" --user + $(PYTHON) setup.py install --prefix="$(DESTDIR)$(prefix)" + +uninstall-hook: + rm -f $(pythondir)/pmix*.so + rm -f $(pythondir)/pypmix-*.egg-info + +CLEANFILES += pmix.c + +clean-local: + rm -rf build + +endif diff --git a/opal/mca/pmix/pmix3x/pmix/bindings/python/README b/opal/mca/pmix/pmix3x/pmix/bindings/python/README new file mode 100644 index 00000000000..0fbdd06fedf --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/bindings/python/README @@ -0,0 +1,49 @@ +=========================================================================== +Cython-based Python wrapper for PMIx +=========================================================================== + + +Example +------- + +This example starts up a persistent DVM and then spawns some tasks using +Python. + +$ virtualenv ve +$ source ve/bin/activate +$ pip install orte-cffi +$ orte-dvm --report-uri dvm_uri +$ python examples/submit.py + + +Create a distfile +---------------------------------------- + +If you want to create a sdist file: + +$ virtualenv ve +$ source ve/bin/activate +$ python setup.py sdist + + +Uploading sdist to pypi +----------------------- + +Assuming you have admin privileges to the pypi package repository for this +package, a new version can be uploaded using twine: + +$ virtualenv ve +$ source ve/bin/activate +$ pip install twine +$ twine upload dist/orte-cffi-`python setup.py --version`.tar.gz + + +Building (for development purposes only) +---------------------------------------- + +If you want to create a non-pip build: + +$ virtualenv ve +$ source ve/bin/activate +$ pip install cffi +$ python src/orte-cffi/build.py diff --git a/opal/mca/pmix/pmix3x/pmix/config/pmix.m4 b/opal/mca/pmix/pmix3x/pmix/config/pmix.m4 index eac9bf59921..7ee051d6f6f 100644 --- a/opal/mca/pmix/pmix3x/pmix/config/pmix.m4 +++ b/opal/mca/pmix/pmix3x/pmix/config/pmix.m4 @@ -727,6 +727,14 @@ AC_DEFUN([PMIX_SETUP_CORE],[ PMIX_LIBEVENT_CONFIG + ################################## + # HWLOC + ################################## + pmix_show_title "HWLOC" + + PMIX_HWLOC_CONFIG + + ################################## # ZLIB COMPRESSION ################################## @@ -825,6 +833,8 @@ AC_DEFUN([PMIX_SETUP_CORE],[ AC_CONFIG_FILES( pmix_config_prefix[Makefile] + pmix_config_prefix[bindings/Makefile] + pmix_config_prefix[bindings/python/Makefile] pmix_config_prefix[config/Makefile] pmix_config_prefix[etc/Makefile] pmix_config_prefix[include/Makefile] @@ -832,7 +842,7 @@ AC_DEFUN([PMIX_SETUP_CORE],[ pmix_config_prefix[src/util/keyval/Makefile] pmix_config_prefix[src/mca/base/Makefile] pmix_config_prefix[src/tools/pevent/Makefile] - pmix_config_prefix[src/tools/pinfo/Makefile] + pmix_config_prefix[src/tools/pmix_info/Makefile] pmix_config_prefix[src/tools/plookup/Makefile] pmix_config_prefix[src/tools/pps/Makefile] ) @@ -1118,6 +1128,44 @@ fi AM_CONDITIONAL([PMIX_INSTALL_BINARIES], [test $WANT_PMIX_BINARIES -eq 1]) +# +# Install Python bindings? +# +AC_MSG_CHECKING([if want install Python bindings]) +AC_ARG_ENABLE(python-bindings, + AC_HELP_STRING([--enable-python-bindings], + [enable Python bindings (default: disabled)])) +if test "$enable_python_bindings" != "yes"; then + AC_MSG_RESULT([no]) + WANT_PYTHON_BINDINGS=0 +else + AC_MSG_RESULT([yes]) + WANT_PYTHON_BINDINGS=1 +fi + +AM_CONDITIONAL([WANT_PYTHON_BINDINGS], [test $WANT_PYTHON_BINDINGS -eq 1]) + +if test "$WANT_PYTHON_BINDINGS" = "1"; then + AM_PATH_PYTHON([2.7], [python_happy=1], [python_happy=0]) + if test "$python_happy" = "0"; then + AC_MSG_WARN([Python bindings were enabled, but no suitable]) + AC_MSG_WARN([interpreter was found. PMIx requires at least]) + AC_MSG_WARN([Python v2.7 to provide Python bindings]) + AC_MSG_ERROR([Cannot continue]) + fi + + AC_MSG_CHECKING([if Cython package installed]) + have_cython=esyscmd(config/pmix_check_cython.py) + if test "$have_cython" = "0"; then + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + AC_MSG_WARN([Python bindings were enabled, but the Cython]) + AC_MSG_WARN([package was not found. PMIx Python bindings]) + AC_MSG_WARN([require that the Cython package be installed]) + AC_MSG_ERROR([Cannot continue]) + fi +fi ])dnl diff --git a/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_hwloc.m4 b/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_hwloc.m4 new file mode 100644 index 00000000000..4ee046d9abd --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_hwloc.m4 @@ -0,0 +1,106 @@ +# -*- shell-script -*- +# +# Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# MCA_hwloc_CONFIG([action-if-found], [action-if-not-found]) +# -------------------------------------------------------------------- +AC_DEFUN([PMIX_HWLOC_CONFIG],[ + PMIX_VAR_SCOPE_PUSH([pmix_hwloc_dir pmix_hwloc_libdir pmix_hwloc_standard_lib_location pmix_hwloc_standard_header_location]) + + AC_ARG_WITH([hwloc], + [AC_HELP_STRING([--with-hwloc=DIR], + [Search for hwloc headers and libraries in DIR ])]) + + AC_ARG_WITH([hwloc-libdir], + [AC_HELP_STRING([--with-hwloc-libdir=DIR], + [Search for hwloc libraries in DIR ])]) + + pmix_hwloc_support=0 + + if test "$with_hwloc" != "no"; then + AC_MSG_CHECKING([for hwloc in]) + if test ! -z "$with_hwloc" && test "$with_hwloc" != "yes"; then + pmix_hwloc_dir=$with_hwloc + pmix_hwloc_standard_header_location=no + pmix_hwloc_standard_lib_location=no + AS_IF([test -z "$with_hwloc_libdir" || test "$with_hwloc_libdir" = "yes"], + [if test -d $with_hwloc/lib; then + pmix_hwloc_libdir=$with_hwloc/lib + elif test -d $with_hwloc/lib64; then + pmix_hwloc_libdir=$with_hwloc/lib64 + else + AC_MSG_RESULT([Could not find $with_hwloc/lib or $with_hwloc/lib64]) + AC_MSG_ERROR([Can not continue]) + fi + AC_MSG_RESULT([$pmix_hwloc_dir and $pmix_hwloc_libdir])], + [AC_MSG_RESULT([$with_hwloc_libdir])]) + else + AC_MSG_RESULT([(default search paths)]) + pmix_hwloc_standard_header_location=yes + pmix_hwloc_standard_lib_location=yes + fi + AS_IF([test ! -z "$with_hwloc_libdir" && test "$with_hwloc_libdir" != "yes"], + [pmix_hwloc_libdir="$with_hwloc_libdir" + pmix_hwloc_standard_lib_location=no]) + + PMIX_CHECK_PACKAGE([pmix_hwloc], + [hwloc.h], + [hwloc], + [hwloc_topology_init], + [-lhwloc], + [$pmix_hwloc_dir], + [$pmix_hwloc_libdir], + [pmix_hwloc_support=1], + [pmix_hwloc_support=0]) + if test $pmix_hwloc_support = "1"; then + LIBS="$LIBS -lhwloc" + PMIX_EMBEDDED_LIBS="$PMIX_EMBEDDED_LIBS -lhwloc" + if test "$pmix_hwloc_standard_header_location" != "yes"; then + PMIX_EMBEDDED_CPPFLAGS="$PMIX_EMBEDDED_CPPFLAGS $pmix_hwloc_CPPFLAGS" + CPPFLAGS="$CPPFLAGS $pmix_hwloc_CPPFLAGS" + fi + if test "$pmix_hwloc_standard_lib_location" != "yes"; then + PMIX_EMBEDDED_LDFLAGS="$PMIX_EMBEDDED_LDFLAGS $pmix_hwloc_LDFLAGS" + LDFLAGS="$LDFLAGS $pmix_hwloc_LDFLAGS" + fi + fi + fi + + if test ! -z "$with_hwloc" && test "$with_hwloc" != "no" && test "$pmix_hwloc_support" != "1"; then + AC_MSG_WARN([HWLOC SUPPORT REQUESTED AND NOT FOUND]) + AC_MSG_ERROR([CANNOT CONTINUE]) + fi + + if test $pmix_hwloc_support = "1"; then + AC_MSG_CHECKING([if external hwloc version is 1.5 or greater]) + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[#include ]], + [[ + #if HWLOC_API_VERSION < 0x00010500 + #error "hwloc API version is less than 0x00010500" + #endif + ]])], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AC_MSG_ERROR([Cannot continue])]) + fi + + AC_MSG_CHECKING([will hwloc support be built]) + if test "$pmix_hwloc_support" != "1"; then + AC_MSG_RESULT([no]) + else + AC_MSG_RESULT([yes]) + fi + + AC_DEFINE_UNQUOTED([PMIX_HAVE_HWLOC], [$pmix_hwloc_support], + [Whether or not we have hwloc support]) + PMIX_VAR_SCOPE_POP +])dnl diff --git a/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/Makefile b/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/Makefile index 2950748fd48..b3b60720a2f 100644 --- a/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/Makefile +++ b/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/Makefile @@ -1,21 +1,29 @@ -PMIX_BASE = +PMIX_BASE = /tmp/artemp/pmix +SLURM_BASE = /tmp/artemp/slurm + PMIX_INC= -I$(PMIX_BASE)/include/ -PMIX_LIB= -L$(PMIX_BASE)/lib/ -lpmix +PMIX_LIB= -L$(PMIX_BASE)/lib/ -L$(PMIX_BASE)/lib64/ -lpmix + +PMI2_BASE = $(SLURM_BASE) +PMI2_INC= -I$(PMI2_BASE)/include/ +PMI2_LIB= -L$(PMI2_BASE)/lib/ -L$(PMI2_BASE)/lib64/ -lpmi2 -PMI2_BASE = /usr/ -#PMI2_INC= -I$(PMI2_BASE)/include/ -#PMI2_LIB= -L$(PMI2_BASE)/lib/ -lpmi2 -PMI2_LIB= -lpmi2 +PMI1_BASE = $(SLURM_BASE) +PMI1_INC= -I$(PMI1_BASE)/include/ +PMI1_LIB= -L$(PMI1_BASE)/lib/ -L$(PMI1_BASE)/lib64/ -lpmi CFLAGS = -O2 -g -all: pmix pmi2 +all: pmix_intra_perf pmi2_intra_perf pmi1_intra_perf -pmix: pmi_intra_perf.c pmi.h pmix.c +pmix_intra_perf: pmi_intra_perf.c pmi.h pmix.c gcc $(PMIX_INC) $(CFLAGS) -o pmix_intra_perf pmi_intra_perf.c pmix.c $(PMIX_LIB) -lrt -pmi2: pmi_intra_perf.c pmi.h pmi2.c pmi2_pmap_parser.c pmi2_pmap_parser.h pmi2_utils.c pmi2_utils.h +pmi2_intra_perf: pmi_intra_perf.c pmi.h pmi2.c pmi2_pmap_parser.c pmi2_pmap_parser.h pmi2_utils.c pmi2_utils.h gcc $(PMI2_INC) $(CFLAGS) -o pmi2_intra_perf pmi_intra_perf.c pmi2.c pmi2_utils.c pmi2_pmap_parser.c -lrt $(PMI2_LIB) +pmi1_intra_perf: pmi_intra_perf.c pmi.h pmi1.c pmi2_utils.c pmi2_utils.h + gcc $(PMI1_INC) $(CFLAGS) -o pmi1_intra_perf pmi_intra_perf.c pmi1.c pmi2_utils.c -lrt $(PMI1_LIB) + clean: - rm -f pmix_intra_perf pmi2_intra_perf + rm -f pmix_intra_perf pmi2_intra_perf pmi1_intra_perf diff --git a/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi.h b/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi.h index 4782a1903c9..38c90df2505 100644 --- a/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi.h +++ b/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi.h @@ -15,6 +15,7 @@ void pmi_init(int *rank, int *size); void pmi_get_local_ranks(int **local_ranks, int *local_cnt); +void pmi_get_shmem_size(char *is_avail, size_t *cum_size); void pmi_put_key_loc(char *key, int *key_val, int key_size); void pmi_put_key_rem(char *key, int *key_val, int key_size); void pmi_put_double(char *key, double val); diff --git a/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi2.c b/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi2.c index b6b34122638..2d2d7761e38 100644 --- a/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi2.c +++ b/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi2.c @@ -75,6 +75,11 @@ void pmi_get_local_ranks(int **local_ranks, int *local_cnt) free(pmapping); } +void pmi_get_shmem_size(char *is_avail, size_t *size) +{ + *is_avail = 0; +} + void pmi_put_key_loc(char *key, int *key_val, int key_size) { char *encoded = pmi_encode(key_val, key_size * sizeof(int)); @@ -169,7 +174,7 @@ void pmi_get_key_rem(int rank, char *key_name, int **key_val, int *key_size) free(tmp); } -float pmi_get_double(int rank, char *key) +double pmi_get_double(int rank, char *key) { int len, rc; size_t tmp_size; diff --git a/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi_intra_perf.c b/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi_intra_perf.c index fe6bc2ac742..e0acc3cf3cd 100644 --- a/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi_intra_perf.c +++ b/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi_intra_perf.c @@ -67,6 +67,10 @@ void parse_options(int argc, char **argv) switch (c) { case 's': key_size = atoi(optarg); + /* Make sure that we transform it to int as + * this is what will be the key value type + */ + key_size = key_size / 4 + !!(key_size % 4); break; case 'c': key_count = atoi(optarg); @@ -159,9 +163,11 @@ int main(int argc, char **argv) int *local_ranks, local_cnt; int *remote_ranks, remote_cnt; double start, total_start, get_loc_time = 0, get_rem_time = 0, put_loc_time = 0, - put_rem_time = 0, commit_time = 0, fence_time = 0, init_time = 0, total_time = 0; + put_rem_time = 0, commit_time = 0, fence_time = 0, init_time = 0, total_time = 0; int get_loc_cnt = 0, get_rem_cnt = 0, put_loc_cnt = 0, put_rem_cnt = 0; double mem_pss = 0.0, mem_rss = 0.0; + char have_shmem; + size_t shmem_job_info, shmem_all; parse_options(argc, argv); @@ -177,6 +183,14 @@ int main(int argc, char **argv) fill_remote_ranks(local_ranks, local_cnt, remote_ranks, nproc); } + pmi_get_shmem_size(&have_shmem, &shmem_job_info); + + /* + * Make sure that no other rank started publishing keys in the dstore + * before we finished with shmem size screening + */ + pmi_fence( 0 ); + if( 0 == rank && debug_on ){ int i; fprintf(stderr,"%d: local ranks: ", rank); @@ -302,6 +316,7 @@ int main(int argc, char **argv) commit_time, fence_time); } + pmi_get_shmem_size(&have_shmem, &shmem_all); /* * The barrier ensures that all procs finished key fetching * we had issues with dstor/lockless case evaluation @@ -501,10 +516,12 @@ int main(int argc, char **argv) fprintf(stderr,"total: max %lf min %lf\n", max_total_time, min_total_time); fprintf(stderr,"mem: loc %0.2lf avg %0.2lf min %0.2lf max %0.2lf total %0.2lf Kb\n", mem_pss, cum_mem_pss / nproc, min_mem_pss, max_mem_pss, cum_mem_pss); + if( have_shmem ) { + fprintf(stderr,"shmem: job_info: %0.2lf total %0.2lf Kb\n", + (double)shmem_job_info / 1024, (double)shmem_all / 1024); + } - - /* debug printout */ -/* + /* debug printout *//* for(i = 0; i < nproc; i++){ double val; printf("%d: ", i); diff --git a/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmix.c b/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmix.c index 9bac8fd4ac0..b1865bf9843 100644 --- a/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmix.c +++ b/opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmix.c @@ -10,9 +10,16 @@ */ #include +#include +#include +#include +#include +#include +#include #include pmix_proc_t this_proc; +static int _int_size = 0; void pmi_init(int *rank, int *size) { @@ -40,7 +47,7 @@ void pmi_init(int *rank, int *size) fprintf(stderr, "Client ns %s rank %d: PMIx_Get job size failed: %d", this_proc.nspace, this_proc.rank, rc); abort(); } - *size = val->data.uint32; + _int_size = *size = val->data.uint32; *rank = this_proc.rank; PMIX_VALUE_RELEASE(val); } @@ -81,6 +88,94 @@ void pmi_get_local_ranks(int **local_ranks, int *local_cnt) } } +/* WARNING: should match one in + * src/mca/gds/ds12/gds_dstore.h + */ +typedef struct { + size_t rank; + size_t offset; + size_t count; +} rank_meta_info; + +#define DSTORE_INIT_SEG "initial-pmix_shared-segment" +#define DSTORE_META_SEG "smseg" +#define DSTORE_DATA_SEG "smdataseg" + +typedef enum { other_seg, init_seg, meta_seg, data_seg } seg_type_t; + +struct { + char *sign; + seg_type_t type; +} segments[] = { + {DSTORE_INIT_SEG, init_seg}, + {DSTORE_META_SEG, meta_seg}, + {DSTORE_DATA_SEG, data_seg}, +}; + +static char *_get_maps_file(char *line) +{ + char *token = NULL; + char *saveptr = NULL; + int i = 0; + + token = strtok_r(line, "-\n", &saveptr); + if(NULL == token) { + return NULL; + } + return (char*)strtoul(token, &saveptr, 16); +} + +static seg_type_t +is_dstor_region(char *line, char **base) +{ + int i; + seg_type_t type = other_seg; + for(i=0; i<3; i++){ + if( strstr(line, segments[i].sign) ){ + if(!(*base = _get_maps_file(line)) ){ + return other_seg; + } + return segments[i].type; + } + } + return other_seg; +} + +void pmi_get_shmem_size(char *is_avail, size_t *cum_size) +{ + char maps_path[PATH_MAX] = "/proc/self/maps"; + FILE *maps_fp = NULL; + char *line = NULL; + size_t size = 0, meta_size = 0, data_size = 0; + char *base; + + if (NULL == (maps_fp = fopen(maps_path, "r"))) { + abort(); + } + + while ((size = getline(&line, &size, maps_fp)) != -1) { + seg_type_t type = is_dstor_region(line, &base); + switch(type) { + case other_seg: + case init_seg: + case meta_seg: + break; + case data_seg:{ + data_size += *((size_t*)base); + break; + } + } + } + free(line); + fclose(maps_fp); + *is_avail = 0; + *cum_size = data_size; + if( *cum_size > 0 ) { + *is_avail = 1; + *cum_size += sizeof(rank_meta_info) * _int_size; + } +} + static void _put_key(char *key, int *key_val, int key_size, pmix_scope_t scope) { pmix_value_t value; diff --git a/opal/mca/pmix/pmix3x/pmix/contrib/pmix.spec b/opal/mca/pmix/pmix3x/pmix/contrib/pmix.spec old mode 100755 new mode 100644 index 98cc55d2bc0..5f4613b6c26 --- a/opal/mca/pmix/pmix3x/pmix/contrib/pmix.spec +++ b/opal/mca/pmix/pmix3x/pmix/contrib/pmix.spec @@ -12,7 +12,7 @@ # Copyright (c) 2006-2016 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2013 Mellanox Technologies, Inc. # All rights reserved. -# Copyright (c) 2015-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2015-2018 Intel, Inc. All rights reserved. # Copyright (c) 2015 Research Organization for Information Science # and Technology (RIST). All rights reserved. # $COPYRIGHT$ diff --git a/opal/mca/pmix/pmix3x/pmix/examples/Makefile.am b/opal/mca/pmix/pmix3x/pmix/examples/Makefile.am index d2e9597a9cd..72ed7216131 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/examples/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -21,7 +21,7 @@ AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/include -I$(top_builddir)/include/pmix -noinst_PROGRAMS = client client2 dmodex dynamic fault pub tool debugger debuggerd alloc jctrl +noinst_PROGRAMS = client client2 dmodex dynamic fault pub pubi tool debugger debuggerd alloc jctrl if !WANT_HIDDEN # these examples use internal symbols # use --disable-visibility @@ -68,6 +68,10 @@ pub_SOURCES = pub.c pub_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) pub_LDADD = $(top_builddir)/src/libpmix.la +pubi_SOURCES = pubi.c +pubi_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) +pubi_LDADD = $(top_builddir)/src/libpmix.la + tool_SOURCES = tool.c tool_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) tool_LDADD = $(top_builddir)/src/libpmix.la diff --git a/opal/mca/pmix/pmix3x/pmix/examples/debuggerd.c b/opal/mca/pmix/pmix3x/pmix/examples/debuggerd.c index 2d21f241e99..80f0aeeff73 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/debuggerd.c +++ b/opal/mca/pmix/pmix3x/pmix/examples/debuggerd.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * diff --git a/opal/mca/pmix/pmix3x/pmix/examples/dynamic.c b/opal/mca/pmix/pmix3x/pmix/examples/dynamic.c index 2e31fd17329..4ffe501f02f 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/dynamic.c +++ b/opal/mca/pmix/pmix3x/pmix/examples/dynamic.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -44,7 +44,7 @@ int main(int argc, char **argv) int rc; pmix_value_t value; pmix_value_t *val = &value; - pmix_proc_t proc, newproc; + pmix_proc_t proc; uint32_t nprocs; char nsp2[PMIX_MAX_NSLEN+1]; pmix_app_t *app; @@ -135,13 +135,13 @@ int main(int argc, char **argv) /* just cycle the connect/disconnect functions */ (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; - if (PMIX_SUCCESS != (rc = PMIx_Connect(&proc, 1, NULL, 0, newproc.nspace, &newproc.rank))) { + if (PMIX_SUCCESS != (rc = PMIx_Connect(&proc, 1, NULL, 0))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Connect failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; } - fprintf(stderr, "Client ns %s rank %d: PMIx_Connect succeeded - new ID: %s:%d\n", - myproc.nspace, myproc.rank, newproc.nspace, newproc.rank); - if (PMIX_SUCCESS != (rc = PMIx_Disconnect(newproc.nspace, NULL, 0))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Connect succeeded\n", + myproc.nspace, myproc.rank); + if (PMIX_SUCCESS != (rc = PMIx_Disconnect(&proc, 1, NULL, 0))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Disonnect failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; } diff --git a/opal/mca/pmix/pmix3x/pmix/examples/pub.c b/opal/mca/pmix/pmix3x/pmix/examples/pub.c index d7e02c90a11..89589ac08ca 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/pub.c +++ b/opal/mca/pmix/pmix3x/pmix/examples/pub.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -69,6 +69,7 @@ int main(int argc, char **argv) /* publish something */ if (0 == myproc.rank) { + fprintf(stderr, "%s:%d publishing two keys\n", myproc.nspace, myproc.rank); PMIX_INFO_CREATE(info, 2); (void)strncpy(info[0].key, "FOOBAR", PMIX_MAX_KEYLEN); info[0].value.type = PMIX_UINT8; @@ -80,6 +81,7 @@ int main(int argc, char **argv) fprintf(stderr, "Client ns %s rank %d: PMIx_Publish failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; } + fprintf(stderr, "%s:%d publish complete\n", myproc.nspace, myproc.rank); PMIX_INFO_FREE(info, 2); } @@ -93,6 +95,7 @@ int main(int argc, char **argv) /* lookup something */ if (0 != myproc.rank) { PMIX_PDATA_CREATE(pdata, 1); + fprintf(stderr, "%s:%d looking up key FOOBAR\n", myproc.nspace, myproc.rank); (void)strncpy(pdata[0].key, "FOOBAR", PMIX_MAX_KEYLEN); if (PMIX_SUCCESS != (rc = PMIx_Lookup(pdata, 1, NULL, 0))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup failed: %d\n", myproc.nspace, myproc.rank, rc); @@ -136,6 +139,7 @@ int main(int argc, char **argv) keys[1] = "PANDA"; keys[2] = NULL; + fprintf(stderr, "%s:%d unpublishing two keys\n", myproc.nspace, myproc.rank); if (PMIX_SUCCESS != (rc = PMIx_Unpublish(keys, NULL, 0))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Unpublish failed: %d\n", myproc.nspace, myproc.rank, rc); free(keys); diff --git a/opal/mca/pmix/pmix3x/pmix/examples/pubi.c b/opal/mca/pmix/pmix3x/pmix/examples/pubi.c new file mode 100644 index 00000000000..bbb66e66978 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/examples/pubi.c @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include +#include +#include +#include +#include + +#include + +int main(int argc, char **argv) +{ + pmix_proc_t myproc; + int rc; + pmix_value_t value; + pmix_value_t *val = &value; + pmix_proc_t proc; + uint32_t nprocs; + pmix_info_t *info; + pmix_pdata_t *pdata; + size_t n; + + /* init us */ + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, rc); + exit(0); + } + fprintf(stderr, "Client ns %s rank %d: Running\n", myproc.nspace, myproc.rank); + + /* get our universe size */ + PMIX_PROC_CONSTRUCT(&proc); + (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &val))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get universe size failed: %d\n", myproc.nspace, myproc.rank, rc); + goto done; + } + nprocs = val->data.uint32; + PMIX_VALUE_RELEASE(val); + fprintf(stderr, "Client %s:%d universe size %d\n", myproc.nspace, myproc.rank, nprocs); + + /* publish something */ + if (0 == myproc.rank) { + fprintf(stderr, "%s:%d publishing two keys\n", myproc.nspace, myproc.rank); + PMIX_INFO_CREATE(info, 2); + (void)strncpy(info[0].key, "FOOBAR", PMIX_MAX_KEYLEN); + info[0].value.type = PMIX_UINT8; + info[0].value.data.uint8 = 1; + (void)strncpy(info[1].key, "PANDA", PMIX_MAX_KEYLEN); + info[1].value.type = PMIX_SIZE; + info[1].value.data.size = 123456; + if (PMIX_SUCCESS != (rc = PMIx_Publish(info, 2))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Publish failed: %d\n", myproc.nspace, myproc.rank, rc); + goto done; + } + fprintf(stderr, "%s:%d publish complete\n", myproc.nspace, myproc.rank); + PMIX_INFO_FREE(info, 2); + } + + /* lookup something */ + if (0 != myproc.rank) { + PMIX_PDATA_CREATE(pdata, 2); + fprintf(stderr, "%s:%d looking up key FOOBAR\n", myproc.nspace, myproc.rank); + (void)strncpy(pdata[0].key, "FOOBAR", PMIX_MAX_KEYLEN); + (void)strncpy(pdata[1].key, "PANDA", PMIX_MAX_KEYLEN); + PMIX_INFO_CREATE(info, 1); + rc = 0; + PMIX_INFO_LOAD(&info[0], PMIX_WAIT, &rc, PMIX_INT); + if (PMIX_SUCCESS != (rc = PMIx_Lookup(pdata, 2, info, 1))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup failed: %d\n", myproc.nspace, myproc.rank, rc); + goto done; + } + PMIX_INFO_FREE(info, 1); + /* check the return for value and source */ + for (n=0; n < 2; n++) { + if (0 != strncmp(myproc.nspace, pdata[n].proc.nspace, PMIX_MAX_NSLEN)) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup returned wrong nspace: %s\n", + myproc.nspace, myproc.rank, pdata[n].proc.nspace); + goto done; + } + if (0 != pdata[n].proc.rank) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup returned wrong rank: %d\n", + myproc.nspace, myproc.rank, pdata[n].proc.rank); + goto done; + } + } + if (PMIX_UINT8 != pdata[0].value.type) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup returned wrong type: %d\n", + myproc.nspace, myproc.rank, pdata[0].value.type); + goto done; + } + if (1 != pdata[0].value.data.uint8) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup returned wrong value: %d\n", + myproc.nspace, myproc.rank, (int)pdata[0].value.data.uint8); + goto done; + } + if (PMIX_SIZE != pdata[1].value.type) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup returned wrong type: %d\n", + myproc.nspace, myproc.rank, pdata[1].value.type); + goto done; + } + if (123456 != pdata[1].value.data.size) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup returned wrong value: %d\n", + myproc.nspace, myproc.rank, (int)pdata[1].value.data.size); + goto done; + } + PMIX_PDATA_FREE(pdata, 2); + fprintf(stderr, "PUBLISH-LOOKUP SUCCEEDED\n"); + } + + /* call fence so rank 0 waits before leaving */ + if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, NULL, 0))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d\n", myproc.nspace, myproc.rank, rc); + goto done; + } + + if (0 == myproc.rank) { + char **keys; + keys = (char**)malloc(3 * sizeof(char*)); + keys[0] = "FOOBAR"; + keys[1] = "PANDA"; + keys[2] = NULL; + + fprintf(stderr, "%s:%d unpublishing two keys\n", myproc.nspace, myproc.rank); + if (PMIX_SUCCESS != (rc = PMIx_Unpublish(keys, NULL, 0))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Unpublish failed: %d\n", myproc.nspace, myproc.rank, rc); + free(keys); + goto done; + } + free(keys); + fprintf(stderr, "UNPUBLISH SUCCEEDED\n"); + } + + /* call fence again so everyone waits for rank 0 before leaving */ + if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, NULL, 0))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d\n", myproc.nspace, myproc.rank, rc); + goto done; + } + + done: + /* finalize us */ + fprintf(stderr, "Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank); + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { + fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); + } else { + fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); + } + fflush(stderr); + return(0); +} diff --git a/opal/mca/pmix/pmix3x/pmix/examples/server.c b/opal/mca/pmix/pmix3x/pmix/examples/server.c index 56b82be1eee..a720384f5da 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/server.c +++ b/opal/mca/pmix/pmix3x/pmix/examples/server.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -80,8 +80,8 @@ static pmix_status_t spawn_fn(const pmix_proc_t *proc, pmix_spawn_cbfunc_t cbfunc, void *cbdata); static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, - pmix_connect_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t disconnect_fn(const char nspace[], + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); static pmix_status_t register_event_fn(pmix_status_t *codes, size_t ncodes, @@ -134,6 +134,14 @@ PMIX_CLASS_INSTANCE(pmix_locdat_t, pmix_list_item_t, NULL, NULL); +#define PMIX_WAIT_FOR_COMPLETION(a) \ + do { \ + while ((a)) { \ + usleep(10); \ + } \ + PMIX_ACQUIRE_OBJECT((a)); \ + } while (0) + typedef struct { pmix_object_t super; volatile bool active; @@ -740,7 +748,7 @@ static pmix_status_t spawn_fn(const pmix_proc_t *proc, static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, - pmix_connect_cbfunc_t cbfunc, void *cbdata) + pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_output(0, "SERVER: CONNECT"); @@ -748,14 +756,14 @@ static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, * resource manager for handling */ if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, "FOOBAR", 1, cbdata); + cbfunc(PMIX_SUCCESS, cbdata); } return PMIX_SUCCESS; } -static pmix_status_t disconnect_fn(const char nspace[], +static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { diff --git a/opal/mca/pmix/pmix3x/pmix/include/pmix.h b/opal/mca/pmix/pmix3x/pmix/include/pmix.h index 69f733ed8c0..ae54d85b70a 100644 --- a/opal/mca/pmix/pmix3x/pmix/include/pmix.h +++ b/opal/mca/pmix/pmix3x/pmix/include/pmix.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * @@ -353,9 +353,11 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t nin * appropriate action. Note that different resource managers may respond to * failures in different manners. * - * The host RM will assign a unique nspace to the resulting process group. The - * new nspace must be provided when disconnecting from the group. All procs are - * required to disconnect from the nspace prior to terminating. + * The callback function is to be called once all participating processes have + * called connect. The server is required to return any job-level info for the + * connecting processes that might not already have - i.e., if the connect + * request involves procs from different nspaces, then each proc shall receive + * the job-level info from those nspaces other than their own. * * Note: a process can only engage in _one_ connect operation involving the identical * set of processes at a time. However, a process _can_ be simultaneously engaged @@ -364,61 +366,24 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t nin * As in the case of the fence operation, the info array can be used to pass * user-level directives regarding the algorithm to be used for the collective * operation involved in the "connect", timeout constraints, and other options - * available from the host RM. - * - * The server is required to return any job-level info for the connecting - * processes that they might not already have - i.e., if the connect request - * involves procs from different nspaces, then each proc shall receive the - * job-level info from those nspaces other than their own. - */ - - -/* The blocking form of this call must provide a character array of size - * PMIX_MAX_NSLEN+1 for the assigned nspace of the resulting group, and a pointer - * to an pmix_rank_t location where the new rank of this process in the assigned - * nspace can be returned. Calls will return once the specified operation - * is complete (i.e., all participants have called PMIx_Connect). - */ - + * available from the host RM */ PMIX_EXPORT pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs, - const pmix_info_t info[], size_t ninfo, - char nspace[], pmix_rank_t *newrank); - -/* The callback function for the non-blocking form of the PMIx_Connect - * operation will be called once the operation is complete. Any participant - * that fails to call "connect" prior to terminating will cause the - * operation to return a "failed" status to all other participants. This - * is the default behavior in the absence of any provided directive. - * - * Some additional info keys are provided for this operation: - * - * (a) PMIX_CONNECT_NOTIFY_EACH: generate a local event notification using - * the PMIX_PROC_HAS_CONNECTED event each time a process connects - * - * (b) PMIX_CONNECT_NOTIFY_REQ: notify each of the indicated procs that - * they are requested to connect using the PMIX_CONNECT_REQUESTED event - * - * (c) PMIX_CONNECT_OPTIONAL: participation is optional - do not return - * error if procs terminate without having connected - */ + const pmix_info_t info[], size_t ninfo); PMIX_EXPORT pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, - pmix_connect_cbfunc_t cbfunc, void *cbdata); + pmix_op_cbfunc_t cbfunc, void *cbdata); -/* Disconnect this process from a specified nspace. An error will be returned - * if the specified nspace is not recognized. The info array is used as above. - * - * Processes that terminate while connected to other processes will generate a - * "termination error" event that will be reported to any process in the connected - * group that has registered for such events. Calls to "disconnect" that include the - * PMIX_CONNECT_NOTIFY_EACH info key will cause other processes in the nspace to receive - * an event notification of the disconnect, if they are registered for such events. - */ -PMIX_EXPORT pmix_status_t PMIx_Disconnect(const char nspace[], +/* Disconnect a previously connected set of processes. An error will be returned + * if the specified set of procs was not previously "connected". As above, a process + * may be involved in multiple simultaneous disconnect operations. However, a process + * is not allowed to reconnect to a set of procs that has not fully completed + * disconnect - i.e., you have to fully disconnect before you can reconnect to the + * _same_ group of processes. The info array is used as above. */ +PMIX_EXPORT pmix_status_t PMIx_Disconnect(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo); -PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const char nspace[], +PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const pmix_proc_t ranges[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); @@ -468,6 +433,9 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque * has been completed. The data array must be maintained until * the callback is provided */ +PMIX_EXPORT pmix_status_t PMIx_Log(const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs); + PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata, const pmix_info_t directives[], size_t ndirs, pmix_op_cbfunc_t cbfunc, void *cbdata); @@ -504,6 +472,9 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata, * lieue of preemption. A corresponding ability to "reacquire" resources * previously released is included. */ +PMIX_EXPORT pmix_status_t PMIx_Allocation_request(pmix_alloc_directive_t directive, + pmix_info_t *info, size_t ninfo); + PMIX_EXPORT pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t directive, pmix_info_t *info, size_t ninfo, pmix_info_cbfunc_t cbfunc, void *cbdata); @@ -522,6 +493,9 @@ PMIX_EXPORT pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t dire * when the callback function completes - this will be used to release * any provided pmix_info_t array. */ +PMIX_EXPORT pmix_status_t PMIx_Job_control(const pmix_proc_t targets[], size_t ntargets, + const pmix_info_t directives[], size_t ndirs); + PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_t ntargets, const pmix_info_t directives[], size_t ndirs, pmix_info_cbfunc_t cbfunc, void *cbdata); @@ -553,6 +527,9 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_ * * Note: a process can send a heartbeat to the server using the PMIx_Heartbeat * macro provided below*/ +PMIX_EXPORT pmix_status_t PMIx_Process_monitor(const pmix_info_t *monitor, pmix_status_t error, + const pmix_info_t directives[], size_t ndirs); + PMIX_EXPORT pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pmix_status_t error, const pmix_info_t directives[], size_t ndirs, pmix_info_cbfunc_t cbfunc, void *cbdata); diff --git a/opal/mca/pmix/pmix3x/pmix/include/pmix_common.h.in b/opal/mca/pmix/pmix3x/pmix/include/pmix_common.h.in index 8f3d9515d60..93146a45b33 100644 --- a/opal/mca/pmix/pmix3x/pmix/include/pmix_common.h.in +++ b/opal/mca/pmix/pmix3x/pmix/include/pmix_common.h.in @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2016-2017 Research Organization for Information Science + * Copyright (c) 2016-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2016-2017 Mellanox Technologies, Inc. @@ -99,11 +99,23 @@ typedef uint32_t pmix_rank_t; * data for the given key from every rank that posted * that key */ #define PMIX_RANK_WILDCARD UINT32_MAX-1 - /* other special rank values will be used to define * groups of ranks for use in collectives */ #define PMIX_RANK_LOCAL_NODE UINT32_MAX-2 // all ranks on local node +/* define an invalid value */ +#define PMIX_RANK_INVALID UINT32_MAX-3 +/**** PMIX ENVIRONMENTAL PARAMETERS ****/ +/* There are a few environmental parameters used by PMIx for + * various operations. While there is no "definition" of them + * as values, we do record them here for informational purposes. + * + * PMIX_LAUNCHER_PAUSE_FOR_TOOL - if set to non-zero value, instructs + * launchers (e.g., "prun") to stop prior to spawning the application until + * a tool can connect with further instructions. This envar will be + * set by the tool and is _not_ intended for the direct use of users. + * + */ /* define a set of "standard" PMIx attributes that can * be queried. Implementations (and users) are free to extend as @@ -132,7 +144,9 @@ typedef uint32_t pmix_rank_t; #define PMIX_SERVER_ENABLE_MONITORING "pmix.srv.monitor" // (bool) Enable PMIx internal monitoring by server #define PMIX_SERVER_NSPACE "pmix.srv.nspace" // (char*) Name of the nspace to use for this server #define PMIX_SERVER_RANK "pmix.srv.rank" // (pmix_rank_t) Rank of this server - +#define PMIX_SERVER_GATEWAY "pmix.srv.gway" // (bool) Server is acting as a gateway for PMIx requests + // that cannot be serviced on backend nodes + // (e.g., logging to email) /* tool-related attributes */ #define PMIX_TOOL_NSPACE "pmix.tool.nspace" // (char*) Name of the nspace to use for this tool @@ -148,6 +162,8 @@ typedef uint32_t pmix_rank_t; #define PMIX_TOOL_DO_NOT_CONNECT "pmix.tool.nocon" // (bool) the tool wants to use internal PMIx support, but does // not want to connect to a PMIx server // from the specified processes to this tool +#define PMIX_RECONNECT_SERVER "pmix.cnct.recon" // (bool) tool is requesting to change server connections +#define PMIX_LAUNCHER "pmix.tool.launcher" // (bool) tool is a launcher and needs rendezvous files created /* identification attributes */ #define PMIX_USERID "pmix.euid" // (uint32_t) effective user id @@ -189,6 +205,7 @@ typedef uint32_t pmix_rank_t; #define PMIX_TCP_DISABLE_IPV4 "pmix.tcp.disipv4" // (bool) true to disable IPv4 family #define PMIX_TCP_DISABLE_IPV6 "pmix.tcp.disipv6" // (bool) true to disable IPv6 family + /* attributes for GDS */ #define PMIX_GDS_MODULE "pmix.gds.mod" // (char*) comma-delimited string of desired modules @@ -207,7 +224,7 @@ typedef uint32_t pmix_rank_t; /* information about relative ranks as assigned by the RM */ #define PMIX_CLUSTER_ID "pmix.clid" // (char*) a string name for the cluster this proc is executing on -#define PMIX_PROCID "pmix.procid" // (pmix_proc_t) process identifier +#define PMIX_PROCID "pmix.procid" // (pmix_proc_t*) process identifier #define PMIX_NSPACE "pmix.nspace" // (char*) nspace of a job #define PMIX_JOBID "pmix.jobid" // (char*) jobid assigned by scheduler #define PMIX_APPNUM "pmix.appnum" // (uint32_t) app number within the job @@ -234,6 +251,7 @@ typedef uint32_t pmix_rank_t; #define PMIX_LOCALITY "pmix.loc" // (uint16_t) relative locality of two procs #define PMIX_PARENT_ID "pmix.parent" // (pmix_proc_t*) identifier of the process that called PMIx_Spawn // to launch this proc's application +#define PMIX_EXIT_CODE "pmix.exit.code" // (int) exit code returned when proc terminated /* size info */ @@ -246,16 +264,20 @@ typedef uint32_t pmix_rank_t; #define PMIX_MAX_PROCS "pmix.max.size" // (uint32_t) max #procs for this job #define PMIX_NUM_NODES "pmix.num.nodes" // (uint32_t) #nodes in this nspace + /* Memory info */ #define PMIX_AVAIL_PHYS_MEMORY "pmix.pmem" // (uint64_t) total available physical memory on this node #define PMIX_DAEMON_MEMORY "pmix.dmn.mem" // (float) Mbytes of memory currently used by daemon #define PMIX_CLIENT_AVG_MEMORY "pmix.cl.mem.avg" // (float) Average Mbytes of memory used by client processes + /* topology info */ #define PMIX_NET_TOPO "pmix.ntopo" // (char*) xml-representation of network topology #define PMIX_LOCAL_TOPO "pmix.ltopo" // (char*) xml-representation of local node topology #define PMIX_NODE_LIST "pmix.nlist" // (char*) comma-delimited list of nodes running procs for this job #define PMIX_TOPOLOGY "pmix.topo" // (hwloc_topology_t) pointer to the PMIx client's internal topology object +#define PMIX_TOPOLOGY_XML "pmix.topo.xml" // (char*) XML-based description of topology +#define PMIX_TOPOLOGY_FILE "pmix.topo.file" // (char*) full path to file containing XML topology description #define PMIX_TOPOLOGY_SIGNATURE "pmix.toposig" // (char*) topology signature string #define PMIX_LOCALITY_STRING "pmix.locstr" // (char*) string describing a proc's location #define PMIX_HWLOC_SHMEM_ADDR "pmix.hwlocaddr" // (size_t) address of HWLOC shared memory segment @@ -263,6 +285,8 @@ typedef uint32_t pmix_rank_t; #define PMIX_HWLOC_SHMEM_FILE "pmix.hwlocfile" // (char*) path to HWLOC shared memory file #define PMIX_HWLOC_XML_V1 "pmix.hwlocxml1" // (char*) XML representation of local topology using HWLOC v1.x format #define PMIX_HWLOC_XML_V2 "pmix.hwlocxml2" // (char*) XML representation of local topology using HWLOC v2.x format +#define PMIX_HWLOC_SHARE_TOPO "pmix.hwlocsh" // (bool) Share the HWLOC topology via shared memory +#define PMIX_HWLOC_HOLE_KIND "pmix.hwlocholek" // (char*) Kind of VM "hole" HWLOC should use for shared memory /* request-related info */ @@ -291,7 +315,7 @@ typedef uint32_t pmix_rank_t; /* attributes used by host server to pass data to the server convenience library - the * data will then be parsed and provided to the local clients */ #define PMIX_REGISTER_NODATA "pmix.reg.nodata" // (bool) Registration is for nspace only, do not copy job data -#define PMIX_PROC_DATA "pmix.pdata" // (pmix_data_array_t) starts with rank, then contains more data +#define PMIX_PROC_DATA "pmix.pdata" // (pmix_data_array_t*) starts with rank, then contains more data #define PMIX_NODE_MAP "pmix.nmap" // (char*) regex of nodes containing procs for this job #define PMIX_PROC_MAP "pmix.pmap" // (char*) regex describing procs on each node within this job #define PMIX_ANL_MAP "pmix.anlmap" // (char*) process mapping in ANL notation (used in PMI-1/PMI-2) @@ -317,7 +341,7 @@ typedef uint32_t pmix_rank_t; #define PMIX_EVENT_HDLR_PREPEND "pmix.evprepend" // (bool) prepend this handler to the precedence list within its category #define PMIX_EVENT_HDLR_APPEND "pmix.evappend" // (bool) append this handler to the precedence list within its category #define PMIX_EVENT_CUSTOM_RANGE "pmix.evrange" // (pmix_data_array_t*) array of pmix_proc_t defining range of event notification -#define PMIX_EVENT_AFFECTED_PROC "pmix.evproc" // (pmix_proc_t) single proc that was affected +#define PMIX_EVENT_AFFECTED_PROC "pmix.evproc" // (pmix_proc_t*) single proc that was affected #define PMIX_EVENT_AFFECTED_PROCS "pmix.evaffected" // (pmix_data_array_t*) array of pmix_proc_t defining affected procs #define PMIX_EVENT_NON_DEFAULT "pmix.evnondef" // (bool) event is not to be delivered to default event handlers #define PMIX_EVENT_RETURN_OBJECT "pmix.evobject" // (void*) object to be returned whenever the registered cbfunc is invoked @@ -335,7 +359,8 @@ typedef uint32_t pmix_rank_t; #define PMIX_EVENT_ACTION_TIMEOUT "pmix.evtimeout" // (int) time in sec before RM will execute error response #define PMIX_EVENT_NO_TERMINATION "pmix.evnoterm" // (bool) indicates that the handler has satisfactorily handled // the event and believes termination of the application is not required -#define PMIX_EVENT_WANT_TERMINATION "pmix.evterm" // (bool) indicates that the handler has determined that the application should be terminated +#define PMIX_EVENT_WANT_TERMINATION "pmix.evterm" // (bool) indicates that the handler has determined that the + // application should be terminated /* attributes used to describe "spawn" directives */ @@ -355,8 +380,8 @@ typedef uint32_t pmix_rank_t; #define PMIX_PRELOAD_BIN "pmix.preloadbin" // (bool) preload binaries #define PMIX_PRELOAD_FILES "pmix.preloadfiles" // (char*) comma-delimited list of files to pre-position #define PMIX_NON_PMI "pmix.nonpmi" // (bool) spawned procs will not call PMIx_Init -#define PMIX_STDIN_TGT "pmix.stdin" // (pmix_proc_t) proc that is to receive stdin - // (PMIX_RANK_WILDCARD = all in given nspace) +#define PMIX_STDIN_TGT "pmix.stdin" // (pmix_proc_t*) proc that is to receive stdin + // (PMIX_RANK_WILDCARD = all in given nspace) #define PMIX_DEBUGGER_DAEMONS "pmix.debugger" // (bool) spawned app consists of debugger daemons #define PMIX_COSPAWN_APP "pmix.cospawn" // (bool) designated app is to be spawned as a disconnected // job - i.e., not part of the "comm_world" of the job @@ -376,7 +401,7 @@ typedef uint32_t pmix_rank_t; #define PMIX_JOB_CONTINUOUS "pmix.continuous" // (bool) application is continuous, all failed procs should // be immediately restarted #define PMIX_MAX_RESTARTS "pmix.maxrestarts" // (uint32_t) max number of times to restart a job -#define PMIX_FWD_STDIN "pmix.fwd.stdin" // (bool) forward the stdin from this process to the spawned processes +#define PMIX_FWD_STDIN "pmix.fwd.stdin" // (bool) forward the stdin from this process to the target processes #define PMIX_FWD_STDOUT "pmix.fwd.stdout" // (bool) forward stdout from the spawned processes to this process (typically used by a tool) #define PMIX_FWD_STDERR "pmix.fwd.stderr" // (bool) forward stderr from the spawned processes to this process (typically used by a tool) #define PMIX_FWD_STDDIAG "pmix.fwd.stddiag" // (bool) if a diagnostic channel exists, forward any output on it @@ -401,9 +426,9 @@ typedef uint32_t pmix_rank_t; #define PMIX_QUERY_QUEUE_LIST "pmix.qry.qlst" // (char*) request a comma-delimited list of scheduler queues #define PMIX_QUERY_QUEUE_STATUS "pmix.qry.qst" // (TBD) status of a specified scheduler queue #define PMIX_QUERY_PROC_TABLE "pmix.qry.ptable" // (char*) input nspace of job whose info is being requested - // returns (pmix_data_array_t) an array of pmix_proc_info_t + // returns (pmix_data_array_t*) an array of pmix_proc_info_t #define PMIX_QUERY_LOCAL_PROC_TABLE "pmix.qry.lptable" // (char*) input nspace of job whose info is being requested - // returns (pmix_data_array_t) an array of pmix_proc_info_t for + // returns (pmix_data_array_t*) an array of pmix_proc_info_t for // procs in job on same node #define PMIX_QUERY_AUTHORIZATIONS "pmix.qry.auths" // (bool) return operations tool is authorized to perform #define PMIX_QUERY_SPAWN_SUPPORT "pmix.qry.spawn" // (bool) return a comma-delimited list of supported spawn attributes @@ -418,21 +443,46 @@ typedef uint32_t pmix_rank_t; // for the specified nspace /* log attributes */ +#define PMIX_LOG_SOURCE "pmix.log.source" // (pmix_proc_t*) ID of source of the log request #define PMIX_LOG_STDERR "pmix.log.stderr" // (char*) log string to stderr #define PMIX_LOG_STDOUT "pmix.log.stdout" // (char*) log string to stdout -#define PMIX_LOG_SYSLOG "pmix.log.syslog" // (char*) log data to syslog - defaults to ERROR priority unless +#define PMIX_LOG_SYSLOG "pmix.log.syslog" // (char*) log message to syslog - defaults to ERROR priority. Will log + // to global syslog if available, otherwise to local syslog +#define PMIX_LOG_LOCAL_SYSLOG "pmix.log.lsys" // (char*) log msg to local syslog - defaults to ERROR priority +#define PMIX_LOG_GLOBAL_SYSLOG "pmix.log.gsys" // (char*) forward data to system "master" and log msg to that syslog +#define PMIX_LOG_SYSLOG_PRI "pmix.log.syspri" // (int) syslog priority level + +#define PMIX_LOG_TIMESTAMP "pmix.log.tstmp" // (time_t) timestamp for log report +#define PMIX_LOG_GENERATE_TIMESTAMP "pmix.log.gtstmp" // (bool) generate timestamp for log +#define PMIX_LOG_TAG_OUTPUT "pmix.log.tag" // (bool) label the output stream with the channel name (e.g., "stdout") +#define PMIX_LOG_TIMESTAMP_OUTPUT "pmix.log.tsout" // (bool) print timestamp in output string +#define PMIX_LOG_XML_OUTPUT "pmix.log.xml" // (bool) print the output stream in xml format +#define PMIX_LOG_ONCE "pmix.log.once" // (bool) only log this once with whichever channel can first support it #define PMIX_LOG_MSG "pmix.log.msg" // (pmix_byte_object_t) message blob to be sent somewhere -#define PMIX_LOG_EMAIL "pmix.log.email" // (pmix_data_array_t) log via email based on pmix_info_t containing directives + +#define PMIX_LOG_EMAIL "pmix.log.email" // (pmix_data_array_t*) log via email based on array of pmix_info_t + // containing directives #define PMIX_LOG_EMAIL_ADDR "pmix.log.emaddr" // (char*) comma-delimited list of email addresses that are to recv msg +#define PMIX_LOG_EMAIL_SENDER_ADDR "pmix.log.emfaddr" // (char*) return email address of sender #define PMIX_LOG_EMAIL_SUBJECT "pmix.log.emsub" // (char*) subject line for email #define PMIX_LOG_EMAIL_MSG "pmix.log.emmsg" // (char*) msg to be included in email +#define PMIX_LOG_EMAIL_SERVER "pmix.log.esrvr" // (char*) hostname (or IP addr) of estmp server +#define PMIX_LOG_EMAIL_SRVR_PORT "pmix.log.esrvrprt" // (int32_t) port the email server is listening to + +#define PMIX_LOG_GLOBAL_DATASTORE "pmix.log.gstore" // (bool) +#define PMIX_LOG_JOB_RECORD "pmix.log.jrec" // (bool) log the provided information to the RM's job record + /* debugger attributes */ #define PMIX_DEBUG_STOP_ON_EXEC "pmix.dbg.exec" // (bool) job is being spawned under debugger - instruct it to pause on start #define PMIX_DEBUG_STOP_IN_INIT "pmix.dbg.init" // (bool) instruct job to stop during PMIx init #define PMIX_DEBUG_WAIT_FOR_NOTIFY "pmix.dbg.notify" // (bool) block at desired point until receiving debugger release notification -#define PMIX_DEBUG_JOB "pmix.dbg.job" // (char*) nspace of the job to be debugged - the RM/PMIx server are +#define PMIX_DEBUG_JOB "pmix.dbg.job" // (char*) nspace of the job assigned to this debugger to be debugged. Note + // that id's, pids, and other info on the procs is available + // via a query for the nspace's local or global proctable #define PMIX_DEBUG_WAITING_FOR_NOTIFY "pmix.dbg.waiting" // (bool) job to be debugged is waiting for a release +#define PMIX_DEBUG_JOB_DIRECTIVES "pmix.dbg.jdirs" // (pmix_data_array_t*) array of job-level directives +#define PMIX_DEBUG_APP_DIRECTIVES "pmix.dbg.adirs" // (pmix_data_array_t*) array of app-level directives /* Resource Manager identification */ @@ -442,6 +492,7 @@ typedef uint32_t pmix_rank_t; /* environmental variable operation attributes */ #define PMIX_SET_ENVAR "pmix.envar.set" // (pmix_envar_t*) set the envar to the given value, // overwriting any pre-existing one +#define PMIX_ADD_ENVAR "pmix.envar.add" // (pmix_envar_t*) add envar, but do not overwrite any existing one #define PMIX_UNSET_ENVAR "pmix.envar.unset" // (char*) unset the envar, if present #define PMIX_PREPEND_ENVAR "pmix.envar.prepnd" // (pmix_envar_t*) prepend the given value to the // specified envar using the separator @@ -459,14 +510,41 @@ typedef uint32_t pmix_rank_t; #define PMIX_ALLOC_NUM_CPU_LIST "pmix.alloc.ncpulist" // (char*) regex of #cpus for each node #define PMIX_ALLOC_CPU_LIST "pmix.alloc.cpulist" // (char*) regex of specific cpus indicating the cpus involved. #define PMIX_ALLOC_MEM_SIZE "pmix.alloc.msize" // (float) number of Mbytes -#define PMIX_ALLOC_NETWORK "pmix.alloc.net" // (array) array of pmix_info_t describing network resources. If not - // given as part of an info struct that identifies the - // impacted nodes, then the description will be applied - // across all nodes in the requestor's allocation -#define PMIX_ALLOC_NETWORK_ID "pmix.alloc.netid" // (char*) name of network +#define PMIX_ALLOC_NETWORK "pmix.alloc.net" // (pmix_data_array_t*) Array of pmix_info_t describing + // network resource request. This must include at least: + // * PMIX_ALLOC_NETWORK_ID + // * PMIX_ALLOC_NETWORK_TYPE + // * PMIX_ALLOC_NETWORK_ENDPTS + // plus whatever other descriptors are desired +#define PMIX_ALLOC_NETWORK_ID "pmix.alloc.netid" // (char*) key to be used when accessing this requested network allocation. The + // allocation will be returned/stored as a pmix_data_array_t of + // pmix_info_t indexed by this key and containing at least one + // entry with the same key and the allocated resource description. + // The type of the included value depends upon the network + // support. For example, a TCP allocation might consist of a + // comma-delimited string of socket ranges such as + // "32000-32100,33005,38123-38146". Additional entries will consist + // of any provided resource request directives, along with their + // assigned values. Examples include: + // * PMIX_ALLOC_NETWORK_TYPE - the type of resources provided + // * PMIX_ALLOC_NETWORK_PLANE - if applicable, what plane the + // resources were assigned from + // * PMIX_ALLOC_NETWORK_QOS - the assigned QoS + // * PMIX_ALLOC_BANDWIDTH - the allocated bandwidth + // * PMIX_ALLOC_NETWORK_SEC_KEY - a security key for the requested + // network allocation + // NOTE: the assigned values may differ from those requested, + // especially if the "required" flag was not set in the request #define PMIX_ALLOC_BANDWIDTH "pmix.alloc.bw" // (float) Mbits/sec #define PMIX_ALLOC_NETWORK_QOS "pmix.alloc.netqos" // (char*) quality of service level -#define PMIX_ALLOC_TIME "pmix.alloc.time" // (uint32_t) time in seconds +#define PMIX_ALLOC_TIME "pmix.alloc.time" // (uint32_t) time in seconds that the allocation shall remain valid +#define PMIX_ALLOC_NETWORK_TYPE "pmix.alloc.nettype" // (char*) type of desired transport (e.g., tcp, udp) +#define PMIX_ALLOC_NETWORK_PLANE "pmix.alloc.netplane" // (char*) id string for the NIC (aka plane) to be used for this allocation + // (e.g., CIDR for Ethernet) +#define PMIX_ALLOC_NETWORK_ENDPTS "pmix.alloc.endpts" // (size_t) number of endpoints to allocate per process +#define PMIX_ALLOC_NETWORK_ENDPTS_NODE "pmix.alloc.endpts.nd" // (size_t) number of endpoints to allocate per node +#define PMIX_ALLOC_NETWORK_SEC_KEY "pmix.alloc.nsec" // (pmix_byte_object_t) network security key + /* job control attributes */ #define PMIX_JOB_CTRL_ID "pmix.jctrl.id" // (char*) provide a string identifier for this request @@ -520,32 +598,44 @@ typedef uint32_t pmix_rank_t; // generating the event /* security attributes */ -#define PMIX_CRED_TYPE "pmix.sec.ctype" // when passed in PMIx_Get_credential, a prioritized, +#define PMIX_CRED_TYPE "pmix.sec.ctype" // (char*) when passed in PMIx_Get_credential, a prioritized, // comma-delimited list of desired credential types for use // in environments where multiple authentication mechanisms // may be available. When returned in a callback function, a // string identifier of the credential type +#define PMIX_CRYPTO_KEY "pmix.sec.key" // (pmix_byte_object_t) blob containing crypto key + /* IO Forwarding Attributes */ -#define PMIX_IOF_CACHE_SIZE "pmix.iof.csize" // (uint32_t) requested size of the server cache in bytes for each specified channel. - // By default, the server is allowed (but not required) to drop - // all bytes received beyond the max size -#define PMIX_IOF_DROP_OLDEST "pmix.iof.old" // (bool) in an overflow situation, drop the oldest bytes to make room in the cache -#define PMIX_IOF_DROP_NEWEST "pmix.iof.new" // (bool) in an overflow situation, drop any new bytes received until room becomes - // available in the cache (default) -#define PMIX_IOF_BUFFERING_SIZE "pmix.iof.bsize" // (uint32_t) basically controls grouping of IO on the specified channel(s) to - // avoid being called every time a bit of IO arrives. The library - // will execute the callback whenever the specified number of bytes - // becomes available. Any remaining buffered data will be "flushed" - // upon call to deregister the respective channel -#define PMIX_IOF_BUFFERING_TIME "pmix.iof.btime" // (uint32_t) max time in seconds to buffer IO before delivering it. Used in conjunction - // with buffering size, this prevents IO from being held indefinitely - // while waiting for another payload to arrive -#define PMIX_IOF_COMPLETE "pmix.iof.cmp" // (bool) indicates whether or not the specified IO channel has been closed - // by the source -#define PMIX_IOF_PUSH_STDIN "pmix.iof.stdin" // (bool) Used by a tool to request that the PMIx library collect - // the tool's stdin and forward it to the procs specified in - // the PMIx_IOF_push call +#define PMIX_IOF_CACHE_SIZE "pmix.iof.csize" // (uint32_t) requested size of the server cache in bytes for each specified channel. + // By default, the server is allowed (but not required) to drop + // all bytes received beyond the max size +#define PMIX_IOF_DROP_OLDEST "pmix.iof.old" // (bool) in an overflow situation, drop the oldest bytes to make room in the cache +#define PMIX_IOF_DROP_NEWEST "pmix.iof.new" // (bool) in an overflow situation, drop any new bytes received until room becomes + // available in the cache (default) +#define PMIX_IOF_BUFFERING_SIZE "pmix.iof.bsize" // (uint32_t) basically controls grouping of IO on the specified channel(s) to + // avoid being called every time a bit of IO arrives. The library + // will execute the callback whenever the specified number of bytes + // becomes available. Any remaining buffered data will be "flushed" + // upon call to deregister the respective channel +#define PMIX_IOF_BUFFERING_TIME "pmix.iof.btime" // (uint32_t) max time in seconds to buffer IO before delivering it. Used in conjunction + // with buffering size, this prevents IO from being held indefinitely + // while waiting for another payload to arrive +#define PMIX_IOF_COMPLETE "pmix.iof.cmp" // (bool) indicates whether or not the specified IO channel has been closed + // by the source +#define PMIX_IOF_PUSH_STDIN "pmix.iof.stdin" // (bool) Used by a tool to request that the PMIx library collect + // the tool's stdin and forward it to the procs specified in + // the PMIx_IOF_push call +#define PMIX_IOF_TAG_OUTPUT "pmix.iof.tag" // (bool) Tag output with the channel it comes from +#define PMIX_IOF_TIMESTAMP_OUTPUT "pmix.iof.ts" // (bool) Timestamp output +#define PMIX_IOF_XML_OUTPUT "pmix.iof.xml" // (bool) Format output in XML + + +/* Attributes for controlling contents of application setup data */ +#define PMIX_SETUP_APP_ENVARS "pmix.setup.env" // (bool) harvest and include relevant envars +#define PMIX_SETUP_APP_NONENVARS "pmix.setup.nenv" // (bool) include all non-envar data +#define PMIX_SETUP_APP_ALL "pmix.setup.all" // (bool) include all relevant data + /**** PROCESS STATE DEFINITIONS ****/ typedef uint8_t pmix_proc_state_t; @@ -576,11 +666,13 @@ typedef uint8_t pmix_proc_state_t; #define PMIX_PROC_STATE_ABORTED_BY_SIG (PMIX_PROC_STATE_ERROR + 4) /* process aborted by signal */ #define PMIX_PROC_STATE_TERM_WO_SYNC (PMIX_PROC_STATE_ERROR + 5) /* process exit'd w/o calling PMIx_Finalize */ #define PMIX_PROC_STATE_COMM_FAILED (PMIX_PROC_STATE_ERROR + 6) /* process communication has failed */ -#define PMIX_PROC_STATE_CALLED_ABORT (PMIX_PROC_STATE_ERROR + 7) /* process called "PMIx_Abort" */ -#define PMIX_PROC_STATE_MIGRATING (PMIX_PROC_STATE_ERROR + 8) /* process failed and is waiting for resources before restarting */ -#define PMIX_PROC_STATE_CANNOT_RESTART (PMIX_PROC_STATE_ERROR + 9) /* process failed and cannot be restarted */ -#define PMIX_PROC_STATE_TERM_NON_ZERO (PMIX_PROC_STATE_ERROR + 10) /* process exited with a non-zero status, indicating abnormal */ -#define PMIX_PROC_STATE_FAILED_TO_LAUNCH (PMIX_PROC_STATE_ERROR + 11) /* unable to launch process */ +#define PMIX_PROC_STATE_SENSOR_BOUND_EXCEEDED (PMIX_PROC_STATE_ERROR + 7) /* process exceeded a sensor limit */ +#define PMIX_PROC_STATE_CALLED_ABORT (PMIX_PROC_STATE_ERROR + 8) /* process called "PMIx_Abort" */ +#define PMIX_PROC_STATE_HEARTBEAT_FAILED (PMIX_PROC_STATE_ERROR + 9) /* process failed to send heartbeat w/in time limit */ +#define PMIX_PROC_STATE_MIGRATING (PMIX_PROC_STATE_ERROR + 10) /* process failed and is waiting for resources before restarting */ +#define PMIX_PROC_STATE_CANNOT_RESTART (PMIX_PROC_STATE_ERROR + 11) /* process failed and cannot be restarted */ +#define PMIX_PROC_STATE_TERM_NON_ZERO (PMIX_PROC_STATE_ERROR + 12) /* process exited with a non-zero status, indicating abnormal */ +#define PMIX_PROC_STATE_FAILED_TO_LAUNCH (PMIX_PROC_STATE_ERROR + 13) /* unable to launch process */ /**** PMIX ERROR CONSTANTS ****/ @@ -686,6 +778,10 @@ typedef int pmix_status_t; #define PMIX_MODEL_RESOURCES (PMIX_ERR_OP_BASE - 21) // model resource usage has changed #define PMIX_OPENMP_PARALLEL_ENTERED (PMIX_ERR_OP_BASE - 22) // an OpenMP parallel region has been entered #define PMIX_OPENMP_PARALLEL_EXITED (PMIX_ERR_OP_BASE - 23) // an OpenMP parallel region has completed +#define PMIX_LAUNCH_DIRECTIVE (PMIX_ERR_OP_BASE - 24) +#define PMIX_LAUNCHER_READY (PMIX_ERR_OP_BASE - 25) +#define PMIX_OPERATION_IN_PROGRESS (PMIX_ERR_OP_BASE - 26) + /* define a starting point for system error constants so * we avoid renumbering when making additions */ @@ -806,6 +902,7 @@ typedef uint8_t pmix_data_range_t; #define PMIX_RANGE_GLOBAL 5 // data available to all procs #define PMIX_RANGE_CUSTOM 6 // range is specified in a pmix_info_t #define PMIX_RANGE_PROC_LOCAL 7 // restrict range to the local proc +#define PMIX_RANGE_INVALID UINT8_MAX /* define a "persistence" policy for data published by clients */ typedef uint8_t pmix_persistence_t; @@ -814,12 +911,16 @@ typedef uint8_t pmix_persistence_t; #define PMIX_PERSIST_PROC 2 // retain until publishing process terminates #define PMIX_PERSIST_APP 3 // retain until application terminates #define PMIX_PERSIST_SESSION 4 // retain until session/allocation terminates +#define PMIX_PERSIST_INVALID UINT8_MAX /* define a set of bit-mask flags for specifying behavior of * command directives via pmix_info_t arrays */ typedef uint32_t pmix_info_directives_t; -#define PMIX_INFO_REQD 0x0001 - +#define PMIX_INFO_REQD 0x00000001 +/* the top 16-bits are reserved for internal use by + * implementers - these may be changed inside the + * PMIx library */ +#define PMIX_INFO_DIR_RESERVED 0xffff0000 /* define a set of directives for allocation requests */ typedef uint8_t pmix_alloc_directive_t; @@ -935,9 +1036,11 @@ typedef struct { do { \ if (NULL != (m)->envar) { \ free((m)->envar); \ + (m)->envar = NULL; \ } \ if (NULL != (m)->value) { \ free((m)->value); \ + (m)->value = NULL; \ } \ } while(0) #define PMIX_ENVAR_LOAD(m, e, v, s) \ @@ -986,9 +1089,28 @@ typedef struct pmix_data_buffer { do { \ if (NULL != (m)->base_ptr) { \ free((m)->base_ptr); \ + (m)->base_ptr = NULL; \ } \ + (m)->pack_ptr = NULL; \ + (m)->unpack_ptr = NULL; \ + (m)->bytes_allocated = 0; \ + (m)->bytes_used = 0; \ } while (0) +#define PMIX_DATA_BUFFER_LOAD(b, d, s) \ + do { \ + (b)->base_ptr = (char*)(d); \ + (b)->pack_ptr = (b)->base_ptr + (s); \ + (b)->unpack_ptr = (b)->base_ptr; \ + (b)->bytes_allocated = (s); \ + (b)->bytes_used = (s); \ + } while(0) +#define PMIX_DATA_BUFFER_UNLOAD(b, d, s) \ + do { \ + (d) = (b)->base_ptr; \ + (s) = (b)->bytes_used; \ + (b)->base_ptr = NULL; \ + } while(0) /**** PMIX PROC OBJECT ****/ typedef struct pmix_proc { @@ -1017,6 +1139,7 @@ typedef struct pmix_proc { do { \ if (NULL != (m)) { \ free((m)); \ + (m) = NULL; \ } \ } while (0) @@ -1082,9 +1205,11 @@ typedef struct pmix_proc_info { do { \ if (NULL != (m)->hostname) { \ free((m)->hostname); \ + (m)->hostname = NULL; \ } \ if (NULL != (m)->executable_name) { \ free((m)->executable_name); \ + (m)->executable_name = NULL; \ } \ } while(0) @@ -1108,6 +1233,16 @@ typedef struct pmix_data_array { size_t size; void *array; } pmix_data_array_t; +#define PMIX_DATA_ARRAY_CONSTRUCT(m, n, t) \ + do { \ + (m)->type = (t); \ + (m)->size = (n); \ + } while(0) +#define PMIX_DATA_ARRAY_CREATE(m, n, t) \ + do { \ + (m) = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); \ + PMIX_DATA_ARRAY_CONSTRUCT((m), (n), (t)); \ + } while(0) typedef struct pmix_info_array { size_t size; @@ -1179,6 +1314,7 @@ typedef struct pmix_value { do { \ PMIX_VALUE_DESTRUCT((m)); \ free((m)); \ + (m) = NULL; \ } while (0) /* initialize a single value struct */ @@ -1189,86 +1325,7 @@ typedef struct pmix_value { } while (0) /* release the memory in the value struct data field */ -#define PMIX_VALUE_DESTRUCT(m) \ - do { \ - size_t _n; \ - if (PMIX_STRING == (m)->type) { \ - if (NULL != (m)->data.string) { \ - free((m)->data.string); \ - } \ - } else if ((PMIX_BYTE_OBJECT == (m)->type) || \ - (PMIX_COMPRESSED_STRING == (m)->type)) { \ - if (NULL != (m)->data.bo.bytes) { \ - free((m)->data.bo.bytes); \ - } \ - } else if (PMIX_DATA_ARRAY == (m)->type) { \ - if (NULL != (m)->data.darray && NULL != (m)->data.darray->array) { \ - if (PMIX_STRING == (m)->data.darray->type) { \ - char **_str = (char**)(m)->data.darray->array; \ - for (_n=0; _n < (m)->data.darray->size; _n++) { \ - if (NULL != _str[_n]) { \ - free(_str[_n]); \ - } \ - } \ - } else if (PMIX_PROC_INFO == (m)->data.darray->type) { \ - pmix_proc_info_t *_info = \ - (pmix_proc_info_t*)(m)->data.darray->array; \ - for (_n=0; _n < (m)->data.darray->size; _n++) { \ - PMIX_PROC_INFO_DESTRUCT(&_info[_n]); \ - } \ - } else if (PMIX_INFO == (m)->data.darray->type) { \ - pmix_info_t *_info = \ - (pmix_info_t*)(m)->data.darray->array; \ - for (_n=0; _n < (m)->data.darray->size; _n++) { \ - /* cannot use info destruct as that loops back */ \ - if (PMIX_STRING == _info[_n].value.type) { \ - if (NULL != _info[_n].value.data.string) { \ - free(_info[_n].value.data.string); \ - } \ - } else if (PMIX_BYTE_OBJECT == _info[_n].value.type) { \ - if (NULL != _info[_n].value.data.bo.bytes) { \ - free(_info[_n].value.data.bo.bytes); \ - } \ - } else if (PMIX_PROC_INFO == _info[_n].value.type) { \ - PMIX_PROC_INFO_DESTRUCT(_info[_n].value.data.pinfo); \ - } \ - } \ - } else if (PMIX_BYTE_OBJECT == (m)->data.darray->type) { \ - pmix_byte_object_t *_obj = \ - (pmix_byte_object_t*)(m)->data.darray->array; \ - for (_n=0; _n < (m)->data.darray->size; _n++) { \ - if (NULL != _obj[_n].bytes) { \ - free(_obj[_n].bytes); \ - } \ - } \ - } \ - free((m)->data.darray->array); \ - } \ - if (NULL != (m)->data.darray) { \ - free((m)->data.darray); \ - } \ - /**** DEPRECATED ****/ \ - } else if (PMIX_INFO_ARRAY == (m)->type) { \ - pmix_info_t *_p = (pmix_info_t*)((m)->data.array->array); \ - for (_n=0; _n < (m)->data.array->size; _n++) { \ - if (PMIX_STRING == _p[_n].value.type) { \ - if (NULL != _p[_n].value.data.string) { \ - free(_p[_n].value.data.string); \ - } \ - } else if (PMIX_BYTE_OBJECT == _p[_n].value.type) { \ - if (NULL != _p[_n].value.data.bo.bytes) { \ - free(_p[_n].value.data.bo.bytes); \ - } \ - } else if (PMIX_PROC_INFO == _p[_n].value.type) { \ - PMIX_PROC_INFO_DESTRUCT(_p[_n].value.data.pinfo); \ - } \ - } \ - free(_p); \ - /********************/ \ - } else if (PMIX_ENVAR == (m)->type) { \ - PMIX_ENVAR_DESTRUCT(&(m)->data.envar); \ - } \ - } while (0) +#define PMIX_VALUE_DESTRUCT(m) pmix_value_destruct(m) #define PMIX_VALUE_FREE(m, n) \ do { \ @@ -1278,9 +1335,44 @@ typedef struct pmix_value { PMIX_VALUE_DESTRUCT(&((m)[_s])); \ } \ free((m)); \ + (m) = NULL; \ } \ } while (0) +#define PMIX_VALUE_GET_NUMBER(s, m, n, t) \ + do { \ + (s) = PMIX_SUCCESS; \ + if (PMIX_SIZE == (m)->type) { \ + (n) = (t)((m)->data.size); \ + } else if (PMIX_INT == (m)->type) { \ + (n) = (t)((m)->data.integer); \ + } else if (PMIX_INT8 == (m)->type) { \ + (n) = (t)((m)->data.int8); \ + } else if (PMIX_INT16 == (m)->type) { \ + (n) = (t)((m)->data.int16); \ + } else if (PMIX_INT32 == (m)->type) { \ + (n) = (t)((m)->data.int32); \ + } else if (PMIX_INT64 == (m)->type) { \ + (n) = (t)((m)->data.int64); \ + } else if (PMIX_UINT == (m)->type) { \ + (n) = (t)((m)->data.uint); \ + } else if (PMIX_UINT8 == (m)->type) { \ + (n) = (t)((m)->data.uint8); \ + } else if (PMIX_UINT16 == (m)->type) { \ + (n) = (t)((m)->data.uint16); \ + } else if (PMIX_UINT32 == (m)->type) { \ + (n) = (t)((m)->data.uint32); \ + } else if (PMIX_UINT64 == (m)->type) { \ + (n) = (t)((m)->data.uint64); \ + } else if (PMIX_FLOAT == (m)->type) { \ + (n) = (t)((m)->data.fval); \ + } else if (PMIX_DOUBLE == (m)->type) { \ + (n) = (t)((m)->data.dval); \ + } else { \ + (s) = PMIX_ERR_BAD_PARAM; \ + } \ + } while(0) + /* expose some functions that are resolved in the * PMIx library, but part of a header that * includes internal functions - we don't @@ -1291,6 +1383,10 @@ void pmix_value_load(pmix_value_t *v, const void *data, pmix_data_type_t type); #define PMIX_VALUE_LOAD(v, d, t) \ pmix_value_load((v), (d), (t)) +pmix_status_t pmix_value_unload(pmix_value_t *kv, void **data, size_t *sz); +#define PMIX_VALUE_UNLOAD(r, k, d, s) \ + (r) = pmix_value_unload((k), (d), (s)) + pmix_status_t pmix_value_xfer(pmix_value_t *kv, pmix_value_t *src); #define PMIX_VALUE_XFER(r, v, s) \ do { \ @@ -1347,25 +1443,36 @@ struct pmix_info_t { PMIX_INFO_DESTRUCT(&((m)[_s])); \ } \ free((m)); \ + (m) = NULL; \ } \ } while (0) -#define PMIX_INFO_LOAD(m, k, v, t) \ - do { \ - (void)strncpy((m)->key, (k), PMIX_MAX_KEYLEN); \ - pmix_value_load(&((m)->value), (v), (t)); \ - } while (0) -#define PMIX_INFO_XFER(d, s) \ +#define PMIX_INFO_LOAD(m, k, v, t) \ do { \ - (void)strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \ - (d)->flags = (s)->flags; \ - pmix_value_xfer(&(d)->value, &(s)->value); \ + if (NULL != (k)) { \ + (void)strncpy((m)->key, (k), PMIX_MAX_KEYLEN); \ + } \ + (m)->flags = 0; \ + pmix_value_load(&((m)->value), (v), (t)); \ + } while (0) +#define PMIX_INFO_XFER(d, s) \ + do { \ + if (NULL != (s)->key) { \ + (void)strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \ + } \ + (d)->flags = (s)->flags; \ + pmix_value_xfer(&(d)->value, &(s)->value); \ } while(0) #define PMIX_INFO_REQUIRED(m) \ - (m)->flags |= PMIX_INFO_REQD; + (m)->flags |= PMIX_INFO_REQD #define PMIX_INFO_OPTIONAL(m) \ - (m)->flags &= ~PMIX_INFO_REQD; + (m)->flags &= ~PMIX_INFO_REQD + +#define PMIX_INFO_IS_REQUIRED(m) \ + (m)->flags & PMIX_INFO_REQD +#define PMIX_INFO_IS_OPTIONAL(m) \ + !((m)->flags & PMIX_INFO_REQD) #define PMIX_INFO_UNLOAD(r, v, l) \ do { \ @@ -1380,7 +1487,9 @@ struct pmix_info_t { (r) = PMIX_ERR_NOMEM; \ break; \ } \ - _kv->key = strdup(_info[_n].key); \ + if (NULL != _info[_n].key) { \ + _kv->key = strdup(_info[_n].key); \ + } \ PMIX_VALUE_XFER((r), _kv->value, &_info[_n].value);\ if (PMIX_SUCCESS != (r)) { \ PMIX_RELEASE(_kv); \ @@ -1415,6 +1524,7 @@ typedef struct pmix_pdata { do { \ PMIX_VALUE_DESTRUCT(&(m)->value); \ free((m)); \ + (m) = NULL; \ } while (0) #define PMIX_PDATA_CONSTRUCT(m) \ @@ -1436,6 +1546,7 @@ typedef struct pmix_pdata { PMIX_PDATA_DESTRUCT(&((m)[_s])); \ } \ free((m)); \ + (m) = NULL; \ } \ } while (0) @@ -1482,6 +1593,7 @@ typedef struct pmix_app { do { \ PMIX_APP_DESTRUCT((m)); \ free((m)); \ + (m) = NULL; \ } while (0) #define PMIX_APP_CONSTRUCT(m) \ @@ -1494,27 +1606,32 @@ typedef struct pmix_app { size_t _ii; \ if (NULL != (m)->cmd) { \ free((m)->cmd); \ + (m)->cmd = NULL; \ } \ if (NULL != (m)->argv) { \ for (_ii=0; NULL != (m)->argv[_ii]; _ii++) { \ free((m)->argv[_ii]); \ } \ free((m)->argv); \ + (m)->argv = NULL; \ } \ if (NULL != (m)->env) { \ for (_ii=0; NULL != (m)->env[_ii]; _ii++) { \ free((m)->env[_ii]); \ } \ free((m)->env); \ + (m)->env = NULL; \ } \ if (NULL != (m)->cwd) { \ free((m)->cwd); \ + (m)->cwd = NULL; \ } \ if (NULL != (m)->info) { \ for (_ii=0; _ii < (m)->ninfo; _ii++) { \ PMIX_INFO_DESTRUCT(&(m)->info[_ii]); \ } \ free((m)->info); \ + (m)->info = NULL; \ } \ } while (0) @@ -1526,6 +1643,7 @@ typedef struct pmix_app { PMIX_APP_DESTRUCT(&((m)[_s])); \ } \ free((m)); \ + (m) = NULL; \ } \ } while (0) @@ -1546,6 +1664,7 @@ typedef struct pmix_query { do { \ PMIX_QUERY_DESTRUCT((m)); \ free((m)); \ + (m) = NULL; \ } while (0) #define PMIX_QUERY_CONSTRUCT(m) \ @@ -1561,12 +1680,14 @@ typedef struct pmix_query { free((m)->keys[_ii]); \ } \ free((m)->keys); \ + (m)->keys = NULL; \ } \ if (NULL != (m)->qualifiers) { \ for (_ii=0; _ii < (m)->nqual; _ii++) { \ PMIX_INFO_DESTRUCT(&(m)->qualifiers[_ii]); \ } \ free((m)->qualifiers); \ + (m)->qualifiers = NULL; \ } \ } while (0) @@ -1578,6 +1699,7 @@ typedef struct pmix_query { PMIX_QUERY_DESTRUCT(&((m)[_s])); \ } \ free((m)); \ + (m) = NULL; \ } \ } while (0) @@ -1601,6 +1723,7 @@ typedef struct pmix_modex_data { do { \ PMIX_MODEX_DESTRUCT((m)); \ free((m)); \ + (m) = NULL; \ } while (0) #define PMIX_MODEX_CONSTRUCT(m) \ @@ -1612,6 +1735,7 @@ typedef struct pmix_modex_data { do { \ if (NULL != (m)->blob) { \ free((m)->blob); \ + (m)->blob = NULL; \ } \ } while (0) @@ -1623,6 +1747,7 @@ typedef struct pmix_modex_data { PMIX_MODEX_DESTRUCT(&((m)[_s])); \ } \ free((m)); \ + (m) = NULL; \ } \ } while (0) @@ -1655,18 +1780,6 @@ typedef void (*pmix_modex_cbfunc_t)(pmix_status_t status, typedef void (*pmix_spawn_cbfunc_t)(pmix_status_t status, char nspace[], void *cbdata); -/* define a callback function for calls to PMIx_Connect_nb - the function - * will be called upon completion of the command. The status will indicate - * whether or not the connect operation succeeded. The nspace will contain - * the new identity assigned by the host RM to the specified group of - * processes, and the rank will be the rank of this process within that new - * group. Note that the returned nspace value may be - * released by the library upon return from the callback function, so - * the receiver must copy it if it needs to be retained */ -typedef void (*pmix_connect_cbfunc_t)(pmix_status_t status, - char nspace[], int rank, - void *cbdata); - /* define a callback for common operations that simply return * a status. Examples include the non-blocking versions of * Fence, Connect, and Disconnect */ @@ -1908,13 +2021,8 @@ PMIX_EXPORT void PMIx_Deregister_event_handler(size_t evhdlr_ref, pmix_op_cbfunc_t cbfunc, void *cbdata); -/* Report an event to a process for notification via any - * registered evhdlr. The evhdlr registration can be - * called by both the server and the client application. On the - * server side, the evhdlr is used to report events detected - * by PMIx to the host server for handling. On the client side, - * the evhdlr is used to notify the process of events - * reported by the server - e.g., the failure of another process. +/* Report an event for notification via any + * registered evhdlr. * * This function allows the host server to direct the server * convenience library to notify all registered local procs of @@ -1922,22 +2030,37 @@ PMIX_EXPORT void PMIx_Deregister_event_handler(size_t evhdlr_ref, * The status indicates the event being reported. * * The client application can also call this function to notify the - * resource manager of an event it encountered. It can request the host - * server to notify the indicated processes about the event. + * resource manager and/or other processes of an event it encountered. + * It can also be used to asynchronously notify other parts of its + * own internal process - e.g., for one library to notify another + * when initialized inside the process. * - * The array of procs identifies the processes that will be impacted - * by the event. This could consist of a single process, or a number - * of processes. + * status - status code indicating the event being reported * - * The info array contains any further info the RM can and/or chooses - * to provide. + * source - the process that generated the event * - * The callback function will be called upon completion of the - * notify_event function's actions. Note that any messages will - * have been queued, but may not have been transmitted by this - * time. Note that the caller is required to maintain the input - * data until the callback function has been executed! -*/ + * range - the range in which the event is to be reported. For example, + * a value of PMIX_RANGE_LOCAL would instruct the system + * to only notify procs on the same local node as the + * event generator. + * + * info - an array of pmix_info_t structures provided by the event + * generator to pass any additional information about the + * event. This can include an array of pmix_proc_t structs + * describing the processes impacted by the event, the nature + * of the event and its severity, etc. The precise contents + * of the array will depend on the event generator. + * + * ninfo - number of elements in the info array + * + * cbfunc - callback function to be called upon completion of the + * notify_event function's actions. Note that any messages + * will have been queued, but may not have been transmitted + * by this time. Note that the caller is required to maintain + * the input data until the callback function has been executed! + * + * cbdata - the caller's provided void* object + */ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, const pmix_proc_t *source, pmix_data_range_t range, @@ -2268,6 +2391,81 @@ PMIX_EXPORT pmix_status_t PMIx_Data_copy_payload(pmix_data_buffer_t *dest, #define PMIX_VAL_FREE(_v) \ PMIx_free_value_data(_v) +static inline void pmix_value_destruct(pmix_value_t * m) { + size_t _n; + if (PMIX_STRING == (m)->type) { + if (NULL != (m)->data.string) { + free((m)->data.string); + (m)->data.string = NULL; + } + } else if ((PMIX_BYTE_OBJECT == (m)->type) || + (PMIX_COMPRESSED_STRING == (m)->type)) { + if (NULL != (m)->data.bo.bytes) { + free((m)->data.bo.bytes); + (m)->data.bo.bytes = NULL; + (m)->data.bo.size = 0; + } + } else if (PMIX_DATA_ARRAY == (m)->type) { + if (NULL != (m)->data.darray && NULL != (m)->data.darray->array) { + if (PMIX_STRING == (m)->data.darray->type) { + char **_str = (char**)(m)->data.darray->array; + for (_n=0; _n < (m)->data.darray->size; _n++) { + if (NULL != _str[_n]) { + free(_str[_n]); + } + } + } else if (PMIX_PROC_INFO == (m)->data.darray->type) { + pmix_proc_info_t *_info = + (pmix_proc_info_t*)(m)->data.darray->array; + for (_n=0; _n < (m)->data.darray->size; _n++) { + PMIX_PROC_INFO_DESTRUCT(&_info[_n]); + } + } else if (PMIX_INFO == (m)->data.darray->type) { + pmix_info_t *_info = + (pmix_info_t*)(m)->data.darray->array; + for (_n=0; _n < (m)->data.darray->size; _n++) { + pmix_value_destruct(&_info[_n].value); + } + } else if (PMIX_BYTE_OBJECT == (m)->data.darray->type) { + pmix_byte_object_t *_obj = + (pmix_byte_object_t*)(m)->data.darray->array; + for (_n=0; _n < (m)->data.darray->size; _n++) { + if (NULL != _obj[_n].bytes) { + free(_obj[_n].bytes); + } + } + } + free((m)->data.darray->array); + (m)->data.darray->array = NULL; + (m)->data.darray->size = 0; + } + if (NULL != (m)->data.darray) { + free((m)->data.darray); + (m)->data.darray = NULL; + } + /**** DEPRECATED ****/ + } else if (PMIX_INFO_ARRAY == (m)->type) { + pmix_info_t *_p = (pmix_info_t*)((m)->data.array->array); + for (_n=0; _n < (m)->data.array->size; _n++) { + if (PMIX_STRING == _p[_n].value.type) { + if (NULL != _p[_n].value.data.string) { + free(_p[_n].value.data.string); + } + } else if (PMIX_BYTE_OBJECT == _p[_n].value.type) { + if (NULL != _p[_n].value.data.bo.bytes) { + free(_p[_n].value.data.bo.bytes); + } + } else if (PMIX_PROC_INFO == _p[_n].value.type) { + PMIX_PROC_INFO_DESTRUCT(_p[_n].value.data.pinfo); + } + } + free(_p); + /********************/ + } else if (PMIX_ENVAR == (m)->type) { + PMIX_ENVAR_DESTRUCT(&(m)->data.envar); + } +} + #if defined(c_plusplus) || defined(__cplusplus) } #endif diff --git a/opal/mca/pmix/pmix3x/pmix/include/pmix_server.h b/opal/mca/pmix/pmix3x/pmix/include/pmix_server.h index 5cebee6aae0..ecc08307fec 100644 --- a/opal/mca/pmix/pmix3x/pmix/include/pmix_server.h +++ b/opal/mca/pmix/pmix3x/pmix/include/pmix_server.h @@ -198,48 +198,38 @@ typedef pmix_status_t (*pmix_server_spawn_fn_t)(const pmix_proc_t *proc, const pmix_app_t apps[], size_t napps, pmix_spawn_cbfunc_t cbfunc, void *cbdata); -/* Record the specified processes as "connected". This means that: - * - * (a) the resource manager should treat the specified group as - * a group when reporting events. - * - * (b) processes can address the group by the newly assigned nspace - * when passing requests - * - * As in the case of the fence operation, the info array can be used to pass - * user-level directives regarding the algorithm to be used for the collective - * operation involved in the "connect", timeout constraints, and other options - * available from the host RM. - * - * The callback function will be called once the operation is complete. Any - * participant that fails to call "connect" prior to terminating will cause the - * operation to return a "failed" status to all other participants. This - * is the default behavior in the absence of any provided directive. - * - * Some additional info keys are provided for this operation: - * - * (a) PMIX_CONNECT_NOTIFY_EACH: generate a local event notification using - * the PMIX_PROC_HAS_CONNECTED event each time a process connects - * - * (b) PMIX_CONNECT_NOTIFY_REQ: notify each of the indicated procs that - * they are requested to connect using the PMIX_CONNECT_REQUESTED event - * - * (c) PMIX_CONNECT_OPTIONAL: participation is optional - do not return - * error if procs terminate without having connected - */ +/* Record the specified processes as "connected". This means that the resource + * manager should treat the failure of any process in the specified group as + * a reportable event, and take appropriate action. The callback function is + * to be called once all participating processes have called connect. Note that + * a process can only engage in *one* connect operation involving the identical + * set of procs at a time. However, a process *can* be simultaneously engaged + * in multiple connect operations, each involving a different set of procs + * + * Note also that this is a collective operation within the client library, and + * thus the client will be blocked until all procs participate. Thus, the info + * array can be used to pass user directives, including a timeout. + * The directives are optional _unless_ the _mandatory_ flag + * has been set - in such cases, the host RM is required to return an error + * if the directive cannot be met. */ typedef pmix_status_t (*pmix_server_connect_fn_t)(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, - pmix_connect_cbfunc_t cbfunc, void *cbdata); + pmix_op_cbfunc_t cbfunc, void *cbdata); -/* Disconnect this process from a specified nspace. An error will be returned - * if the specified nspace is not recognized. The info array is used as above. - * - * Processes that terminate while connected to other processes will generate a - * "termination error" event that will be reported to any process in the connected - * group that has registered for such events. Calls to "disconnect" that include the - * PMIX_CONNECT_NOTIFY_EACH info key will cause other processes in the nspace to receive - * an event notification of the disconnect, if they are registered for such events. */ -typedef pmix_status_t (*pmix_server_disconnect_fn_t)(const char nspace[], +/* Disconnect a previously connected set of processes. An error should be returned + * if the specified set of procs was not previously "connected". As above, a process + * may be involved in multiple simultaneous disconnect operations. However, a process + * is not allowed to reconnect to a set of ranges that has not fully completed + * disconnect - i.e., you have to fully disconnect before you can reconnect to the + * same group of processes. + * + * Note also that this is a collective operation within the client library, and + * thus the client will be blocked until all procs participate. Thus, the info + * array can be used to pass user directives, including a timeout. + * The directives are optional _unless_ the _mandatory_ flag + * has been set - in such cases, the host RM is required to return an error + * if the directive cannot be met. */ +typedef pmix_status_t (*pmix_server_disconnect_fn_t)(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); @@ -326,7 +316,24 @@ typedef void (*pmix_server_tool_connection_fn_t)(pmix_info_t *info, size_t ninfo pmix_tool_connection_cbfunc_t cbfunc, void *cbdata); -/* Log data on behalf of a client */ +/* Log data on behalf of a client. Calls to the host thru this + * function must _NOT_ call the PMIx_Log API as this will + * trigger an infinite loop. Instead, the implementation must + * perform one of three operations: + * + * (a) transfer the data+directives to a "gateway" server + * where they can be logged. Gateways are designated + * servers on nodes (typically service nodes) where + * centralized logging is supported. The data+directives + * may be passed to the PMIx_Log API once arriving at + * that destination. + * + * (b) transfer the data to a logging channel outside of + * PMIx, but directly supported by the host + * + * (c) return an error to the caller indicating that the + * requested action is not supported + */ typedef void (*pmix_server_log_fn_t)(const pmix_proc_t *client, const pmix_info_t data[], size_t ndata, const pmix_info_t directives[], size_t ndirs, @@ -662,12 +669,13 @@ typedef void (*pmix_setup_application_cbfunc_t)(pmix_status_t status, pmix_op_cbfunc_t cbfunc, void *cbdata); /* Provide a function by which the resource manager can request - * any application-specific environmental variables prior to - * launch of an application. For example, network libraries may - * opt to provide security credentials for the application. This - * is defined as a non-blocking operation in case network - * libraries need to perform some action before responding. The - * returned env will be distributed along with the application */ + * any application-specific environmental variables, resource + * assignments, and/or other data prior to launch of an application. + * For example, network libraries may opt to provide security + * credentials for the application. This is defined as a non-blocking + * operation in case network libraries need to perform some action + * before responding. Any returned env will be distributed along + * with the application */ PMIX_EXPORT pmix_status_t PMIx_server_setup_application(const char nspace[], pmix_info_t info[], size_t ninfo, pmix_setup_application_cbfunc_t cbfunc, void *cbdata); @@ -676,6 +684,13 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_application(const char nspace[], * any application-specific operations prior to spawning local * clients of a given application. For example, a network library * might need to setup the local driver for "instant on" addressing. + * Data provided in the info array will be stored in the job-info + * region for the nspace. Operations included in the info array + * will be cached until the server calls PMIx_server_setup_fork, + * thereby indicating that local clients of this nspace will exist. + * Operations indicated by the provided data will only be executed + * for the first local client - i.e., they will only be executed + * once for a given nspace */ PMIX_EXPORT pmix_status_t PMIx_server_setup_local_support(const char nspace[], pmix_info_t info[], size_t ninfo, @@ -713,6 +728,26 @@ PMIX_EXPORT pmix_status_t PMIx_server_IOF_deliver(const pmix_proc_t *source, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); +/* Collect inventory of local resources. This is a non-blocking + * API as it may involve somewhat lengthy operations to obtain + * the requested information. Servers designated as "gateways" + * and whose plugins support collection of infrastructure info + * (e.g., switch and fabric topology, connectivity maps) shall + * return that information - plugins on non-gateway servers + * shall only return the node-local inventory. */ +PMIX_EXPORT pmix_status_t PMIx_server_collect_inventory(pmix_info_t directives[], size_t ndirs, + pmix_info_cbfunc_t cbfunc, void *cbdata); + +/* Deliver collected inventory for archiving by the corresponding + * plugins. Typically executed on a "gateway" associated with the + * system scheduler to enable use of inventory information by the + * the scheduling algorithm. May also be used on compute nodes to + * store a broader picture of the system for access by applications, + * if desired */ +PMIX_EXPORT pmix_status_t PMIx_server_deliver_inventory(pmix_info_t info[], size_t ninfo, + pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + #if defined(c_plusplus) || defined(__cplusplus) } #endif diff --git a/opal/mca/pmix/pmix3x/pmix/include/pmix_tool.h b/opal/mca/pmix/pmix3x/pmix/include/pmix_tool.h index ef05389f76d..7ea0062a01d 100644 --- a/opal/mca/pmix/pmix3x/pmix/include/pmix_tool.h +++ b/opal/mca/pmix/pmix3x/pmix/include/pmix_tool.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2015 Research Organization for Information Science @@ -98,6 +98,36 @@ PMIX_EXPORT pmix_status_t PMIx_tool_init(pmix_proc_t *proc, * operation. */ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void); +/* Switch server connection. Closes the connection, if existing, to a server + * and establishes a connection to the specified server. The target server can + * be given as: + * + * - PMIX_CONNECT_TO_SYSTEM: connect solely to the system server + * + * - PMIX_CONNECT_SYSTEM_FIRST: a request to use the system server first, + * if existing, and then look for the server specified in a different + * attribute + * + * - PMIX_SERVER_URI: connect to the server at the given URI + * + * - PMIX_SERVER_NSPACE: connect to the server of a given nspace + * + * - PMIX_SERVER_PIDINFO: connect to a server embedded in the process with + * the given pid + * + * Passing a _NULL_ value for the info array pointer is not allowed and will + * result in return of an error. + * + * NOTE: PMIx does not currently support on-the-fly changes to the tool's + * identifier. Thus, the new server must be under the same nspace manager + * (e.g., host RM) as the prior server so that the original nspace remains + * a unique assignment. The proc parameter is included here for obsolence + * protection in case this constraint is someday removed. Meantime, the + * proc parameter will be filled with the tool's existing nspace/rank, and + * the caller is welcome to pass _NULL_ in that location + */ +PMIX_EXPORT pmix_status_t PMIx_tool_connect_to_server(pmix_proc_t *proc, + pmix_info_t info[], size_t ninfo); #if defined(c_plusplus) || defined(__cplusplus) } diff --git a/opal/mca/pmix/pmix3x/pmix/src/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/Makefile.am index a5c16d7d799..9522167dfc1 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/src/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2006-2016 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -103,6 +103,7 @@ include runtime/Makefile.include include tool/Makefile.include include tools/Makefile.include include common/Makefile.include +include hwloc/Makefile.include MAINTAINERCLEANFILES = Makefile.in config.h config.h.in DISTCLEANFILES = Makefile diff --git a/opal/mca/pmix/pmix3x/pmix/src/class/Makefile.include b/opal/mca/pmix/pmix3x/pmix/src/class/Makefile.include index 904995173d3..4fcaf6845d2 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/class/Makefile.include +++ b/opal/mca/pmix/pmix3x/pmix/src/class/Makefile.include @@ -10,7 +10,7 @@ # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. # Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. # $COPYRIGHT$ # @@ -24,6 +24,7 @@ # Source code files headers += \ + class/pmix_bitmap.h \ class/pmix_object.h \ class/pmix_list.h \ class/pmix_pointer_array.h \ @@ -33,6 +34,7 @@ headers += \ class/pmix_value_array.h sources += \ + class/pmix_bitmap.c \ class/pmix_object.c \ class/pmix_list.c \ class/pmix_pointer_array.c \ diff --git a/opal/mca/pmix/pmix3x/pmix/src/class/pmix_bitmap.c b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_bitmap.c new file mode 100644 index 00000000000..ed44afc934b --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_bitmap.c @@ -0,0 +1,412 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010-2012 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#include + +#include "pmix_common.h" +#include "src/class/pmix_bitmap.h" + +/* The number of bits in the underlying type of the bitmap field + * in the pmix_bitmap_t struct + */ +#define SIZE_OF_BASE_TYPE 64 + +static void pmix_bitmap_construct(pmix_bitmap_t *bm); +static void pmix_bitmap_destruct(pmix_bitmap_t *bm); + +PMIX_CLASS_INSTANCE(pmix_bitmap_t, pmix_object_t, + pmix_bitmap_construct, pmix_bitmap_destruct); + + +static void +pmix_bitmap_construct(pmix_bitmap_t *bm) +{ + bm->bitmap = NULL; + bm->array_size = 0; + bm->max_size = INT_MAX; +} + + +static void +pmix_bitmap_destruct(pmix_bitmap_t *bm) +{ + if (NULL != bm->bitmap) { + free(bm->bitmap); + bm->bitmap = NULL; + } +} + + +int pmix_bitmap_set_max_size (pmix_bitmap_t *bm, int max_size) +{ + if (NULL == bm) { + return PMIX_ERR_BAD_PARAM; + } + + /* + * Only if the caller wants to set the maximum size, + * we set it (in numbers of bits!), otherwise it is + * set to INT_MAX in the constructor. + */ + bm->max_size = (int)(((size_t)max_size + SIZE_OF_BASE_TYPE - 1) / SIZE_OF_BASE_TYPE); + + return PMIX_SUCCESS; +} + + +int +pmix_bitmap_init(pmix_bitmap_t *bm, int size) +{ + /* + * Only if the caller set the maximum size before initializing, + * we test here (in numbers of bits!) + * By default, the max size is INT_MAX, set in the constructor. + */ + if ((size <= 0) || (NULL == bm) || (size > bm->max_size)) { + return PMIX_ERR_BAD_PARAM; + } + + bm->array_size = (int)(((size_t)size + SIZE_OF_BASE_TYPE - 1) / SIZE_OF_BASE_TYPE); + if( NULL != bm->bitmap ) { + free(bm->bitmap); + if(bm->max_size < bm->array_size) + bm->max_size = bm->array_size; + } + bm->bitmap = (uint64_t*) malloc(bm->array_size * sizeof(uint64_t)); + if (NULL == bm->bitmap) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + + pmix_bitmap_clear_all_bits(bm); + return PMIX_SUCCESS; +} + + +int +pmix_bitmap_set_bit(pmix_bitmap_t *bm, int bit) +{ + int index, offset, new_size; + + if ((bit < 0) || (NULL == bm) || (bit > bm->max_size)) { + return PMIX_ERR_BAD_PARAM; + } + + index = bit / SIZE_OF_BASE_TYPE; + offset = bit % SIZE_OF_BASE_TYPE; + + if (index >= bm->array_size) { + + /* We need to allocate more space for the bitmap, since we are + out of range. We don't throw any error here, because this is + valid and we simply expand the bitmap */ + + new_size = index + 1; + if( new_size > bm->max_size ) + new_size = bm->max_size; + + /* New size is just a multiple of the original size to fit in + the index. */ + bm->bitmap = (uint64_t*)realloc(bm->bitmap, new_size*sizeof(uint64_t)); + if (NULL == bm->bitmap) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + + /* zero out the new elements */ + memset(&bm->bitmap[bm->array_size], 0, (new_size - bm->array_size) * sizeof(uint64_t)); + + /* Update the array_size */ + bm->array_size = new_size; + } + + /* Now set the bit */ + bm->bitmap[index] |= (1UL << offset); + + return PMIX_SUCCESS; +} + + +int +pmix_bitmap_clear_bit(pmix_bitmap_t *bm, int bit) +{ + int index, offset; + + if ((bit < 0) || NULL == bm || (bit >= (bm->array_size * SIZE_OF_BASE_TYPE))) { + return PMIX_ERR_BAD_PARAM; + } + + index = bit / SIZE_OF_BASE_TYPE; + offset = bit % SIZE_OF_BASE_TYPE; + + bm->bitmap[index] &= ~(1UL << offset); + return PMIX_SUCCESS; +} + + +bool +pmix_bitmap_is_set_bit(pmix_bitmap_t *bm, int bit) +{ + int index, offset; + + if ((bit < 0) || NULL == bm || (bit >= (bm->array_size * SIZE_OF_BASE_TYPE))) { + return false; + } + + index = bit / SIZE_OF_BASE_TYPE; + offset = bit % SIZE_OF_BASE_TYPE; + + if (0 != (bm->bitmap[index] & (1UL << offset))) { + return true; + } + + return false; +} + + +int +pmix_bitmap_clear_all_bits(pmix_bitmap_t *bm) +{ + if (NULL == bm) { + return PMIX_ERR_BAD_PARAM; + } + + memset(bm->bitmap, 0, bm->array_size * sizeof(uint64_t)); + return PMIX_SUCCESS; +} + + +int +pmix_bitmap_set_all_bits(pmix_bitmap_t *bm) +{ + if (NULL == bm) { + return PMIX_ERR_BAD_PARAM; + } + + memset(bm->bitmap, 0xff, bm->array_size * sizeof(uint64_t)); + + return PMIX_SUCCESS; +} + + +int +pmix_bitmap_find_and_set_first_unset_bit(pmix_bitmap_t *bm, int *position) +{ + int i = 0; + uint64_t temp, all_ones = 0xffffffffffffffffUL; + + if (NULL == bm) { + return PMIX_ERR_BAD_PARAM; + } + + /* Neglect all which don't have an unset bit */ + *position = 0; + while((i < bm->array_size) && (bm->bitmap[i] == all_ones)) { + ++i; + } + + if (i == bm->array_size) { + /* increase the bitmap size then */ + *position = bm->array_size * SIZE_OF_BASE_TYPE; + return pmix_bitmap_set_bit(bm, *position); + } + + /* This one has an unset bit, find its bit number */ + + temp = bm->bitmap[i]; + bm->bitmap[i] |= (bm->bitmap[i] + 1); /* Set the first zero bit */ + temp ^= bm->bitmap[i]; /* Compute the change: the first unset bit in the original number */ + while( !(temp & 0x1) ) { + ++(*position); + temp >>= 1; + } + + (*position) += i * SIZE_OF_BASE_TYPE; + return PMIX_SUCCESS; +} + +int pmix_bitmap_bitwise_and_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right) +{ + int i; + + /* + * Sanity check + */ + if( NULL == dest || NULL == right ) { + return PMIX_ERR_BAD_PARAM; + } + if( dest->array_size != right->array_size ) { + return PMIX_ERR_BAD_PARAM; + } + + /* + * Bitwise AND + */ + for(i = 0; i < dest->array_size; ++i) { + dest->bitmap[i] &= right->bitmap[i]; + } + + return PMIX_SUCCESS; +} + +int pmix_bitmap_bitwise_or_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right) +{ + int i; + + /* + * Sanity check + */ + if( NULL == dest || NULL == right ) { + return PMIX_ERR_BAD_PARAM; + } + if( dest->array_size != right->array_size ) { + return PMIX_ERR_BAD_PARAM; + } + + /* + * Bitwise OR + */ + for(i = 0; i < dest->array_size; ++i) { + dest->bitmap[i] |= right->bitmap[i]; + } + + return PMIX_SUCCESS; +} + +int pmix_bitmap_bitwise_xor_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right) +{ + int i; + + /* + * Sanity check + */ + if( NULL == dest || NULL == right ) { + return PMIX_ERR_BAD_PARAM; + } + if( dest->array_size != right->array_size ) { + return PMIX_ERR_BAD_PARAM; + } + + /* + * Bitwise XOR + */ + for(i = 0; i < dest->array_size; ++i) { + dest->bitmap[i] ^= right->bitmap[i]; + } + + return PMIX_SUCCESS; +} + +bool pmix_bitmap_are_different(pmix_bitmap_t *left, pmix_bitmap_t *right) +{ + int i; + + /* + * Sanity check + */ + if( NULL == left || NULL == right ) { + return PMIX_ERR_BAD_PARAM; + } + + if( pmix_bitmap_size(left) != pmix_bitmap_size(right) ) { + return true; + } + + /* + * Direct comparison + */ + for(i = 0; i < left->array_size; ++i) { + if( left->bitmap[i] != right->bitmap[i] ) { + return true; + } + } + + return false; +} + +char * pmix_bitmap_get_string(pmix_bitmap_t *bitmap) +{ + int i; + char *bitmap_str = NULL; + + if( NULL == bitmap) { + return NULL; + } + + bitmap_str = malloc(bitmap->array_size * SIZE_OF_BASE_TYPE + 1); + if (NULL == bitmap_str) { + return NULL; + } + bitmap_str[bitmap->array_size * SIZE_OF_BASE_TYPE] = '\0'; + + for( i = 0; i < (bitmap->array_size * SIZE_OF_BASE_TYPE); ++i) { + if( pmix_bitmap_is_set_bit(bitmap, i) ) { + bitmap_str[i] = 'X'; + } else { + bitmap_str[i] = '_'; + } + } + + return bitmap_str; +} + +int pmix_bitmap_num_unset_bits(pmix_bitmap_t *bm, int len) +{ + return (len - pmix_bitmap_num_set_bits(bm, len)); +} + +int pmix_bitmap_num_set_bits(pmix_bitmap_t *bm, int len) +{ + int i, cnt = 0; + uint64_t val; + +#if PMIX_ENABLE_DEBUG + if ((len < 0) || NULL == bm || (len >= (bm->array_size * SIZE_OF_BASE_TYPE))) { + return 0; + } +#endif + + for(i = 0; i < len; ++i) { + if( 0 == (val = bm->bitmap[i]) ) continue; + /* Peter Wegner in CACM 3 (1960), 322. This method goes through as many + * iterations as there are set bits. */ + for( ; val; cnt++ ) { + val &= val - 1; /* clear the least significant bit set */ + } + } + + return cnt; +} + +bool pmix_bitmap_is_clear(pmix_bitmap_t *bm) +{ + int i; + + for (i = 0; i < bm->array_size; ++i) { + if (0 != bm->bitmap[i]) { + return false; + } + } + return true; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/class/pmix_bitmap.h b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_bitmap.h new file mode 100644 index 00000000000..9017b3b88da --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_bitmap.h @@ -0,0 +1,259 @@ +/* -*- Mode: C; c-basic-offset:4 ; -*- */ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010-2012 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +/** @file + * + * A bitmap implementation. The bits start off with 0, so this bitmap + * has bits numbered as bit 0, bit 1, bit 2 and so on. This bitmap + * has auto-expansion capabilities, that is once the size is set + * during init, it can be automatically expanded by setting the bit + * beyond the current size. But note, this is allowed just when the + * bit is set -- so the valid functions are set_bit and + * find_and_set_bit. Other functions like clear, if passed a bit + * outside the initialized range will result in an error. + * + * To allow these bitmaps to track fortran handles (which MPI defines + * to be Fortran INTEGER), we offer a pmix_bitmap_set_max_size, so that + * the upper layer can ask to never have more than + * OMPI_FORTRAN_HANDLE_MAX, which is min(INT_MAX, fortran INTEGER max). + */ + +#ifndef PMIX_BITMAP_H +#define PMIX_BITMAP_H + +#include + +#include + +#include "src/class/pmix_object.h" + +BEGIN_C_DECLS + +struct pmix_bitmap_t { + pmix_object_t super; /**< Subclass of pmix_object_t */ + uint64_t *bitmap; /**< The actual bitmap array of characters */ + int array_size; /**< The actual array size that maintains the bitmap */ + int max_size; /**< The maximum size that this bitmap may grow (optional) */ +}; + +typedef struct pmix_bitmap_t pmix_bitmap_t; + +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_bitmap_t); + +/** + * Set the maximum size of the bitmap. + * May be reset any time, but HAS TO BE SET BEFORE pmix_bitmap_init! + * + * @param bitmap The input bitmap (IN) + * @param max_size The maximum size of the bitmap in terms of bits (IN) + * @return PMIX error code or success + * + */ +PMIX_EXPORT int pmix_bitmap_set_max_size (pmix_bitmap_t *bm, int max_size); + + +/** + * Initializes the bitmap and sets its size. This must be called + * before the bitmap can be actually used + * + * @param bitmap The input bitmap (IN) + * @param size The initial size of the bitmap in terms of bits (IN) + * @return PMIX error code or success + * + */ +PMIX_EXPORT int pmix_bitmap_init (pmix_bitmap_t *bm, int size); + + +/** + * Set a bit of the bitmap. If the bit asked for is beyond the current + * size of the bitmap, then the bitmap is extended to accomodate the + * bit + * + * @param bitmap The input bitmap (IN) + * @param bit The bit which is to be set (IN) + * @return PMIX error code or success + * + */ +PMIX_EXPORT int pmix_bitmap_set_bit(pmix_bitmap_t *bm, int bit); + + +/** + * Clear/unset a bit of the bitmap. If the bit is beyond the current + * size of the bitmap, an error is returned + * + * @param bitmap The input bitmap (IN) + * @param bit The bit which is to be cleared (IN) + * @return PMIX error code if the bit is out of range, else success + * + */ +PMIX_EXPORT int pmix_bitmap_clear_bit(pmix_bitmap_t *bm, int bit); + + +/** + * Find out if a bit is set in the bitmap + * + * @param bitmap The input bitmap (IN) + * @param bit The bit which is to be checked (IN) + * @return true if the bit is set + * false if the bit is not set OR the index + * is outside the bounds of the provided + * bitmap + * + */ +PMIX_EXPORT bool pmix_bitmap_is_set_bit(pmix_bitmap_t *bm, int bit); + + +/** + * Find the first clear bit in the bitmap and set it + * + * @param bitmap The input bitmap (IN) + * @param position Position of the first clear bit (OUT) + + * @return err PMIX_SUCCESS on success + */ +PMIX_EXPORT int pmix_bitmap_find_and_set_first_unset_bit(pmix_bitmap_t *bm, + int *position); + + +/** + * Clear all bits in the bitmap + * + * @param bitmap The input bitmap (IN) + * @return PMIX error code if bm is NULL + * + */ +PMIX_EXPORT int pmix_bitmap_clear_all_bits(pmix_bitmap_t *bm); + + +/** + * Set all bits in the bitmap + * @param bitmap The input bitmap (IN) + * @return PMIX error code if bm is NULL + * + */ +PMIX_EXPORT int pmix_bitmap_set_all_bits(pmix_bitmap_t *bm); + + +/** + * Gives the current size (number of bits) in the bitmap. This is the + * legal (accessible) number of bits + * + * @param bitmap The input bitmap (IN) + * @return PMIX error code if bm is NULL + * + */ +static inline int pmix_bitmap_size(pmix_bitmap_t *bm) +{ + return (NULL == bm) ? 0 : (bm->array_size * ((int) (sizeof(*bm->bitmap) * 8))); +} + + +/** + * Copy a bitmap + * + * @param dest Pointer to the destination bitmap + * @param src Pointer to the source bitmap + * @ return PMIX error code if something goes wrong + */ +static inline void pmix_bitmap_copy(pmix_bitmap_t *dest, pmix_bitmap_t *src) +{ + if( dest->array_size < src->array_size ) { + if( NULL != dest->bitmap) free(dest->bitmap); + dest->max_size = src->max_size; + dest->bitmap = (uint64_t*)malloc(src->array_size*sizeof(uint64_t)); + } + memcpy(dest->bitmap, src->bitmap, src->array_size * sizeof(uint64_t)); + dest->array_size = src->array_size; +} + +/** + * Bitwise AND operator (inplace) + * + * @param dest Pointer to the bitmap that should be modified + * @param right Point to the other bitmap in the operation + * @return PMIX error code if the length of the two bitmaps is not equal or one is NULL. + */ +PMIX_EXPORT int pmix_bitmap_bitwise_and_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right); + +/** + * Bitwise OR operator (inplace) + * + * @param dest Pointer to the bitmap that should be modified + * @param right Point to the other bitmap in the operation + * @return PMIX error code if the length of the two bitmaps is not equal or one is NULL. + */ +PMIX_EXPORT int pmix_bitmap_bitwise_or_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right); + +/** + * Bitwise XOR operator (inplace) + * + * @param dest Pointer to the bitmap that should be modified + * @param right Point to the other bitmap in the operation + * @return PMIX error code if the length of the two bitmaps is not equal or one is NULL. + */ +PMIX_EXPORT int pmix_bitmap_bitwise_xor_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right); + +/** + * If the bitmaps are different + * + * @param left Pointer to a bitmap + * @param right Pointer to another bitmap + * @return true if different, false if the same + */ +PMIX_EXPORT bool pmix_bitmap_are_different(pmix_bitmap_t *left, pmix_bitmap_t *right); + +/** + * Get a string representation of the bitmap. + * Useful for debugging. + * + * @param bitmap Point to the bitmap to represent + * @return Pointer to the string (caller must free if not NULL) + */ +PMIX_EXPORT char * pmix_bitmap_get_string(pmix_bitmap_t *bitmap); + +/** + * Return the number of 'unset' bits, upto the specified length + * + * @param bitmap Pointer to the bitmap + * @param len Number of bits to check + * @return Integer + */ +PMIX_EXPORT int pmix_bitmap_num_unset_bits(pmix_bitmap_t *bm, int len); + +/** + * Return the number of 'set' bits, upto the specified length + * + * @param bitmap Pointer to the bitmap + * @param len Number of bits to check + * @return Integer + */ +PMIX_EXPORT int pmix_bitmap_num_set_bits(pmix_bitmap_t *bm, int len); + +/** + * Check a bitmap to see if any bit is set + */ +PMIX_EXPORT bool pmix_bitmap_is_clear(pmix_bitmap_t *bm); + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.c b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.c index 93fbaab66d5..fd114a77aa3 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.c +++ b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.c @@ -2,7 +2,7 @@ /* * Copyright (c) 2012-2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.h b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.h index a4c65fc38c7..02a246bfdd5 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.h +++ b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.h @@ -2,7 +2,7 @@ /* * Copyright (c) 2012-2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix3x/pmix/src/client/pmi2.c b/opal/mca/pmix/pmix3x/pmix/src/client/pmi2.c index 080c5bf5077..2ad443c7382 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/client/pmi2.c +++ b/opal/mca/pmix/pmix3x/pmix/src/client/pmi2.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -327,8 +327,6 @@ PMIX_EXPORT int PMI2_Job_Connect(const char jobid[], PMI2_Connect_comm_t *conn) { pmix_status_t rc = PMIX_SUCCESS; pmix_proc_t proc; - char nspace[PMIX_MAX_NSLEN+1]; - pmix_rank_t rank; PMI2_CHECK(); @@ -343,14 +341,14 @@ PMIX_EXPORT int PMI2_Job_Connect(const char jobid[], PMI2_Connect_comm_t *conn) memset(proc.nspace, 0, sizeof(proc.nspace)); (void)strncpy(proc.nspace, (jobid ? jobid : proc.nspace), sizeof(proc.nspace)-1); proc.rank = PMIX_RANK_WILDCARD; - rc = PMIx_Connect(&proc, 1, NULL, 0, nspace, &rank); + rc = PMIx_Connect(&proc, 1, NULL, 0); return convert_err(rc); } PMIX_EXPORT int PMI2_Job_Disconnect(const char jobid[]) { pmix_status_t rc = PMIX_SUCCESS; - char nspace[PMIX_MAX_NSLEN+1]; + pmix_proc_t proc; PMI2_CHECK(); @@ -358,9 +356,10 @@ PMIX_EXPORT int PMI2_Job_Disconnect(const char jobid[]) return PMI2_SUCCESS; } - memset(nspace, 0, sizeof(nspace)); - (void)strncpy(nspace, (jobid ? jobid : nspace), sizeof(nspace)-1); - rc = PMIx_Disconnect(nspace, NULL, 0); + memset(proc.nspace, 0, sizeof(proc.nspace)); + (void)strncpy(proc.nspace, (jobid ? jobid : proc.nspace), sizeof(proc.nspace)-1); + proc.rank = PMIX_RANK_WILDCARD; + rc = PMIx_Disconnect(&proc, 1, NULL, 0); return convert_err(rc); } diff --git a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client.c b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client.c index 808f52e5c6c..0ac9c39160c 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client.c +++ b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client.c @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2014-2017 Research Organization for Information Science + * Copyright (c) 2014-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . * All rights reserved. @@ -66,6 +66,7 @@ static const char pmix_version_string[] = PMIX_VERSION; #include "src/util/compress.h" #include "src/util/error.h" #include "src/util/hash.h" +#include "src/util/name_fns.h" #include "src/util/output.h" #include "src/runtime/pmix_progress_threads.h" #include "src/runtime/pmix_rte.h" @@ -154,9 +155,9 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer, goto error; } - /* we always leave space for a callback object */ - chain->ninfo = ninfo + 1; - PMIX_INFO_CREATE(chain->info, chain->ninfo); + /* we always leave space for event hdlr name and a callback object */ + chain->nallocated = ninfo + 2; + PMIX_INFO_CREATE(chain->info, chain->nallocated); if (NULL == chain->info) { PMIX_ERROR_LOG(PMIX_ERR_NOMEM); PMIX_RELEASE(chain); @@ -164,6 +165,7 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer, } if (0 < ninfo) { + chain->ninfo = ninfo; cnt = ninfo; PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver, buf, chain->info, &cnt, PMIX_INFO); @@ -180,12 +182,10 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer, } } } - /* now put the callback object tag in the last element */ - PMIX_INFO_LOAD(&chain->info[ninfo], PMIX_EVENT_RETURN_OBJECT, NULL, PMIX_POINTER); pmix_output_verbose(2, pmix_client_globals.base_output, - "[%s:%d] pmix:client_notify_recv - processing event %d, calling errhandler", - pmix_globals.myid.nspace, pmix_globals.myid.rank, chain->status); + "[%s:%d] pmix:client_notify_recv - processing event %s, calling errhandler", + pmix_globals.myid.nspace, pmix_globals.myid.rank, PMIx_Error_string(chain->status)); pmix_invoke_local_event_hdlr(chain); return; @@ -243,6 +243,7 @@ static void job_data(struct pmix_peer_t *pr, PMIX_GDS_STORE_JOB_INFO(cb->status, pmix_client_globals.myserver, nspace, buf); + free(nspace); cb->status = PMIX_SUCCESS; PMIX_POST_OBJECT(cb); PMIX_WAKEUP_THREAD(&cb->lock); @@ -253,6 +254,18 @@ PMIX_EXPORT const char* PMIx_Get_version(void) return pmix_version_string; } +/* event handler registration callback */ +static void evhandler_reg_callbk(pmix_status_t status, + size_t evhandler_ref, + void *cbdata) +{ + pmix_lock_t *lock = (pmix_lock_t*)cbdata; + + lock->status = status; + PMIX_WAKEUP_THREAD(lock); +} + + static void notification_fn(size_t evhdlr_registration_id, pmix_status_t status, const pmix_proc_t *source, @@ -261,13 +274,38 @@ static void notification_fn(size_t evhdlr_registration_id, pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata) { - pmix_lock_t *reglock = (pmix_lock_t*)cbdata; + pmix_lock_t *lock=NULL; + char *name = NULL; + size_t n; + + if (NULL != info) { + lock = NULL; + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { + lock = (pmix_lock_t*)info[n].value.data.ptr; + } else if (0 == strncmp(info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) { + name = info[n].value.data.string; + } + } + /* if the object wasn't returned, then that is an error */ + if (NULL == lock) { + pmix_output_verbose(2, pmix_client_globals.base_output, + "event handler %s failed to return object", + (NULL == name) ? "NULL" : name); + /* let the event handler progress */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } + return; + } + } + if (NULL != lock) { + PMIX_WAKEUP_THREAD(lock); + } if (NULL != cbfunc) { cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata); } - PMIX_WAKEUP_THREAD(reglock); - } typedef struct { @@ -381,11 +419,7 @@ static void client_iof_handler(struct pmix_peer_t *pr, return; } if (NULL != bo.bytes && 0 < bo.size) { - if (channel & PMIX_FWD_STDOUT_CHANNEL) { - pmix_iof_write_output(&source, channel, &bo, &pmix_client_globals.iof_stdout.wev); - } else { - pmix_iof_write_output(&source, channel, &bo, &pmix_client_globals.iof_stderr.wev); - } + pmix_iof_write_output(&source, channel, &bo, NULL); } PMIX_BYTE_OBJECT_DESTRUCT(&bo); } @@ -395,15 +429,14 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, { char *evar; pmix_status_t rc; - pmix_nspace_t *nsptr; pmix_cb_t cb; pmix_buffer_t *req; pmix_cmd_t cmd = PMIX_REQ_CMD; pmix_status_t code = PMIX_ERR_DEBUGGER_RELEASE; pmix_proc_t wildcard; - pmix_info_t ginfo; + pmix_info_t ginfo, evinfo[2]; pmix_value_t *val = NULL; - pmix_lock_t reglock; + pmix_lock_t reglock, releaselock; size_t n; bool found; pmix_ptl_posted_recv_t *rcv; @@ -430,7 +463,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, return PMIX_SUCCESS; } /* if we don't see the required info, then we cannot init */ - if (NULL == getenv("PMIX_NAMESPACE")) { + if (NULL == (evar = getenv("PMIX_NAMESPACE"))) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INVALID_NAMESPACE; } @@ -472,9 +505,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOMEM; } - /* construct the global notification ring buffer */ - PMIX_CONSTRUCT(&pmix_globals.notifications, pmix_ring_buffer_t); - pmix_ring_buffer_init(&pmix_globals.notifications, 256); pmix_output_verbose(2, pmix_client_globals.base_output, "pmix: init called"); @@ -487,23 +517,12 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, } /* we require our nspace */ - if (NULL == (evar = getenv("PMIX_NAMESPACE"))) { - /* let the caller know that the server isn't available yet */ - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_INVALID_NAMESPACE; - } if (NULL != proc) { (void)strncpy(proc->nspace, evar, PMIX_MAX_NSLEN); } (void)strncpy(pmix_globals.myid.nspace, evar, PMIX_MAX_NSLEN); - /* create a pmix_nspace_t object for our peer */ - nsptr = PMIX_NEW(pmix_nspace_t); - if (NULL == nsptr){ - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_NOMEM; - } - nsptr->nspace = strdup(evar); - pmix_globals.mypeer->nptr = nsptr; + /* set the global pmix_nspace_t object for our peer */ + pmix_globals.mypeer->nptr->nspace = strdup(evar); /* we also require our rank */ if (NULL == (evar = getenv("PMIX_RANK"))) { @@ -643,12 +662,20 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, /* if the value was found, then we need to wait for debugger attach here */ /* register for the debugger release notification */ PMIX_CONSTRUCT_LOCK(®lock); - PMIX_POST_OBJECT(®lock); - PMIx_Register_event_handler(&code, 1, NULL, 0, - notification_fn, NULL, (void*)®lock); - /* wait for it to arrive */ + PMIX_CONSTRUCT_LOCK(&releaselock); + PMIX_INFO_LOAD(&evinfo[0], PMIX_EVENT_RETURN_OBJECT, &releaselock, PMIX_POINTER); + PMIX_INFO_LOAD(&evinfo[1], PMIX_EVENT_HDLR_NAME, "WAIT-FOR-DEBUGGER", PMIX_STRING); + + PMIx_Register_event_handler(&code, 1, evinfo, 2, + notification_fn, evhandler_reg_callbk, (void*)®lock); + /* wait for registration to complete */ PMIX_WAIT_THREAD(®lock); PMIX_DESTRUCT_LOCK(®lock); + PMIX_INFO_DESTRUCT(&evinfo[0]); + PMIX_INFO_DESTRUCT(&evinfo[1]); + /* wait for release to arrive */ + PMIX_WAIT_THREAD(&releaselock); + PMIX_DESTRUCT_LOCK(&releaselock); } PMIX_INFO_DESTRUCT(&ginfo); @@ -656,7 +683,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, if (NULL != info) { _check_for_notify(info, ninfo); } - return PMIX_SUCCESS; } @@ -810,6 +836,7 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo) PMIX_RELEASE(peer); } } + PMIX_DESTRUCT(&pmix_client_globals.peers); if (0 <= pmix_client_globals.myserver->sd) { CLOSE_THE_SOCKET(pmix_client_globals.myserver->sd); @@ -820,8 +847,15 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo) pmix_rte_finalize(); + if (NULL != pmix_globals.mypeer) { + PMIX_RELEASE(pmix_globals.mypeer); + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + /* finalize the class/object system */ + pmix_class_finalize(); + return PMIX_SUCCESS; } @@ -1175,11 +1209,26 @@ static void _commitfn(int sd, short args, void *cbdata) return rc; } +static void _resolve_peers(int sd, short args, void *cbdata) +{ + pmix_cb_t *cb = (pmix_cb_t*)cbdata; + + cb->status = pmix_preg.resolve_peers(cb->key, cb->pname.nspace, + &cb->procs, &cb->nprocs); + /* post the data so the receiving thread can acquire it */ + PMIX_POST_OBJECT(cb); + PMIX_WAKEUP_THREAD(&cb->lock); +} + /* need to thread-shift this request */ PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename, const char *nspace, pmix_proc_t **procs, size_t *nprocs) { + pmix_cb_t *cb; + pmix_status_t rc; + pmix_proc_t proc; + PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { PMIX_RELEASE_THREAD(&pmix_global_lock); @@ -1187,16 +1236,71 @@ PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename, } PMIX_RELEASE_THREAD(&pmix_global_lock); - /* set default */ - *procs = NULL; - *nprocs = 0; - return pmix_preg.resolve_peers(nodename, nspace, procs, nprocs); + cb = PMIX_NEW(pmix_cb_t); + cb->key = (char*)nodename; + cb->pname.nspace = strdup(nspace); + + PMIX_THREADSHIFT(cb, _resolve_peers); + + /* wait for the result */ + PMIX_WAIT_THREAD(&cb->lock); + + /* if the nspace wasn't found, then we need to + * ask the server for that info */ + if (PMIX_ERR_INVALID_NAMESPACE == cb->status) { + (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + /* any key will suffice as it will bring down + * the entire data blob */ + rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, NULL); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(cb); + return rc; + } + /* retry the fetch */ + cb->lock.active = true; + PMIX_THREADSHIFT(cb, _resolve_peers); + PMIX_WAIT_THREAD(&cb->lock); + } + *procs = cb->procs; + *nprocs = cb->nprocs; + + rc = cb->status; + PMIX_RELEASE(cb); + return rc; +} + +static void _resolve_nodes(int fd, short args, void *cbdata) +{ + pmix_cb_t *cb = (pmix_cb_t*)cbdata; + char *regex, **names; + + /* get a regular expression describing the PMIX_NODE_MAP */ + cb->status = pmix_preg.resolve_nodes(cb->pname.nspace, ®ex); + if (PMIX_SUCCESS == cb->status) { + /* parse it into an argv array of names */ + cb->status = pmix_preg.parse_nodes(regex, &names); + if (PMIX_SUCCESS == cb->status) { + /* assemble it into a comma-delimited list */ + cb->key = pmix_argv_join(names, ','); + pmix_argv_free(names); + } else { + free(regex); + } + } + /* post the data so the receiving thread can acquire it */ + PMIX_POST_OBJECT(cb); + PMIX_WAKEUP_THREAD(&cb->lock); } /* need to thread-shift this request */ PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist) { + pmix_cb_t *cb; + pmix_status_t rc; + pmix_proc_t proc; + PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { PMIX_RELEASE_THREAD(&pmix_global_lock); @@ -1204,8 +1308,35 @@ PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist } PMIX_RELEASE_THREAD(&pmix_global_lock); - /* set default */ - *nodelist = NULL; + cb = PMIX_NEW(pmix_cb_t); + cb->pname.nspace = strdup(nspace); + + PMIX_THREADSHIFT(cb, _resolve_nodes); - return pmix_preg.resolve_nodes(nspace, nodelist); + /* wait for the result */ + PMIX_WAIT_THREAD(&cb->lock); + + /* if the nspace wasn't found, then we need to + * ask the server for that info */ + if (PMIX_ERR_INVALID_NAMESPACE == cb->status) { + (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + /* any key will suffice as it will bring down + * the entire data blob */ + rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, NULL); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(cb); + return rc; + } + /* retry the fetch */ + cb->lock.active = true; + PMIX_THREADSHIFT(cb, _resolve_nodes); + PMIX_WAIT_THREAD(&cb->lock); + } + /* the string we want is in the key field */ + *nodelist = cb->key; + + rc = cb->status; + PMIX_RELEASE(cb); + return rc; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_connect.c b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_connect.c index b4ee7f8aa43..3938f5f513b 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_connect.c +++ b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_connect.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . @@ -62,26 +62,17 @@ static void wait_cbfunc(struct pmix_peer_t *pr, pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata); -static void discbfunc(struct pmix_peer_t *pr, - pmix_ptl_hdr_t *hdr, - pmix_buffer_t *buf, void *cbdata); -static void cnct_cbfunc(pmix_status_t status, - char nspace[], int rank, - void *cbdata); - static void op_cbfunc(pmix_status_t status, void *cbdata); PMIX_EXPORT pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs, - const pmix_info_t info[], size_t ninfo, - char nspace[], pmix_rank_t *newrank) + const pmix_info_t info[], size_t ninfo) { pmix_status_t rc; pmix_cb_t *cb; - size_t n; PMIX_ACQUIRE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.connect_output, "pmix: connect called"); if (pmix_globals.init_cntr <= 0) { @@ -100,18 +91,9 @@ PMIX_EXPORT pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs, * recv routine so we know which callback to use when * the return message is recvd */ cb = PMIX_NEW(pmix_cb_t); - /* see if this connect request was to return a new nspace/rank, or - * was just an exchange of info */ - cb->checked = true; - for (n=0; n < ninfo; n++) { - if (0 == strncmp(info[n].key, PMIX_CONNECT_XCHG_ONLY, PMIX_MAX_KEYLEN)) { - cb->checked = false; - break; - } - } /* push the message into our event base to send to the server */ - if (PMIX_SUCCESS != (rc = PMIx_Connect_nb(procs, nprocs, info, ninfo, cnct_cbfunc, cb))) { + if (PMIX_SUCCESS != (rc = PMIx_Connect_nb(procs, nprocs, info, ninfo, op_cbfunc, cb))) { PMIX_RELEASE(cb); return rc; } @@ -119,15 +101,6 @@ PMIX_EXPORT pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs, /* wait for the connect to complete */ PMIX_WAIT_THREAD(&cb->lock); rc = cb->status; - - if (cb->checked && PMIX_SUCCESS == rc) { - if (NULL != nspace) { - (void)strncpy(nspace, cb->pname.nspace, PMIX_MAX_NSLEN); - } - if (NULL != newrank) { - *newrank = cb->pname.rank; - } - } PMIX_RELEASE(cb); pmix_output_verbose(2, pmix_globals.debug_output, @@ -138,18 +111,17 @@ PMIX_EXPORT pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs, PMIX_EXPORT pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, - pmix_connect_cbfunc_t cbfunc, void *cbdata) + pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_buffer_t *msg; pmix_cmd_t cmd = PMIX_CONNECTNB_CMD; pmix_status_t rc; pmix_cb_t *cb; - size_t n; PMIX_ACQUIRE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: connect called"); + pmix_output_verbose(2, pmix_client_globals.connect_output, + "pmix:connect_nb called"); if (pmix_globals.init_cntr <= 0) { PMIX_RELEASE_THREAD(&pmix_global_lock); @@ -213,19 +185,9 @@ PMIX_EXPORT pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t npro * recv routine so we know which callback to use when * the return message is recvd */ cb = PMIX_NEW(pmix_cb_t); - cb->cbfunc.cnctfn = cbfunc; + cb->cbfunc.opfn = cbfunc; cb->cbdata = cbdata; - /* see if this connect request was to return a new nspace/rank, or - * was just an exchange of info */ - cb->checked = true; - for (n=0; n < ninfo; n++) { - if (0 == strncmp(info[n].key, PMIX_CONNECT_XCHG_ONLY, PMIX_MAX_KEYLEN)) { - cb->checked = false; - break; - } - } - /* push the message into our event base to send to the server */ PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb); @@ -237,7 +199,7 @@ PMIX_EXPORT pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t npro return rc; } -PMIX_EXPORT pmix_status_t PMIx_Disconnect(const char nspace[], +PMIX_EXPORT pmix_status_t PMIx_Disconnect(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo) { pmix_status_t rc; @@ -261,7 +223,7 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect(const char nspace[], * the return message is recvd */ cb = PMIX_NEW(pmix_cb_t); - if (PMIX_SUCCESS != (rc = PMIx_Disconnect_nb(nspace, info, ninfo, op_cbfunc, cb))) { + if (PMIX_SUCCESS != (rc = PMIx_Disconnect_nb(procs, nprocs, info, ninfo, op_cbfunc, cb))) { PMIX_RELEASE(cb); return rc; } @@ -277,7 +239,7 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect(const char nspace[], return rc; } -PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const char nspace[], +PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { @@ -291,6 +253,13 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const char nspace[], pmix_output_verbose(2, pmix_globals.debug_output, "pmix: disconnect called"); + size_t cnt; + for (cnt = 0; cnt < nprocs; cnt++) { + if (0 != strcmp(pmix_globals.myid.nspace, procs[cnt].nspace)) { + PMIX_GDS_DEL_NSPACE(rc, procs[cnt].nspace); + } + } + if (pmix_globals.init_cntr <= 0) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; @@ -304,15 +273,10 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const char nspace[], PMIX_RELEASE_THREAD(&pmix_global_lock); /* check for bozo input */ - if (NULL == nspace) { + if (NULL == procs || 0 >= nprocs) { return PMIX_ERR_BAD_PARAM; } - /* release our internal resources */ - if (0 != strncmp(pmix_globals.myid.nspace, nspace, PMIX_MAX_NSLEN)) { - PMIX_GDS_DEL_NSPACE(rc, nspace); - } - msg = PMIX_NEW(pmix_buffer_t); /* pack the cmd */ PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, @@ -322,9 +286,15 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const char nspace[], return rc; } - /* pack the nspace */ + /* pack the number of procs */ PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, - msg, &nspace, 1, PMIX_STRING); + msg, &nprocs, 1, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + msg, procs, nprocs, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; @@ -357,7 +327,7 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const char nspace[], /* push the message into our event base to send to the server */ PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, - msg, discbfunc, (void*)cb); + msg, wait_cbfunc, (void*)cb); if (PMIX_SUCCESS != rc) { PMIX_RELEASE(msg); PMIX_RELEASE(cb); @@ -380,7 +350,6 @@ static void wait_cbfunc(struct pmix_peer_t *pr, char *nspace; pmix_buffer_t bkt; pmix_byte_object_t bo; - pmix_proc_t pname; pmix_output_verbose(2, pmix_globals.debug_output, "pmix:client recv callback activated with %d bytes", @@ -398,10 +367,6 @@ static void wait_cbfunc(struct pmix_peer_t *pr, goto report; } - /* set the default nspace/rank */ - memset(pname.nspace, 0, PMIX_MAX_NSLEN+1); - pname.rank = PMIX_RANK_UNDEF; - /* unpack the returned status */ cnt = 1; PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver, @@ -410,23 +375,6 @@ static void wait_cbfunc(struct pmix_peer_t *pr, PMIX_ERROR_LOG(rc); ret = rc; } - - if (PMIX_SUCCESS != ret) { - goto report; - } - - if (cb->checked) { - /* unpack the returned nspace/rank */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver, - buf, &pname, &cnt, PMIX_PROC); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - ret = rc; - goto report; - } - } - /* connect has to also pass back data from all nspace's involved in * the operation, including our own. Each will come as a byte object */ cnt = 1; @@ -463,47 +411,6 @@ static void wait_cbfunc(struct pmix_peer_t *pr, ret = rc; } - report: - if (NULL != cb->cbfunc.cnctfn) { - cb->cbfunc.cnctfn(ret, pname.nspace, pname.rank, cb->cbdata); - } - PMIX_RELEASE(cb); -} - -static void discbfunc(struct pmix_peer_t *pr, - pmix_ptl_hdr_t *hdr, - pmix_buffer_t *buf, void *cbdata) -{ - pmix_cb_t *cb = (pmix_cb_t*)cbdata; - pmix_status_t rc; - pmix_status_t ret; - int32_t cnt; - - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:client recv callback activated with %d bytes", - (NULL == buf) ? -1 : (int)buf->bytes_used); - - if (NULL == buf) { - ret = PMIX_ERR_BAD_PARAM; - goto report; - } - - /* a zero-byte buffer indicates that this recv is being - * completed due to a lost connection */ - if (PMIX_BUFFER_IS_EMPTY(buf)) { - ret = PMIX_ERR_UNREACH; - goto report; - } - - /* unpack the returned status */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver, - buf, &ret, &cnt, PMIX_STATUS); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - ret = rc; - } - report: if (NULL != cb->cbfunc.opfn) { cb->cbfunc.opfn(ret, cb->cbdata); @@ -519,18 +426,3 @@ static void op_cbfunc(pmix_status_t status, void *cbdata) PMIX_POST_OBJECT(cb); PMIX_WAKEUP_THREAD(&cb->lock); } - -static void cnct_cbfunc(pmix_status_t status, - char nspace[], int rank, - void *cbdata) -{ - pmix_cb_t *cb = (pmix_cb_t*)cbdata; - - cb->status = status; - if (NULL != nspace) { - cb->pname.nspace = strdup(nspace); - } - cb->pname.rank = rank; - PMIX_POST_OBJECT(cb); - PMIX_WAKEUP_THREAD(&cb->lock); -} diff --git a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_get.c b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_get.c index 951804767e9..04a2e09184a 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_get.c +++ b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_get.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . @@ -111,6 +111,7 @@ PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], rc = cb->status; if (NULL != val) { *val = cb->value; + cb->value = NULL; } PMIX_RELEASE(cb); diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_control.c b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_control.c index 44803eff7ae..26123cca2a6 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_control.c +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_control.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -24,6 +24,7 @@ #include "src/threads/threads.h" #include "src/util/argv.h" #include "src/util/error.h" +#include "src/util/name_fns.h" #include "src/util/output.h" #include "src/mca/bfrops/bfrops.h" #include "src/mca/ptl/ptl.h" @@ -108,6 +109,59 @@ static void query_cbfunc(struct pmix_peer_t *peer, PMIX_RELEASE(cd); } +static void acb(pmix_status_t status, + pmix_info_t *info, size_t ninfo, + void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + pmix_cb_t *cb = (pmix_cb_t*)cbdata; + cb->status = status; + if (NULL != release_fn) { + release_fn(release_cbdata); + } + PMIX_WAKEUP_THREAD(&cb->lock); +} + +PMIX_EXPORT pmix_status_t PMIx_Job_control(const pmix_proc_t targets[], size_t ntargets, + const pmix_info_t directives[], size_t ndirs) +{ + pmix_cb_t cb; + pmix_status_t rc; + + PMIX_ACQUIRE_THREAD(&pmix_global_lock); + + if (pmix_globals.init_cntr <= 0) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_INIT; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + + pmix_output_verbose(2, pmix_globals.debug_output, + "%s pmix:job_ctrl", PMIX_NAME_PRINT(&pmix_globals.myid)); + + /* create a callback object as we need to pass it to the + * recv routine so we know which callback to use when + * the return message is recvd */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + if (PMIX_SUCCESS != (rc = PMIx_Job_control_nb(targets, ntargets, + directives, ndirs, + acb, &cb))) { + PMIX_DESTRUCT(&cb); + return rc; + } + + /* wait for the operation to complete */ + PMIX_WAIT_THREAD(&cb.lock); + rc = cb.status; + PMIX_DESTRUCT(&cb); + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:job_ctrl completed"); + + return rc; +} + PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_t ntargets, const pmix_info_t directives[], size_t ndirs, pmix_info_cbfunc_t cbfunc, void *cbdata) @@ -127,16 +181,11 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_ return PMIX_ERR_INIT; } - /* if we aren't connected, don't attempt to send */ - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !pmix_globals.connected) { - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_UNREACH; - } - PMIX_RELEASE_THREAD(&pmix_global_lock); - /* if we are the server, then we just issue the request and * return the response */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + PMIX_RELEASE_THREAD(&pmix_global_lock); if (NULL == pmix_host_server.job_control) { /* nothing we can do */ return PMIX_ERR_NOT_SUPPORTED; @@ -150,6 +199,13 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_ return rc; } + /* we need to send, so check for connection */ + if (!pmix_globals.connected) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_UNREACH; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + /* if we are a client, then relay this request to the server */ msg = PMIX_NEW(pmix_buffer_t); /* pack the cmd */ @@ -171,7 +227,7 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_ } /* remember, the targets can be NULL to indicate that the operation * is to be done against all members of our nspace */ - if (0 < ntargets) { + if (NULL != targets && 0 < ntargets) { /* pack the targets */ PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, targets, ntargets, PMIX_PROC); @@ -190,7 +246,7 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_ PMIX_RELEASE(msg); return rc; } - if (0 < ndirs) { + if (NULL != directives && 0 < ndirs) { PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, directives, ndirs, PMIX_INFO); if (PMIX_SUCCESS != rc) { @@ -218,6 +274,45 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_ return rc; } +PMIX_EXPORT pmix_status_t PMIx_Process_monitor(const pmix_info_t *monitor, pmix_status_t error, + const pmix_info_t directives[], size_t ndirs) +{ + pmix_cb_t cb; + pmix_status_t rc; + + PMIX_ACQUIRE_THREAD(&pmix_global_lock); + + if (pmix_globals.init_cntr <= 0) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_INIT; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + + pmix_output_verbose(2, pmix_globals.debug_output, + "%s pmix:monitor", PMIX_NAME_PRINT(&pmix_globals.myid)); + + /* create a callback object as we need to pass it to the + * recv routine so we know which callback to use when + * the return message is recvd */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + if (PMIX_SUCCESS != (rc = PMIx_Process_monitor_nb(monitor, error, + directives, ndirs, + acb, &cb))) { + PMIX_DESTRUCT(&cb); + return rc; + } + + /* wait for the operation to complete */ + PMIX_WAIT_THREAD(&cb.lock); + rc = cb.status; + PMIX_DESTRUCT(&cb); + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:monitor completed"); + + return rc; +} + PMIX_EXPORT pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pmix_status_t error, const pmix_info_t directives[], size_t ndirs, pmix_info_cbfunc_t cbfunc, void *cbdata) @@ -237,16 +332,11 @@ PMIX_EXPORT pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pm return PMIX_ERR_INIT; } - /* if we aren't connected, don't attempt to send */ - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !pmix_globals.connected) { - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_UNREACH; - } - PMIX_RELEASE_THREAD(&pmix_global_lock); - /* if we are the server, then we just issue the request and * return the response */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + PMIX_RELEASE_THREAD(&pmix_global_lock); if (NULL == pmix_host_server.monitor) { /* nothing we can do */ return PMIX_ERR_NOT_SUPPORTED; @@ -258,6 +348,13 @@ PMIX_EXPORT pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pm return rc; } + /* we need to send, so check for connection */ + if (!pmix_globals.connected) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_UNREACH; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + /* if we are a client, then relay this request to the server */ msg = PMIX_NEW(pmix_buffer_t); /* pack the cmd */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_data.c b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_data.c index f1e9c00cec5..39ca2bcf291 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_data.c +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_data.c @@ -76,7 +76,12 @@ static pmix_peer_t* find_peer(const pmix_proc_t *proc) pmix_value_t *value; int i; - if (NULL == proc) { + if (NULL == proc ) { + return pmix_globals.mypeer; + } + + /* if the target is someone in my nspace, then use my own peer */ + if (0 == strncmp(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN)) { return pmix_globals.mypeer; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.c b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.c index c0b4199d053..03f38eddc60 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.c +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.c @@ -83,7 +83,8 @@ PMIX_EXPORT pmix_status_t PMIx_IOF_pull(const pmix_proc_t procs[], size_t nprocs } /* if we are a server, we cannot do this */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOT_SUPPORTED; } @@ -236,7 +237,8 @@ pmix_status_t PMIx_IOF_push(const pmix_proc_t targets[], size_t ntargets, /* if we are not a server, then we send the provided * data to our server for processing */ - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) || + PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { msg = PMIX_NEW(pmix_buffer_t); if (NULL == msg) { return PMIX_ERR_NOMEM; @@ -318,7 +320,7 @@ pmix_status_t PMIx_IOF_push(const pmix_proc_t targets[], size_t ntargets, pmix_status_t pmix_iof_write_output(const pmix_proc_t *name, pmix_iof_channel_t stream, const pmix_byte_object_t *bo, - pmix_iof_write_event_t *channel) + pmix_iof_flags_t *flags) { char starttag[PMIX_IOF_BASE_TAG_MAX], endtag[PMIX_IOF_BASE_TAG_MAX], *suffix; pmix_iof_write_output_t *output; @@ -326,6 +328,25 @@ pmix_status_t pmix_iof_write_output(const pmix_proc_t *name, int j, k, starttaglen, endtaglen, num_buffered; bool endtagged; char qprint[10]; + pmix_iof_write_event_t *channel; + pmix_iof_flags_t myflags; + + if (PMIX_FWD_STDOUT_CHANNEL & stream) { + channel = &pmix_client_globals.iof_stdout.wev; + } else { + channel = &pmix_client_globals.iof_stderr.wev; + } + if (NULL == flags) { + myflags.xml = pmix_globals.xml_output; + if (pmix_globals.timestamp_output) { + time(&myflags.timestamp); + } else { + myflags.timestamp = 0; + } + myflags.tag = pmix_globals.tag_output; + } else { + myflags = *flags; + } PMIX_OUTPUT_VERBOSE((1, pmix_client_globals.iof_output, "%s write:output setting up to write %lu bytes to %s for %s on fd %d", @@ -337,6 +358,8 @@ pmix_status_t pmix_iof_write_output(const pmix_proc_t *name, /* setup output object */ output = PMIX_NEW(pmix_iof_write_output_t); + memset(starttag, 0, PMIX_IOF_BASE_TAG_MAX); + memset(endtag, 0, PMIX_IOF_BASE_TAG_MAX); /* write output data to the corresponding tag */ if (PMIX_FWD_STDIN_CHANNEL & stream) { @@ -370,22 +393,20 @@ pmix_status_t pmix_iof_write_output(const pmix_proc_t *name, /* if this is to be xml tagged, create a tag with the correct syntax - we do not allow * timestamping of xml output */ - if (pmix_globals.xml_output) { + if (myflags.xml) { snprintf(starttag, PMIX_IOF_BASE_TAG_MAX, "<%s rank=\"%s\">", suffix, PMIX_RANK_PRINT(name->rank)); snprintf(endtag, PMIX_IOF_BASE_TAG_MAX, "", suffix); goto construct; } /* if we are to timestamp output, start the tag with that */ - if (pmix_globals.timestamp_output) { - time_t mytime; + if (0 < myflags.timestamp) { char *cptr; /* get the timestamp */ - time(&mytime); - cptr = ctime(&mytime); + cptr = ctime(&myflags.timestamp); cptr[strlen(cptr)-1] = '\0'; /* remove trailing newline */ - if (pmix_globals.tag_output) { + if (myflags.tag) { /* if we want it tagged as well, use both */ snprintf(starttag, PMIX_IOF_BASE_TAG_MAX, "%s[%s]<%s>:", cptr, PMIX_NAME_PRINT(name), suffix); @@ -398,7 +419,7 @@ pmix_status_t pmix_iof_write_output(const pmix_proc_t *name, goto construct; } - if (pmix_globals.tag_output) { + if (myflags.tag) { snprintf(starttag, PMIX_IOF_BASE_TAG_MAX, "[%s]<%s>:", PMIX_NAME_PRINT(name), suffix); /* no endtag for this option */ @@ -431,7 +452,7 @@ pmix_status_t pmix_iof_write_output(const pmix_proc_t *name, * and replace those with the tag */ for (i=0; i < bo->size && k < PMIX_IOF_BASE_TAGGED_OUT_MAX; i++) { - if (pmix_globals.xml_output) { + if (myflags.xml) { if ('&' == bo->bytes[i]) { if (k+5 >= PMIX_IOF_BASE_TAGGED_OUT_MAX) { PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.h b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.h index 1a91b12baac..fcc5f7b3da8 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.h +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.h @@ -104,6 +104,15 @@ typedef struct { PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_iof_read_event_t); +/* define a struct to hold booleans controlling the + * format/contents of the output */ +typedef struct { + bool xml; + time_t timestamp; + bool tag; +} pmix_iof_flags_t; + + /* Write event macro's */ static inline bool @@ -184,7 +193,7 @@ PMIX_EXPORT pmix_status_t pmix_iof_flush(void); PMIX_EXPORT pmix_status_t pmix_iof_write_output(const pmix_proc_t *name, pmix_iof_channel_t stream, const pmix_byte_object_t *bo, - pmix_iof_write_event_t *channel); + pmix_iof_flags_t *flags); PMIX_EXPORT void pmix_iof_static_dump_output(pmix_iof_sink_t *sink); PMIX_EXPORT void pmix_iof_write_handler(int fd, short event, void *cbdata); PMIX_EXPORT void pmix_iof_stdin_write_handler(int fd, short event, void *cbdata); diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_log.c b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_log.c index b32c4cc5cf7..582a64e4eaa 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_log.c +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_log.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -24,14 +24,22 @@ #include "src/threads/threads.h" #include "src/util/argv.h" #include "src/util/error.h" +#include "src/util/name_fns.h" #include "src/util/output.h" #include "src/mca/bfrops/bfrops.h" -#include "src/mca/ptl/ptl.h" +#include "src/mca/plog/base/base.h" #include "src/client/pmix_client_ops.h" #include "src/server/pmix_server_ops.h" #include "src/include/pmix_globals.h" +static void opcbfunc(pmix_status_t status, void *cbdata) +{ + pmix_cb_t *cb = (pmix_cb_t*)cbdata; + cb->status = status; + PMIX_WAKEUP_THREAD(&cb->lock); +} + static void log_cbfunc(struct pmix_peer_t *peer, pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) @@ -53,6 +61,55 @@ static void log_cbfunc(struct pmix_peer_t *peer, PMIX_RELEASE(cd); } +PMIX_EXPORT pmix_status_t PMIx_Log(const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs) +{ + pmix_cb_t cb; + pmix_status_t rc; + + PMIX_ACQUIRE_THREAD(&pmix_global_lock); + + if (pmix_globals.init_cntr <= 0) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_INIT; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + + pmix_output_verbose(2, pmix_plog_base_framework.framework_output, + "%s pmix:log", PMIX_NAME_PRINT(&pmix_globals.myid)); + + /* create a callback object as we need to pass it to the + * recv routine so we know which callback to use when + * the return message is recvd */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + if (PMIX_SUCCESS != (rc = PMIx_Log_nb(data, ndata, directives, + ndirs, opcbfunc, &cb))) { + PMIX_DESTRUCT(&cb); + return rc; + } + + /* wait for the operation to complete */ + PMIX_WAIT_THREAD(&cb.lock); + rc = cb.status; + PMIX_DESTRUCT(&cb); + + pmix_output_verbose(2, pmix_plog_base_framework.framework_output, + "pmix:log completed"); + + return rc; +} + +static void localcbfunc(pmix_status_t status, void *cbdata) +{ + pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata; + + PMIX_INFO_FREE(cd->directives, cd->ndirs); + if (NULL != cd->cbfunc.opcbfn) { + cd->cbfunc.opcbfn(status, cd->cbdata); + } + PMIX_RELEASE(cd); +} + PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata, const pmix_info_t directives[], size_t ndirs, pmix_op_cbfunc_t cbfunc, void *cbdata) @@ -62,6 +119,9 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata, pmix_cmd_t cmd = PMIX_LOG_CMD; pmix_buffer_t *msg; pmix_status_t rc; + size_t n; + time_t timestamp = 0; + pmix_proc_t *source = NULL; PMIX_ACQUIRE_THREAD(&pmix_global_lock); @@ -73,32 +133,38 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata, return PMIX_ERR_INIT; } - /* if we aren't connected, don't attempt to send */ - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !pmix_globals.connected) { - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_UNREACH; - } - PMIX_RELEASE_THREAD(&pmix_global_lock); - if (0 == ndata || NULL == data) { + PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_BAD_PARAM; } - /* if we are the server, then we just log and - * return the response */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - if (NULL == pmix_host_server.log) { - /* nothing we can do */ - return PMIX_ERR_NOT_SUPPORTED; + /* check the directives - if they requested a timestamp, then + * get the time, also look for a source */ + if (NULL != directives) { + for (n=0; n < ndirs; n++) { + if (0 == strncmp(directives[n].key, PMIX_LOG_GENERATE_TIMESTAMP, PMIX_MAX_KEYLEN)) { + if (PMIX_INFO_TRUE(&directives[n])) { + /* pickup the timestamp */ + timestamp = time(NULL); + } + } else if (0 == strncmp(directives[n].key, PMIX_LOG_SOURCE, PMIX_MAX_KEYLEN)) { + source = directives[n].value.data.proc; } - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:log handed to RM"); - pmix_host_server.log(&pmix_globals.myid, - data, ndata, directives, ndirs, - cbfunc, cbdata); - rc = PMIX_SUCCESS; - } else { - /* if we are a client, then relay this request to the server */ + } + } + + /* if we are a client or tool, we never do this ourselves - we + * always pass this request to our server for execution */ + if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + /* if we aren't connected, don't attempt to send */ + if (!pmix_globals.connected) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_UNREACH; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + + /* if we are not a server, then relay this request to the server */ cd = PMIX_NEW(pmix_shift_caddy_t); cd->cbfunc.opcbfn = cbfunc; cd->cbdata = cbdata; @@ -111,22 +177,35 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata, PMIX_RELEASE(cd); return rc; } + /* provide the timestamp - zero will indicate + * that it wasn't taken */ PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, - msg, &ndata, 1, PMIX_SIZE); + msg, ×tamp, 1, PMIX_TIME); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); PMIX_RELEASE(cd); return rc; } + /* pack the number of data entries */ PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, - msg, data, ndata, PMIX_INFO); + msg, &ndata, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); PMIX_RELEASE(cd); return rc; } + if (0 < ndata) { + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + msg, data, ndata, PMIX_INFO); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + PMIX_RELEASE(cd); + return rc; + } + } PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, &ndirs, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { @@ -146,7 +225,7 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata, } } - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_plog_base_framework.framework_output, "pmix:log sending to server"); PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg, log_cbfunc, (void*)cd); @@ -154,6 +233,41 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata, PMIX_ERROR_LOG(rc); PMIX_RELEASE(cd); } + return rc; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + + /* if no recorded source was found, then we must be it */ + if (NULL == source) { + source = &pmix_globals.myid; + cd = PMIX_NEW(pmix_shift_caddy_t); + cd->cbfunc.opcbfn = cbfunc; + cd->cbdata = cbdata; + cd->ndirs = ndirs + 1; + PMIX_INFO_CREATE(cd->directives, cd->ndirs); + for (n=0; n < ndirs; n++) { + PMIX_INFO_XFER(&cd->directives[n], (pmix_info_t*)&directives[n]); + } + PMIX_INFO_LOAD(&cd->directives[ndirs], PMIX_LOG_SOURCE, &source, PMIX_PROC); + /* call down to process the request - the various components + * will thread shift as required */ + rc = pmix_plog.log(source, data, ndata, cd->directives, cd->ndirs, localcbfunc, cd); + if (PMIX_SUCCESS != rc) { + PMIX_INFO_FREE(cd->directives, cd->ndirs); + PMIX_RELEASE(cd); + } + } else if (0 == strncmp(source->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN) && + source->rank == pmix_globals.myid.rank) { + /* if I am the recorded source, then this is a re-submission of + * something that got "upcalled" by a prior call. In this case, + * we return a "not supported" error as clearly we couldn't + * handle it, and neither could our host */ + rc = PMIX_ERR_NOT_SUPPORTED; + } else { + /* call down to process the request - the various components + * will thread shift as required */ + rc = pmix_plog.log(source, data, ndata, directives, ndirs, cbfunc, cbdata); } + return rc; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_query.c b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_query.c index 5aa14a532a7..33bc025dafd 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_query.c +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_query.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -24,6 +24,7 @@ #include "src/threads/threads.h" #include "src/util/argv.h" #include "src/util/error.h" +#include "src/util/name_fns.h" #include "src/util/output.h" #include "src/mca/bfrops/bfrops.h" #include "src/mca/ptl/ptl.h" @@ -63,6 +64,7 @@ static void query_cbfunc(struct pmix_peer_t *peer, PMIX_BFROPS_UNPACK(rc, peer, buf, &results->status, &cnt, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); + results->status = rc; goto complete; } if (PMIX_SUCCESS != results->status) { @@ -74,6 +76,7 @@ static void query_cbfunc(struct pmix_peer_t *peer, PMIX_BFROPS_UNPACK(rc, peer, buf, &results->ninfo, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); + results->status = rc; goto complete; } if (0 < results->ninfo) { @@ -82,6 +85,7 @@ static void query_cbfunc(struct pmix_peer_t *peer, PMIX_BFROPS_UNPACK(rc, peer, buf, results->info, &cnt, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); + results->status = rc; goto complete; } } @@ -115,20 +119,16 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque return PMIX_ERR_INIT; } - /* if we aren't connected, don't attempt to send */ - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !pmix_globals.connected) { - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_UNREACH; - } - PMIX_RELEASE_THREAD(&pmix_global_lock); - if (0 == nqueries || NULL == queries) { + PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_BAD_PARAM; } /* if we are the server, then we just issue the query and * return the response */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + PMIX_RELEASE_THREAD(&pmix_global_lock); if (NULL == pmix_host_server.query) { /* nothing we can do */ return PMIX_ERR_NOT_SUPPORTED; @@ -138,45 +138,104 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque pmix_host_server.query(&pmix_globals.myid, queries, nqueries, cbfunc, cbdata); - rc = PMIX_SUCCESS; - } else { - /* if we are a client, then relay this request to the server */ - cd = PMIX_NEW(pmix_query_caddy_t); - cd->cbfunc = cbfunc; - cd->cbdata = cbdata; - msg = PMIX_NEW(pmix_buffer_t); - PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, - msg, &cmd, 1, PMIX_COMMAND); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - PMIX_RELEASE(cd); - return rc; - } - PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, - msg, &nqueries, 1, PMIX_SIZE); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - PMIX_RELEASE(cd); - return rc; - } - PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, - msg, queries, nqueries, PMIX_QUERY); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - PMIX_RELEASE(cd); - return rc; - } - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:query sending to server"); - PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, - msg, query_cbfunc, (void*)cd); - if (PMIX_SUCCESS != rc) { - PMIX_RELEASE(cd); - } + return PMIX_SUCCESS; + } + + /* if we aren't connected, don't attempt to send */ + if (!pmix_globals.connected) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_UNREACH; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + + /* if we are a client, then relay this request to the server */ + cd = PMIX_NEW(pmix_query_caddy_t); + cd->cbfunc = cbfunc; + cd->cbdata = cbdata; + msg = PMIX_NEW(pmix_buffer_t); + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + msg, &cmd, 1, PMIX_COMMAND); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + PMIX_RELEASE(cd); + return rc; } + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + msg, &nqueries, 1, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + PMIX_RELEASE(cd); + return rc; + } + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + msg, queries, nqueries, PMIX_QUERY); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + PMIX_RELEASE(cd); + return rc; + } + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:query sending to server"); + PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, + msg, query_cbfunc, (void*)cd); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(cd); + } + return rc; +} + +static void acb(pmix_status_t status, + pmix_info_t *info, size_t ninfo, + void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + pmix_cb_t *cb = (pmix_cb_t*)cbdata; + cb->status = status; + if (NULL != release_fn) { + release_fn(release_cbdata); + } + PMIX_WAKEUP_THREAD(&cb->lock); +} + +PMIX_EXPORT pmix_status_t PMIx_Allocation_request(pmix_alloc_directive_t directive, + pmix_info_t *info, size_t ninfo) +{ + pmix_cb_t cb; + pmix_status_t rc; + + PMIX_ACQUIRE_THREAD(&pmix_global_lock); + + if (pmix_globals.init_cntr <= 0) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_INIT; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + + pmix_output_verbose(2, pmix_globals.debug_output, + "%s pmix:allocate", PMIX_NAME_PRINT(&pmix_globals.myid)); + + /* create a callback object as we need to pass it to the + * recv routine so we know which callback to use when + * the return message is recvd */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + if (PMIX_SUCCESS != (rc = PMIx_Allocation_request_nb(directive, info, ninfo, + acb, &cb))) { + PMIX_DESTRUCT(&cb); + return rc; + } + + /* wait for the operation to complete */ + PMIX_WAIT_THREAD(&cb.lock); + rc = cb.status; + PMIX_DESTRUCT(&cb); + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:allocate completed"); + return rc; } @@ -192,13 +251,18 @@ PMIX_EXPORT pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t dire pmix_output_verbose(2, pmix_globals.debug_output, "pmix: allocate called"); + PMIX_ACQUIRE_THREAD(&pmix_global_lock); + if (pmix_globals.init_cntr <= 0) { + PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; } /* if we are the server, then we just issue the request and * return the response */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + PMIX_RELEASE_THREAD(&pmix_global_lock); if (NULL == pmix_host_server.allocate) { /* nothing we can do */ return PMIX_ERR_NOT_SUPPORTED; @@ -216,8 +280,10 @@ PMIX_EXPORT pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t dire /* if we aren't connected, don't attempt to send */ if (!pmix_globals.connected) { + PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_UNREACH; } + PMIX_RELEASE_THREAD(&pmix_global_lock); msg = PMIX_NEW(pmix_buffer_t); /* pack the cmd */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_security.c b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_security.c index 52091e14c5a..c4797c1cd05 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_security.c +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_security.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -131,7 +131,8 @@ PMIX_EXPORT pmix_status_t PMIx_Get_credential(const pmix_info_t info[], size_t n } /* if we are the server */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { PMIX_RELEASE_THREAD(&pmix_global_lock); /* if the host doesn't support this operation, * see if we can generate it ourselves */ @@ -316,7 +317,8 @@ PMIX_EXPORT pmix_status_t PMIx_Validate_credential(const pmix_byte_object_t *cre } /* if we are the server */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { PMIX_RELEASE_THREAD(&pmix_global_lock); /* if the host doesn't support this operation, * see if we can validate it ourselves */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_strings.c b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_strings.c index 18ae2a34d1c..7ee50b50fa9 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_strings.c +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_strings.c @@ -71,8 +71,12 @@ PMIX_EXPORT const char* PMIx_Proc_state_string(pmix_proc_state_t state) return "PROC TERMINATED WITHOUT CALLING PMIx_Finalize"; case PMIX_PROC_STATE_COMM_FAILED: return "PROC LOST COMMUNICATION"; + case PMIX_PROC_STATE_SENSOR_BOUND_EXCEEDED: + return "PROC SENSOR BOUND EXCEEDED"; case PMIX_PROC_STATE_CALLED_ABORT: return "PROC CALLED PMIx_Abort"; + case PMIX_PROC_STATE_HEARTBEAT_FAILED: + return "PROC FAILED TO REPORT HEARTBEAT"; case PMIX_PROC_STATE_MIGRATING: return "PROC WAITING TO MIGRATE"; case PMIX_PROC_STATE_CANNOT_RESTART: @@ -117,6 +121,8 @@ PMIX_EXPORT const char* PMIx_Persistence_string(pmix_persistence_t persist) return "RETAIN UNTIL APPLICATION OF PUBLISHING PROCESS TERMINATES"; case PMIX_PERSIST_SESSION: return "RETAIN UNTIL ALLOCATION OF PUBLISHING PROCESS TERMINATES"; + case PMIX_PERSIST_INVALID: + return "INVALID"; default: return "UNKNOWN PERSISTENCE"; } @@ -141,6 +147,8 @@ PMIX_EXPORT const char* PMIx_Data_range_string(pmix_data_range_t range) return "AVAIL AS SPECIFIED IN DIRECTIVES"; case PMIX_RANGE_PROC_LOCAL: return "AVAIL ON LOCAL PROC ONLY"; + case PMIX_RANGE_INVALID: + return "INVALID"; default: return "UNKNOWN"; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event.h b/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event.h index 3e45197a41f..b4ee30b0c0e 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event.h +++ b/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event.h @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -53,7 +53,22 @@ typedef struct { size_t index; uint8_t precedence; char *locator; + pmix_proc_t source; // who generated this event + /* When registering for events, callers can specify + * the range of sources from which they are willing + * to receive notifications - e.g., for callers to + * define different handlers for events coming from + * the RM vs those coming from their peers. We use + * the rng field to track these values upon registration. + */ pmix_range_trkr_t rng; + /* For registration, we use the affected field to track + * the range of procs that, if affected by the event, + * should cause the handler to be called (subject, of + * course, to any rng constraints). + */ + pmix_proc_t *affected; + size_t naffected; pmix_notification_fn_t evhdlr; void *cbobject; pmix_status_t *codes; @@ -102,8 +117,11 @@ typedef struct pmix_event_chain_t { bool endchain; pmix_proc_t source; pmix_data_range_t range; + pmix_proc_t *affected; + size_t naffected; pmix_info_t *info; size_t ninfo; + size_t nallocated; pmix_info_t *results; size_t nresults; pmix_event_hdlr_t *evhdlr; @@ -117,6 +135,13 @@ PMIX_CLASS_DECLARATION(pmix_event_chain_t); * affected, plus any additional info provided by the server */ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain); +bool pmix_notify_check_range(pmix_range_trkr_t *rng, + const pmix_proc_t *proc); + +bool pmix_notify_check_affected(pmix_proc_t *interested, size_t ninterested, + pmix_proc_t *affected, size_t naffected); + + /* invoke the server event notification handler */ pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status, const pmix_proc_t *source, @@ -150,16 +175,11 @@ void pmix_event_timeout_cb(int fd, short flags, void *arg); (p)->nptr->nspace, \ PMIX_MAX_NSLEN); \ ch->source.rank = (p)->info->pname.rank; \ - ch->ninfo = 2; \ + ch->ninfo = 0; \ + ch->nallocated = 2; \ ch->final_cbfunc = (f); \ ch->final_cbdata = ch; \ - PMIX_INFO_CREATE(ch->info, ch->ninfo); \ - PMIX_INFO_LOAD(&ch->info[0], \ - PMIX_EVENT_HDLR_NAME, \ - NULL, PMIX_STRING); \ - PMIX_INFO_LOAD(&ch->info[1], \ - PMIX_EVENT_RETURN_OBJECT, \ - NULL, PMIX_POINTER); \ + PMIX_INFO_CREATE(ch->info, ch->nallocated); \ /* cache it */ \ pmix_list_append(&pmix_globals.cached_events, &ch->super); \ ch->timer_active = true; \ @@ -171,7 +191,7 @@ void pmix_event_timeout_cb(int fd, short flags, void *arg); /* add this peer to the array of sources */ \ (void)strncpy(proc.nspace, (p)->nptr->nspace, PMIX_MAX_NSLEN); \ proc.rank = (p)->info->pname.rank; \ - ninfo = ch->ninfo + 1; \ + ninfo = ch->nallocated + 1; \ PMIX_INFO_CREATE(info, ninfo); \ /* must keep the hdlr name and return object at the end, so prepend */ \ PMIX_INFO_LOAD(&info[0], PMIX_PROCID, \ @@ -179,9 +199,10 @@ void pmix_event_timeout_cb(int fd, short flags, void *arg); for (n=0; n < ch->ninfo; n++) { \ PMIX_INFO_XFER(&info[n+1], &ch->info[n]); \ } \ - PMIX_INFO_FREE(ch->info, ch->ninfo); \ + PMIX_INFO_FREE(ch->info, ch->nallocated); \ + ch->nallocated = ninfo; \ ch->info = info; \ - ch->ninfo = ninfo; \ + ch->ninfo = ninfo - 2; \ /* reset the timer */ \ pmix_event_del(&ch->ev); \ PMIX_POST_OBJECT(ch); \ diff --git a/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_notification.c b/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_notification.c index 0fa953a99fb..b4406f21e70 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_notification.c +++ b/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_notification.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. @@ -33,8 +33,6 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); -static bool check_range(pmix_range_trkr_t *range, const pmix_proc_t *proc); - /* if we are a client, we call this function to notify the server of * an event. If we are a server, our host RM will call this function * to notify us of an event */ @@ -53,15 +51,10 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, return PMIX_ERR_INIT; } - /* if we aren't connected, don't attempt to send */ - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !pmix_globals.connected) { - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_UNREACH; - } - PMIX_RELEASE_THREAD(&pmix_global_lock); - - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + PMIX_RELEASE_THREAD(&pmix_global_lock); rc = pmix_server_notify_client_of_event(status, source, range, info, ninfo, cbfunc, cbdata); @@ -69,15 +62,23 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, "pmix_server_notify_event source = %s:%d event_status = %d, rc= %d", (NULL == source) ? "UNKNOWN" : source->nspace, (NULL == source) ? PMIX_RANK_WILDCARD : source->rank, status, rc); - } else { - rc = notify_server_of_event(status, source, range, - info, ninfo, - cbfunc, cbdata); - pmix_output_verbose(2, pmix_client_globals.event_output, - "pmix_client_notify_event source = %s:%d event_status =%d, rc=%d", - (NULL == source) ? pmix_globals.myid.nspace : source->nspace, - (NULL == source) ? pmix_globals.myid.rank : source->rank, status, rc); + return rc; } + + /* if we aren't connected, don't attempt to send */ + if (!pmix_globals.connected) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_UNREACH; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + + rc = notify_server_of_event(status, source, range, + info, ninfo, + cbfunc, cbdata); + pmix_output_verbose(2, pmix_client_globals.event_output, + "pmix_client_notify_event source = %s:%d event_status =%d, rc=%d", + (NULL == source) ? pmix_globals.myid.nspace : source->nspace, + (NULL == source) ? pmix_globals.myid.rank : source->rank, status, rc); return rc; } @@ -167,22 +168,17 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, chain->status = status; (void)strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); chain->source.rank = pmix_globals.myid.rank; - /* we always leave space for a callback object and - * the evhandler name. */ - chain->ninfo = ninfo + 2; - PMIX_INFO_CREATE(chain->info, chain->ninfo); + /* we always leave space for event hdlr name and a callback object */ + chain->nallocated = ninfo + 2; + PMIX_INFO_CREATE(chain->info, chain->nallocated); if (0 < ninfo) { + chain->ninfo = ninfo; /* need to copy the info */ for (n=0; n < ninfo; n++) { PMIX_INFO_XFER(&chain->info[n], &info[n]); } } - /* add the evhandler name tag - we - * will fill it in as each handler is called */ - PMIX_INFO_LOAD(&chain->info[chain->ninfo-2], PMIX_EVENT_HDLR_NAME, NULL, PMIX_STRING); - /* now add the callback object tag */ - PMIX_INFO_LOAD(&chain->info[chain->ninfo-1], PMIX_EVENT_RETURN_OBJECT, NULL, PMIX_POINTER); /* we need to cache this event so we can pass it into * ourselves should someone later register for it */ @@ -204,6 +200,7 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, PMIX_INFO_XFER(&cd->info[n], &chain->info[n]); if (0 == strncmp(cd->info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) { cd->nondefault = true; + chain->nondefault = true; } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_CUSTOM_RANGE, PMIX_MAX_KEYLEN)) { /* provides an array of pmix_proc_t identifying the procs * that are to receive this notification, or a single pmix_proc_t */ @@ -222,6 +219,40 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); return PMIX_ERR_BAD_PARAM; } + } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + PMIX_PROC_CREATE(cd->affected, 1); + if (NULL == cd->affected) { + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + cd->naffected = 1; + memcpy(cd->affected, cd->info[n].value.data.proc, sizeof(pmix_proc_t)); + /* need to do the same for chain so it can be correctly processed */ + PMIX_PROC_CREATE(chain->affected, 1); + if (NULL == chain->affected) { + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + chain->naffected = 1; + memcpy(chain->affected, cd->info[n].value.data.proc, sizeof(pmix_proc_t)); + } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) { + cd->naffected = cd->info[n].value.data.darray->size; + PMIX_PROC_CREATE(cd->affected, cd->naffected); + if (NULL == cd->affected) { + cd->naffected = 0; + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + memcpy(cd->affected, cd->info[n].value.data.darray->array, cd->naffected * sizeof(pmix_proc_t)); + /* need to do the same for chain so it can be correctly processed */ + chain->naffected = cd->info[n].value.data.darray->size; + PMIX_PROC_CREATE(chain->affected, chain->naffected); + if (NULL == chain->affected) { + chain->naffected = 0; + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + memcpy(chain->affected, cd->info[n].value.data.darray->array, chain->naffected * sizeof(pmix_proc_t)); } } } @@ -265,7 +296,9 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, cleanup: pmix_output_verbose(2, pmix_client_globals.event_output, "client: notifying server - unable to send"); - PMIX_RELEASE(msg); + if (NULL != msg) { + PMIX_RELEASE(msg); + } /* we were unable to send anything, so we just return the error */ return rc; } @@ -332,6 +365,10 @@ static void progress_local_event_hdlr(pmix_status_t status, /* pass along the new ones */ chain->results = newinfo; chain->nresults = cnt; + /* clear any loaded name and object */ + chain->ninfo = chain->nallocated - 2; + PMIX_INFO_DESTRUCT(&chain->info[chain->nallocated-2]); + PMIX_INFO_DESTRUCT(&chain->info[chain->nallocated-1]); /* if the caller indicates that the chain is completed, * or we completed the "last" event */ @@ -348,28 +385,22 @@ static void progress_local_event_hdlr(pmix_status_t status, while (pmix_list_get_end(&pmix_globals.events.single_events) != (item = pmix_list_get_next(item))) { nxt = (pmix_event_hdlr_t*)item; if (nxt->codes[0] == chain->status && - check_range(&nxt->rng, &chain->source)) { + pmix_notify_check_range(&nxt->rng, &chain->source) && + pmix_notify_check_affected(nxt->affected, nxt->naffected, + chain->affected, chain->naffected)) { chain->evhdlr = nxt; - /* update the handler name in case they want to reference it */ - for (n=0; n < chain->ninfo; n++) { - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) { - if (NULL != chain->info[n].value.data.string) { - free(chain->info[n].value.data.string); - } - if (NULL != chain->evhdlr->name) { - chain->info[n].value.data.string = strdup(chain->evhdlr->name); - } - break; - } + /* reset our count to the info provided by the caller */ + chain->ninfo = chain->nallocated - 2; + /* if the handler has a name, then provide it */ + if (NULL != chain->evhdlr->name) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING); + chain->ninfo++; } - /* update the evhdlr cbobject */ - for (n=0; n < chain->ninfo; n++) { - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { - if (NULL != chain->evhdlr->name) { - chain->info[n].value.data.ptr = chain->evhdlr->cbobject; - } - break; - } + + /* if there is an evhdlr cbobject, provide it */ + if (NULL != chain->evhdlr->cbobject) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER); + chain->ninfo++; } nxt->evhdlr(nxt->index, chain->status, &chain->source, @@ -394,7 +425,9 @@ static void progress_local_event_hdlr(pmix_status_t status, } while (pmix_list_get_end(&pmix_globals.events.multi_events) != (item = pmix_list_get_next(item))) { nxt = (pmix_event_hdlr_t*)item; - if (!check_range(&nxt->rng, &chain->source)) { + if (!pmix_notify_check_range(&nxt->rng, &chain->source) && + !pmix_notify_check_affected(nxt->affected, nxt->naffected, + chain->affected, chain->naffected)) { continue; } for (n=0; n < nxt->ncodes; n++) { @@ -402,26 +435,18 @@ static void progress_local_event_hdlr(pmix_status_t status, * the source fits within it */ if (nxt->codes[n] == chain->status) { chain->evhdlr = nxt; - /* update the handler name in case they want to reference it */ - for (n=0; n < chain->ninfo; n++) { - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) { - if (NULL != chain->info[n].value.data.string) { - free(chain->info[n].value.data.string); - } - if (NULL != chain->evhdlr->name) { - chain->info[n].value.data.string = strdup(chain->evhdlr->name); - } - break; - } + /* reset our count to the info provided by the caller */ + chain->ninfo = chain->nallocated - 2; + /* if the handler has a name, then provide it */ + if (NULL != chain->evhdlr->name) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING); + chain->ninfo++; } - /* update the evhdlr cbobject */ - for (n=0; n < chain->ninfo; n++) { - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { - if (NULL != chain->evhdlr->name) { - chain->info[n].value.data.ptr = chain->evhdlr->cbobject; - } - break; - } + + /* if there is an evhdlr cbobject, provide it */ + if (NULL != chain->evhdlr->cbobject) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER); + chain->ninfo++; } nxt->evhdlr(nxt->index, chain->status, &chain->source, @@ -446,28 +471,22 @@ static void progress_local_event_hdlr(pmix_status_t status, nxt = (pmix_event_hdlr_t*)item; /* if this event handler provided a range, check to see if * the source fits within it */ - if (check_range(&nxt->rng, &chain->source)) { + if (pmix_notify_check_range(&nxt->rng, &chain->source) && + pmix_notify_check_affected(nxt->affected, nxt->naffected, + chain->affected, chain->naffected)) { chain->evhdlr = nxt; - /* update the handler name in case they want to reference it */ - for (n=0; n < chain->ninfo; n++) { - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) { - if (NULL != chain->info[n].value.data.string) { - free(chain->info[n].value.data.string); - } - if (NULL != chain->evhdlr->name) { - chain->info[n].value.data.string = strdup(chain->evhdlr->name); - } - break; - } + /* reset our count to the info provided by the caller */ + chain->ninfo = chain->nallocated - 2; + /* if the handler has a name, then provide it */ + if (NULL != chain->evhdlr->name) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING); + chain->ninfo++; } - /* update the evhdlr cbobject */ - for (n=0; n < chain->ninfo; n++) { - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { - if (NULL != chain->evhdlr->name) { - chain->info[n].value.data.ptr = chain->evhdlr->cbobject; - } - break; - } + + /* if there is an evhdlr cbobject, provide it */ + if (NULL != chain->evhdlr->cbobject) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER); + chain->ninfo++; } nxt->evhdlr(nxt->index, chain->status, &chain->source, @@ -482,31 +501,25 @@ static void progress_local_event_hdlr(pmix_status_t status, /* if we registered a "last" handler, and it fits the given range * and code, then invoke it now */ if (NULL != pmix_globals.events.last && - check_range(&pmix_globals.events.last->rng, &chain->source)) { + pmix_notify_check_range(&pmix_globals.events.last->rng, &chain->source) && + pmix_notify_check_affected(nxt->affected, nxt->naffected, + chain->affected, chain->naffected)) { chain->endchain = true; // ensure we don't do this again if (1 == pmix_globals.events.last->ncodes && pmix_globals.events.last->codes[0] == chain->status) { chain->evhdlr = pmix_globals.events.last; - /* update the handler name in case they want to reference it */ - for (n=0; n < chain->ninfo; n++) { - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) { - if (NULL != chain->info[n].value.data.string) { - free(chain->info[n].value.data.string); - } - if (NULL != chain->evhdlr->name) { - chain->info[n].value.data.string = strdup(chain->evhdlr->name); - } - break; - } + /* reset our count to the info provided by the caller */ + chain->ninfo = chain->nallocated - 2; + /* if the handler has a name, then provide it */ + if (NULL != chain->evhdlr->name) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING); + chain->ninfo++; } - /* update the evhdlr cbobject */ - for (n=0; n < chain->ninfo; n++) { - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { - if (NULL != chain->evhdlr->name) { - chain->info[n].value.data.ptr = chain->evhdlr->cbobject; - } - break; - } + + /* if there is an evhdlr cbobject, provide it */ + if (NULL != chain->evhdlr->cbobject) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER); + chain->ninfo++; } chain->evhdlr->evhdlr(chain->evhdlr->index, chain->status, &chain->source, @@ -519,26 +532,18 @@ static void progress_local_event_hdlr(pmix_status_t status, for (n=0; n < pmix_globals.events.last->ncodes; n++) { if (pmix_globals.events.last->codes[n] == chain->status) { chain->evhdlr = pmix_globals.events.last; - /* update the handler name in case they want to reference it */ - for (n=0; n < chain->ninfo; n++) { - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) { - if (NULL != chain->info[n].value.data.string) { - free(chain->info[n].value.data.string); - } - if (NULL != chain->evhdlr->name) { - chain->info[n].value.data.string = strdup(chain->evhdlr->name); - } - break; - } + /* reset our count to the info provided by the caller */ + chain->ninfo = chain->nallocated - 2; + /* if the handler has a name, then provide it */ + if (NULL != chain->evhdlr->name) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING); + chain->ninfo++; } - /* update the evhdlr cbobject */ - for (n=0; n < chain->ninfo; n++) { - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { - if (NULL != chain->evhdlr->name) { - chain->info[n].value.data.ptr = chain->evhdlr->cbobject; - } - break; - } + + /* if there is an evhdlr cbobject, provide it */ + if (NULL != chain->evhdlr->cbobject) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER); + chain->ninfo++; } chain->evhdlr->evhdlr(chain->evhdlr->index, chain->status, &chain->source, @@ -551,26 +556,18 @@ static void progress_local_event_hdlr(pmix_status_t status, } else { /* gets run for all codes */ chain->evhdlr = pmix_globals.events.last; - /* update the handler name in case they want to reference it */ - for (n=0; n < chain->ninfo; n++) { - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) { - if (NULL != chain->info[n].value.data.string) { - free(chain->info[n].value.data.string); - } - if (NULL != chain->evhdlr->name) { - chain->info[n].value.data.string = strdup(chain->evhdlr->name); - } - break; - } + /* reset our count to the info provided by the caller */ + chain->ninfo = chain->nallocated - 2; + /* if the handler has a name, then provide it */ + if (NULL != chain->evhdlr->name) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING); + chain->ninfo++; } - /* update the evhdlr cbobject */ - for (n=0; n < chain->ninfo; n++) { - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { - if (NULL != chain->evhdlr->name) { - chain->info[n].value.data.ptr = chain->evhdlr->cbobject; - } - break; - } + + /* if there is an evhdlr cbobject, provide it */ + if (NULL != chain->evhdlr->cbobject) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER); + chain->ninfo++; } chain->evhdlr->evhdlr(chain->evhdlr->index, chain->status, &chain->source, @@ -613,15 +610,15 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) pmix_status_t rc = PMIX_SUCCESS; bool found; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "%s:%d invoke_local_event_hdlr for status %s", pmix_globals.myid.nspace, pmix_globals.myid.rank, PMIx_Error_string(chain->status)); /* sanity check */ if (NULL == chain->info) { - /* should never happen as the return object must - * at least be there, even if it is NULL */ + /* should never happen as space must always be + * reserved for handler name and callback object*/ rc = PMIX_ERR_BAD_PARAM; goto complete; } @@ -638,7 +635,9 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) if (NULL != pmix_globals.events.first) { if (1 == pmix_globals.events.first->ncodes && pmix_globals.events.first->codes[0] == chain->status && - check_range(&pmix_globals.events.first->rng, &chain->source)) { + pmix_notify_check_range(&pmix_globals.events.first->rng, &chain->source) && + pmix_notify_check_affected(pmix_globals.events.first->affected, pmix_globals.events.first->naffected, + chain->affected, chain->naffected)) { /* invoke the handler */ chain->evhdlr = pmix_globals.events.first; goto invk; @@ -653,14 +652,14 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) } /* if this event handler provided a range, check to see if * the source fits within it */ - if (found && check_range(&pmix_globals.events.first->rng, &chain->source)) { + if (found && pmix_notify_check_range(&pmix_globals.events.first->rng, &chain->source)) { /* invoke the handler */ chain->evhdlr = pmix_globals.events.first; goto invk; } } else { /* take all codes for a default handler */ - if (check_range(&pmix_globals.events.first->rng, &chain->source)) { + if (pmix_notify_check_range(&pmix_globals.events.first->rng, &chain->source)) { /* invoke the handler */ chain->evhdlr = pmix_globals.events.first; goto invk; @@ -672,7 +671,9 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) /* cycle thru the single-event registrations first */ PMIX_LIST_FOREACH(evhdlr, &pmix_globals.events.single_events, pmix_event_hdlr_t) { if (evhdlr->codes[0] == chain->status) { - if (check_range(&evhdlr->rng, &chain->source)) { + if (pmix_notify_check_range(&evhdlr->rng, &chain->source) && + pmix_notify_check_affected(evhdlr->affected, evhdlr->naffected, + chain->affected, chain->naffected)) { /* invoke the handler */ chain->evhdlr = evhdlr; goto invk; @@ -685,7 +686,9 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) PMIX_LIST_FOREACH(evhdlr, &pmix_globals.events.multi_events, pmix_event_hdlr_t) { for (i=0; i < evhdlr->ncodes; i++) { if (evhdlr->codes[i] == chain->status) { - if (check_range(&evhdlr->rng, &chain->source)) { + if (pmix_notify_check_range(&evhdlr->rng, &chain->source) && + pmix_notify_check_affected(evhdlr->affected, evhdlr->naffected, + chain->affected, chain->naffected)) { /* invoke the handler */ chain->evhdlr = evhdlr; goto invk; @@ -698,7 +701,9 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) if (!chain->nondefault) { /* pass it to any default handlers */ PMIX_LIST_FOREACH(evhdlr, &pmix_globals.events.default_events, pmix_event_hdlr_t) { - if (check_range(&evhdlr->rng, &chain->source)) { + if (pmix_notify_check_range(&evhdlr->rng, &chain->source) && + pmix_notify_check_affected(evhdlr->affected, evhdlr->naffected, + chain->affected, chain->naffected)) { /* invoke the handler */ chain->evhdlr = evhdlr; goto invk; @@ -709,7 +714,9 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) /* if we registered a "last" handler, and it fits the given range * and code, then invoke it now */ if (NULL != pmix_globals.events.last && - check_range(&pmix_globals.events.last->rng, &chain->source)) { + pmix_notify_check_range(&pmix_globals.events.last->rng, &chain->source) && + pmix_notify_check_affected(pmix_globals.events.last->affected, pmix_globals.events.last->naffected, + chain->affected, chain->naffected)) { chain->endchain = true; // ensure we don't do this again if (1 == pmix_globals.events.last->ncodes && pmix_globals.events.last->codes[0] == chain->status) { @@ -741,29 +748,23 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) invk: - /* update the handler name in case they want to reference it */ - for (i=0; i < chain->ninfo; i++) { - if (0 == strncmp(chain->info[i].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) { - if (NULL != chain->info[i].value.data.string) { - free(chain->info[i].value.data.string); - } - if (NULL != chain->evhdlr->name) { - chain->info[i].value.data.string = strdup(chain->evhdlr->name); - } - break; - } + /* start with the chain holding only the given info */ + chain->ninfo = chain->nallocated - 2; + + /* if the handler has a name, then provide it */ + if (NULL != chain->evhdlr->name) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING); + chain->ninfo++; } - /* update the evhdlr cbobject */ - for (i=0; i < chain->ninfo; i++) { - if (0 == strncmp(chain->info[i].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { - if (NULL != chain->evhdlr->name) { - chain->info[i].value.data.ptr = chain->evhdlr->cbobject; - } - break; - } + + /* if there is an evhdlr cbobject, provide it */ + if (NULL != chain->evhdlr->cbobject) { + PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER); + chain->ninfo++; } + /* invoke the handler */ - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "[%s:%d] INVOKING EVHDLR %s", __FILE__, __LINE__, (NULL == chain->evhdlr->name) ? "NULL" : chain->evhdlr->name); @@ -822,7 +823,6 @@ static void _notify_client_event(int sd, short args, void *cbdata) } } } - if (holdcd) { /* we cannot know if everyone who wants this notice has had a chance * to register for it - the notice may be coming too early. So cache @@ -965,19 +965,72 @@ static void _notify_client_event(int sd, short args, void *cbdata) chain->source.rank = cd->source.rank; /* we always leave space for a callback object and * the evhandler name. */ - chain->ninfo = cd->ninfo + 2; - PMIX_INFO_CREATE(chain->info, chain->ninfo); + chain->nallocated = cd->ninfo + 2; + PMIX_INFO_CREATE(chain->info, chain->nallocated); if (0 < cd->ninfo) { + chain->ninfo = cd->ninfo; /* need to copy the info */ for (n=0; n < cd->ninfo; n++) { PMIX_INFO_XFER(&chain->info[n], &cd->info[n]); + if (0 == strncmp(cd->info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) { + cd->nondefault = true; + chain->nondefault = true; + } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_CUSTOM_RANGE, PMIX_MAX_KEYLEN)) { + /* provides an array of pmix_proc_t identifying the procs + * that are to receive this notification, or a single pmix_proc_t */ + if (PMIX_DATA_ARRAY == cd->info[n].value.type && + NULL != cd->info[n].value.data.darray && + NULL != cd->info[n].value.data.darray->array) { + cd->ntargets = cd->info[n].value.data.darray->size; + PMIX_PROC_CREATE(cd->targets, cd->ntargets); + memcpy(cd->targets, cd->info[n].value.data.darray->array, cd->ntargets * sizeof(pmix_proc_t)); + } else if (PMIX_PROC == cd->info[n].value.type) { + cd->ntargets = 1; + PMIX_PROC_CREATE(cd->targets, cd->ntargets); + memcpy(cd->targets, cd->info[n].value.data.proc, sizeof(pmix_proc_t)); + } else { + /* this is an error */ + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + PMIX_RELEASE(chain); + return; + } + } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + PMIX_PROC_CREATE(cd->affected, 1); + if (NULL == cd->affected) { + PMIX_RELEASE(chain); + return; + } + cd->naffected = 1; + memcpy(cd->affected, cd->info[n].value.data.proc, sizeof(pmix_proc_t)); + /* need to do the same for chain so it can be correctly processed */ + PMIX_PROC_CREATE(chain->affected, 1); + if (NULL == chain->affected) { + PMIX_RELEASE(chain); + return; + } + chain->naffected = 1; + memcpy(chain->affected, cd->info[n].value.data.proc, sizeof(pmix_proc_t)); + } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) { + cd->naffected = cd->info[n].value.data.darray->size; + PMIX_PROC_CREATE(cd->affected, cd->naffected); + if (NULL == cd->affected) { + cd->naffected = 0; + PMIX_RELEASE(chain); + return; + } + memcpy(cd->affected, cd->info[n].value.data.darray->array, cd->naffected * sizeof(pmix_proc_t)); + /* need to do the same for chain so it can be correctly processed */ + chain->naffected = cd->info[n].value.data.darray->size; + PMIX_PROC_CREATE(chain->affected, chain->naffected); + if (NULL == chain->affected) { + chain->naffected = 0; + PMIX_RELEASE(chain); + return; + } + memcpy(chain->affected, cd->info[n].value.data.darray->array, chain->naffected * sizeof(pmix_proc_t)); + } } } - /* put the evhandler name tag in the next-to-last element - we - * will fill it in as each handler is called */ - PMIX_INFO_LOAD(&chain->info[chain->ninfo-2], PMIX_EVENT_HDLR_NAME, NULL, PMIX_STRING); - /* now put the callback object tag in the last element */ - PMIX_INFO_LOAD(&chain->info[chain->ninfo-1], PMIX_EVENT_RETURN_OBJECT, NULL, PMIX_POINTER); /* process it */ pmix_invoke_local_event_hdlr(chain); @@ -1057,6 +1110,7 @@ pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status, } } } + /* * If the range is PMIX_RANGE_NAMESPACE, then they should not have set a * PMIX_EVENT_CUSTOM_RANGE info object or at least we should ignore it @@ -1089,8 +1143,8 @@ pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status, return PMIX_SUCCESS; } -static bool check_range(pmix_range_trkr_t *rng, - const pmix_proc_t *proc) +bool pmix_notify_check_range(pmix_range_trkr_t *rng, + const pmix_proc_t *proc) { size_t n; @@ -1138,6 +1192,37 @@ static bool check_range(pmix_range_trkr_t *rng, return false; } +bool pmix_notify_check_affected(pmix_proc_t *interested, size_t ninterested, + pmix_proc_t *affected, size_t naffected) +{ + size_t m, n; + + /* if they didn't restrict their interests, then accept it */ + if (NULL == interested) { + return true; + } + /* if we weren't given the affected procs, then accept it */ + if (NULL == affected) { + return true; + } + /* check if the two overlap */ + for (n=0; n < naffected; n++) { + for (m=0; m < ninterested; m++) { + if (0 != strncmp(affected[n].nspace, interested[m].nspace, PMIX_MAX_NSLEN)) { + continue; + } + if (PMIX_RANK_WILDCARD == interested[m].rank || + PMIX_RANK_WILDCARD == affected[n].rank || + affected[n].rank == interested[m].rank) { + return true; + } + } + } + /* if we get here, then this proc isn't in range */ + return false; + +} + void pmix_event_timeout_cb(int fd, short flags, void *arg) { pmix_event_chain_t *ch = (pmix_event_chain_t*)arg; @@ -1151,7 +1236,8 @@ void pmix_event_timeout_cb(int fd, short flags, void *arg) pmix_list_remove_item(&pmix_globals.cached_events, &ch->super); /* process this event thru the regular channels */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { pmix_server_notify_client_of_event(ch->status, &ch->source, ch->range, ch->info, ch->ninfo, ch->final_cbfunc, ch->final_cbdata); @@ -1171,6 +1257,8 @@ static void sevcon(pmix_event_hdlr_t *p) p->rng.range = PMIX_RANGE_UNDEF; p->rng.procs = NULL; p->rng.nprocs = 0; + p->affected = NULL; + p->naffected = 0; p->evhdlr = NULL; p->cbobject = NULL; p->codes = NULL; @@ -1187,6 +1275,9 @@ static void sevdes(pmix_event_hdlr_t *p) if (NULL != p->rng.procs) { free(p->rng.procs); } + if (NULL != p->affected) { + PMIX_PROC_FREE(p->affected, p->naffected); + } if (NULL != p->codes) { free(p->codes); } @@ -1238,8 +1329,11 @@ static void chcon(pmix_event_chain_t *p) p->nondefault = false; p->endchain = false; p->range = PMIX_RANGE_UNDEF; + p->affected = NULL; + p->naffected = 0; p->info = NULL; p->ninfo = 0; + p->nallocated = 0; p->results = NULL; p->nresults = 0; p->evhdlr = NULL; @@ -1251,8 +1345,11 @@ static void chdes(pmix_event_chain_t *p) if (p->timer_active) { pmix_event_del(&p->ev); } + if (NULL != p->affected) { + PMIX_PROC_FREE(p->affected, p->naffected); + } if (NULL != p->info) { - PMIX_INFO_FREE(p->info, p->ninfo); + PMIX_INFO_FREE(p->info, p->nallocated); } if (NULL != p->results) { PMIX_INFO_FREE(p->results, p->nresults); diff --git a/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_registration.c b/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_registration.c index 0dceb4f9e34..d1c95358ffb 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_registration.c +++ b/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_registration.c @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2017 Research Organization for Information Science + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -40,6 +40,8 @@ size_t ncodes; pmix_info_t *info; size_t ninfo; + pmix_proc_t *affected; + size_t naffected; pmix_notification_fn_t evhdlr; pmix_hdlr_reg_cbfunc_t evregcbfn; void *cbdata; @@ -55,12 +57,17 @@ static void rscon(pmix_rshift_caddy_t *p) p->ncodes = 0; p->info = NULL; p->ninfo = 0; + p->affected = NULL; + p->naffected = 0; p->evhdlr = NULL; p->evregcbfn = NULL; p->cbdata = NULL; } static void rsdes(pmix_rshift_caddy_t *p) { + if (0 < p->ncodes) { + free(p->codes); + } if (NULL != p->cd) { PMIX_RELEASE(p->cd); } @@ -80,7 +87,7 @@ static void regevents_cbfunc(struct pmix_peer_t *peer, pmix_ptl_hdr_t *hdr, int cnt; size_t index = rb->index; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: regevents callback recvd"); /* unpack the status code */ @@ -230,7 +237,7 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer) pmix_active_code_t *active; pmix_status_t rc; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: _add_hdlr"); /* check to see if we have an active registration on these codes */ @@ -299,15 +306,16 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer) * type with our server, or if we have directives, then we need to notify * the server - however, don't do this for a v1 server as the event * notification system there doesn't work */ - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && pmix_globals.connected && + if ((!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) || PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) && + pmix_globals.connected && !PMIX_PROC_IS_V1(pmix_client_globals.myserver) && (need_register || 0 < pmix_list_get_size(xfer))) { - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: _add_hdlr sending to server"); /* send the directives to the server - we will ack this * registration upon return from there */ if (PMIX_SUCCESS != (rc = _send_to_server(cd2))) { - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: add_hdlr - pack send_to_server failed status=%d", rc); if (NULL != cd2->info) { PMIX_INFO_FREE(cd2->info, cd2->ninfo); @@ -320,9 +328,10 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer) /* if we are a server and are registering for events, then we only contact * our host if we want environmental events */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && cd->enviro && + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer) && cd->enviro && NULL != pmix_host_server.register_events) { - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: _add_hdlr registering with server"); if (PMIX_SUCCESS != (rc = pmix_host_server.register_events(cd->codes, cd->ncodes, cd2->info, cd2->ninfo, @@ -357,8 +366,10 @@ static void check_cached_events(pmix_rshift_caddy_t *cd) } found = false; if (NULL == cd->codes) { - /* they registered a default event handler - always matches */ - found = true; + if (!ncd->nondefault) { + /* they registered a default event handler - always matches */ + found = true; + } } else { for (n=0; n < cd->ncodes; n++) { if (cd->codes[n] == ncd->status) { @@ -367,49 +378,72 @@ static void check_cached_events(pmix_rshift_caddy_t *cd) } } } - if (found) { - /* if we were given specific targets, check if we are one */ - if (NULL != ncd->targets) { - matched = false; - for (n=0; n < ncd->ntargets; n++) { - if (0 != strncmp(pmix_globals.myid.nspace, ncd->targets[n].nspace, PMIX_MAX_NSLEN)) { - continue; - } - if (PMIX_RANK_WILDCARD == ncd->targets[n].rank || - pmix_globals.myid.rank == ncd->targets[n].rank) { - matched = true; - break; - } - } - if (!matched) { - /* do not notify this one */ + if (!found) { + continue; + } + /* if we were given specific targets, check if we are one */ + if (NULL != ncd->targets) { + matched = false; + for (n=0; n < ncd->ntargets; n++) { + if (0 != strncmp(pmix_globals.myid.nspace, ncd->targets[n].nspace, PMIX_MAX_NSLEN)) { continue; } + if (PMIX_RANK_WILDCARD == ncd->targets[n].rank || + pmix_globals.myid.rank == ncd->targets[n].rank) { + matched = true; + break; + } } - /* all matches - notify */ - chain = PMIX_NEW(pmix_event_chain_t); - chain->status = ncd->status; - (void)strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); - chain->source.rank = pmix_globals.myid.rank; - /* we already left space for evhandler name plus - * a callback object when we cached the notification */ + if (!matched) { + /* do not notify this one */ + continue; + } + } + /* if they specified affected proc(s) they wanted to know about, check */ + if (!pmix_notify_check_affected(cd->affected, cd->naffected, + ncd->affected, ncd->naffected)) { + continue; + } + /* create the chain */ + chain = PMIX_NEW(pmix_event_chain_t); + chain->status = ncd->status; + (void)strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + chain->source.rank = pmix_globals.myid.rank; + /* we always leave space for event hdlr name and a callback object */ + chain->nallocated = ncd->ninfo + 2; + PMIX_INFO_CREATE(chain->info, chain->nallocated); + if (0 < cd->ninfo) { chain->ninfo = ncd->ninfo; - PMIX_INFO_CREATE(chain->info, chain->ninfo); - if (0 < cd->ninfo) { - /* need to copy the info */ - for (n=0; n < ncd->ninfo; n++) { - PMIX_INFO_XFER(&chain->info[n], &ncd->info[n]); - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) { - chain->nondefault = true; + /* need to copy the info */ + for (n=0; n < ncd->ninfo; n++) { + PMIX_INFO_XFER(&chain->info[n], &ncd->info[n]); + if (0 == strncmp(ncd->info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) { + chain->nondefault = true; + } else if (0 == strncmp(ncd->info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + PMIX_PROC_CREATE(chain->affected, 1); + if (NULL == chain->affected) { + PMIX_RELEASE(chain); + return; } + chain->naffected = 1; + memcpy(chain->affected, ncd->info[n].value.data.proc, sizeof(pmix_proc_t)); + } else if (0 == strncmp(ncd->info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) { + chain->naffected = ncd->info[n].value.data.darray->size; + PMIX_PROC_CREATE(chain->affected, chain->naffected); + if (NULL == chain->affected) { + chain->naffected = 0; + PMIX_RELEASE(chain); + return; + } + memcpy(chain->affected, ncd->info[n].value.data.darray->array, chain->naffected * sizeof(pmix_proc_t)); } } - /* we don't want this chain to propagate, so indicate it - * should only be run as a single-shot */ - chain->endchain = true; - /* now notify any matching registered callbacks we have */ - pmix_invoke_local_event_hdlr(chain); } + /* we don't want this chain to propagate, so indicate it + * should only be run as a single-shot */ + chain->endchain = true; + /* now notify any matching registered callbacks we have */ + pmix_invoke_local_event_hdlr(chain); } } @@ -433,7 +467,7 @@ static void reg_event_hdlr(int sd, short args, void *cbdata) /* need to acquire the object from its originating thread */ PMIX_ACQUIRE_OBJECT(cd); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: register event_hdlr with %d infos", (int)cd->ninfo); PMIX_CONSTRUCT(&xfer, pmix_list_t); @@ -482,6 +516,12 @@ static void reg_event_hdlr(int sd, short args, void *cbdata) } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_CUSTOM_RANGE, PMIX_MAX_KEYLEN)) { parray = (pmix_proc_t*)cd->info[n].value.data.darray->array; nprocs = cd->info[n].value.data.darray->size; + } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + cd->affected = cd->info[n].value.data.proc; + cd->naffected = 1; + } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) { + cd->affected = (pmix_proc_t*)cd->info[n].value.data.darray->array; + cd->naffected = cd->info[n].value.data.darray->size; } else { ixfer = PMIX_NEW(pmix_info_caddy_t); ixfer->info = &cd->info[n]; @@ -525,6 +565,17 @@ static void reg_event_hdlr(int sd, short args, void *cbdata) } memcpy(evhdlr->rng.procs, parray, nprocs * sizeof(pmix_proc_t)); } + if (NULL != cd->affected && 0 < cd->naffected) { + evhdlr->naffected = cd->naffected; + PMIX_PROC_CREATE(evhdlr->affected, cd->naffected); + if (NULL == evhdlr->affected) { + index = UINT_MAX; + rc = PMIX_ERR_EVENT_REGISTRATION; + PMIX_RELEASE(evhdlr); + goto ack; + } + memcpy(evhdlr->affected, cd->affected, cd->naffected * sizeof(pmix_proc_t)); + } evhdlr->evhdlr = cd->evhdlr; evhdlr->cbobject = cbobject; if (NULL != cd->codes) { @@ -599,6 +650,17 @@ static void reg_event_hdlr(int sd, short args, void *cbdata) } memcpy(evhdlr->rng.procs, parray, nprocs * sizeof(pmix_proc_t)); } + if (NULL != cd->affected && 0 < cd->naffected) { + evhdlr->naffected = cd->naffected; + PMIX_PROC_CREATE(evhdlr->affected, cd->naffected); + if (NULL == evhdlr->affected) { + index = UINT_MAX; + rc = PMIX_ERR_EVENT_REGISTRATION; + PMIX_RELEASE(evhdlr); + goto ack; + } + memcpy(evhdlr->affected, cd->affected, cd->naffected * sizeof(pmix_proc_t)); + } evhdlr->evhdlr = cd->evhdlr; evhdlr->cbobject = cbobject; if (NULL == cd->codes) { @@ -810,7 +872,7 @@ PMIX_EXPORT void PMIx_Register_event_handler(pmix_status_t codes[], size_t ncode cd->evregcbfn = cbfunc; cd->cbdata = cbdata; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix_register_event_hdlr shifting to progress thread"); PMIX_THREADSHIFT(cd, reg_event_hdlr); @@ -832,7 +894,8 @@ static void dereg_event_hdlr(int sd, short args, void *cbdata) /* if I am not the server, and I am connected, then I need * to notify the server to remove my registration */ - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && pmix_globals.connected) { + if ((!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) || PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) && + pmix_globals.connected) { msg = PMIX_NEW(pmix_buffer_t); PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, &cmd, 1, PMIX_COMMAND); @@ -1025,7 +1088,7 @@ PMIX_EXPORT void PMIx_Deregister_event_handler(size_t event_hdlr_ref, cd->cbdata = cbdata; cd->ref = event_hdlr_ref; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix_deregister_event_hdlr shifting to progress thread"); PMIX_THREADSHIFT(cd, dereg_event_hdlr); } diff --git a/opal/mca/pmix/pmix3x/pmix/src/hwloc/Makefile.include b/opal/mca/pmix/pmix3x/pmix/src/hwloc/Makefile.include new file mode 100644 index 00000000000..989e00fd797 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/hwloc/Makefile.include @@ -0,0 +1,15 @@ +# -*- makefile -*- +# +# Copyright (c) 2018 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +sources += \ + hwloc/hwloc.c + +headers += \ + hwloc/hwloc-internal.h diff --git a/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc-internal.h b/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc-internal.h new file mode 100644 index 00000000000..1e731323b71 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc-internal.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2011-2017 Cisco Systems, Inc. All rights reserved + * Copyright (c) 2016 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * this file is included in the rest of + * the code base via pmix/hwloc/hwloc-internal.h. + */ + +#ifndef PMIX_HWLOC_INTERNAL_H +#define PMIX_HWLOC_INTERNAL_H + + +#include +#include + +#if PMIX_HAVE_HWLOC +#include + +#if HWLOC_API_VERSION < 0x00010b00 +#define HWLOC_OBJ_NUMANODE HWLOC_OBJ_NODE +#define HWLOC_OBJ_PACKAGE HWLOC_OBJ_SOCKET +#endif + +extern hwloc_topology_t pmix_hwloc_topology; +#endif + +BEGIN_C_DECLS + +typedef enum { + VM_HOLE_NONE = -1, + VM_HOLE_BEGIN = 0, /* use hole at the very beginning */ + VM_HOLE_AFTER_HEAP = 1, /* use hole right after heap */ + VM_HOLE_BEFORE_STACK = 2, /* use hole right before stack */ + VM_HOLE_BIGGEST = 3, /* use biggest hole */ + VM_HOLE_IN_LIBS = 4, /* use biggest hole between heap and stack */ + VM_HOLE_CUSTOM = 5, /* use given address if available */ +} pmix_hwloc_vm_hole_kind_t; + +typedef enum { + VM_MAP_FILE = 0, + VM_MAP_ANONYMOUS = 1, + VM_MAP_HEAP = 2, + VM_MAP_STACK = 3, + VM_MAP_OTHER = 4 /* vsyscall/vdso/vvar shouldn't occur since we stop after stack */ +} pmix_hwloc_vm_map_kind_t; + +PMIX_EXPORT pmix_status_t pmix_hwloc_get_topology(pmix_info_t *info, size_t ninfo); +PMIX_EXPORT void pmix_hwloc_cleanup(void); + +END_C_DECLS + +#endif /* PMIX_HWLOC_INTERNAL_H */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc.c b/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc.c new file mode 100644 index 00000000000..223347ffe08 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc.c @@ -0,0 +1,741 @@ +/* + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2017 Cisco Systems, Inc. All rights reserved + * Copyright (c) 2017 Inria. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include + +#include +#include +#include +#include +#include +#ifdef HAVE_UNISTD_H +#include +#endif /* HAVE_UNISTD_H */ +#include +#include +#ifdef HAVE_SYS_STAT_H +#include +#endif +#if HAVE_FCNTL_H +#include +#endif + +#include "src/util/error.h" +#include "src/util/fd.h" +#include "src/util/path.h" +#include "src/mca/bfrops/bfrops_types.h" +#include "src/server/pmix_server_ops.h" +#include "hwloc-internal.h" + +#if PMIX_HAVE_HWLOC + +#if HWLOC_API_VERSION >= 0x20000 +#include +#endif + + +PMIX_EXPORT hwloc_topology_t pmix_hwloc_topology = NULL; +static bool external_topology = false; + +#if HWLOC_API_VERSION >= 0x20000 +static size_t shmemsize = 0; +static size_t shmemaddr; +static char *shmemfile = NULL; +static int shmemfd = -1; + +static int parse_map_line(const char *line, + unsigned long *beginp, + unsigned long *endp, + pmix_hwloc_vm_map_kind_t *kindp); +static int use_hole(unsigned long holebegin, + unsigned long holesize, + unsigned long *addrp, + unsigned long size); +static int find_hole(pmix_hwloc_vm_hole_kind_t hkind, + size_t *addrp, + size_t size); +static int enough_space(const char *filename, + size_t space_req, + uint64_t *space_avail, + bool *result); +#endif + +static int set_flags(hwloc_topology_t topo, unsigned int flags) +{ + #if HWLOC_API_VERSION < 0x20000 + flags = HWLOC_TOPOLOGY_FLAG_IO_DEVICES; + #else + int ret = hwloc_topology_set_io_types_filter(topo, HWLOC_TYPE_FILTER_KEEP_IMPORTANT); + if (0 != ret) return ret; + #endif + if (0 != hwloc_topology_set_flags(topo, flags)) { + return PMIX_ERR_INIT; + } + return PMIX_SUCCESS; +} +#endif + +pmix_status_t pmix_hwloc_get_topology(pmix_info_t *info, size_t ninfo) +{ +#if PMIX_HAVE_HWLOC + size_t n; + bool save_xml_v1 = false; + bool save_xml_v2 = false; +#if HWLOC_API_VERSION < 0x20000 + bool save_xml_v2_reqd = false; +#endif + bool share_topo = false; + bool share_reqd = false; + pmix_kval_t *kp2; + char *xml; + int sz; + pmix_status_t rc; +#if HWLOC_API_VERSION >= 0x20000 + pmix_hwloc_vm_hole_kind_t hole = VM_HOLE_BIGGEST; +#endif + + /* we only do something if specifically requested */ + if (NULL == info) { + return PMIX_SUCCESS; + } + + /* check for directives */ + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_TOPOLOGY, PMIX_MAX_KEYLEN)) { + /* if the pointer is NULL, then they want us to + * get the topology - it not NULL, then they + * are giving us the topology */ + if (NULL != pmix_hwloc_topology) { + /* cannot have two topologies */ + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } + if (NULL != info[n].value.data.ptr) { + pmix_hwloc_topology = (hwloc_topology_t)info[n].value.data.ptr; + external_topology = true; + } else { + if (0 != hwloc_topology_init(&pmix_hwloc_topology)) { + return PMIX_ERR_INIT; + } + + if (0 != set_flags(pmix_hwloc_topology, 0)) { + hwloc_topology_destroy(pmix_hwloc_topology); + return PMIX_ERR_INIT; + } + + if (0 != hwloc_topology_load(pmix_hwloc_topology)) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); + hwloc_topology_destroy(pmix_hwloc_topology); + return PMIX_ERR_NOT_SUPPORTED; + } + } + } else if (0 == strncmp(info[n].key, PMIX_HWLOC_XML_V1, PMIX_MAX_KEYLEN)) { + /* if the string pointer is NULL or empty, then they + * want us to create it and store it for later sharing. + * if non-NULL, then this is the topology we are to use */ + if (NULL == info[n].value.data.string) { + save_xml_v1 = true; + } else if (NULL != pmix_hwloc_topology) { + /* cannot have two topologies */ + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } else { + /* load the topology */ + if (0 != hwloc_topology_init(&pmix_hwloc_topology)) { + return PMIX_ERROR; + } + if (0 != hwloc_topology_set_xmlbuffer(pmix_hwloc_topology, + info[n].value.data.string, + strlen(info[n].value.data.string))) { + hwloc_topology_destroy(pmix_hwloc_topology); + return PMIX_ERROR; + } + /* since we are loading this from an external source, we have to + * explicitly set a flag so hwloc sets things up correctly + */ + if (0 != set_flags(pmix_hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM)) { + hwloc_topology_destroy(pmix_hwloc_topology); + return PMIX_ERROR; + } + /* now load the topology */ + if (0 != hwloc_topology_load(pmix_hwloc_topology)) { + hwloc_topology_destroy(pmix_hwloc_topology); + return PMIX_ERROR; + } + /* store the string */ + kp2 = PMIX_NEW(pmix_kval_t); + if (NULL == kp2) { + return PMIX_ERR_NOMEM; + } + kp2->key = strdup(info[n].key); + PMIX_VALUE_XFER(rc, kp2->value, &info[n].value); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + pmix_list_append(&pmix_server_globals.gdata, &kp2->super); + } + } else if (0 == strncmp(info[n].key, PMIX_HWLOC_XML_V2, PMIX_MAX_KEYLEN)) { + /* if the string pointer is NULL or empty, then they + * want us to create it and store it for later sharing. + * if non-NULL, then this is the topology we are to use */ + if (NULL == info[n].value.data.string) { + save_xml_v2 = true; +#if HWLOC_API_VERSION < 0x20000 + save_xml_v2_reqd = PMIX_INFO_REQUIRED(&info[n]); +#endif + } else if (NULL != pmix_hwloc_topology) { + /* cannot have two topologies */ + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } else { + /* load the topology */ + if (0 != hwloc_topology_init(&pmix_hwloc_topology)) { + return PMIX_ERROR; + } + if (0 != hwloc_topology_set_xmlbuffer(pmix_hwloc_topology, + info[n].value.data.string, + strlen(info[n].value.data.string))) { + hwloc_topology_destroy(pmix_hwloc_topology); + return PMIX_ERROR; + } + /* since we are loading this from an external source, we have to + * explicitly set a flag so hwloc sets things up correctly + */ + if (0 != set_flags(pmix_hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM)) { + hwloc_topology_destroy(pmix_hwloc_topology); + return PMIX_ERROR; + } + /* now load the topology */ + if (0 != hwloc_topology_load(pmix_hwloc_topology)) { + hwloc_topology_destroy(pmix_hwloc_topology); + return PMIX_ERROR; + } + /* store the string */ + kp2 = PMIX_NEW(pmix_kval_t); + if (NULL == kp2) { + return PMIX_ERR_NOMEM; + } + kp2->key = strdup(info[n].key); + PMIX_VALUE_XFER(rc, kp2->value, &info[n].value); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + pmix_list_append(&pmix_server_globals.gdata, &kp2->super); + } + } else if (0 == strncmp(info[n].key, PMIX_TOPOLOGY_FILE, PMIX_MAX_KEYLEN)) { + if (NULL == info[n].value.data.string) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } else if (NULL != pmix_hwloc_topology) { + /* cannot have two topologies */ + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } else { + if (0 != hwloc_topology_init(&pmix_hwloc_topology)) { + return PMIX_ERR_NOT_SUPPORTED; + } + if (0 != hwloc_topology_set_xml(pmix_hwloc_topology, info[n].value.data.string)) { + hwloc_topology_destroy(pmix_hwloc_topology); + return PMIX_ERR_NOT_SUPPORTED; + } + /* since we are loading this from an external source, we have to + * explicitly set a flag so hwloc sets things up correctly + */ + if (0 != set_flags(pmix_hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM)) { + hwloc_topology_destroy(pmix_hwloc_topology); + return PMIX_ERR_NOT_SUPPORTED; + } + if (0 != hwloc_topology_load(pmix_hwloc_topology)) { + hwloc_topology_destroy(pmix_hwloc_topology); + return PMIX_ERR_NOT_SUPPORTED; + } + /* store the filename */ + kp2 = PMIX_NEW(pmix_kval_t); + if (NULL == kp2) { + return PMIX_ERR_NOMEM; + } + kp2->key = strdup(info[n].key); + PMIX_VALUE_XFER(rc, kp2->value, &info[n].value); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + pmix_list_append(&pmix_server_globals.gdata, &kp2->super); + } + } else if (0 == strncmp(info[n].key, PMIX_HWLOC_SHARE_TOPO, PMIX_MAX_KEYLEN)) { + share_topo = PMIX_INFO_TRUE(&info[n]); + share_reqd = PMIX_INFO_IS_REQUIRED(&info[n]); + } else if (0 == strncmp(info[n].key, PMIX_HWLOC_HOLE_KIND, PMIX_MAX_KEYLEN)) { +#if HWLOC_API_VERSION >= 0x20000 + if (0 == strcasecmp(info[n].value.data.string, "none")) { + hole = VM_HOLE_NONE; + } else if (0 == strcasecmp(info[n].value.data.string, "begin")) { + hole = VM_HOLE_BEGIN; + } else if (0 == strcasecmp(info[n].value.data.string, "biggest")) { + hole = VM_HOLE_BIGGEST; + } else if (0 == strcasecmp(info[n].value.data.string, "libs")) { + hole = VM_HOLE_IN_LIBS; + } else if (0 == strcasecmp(info[n].value.data.string, "heap")) { + hole = VM_HOLE_AFTER_HEAP; + } else if (0 == strcasecmp(info[n].value.data.string, "stack")) { + hole = VM_HOLE_BEFORE_STACK; + } else { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } +#endif + } + } + + if (save_xml_v1) { + /* create the XML string */ +#if HWLOC_API_VERSION >= 0x20000 + if (0 != hwloc_topology_export_xmlbuffer(pmix_hwloc_topology, &xml, &sz, HWLOC_TOPOLOGY_EXPORT_XML_FLAG_V1)) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); + return PMIX_ERR_NOT_SUPPORTED; + } +#else + if (0 != hwloc_topology_export_xmlbuffer(pmix_hwloc_topology, &xml, &sz)) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); + return PMIX_ERR_NOT_SUPPORTED; + } +#endif + /* store it */ + kp2 = PMIX_NEW(pmix_kval_t); + if (NULL == kp2) { + return PMIX_ERR_NOMEM; + } + kp2->key = strdup(PMIX_HWLOC_XML_V1); + PMIX_VALUE_LOAD(kp2->value, xml, PMIX_STRING); + hwloc_free_xmlbuffer(pmix_hwloc_topology, xml); + pmix_list_append(&pmix_server_globals.gdata, &kp2->super); + } + if (save_xml_v2) { + /* create the XML string */ +#if HWLOC_API_VERSION >= 0x20000 + if (0 != hwloc_topology_export_xmlbuffer(pmix_hwloc_topology, &xml, &sz, 0)) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); + return PMIX_ERR_NOT_SUPPORTED; + } + /* store it */ + kp2 = PMIX_NEW(pmix_kval_t); + if (NULL == kp2) { + return PMIX_ERR_NOMEM; + } + kp2->key = strdup(PMIX_HWLOC_XML_V1); + PMIX_VALUE_LOAD(kp2->value, xml, PMIX_STRING); + hwloc_free_xmlbuffer(pmix_hwloc_topology, xml); + pmix_list_append(&pmix_server_globals.gdata, &kp2->super); +#else + if (save_xml_v2_reqd) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); + return PMIX_ERR_NOT_SUPPORTED; + } +#endif + } + + if (share_topo) { +#if HWLOC_API_VERSION < 0x20000 + if (share_reqd) { + return PMIX_ERR_NOT_SUPPORTED; + } +#else + pmix_status_t rc; + bool space_available = false; + uint64_t amount_space_avail = 0; + + if (VM_HOLE_NONE == hole) { + return PMIX_SUCCESS; + } + + /* get the size of the topology shared memory segment */ + if (0 != hwloc_shmem_topology_get_length(pmix_hwloc_topology, &shmemsize, 0)) { + if (share_reqd) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); + return PMIX_ERR_NOT_SUPPORTED; + } + return PMIX_SUCCESS; + } + + if (PMIX_SUCCESS != (rc = find_hole(hole, &shmemaddr, shmemsize))) { + /* we couldn't find a hole, so don't use the shmem support */ + if (share_reqd) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); + return PMIX_ERR_NOT_SUPPORTED; + } + return PMIX_SUCCESS; + } + /* create the shmem file in our session dir so it + * will automatically get cleaned up */ + asprintf(&shmemfile, "%s/hwloc.sm", pmix_server_globals.tmpdir); + /* let's make sure we have enough space for the backing file */ + if (PMIX_SUCCESS != (rc = enough_space(shmemfile, shmemsize, + &amount_space_avail, + &space_available))) { + free(shmemfile); + shmemfile = NULL; + if (share_reqd) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); + return PMIX_ERR_NOT_SUPPORTED; + } else { + return PMIX_SUCCESS; + } + } + if (!space_available) { + free(shmemfile); + shmemfile = NULL; + if (share_reqd) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); + return PMIX_ERR_NOT_SUPPORTED; + } else { + return PMIX_SUCCESS; + } + } + /* enough space is available, so create the segment */ + if (-1 == (shmemfd = open(shmemfile, O_CREAT | O_RDWR, 0600))) { + free(shmemfile); + shmemfile = NULL; + if (share_reqd) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); + return PMIX_ERR_NOT_SUPPORTED; + } else { + return PMIX_SUCCESS; + } + } + /* ensure nobody inherits this fd */ + pmix_fd_set_cloexec(shmemfd); + /* populate the shmem segment with the topology */ + if (0 != (rc = hwloc_shmem_topology_write(pmix_hwloc_topology, shmemfd, 0, + (void*)shmemaddr, shmemsize, 0))) { + unlink(shmemfile); + free(shmemfile); + shmemfile = NULL; + close(shmemfd); + shmemfd = -1; + if (share_reqd) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); + return PMIX_ERR_NOT_SUPPORTED; + } else { + return PMIX_SUCCESS; + } + } + /* store the rendezvous info */ + kp2 = PMIX_NEW(pmix_kval_t); + if (NULL == kp2) { + return PMIX_ERR_NOMEM; + } + kp2->key = strdup(PMIX_HWLOC_SHMEM_FILE); + PMIX_VALUE_CREATE(kp2->value, 1); + PMIX_VALUE_LOAD(kp2->value, shmemfile, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + pmix_list_append(&pmix_server_globals.gdata, &kp2->super); + kp2 = PMIX_NEW(pmix_kval_t); + if (NULL == kp2) { + return PMIX_ERR_NOMEM; + } + kp2->key = strdup(PMIX_HWLOC_SHMEM_ADDR); + PMIX_VALUE_CREATE(kp2->value, 1); + PMIX_VALUE_LOAD(kp2->value, &shmemaddr, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + pmix_list_append(&pmix_server_globals.gdata, &kp2->super); + kp2 = PMIX_NEW(pmix_kval_t); + if (NULL == kp2) { + return PMIX_ERR_NOMEM; + } + kp2->key = strdup(PMIX_HWLOC_SHMEM_SIZE); + PMIX_VALUE_CREATE(kp2->value, 1); + PMIX_VALUE_LOAD(kp2->value, &shmemsize, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + pmix_list_append(&pmix_server_globals.gdata, &kp2->super); + + +#endif + } + + return PMIX_SUCCESS; +#else // PMIX_HAVE_HWLOC + return PMIX_ERR_NOT_SUPPORTED; +#endif +} + +void pmix_hwloc_cleanup(void) +{ +#if PMIX_HAVE_HWLOC +#if HWLOC_API_VERSION >= 0x20000 + if (NULL != shmemfile) { + unlink(shmemfile); + free(shmemfile); + } + if (0 <= shmemfd) { + close(shmemfd); + } +#endif + if (NULL != pmix_hwloc_topology && !external_topology) { + hwloc_topology_destroy(pmix_hwloc_topology); + } +#endif + return; +} + +#if PMIX_HAVE_HWLOC +#if HWLOC_API_VERSION >= 0x20000 + +static int parse_map_line(const char *line, + unsigned long *beginp, + unsigned long *endp, + pmix_hwloc_vm_map_kind_t *kindp) +{ + const char *tmp = line, *next; + unsigned long value; + + /* "beginaddr-endaddr " */ + value = strtoull(tmp, (char **) &next, 16); + if (next == tmp) { + return PMIX_ERROR; + } + + *beginp = (unsigned long) value; + + if (*next != '-') { + return PMIX_ERROR; + } + + tmp = next + 1; + + value = strtoull(tmp, (char **) &next, 16); + if (next == tmp) { + return PMIX_ERROR; + } + *endp = (unsigned long) value; + tmp = next; + + if (*next != ' ') { + return PMIX_ERROR; + } + tmp = next + 1; + + /* look for ending absolute path */ + next = strchr(tmp, '/'); + if (next) { + *kindp = VM_MAP_FILE; + } else { + /* look for ending special tag [foo] */ + next = strchr(tmp, '['); + if (next) { + if (!strncmp(next, "[heap]", 6)) { + *kindp = VM_MAP_HEAP; + } else if (!strncmp(next, "[stack]", 7)) { + *kindp = VM_MAP_STACK; + } else { + char *end; + if ((end = strchr(next, '\n')) != NULL) { + *end = '\0'; + } + *kindp = VM_MAP_OTHER; + } + } else { + *kindp = VM_MAP_ANONYMOUS; + } + } + + return PMIX_SUCCESS; +} + +#define ALIGN2MB (2*1024*1024UL) + +static int use_hole(unsigned long holebegin, + unsigned long holesize, + unsigned long *addrp, + unsigned long size) +{ + unsigned long aligned; + unsigned long middle = holebegin+holesize/2; + + if (holesize < size) { + return PMIX_ERROR; + } + + /* try to align the middle of the hole on 64MB for POWER's 64k-page PMD */ + #define ALIGN64MB (64*1024*1024UL) + aligned = (middle + ALIGN64MB) & ~(ALIGN64MB-1); + if (aligned + size <= holebegin + holesize) { + *addrp = aligned; + return PMIX_SUCCESS; + } + + /* try to align the middle of the hole on 2MB for x86 PMD */ + aligned = (middle + ALIGN2MB) & ~(ALIGN2MB-1); + if (aligned + size <= holebegin + holesize) { + *addrp = aligned; + return PMIX_SUCCESS; + } + + /* just use the end of the hole */ + *addrp = holebegin + holesize - size; + return PMIX_SUCCESS; +} + +static int find_hole(pmix_hwloc_vm_hole_kind_t hkind, + size_t *addrp, size_t size) +{ + unsigned long biggestbegin = 0; + unsigned long biggestsize = 0; + unsigned long prevend = 0; + pmix_hwloc_vm_map_kind_t prevmkind = VM_MAP_OTHER; + int in_libs = 0; + FILE *file; + char line[96]; + + file = fopen("/proc/self/maps", "r"); + if (!file) { + return PMIX_ERROR; + } + + while (fgets(line, sizeof(line), file) != NULL) { + unsigned long begin=0, end=0; + pmix_hwloc_vm_map_kind_t mkind=VM_MAP_OTHER; + + if (!parse_map_line(line, &begin, &end, &mkind)) { + switch (hkind) { + case VM_HOLE_BEGIN: + fclose(file); + return use_hole(0, begin, addrp, size); + + case VM_HOLE_AFTER_HEAP: + if (prevmkind == VM_MAP_HEAP && mkind != VM_MAP_HEAP) { + /* only use HEAP when there's no other HEAP after it + * (there can be several of them consecutively). + */ + fclose(file); + return use_hole(prevend, begin-prevend, addrp, size); + } + break; + + case VM_HOLE_BEFORE_STACK: + if (mkind == VM_MAP_STACK) { + fclose(file); + return use_hole(prevend, begin-prevend, addrp, size); + } + break; + + case VM_HOLE_IN_LIBS: + /* see if we are between heap and stack */ + if (prevmkind == VM_MAP_HEAP) { + in_libs = 1; + } + if (mkind == VM_MAP_STACK) { + in_libs = 0; + } + if (!in_libs) { + /* we're not in libs, ignore this entry */ + break; + } + /* we're in libs, consider this entry for searching the biggest hole below */ + /* fallthrough */ + + case VM_HOLE_BIGGEST: + if (begin-prevend > biggestsize) { + biggestbegin = prevend; + biggestsize = begin-prevend; + } + break; + + default: + assert(0); + } + } + + while (!strchr(line, '\n')) { + if (!fgets(line, sizeof(line), file)) { + goto done; + } + } + + if (mkind == VM_MAP_STACK) { + /* Don't go beyond the stack. Other VMAs are special (vsyscall, vvar, vdso, etc), + * There's no spare room there. And vsyscall is even above the userspace limit. + */ + break; + } + + prevend = end; + prevmkind = mkind; + + } + + done: + fclose(file); + if (hkind == VM_HOLE_IN_LIBS || hkind == VM_HOLE_BIGGEST) { + return use_hole(biggestbegin, biggestsize, addrp, size); + } + + return PMIX_ERROR; +} + +static int enough_space(const char *filename, + size_t space_req, + uint64_t *space_avail, + bool *result) +{ + uint64_t avail = 0; + size_t fluff = (size_t)(.05 * space_req); + bool enough = false; + char *last_sep = NULL; + /* the target file name is passed here, but we need to check the parent + * directory. store it so we can extract that info later. */ + char *target_dir = strdup(filename); + int rc; + + if (NULL == target_dir) { + rc = PMIX_ERR_OUT_OF_RESOURCE; + goto out; + } + /* get the parent directory */ + last_sep = strrchr(target_dir, PMIX_PATH_SEP[0]); + *last_sep = '\0'; + /* now check space availability */ + if (PMIX_SUCCESS != (rc = pmix_path_df(target_dir, &avail))) { + goto out; + } + /* do we have enough space? */ + if (avail >= space_req + fluff) { + enough = true; + } + +out: + if (NULL != target_dir) { + free(target_dir); + } + *result = enough; + *space_avail = avail; + return rc; +} +#endif + +#endif // PMIX_HAVE_HWLOC diff --git a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.c b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.c index 2f32cd4bd76..8096896b24e 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.c +++ b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.c @@ -2,7 +2,7 @@ /* * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Research Organization for Information Science - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Artem Y. Polyakov . * All rights reserved. @@ -99,16 +99,19 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_cleanup_dir_t, static void nscon(pmix_nspace_t *p) { p->nspace = NULL; + p->nprocs = 0; p->nlocalprocs = 0; p->all_registered = false; p->version_stored = false; p->jobbkt = NULL; p->ndelivered = 0; + p->nfinalized = 0; PMIX_CONSTRUCT(&p->ranks, pmix_list_t); memset(&p->compat, 0, sizeof(p->compat)); PMIX_CONSTRUCT(&p->epilog.cleanup_dirs, pmix_list_t); PMIX_CONSTRUCT(&p->epilog.cleanup_files, pmix_list_t); PMIX_CONSTRUCT(&p->epilog.ignores, pmix_list_t); + PMIX_CONSTRUCT(&p->setup_data, pmix_list_t); } static void nsdes(pmix_nspace_t *p) { @@ -125,6 +128,7 @@ static void nsdes(pmix_nspace_t *p) PMIX_LIST_DESTRUCT(&p->epilog.cleanup_dirs); PMIX_LIST_DESTRUCT(&p->epilog.cleanup_files); PMIX_LIST_DESTRUCT(&p->epilog.ignores); + PMIX_LIST_DESTRUCT(&p->setup_data); } PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_nspace_t, pmix_list_item_t, @@ -173,7 +177,6 @@ static void pcon(pmix_peer_t *p) p->proc_cnt = 0; p->index = 0; p->sd = -1; - p->finalized = false; p->send_ev_active = false; p->recv_ev_active = false; PMIX_CONSTRUCT(&p->send_queue, pmix_list_t); @@ -183,6 +186,7 @@ static void pcon(pmix_peer_t *p) PMIX_CONSTRUCT(&p->epilog.cleanup_dirs, pmix_list_t); PMIX_CONSTRUCT(&p->epilog.cleanup_files, pmix_list_t); PMIX_CONSTRUCT(&p->epilog.ignores, pmix_list_t); + } static void pdes(pmix_peer_t *p) @@ -214,6 +218,9 @@ static void pdes(pmix_peer_t *p) PMIX_LIST_DESTRUCT(&p->epilog.cleanup_dirs); PMIX_LIST_DESTRUCT(&p->epilog.cleanup_files); PMIX_LIST_DESTRUCT(&p->epilog.ignores); + if (NULL != p->nptr) { + PMIX_RELEASE(p->nptr); + } } PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_peer_t, pmix_object_t, diff --git a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.h b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.h index f60947ec79a..9f565214a76 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.h +++ b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.h @@ -56,6 +56,12 @@ BEGIN_C_DECLS #define PMIX_PNET_SETUP_APP "pmix.pnet.setapp" // (pmix_byte_object_t) blob containing info to be given to // pnet framework on remote nodes +#define PMIX_INFO_OP_COMPLETE 0x80000000 +#define PMIX_INFO_OP_COMPLETED(m) \ + ((pmix_info_t*)(m))->flags |= PMIX_INFO_OP_COMPLETE +#define PMIX_INFO_OP_IS_COMPLETE(m) \ + ((m)->flags & PMIX_INFO_OP_COMPLETE) + /* define an internal-only process name that has * a dynamically-sized nspace field to save memory */ typedef struct { @@ -155,11 +161,13 @@ PMIX_CLASS_DECLARATION(pmix_cleanup_dir_t); typedef struct { pmix_list_item_t super; char *nspace; + pmix_rank_t nprocs; // num procs in this nspace size_t nlocalprocs; bool all_registered; // all local ranks have been defined bool version_stored; // the version string used by this nspace has been stored pmix_buffer_t *jobbkt; // packed version of jobinfo size_t ndelivered; // count of #local clients that have received the jobinfo + size_t nfinalized; // count of #local clients that have finalized pmix_list_t ranks; // list of pmix_rank_info_t for connection support of my clients /* all members of an nspace are required to have the * same personality, but it can differ between nspaces. @@ -168,6 +176,8 @@ typedef struct { pmix_personality_t compat; pmix_epilog_t epilog; // things to do upon termination of all local clients // from this nspace + pmix_list_t setup_data; // list of pmix_kval_t containing info structs having blobs + // for setting up the local node for this nspace/application } pmix_nspace_t; PMIX_CLASS_DECLARATION(pmix_nspace_t); @@ -285,7 +295,6 @@ typedef struct { pmix_collect_t collect_type; // whether or not data is to be returned at completion pmix_modex_cbfunc_t modexcbfunc; pmix_op_cbfunc_t op_cbfunc; - pmix_connect_cbfunc_t cnct_cbfunc; } pmix_server_trkr_t; PMIX_CLASS_DECLARATION(pmix_server_trkr_t); @@ -353,7 +362,6 @@ typedef struct { pmix_value_cbfunc_t valuefn; pmix_lookup_cbfunc_t lookupfn; pmix_spawn_cbfunc_t spawnfn; - pmix_connect_cbfunc_t cnctfn; pmix_hdlr_reg_cbfunc_t hdlrregfn; } cbfunc; size_t errhandler_ref; @@ -382,14 +390,6 @@ PMIX_CLASS_DECLARATION(pmix_cb_t); } while (0) -#define PMIX_WAIT_FOR_COMPLETION(a) \ - do { \ - while ((a)) { \ - usleep(10); \ - } \ - PMIX_ACQUIRE_OBJECT((a)); \ - } while (0) - typedef struct { pmix_object_t super; pmix_event_t ev; @@ -397,9 +397,28 @@ typedef struct { pmix_status_t status; pmix_proc_t source; pmix_data_range_t range; + /* For notification, we use the targets field to track + * any custom range of procs that are to receive the + * event. + */ pmix_proc_t *targets; size_t ntargets; + /* When generating a notification, the originator can + * specify the range of procs affected by this event. + * For example, when creating a JOB_TERMINATED event, + * the RM can specify the nspace of the job that has + * ended, thus allowing users to provide a different + * callback object based on the nspace being monitored. + * We use the "affected" field to track these values + * when processing the event chain. + */ + pmix_proc_t *affected; + size_t naffected; + /* track if the event generator stipulates that default + * event handlers are/are not to be given the event */ bool nondefault; + /* carry along any other provided info so the individual + * handlers can look at it */ pmix_info_t *info; size_t ninfo; pmix_buffer_t *buf; diff --git a/opal/mca/pmix/pmix3x/pmix/src/include/types.h b/opal/mca/pmix/pmix3x/pmix/src/include/types.h index 883c732cf45..cf8d082c34d 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/include/types.h +++ b/opal/mca/pmix/pmix3x/pmix/src/include/types.h @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -232,6 +232,8 @@ typedef struct event pmix_event_t; #define pmix_event_base_free(b) event_base_free(b) +#define pmix_event_free(x) event_free(x) + #define pmix_event_base_loopbreak(b) event_base_loopbreak(b) #define pmix_event_base_loopexit(b) event_base_loopexit(b, NULL) diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/base/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/mca/base/Makefile.am index 0aa1d58490c..3531eb898e2 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/base/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/base/Makefile.am @@ -10,7 +10,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2016 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2017 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -26,7 +26,7 @@ AM_CPPFLAGS = \ noinst_LTLIBRARIES = libpmix_mca_base.la -dist_pmixdata_DATA = help-mca-base.txt help-mca-var.txt +dist_pmixdata_DATA = help-pmix-mca-base.txt help-pmix-mca-var.txt # Source code files diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/base/help-mca-base.txt b/opal/mca/pmix/pmix3x/pmix/src/mca/base/help-pmix-mca-base.txt similarity index 97% rename from opal/mca/pmix/pmix3x/pmix/src/mca/base/help-mca-base.txt rename to opal/mca/pmix/pmix3x/pmix/src/mca/base/help-pmix-mca-base.txt index c0b82510767..7a96e7ace8f 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/base/help-mca-base.txt +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/base/help-pmix-mca-base.txt @@ -11,6 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2008-2014 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/base/help-mca-var.txt b/opal/mca/pmix/pmix3x/pmix/src/mca/base/help-pmix-mca-var.txt similarity index 98% rename from opal/mca/pmix/pmix3x/pmix/src/mca/base/help-mca-var.txt rename to opal/mca/pmix/pmix3x/pmix/src/mca/base/help-pmix-mca-var.txt index b306c31ff90..37ac4447ec0 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/base/help-mca-var.txt +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/base/help-pmix-mca-var.txt @@ -13,6 +13,7 @@ # Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2013 Los Alamos National Security, LLC. All rights # reserved. +# Copyright (c) 2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_component_find.c b/opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_component_find.c index 981511ee5a5..fed38f988fd 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_component_find.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_component_find.c @@ -16,7 +16,7 @@ * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -341,7 +341,7 @@ static int component_find_check (pmix_mca_base_framework_t *framework, char **re if (!found) { char h[MAXHOSTNAMELEN]; gethostname(h, sizeof(h)); - pmix_show_help("help-mca-base.txt", + pmix_show_help("help-pmix-mca-base.txt", "find-available:not-valid", true, h, framework->framework_name, requested_component_names[i]); return PMIX_ERR_NOT_FOUND; @@ -375,7 +375,7 @@ int pmix_mca_base_component_parse_requested (const char *requested, bool *includ /* Double check to ensure that the user did not specify the negate character anywhere else in the value. */ if (NULL != strstr (requested, negate)) { - pmix_show_help("help-mca-base.txt", + pmix_show_help("help-pmix-mca-base.txt", "framework-param:too-many-negates", true, requested_orig); return PMIX_ERROR; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_var.c b/opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_var.c index f613090e50b..f1736447a0d 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_var.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_var.c @@ -330,7 +330,7 @@ static void process_env_list(char *env_list, char ***argv, char sep) if (NULL == (ptr = strchr(tokens[i], '='))) { value = getenv(tokens[i]); if (NULL == value) { - pmix_show_help("help-mca-var.txt", "incorrect-env-list-param", + pmix_show_help("help-pmix-mca-var.txt", "incorrect-env-list-param", true, tokens[i], env_list); break; } @@ -368,7 +368,7 @@ int pmix_mca_base_var_process_env_list(char ***argv) if (1 == strlen(pmix_mca_base_env_list_sep)) { sep = pmix_mca_base_env_list_sep[0]; } else { - pmix_show_help("help-mca-var.txt", "incorrect-env-list-sep", + pmix_show_help("help-pmix-mca-var.txt", "incorrect-env-list-sep", true, pmix_mca_base_env_list_sep); return PMIX_SUCCESS; } @@ -742,11 +742,11 @@ static int var_set_from_string (pmix_mca_base_var_t *var, char *src) if (var->mbv_enumerator) { char *valid_values; (void) var->mbv_enumerator->dump(var->mbv_enumerator, &valid_values); - pmix_show_help("help-mca-var.txt", "invalid-value-enum", + pmix_show_help("help-pmix-mca-var.txt", "invalid-value-enum", true, var->mbv_full_name, src, valid_values); free(valid_values); } else { - pmix_show_help("help-mca-var.txt", "invalid-value", + pmix_show_help("help-pmix-mca-var.txt", "invalid-value", true, var->mbv_full_name, src); } @@ -1229,7 +1229,7 @@ static int fixup_files(char **file_list, char * path, bool rel_path_search, char } if (NULL == tmp_file) { - pmix_show_help("help-mca-var.txt", "missing-param-file", + pmix_show_help("help-pmix-mca-var.txt", "missing-param-file", true, getpid(), files[i], msg_path); exit_status = PMIX_ERROR; break; @@ -1386,7 +1386,7 @@ static int register_variable (const char *project_name, const char *framework_na /* Read-only and constant variables can't be settable */ if (scope < PMIX_MCA_BASE_VAR_SCOPE_LOCAL || (flags & PMIX_MCA_BASE_VAR_FLAG_DEFAULT_ONLY)) { if ((flags & PMIX_MCA_BASE_VAR_FLAG_DEFAULT_ONLY) && (flags & PMIX_MCA_BASE_VAR_FLAG_SETTABLE)) { - pmix_show_help("help-mca-var.txt", "invalid-flag-combination", + pmix_show_help("help-pmix-mca-var.txt", "invalid-flag-combination", true, "PMIX_MCA_BASE_VAR_FLAG_DEFAULT_ONLY", "PMIX_MCA_BASE_VAR_FLAG_SETTABLE"); return PMIX_ERROR; } @@ -1476,7 +1476,7 @@ static int register_variable (const char *project_name, const char *framework_na if (0 != compare_strings(framework_name, group->group_framework) || 0 != compare_strings(component_name, group->group_component) || 0 != compare_strings(variable_name, var->mbv_variable_name)) { - pmix_show_help("help-mca-var.txt", "var-name-conflict", + pmix_show_help("help-pmix-mca-var.txt", "var-name-conflict", true, var->mbv_full_name, framework_name, component_name, variable_name, group->group_framework, group->group_component, @@ -1488,7 +1488,7 @@ static int register_variable (const char *project_name, const char *framework_na if (var->mbv_type != type) { #if PMIX_ENABLE_DEBUG - pmix_show_help("help-mca-var.txt", + pmix_show_help("help-pmix-mca-var.txt", "re-register-with-different-type", true, var->mbv_full_name); #endif @@ -1660,7 +1660,7 @@ static int var_set_from_env (pmix_mca_base_var_t *var, pmix_mca_base_var_t *orig /* we found an environment variable but this variable is default-only. print a warning. */ if (PMIX_VAR_IS_DEFAULT_ONLY(original[0])) { - pmix_show_help("help-mca-var.txt", "default-only-param-set", + pmix_show_help("help-pmix-mca-var.txt", "default-only-param-set", true, var_full_name); return PMIX_ERR_NOT_FOUND; @@ -1668,7 +1668,7 @@ static int var_set_from_env (pmix_mca_base_var_t *var, pmix_mca_base_var_t *orig if (PMIX_MCA_BASE_VAR_SOURCE_OVERRIDE == original->mbv_source) { if (!pmix_mca_base_var_suppress_override_warning) { - pmix_show_help("help-mca-var.txt", "overridden-param-set", + pmix_show_help("help-pmix-mca-var.txt", "overridden-param-set", true, var_full_name); } @@ -1699,16 +1699,16 @@ static int var_set_from_env (pmix_mca_base_var_t *var, pmix_mca_base_var_t *orig switch (var->mbv_source) { case PMIX_MCA_BASE_VAR_SOURCE_ENV: - pmix_show_help("help-mca-var.txt", "deprecated-mca-env", + pmix_show_help("help-pmix-mca-var.txt", "deprecated-mca-env", true, var_full_name, new_variable); break; case PMIX_MCA_BASE_VAR_SOURCE_COMMAND_LINE: - pmix_show_help("help-mca-var.txt", "deprecated-mca-cli", + pmix_show_help("help-pmix-mca-var.txt", "deprecated-mca-cli", true, var_full_name, new_variable); break; case PMIX_MCA_BASE_VAR_SOURCE_FILE: case PMIX_MCA_BASE_VAR_SOURCE_OVERRIDE: - pmix_show_help("help-mca-var.txt", "deprecated-mca-file", + pmix_show_help("help-pmix-mca-var.txt", "deprecated-mca-file", true, var_full_name, pmix_mca_base_var_source_file (var), new_variable); break; @@ -1747,14 +1747,14 @@ static int var_set_from_file (pmix_mca_base_var_t *var, pmix_mca_base_var_t *ori /* found it */ if (PMIX_VAR_IS_DEFAULT_ONLY(var[0])) { - pmix_show_help("help-mca-var.txt", "default-only-param-set", + pmix_show_help("help-pmix-mca-var.txt", "default-only-param-set", true, var_full_name); return PMIX_ERR_NOT_FOUND; } if (PMIX_MCA_BASE_VAR_FLAG_ENVIRONMENT_ONLY & original->mbv_flags) { - pmix_show_help("help-mca-var.txt", "environment-only-param", + pmix_show_help("help-pmix-mca-var.txt", "environment-only-param", true, var_full_name, fv->mbvfv_value, fv->mbvfv_file); @@ -1763,7 +1763,7 @@ static int var_set_from_file (pmix_mca_base_var_t *var, pmix_mca_base_var_t *ori if (PMIX_MCA_BASE_VAR_SOURCE_OVERRIDE == original->mbv_source) { if (!pmix_mca_base_var_suppress_override_warning) { - pmix_show_help("help-mca-var.txt", "overridden-param-set", + pmix_show_help("help-pmix-mca-var.txt", "overridden-param-set", true, var_full_name); } @@ -1777,7 +1777,7 @@ static int var_set_from_file (pmix_mca_base_var_t *var, pmix_mca_base_var_t *ori new_variable = original->mbv_full_name; } - pmix_show_help("help-mca-var.txt", "deprecated-mca-file", + pmix_show_help("help-pmix-mca-var.txt", "deprecated-mca-file", true, var_full_name, fv->mbvfv_file, new_variable); } @@ -2041,7 +2041,7 @@ int pmix_mca_base_var_check_exclusive (const char *project, str_b = source_name(var_b); /* Print it all out */ - pmix_show_help("help-mca-var.txt", + pmix_show_help("help-pmix-mca-var.txt", "mutually-exclusive-vars", true, var_a->mbv_full_name, str_a, var_b->mbv_full_name, diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/base.h b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/base.h index 3a4c81a190f..318f076fdca 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/base.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/base.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_copy.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_copy.c index 850b4ed0012..481eb69e76a 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_copy.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_copy.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -367,18 +367,23 @@ pmix_status_t pmix_bfrops_base_copy_pinfo(pmix_proc_info_t **dest, pmix_proc_info_t *src, pmix_data_type_t type) { - *dest = (pmix_proc_info_t*)malloc(sizeof(pmix_proc_info_t)); - (void)strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); - (*dest)->proc.rank = src->proc.rank; + pmix_proc_info_t *p; + + PMIX_PROC_INFO_CREATE(p, 1); + if (NULL == p) { + return PMIX_ERR_NOMEM; + } + memcpy(&p->proc, &src->proc, sizeof(pmix_proc_t)); if (NULL != src->hostname) { - (*dest)->hostname = strdup(src->hostname); + p->hostname = strdup(src->hostname); } if (NULL != src->executable_name) { - (*dest)->executable_name = strdup(src->executable_name); + p->executable_name = strdup(src->executable_name); } - (*dest)->pid = src->pid; - (*dest)->exit_code = src->exit_code; - (*dest)->state = src->state; + memcpy(&p->pid, &src->pid, sizeof(pid_t)); + memcpy(&p->exit_code, &src->exit_code, sizeof(int)); + memcpy(&p->state, &src->state, sizeof(pmix_proc_state_t)); + *dest = p; return PMIX_SUCCESS; } @@ -619,7 +624,7 @@ pmix_status_t pmix_bfrops_base_copy_darray(pmix_data_array_t **dest, p1 = (pmix_info_t*)p->array; s1 = (pmix_info_t*)src->array; for (n=0; n < src->size; n++) { - PMIX_INFO_LOAD(&p1[n], s1[n].key, &s1[n].value.data.flag, s1[n].value.type); + PMIX_INFO_XFER(&p1[n], &s1[n]); } break; case PMIX_PDATA: @@ -631,7 +636,7 @@ pmix_status_t pmix_bfrops_base_copy_darray(pmix_data_array_t **dest, pd = (pmix_pdata_t*)p->array; sd = (pmix_pdata_t*)src->array; for (n=0; n < src->size; n++) { - PMIX_PDATA_LOAD(&pd[n], &sd[n].proc, sd[n].key, &sd[n].value.data.flag, sd[n].value.type); + PMIX_PDATA_XFER(&pd[n], &sd[n]); } break; case PMIX_BUFFER: @@ -901,7 +906,7 @@ pmix_status_t pmix_bfrops_base_copy_envar(pmix_envar_t **dest, pmix_envar_t *src, pmix_data_type_t type) { - *dest = (pmix_envar_t*)malloc(sizeof(pmix_envar_t)); + PMIX_ENVAR_CREATE(*dest, 1); if (NULL == (*dest)) { return PMIX_ERR_NOMEM; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_fns.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_fns.c index b6cbd1ff5f4..bcd083baf57 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_fns.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_fns.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -38,6 +38,13 @@ PMIX_EXPORT void pmix_value_load(pmix_value_t *v, const void *data, pmix_bfrops_base_value_load(v, data, type); } +PMIX_EXPORT pmix_status_t pmix_value_unload(pmix_value_t *kv, + void **data, + size_t *sz) +{ + return pmix_bfrops_base_value_unload(kv, data, sz); +} + PMIX_EXPORT pmix_status_t pmix_value_xfer(pmix_value_t *dest, pmix_value_t *src) { @@ -50,6 +57,8 @@ void pmix_bfrops_base_value_load(pmix_value_t *v, const void *data, pmix_byte_object_t *bo; pmix_proc_info_t *pi; pmix_envar_t *envar; + pmix_data_array_t *darray; + pmix_status_t rc; v->type = type; if (NULL == data) { @@ -173,7 +182,7 @@ void pmix_bfrops_base_value_load(pmix_value_t *v, const void *data, memcpy(&(v->data.pinfo->exit_code), &pi->exit_code, sizeof(int)); break; case PMIX_POINTER: - memcpy(&(v->data.ptr), data, sizeof(void*)); + v->data.ptr = (void*)data; break; case PMIX_ENVAR: envar = (pmix_envar_t*)data; @@ -185,6 +194,14 @@ void pmix_bfrops_base_value_load(pmix_value_t *v, const void *data, } v->data.envar.separator = envar->separator; break; + case PMIX_DATA_ARRAY: + darray = (pmix_data_array_t*)data; + rc = pmix_bfrops_base_copy_darray(&v->data.darray, darray, PMIX_DATA_ARRAY); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + break; + default: /* silence warnings */ break; @@ -199,6 +216,7 @@ pmix_status_t pmix_bfrops_base_value_unload(pmix_value_t *kv, { pmix_status_t rc; pmix_envar_t *envar; + pmix_data_array_t **darray; rc = PMIX_SUCCESS; if (NULL == data || @@ -313,9 +331,14 @@ pmix_status_t pmix_bfrops_base_value_unload(pmix_value_t *kv, *sz = sizeof(pmix_proc_state_t); break; case PMIX_POINTER: - memcpy(*data, &(kv->data.ptr), sizeof(void*)); + *data = (void*)kv->data.ptr; *sz = sizeof(void*); break; + case PMIX_DATA_ARRAY: + darray = (pmix_data_array_t**)data; + rc = pmix_bfrops_base_copy_darray(darray, kv->data.darray, PMIX_DATA_ARRAY); + *sz = sizeof(pmix_data_array_t); + break; case PMIX_ENVAR: PMIX_ENVAR_CREATE(envar, 1); if (NULL == envar) { @@ -345,6 +368,7 @@ pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p, pmix_value_t *p1) { pmix_value_cmp_t rc = PMIX_VALUE1_GREATER; + int ret; if (p->type != p1->type) { return rc; @@ -441,10 +465,10 @@ pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p, if (NULL == p1->data.envar.envar) { return PMIX_VALUE1_GREATER; } - rc = strcmp(p->data.envar.envar, p1->data.envar.envar); - if (rc < 0) { + ret = strcmp(p->data.envar.envar, p1->data.envar.envar); + if (ret < 0) { return PMIX_VALUE2_GREATER; - } else if (0 < rc) { + } else if (0 < ret) { return PMIX_VALUE1_GREATER; } } else if (NULL != p1->data.envar.envar) { @@ -457,10 +481,10 @@ pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p, if (NULL == p1->data.envar.value) { return PMIX_VALUE1_GREATER; } - rc = strcmp(p->data.envar.value, p1->data.envar.value); - if (rc < 0) { + ret = strcmp(p->data.envar.value, p1->data.envar.value); + if (ret < 0) { return PMIX_VALUE2_GREATER; - } else if (0 < rc) { + } else if (0 < ret) { return PMIX_VALUE1_GREATER; } } else if (NULL != p1->data.envar.value) { @@ -487,20 +511,8 @@ pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p, pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p, pmix_value_t *src) { - size_t n, m; - pmix_status_t rc; - char **prarray, **strarray; - pmix_value_t *pv, *sv; + size_t n; pmix_info_t *p1, *s1; - pmix_app_t *pa, *sa; - pmix_pdata_t *pd, *sd; - pmix_buffer_t *pb, *sb; - pmix_byte_object_t *pbo, *sbo; - pmix_kval_t *pk, *sk; - pmix_modex_data_t *pm, *sm; - pmix_proc_info_t *pi, *si; - pmix_query_t *pq, *sq; - pmix_envar_t *pe, *se; /* copy the right field */ p->type = src->type; @@ -614,411 +626,11 @@ pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p, memcpy(&p->data.state, &src->data.state, sizeof(pmix_proc_state_t)); break; case PMIX_PROC_INFO: - PMIX_PROC_INFO_CREATE(p->data.pinfo, 1); - if (NULL != src->data.pinfo->hostname) { - p->data.pinfo->hostname = strdup(src->data.pinfo->hostname); - } - if (NULL != src->data.pinfo->executable_name) { - p->data.pinfo->executable_name = strdup(src->data.pinfo->executable_name); - } - memcpy(&p->data.pinfo->pid, &src->data.pinfo->pid, sizeof(pid_t)); - memcpy(&p->data.pinfo->exit_code, &src->data.pinfo->exit_code, sizeof(int)); - memcpy(&p->data.pinfo->state, &src->data.pinfo->state, sizeof(pmix_proc_state_t)); - break; + return pmix_bfrops_base_copy_pinfo(&p->data.pinfo, src->data.pinfo, PMIX_PROC_INFO); case PMIX_DATA_ARRAY: - p->data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); - p->data.darray->type = src->data.darray->type; - p->data.darray->size = src->data.darray->size; - if (0 == p->data.darray->size || NULL == src->data.darray->array) { - p->data.darray->array = NULL; - p->data.darray->size = 0; - break; - } - /* allocate space and do the copy */ - switch (src->data.darray->type) { - case PMIX_UINT8: - case PMIX_INT8: - case PMIX_BYTE: - p->data.darray->array = (char*)malloc(src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size); - break; - case PMIX_UINT16: - case PMIX_INT16: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint16_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint16_t)); - break; - case PMIX_UINT32: - case PMIX_INT32: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint32_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint32_t)); - break; - case PMIX_UINT64: - case PMIX_INT64: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint64_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint64_t)); - break; - case PMIX_BOOL: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(bool)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(bool)); - break; - case PMIX_SIZE: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(size_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(size_t)); - break; - case PMIX_PID: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(pid_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pid_t)); - break; - case PMIX_STRING: - p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - prarray = (char**)p->data.darray->array; - strarray = (char**)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != strarray[n]) { - prarray[n] = strdup(strarray[n]); - } - } - break; - case PMIX_INT: - case PMIX_UINT: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(int)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(int)); - break; - case PMIX_FLOAT: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(float)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(float)); - break; - case PMIX_DOUBLE: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(double)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(double)); - break; - case PMIX_TIMEVAL: - p->data.darray->array = (struct timeval*)malloc(src->data.darray->size * sizeof(struct timeval)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(struct timeval)); - break; - case PMIX_TIME: - p->data.darray->array = (time_t*)malloc(src->data.darray->size * sizeof(time_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(time_t)); - break; - case PMIX_STATUS: - p->data.darray->array = (pmix_status_t*)malloc(src->data.darray->size * sizeof(pmix_status_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_status_t)); - break; - case PMIX_VALUE: - PMIX_VALUE_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pv = (pmix_value_t*)p->data.darray->array; - sv = (pmix_value_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (PMIX_SUCCESS != (rc = pmix_value_xfer(&pv[n], &sv[n]))) { - PMIX_VALUE_FREE(pv, src->data.darray->size); - return rc; - } - } - break; - case PMIX_PROC: - PMIX_PROC_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_proc_t)); - break; - case PMIX_APP: - PMIX_APP_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pa = (pmix_app_t*)p->data.darray->array; - sa = (pmix_app_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sa[n].cmd) { - pa[n].cmd = strdup(sa[n].cmd); - } - if (NULL != sa[n].argv) { - pa[n].argv = pmix_argv_copy(sa[n].argv); - } - if (NULL != sa[n].env) { - pa[n].env = pmix_argv_copy(sa[n].env); - } - if (NULL != sa[n].cwd) { - pa[n].cwd = strdup(sa[n].cwd); - } - pa[n].maxprocs = sa[n].maxprocs; - if (0 < sa[n].ninfo && NULL != sa[n].info) { - PMIX_INFO_CREATE(pa[n].info, sa[n].ninfo); - if (NULL == pa[n].info) { - PMIX_APP_FREE(pa, src->data.darray->size); - return PMIX_ERR_NOMEM; - } - pa[n].ninfo = sa[n].ninfo; - for (m=0; m < pa[n].ninfo; m++) { - PMIX_INFO_XFER(&pa[n].info[m], &sa[n].info[m]); - } - } - } - break; - case PMIX_INFO: - PMIX_INFO_CREATE(p->data.darray->array, src->data.darray->size); - p1 = (pmix_info_t*)p->data.darray->array; - s1 = (pmix_info_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_INFO_XFER(&p1[n], &s1[n]); - } - break; - case PMIX_PDATA: - PMIX_PDATA_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pd = (pmix_pdata_t*)p->data.darray->array; - sd = (pmix_pdata_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_PDATA_XFER(&pd[n], &sd[n]); - } - break; - case PMIX_BUFFER: - p->data.darray->array = (pmix_buffer_t*)malloc(src->data.darray->size * sizeof(pmix_buffer_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pb = (pmix_buffer_t*)p->data.darray->array; - sb = (pmix_buffer_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_CONSTRUCT(&pb[n], pmix_buffer_t); - pmix_bfrops_base_copy_payload(&pb[n], &sb[n]); - } - break; - case PMIX_BYTE_OBJECT: - case PMIX_COMPRESSED_STRING: - p->data.darray->array = (pmix_byte_object_t*)malloc(src->data.darray->size * sizeof(pmix_byte_object_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pbo = (pmix_byte_object_t*)p->data.darray->array; - sbo = (pmix_byte_object_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sbo[n].bytes && 0 < sbo[n].size) { - pbo[n].size = sbo[n].size; - pbo[n].bytes = (char*)malloc(pbo[n].size); - memcpy(pbo[n].bytes, sbo[n].bytes, pbo[n].size); - } else { - pbo[n].bytes = NULL; - pbo[n].size = 0; - } - } - break; - case PMIX_KVAL: - p->data.darray->array = (pmix_kval_t*)calloc(src->data.darray->size , sizeof(pmix_kval_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pk = (pmix_kval_t*)p->data.darray->array; - sk = (pmix_kval_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sk[n].key) { - pk[n].key = strdup(sk[n].key); - } - if (NULL != sk[n].value) { - PMIX_VALUE_CREATE(pk[n].value, 1); - if (NULL == pk[n].value) { - free(p->data.darray->array); - return PMIX_ERR_NOMEM; - } - if (PMIX_SUCCESS != (rc = pmix_value_xfer(pk[n].value, sk[n].value))) { - return rc; - } - } - } - break; - case PMIX_MODEX: - PMIX_MODEX_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pm = (pmix_modex_data_t*)p->data.darray->array; - sm = (pmix_modex_data_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - memcpy(&pm[n], &sm[n], sizeof(pmix_modex_data_t)); - if (NULL != sm[n].blob && 0 < sm[n].size) { - pm[n].blob = (uint8_t*)malloc(sm[n].size); - if (NULL == pm[n].blob) { - return PMIX_ERR_NOMEM; - } - memcpy(pm[n].blob, sm[n].blob, sm[n].size); - pm[n].size = sm[n].size; - } else { - pm[n].blob = NULL; - pm[n].size = 0; - } - } - break; - case PMIX_PERSIST: - p->data.darray->array = (pmix_persistence_t*)malloc(src->data.darray->size * sizeof(pmix_persistence_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_persistence_t)); - break; - case PMIX_POINTER: - p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - prarray = (char**)p->data.darray->array; - strarray = (char**)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - prarray[n] = strarray[n]; - } - break; - case PMIX_SCOPE: - p->data.darray->array = (pmix_scope_t*)malloc(src->data.darray->size * sizeof(pmix_scope_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_scope_t)); - break; - case PMIX_DATA_RANGE: - p->data.darray->array = (pmix_data_range_t*)malloc(src->data.darray->size * sizeof(pmix_data_range_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_data_range_t)); - break; - case PMIX_COMMAND: - p->data.darray->array = (pmix_cmd_t*)malloc(src->data.darray->size * sizeof(pmix_cmd_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_cmd_t)); - break; - case PMIX_INFO_DIRECTIVES: - p->data.darray->array = (pmix_info_directives_t*)malloc(src->data.darray->size * sizeof(pmix_info_directives_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_info_directives_t)); - break; - case PMIX_PROC_INFO: - PMIX_PROC_INFO_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pi = (pmix_proc_info_t*)p->data.darray->array; - si = (pmix_proc_info_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - memcpy(&pi[n].proc, &si[n].proc, sizeof(pmix_proc_t)); - if (NULL != si[n].hostname) { - pi[n].hostname = strdup(si[n].hostname); - } else { - pi[n].hostname = NULL; - } - if (NULL != si[n].executable_name) { - pi[n].executable_name = strdup(si[n].executable_name); - } else { - pi[n].executable_name = NULL; - } - pi[n].pid = si[n].pid; - pi[n].exit_code = si[n].exit_code; - pi[n].state = si[n].state; - } - break; - case PMIX_DATA_ARRAY: - PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); - return PMIX_ERR_NOT_SUPPORTED; // don't support iterative arrays - case PMIX_QUERY: - PMIX_QUERY_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pq = (pmix_query_t*)p->data.darray->array; - sq = (pmix_query_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sq[n].keys) { - pq[n].keys = pmix_argv_copy(sq[n].keys); - } - if (NULL != sq[n].qualifiers && 0 < sq[n].nqual) { - PMIX_INFO_CREATE(pq[n].qualifiers, sq[n].nqual); - if (NULL == pq[n].qualifiers) { - PMIX_QUERY_FREE(pq, src->data.darray->size); - return PMIX_ERR_NOMEM; - } - for (m=0; m < sq[n].nqual; m++) { - PMIX_INFO_XFER(&pq[n].qualifiers[m], &sq[n].qualifiers[m]); - } - pq[n].nqual = sq[n].nqual; - } else { - pq[n].qualifiers = NULL; - pq[n].nqual = 0; - } - } - break; - case PMIX_ENVAR: - PMIX_ENVAR_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pe = (pmix_envar_t*)p->data.darray->array; - se = (pmix_envar_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != se[n].envar) { - pe[n].envar = strdup(se[n].envar); - } - if (NULL != se[n].value) { - pe[n].value = strdup(se[n].value); - } - pe[n].separator = se[n].separator; - } - break; - default: - return PMIX_ERR_UNKNOWN_DATA_TYPE; - } - break; + return pmix_bfrops_base_copy_darray(&p->data.darray, src->data.darray, PMIX_DATA_ARRAY); case PMIX_POINTER: - memcpy(&p->data.ptr, &src->data.ptr, sizeof(void*)); + p->data.ptr = src->data.ptr; break; case PMIX_ENVAR: PMIX_ENVAR_CONSTRUCT(&p->data.envar); @@ -1048,8 +660,7 @@ pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p, break; /********************/ default: - pmix_output(0, "XFER-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)src->type); - assert(0); + pmix_output(0, "PMIX-XFER-VALUE: UNSUPPORTED TYPE %d", (int)src->type); return PMIX_ERROR; } return PMIX_SUCCESS; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_frame.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_frame.c index b1c1d5439de..952ca015bb8 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_frame.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_frame.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -90,7 +90,7 @@ static pmix_status_t pmix_bfrop_close(void) pmix_bfrops_globals.initialized = false; /* the components will cleanup when closed */ - PMIX_DESTRUCT(&pmix_bfrops_globals.actives); + PMIX_LIST_DESTRUCT(&pmix_bfrops_globals.actives); return pmix_mca_base_framework_components_close(&pmix_bfrops_base_framework, NULL); } diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_pack.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_pack.c index d5dda0cbb82..2e66dfe57b2 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_pack.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_pack.c @@ -454,6 +454,7 @@ pmix_status_t pmix_bfrops_base_pack_status(pmix_buffer_t *buffer, const void *sr for (i = 0; i < num_vals; ++i) { status = (int32_t)ssrc[i]; if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_int32(buffer, &status, 1, PMIX_INT32))) { + PMIX_ERROR_LOG(ret); return ret; } } @@ -1310,5 +1311,5 @@ pmix_status_t pmix_bfrops_base_pack_envar(pmix_buffer_t *buffer, const void *src return ret; } } - return pmix_bfrops_base_pack_int16(buffer, src, num_vals, PMIX_UINT16); + return PMIX_SUCCESS; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_print.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_print.c index 25129fef47f..c02fce285b1 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_print.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_print.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c index bfda0b8e868..afd685b4374 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -111,6 +111,7 @@ pmix_status_t pmix_bfrops_base_unpack(pmix_pointer_array_t *regtypes, } if (PMIX_INT32 != local_type) { /* if the length wasn't first, then error */ *num_vals = 0; + PMIX_ERROR_LOG(PMIX_ERR_UNPACK_FAILURE); return PMIX_ERR_UNPACK_FAILURE; } } @@ -753,13 +754,13 @@ pmix_status_t pmix_bfrops_base_unpack_val(pmix_buffer_t *buffer, return ret; } break; - case PMIX_QUERY: - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_query(buffer, val->data.darray, &m, PMIX_QUERY))) { + case PMIX_ALLOC_DIRECTIVE: + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_alloc_directive(buffer, &val->data.adir, &m, PMIX_ALLOC_DIRECTIVE))) { return ret; } break; case PMIX_ENVAR: - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_envar(buffer, val->data.darray, &m, PMIX_ENVAR))) { + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_envar(buffer, &val->data.envar, &m, PMIX_ENVAR))) { return ret; } break; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/bfrops.h b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/bfrops.h index 3aeeebfe12e..bfb4013b875 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/bfrops.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/bfrops.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -32,7 +32,7 @@ #define PMIX_BFROP_H_ #include - +#include #include #include "src/mca/mca.h" @@ -100,11 +100,6 @@ BEGIN_C_DECLS * NOTE: THESE FUNCTIONS ARE NOT TO BE USED INTERNALLY - * USE THE MACROS INSTEAD */ -pmix_status_t pmix_value_xfer(pmix_value_t *kv, pmix_value_t *src); -void pmix_value_load(pmix_value_t *v, const void *data, - pmix_data_type_t type); -pmix_status_t pmix_value_unload(pmix_value_t *kv, void **data, - size_t *sz, pmix_data_type_t type); bool pmix_value_cmp(pmix_value_t *p, pmix_value_t *p1); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.c index 1a22226b7fa..20ef97e23c7 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.c @@ -3,6 +3,8 @@ * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2016-2017 Mellanox Technologies, Inc. * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * * $COPYRIGHT$ * @@ -2269,6 +2271,10 @@ static void dstore_finalize(void) PMIX_RELEASE(_clients_peer->nptr); PMIX_RELEASE(_clients_peer); } + /* close the pshmem framework */ + if( PMIX_SUCCESS != (rc = pmix_mca_base_framework_close(&pmix_pshmem_base_framework)) ) { + PMIX_ERROR_LOG(rc); + } } static pmix_status_t _dstore_store(const char *nspace, diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore_component.c index da955113b50..35d984e1b90 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore_component.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore_component.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * Copyright (c) 2017 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -31,7 +31,7 @@ #include #include "pmix_common.h" - +#include "src/include/pmix_globals.h" #include "src/mca/gds/gds.h" #include "gds_dstore.h" @@ -74,6 +74,13 @@ static int component_open(void) static int component_query(pmix_mca_base_module_t **module, int *priority) { + /* launchers cannot use the dstore */ + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + *priority = 0; + *module = NULL; + return PMIX_ERROR; + } + *priority = 20; *module = (pmix_mca_base_module_t *)&pmix_ds12_module; return PMIX_SUCCESS; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/hash/gds_hash.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/hash/gds_hash.c index 95bcec0a089..da9608fb3e5 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/hash/gds_hash.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/hash/gds_hash.c @@ -1,6 +1,8 @@ /* * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * * $COPYRIGHT$ * @@ -135,8 +137,11 @@ static void htdes(pmix_hash_trkr_t *p) if (NULL != p->nptr) { PMIX_RELEASE(p->nptr); } + pmix_hash_remove_data(&p->internal, PMIX_RANK_WILDCARD, NULL); PMIX_DESTRUCT(&p->internal); + pmix_hash_remove_data(&p->remote, PMIX_RANK_WILDCARD, NULL); PMIX_DESTRUCT(&p->remote); + pmix_hash_remove_data(&p->local, PMIX_RANK_WILDCARD, NULL); PMIX_DESTRUCT(&p->local); } static PMIX_CLASS_INSTANCE(pmix_hash_trkr_t, @@ -327,13 +332,14 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, pmix_argv_free(procs); return rc; } - PMIX_RELEASE(kp2); + PMIX_RELEASE(kp2); // maintain acctg } pmix_argv_free(procs); } /* store the comma-delimited list of nodes hosting - * procs in this nspace */ + * procs in this nspace in case someone using PMIx v2 + * requests it */ kp2 = PMIX_NEW(pmix_kval_t); kp2->key = strdup(PMIX_NODE_LIST); kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); @@ -344,6 +350,7 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, PMIX_RELEASE(kp2); return rc; } + PMIX_RELEASE(kp2); // maintain acctg return PMIX_SUCCESS; } @@ -397,6 +404,20 @@ pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, ht = &trk->internal; for (n=0; n < ninfo; n++) { if (0 == strcmp(info[n].key, PMIX_NODE_MAP)) { + /* store the node map itself since that is + * what v3 uses */ + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_NODE_MAP); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_STRING; + kp2->value->data.string = strdup(info[n].value.data.string); + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + /* parse the regex to get the argv array of node names */ if (PMIX_SUCCESS != (rc = pmix_preg.parse_nodes(info[n].value.data.string, &nodes))) { PMIX_ERROR_LOG(rc); @@ -513,6 +534,11 @@ pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, PMIX_RELEASE(kp2); goto release; } + PMIX_RELEASE(kp2); // maintain acctg + /* if this is the job size, then store it */ + if (0 == strncmp(info[n].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN)) { + nptr->nprocs = info[n].value.data.uint32; + } } } @@ -538,6 +564,7 @@ pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, PMIX_RELEASE(kp2); break; } + PMIX_RELEASE(kp2); // maintain acctg } trk->gdata_added = true; } @@ -560,7 +587,6 @@ static pmix_status_t register_info(pmix_peer_t *peer, pmix_hash_table_t *ht; pmix_value_t *val, blob; pmix_status_t rc = PMIX_SUCCESS; - pmix_rank_info_t *rinfo; pmix_info_t *info; size_t ninfo, n; pmix_kval_t kv; @@ -607,9 +633,9 @@ static pmix_status_t register_info(pmix_peer_t *peer, PMIX_VALUE_RELEASE(val); } - PMIX_LIST_FOREACH(rinfo, &ns->ranks, pmix_rank_info_t) { + for (rank=0; rank < ns->nprocs; rank++) { val = NULL; - rc = pmix_hash_fetch(ht, rinfo->pname.rank, NULL, &val); + rc = pmix_hash_fetch(ht, rank, NULL, &val); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); if (NULL != val) { @@ -621,7 +647,6 @@ static pmix_status_t register_info(pmix_peer_t *peer, return PMIX_ERR_NOT_FOUND; } PMIX_CONSTRUCT(&buf, pmix_buffer_t); - rank = rinfo->pname.rank; PMIX_BFROPS_PACK(rc, peer, &buf, &rank, 1, PMIX_PROC_RANK); info = (pmix_info_t*)val->data.darray->array; @@ -658,7 +683,8 @@ static pmix_status_t hash_register_job_info(struct pmix_peer_t *pr, pmix_status_t rc; pmix_hash_trkr_t *trk, *t2; - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { /* this function is only available on servers */ PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); return PMIX_ERR_NOT_SUPPORTED; @@ -758,7 +784,8 @@ static pmix_status_t hash_store_job_info(const char *nspace, "[%s:%u] pmix:gds:hash store job info for nspace %s", pmix_globals.myid.nspace, pmix_globals.myid.rank, nspace); - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { /* this function is NOT available on servers */ PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); return PMIX_ERR_NOT_SUPPORTED; @@ -913,7 +940,7 @@ static pmix_status_t hash_store_job_info(const char *nspace, PMIX_DESTRUCT(&buf2); return rc; } - PMIX_RELEASE(kp2); + PMIX_RELEASE(kp2); // maintain acctg } else { /* nope - so add this by itself */ kp2 = PMIX_NEW(pmix_kval_t); @@ -945,7 +972,7 @@ static pmix_status_t hash_store_job_info(const char *nspace, PMIX_DESTRUCT(&buf2); return rc; } - PMIX_RELEASE(kp2); + PMIX_RELEASE(kp2); // maintain acctg } /* split the list of procs so we can store their * individual location data */ @@ -968,7 +995,7 @@ static pmix_status_t hash_store_job_info(const char *nspace, pmix_argv_free(procs); return rc; } - PMIX_RELEASE(kp2); + PMIX_RELEASE(kp2); // maintain acctg } pmix_argv_free(procs); PMIX_DESTRUCT(&kv); @@ -989,7 +1016,7 @@ static pmix_status_t hash_store_job_info(const char *nspace, PMIX_DESTRUCT(&buf2); return rc; } - PMIX_RELEASE(kp2); + PMIX_RELEASE(kp2); // maintain acctg } /* cleanup */ PMIX_DESTRUCT(&buf2); @@ -1044,10 +1071,10 @@ static pmix_status_t hash_store(const pmix_proc_t *proc, pmix_kval_t *kp; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, - "[%s:%d] gds:hash:hash_store for proc [%s:%d] key %s scope %s", + "[%s:%d] gds:hash:hash_store for proc [%s:%d] key %s type %s scope %s", pmix_globals.myid.nspace, pmix_globals.myid.rank, proc->nspace, proc->rank, kv->key, - PMIx_Scope_string(scope)); + PMIx_Data_type_string(kv->value->type), PMIx_Scope_string(scope)); if (NULL == kv->key) { return PMIX_ERR_BAD_PARAM; @@ -1091,8 +1118,10 @@ static pmix_status_t hash_store(const pmix_proc_t *proc, } if (PMIX_SUCCESS != (rc = pmix_hash_store(&trk->internal, proc->rank, kp))) { PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp); return rc; } + PMIX_RELEASE(kp); // maintain accounting } } @@ -1442,7 +1471,17 @@ static pmix_status_t nspace_add(const char *nspace, static pmix_status_t nspace_del(const char *nspace) { - /* we don't need to do anything here */ + pmix_hash_trkr_t *t; + + /* find the hash table for this nspace */ + PMIX_LIST_FOREACH(t, &myhashes, pmix_hash_trkr_t) { + if (0 == strcmp(nspace, t->ns)) { + /* release it */ + pmix_list_remove_item(&myhashes, &t->super); + PMIX_RELEASE(t); + break; + } + } return PMIX_SUCCESS; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/posix_ipv4/pif_posix.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/posix_ipv4/pif_posix.c index 3c2e4603a66..c338b4f86a1 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/posix_ipv4/pif_posix.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/posix_ipv4/pif_posix.c @@ -4,7 +4,7 @@ * Copyright (c) 2013 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -333,7 +333,8 @@ static int if_posix_open(void) } intf->ifmtu = ifr->ifr_mtu; #endif - + pmix_output_verbose(1, pmix_pif_base_framework.framework_output, + "adding interface %s", intf->if_name); pmix_list_append(&pmix_if_list, &(intf->super)); } free(ifconf.ifc_req); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/Makefile.am new file mode 100644 index 00000000000..3d6a2c0e49b --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/Makefile.am @@ -0,0 +1,44 @@ +# -*- makefile -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +AM_CPPFLAGS = $(LTDLINCL) + +# main library setup +noinst_LTLIBRARIES = libmca_plog.la +libmca_plog_la_SOURCES = + +# local files +headers = plog.h +sources = + +# Conditionally install the header files +if WANT_INSTALL_HEADERS +pmixdir = $(pmixincludedir)/$(subdir) +nobase_pmix_HEADERS = $(headers) +endif + +include base/Makefile.include + +libmca_plog_la_SOURCES += $(headers) $(sources) + +distclean-local: + rm -f base/static-components.h diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/Makefile.include b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/Makefile.include new file mode 100644 index 00000000000..8f0ecf0a4ee --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/Makefile.include @@ -0,0 +1,34 @@ +# -*- makefile -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This makefile.am does not stand on its own - it is included from +# src/Makefile.am + +dist_pmixdata_DATA = base/help-pmix-plog.txt + +headers += \ + base/base.h + +sources += \ + base/plog_base_frame.c \ + base/plog_base_select.c \ + base/plog_base_stubs.c diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/base.h b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/base.h new file mode 100644 index 00000000000..6c865731eff --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/base.h @@ -0,0 +1,93 @@ +/* -*- C -*- + * + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ +#ifndef PMIX_PLOG_BASE_H_ +#define PMIX_PLOG_BASE_H_ + +#include + + +#ifdef HAVE_SYS_TIME_H +#include /* for struct timeval */ +#endif +#ifdef HAVE_STRING_H +#include +#endif + +#include "src/class/pmix_list.h" +#include "src/class/pmix_pointer_array.h" +#include "src/threads/threads.h" +#include "src/mca/mca.h" +#include "src/mca/base/pmix_mca_base_framework.h" + +#include "src/mca/plog/plog.h" + + +BEGIN_C_DECLS + +/* + * MCA Framework + */ +PMIX_EXPORT extern pmix_mca_base_framework_t pmix_plog_base_framework; +/** + * PLOG select function + * + * Cycle across available components and construct the array + * of active modules + */ +PMIX_EXPORT pmix_status_t pmix_plog_base_select(void); + +/** + * Track an active component / module + */ +struct pmix_plog_base_active_module_t { + pmix_list_item_t super; + bool reqd; + bool added; + int pri; + pmix_plog_module_t *module; + pmix_plog_base_component_t *component; +}; +typedef struct pmix_plog_base_active_module_t pmix_plog_base_active_module_t; +PMIX_CLASS_DECLARATION(pmix_plog_base_active_module_t); + + +/* framework globals */ +struct pmix_plog_globals_t { + pmix_lock_t lock; + pmix_pointer_array_t actives; + bool initialized; + char **channels; +}; +typedef struct pmix_plog_globals_t pmix_plog_globals_t; + +PMIX_EXPORT extern pmix_plog_globals_t pmix_plog_globals; + +PMIX_EXPORT pmix_status_t pmix_plog_base_log(const pmix_proc_t *source, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/help-pmix-plog.txt b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/help-pmix-plog.txt new file mode 100644 index 00000000000..211b4521756 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/help-pmix-plog.txt @@ -0,0 +1,56 @@ +# -*- text -*- +# +# Copyright (c) 2018 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# +# This is a US/English help file +# +[reqd-not-found] +The plog_base_order MCA parameter included a required logging +channel that is not available: + + Channel: %s + +Please update the parameter and try again. +# +[syslog:unrec-level] +An unrecognized syslog level was given: + + Level: %s + +Please see "man syslog" for a list of defined levels. Input +parameter strings and their corresponding syslog levels +recognized by PMIx include: + + Parameter Level + err LOG_ERR (default) + alert LOG_ALERT + crit LOG_CRIT + emerg LOG_EMERG + warn LOG_WARNING + not LOG_NOTICE + info LOG_INFO + debug LOG_DEBUG + +Please redefine the MCA parameter and try again. +# +[syslog:unrec-facility] +An unsupported or unrecognized value was given for the +syslog facility (i.e., the type of program calling syslog): + + Value: %s + +Please see "man syslog" for a list of defined facility values. +PMIx currently supports only the following designations: + + Parameter Level + auth LOG_AUTH + priv LOG_AUTHPRIV + daemon LOG_DAEMON + user LOG_USER (default) + +Please redefine the MCA parameter and try again. diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_frame.c b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_frame.c new file mode 100644 index 00000000000..906b8ccaa72 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_frame.c @@ -0,0 +1,105 @@ +/* -*- Mode: C; c-basic-offset:4 ; -*- */ +/* + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include + +#include + +#ifdef HAVE_STRING_H +#include +#endif + +#include "src/class/pmix_list.h" +#include "src/threads/threads.h" +#include "src/util/argv.h" +#include "src/mca/base/base.h" +#include "src/mca/plog/base/base.h" + +/* + * The following file was created by configure. It contains extern + * statements and the definition of an array of pointers to each + * component's public mca_base_component_t struct. + */ + +#include "src/mca/plog/base/static-components.h" + +/* Instantiate the global vars */ +pmix_plog_globals_t pmix_plog_globals = {{0}}; +pmix_plog_API_module_t pmix_plog = { + .log = pmix_plog_base_log +}; + +static char *order = NULL; +static int pmix_plog_register(pmix_mca_base_register_flag_t flags) +{ + pmix_mca_base_var_register("pmix", "plog", "base", "order", + "Comma-delimited, prioritized list of logging channels", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &order); + if (NULL != order) { + pmix_plog_globals.channels = pmix_argv_split(order, ','); + } + return PMIX_SUCCESS; +} + +static pmix_status_t pmix_plog_close(void) +{ + pmix_plog_base_active_module_t *active; + int n; + + if (!pmix_plog_globals.initialized) { + return PMIX_SUCCESS; + } + pmix_plog_globals.initialized = false; + + for (n=0; n < pmix_plog_globals.actives.size; n++) { + if (NULL == (active = (pmix_plog_base_active_module_t*)pmix_pointer_array_get_item(&pmix_plog_globals.actives, n))) { + continue; + } + if (NULL != active->module->finalize) { + active->module->finalize(); + } + PMIX_RELEASE(active); + pmix_pointer_array_set_item(&pmix_plog_globals.actives, n, NULL); + } + PMIX_DESTRUCT(&pmix_plog_globals.actives); + + PMIX_DESTRUCT_LOCK(&pmix_plog_globals.lock); + + return pmix_mca_base_framework_components_close(&pmix_plog_base_framework, NULL); +} + +static pmix_status_t pmix_plog_open(pmix_mca_base_open_flag_t flags) +{ + /* initialize globals */ + pmix_plog_globals.initialized = true; + pmix_plog_globals.channels = NULL; + PMIX_CONSTRUCT(&pmix_plog_globals.actives, pmix_pointer_array_t); + pmix_pointer_array_init(&pmix_plog_globals.actives, 1, INT_MAX, 1); + PMIX_CONSTRUCT_LOCK(&pmix_plog_globals.lock); + pmix_plog_globals.lock.active = false; + + /* Open up all available components */ + return pmix_mca_base_framework_components_open(&pmix_plog_base_framework, flags); +} + +PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, plog, "PMIx Logging Operations", + pmix_plog_register, pmix_plog_open, pmix_plog_close, + mca_plog_base_static_components, 0); + +static void acon(pmix_plog_base_active_module_t *p) +{ + p->reqd = false; + p->added = false; +} +PMIX_CLASS_INSTANCE(pmix_plog_base_active_module_t, + pmix_list_item_t, + acon, NULL); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_select.c b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_select.c new file mode 100644 index 00000000000..fe947938e4f --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_select.c @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include + +#include + +#include "src/mca/mca.h" +#include "src/mca/base/base.h" +#include "src/util/show_help.h" + +#include "src/mca/plog/base/base.h" + +static bool selected = false; + +/* Function for selecting a prioritized array of components + * from all those that are available. */ +int pmix_plog_base_select(void) +{ + pmix_mca_base_component_list_item_t *cli = NULL; + pmix_mca_base_component_t *component = NULL; + pmix_mca_base_module_t *module = NULL; + pmix_plog_module_t *nmodule; + pmix_plog_base_active_module_t *newmodule, *mod, *default_mod = NULL; + int rc, priority, n; + bool inserted, default_added, reqd; + pmix_list_t actives; + char *ptr; + size_t len; + + if (selected) { + /* ensure we don't do this twice */ + return PMIX_SUCCESS; + } + selected = true; + + PMIX_CONSTRUCT(&actives, pmix_list_t); + + /* Query all available components and ask if they have a module */ + PMIX_LIST_FOREACH(cli, &pmix_plog_base_framework.framework_components, pmix_mca_base_component_list_item_t) { + component = (pmix_mca_base_component_t *) cli->cli_component; + + pmix_output_verbose(5, pmix_plog_base_framework.framework_output, + "mca:plog:select: checking available component %s", component->pmix_mca_component_name); + + /* If there's no query function, skip it */ + if (NULL == component->pmix_mca_query_component) { + pmix_output_verbose(5, pmix_plog_base_framework.framework_output, + "mca:plog:select: Skipping component [%s]. It does not implement a query function", + component->pmix_mca_component_name ); + continue; + } + + /* Query the component */ + pmix_output_verbose(5, pmix_plog_base_framework.framework_output, + "mca:plog:select: Querying component [%s]", + component->pmix_mca_component_name); + rc = component->pmix_mca_query_component(&module, &priority); + + /* If no module was returned, then skip component */ + if (PMIX_SUCCESS != rc || NULL == module) { + pmix_output_verbose(5, pmix_plog_base_framework.framework_output, + "mca:plog:select: Skipping component [%s]. Query failed to return a module", + component->pmix_mca_component_name ); + continue; + } + + /* If we got a module, keep it */ + nmodule = (pmix_plog_module_t*) module; + /* let it initialize */ + if (NULL != nmodule->init && PMIX_SUCCESS != nmodule->init()) { + continue; + } + /* add to the list of selected modules */ + newmodule = PMIX_NEW(pmix_plog_base_active_module_t); + newmodule->pri = priority; + newmodule->module = nmodule; + newmodule->component = (pmix_plog_base_component_t*)cli->cli_component; + + /* maintain priority order */ + inserted = false; + PMIX_LIST_FOREACH(mod, &actives, pmix_plog_base_active_module_t) { + if (priority > mod->pri) { + pmix_list_insert_pos(&actives, (pmix_list_item_t*)mod, &newmodule->super); + inserted = true; + break; + } + } + if (!inserted) { + /* must be lowest priority - add to end */ + pmix_list_append(&actives, &newmodule->super); + } + + /* if this is the default module, track it */ + if (0 == strcmp(newmodule->module->name, "default")) { + default_mod = newmodule; + } + } + + /* if they gave us a desired ordering, then impose it here */ + if (NULL != pmix_plog_globals.channels) { + default_added = false; + for (n=0; NULL != pmix_plog_globals.channels[n]; n++) { + len = strlen(pmix_plog_globals.channels[n]); + /* check for the "req" modifier */ + reqd = false; + ptr = strrchr(pmix_plog_globals.channels[n], ':'); + if (NULL != ptr) { + /* get the length of the remaining string so we + * can constrain our comparison of the channel + * name itself */ + len = len - strlen(ptr); + /* move over the ':' */ + ++ptr; + /* we accept anything that starts with "req" */ + if (0 == strncasecmp(ptr, "req", 3)) { + reqd = true; + } + } + /* now search for this channel in our list of actives */ + inserted = false; + PMIX_LIST_FOREACH(mod, &actives, pmix_plog_base_active_module_t) { + if (0 == strncasecmp(pmix_plog_globals.channels[n], mod->module->name, len)) { + pmix_list_remove_item(&actives, &mod->super); + pmix_pointer_array_add(&pmix_plog_globals.actives, mod); + mod->reqd = reqd; + inserted = true; + break; + } + } + if (!inserted) { + /* we didn't find a supporting module - this + * still might be okay because it could be something + * the RM itself supports, so just insert the default + * module here if it hasn't already been inserted */ + if (!default_added) { + /* if the default module isn't available and this + * channel isn't optional, then there is nothing + * we can do except report an error */ + if (NULL == default_mod && reqd) { + pmix_show_help("help-pmix-plog.txt", "reqd-not-found", + true, pmix_plog_globals.channels[n]); + PMIX_LIST_DESTRUCT(&actives); + return PMIX_ERR_NOT_FOUND; + } else if (NULL != default_mod) { + pmix_pointer_array_add(&pmix_plog_globals.actives, default_mod); + default_added = true; + default_mod->reqd = reqd; + } + } else if (reqd) { + /* if we already added it, we still have to check the + * reqd status - if any citation requires that the + * default be used, then we set it, but be sure we + * don't overwrite it with a "not required" if it + * was already set as "required" */ + default_mod->reqd = reqd; + } + } + } + /* if there are any modules left over, we need to discard them */ + PMIX_LIST_DESTRUCT(&actives); + } else { + /* insert the modules into the global array in priority order */ + while (NULL != (mod = (pmix_plog_base_active_module_t*)pmix_list_remove_first(&actives))) { + pmix_pointer_array_add(&pmix_plog_globals.actives, mod); + } + PMIX_DESTRUCT(&actives); + } + + if (4 < pmix_output_get_verbosity(pmix_plog_base_framework.framework_output)) { + pmix_output(0, "Final plog order"); + /* show the prioritized order */ + for (n=0; n < pmix_plog_globals.actives.size; n++) { + if (NULL != (mod = (pmix_plog_base_active_module_t*)pmix_pointer_array_get_item(&pmix_plog_globals.actives, n))) { + pmix_output(0, "\tplog[%d]: %s", n, mod->component->base.pmix_mca_component_name); + } + } + } + + + return PMIX_SUCCESS;; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_stubs.c b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_stubs.c new file mode 100644 index 00000000000..d7f0317582d --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_stubs.c @@ -0,0 +1,252 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#include "src/include/pmix_globals.h" + +#include "src/class/pmix_list.h" +#include "src/util/error.h" +#include "src/server/pmix_server_ops.h" + +#include "src/mca/plog/base/base.h" + +typedef struct { + pmix_object_t super; + pmix_lock_t lock; + size_t nreqs; + pmix_status_t status; + pmix_op_cbfunc_t cbfunc; + void *cbdata; +} pmix_mycount_t; +static void mycon(pmix_mycount_t *p) +{ + PMIX_CONSTRUCT_LOCK(&p->lock); + p->lock.active = false; + p->nreqs = 0; + p->status = PMIX_ERR_NOT_AVAILABLE; + p->cbfunc = NULL; + p->cbdata = NULL; +} +static void mydes(pmix_mycount_t *p) +{ + PMIX_DESTRUCT_LOCK(&p->lock); +} +static PMIX_CLASS_INSTANCE(pmix_mycount_t, + pmix_object_t, + mycon, mydes); + +static void localcbfunc(pmix_status_t status, void *cbdata) +{ + pmix_mycount_t *mycount = (pmix_mycount_t*)cbdata; + + PMIX_ACQUIRE_THREAD(&mycount->lock); + mycount->nreqs--; + if (PMIX_SUCCESS != status && PMIX_SUCCESS == mycount->status) { + mycount->status = status; + } + if (0 == mycount->nreqs) { + /* execute their callback */ + if (NULL != mycount->cbfunc) { + mycount->cbfunc(mycount->status, mycount->cbdata); + } + PMIX_RELEASE(mycount); + return; + } + PMIX_RELEASE_THREAD(&mycount->lock); +} + +pmix_status_t pmix_plog_base_log(const pmix_proc_t *source, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_plog_base_active_module_t *active; + pmix_status_t rc = PMIX_ERR_NOT_AVAILABLE; + size_t n, k; + int m; + bool logonce = false; + pmix_mycount_t *mycount; + pmix_list_t channels; + bool all_complete = true; + + if (!pmix_plog_globals.initialized) { + return PMIX_ERR_INIT; + } + + /* we have to serialize our way thru here as we are going + * to construct a list of the available modules, and those + * can only be on one list at a time */ + PMIX_ACQUIRE_THREAD(&pmix_plog_globals.lock); + + pmix_output_verbose(2, pmix_plog_base_framework.framework_output, + "plog:log called"); + + /* initialize the tracker */ + mycount = PMIX_NEW(pmix_mycount_t); + if (NULL == mycount) { + PMIX_RELEASE_THREAD(&pmix_plog_globals.lock); + return PMIX_ERR_NOMEM; + } + mycount->cbfunc = cbfunc; + mycount->cbdata = cbdata; + /* initialize the list of channels */ + PMIX_CONSTRUCT(&channels, pmix_list_t); + + if (NULL != directives) { + /* scan the directives for the PMIX_LOG_ONCE attribute + * which indicates we should stop with the first log + * channel that can successfully handle this request, + * and any channel directives */ + for (n=0; n < ndirs; n++) { + if (0 == strncmp(directives[n].key, PMIX_LOG_ONCE, PMIX_MAX_KEYLEN)) { + logonce = true; + break; + } + } + } + + /* scan the incoming logging requests and assemble the modules in + * the corresponding order - this will ensure that the one they + * requested first gets first shot at "log once" */ + for (n=0; n < ndata; n++) { + if (PMIX_INFO_OP_IS_COMPLETE(&data[n])) { + continue; + } + all_complete = false; + for (m=0; m < pmix_plog_globals.actives.size; m++) { + if (NULL == (active = (pmix_plog_base_active_module_t*)pmix_pointer_array_get_item(&pmix_plog_globals.actives, m))) { + continue; + } + /* if this channel is included in the ones serviced by this + * module, then include the module */ + if (NULL == active->module->channels) { + if (!active->added) { + /* add this channel to the list */ + pmix_list_append(&channels, &active->super); + /* mark it as added */ + active->added = true; + } + } else { + for (k=0; NULL != active->module->channels[k]; k++) { + if (NULL != strstr(data[n].key, active->module->channels[k])) { + if (!active->added) { + /* add this channel to the list */ + pmix_list_append(&channels, &active->super); + /* mark it as added */ + active->added = true; + break; + } + } + } + } + } + } + /* reset the added marker for the next time we are called */ + PMIX_LIST_FOREACH(active, &channels, pmix_plog_base_active_module_t) { + active->added = false; + } + if (all_complete) { + /* nothing we need do */ + while (NULL != pmix_list_remove_first(&channels)); + PMIX_DESTRUCT(&channels); + PMIX_RELEASE(mycount); + PMIX_RELEASE_THREAD(&pmix_plog_globals.lock); + return PMIX_SUCCESS; + } + PMIX_ACQUIRE_THREAD(&mycount->lock); + PMIX_LIST_FOREACH(active, &channels, pmix_plog_base_active_module_t) { + if (NULL != active->module->log) { + mycount->nreqs++; + rc = active->module->log(source, data, ndata, directives, ndirs, + localcbfunc, (void*)mycount); + /* The plugins are required to return: + * + * PMIX_SUCCESS - indicating that the logging operation for + * that component was very quick, and therefore + * done atomically. No callback will be issued + * + * PMIX_OPERATION_IN_PROGRESS - indicates that the plugin + * expects to execute the desired logging request, + * but must do so asynchronously. The provided + * callback _must_ be executed upon completion + * of the operation, indicating success or failure. + * + * PMIX_ERR_NOT_AVAILABLE - indicates that the plugin is unable + * to process the request. + * No callback will be issued. + * + * PMIX_ERR_TAKE_NEXT_OPTION - indicates that the plugin didn't + * find any directives that it supports. + * No callback will be issued. + * + * PMIX_ERR_NOT_SUPPORTED - indicates that the request cannot be + * supported. The list will cease processing at + * that point and return this error + * + * All other returned errors indicate that the plugin should + * have attempted to perform the requested operation, but determined + * that it could not do so. Note that this differs from the case + * where a plugin asynchronously attempts an operation that subsequently + * fails - that error would be returned in the callback function. + * In this case, the error indicates that the request contained + * an incorrect/invalid element that prevents the plugin from + * executing it. The first such retured error will be cached and + * returned to the caller upon completion of all pending operations. + * No callback from failed plugins shall be executed. + */ + if (PMIX_SUCCESS == rc) { + mycount->nreqs--; + mycount->status = rc; + if (logonce) { + break; + } + } else if (PMIX_ERR_NOT_AVAILABLE == rc || + PMIX_ERR_TAKE_NEXT_OPTION == rc) { + mycount->nreqs--; + } else if (PMIX_OPERATION_IN_PROGRESS == rc) { + /* even though the operation hasn't completed, + * we still treat this as a completed request */ + mycount->status = PMIX_SUCCESS; + if (logonce) { + break; + } + } else { + /* we may have outstanding requests we need + * to wait for, so mark that there was an error + * for reporting purposes */ + mycount->nreqs--; + mycount->status = rc; + } + } + } + + /* cannot release the modules - just remove everything from the list */ + while (NULL != pmix_list_remove_first(&channels)); + PMIX_DESTRUCT(&channels); + + rc = mycount->status; // save the status as it could change when the lock is released + if (0 == mycount->nreqs) { + /* execute their callback */ + if (NULL != mycount->cbfunc) { + mycount->cbfunc(mycount->status, mycount->cbdata); + } + PMIX_RELEASE_THREAD(&mycount->lock); + PMIX_RELEASE(mycount); + PMIX_RELEASE_THREAD(&pmix_plog_globals.lock); + return PMIX_SUCCESS; + } + PMIX_RELEASE_THREAD(&mycount->lock); + PMIX_RELEASE_THREAD(&pmix_plog_globals.lock); + + return rc; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/Makefile.am new file mode 100644 index 00000000000..aa141f9d8ff --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/Makefile.am @@ -0,0 +1,46 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2014-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2017 IBM Corporation. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +sources = \ + plog_default.h \ + plog_default.c \ + plog_default_component.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_pmix_plog_default_DSO +component_noinst = +component_install = mca_plog_default.la +else +component_noinst = libmca_plog_default.la +component_install = +endif + +mcacomponentdir = $(pmixlibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_plog_default_la_SOURCES = $(sources) +mca_plog_default_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(component_noinst) +libmca_plog_default_la_SOURCES =$(sources) +libmca_plog_default_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default.c b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default.c new file mode 100644 index 00000000000..ac4cf4fec0d --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default.c @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "pmix_config.h" +#include "pmix_common.h" + +#include +#ifdef HAVE_SYS_TIME_H +#include +#endif /* HAVE_SYS_TIME_H */ +#include + +#include "src/include/pmix_globals.h" +#include "src/util/show_help.h" +#include "src/util/error.h" +#include "src/server/pmix_server_ops.h" + +#include "src/mca/plog/base/base.h" +#include "plog_default.h" + + +/* Static API's */ +static int init(void); +static pmix_status_t mylog(const pmix_proc_t *source, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +/* Module def */ +pmix_plog_module_t pmix_plog_default_module = { + .name = "default", + .channels = NULL, + .init = init, + .finalize = NULL, + .log = mylog +}; + +/* local object */ +typedef struct { + pmix_object_t super; + pmix_info_t *data; + size_t ndata; + pmix_op_cbfunc_t cbfunc; + void *cbdata; +} local_caddy_t; +static void lcon(local_caddy_t *p) +{ + p->data = NULL; + p->ndata = 0; +} +static void ldes(local_caddy_t *p) +{ + if (NULL != p->data) { + PMIX_INFO_FREE(p->data, p->ndata); + } +} +static PMIX_CLASS_INSTANCE(local_caddy_t, + pmix_object_t, + lcon, ldes); + + +static int init(void) +{ + /* we cannot operate if our host doesn't support log */ + if (NULL == pmix_host_server.log) { + return PMIX_ERR_NOT_AVAILABLE; + } + return PMIX_SUCCESS; +} + +static void localcbfn(pmix_status_t status, void *cbdata) +{ + local_caddy_t *cd = (local_caddy_t*)cbdata; + + if (NULL != cd->cbfunc) { + cd->cbfunc(status, cd->cbdata); + } + PMIX_RELEASE(cd); +} + +static pmix_status_t mylog(const pmix_proc_t *source, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + local_caddy_t *cd; + size_t ntodo, n; + + /* if none of the prior modules performed a requested logging + * operation, then we will try here */ + ntodo = 0; + for (n=0; n < ndata; n++) { + if (!PMIX_INFO_OP_IS_COMPLETE(&data[n])) { + ++ntodo; + } + } + if (0 == ntodo) { + return PMIX_ERR_TAKE_NEXT_OPTION; + } + + /* send it upwards for potential handling. This might seem + * odd in the case where we are a gateway, but we must allow + * for the possibility that the host has a channel we don't + * directly support */ + cd = PMIX_NEW(local_caddy_t); + if (NULL == cd) { + return PMIX_ERR_NOMEM; + } + cd->cbfunc = cbfunc; + cd->cbdata = cbdata; + + /* separate out the ones that weren't completed */ + PMIX_INFO_CREATE(cd->data, ntodo); + if (NULL == cd->data) { + PMIX_RELEASE(cd); + return PMIX_ERR_NOMEM; + } + cd->ndata = ntodo; + ntodo = 0; + for (n=0; n < ndata; n++) { + if (!PMIX_INFO_OP_IS_COMPLETE(&data[n])) { + PMIX_INFO_XFER(&cd->data[ntodo], (pmix_info_t*)&data[n]); + ++ntodo; + } + } + + /* ask the host to log the remainder */ + pmix_host_server.log(source, cd->data, cd->ndata, + directives, ndirs, + localcbfn, (void*)cd); + + return PMIX_OPERATION_IN_PROGRESS; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default.h b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default.h new file mode 100644 index 00000000000..f2e435db52f --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default.h @@ -0,0 +1,40 @@ +/* -*- C -*- + * + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ +#ifndef PLOG_DEFAULT_H +#define PLOG_DEFAULT_H + +#include "pmix_config.h" + +#include "src/mca/plog/plog.h" + +BEGIN_C_DECLS + +/* + * Plog interfaces + */ + +PMIX_EXPORT extern pmix_plog_base_component_t mca_plog_default_component; +extern pmix_plog_module_t pmix_plog_default_module; + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default_component.c new file mode 100644 index 00000000000..c475e7c697a --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default_component.c @@ -0,0 +1,47 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ +*/ + +/* + * includes + */ +#include "pmix_config.h" +#include "pmix_common.h" + +#include "plog_default.h" + + +static pmix_status_t component_query(pmix_mca_base_module_t **module, + int *priority); + +/* + * Struct of function pointers that need to be initialized + */ +pmix_plog_base_component_t mca_plog_default_component = { + .base = { + PMIX_PLOG_BASE_VERSION_1_0_0, + + .pmix_mca_component_name = "default", + PMIX_MCA_BASE_MAKE_VERSION(component, PMIX_MAJOR_VERSION, PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + .pmix_mca_query_component = component_query + }, + .data = { + /* The component is checkpoint ready */ + PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT + }, +}; + +static pmix_status_t component_query(pmix_mca_base_module_t **module, + int *priority) +{ + *priority = 1; + *module = (pmix_mca_base_module_t *)&pmix_plog_default_module; + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/plog.h b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/plog.h new file mode 100644 index 00000000000..3c1ba6defb0 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/plog.h @@ -0,0 +1,102 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/** + * @file + * + * This interface is for use by PMIx servers to obtain network-related info + * such as security keys that need to be shared across applications, and to + * setup network support for applications prior to launch + * + * Available plugins may be defined at runtime via the typical MCA parameter + * syntax. + */ + +#ifndef PMIX_PLOG_H +#define PMIX_PLOG_H + +#include +#include "pmix_common.h" + +#include "src/class/pmix_list.h" +#include "src/mca/mca.h" +#include "src/mca/base/pmix_mca_base_var.h" +#include "src/mca/base/pmix_mca_base_framework.h" +#include "src/include/pmix_globals.h" + +BEGIN_C_DECLS + +/****** MODULE DEFINITION ******/ + +/** + * Initialize the module. Returns an error if the module cannot + * run, success if it can and wants to be used. + */ +typedef pmix_status_t (*pmix_plog_base_module_init_fn_t)(void); + + +/** + * Finalize the module + */ +typedef void (*pmix_plog_base_module_fini_fn_t)(void); + +/** + * Log data to channel, if possible + */ +typedef pmix_status_t (*pmix_plog_base_module_log_fn_t)(const pmix_proc_t *source, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +/** + * Base structure for a PLOG module + */ +typedef struct { + char *name; + char **channels; + /* init/finalize */ + pmix_plog_base_module_init_fn_t init; + pmix_plog_base_module_fini_fn_t finalize; + pmix_plog_base_module_log_fn_t log; +} pmix_plog_module_t; + +/** + * Base structure for a PLOG API + */ +typedef struct { + pmix_plog_base_module_log_fn_t log; +} pmix_plog_API_module_t; + + +/* declare the global APIs */ +PMIX_EXPORT extern pmix_plog_API_module_t pmix_plog; + +/* + * the standard component data structure + */ +struct pmix_plog_base_component_t { + pmix_mca_base_component_t base; + pmix_mca_base_component_data_t data; +}; +typedef struct pmix_plog_base_component_t pmix_plog_base_component_t; + +/* + * Macro for use in components that are of type plog + */ +#define PMIX_PLOG_BASE_VERSION_1_0_0 \ + PMIX_MCA_BASE_VERSION_1_0_0("plog", 1, 0, 0) + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/Makefile.am new file mode 100644 index 00000000000..497dfaaf1a7 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/Makefile.am @@ -0,0 +1,46 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2014-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2017 IBM Corporation. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +sources = \ + plog_stdfd.h \ + plog_stdfd.c \ + plog_stdfd_component.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_pmix_plog_stdfd_DSO +component_noinst = +component_install = mca_plog_stdfd.la +else +component_noinst = libmca_plog_stdfd.la +component_install = +endif + +mcacomponentdir = $(pmixlibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_plog_stdfd_la_SOURCES = $(sources) +mca_plog_stdfd_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(component_noinst) +libmca_plog_stdfd_la_SOURCES =$(sources) +libmca_plog_stdfd_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.c b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.c new file mode 100644 index 00000000000..e6ed5a60ce3 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.c @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "pmix_config.h" +#include "pmix_common.h" + +#include +#ifdef HAVE_SYS_TIME_H +#include +#endif /* HAVE_SYS_TIME_H */ +#include + +#include "src/util/argv.h" +#include "src/util/error.h" +#include "src/util/name_fns.h" +#include "src/util/show_help.h" +#include "src/common/pmix_iof.h" + +#include "src/mca/plog/base/base.h" +#include "plog_stdfd.h" + + +/* Static API's */ +static int init(void); +static void finalize(void); +static pmix_status_t mylog(const pmix_proc_t *source, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +/* Module def */ +pmix_plog_module_t pmix_plog_stdfd_module = { + .name = "stdfd", + .init = init, + .finalize = finalize, + .log = mylog +}; + + +static int init(void) +{ + char *mychannels = "stdout,stderr"; + + pmix_plog_stdfd_module.channels = pmix_argv_split(mychannels, ','); + return PMIX_SUCCESS; +} + +static void finalize(void) +{ + pmix_argv_free(pmix_plog_stdfd_module.channels); +} + +static pmix_status_t mylog(const pmix_proc_t *source, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + size_t n; + pmix_status_t rc; + pmix_byte_object_t bo; + pmix_iof_flags_t flags= {0}; + + /* if there is no data, then we don't handle it */ + if (NULL == data || 0 == ndata) { + return PMIX_ERR_NOT_AVAILABLE; + } + + /* if we are not a gateway, then we don't handle this */ + if (!PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { + return PMIX_ERR_TAKE_NEXT_OPTION; + } + + /* check to see if there are any relevant directives */ + for (n=0; n < ndirs; n++) { + if (0 == strncmp(directives[n].key, PMIX_LOG_TIMESTAMP, PMIX_MAX_KEYLEN)) { + flags.timestamp = data[n].value.data.time; + } else if (0 == strncmp(directives[n].key, PMIX_LOG_XML_OUTPUT, PMIX_MAX_KEYLEN)) { + flags.xml = PMIX_INFO_TRUE(&directives[n]); + } else if (0 == strncmp(directives[n].key, PMIX_LOG_TAG_OUTPUT, PMIX_MAX_KEYLEN)) { + flags.tag = PMIX_INFO_TRUE(&directives[n]); + } + } + + /* check to see if there are any stdfd entries */ + rc = PMIX_ERR_TAKE_NEXT_OPTION; + for (n=0; n < ndata; n++) { + if (0 == strncmp(data[n].key, PMIX_LOG_STDERR, PMIX_MAX_KEYLEN)) { + bo.bytes = data[n].value.data.string; + bo.size = strlen(bo.bytes); + pmix_iof_write_output(source, PMIX_FWD_STDERR_CHANNEL, &bo, &flags); + /* flag that we did this one */ + PMIX_INFO_OP_COMPLETED(&data[n]); + rc = PMIX_SUCCESS; + } else if (0 == strncmp(data[n].key, PMIX_LOG_STDOUT, PMIX_MAX_KEYLEN)) { + bo.bytes = data[n].value.data.string; + bo.size = strlen(bo.bytes); + pmix_iof_write_output(source, PMIX_FWD_STDOUT_CHANNEL, &bo, &flags); + /* flag that we did this one */ + PMIX_INFO_OP_COMPLETED(&data[n]); + rc = PMIX_SUCCESS; + } + } + + return rc; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.h b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.h new file mode 100644 index 00000000000..b11048fa40f --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.h @@ -0,0 +1,40 @@ +/* -*- C -*- + * + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ +#ifndef PLOG_STDFD_H +#define PLOG_STDFD_H + +#include "pmix_config.h" + +#include "src/mca/plog/plog.h" + +BEGIN_C_DECLS + +/* + * Plog interfaces + */ + +PMIX_EXPORT extern pmix_plog_base_component_t mca_plog_stdfd_component; +extern pmix_plog_module_t pmix_plog_stdfd_module; + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd_component.c new file mode 100644 index 00000000000..9fb69c18c33 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd_component.c @@ -0,0 +1,47 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ +*/ + +/* + * includes + */ +#include "pmix_config.h" +#include "pmix_common.h" + +#include "plog_stdfd.h" + + +static pmix_status_t component_query(pmix_mca_base_module_t **module, + int *priority); + +/* + * Struct of function pointers that need to be initialized + */ +pmix_plog_base_component_t mca_plog_stdfd_component = { + .base = { + PMIX_PLOG_BASE_VERSION_1_0_0, + + .pmix_mca_component_name = "stdfd", + PMIX_MCA_BASE_MAKE_VERSION(component, PMIX_MAJOR_VERSION, PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + .pmix_mca_query_component = component_query, + }, + .data = { + /* The component is checkpoint ready */ + PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT + }, +}; + +static pmix_status_t component_query(pmix_mca_base_module_t **module, + int *priority) +{ + *priority = 5; + *module = (pmix_mca_base_module_t *)&pmix_plog_stdfd_module; + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/Makefile.am new file mode 100644 index 00000000000..7a09d28fac1 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/Makefile.am @@ -0,0 +1,46 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2014-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2017 IBM Corporation. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +sources = \ + plog_syslog.h \ + plog_syslog.c \ + plog_syslog_component.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_pmix_plog_syslog_DSO +component_noinst = +component_install = mca_plog_syslog.la +else +component_noinst = libmca_plog_syslog.la +component_install = +endif + +mcacomponentdir = $(pmixlibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_plog_syslog_la_SOURCES = $(sources) +mca_plog_syslog_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(component_noinst) +libmca_plog_syslog_la_SOURCES =$(sources) +libmca_plog_syslog_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/configure.m4 b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/configure.m4 new file mode 100644 index 00000000000..9a649b49354 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/configure.m4 @@ -0,0 +1,30 @@ +# -*- shell-script -*- +# +# Copyright (c) 2017 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2017-2018 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# MCA_plog_syslog_CONFIG([action-if-found], [action-if-not-found]) +# ----------------------------------------------------------- +AC_DEFUN([MCA_pmix_plog_syslog_CONFIG], [ + AC_CONFIG_FILES([src/mca/plog/syslog/Makefile]) + + PMIX_VAR_SCOPE_PUSH([pmix_plog_syslog_happy]) + + # if syslog.h is not compilable, + # disable this component. + AC_CHECK_HEADER([syslog.h], + [pmix_plog_syslog_happy=1], + [pmix_plog_syslog_happy=0]) + + AS_IF([test $pmix_plog_syslog_happy -eq 1], + [$1], + [$2]) + + PMIX_VAR_SCOPE_POP +])dnl diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog.c b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog.c new file mode 100644 index 00000000000..64edceed18c --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog.c @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "pmix_config.h" +#include "pmix_common.h" + +#include +#ifdef HAVE_TIME_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif /* HAVE_SYS_TIME_H */ +#ifdef HAVE_SYSLOG_H +#include +#endif +#include + +#include "src/util/argv.h" +#include "src/util/error.h" +#include "src/util/name_fns.h" +#include "src/util/show_help.h" +#include "src/mca/bfrops/bfrops.h" +#include "src/server/pmix_server_ops.h" + +#include "src/mca/plog/base/base.h" +#include "plog_syslog.h" + + +/* Static API's */ +static pmix_status_t init(void); +static void finalize(void); +static pmix_status_t mylog(const pmix_proc_t *source, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +/* Module def */ +pmix_plog_module_t pmix_plog_syslog_module = { + .name = "syslog", + .init = init, + .finalize = finalize, + .log = mylog +}; + + +static pmix_status_t init(void) +{ + int opts; + char *mychannels = "lsys,gsys,syslog,local_syslog,global_syslog"; + + pmix_plog_syslog_module.channels = pmix_argv_split(mychannels, ','); + + opts = LOG_CONS | LOG_PID; + openlog("PMIx Log Report:", opts, LOG_USER); + + return PMIX_SUCCESS; +} + +static void finalize(void) +{ + closelog(); + pmix_argv_free(pmix_plog_syslog_module.channels); +} + +static pmix_status_t write_local(const pmix_proc_t *source, + time_t timestamp, + int severity, char *msg, + const pmix_info_t *data, size_t ndata); + +/* we only get called if we are a SERVER */ +static pmix_status_t mylog(const pmix_proc_t *source, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + size_t n; + int pri = mca_plog_syslog_component.level; + pmix_status_t rc; + time_t timestamp = 0; + + /* if there is no data, then we don't handle it */ + if (NULL == data || 0 == ndata) { + return PMIX_ERR_NOT_AVAILABLE; + } + + /* check directives */ + if (NULL != directives) { + for (n=0; n < ndirs; n++) { + if (0 == strncmp(directives[n].key, PMIX_LOG_SYSLOG_PRI, PMIX_MAX_KEYLEN)) { + pri = directives[n].value.data.integer; + } else if (0 == strncmp(directives[n].key, PMIX_LOG_TIMESTAMP, PMIX_MAX_KEYLEN)) { + timestamp = directives[n].value.data.time; + } + } + } + + /* check to see if there are any syslog entries */ + for (n=0; n < ndata; n++) { + if (0 == strncmp(data[n].key, PMIX_LOG_SYSLOG, PMIX_MAX_KEYLEN)) { + /* we default to using the local syslog */ + rc = write_local(source, timestamp, pri, data[n].value.data.string, data, ndata); + if (PMIX_SUCCESS == rc) { + /* flag that we did this one */ + PMIX_INFO_OP_COMPLETED(&data[n]); + } + } else if (0 == strncmp(data[n].key, PMIX_LOG_LOCAL_SYSLOG, PMIX_MAX_KEYLEN)) { + rc = write_local(source, timestamp, pri, data[n].value.data.string, data, ndata); + if (PMIX_SUCCESS == rc) { + /* flag that we did this one */ + PMIX_INFO_OP_COMPLETED(&data[n]); + } + } else if (0 == strncmp(data[n].key, PMIX_LOG_GLOBAL_SYSLOG, PMIX_MAX_KEYLEN)) { + /* only do this if we are a gateway server */ + if (PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { + rc = write_local(source, timestamp, pri, data[n].value.data.string, data, ndata); + if (PMIX_SUCCESS == rc) { + /* flag that we did this one */ + PMIX_INFO_OP_COMPLETED(&data[n]); + } + } + } + } + + return PMIX_SUCCESS; +} + +static char* sev2str(int severity) +{ + switch (severity) { + case LOG_EMERG: + return "EMERGENCY"; + case LOG_ALERT: + return "ALERT"; + case LOG_CRIT: + return "CRITICAL"; + case LOG_ERR: + return "ERROR"; + case LOG_WARNING: + return "WARNING"; + case LOG_NOTICE: + return "NOTICE"; + case LOG_INFO: + return "INFO"; + case LOG_DEBUG: + return "DEBUG"; + default: + return "UNKNOWN SEVERITY"; + } +} + +static pmix_status_t write_local(const pmix_proc_t *source, + time_t timestamp, + int severity, char *msg, + const pmix_info_t *data, size_t ndata) +{ + char tod[48], *datastr, *tmp, *tmp2; + pmix_status_t rc; + size_t n; + + pmix_output_verbose(5, pmix_plog_base_framework.framework_output, + "plog:syslog:mylog function called with severity %d", severity); + + if (0 < timestamp) { + /* If there was a message, output it */ + (void)ctime_r(×tamp, tod); + /* trim the newline */ + tod[strlen(tod)] = '\0'; + } + + if (NULL == data) { + syslog(severity, "%s [%s:%d]%s PROC %s:%d REPORTS: %s", + tod, pmix_globals.myid.nspace, pmix_globals.myid.rank, + sev2str(severity), + source->nspace, source->rank, + (NULL == msg) ? "" : msg); + } else { + /* need to print the info from the data, starting + * with any provided msg */ + if (NULL == msg) { + datastr = strdup("\n"); + } else { + if (0 > asprintf(&datastr, "%s", msg)) { + return PMIX_ERR_NOMEM; + } + } + for (n=0; n < ndata; n++) { + PMIX_BFROPS_PRINT(rc, pmix_globals.mypeer, + &tmp, "\t", (pmix_info_t*)&data[n], PMIX_INFO); + if (PMIX_SUCCESS != rc) { + free(datastr); + return rc; + } + if (0 > asprintf(&tmp2, "%s\n%s", datastr, tmp)) { + free(datastr); + return PMIX_ERR_NOMEM; + } + free(datastr); + free(tmp); + datastr = tmp2; + } + /* print out the consolidated msg */ + syslog(severity, "%s [%s:%d]%s PROC %s:%d REPORTS: %s", + tod, pmix_globals.myid.nspace, pmix_globals.myid.rank, + sev2str(severity), source->nspace, source->rank, datastr); + free(datastr); + } + + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog.h b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog.h new file mode 100644 index 00000000000..3a73324384d --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog.h @@ -0,0 +1,47 @@ +/* -*- C -*- + * + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ +#ifndef PLOG_SYSLOG_H +#define PLOG_SYSLOG_H + +#include "pmix_config.h" + +#include "src/mca/plog/plog.h" + +BEGIN_C_DECLS + +/* + * Plog interfaces + */ + +typedef struct { + pmix_plog_base_component_t super; + int console; + int level; + int facility; +} pmix_plog_syslog_component_t; + +PMIX_EXPORT extern pmix_plog_syslog_component_t mca_plog_syslog_component; +extern pmix_plog_module_t pmix_plog_syslog_module; + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog_component.c new file mode 100644 index 00000000000..814c15d0dbc --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog_component.c @@ -0,0 +1,124 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ +*/ + +/* + * includes + */ +#include "pmix_config.h" +#include "pmix_common.h" + +#ifdef HAVE_SYSLOG_H +#include +#endif + +#include "src/util/show_help.h" +#include "plog_syslog.h" + + +static pmix_status_t component_query(pmix_mca_base_module_t **module, + int *priority); +static pmix_status_t syslog_register(void); + +/* + * Struct of function pointers that need to be initialized + */ +pmix_plog_syslog_component_t mca_plog_syslog_component = { + .super = { + .base = { + PMIX_PLOG_BASE_VERSION_1_0_0, + + .pmix_mca_component_name = "syslog", + PMIX_MCA_BASE_MAKE_VERSION(component, PMIX_MAJOR_VERSION, PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + .pmix_mca_query_component = component_query, + .pmix_mca_register_component_params = syslog_register, + }, + .data = { + /* The component is checkpoint ready */ + PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + }, + .console = false, + .level = LOG_ERR, + .facility = LOG_USER +}; + +static char *level = "info"; +static char *facility = "user"; + +static pmix_status_t syslog_register(void) +{ + pmix_status_t rc = PMIX_SUCCESS; + + (void) pmix_mca_base_component_var_register(&mca_plog_syslog_component.super.base, "console", + "Write directly to system console if there is an error while sending to system logger", + PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_plog_syslog_component.console); + + (void) pmix_mca_base_component_var_register(&mca_plog_syslog_component.super.base, "level", + "Default syslog logging level (err, alert, crit, emerg, warning, notice, info[default], or debug)", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &level); + if (0 == strncasecmp(level, "err", 3)) { + mca_plog_syslog_component.level = LOG_ERR; + } else if (0 == strcasecmp(level, "alert")) { + mca_plog_syslog_component.level = LOG_ALERT; + } else if (0 == strncasecmp(level, "crit", 4)) { + mca_plog_syslog_component.level = LOG_CRIT; + } else if (0 == strncasecmp(level, "emerg", 5)) { + mca_plog_syslog_component.level = LOG_EMERG; + } else if (0 == strncasecmp(level, "warn", 4)) { + mca_plog_syslog_component.level = LOG_WARNING; + } else if (0 == strncasecmp(level, "not", 3)) { + mca_plog_syslog_component.level = LOG_NOTICE; + } else if (0 == strcasecmp(level, "info")) { + mca_plog_syslog_component.level = LOG_INFO; + } else if (0 == strcasecmp(level, "debug") || 0 == strcasecmp(level, "dbg")) { + mca_plog_syslog_component.level = LOG_DEBUG; + } else { + pmix_show_help("help-pmix-plog.txt", "syslog:unrec-level", true, level); + rc = PMIX_ERR_NOT_SUPPORTED; + } + + (void) pmix_mca_base_component_var_register(&mca_plog_syslog_component.super.base, "facility", + "Specify what type of program is logging the message " + "(only \"auth\", \"priv\", \"daemon\", and \"user\" are supported)", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &facility); + if (0 == strncasecmp(facility, "auth", 4)) { + mca_plog_syslog_component.facility = LOG_AUTH; + } else if (0 == strncasecmp(facility, "priv", 4)) { + mca_plog_syslog_component.facility = LOG_AUTHPRIV; + } else if (0 == strcasecmp(facility, "daemon")) { + mca_plog_syslog_component.facility = LOG_DAEMON; + } else if (0 == strcasecmp(facility, "user")) { + mca_plog_syslog_component.facility = LOG_USER; + } else { + pmix_show_help("help-pmix-plog.txt", "syslog:unrec-facility", true, facility); + rc = PMIX_ERR_NOT_SUPPORTED; + } + + return rc; +} + + +static pmix_status_t component_query(pmix_mca_base_module_t **module, + int *priority) +{ + *priority = 10; + *module = (pmix_mca_base_module_t *)&pmix_plog_syslog_module; + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/base.h b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/base.h index 24b9349a349..9b4f58fdaf1 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/base.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/base.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -66,25 +66,69 @@ struct pmix_pnet_base_active_module_t { pmix_pnet_base_component_t *component; }; typedef struct pmix_pnet_base_active_module_t pmix_pnet_base_active_module_t; -PMIX_CLASS_DECLARATION(pmix_pnet_base_active_module_t); +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pnet_base_active_module_t); +typedef struct { + pmix_list_item_t super; + char *nspace; + pmix_rank_t *ranks; + size_t np; +} pmix_pnet_local_procs_t; +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pnet_local_procs_t); + +typedef struct { + pmix_list_item_t super; + char *name; + pmix_list_t resources; +}pmix_pnet_resource_t; +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pnet_resource_t); + +typedef struct { + pmix_list_item_t super; + size_t index; + char *name; + pmix_list_t local_jobs; // list of pmix_pnet_local_procs_t + pmix_list_t resources; // list of pmix_pnet_resource_t +} pmix_pnet_node_t; +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pnet_node_t); + +typedef struct { + pmix_list_item_t super; + char *nspace; + pmix_pointer_array_t nodes; +} pmix_pnet_job_t; +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pnet_job_t); /* framework globals */ struct pmix_pnet_globals_t { - pmix_list_t actives; - bool initialized; + pmix_lock_t lock; + pmix_list_t actives; + bool initialized; + pmix_list_t jobs; + pmix_list_t nodes; }; typedef struct pmix_pnet_globals_t pmix_pnet_globals_t; PMIX_EXPORT extern pmix_pnet_globals_t pmix_pnet_globals; -PMIX_EXPORT pmix_status_t pmix_pnet_base_setup_app(char *nspace, pmix_list_t *ilist); +PMIX_EXPORT pmix_status_t pmix_pnet_base_allocate(char *nspace, + pmix_info_t info[], size_t ninfo, + pmix_list_t *ilist); PMIX_EXPORT pmix_status_t pmix_pnet_base_setup_local_network(char *nspace, pmix_info_t info[], size_t ninfo); PMIX_EXPORT pmix_status_t pmix_pnet_base_setup_fork(const pmix_proc_t *peer, char ***env); -PMIX_EXPORT void pmix_pnet_base_child_finalized(pmix_peer_t *peer); -PMIX_EXPORT void pmix_pnet_base_local_app_finalized(char *nspace); +PMIX_EXPORT void pmix_pnet_base_child_finalized(pmix_proc_t *peer); +PMIX_EXPORT void pmix_pnet_base_local_app_finalized(pmix_nspace_t *nptr); +PMIX_EXPORT void pmix_pnet_base_deregister_nspace(char *nspace); +PMIX_EXPORT void pmix_pnet_base_collect_inventory(pmix_info_t directives[], size_t ndirs, + pmix_inventory_cbfunc_t cbfunc, + void *cbdata); +PMIX_EXPORT void pmix_pnet_base_deliver_inventory(pmix_info_t info[], size_t ninfo, + pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); +PMIX_EXPORT pmix_status_t pmix_pnet_base_harvest_envars(char **incvars, char **excvars, + pmix_list_t *ilist); END_C_DECLS diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/pnet_base_fns.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/pnet_base_fns.c index 3572fdf8e82..22ea10829e3 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/pnet_base_fns.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/pnet_base_fns.c @@ -1,8 +1,10 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * * $COPYRIGHT$ * @@ -17,29 +19,114 @@ #include "src/include/pmix_globals.h" #include "src/class/pmix_list.h" +#include "src/mca/preg/preg.h" +#include "src/util/argv.h" #include "src/util/error.h" +#include "src/util/pmix_environ.h" +#include "src/server/pmix_server_ops.h" #include "src/mca/pnet/base/base.h" -pmix_status_t pmix_pnet_base_setup_app(char *nspace, pmix_list_t *ilist) +static pmix_status_t process_maps(char *nspace, char *nregex, char *pregex); + +/* NOTE: a tool (e.g., prun) may call this function to + * harvest local envars for inclusion in a call to + * PMIx_Spawn, or it might be called in response to + * a call to PMIx_Allocate_resources */ +pmix_status_t pmix_pnet_base_allocate(char *nspace, + pmix_info_t info[], size_t ninfo, + pmix_list_t *ilist) { pmix_pnet_base_active_module_t *active; pmix_status_t rc; + pmix_nspace_t *nptr, *ns; + size_t n; + char *nregex, *pregex; if (!pmix_pnet_globals.initialized) { return PMIX_ERR_INIT; } + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:allocate called"); + /* protect against bozo inputs */ if (NULL == nspace || NULL == ilist) { return PMIX_ERR_BAD_PARAM; } + if (PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { + nptr = NULL; + /* find this nspace - note that it may not have + * been registered yet */ + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + if (0 == strcmp(ns->nspace, nspace)) { + nptr = ns; + break; + } + } + if (NULL == nptr) { + /* add it */ + nptr = PMIX_NEW(pmix_nspace_t); + if (NULL == nptr) { + return PMIX_ERR_NOMEM; + } + nptr->nspace = strdup(nspace); + pmix_list_append(&pmix_server_globals.nspaces, &nptr->super); + } - PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { - if (NULL != active->module->setup_app) { - if (PMIX_SUCCESS != (rc = active->module->setup_app(nspace, ilist))) { - return rc; + /* if the info param is NULL, then we make one pass thru the actives + * in case someone specified an allocation or collection of envars + * via MCA param */ + if (NULL == info) { + PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { + if (NULL != active->module->allocate) { + if (PMIX_SUCCESS == (rc = active->module->allocate(nptr, NULL, ilist))) { + break; + } + if (PMIX_ERR_TAKE_NEXT_OPTION != rc) { + /* true error */ + return rc; + } + } + } + } else { + /* check for description of the node and proc maps */ + nregex = NULL; + pregex = NULL; + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_NODE_MAP, PMIX_MAX_KEYLEN)) { + nregex = info[n].value.data.string; + } else if (0 == strncmp(info[n].key, PMIX_PROC_MAP, PMIX_MAX_KEYLEN)) { + pregex = info[n].value.data.string; + } + } + if (NULL != nregex && NULL != pregex) { + /* assemble the pnet node and proc descriptions + * NOTE: this will eventually be folded into the + * new shared memory system, but we do it here + * as the pnet plugins need the information and + * the host will not have registered the clients + * and nspace prior to calling allocate + */ + rc = process_maps(nspace, nregex, pregex); + if (PMIX_SUCCESS != rc) { + return rc; + } + } + /* process the allocation request */ + for (n=0; n < ninfo; n++) { + PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { + if (NULL != active->module->allocate) { + if (PMIX_SUCCESS == (rc = active->module->allocate(nptr, &info[n], ilist))) { + break; + } + if (PMIX_ERR_TAKE_NEXT_OPTION != rc) { + /* true error */ + return rc; + } + } + } } } } @@ -47,25 +134,48 @@ pmix_status_t pmix_pnet_base_setup_app(char *nspace, pmix_list_t *ilist) return PMIX_SUCCESS; } +/* can only be called by a server */ pmix_status_t pmix_pnet_base_setup_local_network(char *nspace, pmix_info_t info[], size_t ninfo) { pmix_pnet_base_active_module_t *active; pmix_status_t rc; + pmix_nspace_t *nptr, *ns; if (!pmix_pnet_globals.initialized) { return PMIX_ERR_INIT; } + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet: setup_local_network called"); + /* protect against bozo inputs */ if (NULL == nspace) { return PMIX_ERR_BAD_PARAM; } + /* find this proc's nspace object */ + nptr = NULL; + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + if (0 == strcmp(ns->nspace, nspace)) { + nptr = ns; + break; + } + } + if (NULL == nptr) { + /* add it */ + nptr = PMIX_NEW(pmix_nspace_t); + if (NULL == nptr) { + return PMIX_ERR_NOMEM; + } + nptr->nspace = strdup(nspace); + pmix_list_append(&pmix_server_globals.nspaces, &nptr->super); + } + PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { if (NULL != active->module->setup_local_network) { - if (PMIX_SUCCESS != (rc = active->module->setup_local_network(nspace, info, ninfo))) { + if (PMIX_SUCCESS != (rc = active->module->setup_local_network(nptr, info, ninfo))) { return rc; } } @@ -74,23 +184,43 @@ pmix_status_t pmix_pnet_base_setup_local_network(char *nspace, return PMIX_SUCCESS; } -pmix_status_t pmix_pnet_base_setup_fork(const pmix_proc_t *peer, char ***env) +/* can only be called by a server */ +pmix_status_t pmix_pnet_base_setup_fork(const pmix_proc_t *proc, char ***env) { pmix_pnet_base_active_module_t *active; pmix_status_t rc; + pmix_nspace_t *nptr, *ns; if (!pmix_pnet_globals.initialized) { return PMIX_ERR_INIT; } /* protect against bozo inputs */ - if (NULL == peer || NULL == env) { + if (NULL == proc || NULL == env) { return PMIX_ERR_BAD_PARAM; } + /* find this proc's nspace object */ + nptr = NULL; + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + if (0 == strcmp(ns->nspace, proc->nspace)) { + nptr = ns; + break; + } + } + if (NULL == nptr) { + /* add it */ + nptr = PMIX_NEW(pmix_nspace_t); + if (NULL == nptr) { + return PMIX_ERR_NOMEM; + } + nptr->nspace = strdup(proc->nspace); + pmix_list_append(&pmix_server_globals.nspaces, &nptr->super); + } + PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { if (NULL != active->module->setup_fork) { - if (PMIX_SUCCESS != (rc = active->module->setup_fork(peer, env))) { + if (PMIX_SUCCESS != (rc = active->module->setup_fork(nptr, proc, env))) { return rc; } } @@ -99,7 +229,7 @@ pmix_status_t pmix_pnet_base_setup_fork(const pmix_proc_t *peer, char ***env) return PMIX_SUCCESS; } -void pmix_pnet_base_child_finalized(pmix_peer_t *peer) +void pmix_pnet_base_child_finalized(pmix_proc_t *peer) { pmix_pnet_base_active_module_t *active; @@ -122,7 +252,7 @@ void pmix_pnet_base_child_finalized(pmix_peer_t *peer) return; } -void pmix_pnet_base_local_app_finalized(char *nspace) +void pmix_pnet_base_local_app_finalized(pmix_nspace_t *nptr) { pmix_pnet_base_active_module_t *active; @@ -131,16 +261,432 @@ void pmix_pnet_base_local_app_finalized(char *nspace) } /* protect against bozo inputs */ - if (NULL == nspace) { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + if (NULL == nptr) { return; } PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { if (NULL != active->module->local_app_finalized) { - active->module->local_app_finalized(nspace); + active->module->local_app_finalized(nptr); + } + } + + return; +} + +void pmix_pnet_base_deregister_nspace(char *nspace) +{ + pmix_pnet_base_active_module_t *active; + pmix_nspace_t *nptr, *ns; + + if (!pmix_pnet_globals.initialized) { + return; + } + + /* protect against bozo inputs */ + if (NULL == nspace) { + return; + } + + /* find this nspace object */ + nptr = NULL; + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + if (0 == strcmp(ns->nspace, nspace)) { + nptr = ns; + break; + } + } + if (NULL == nptr) { + /* nothing we can do */ + return; + } + + PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { + if (NULL != active->module->deregister_nspace) { + active->module->deregister_nspace(nptr); + } + } + + return; +} + +static void cicbfunc(pmix_status_t status, + pmix_list_t *inventory, + void *cbdata) +{ + pmix_inventory_rollup_t *rollup = (pmix_inventory_rollup_t*)cbdata; + pmix_kval_t *kv; + + PMIX_ACQUIRE_THREAD(&rollup->lock); + /* check if they had an error */ + if (PMIX_SUCCESS != status && PMIX_SUCCESS == rollup->status) { + rollup->status = status; + } + /* transfer the inventory */ + if (NULL != inventory) { + while (NULL != (kv = (pmix_kval_t*)pmix_list_remove_first(inventory))) { + pmix_list_append(&rollup->payload, &kv->super); + } + } + /* record that we got a reply */ + rollup->replies++; + /* see if all have replied */ + if (rollup->replies < rollup->requests) { + /* nope - need to wait */ + PMIX_RELEASE_THREAD(&rollup->lock); + return; + } + + /* if we get here, then collection is complete */ + PMIX_RELEASE_THREAD(&rollup->lock); + if (NULL != rollup->cbfunc) { + rollup->cbfunc(rollup->status, &rollup->payload, rollup->cbdata); + } + PMIX_RELEASE(rollup); + return; +} + +void pmix_pnet_base_collect_inventory(pmix_info_t directives[], size_t ndirs, + pmix_inventory_cbfunc_t cbfunc, void *cbdata) +{ + pmix_pnet_base_active_module_t *active; + pmix_inventory_rollup_t *myrollup; + pmix_status_t rc; + + /* we cannot block here as each plugin could take some time to + * complete the request. So instead, we call each active plugin + * and get their immediate response - if "in progress", then + * we record that we have to wait for their answer before providing + * the caller with a response. If "error", then we know we + * won't be getting a response from them */ + + if (!pmix_pnet_globals.initialized) { + /* need to call them back so they know */ + if (NULL != cbfunc) { + cbfunc(PMIX_ERR_INIT, NULL, cbdata); + } + return; + } + /* create the rollup object */ + myrollup = PMIX_NEW(pmix_inventory_rollup_t); + if (NULL == myrollup) { + /* need to call them back so they know */ + if (NULL != cbfunc) { + cbfunc(PMIX_ERR_NOMEM, NULL, cbdata); + } + return; + } + myrollup->cbfunc = cbfunc; + myrollup->cbdata = cbdata; + + /* hold the lock until all active modules have been called + * to avoid race condition where replies come in before + * the requests counter has been fully updated */ + PMIX_ACQUIRE_THREAD(&myrollup->lock); + + PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { + if (NULL != active->module->collect_inventory) { + pmix_output_verbose(5, pmix_pnet_base_framework.framework_output, + "COLLECTING %s", active->module->name); + rc = active->module->collect_inventory(directives, ndirs, cicbfunc, (void*)myrollup); + /* if they return success, then the values were + * placed directly on the payload - nothing + * to wait for here */ + if (PMIX_OPERATION_IN_PROGRESS == rc) { + myrollup->requests++; + } else if (PMIX_SUCCESS != rc && + PMIX_ERR_TAKE_NEXT_OPTION != rc && + PMIX_ERR_NOT_SUPPORTED != rc) { + /* a true error - we need to wait for + * all pending requests to complete + * and then notify the caller of the error */ + if (PMIX_SUCCESS == myrollup->status) { + myrollup->status = rc; + } + } + } + } + if (0 == myrollup->requests) { + /* report back */ + PMIX_RELEASE_THREAD(&myrollup->lock); + if (NULL != cbfunc) { + cbfunc(myrollup->status, &myrollup->payload, cbdata); + } + PMIX_RELEASE(myrollup); + return; + } + + PMIX_RELEASE_THREAD(&myrollup->lock); + return; +} + +static void dlcbfunc(pmix_status_t status, + void *cbdata) +{ + pmix_inventory_rollup_t *rollup = (pmix_inventory_rollup_t*)cbdata; + + PMIX_ACQUIRE_THREAD(&rollup->lock); + /* check if they had an error */ + if (PMIX_SUCCESS != status && PMIX_SUCCESS == rollup->status) { + rollup->status = status; + } + /* record that we got a reply */ + rollup->replies++; + /* see if all have replied */ + if (rollup->replies < rollup->requests) { + /* nope - need to wait */ + PMIX_RELEASE_THREAD(&rollup->lock); + return; + } + + /* if we get here, then delivery is complete */ + PMIX_RELEASE_THREAD(&rollup->lock); + if (NULL != rollup->opcbfunc) { + rollup->opcbfunc(rollup->status, rollup->cbdata); + } + PMIX_RELEASE(rollup); + return; +} + +void pmix_pnet_base_deliver_inventory(pmix_info_t info[], size_t ninfo, + pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_pnet_base_active_module_t *active; + pmix_inventory_rollup_t *myrollup; + pmix_status_t rc; + + /* we cannot block here as each plugin could take some time to + * complete the request. So instead, we call each active plugin + * and get their immediate response - if "in progress", then + * we record that we have to wait for their answer before providing + * the caller with a response. If "error", then we know we + * won't be getting a response from them */ + + if (!pmix_pnet_globals.initialized) { + /* need to call them back so they know */ + if (NULL != cbfunc) { + cbfunc(PMIX_ERR_INIT, cbdata); + } + return; + } + /* create the rollup object */ + myrollup = PMIX_NEW(pmix_inventory_rollup_t); + if (NULL == myrollup) { + /* need to call them back so they know */ + if (NULL != cbfunc) { + cbfunc(PMIX_ERR_NOMEM, cbdata); } + return; + } + myrollup->opcbfunc = cbfunc; + myrollup->cbdata = cbdata; + + /* hold the lock until all active modules have been called + * to avoid race condition where replies come in before + * the requests counter has been fully updated */ + PMIX_ACQUIRE_THREAD(&myrollup->lock); + + PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { + if (NULL != active->module->deliver_inventory) { + pmix_output_verbose(5, pmix_pnet_base_framework.framework_output, + "DELIVERING TO %s", active->module->name); + rc = active->module->deliver_inventory(info, ninfo, directives, ndirs, dlcbfunc, (void*)myrollup); + /* if they return success, then the values were + * immediately archived - nothing to wait for here */ + if (PMIX_OPERATION_IN_PROGRESS == rc) { + myrollup->requests++; + } else if (PMIX_SUCCESS != rc && + PMIX_ERR_TAKE_NEXT_OPTION != rc && + PMIX_ERR_NOT_SUPPORTED != rc) { + /* a true error - we need to wait for + * all pending requests to complete + * and then notify the caller of the error */ + if (PMIX_SUCCESS == myrollup->status) { + myrollup->status = rc; + } + } + } + } + if (0 == myrollup->requests) { + /* report back */ + PMIX_RELEASE_THREAD(&myrollup->lock); + if (NULL != cbfunc) { + cbfunc(myrollup->status, cbdata); + } + PMIX_RELEASE(myrollup); + return; } + PMIX_RELEASE_THREAD(&myrollup->lock); return; } + +pmix_status_t pmix_pnet_base_harvest_envars(char **incvars, char **excvars, + pmix_list_t *ilist) +{ + int i, j; + size_t len; + pmix_kval_t *kv, *next; + char *cs_env, *string_key; + + /* harvest envars to pass along */ + for (j=0; NULL != incvars[j]; j++) { + len = strlen(incvars[j]); + if ('*' == incvars[j][len-1]) { + --len; + } + for (i = 0; NULL != environ[i]; ++i) { + if (0 == strncmp(environ[i], incvars[j], len)) { + cs_env = strdup(environ[i]); + kv = PMIX_NEW(pmix_kval_t); + if (NULL == kv) { + free(cs_env); + return PMIX_ERR_OUT_OF_RESOURCE; + } + kv->key = strdup(PMIX_SET_ENVAR); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + free(cs_env); + return PMIX_ERR_OUT_OF_RESOURCE; + } + kv->value->type = PMIX_ENVAR; + string_key = strchr(cs_env, '='); + if (NULL == string_key) { + free(cs_env); + PMIX_RELEASE(kv); + return PMIX_ERR_BAD_PARAM; + } + *string_key = '\0'; + ++string_key; + PMIX_ENVAR_LOAD(&kv->value->data.envar, cs_env, string_key, ':'); + pmix_list_append(ilist, &kv->super); + free(cs_env); + } + } + } + + /* now check the exclusions and remove any that match */ + if (NULL != excvars) { + for (j=0; NULL != excvars[j]; j++) { + len = strlen(excvars[j]); + if ('*' == excvars[j][len-1]) { + --len; + } + PMIX_LIST_FOREACH_SAFE(kv, next, ilist, pmix_kval_t) { + if (0 == strncmp(kv->value->data.envar.envar, excvars[j], len)) { + pmix_list_remove_item(ilist, &kv->super); + PMIX_RELEASE(kv); + } + } + } + } + return PMIX_SUCCESS; +} + +static pmix_status_t process_maps(char *nspace, char *nregex, char *pregex) +{ + char **nodes, **procs, **ranks; + pmix_status_t rc; + size_t m, n; + pmix_pnet_job_t *jptr, *job; + pmix_pnet_node_t *nd, *ndptr; + pmix_pnet_local_procs_t *lp; + bool needcheck; + + PMIX_ACQUIRE_THREAD(&pmix_pnet_globals.lock); + + /* parse the regex to get the argv array of node names */ + if (PMIX_SUCCESS != (rc = pmix_preg.parse_nodes(nregex, &nodes))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock); + return rc; + } + + /* parse the regex to get the argv array of proc ranks on each node */ + if (PMIX_SUCCESS != (rc = pmix_preg.parse_procs(pregex, &procs))) { + PMIX_ERROR_LOG(rc); + pmix_argv_free(nodes); + PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock); + return rc; + } + + /* see if we already know about this job */ + job = NULL; + if (0 < pmix_list_get_size(&pmix_pnet_globals.jobs)) { + PMIX_LIST_FOREACH(jptr, &pmix_pnet_globals.jobs, pmix_pnet_job_t) { + if (0 == strcmp(nspace, jptr->nspace)) { + job = jptr; + break; + } + } + } + if (NULL == job) { + job = PMIX_NEW(pmix_pnet_job_t); + job->nspace = strdup(nspace); + pmix_list_append(&pmix_pnet_globals.jobs, &job->super); + } + + if (0 < pmix_list_get_size(&pmix_pnet_globals.nodes)) { + needcheck = true; + } else { + needcheck = false; + } + for (n=0; NULL != nodes[n]; n++) { + if (needcheck) { + /* check and see if we already have data for this node */ + nd = NULL; + PMIX_LIST_FOREACH(ndptr, &pmix_pnet_globals.nodes, pmix_pnet_node_t) { + if (0 == strcmp(nodes[n], ndptr->name)) { + nd = ndptr; + break; + } + } + if (NULL == nd) { + nd = PMIX_NEW(pmix_pnet_node_t); + nd->name = strdup(nodes[n]); + pmix_list_append(&pmix_pnet_globals.nodes, &nd->super); + /* add this node to the job */ + PMIX_RETAIN(nd); + nd->index = pmix_pointer_array_add(&job->nodes, nd); + } + } else { + nd = PMIX_NEW(pmix_pnet_node_t); + nd->name = strdup(nodes[n]); + pmix_list_append(&pmix_pnet_globals.nodes, &nd->super); + /* add this node to the job */ + PMIX_RETAIN(nd); + nd->index = pmix_pointer_array_add(&job->nodes, nd); + } + /* check and see if we already have this job on this node */ + PMIX_LIST_FOREACH(lp, &nd->local_jobs, pmix_pnet_local_procs_t) { + if (0 == strcmp(nspace, lp->nspace)) { + /* we assume that the input replaces the prior + * list of ranks */ + pmix_list_remove_item(&nd->local_jobs, &lp->super); + PMIX_RELEASE(lp); + break; + } + } + /* track the local procs */ + lp = PMIX_NEW(pmix_pnet_local_procs_t); + lp->nspace = strdup(nspace); + /* separate out the procs - they are a comma-delimited list + * of rank values */ + ranks = pmix_argv_split(procs[n], ','); + lp->np = pmix_argv_count(ranks); + lp->ranks = (pmix_rank_t*)malloc(lp->np * sizeof(pmix_rank_t)); + for (m=0; m < lp->np; m++) { + lp->ranks[m] = strtoul(ranks[m], NULL, 10); + } + pmix_list_append(&nd->local_jobs, &lp->super); + pmix_argv_free(ranks); + } + + pmix_argv_free(nodes); + pmix_argv_free(procs); + + PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock); + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/pnet_base_frame.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/pnet_base_frame.c index 0dd5410daef..eb1e2f9b52a 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/pnet_base_frame.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/pnet_base_frame.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -44,13 +44,16 @@ #include "src/mca/pnet/base/static-components.h" /* Instantiate the global vars */ -pmix_pnet_globals_t pmix_pnet_globals = {{{0}}}; -pmix_pnet_module_t pmix_pnet = { - .setup_app = pmix_pnet_base_setup_app, +pmix_pnet_globals_t pmix_pnet_globals = {{0}}; +pmix_pnet_API_module_t pmix_pnet = { + .allocate = pmix_pnet_base_allocate, .setup_local_network = pmix_pnet_base_setup_local_network, .setup_fork = pmix_pnet_base_setup_fork, .child_finalized = pmix_pnet_base_child_finalized, - .local_app_finalized = pmix_pnet_base_local_app_finalized + .local_app_finalized = pmix_pnet_base_local_app_finalized, + .deregister_nspace = pmix_pnet_base_deregister_nspace, + .collect_inventory = pmix_pnet_base_collect_inventory, + .deliver_inventory = pmix_pnet_base_deliver_inventory }; static pmix_status_t pmix_pnet_close(void) @@ -71,6 +74,10 @@ static pmix_status_t pmix_pnet_close(void) } PMIX_DESTRUCT(&pmix_pnet_globals.actives); + PMIX_LIST_DESTRUCT(&pmix_pnet_globals.jobs); + PMIX_LIST_DESTRUCT(&pmix_pnet_globals.nodes); + + PMIX_DESTRUCT_LOCK(&pmix_pnet_globals.lock); return pmix_mca_base_framework_components_close(&pmix_pnet_base_framework, NULL); } @@ -78,7 +85,10 @@ static pmix_status_t pmix_pnet_open(pmix_mca_base_open_flag_t flags) { /* initialize globals */ pmix_pnet_globals.initialized = true; + PMIX_CONSTRUCT_LOCK(&pmix_pnet_globals.lock); PMIX_CONSTRUCT(&pmix_pnet_globals.actives, pmix_list_t); + PMIX_CONSTRUCT(&pmix_pnet_globals.jobs, pmix_list_t); + PMIX_CONSTRUCT(&pmix_pnet_globals.nodes, pmix_list_t); /* Open up all available components */ return pmix_mca_base_framework_components_open(&pmix_pnet_base_framework, flags); @@ -91,3 +101,82 @@ PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, pnet, "PMIx Network Operations", PMIX_CLASS_INSTANCE(pmix_pnet_base_active_module_t, pmix_list_item_t, NULL, NULL); + +static void lpcon(pmix_pnet_local_procs_t *p) +{ + p->nspace = NULL; + p->ranks = NULL; + p->np = 0; +} +static void lpdes(pmix_pnet_local_procs_t *p) +{ + if (NULL != p->nspace) { + free(p->nspace); + } + if (NULL != p->ranks) { + free(p->ranks); + } +} +PMIX_CLASS_INSTANCE(pmix_pnet_local_procs_t, + pmix_list_item_t, + lpcon, lpdes); + +static void ndcon(pmix_pnet_node_t *p) +{ + p->name = NULL; + PMIX_CONSTRUCT(&p->local_jobs, pmix_list_t); + PMIX_CONSTRUCT(&p->resources, pmix_list_t); +} +static void nddes(pmix_pnet_node_t *p) +{ + if (NULL != p->name) { + free(p->name); + } + PMIX_LIST_DESTRUCT(&p->local_jobs); + PMIX_LIST_DESTRUCT(&p->resources); +} +PMIX_CLASS_INSTANCE(pmix_pnet_node_t, + pmix_list_item_t, + ndcon, nddes); + +static void jcon(pmix_pnet_job_t *p) +{ + p->nspace = NULL; + PMIX_CONSTRUCT(&p->nodes, pmix_pointer_array_t); + pmix_pointer_array_init(&p->nodes, 1, INT_MAX, 1); +} +static void jdes(pmix_pnet_job_t *p) +{ + int n; + pmix_pnet_node_t *nd; + + if (NULL != p->nspace) { + free(p->nspace); + } + for (n=0; n < p->nodes.size; n++) { + if (NULL != (nd = (pmix_pnet_node_t*)pmix_pointer_array_get_item(&p->nodes, n))) { + pmix_pointer_array_set_item(&p->nodes, n, NULL); + PMIX_RELEASE(nd); + } + } + PMIX_DESTRUCT(&p->nodes); +} +PMIX_CLASS_INSTANCE(pmix_pnet_job_t, + pmix_list_item_t, + jcon, jdes); + +static void rcon(pmix_pnet_resource_t *p) +{ + p->name = NULL; + PMIX_CONSTRUCT(&p->resources, pmix_list_t); +} +static void rdes(pmix_pnet_resource_t *p) +{ + if (NULL != p->name) { + free(p->name); + } + PMIX_LIST_DESTRUCT(&p->resources); +} +PMIX_CLASS_INSTANCE(pmix_pnet_resource_t, + pmix_list_item_t, + rcon, rdes); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/configure.m4 b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/configure.m4 index 4d6d109a9dd..ba03035a074 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/configure.m4 +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/configure.m4 @@ -12,7 +12,7 @@ # All rights reserved. # Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2013 Sandia National Laboratories. All rights reserved. -# Copyright (c) 2014-2016 Intel, Inc. All rights reserved. +# Copyright (c) 2014-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -30,6 +30,67 @@ AC_DEFUN([MCA_pmix_pnet_opa_CONFIG],[ [pnet_opa_happy="yes"], [pnet_opa_happy="no"]) + AC_ARG_WITH([opamgt], + [AC_HELP_STRING([--with-opamgt(=DIR)], + [Build OmniPath Fabric Management support (optionally adding DIR/include, DIR/include/opamgt, DIR/lib, and DIR/lib64 to the search path for headers and libraries])]) + + AC_ARG_WITH([opamgt-libdir], + [AC_HELP_STRING([--with-opamgt-libdir=DIR], + [Search for OmniPath Fabric Management libraries in DIR])]) + PMIX_CHECK_WITHDIR([opamgt-libdir], [$with_opamgt_libdir], [libopamgt.*]) + + pmix_check_opamgt_save_CPPFLAGS="$CPPFLAGS" + pmix_check_opamgt_save_LDFLAGS="$LDFLAGS" + pmix_check_opamgt_save_LIBS="$LIBS" + + pmix_check_opamgt_libdir= + pmix_check_opamgt_dir= + + AS_IF([test "$with_opamgt" != "no"], + [AS_IF([test ! -z "$with_opamgt" && test "$with_opamgt" != "yes"], + [pmix_check_opamgt_dir="$with_opamgt" + AS_IF([test ! -d "$pmix_check_opamgt_dir" -o test ! -f "$pmix_check_opamgt_dir/opamgt.h"], + [$pmix_check_opamgt_dir=$pmix_check_opamgt_dir/include + AS_IF([test ! -d "$pmix_check_opamgt_dir" -o test ! -f "$pmix_check_opamgt_dir/opamgt.h"], + [$pmix_check_opamgt_dir=$pmix_check_opamgt_dir/opamgt + AS_IF([test ! -d "$pmix_check_opamgt_dir" -o test ! -f "$pmix_check_opamgt_dir/opamgt.h"], + [AC_MSG_WARN([OmniPath Fabric Management support requested, but]) + AC_MSG_WARN([required header file opamgt.h not found. Locations tested:]) + AC_MSG_WARN([ $with_opamgt]) + AC_MSG_WARN([ $with_opamgt/include]) + AC_MSG_WARN([ $with_opamgt/include/opamgt]) + AC_MSG_ERROR([Cannot continue])])])])], + [pmix_check_opamgt_dir="/usr/include/opamgt"]) + + AS_IF([test ! -z "$with_opamgt_libdir" && test "$with_opamgt_libdir" != "yes"], + [pmix_check_opamgt_libdir="$with_opamgt_libdir"]) + + PMIX_CHECK_PACKAGE([pnet_opamgt], + [opamgt.h], + [opamgt], + [omgt_query_sa], + [], + [$pmix_check_opamgt_dir], + [$pmix_check_opamgt_libdir], + [pmix_check_opamgt_happy="yes"], + [pmix_check_opamgt_happy="no"])], + [pmix_check_opamgt_happy="no"]) + + pnet_opa_CLFAGS="$pnet_opa_CFLAGS $pnet_opamgt_CFLAGS" + pnet_opa_CPPFLAGS="$pnet_opa_CPPFLAGS $pnet_opamgt_CPPFLAGS" + pnet_opa_LDFLAGS="$pnet_opa_LDFLAGS $pnet_opamgt_LDFLAGS" + pnet_opa_LIBS="$pnet_opa_LIBS $pnet_opamgt_LIBS" + + AS_IF([test "$pmix_check_opamgt_happy" = "yes"], + [pmix_want_opamgt=1], + [pmix_want_opamgt=0]) + AC_DEFINE_UNQUOTED([PMIX_WANT_OPAMGT], [$pmix_want_opamgt], + [Whether or not to include OmniPath Fabric Manager support]) + + CPPFLAGS="$pmix_check_opamgt_save_CPPFLAGS" + LDFLAGS="$pmix_check_opamgt_save_LDFLAGS" + LIBS="$pmix_check_opamgt_save_LIBS" + AS_IF([test "$pnet_opa_happy" = "yes"], [$1], [$2]) diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.c index e3740b5a640..539050727e5 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * * $COPYRIGHT$ @@ -26,6 +26,12 @@ #endif #include +#if 0 +#if PMIX_WANT_OPAMGT +#include "opamgt.h" +#endif +#endif + #include #include "src/mca/base/pmix_mca_base_var.h" @@ -37,40 +43,57 @@ #include "src/util/error.h" #include "src/util/output.h" #include "src/util/pmix_environ.h" +#include "src/mca/preg/preg.h" +#include "src/hwloc/hwloc-internal.h" #include "src/mca/pnet/pnet.h" +#include "src/mca/pnet/base/base.h" #include "pnet_opa.h" static pmix_status_t opa_init(void); static void opa_finalize(void); -static pmix_status_t setup_app(char *nspace, pmix_list_t *ilist); -static pmix_status_t setup_local_network(char *nspace, +static pmix_status_t allocate(pmix_nspace_t *nptr, + pmix_info_t *info, + pmix_list_t *ilist); +static pmix_status_t setup_local_network(pmix_nspace_t *nptr, pmix_info_t info[], size_t ninfo); -static pmix_status_t setup_fork(const pmix_proc_t *peer, char ***env); -static void child_finalized(pmix_peer_t *peer); -static void local_app_finalized(char *nspace); +static pmix_status_t setup_fork(pmix_nspace_t *nptr, + const pmix_proc_t *proc, + char ***env); +static void child_finalized(pmix_proc_t *peer); +static void local_app_finalized(pmix_nspace_t *nptr); +static void deregister_nspace(pmix_nspace_t *nptr); +static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, + pmix_inventory_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, + pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); pmix_pnet_module_t pmix_opa_module = { + .name = "opa", .init = opa_init, .finalize = opa_finalize, - .setup_app = setup_app, + .allocate = allocate, .setup_local_network = setup_local_network, .setup_fork = setup_fork, .child_finalized = child_finalized, - .local_app_finalized = local_app_finalized + .local_app_finalized = local_app_finalized, + .deregister_nspace = deregister_nspace, + .collect_inventory = collect_inventory, + .deliver_inventory = deliver_inventory }; static pmix_status_t opa_init(void) { - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, "pnet: opa init"); return PMIX_SUCCESS; } static void opa_finalize(void) { - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, "pnet: opa finalize"); } @@ -157,84 +180,285 @@ static char* transports_print(uint64_t *unique_key) return string_key; } -static pmix_status_t setup_app(char *nspace, pmix_list_t *ilist) +/* NOTE: if there is any binary data to be transferred, then + * this function MUST pack it for transport as the host will + * not know how to do so */ +static pmix_status_t allocate(pmix_nspace_t *nptr, + pmix_info_t *info, + pmix_list_t *ilist) { uint64_t unique_key[2]; char *string_key, *cs_env; int fd_rand; size_t bytes_read; pmix_kval_t *kv; + bool envars, seckeys; + pmix_status_t rc; - /* put the number here - or else create an appropriate string. this just needs to - * eventually be a string variable - */ - if(-1 == (fd_rand = open("/dev/urandom", O_RDONLY))) { - transports_use_rand(unique_key); - } else { - bytes_read = read(fd_rand, (char *) unique_key, 16); - if(bytes_read != 16) { - transports_use_rand(unique_key); - } - close(fd_rand); + envars = false; + seckeys = false; + if (NULL == info) { + return PMIX_ERR_TAKE_NEXT_OPTION; } - if (NULL == (string_key = transports_print(unique_key))) { - PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); - return PMIX_ERR_OUT_OF_RESOURCE; + if (0 == strncmp(info->key, PMIX_SETUP_APP_ENVARS, PMIX_MAX_KEYLEN)) { + envars = PMIX_INFO_TRUE(info); + } else if (0 == strncmp(info->key, PMIX_SETUP_APP_ALL, PMIX_MAX_KEYLEN)) { + envars = PMIX_INFO_TRUE(info); + seckeys = PMIX_INFO_TRUE(info); + } else if (0 == strncmp(info->key, PMIX_SETUP_APP_NONENVARS, PMIX_MAX_KEYLEN)) { + seckeys = PMIX_INFO_TRUE(info); } - if (PMIX_SUCCESS != pmix_mca_base_var_env_name("pmix_precondition_transports", &cs_env)) { - PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); - free(string_key); - return PMIX_ERR_OUT_OF_RESOURCE; - } + if (seckeys) { + /* put the number here - or else create an appropriate string. this just needs to + * eventually be a string variable + */ + if(-1 == (fd_rand = open("/dev/urandom", O_RDONLY))) { + transports_use_rand(unique_key); + } else { + bytes_read = read(fd_rand, (char *) unique_key, 16); + if(bytes_read != 16) { + transports_use_rand(unique_key); + } + close(fd_rand); + } - kv = PMIX_NEW(pmix_kval_t); - if (NULL == kv) { - free(string_key); + if (NULL == (string_key = transports_print(unique_key))) { + PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); + return PMIX_ERR_OUT_OF_RESOURCE; + } + + if (PMIX_SUCCESS != pmix_mca_base_var_env_name("opa_precondition_transports", &cs_env)) { + PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); + free(string_key); + return PMIX_ERR_OUT_OF_RESOURCE; + } + + kv = PMIX_NEW(pmix_kval_t); + if (NULL == kv) { + free(string_key); + free(cs_env); + return PMIX_ERR_OUT_OF_RESOURCE; + } + kv->key = strdup(PMIX_SET_ENVAR); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + free(string_key); + free(cs_env); + PMIX_RELEASE(kv); + return PMIX_ERR_OUT_OF_RESOURCE; + } + kv->value->type = PMIX_ENVAR; + PMIX_ENVAR_LOAD(&kv->value->data.envar, cs_env, string_key, ':'); + pmix_list_append(ilist, &kv->super); free(cs_env); - return PMIX_ERR_OUT_OF_RESOURCE; - } - kv->key = strdup(PMIX_SET_ENVAR); - kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); - if (NULL == kv->value) { free(string_key); - free(cs_env); - PMIX_RELEASE(kv); - return PMIX_ERR_OUT_OF_RESOURCE; + if (!envars) { + /* providing envars does not constitute allocating resources */ + return PMIX_ERR_TAKE_NEXT_OPTION; + } } - kv->value->type = PMIX_STRING; - if (0 > asprintf(&kv->value->data.string, "%s=%s", cs_env, string_key)) { - free(string_key); - free(cs_env); - PMIX_RELEASE(kv); - return PMIX_ERR_OUT_OF_RESOURCE; + + if (envars) { + /* harvest envars to pass along */ + if (NULL != mca_pnet_opa_component.include) { + rc = pmix_pnet_base_harvest_envars(mca_pnet_opa_component.include, + mca_pnet_opa_component.exclude, + ilist); + if (PMIX_SUCCESS == rc) { + return PMIX_ERR_TAKE_NEXT_OPTION; + } + return rc; + } } - pmix_list_append(ilist, &kv->super); - free(cs_env); - free(string_key); - return PMIX_SUCCESS; + /* we don't currently manage OPA resources */ + return PMIX_ERR_TAKE_NEXT_OPTION; } -static pmix_status_t setup_local_network(char *nspace, +static pmix_status_t setup_local_network(pmix_nspace_t *nptr, pmix_info_t info[], size_t ninfo) { + size_t n; + pmix_kval_t *kv; + + + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_PNET_OPA_BLOB, PMIX_MAX_KEYLEN)) { + /* the byte object contains a packed blob that needs to be + * cached until we determine we have local procs for this + * nspace, and then delivered to the local OPA driver when + * we have a means for doing so */ + kv = PMIX_NEW(pmix_kval_t); + if (NULL == kv) { + return PMIX_ERR_NOMEM; + } + kv->key = strdup(info[n].key); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + return PMIX_ERR_NOMEM; + } + pmix_value_xfer(kv->value, &info[n].value); + pmix_list_append(&nptr->setup_data, &kv->super); + } + } + } + return PMIX_SUCCESS; } -static pmix_status_t setup_fork(const pmix_proc_t *peer, char ***env) +static pmix_status_t setup_fork(pmix_nspace_t *nptr, + const pmix_proc_t *proc, + char ***env) { + pmix_kval_t *kv, *next; + + /* if there are any cached nspace prep blobs, execute them, + * ensuring that we only do so once per nspace - note that + * we don't expect to find any envars here, though we could + * have included some if we needed to set them per-client */ + PMIX_LIST_FOREACH_SAFE(kv, next, &nptr->setup_data, pmix_kval_t) { + if (0 == strcmp(kv->key, PMIX_PNET_OPA_BLOB)) { + pmix_list_remove_item(&nptr->setup_data, &kv->super); + /* deliver to the local lib */ + PMIX_RELEASE(kv); + } + } return PMIX_SUCCESS; } -static void child_finalized(pmix_peer_t *peer) +static void child_finalized(pmix_proc_t *peer) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:opa child finalized"); +} + +static void local_app_finalized(pmix_nspace_t *nptr) +{ + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:opa app finalized"); + +} + +static void deregister_nspace(pmix_nspace_t *nptr) +{ + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:opa deregister nspace"); + +} + +static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, + pmix_inventory_cbfunc_t cbfunc, void *cbdata) +{ +#if PMIX_HAVE_HWLOC + pmix_inventory_rollup_t *cd = (pmix_inventory_rollup_t*)cbdata; + hwloc_obj_t obj; + unsigned n; + pmix_status_t rc; + pmix_kval_t *kv; + pmix_buffer_t bucket, pbkt; + bool found = false; + pmix_byte_object_t pbo; + char nodename[PMIX_MAXHOSTNAMELEN], *foo; + + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:opa collect inventory"); + + if (NULL == pmix_hwloc_topology) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* setup the bucket - we will pass the results as a blob */ + PMIX_CONSTRUCT(&bucket, pmix_buffer_t); + /* pack our node name */ + gethostname(nodename, sizeof(nodename)); + foo = &nodename[0]; + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &bucket, &foo, 1, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_DESTRUCT(&bucket); + return rc; + } + /* search the topology for OPA devices */ + obj = hwloc_get_next_osdev(pmix_hwloc_topology, NULL); + while (NULL != obj) { + if (obj->attr->osdev.type != HWLOC_OBJ_OSDEV_OPENFABRICS || + 0 != strncmp(obj->name, "hfi", 3)) { + obj = hwloc_get_next_osdev(pmix_hwloc_topology, obj); + continue; + } + found = true; + /* pack the name of the device */ + PMIX_CONSTRUCT(&pbkt, pmix_buffer_t); + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &obj->name, 1, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_DESTRUCT(&pbkt); + PMIX_DESTRUCT(&bucket); + return rc; + } + /* pack each descriptive object */ + for (n=0; n < obj->infos_count; n++) { + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &obj->infos[n].name, 1, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_DESTRUCT(&pbkt); + PMIX_DESTRUCT(&bucket); + return rc; + } + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &obj->infos[n].value, 1, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_DESTRUCT(&pbkt); + PMIX_DESTRUCT(&bucket); + return rc; + } + } + /* extract the resulting blob - this is a device unit */ + PMIX_UNLOAD_BUFFER(&pbkt, pbo.bytes, pbo.size); + /* now load that into the blob */ + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &bucket, &pbo, 1, PMIX_BYTE_OBJECT); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_BYTE_OBJECT_DESTRUCT(&pbo); + PMIX_DESTRUCT(&bucket); + return rc; + } + obj = hwloc_get_next_osdev(pmix_hwloc_topology, obj); + } + + /* if we found any devices, then return the blob */ + if (!found) { + PMIX_DESTRUCT(&bucket); + return PMIX_ERR_TAKE_NEXT_OPTION; + } + + /* extract the resulting blob */ + PMIX_UNLOAD_BUFFER(&bucket, pbo.bytes, pbo.size); + kv = PMIX_NEW(pmix_kval_t); + kv->key = strdup(PMIX_PNET_OPA_BLOB); + PMIX_VALUE_CREATE(kv->value, 1); + pmix_value_load(kv->value, &pbo, PMIX_BYTE_OBJECT); + PMIX_BYTE_OBJECT_DESTRUCT(&pbo); + pmix_list_append(&cd->payload, &kv->super); +#else + return PMIX_ERR_TAKE_NEXT_OPTION; +#endif + + return PMIX_SUCCESS; } -static void local_app_finalized(char *nspace) +static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, + pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:opa deliver inventory"); + return PMIX_ERR_NOT_SUPPORTED; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.h b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.h index 278c894863a..f43dab9c0fd 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * * $COPYRIGHT$ * @@ -18,10 +18,19 @@ BEGIN_C_DECLS +typedef struct { + pmix_pnet_base_component_t super; + char **include; + char **exclude; +} pmix_pnet_opa_component_t; + /* the component must be visible data for the linker to find it */ -PMIX_EXPORT extern pmix_pnet_base_component_t mca_pnet_opa_component; +PMIX_EXPORT extern pmix_pnet_opa_component_t mca_pnet_opa_component; extern pmix_pnet_module_t pmix_opa_module; +/* define a key for any blob we need to send in a launch msg */ +#define PMIX_PNET_OPA_BLOB "pmix.pnet.opa.blob" + END_C_DECLS #endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa_component.c index 7d07c400f6e..9a726c3f4ed 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa_component.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa_component.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -29,48 +29,85 @@ #include #include "pmix_common.h" - +#include "src/util/argv.h" #include "src/mca/pnet/pnet.h" #include "pnet_opa.h" static pmix_status_t component_open(void); static pmix_status_t component_close(void); static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority); +static pmix_status_t component_register(void); /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it */ -pmix_pnet_base_component_t mca_pnet_opa_component = { - .base = { - PMIX_PNET_BASE_VERSION_1_0_0, - - /* Component name and version */ - .pmix_mca_component_name = "opa", - PMIX_MCA_BASE_MAKE_VERSION(component, - PMIX_MAJOR_VERSION, - PMIX_MINOR_VERSION, - PMIX_RELEASE_VERSION), - - /* Component open and close functions */ - .pmix_mca_open_component = component_open, - .pmix_mca_close_component = component_close, - .pmix_mca_query_component = component_query, +pmix_pnet_opa_component_t mca_pnet_opa_component = { + .super = { + .base = { + PMIX_PNET_BASE_VERSION_1_0_0, + + /* Component name and version */ + .pmix_mca_component_name = "opa", + PMIX_MCA_BASE_MAKE_VERSION(component, + PMIX_MAJOR_VERSION, + PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + + /* Component open and close functions */ + .pmix_mca_open_component = component_open, + .pmix_mca_close_component = component_close, + .pmix_mca_register_component_params = component_register, + .pmix_mca_query_component = component_query, + }, + .data = { + /* The component is checkpoint ready */ + PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT + } }, - .data = { - /* The component is checkpoint ready */ - PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT - } + .include = NULL, + .exclude = NULL }; +static char *includeparam; +static char *excludeparam; + +static pmix_status_t component_register(void) +{ + pmix_mca_base_component_t *component = &mca_pnet_opa_component.super.base; + + includeparam = "HFI_*,PSM2_*"; + (void)pmix_mca_base_component_var_register(component, "include_envars", + "Comma-delimited list of envars to harvest (\'*\' and \'?\' supported)", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_LOCAL, + &includeparam); + if (NULL != includeparam) { + mca_pnet_opa_component.include = pmix_argv_split(includeparam, ','); + } + + excludeparam = NULL; + (void)pmix_mca_base_component_var_register(component, "exclude_envars", + "Comma-delimited list of envars to exclude (\'*\' and \'?\' supported)", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_LOCAL, + &excludeparam); + if (NULL != excludeparam) { + mca_pnet_opa_component.exclude = pmix_argv_split(excludeparam, ','); + } + + return PMIX_SUCCESS; +} -static int component_open(void) +static pmix_status_t component_open(void) { return PMIX_SUCCESS; } -static int component_query(pmix_mca_base_module_t **module, int *priority) +static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority) { *priority = 10; *module = (pmix_mca_base_module_t *)&pmix_opa_module; @@ -78,7 +115,7 @@ static int component_query(pmix_mca_base_module_t **module, int *priority) } -static int component_close(void) +static pmix_status_t component_close(void) { return PMIX_SUCCESS; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/pnet.h b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/pnet.h index dedb1eb63f5..3313ca67820 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/pnet.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/pnet.h @@ -1,10 +1,10 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. * - * Copyright (c) 2015 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -33,6 +33,7 @@ #include "src/mca/base/pmix_mca_base_var.h" #include "src/mca/base/pmix_mca_base_framework.h" #include "src/include/pmix_globals.h" +#include "src/server/pmix_server_ops.h" BEGIN_C_DECLS @@ -51,18 +52,23 @@ typedef pmix_status_t (*pmix_pnet_base_module_init_fn_t)(void); typedef void (*pmix_pnet_base_module_fini_fn_t)(void); /** - * Provide an opportunity for the network to define values that + * Allocate network resources. This can be called either as a result + * of a call to PMIx_Allocate_resources, or by the scheduler to + * provide an opportunity for the network to define values that * are to be passed to an application. This can include security * tokens required for application processes to communicate with - * each other + * each other, environmental variables picked up at the login node + * for forwarding to compute nodes, or allocation of static endpts */ -typedef pmix_status_t (*pmix_pnet_base_module_setup_app_fn_t)(char *nspace, pmix_list_t *ilist); +typedef pmix_status_t (*pmix_pnet_base_module_allocate_fn_t)(pmix_nspace_t *nptr, + pmix_info_t *info, + pmix_list_t *ilist); /** * Give the local network library an opportunity to setup address information * for the application by passing in the layout type and a regex describing * the layout */ -typedef pmix_status_t (*pmix_pnet_base_module_setup_local_net_fn_t)(char *nspace, +typedef pmix_status_t (*pmix_pnet_base_module_setup_local_net_fn_t)(pmix_nspace_t *nptr, pmix_info_t info[], size_t ninfo); @@ -70,19 +76,75 @@ typedef pmix_status_t (*pmix_pnet_base_module_setup_local_net_fn_t)(char *nspace * Give the local network library an opportunity to add any envars to the * environment of a local application process prior to fork/exec */ -typedef pmix_status_t (*pmix_pnet_base_module_setup_fork_fn_t)(const pmix_proc_t *peer, char ***env); +typedef pmix_status_t (*pmix_pnet_base_module_setup_fork_fn_t)(pmix_nspace_t *nptr, + const pmix_proc_t *proc, + char ***env); /** * Provide an opportunity for the local network library to cleanup when a * local application process terminates */ -typedef void (*pmix_pnet_base_module_child_finalized_fn_t)(pmix_peer_t *peer); +typedef void (*pmix_pnet_base_module_child_finalized_fn_t)(pmix_proc_t *peer); /** * Provide an opportunity for the local network library to cleanup after * all local clients for a given application have terminated */ -typedef void (*pmix_pnet_base_module_local_app_finalized_fn_t)(char *nspace); +typedef void (*pmix_pnet_base_module_local_app_finalized_fn_t)(pmix_nspace_t *nptr); + +/** + * Provide an opportunity for the fabric components to cleanup any + * resource allocations (e.g., static ports) they may have assigned + */ +typedef void (*pmix_pnet_base_module_dregister_nspace_fn_t)(pmix_nspace_t *nptr); + + +/** + * Request that the module report local inventory for its network type. + * + * If the operation can be performed immediately, then the module should just + * add the inventory (as pmix_kval_t's) to the provided object's list and + * return PMIX_SUCCESS. + * + * If the module needs to perform some non-atomic operation + * (e.g., query a fabric manager), then it should shift to its own internal + * thread, return PMIX_OPERATION_IN_PROGRESS, and execute the provided + * callback function when the operation is completed. + * + * If there is no inventory to report, then just return PMIX_SUCCESS. + * + * If the module should be providing inventory but encounters an error, + * then immediately return an error code if the error is immediately detected, + * or execute the callback function with an error code if it is detected later. + */ +typedef pmix_status_t (*pmix_pnet_base_module_collect_inventory_fn_t)(pmix_info_t directives[], size_t ndirs, + pmix_inventory_cbfunc_t cbfunc, + void *cbdata); + +/** + * Deliver inventory for archiving by corresponding modules + * + * Modules are to search the provided inventory to identify + * entries provided by their remote peers, and then store that + * information in a manner that can be queried/retrieved by + * the host RM and/or scheduler. If the operation can be + * performed immediately (e.g., storing the information in + * the local hash table), then the module should just perform + * that operation and return the appropriate status. + * + * If the module needs to perform some non-atomic operation + * (e.g., storing the information in a non-local DHT), then + * it should shift to its own internal thread, return + * PMIX_OPERATION_IN_PROGRESS, and execute the provided + * callback function when the operation is completed. + * + * If there is no relevant inventory to archive, then the module + * should just return PMIX_SUCCESS; + */ +typedef pmix_status_t (*pmix_pnet_base_module_deliver_inventory_fn_t)(pmix_info_t info[], size_t ninfo, + pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + /** * Base structure for a PNET module @@ -92,15 +154,58 @@ typedef struct { /* init/finalize */ pmix_pnet_base_module_init_fn_t init; pmix_pnet_base_module_fini_fn_t finalize; - pmix_pnet_base_module_setup_app_fn_t setup_app; + pmix_pnet_base_module_allocate_fn_t allocate; pmix_pnet_base_module_setup_local_net_fn_t setup_local_network; pmix_pnet_base_module_setup_fork_fn_t setup_fork; pmix_pnet_base_module_child_finalized_fn_t child_finalized; pmix_pnet_base_module_local_app_finalized_fn_t local_app_finalized; + pmix_pnet_base_module_dregister_nspace_fn_t deregister_nspace; + pmix_pnet_base_module_collect_inventory_fn_t collect_inventory; + pmix_pnet_base_module_deliver_inventory_fn_t deliver_inventory; } pmix_pnet_module_t; + +/* define a few API versions of the functions - main difference is the + * string nspace parameter instead of a pointer to pmix_nspace_t. This + * is done as an optimization to avoid having every component look for + * that pointer */ +typedef pmix_status_t (*pmix_pnet_base_API_allocate_fn_t)(char *nspace, + pmix_info_t info[], size_t ninfo, + pmix_list_t *ilist); +typedef pmix_status_t (*pmix_pnet_base_API_setup_local_net_fn_t)(char *nspace, + pmix_info_t info[], + size_t ninfo); +typedef pmix_status_t (*pmix_pnet_base_API_setup_fork_fn_t)(const pmix_proc_t *peer, char ***env); + +typedef void (*pmix_pnet_base_API_deregister_nspace_fn_t)(char *nspace); +typedef void (*pmix_pnet_base_API_collect_inventory_fn_t)(pmix_info_t directives[], size_t ndirs, + pmix_inventory_cbfunc_t cbfunc, + void *cbdata); +typedef void (*pmix_pnet_base_API_deliver_inventory_fn_t)(pmix_info_t info[], size_t ninfo, + pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +/** + * Base structure for a PNET API + */ +typedef struct { + char *name; + /* init/finalize */ + pmix_pnet_base_module_init_fn_t init; + pmix_pnet_base_module_fini_fn_t finalize; + pmix_pnet_base_API_allocate_fn_t allocate; + pmix_pnet_base_API_setup_local_net_fn_t setup_local_network; + pmix_pnet_base_API_setup_fork_fn_t setup_fork; + pmix_pnet_base_module_child_finalized_fn_t child_finalized; + pmix_pnet_base_module_local_app_finalized_fn_t local_app_finalized; + pmix_pnet_base_API_deregister_nspace_fn_t deregister_nspace; + pmix_pnet_base_API_collect_inventory_fn_t collect_inventory; + pmix_pnet_base_API_deliver_inventory_fn_t deliver_inventory; +} pmix_pnet_API_module_t; + + /* declare the global APIs */ -PMIX_EXPORT extern pmix_pnet_module_t pmix_pnet; +PMIX_EXPORT extern pmix_pnet_API_module_t pmix_pnet; /* * the standard component data structure diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/Makefile.am new file mode 100644 index 00000000000..946d81c8fba --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/Makefile.am @@ -0,0 +1,56 @@ +# -*- makefile -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2017 Research Organization for Information Science +# and Technology (RIST). All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +AM_CPPFLAGS = $(pnet_tcp_CPPFLAGS) + +headers = pnet_tcp.h +sources = \ + pnet_tcp_component.c \ + pnet_tcp.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_pmix_pnet_tcp_DSO +lib = +lib_sources = +component = mca_pnet_tcp.la +component_sources = $(headers) $(sources) +else +lib = libmca_pnet_tcp.la +lib_sources = $(headers) $(sources) +component = +component_sources = +endif + +mcacomponentdir = $(pmixlibdir) +mcacomponent_LTLIBRARIES = $(component) +mca_pnet_tcp_la_SOURCES = $(component_sources) +mca_pnet_tcp_la_LIBADD = $(pnet_tcp_LIBS) +mca_pnet_tcp_la_LDFLAGS = -module -avoid-version $(pnet_tcp_LDFLAGS) + +noinst_LTLIBRARIES = $(lib) +libmca_pnet_tcp_la_SOURCES = $(lib_sources) +libmca_pnet_tcp_la_LIBADD = $(pnet_tcp_LIBS) +libmca_pnet_tcp_la_LDFLAGS = -module -avoid-version $(pnet_tcp_LDFLAGS) diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.c new file mode 100644 index 00000000000..eefa462230f --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.c @@ -0,0 +1,1137 @@ +/* + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#ifdef HAVE_FCNTL_H +#include +#endif +#include + +#include + +#include "src/include/pmix_socket_errno.h" +#include "src/include/pmix_globals.h" +#include "src/class/pmix_list.h" +#include "src/util/alfg.h" +#include "src/util/argv.h" +#include "src/util/error.h" +#include "src/util/output.h" +#include "src/util/parse_options.h" +#include "src/util/pif.h" +#include "src/util/pmix_environ.h" +#include "src/mca/preg/preg.h" + +#include "src/mca/pnet/base/base.h" +#include "pnet_tcp.h" + +#define PMIX_TCP_SETUP_APP_KEY "pmix.tcp.setup.app.key" +#define PMIX_TCP_INVENTORY_KEY "pmix.tcp.inventory" + +static pmix_status_t tcp_init(void); +static void tcp_finalize(void); +static pmix_status_t allocate(pmix_nspace_t *nptr, + pmix_info_t *info, + pmix_list_t *ilist); +static pmix_status_t setup_local_network(pmix_nspace_t *nptr, + pmix_info_t info[], + size_t ninfo); +static pmix_status_t setup_fork(pmix_nspace_t *nptr, + const pmix_proc_t *peer, char ***env); +static void child_finalized(pmix_proc_t *peer); +static void local_app_finalized(pmix_nspace_t *nptr); +static void deregister_nspace(pmix_nspace_t *nptr); +static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, + pmix_inventory_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, + pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +pmix_pnet_module_t pmix_tcp_module = { + .name = "tcp", + .init = tcp_init, + .finalize = tcp_finalize, + .allocate = allocate, + .setup_local_network = setup_local_network, + .setup_fork = setup_fork, + .child_finalized = child_finalized, + .local_app_finalized = local_app_finalized, + .deregister_nspace = deregister_nspace, + .collect_inventory = collect_inventory, + .deliver_inventory = deliver_inventory +}; + +typedef struct { + pmix_list_item_t super; + char *device; + char *address; +} tcp_device_t; + +/* local tracker objects */ +typedef struct { + pmix_list_item_t super; + pmix_list_t devices; + char *type; + char *plane; + char **ports; + size_t nports; +} tcp_available_ports_t; + +typedef struct { + pmix_list_item_t super; + char *nspace; + char **ports; + tcp_available_ports_t *src; // source of the allocated ports +} tcp_port_tracker_t; + +static pmix_list_t allocations, available; +static pmix_status_t process_request(pmix_nspace_t *nptr, + char *idkey, int ports_per_node, + tcp_port_tracker_t *trk, + pmix_list_t *ilist); + +static void dcon(tcp_device_t *p) +{ + p->device = NULL; + p->address = NULL; +} +static void ddes(tcp_device_t *p) +{ + if (NULL != p->device) { + free(p->device); + } + if (NULL != p->address) { + free(p->address); + } +} +static PMIX_CLASS_INSTANCE(tcp_device_t, + pmix_list_item_t, + dcon, ddes); + +static void tacon(tcp_available_ports_t *p) +{ + PMIX_CONSTRUCT(&p->devices, pmix_list_t); + p->type = NULL; + p->plane = NULL; + p->ports = NULL; + p->nports = 0; +} +static void tades(tcp_available_ports_t *p) +{ + PMIX_LIST_DESTRUCT(&p->devices); + if (NULL != p->type) { + free(p->type); + } + if (NULL != p->plane) { + free(p->plane); + } + if (NULL != p->ports) { + pmix_argv_free(p->ports); + } +} +static PMIX_CLASS_INSTANCE(tcp_available_ports_t, + pmix_list_item_t, + tacon, tades); + +static void ttcon(tcp_port_tracker_t *p) +{ + p->nspace = NULL; + p->ports = NULL; + p->src = NULL; +} +static void ttdes(tcp_port_tracker_t *p) +{ + size_t n, m, mstart; + + if (NULL != p->nspace) { + free(p->nspace); + } + if (NULL != p->src) { + if (NULL != p->ports) { + mstart = 0; + for (n=0; NULL != p->ports[n]; n++) { + /* find an empty position */ + for (m=mstart; m < p->src->nports; m++) { + if (NULL == p->src->ports[m]) { + p->src->ports[m] = strdup(p->ports[n]); + mstart = m + 1; + break; + } + } + } + pmix_argv_free(p->ports); + } + PMIX_RELEASE(p->src); // maintain accounting + } else if (NULL != p->ports) { + pmix_argv_free(p->ports); + } +} +static PMIX_CLASS_INSTANCE(tcp_port_tracker_t, + pmix_list_item_t, + ttcon, ttdes); + +static pmix_status_t tcp_init(void) +{ + tcp_available_ports_t *trk; + char *p, **grps; + size_t n; + + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet: tcp init"); + + /* if we are not the "gateway", then there is nothing + * for us to do */ + if (!PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { + return PMIX_SUCCESS; + } + + PMIX_CONSTRUCT(&allocations, pmix_list_t); + PMIX_CONSTRUCT(&available, pmix_list_t); + + /* if we have no static ports, then we don't have + * anything to manage. However, we cannot just disqualify + * ourselves as we may still need to provide inventory. + * + * NOTE: need to check inventory in addition to MCA param as + * the inventory may have reported back static ports */ + if (NULL == mca_pnet_tcp_component.static_ports) { + return PMIX_SUCCESS; + } + + /* split on semi-colons */ + grps = pmix_argv_split(mca_pnet_tcp_component.static_ports, ';'); + for (n=0; NULL != grps[n]; n++) { + trk = PMIX_NEW(tcp_available_ports_t); + if (NULL == trk) { + pmix_argv_free(grps); + return PMIX_ERR_NOMEM; + } + /* there must be at least one colon */ + if (NULL == (p = strrchr(grps[n], ':'))) { + pmix_argv_free(grps); + return PMIX_ERR_BAD_PARAM; + } + /* extract the ports */ + *p = '\0'; + ++p; + pmix_util_parse_range_options(p, &trk->ports); + trk->nports = pmix_argv_count(trk->ports); + /* see if they provided a plane */ + if (NULL != (p = strchr(grps[n], ':'))) { + /* yep - save the plane */ + *p = '\0'; + ++p; + trk->plane = strdup(p); + } + /* the type is just what is left at the front */ + trk->type = strdup(grps[n]); + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "TYPE: %s PLANE %s", trk->type, + (NULL == trk->plane) ? "NULL" : trk->plane); + pmix_list_append(&available, &trk->super); + } + pmix_argv_free(grps); + + return PMIX_SUCCESS; +} + +static void tcp_finalize(void) +{ + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet: tcp finalize"); + if (PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { + PMIX_LIST_DESTRUCT(&allocations); + PMIX_LIST_DESTRUCT(&available); + } +} + +/* some network users may want to encrypt their communications + * as a means of securing them, or include a token in their + * messaging headers for some minimal level of security. This + * is far from perfect, but is provided to illustrate how it + * can be done. The resulting info is placed into the + * app_context's env array so it will automatically be pushed + * into the environment of every MPI process when launched. + * + * In a more perfect world, there would be some privileged place + * to store the crypto key and the encryption would occur + * in a non-visible driver - but we don't have a mechanism + * for doing so. + */ + +static inline void generate_key(uint64_t* unique_key) { + pmix_rng_buff_t rng; + pmix_srand(&rng,(unsigned int)time(NULL)); + unique_key[0] = pmix_rand(&rng); + unique_key[1] = pmix_rand(&rng); +} + +/* when allocate is called, we look at our table of available static addresses + * and assign an address to each process on a node based on its node rank. + * This will prevent collisions as the host RM is responsible for correctly + * setting the node rank. Note that node ranks will "rollover" when they + * hit whatever maximum value the host RM supports, and that they will + * increase monotonically as new jobs are launched until hitting that + * max value. So we need to take into account the number of static + * ports we were given and check to ensure we have enough to hand out + * + * NOTE: this implementation is offered as an example that can + * undoubtedly be vastly improved/optimized */ + +static pmix_status_t allocate(pmix_nspace_t *nptr, + pmix_info_t *info, + pmix_list_t *ilist) +{ + uint64_t unique_key[2]; + size_t n, nreqs=0; + int ports_per_node=0; + pmix_kval_t *kv; + pmix_status_t rc; + pmix_info_t *requests = NULL; + char **reqs, *cptr; + bool allocated = false, seckey = false; + tcp_port_tracker_t *trk; + tcp_available_ports_t *avail, *aptr; + pmix_list_t mylist; + pmix_buffer_t buf; + char *type = NULL, *plane = NULL, *idkey = NULL; + + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp:allocate for nspace %s", nptr->nspace); + + /* if I am not the gateway, then ignore this call - should never + * happen, but check to be safe */ + if (!PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { + return PMIX_SUCCESS; + } + + if (NULL == info) { + return PMIX_ERR_TAKE_NEXT_OPTION; + } + + /* check directives to see if a crypto key and/or + * network resource allocations requested */ + PMIX_CONSTRUCT(&mylist, pmix_list_t); + if (0 == strncmp(info->key, PMIX_SETUP_APP_ENVARS, PMIX_MAX_KEYLEN) || + 0 == strncmp(info->key, PMIX_SETUP_APP_ALL, PMIX_MAX_KEYLEN)) { + if (NULL != mca_pnet_tcp_component.include) { + rc = pmix_pnet_base_harvest_envars(mca_pnet_tcp_component.include, + mca_pnet_tcp_component.exclude, + ilist); + return rc; + } + return PMIX_SUCCESS; + } else if (0 != strncmp(info->key, PMIX_ALLOC_NETWORK, PMIX_MAX_KEYLEN)) { + /* not a network allocation request */ + return PMIX_SUCCESS; + } + + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp:allocate alloc_network for nspace %s", + nptr->nspace); + /* this info key includes an array of pmix_info_t, each providing + * a key (that is to be used as the key for the allocated ports) and + * a number of ports to allocate for that key */ + if (PMIX_DATA_ARRAY != info->value.type || + NULL == info->value.data.darray || + PMIX_INFO != info->value.data.darray->type || + NULL == info->value.data.darray->array) { + /* they made an error */ + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } + requests = (pmix_info_t*)info->value.data.darray->array; + nreqs = info->value.data.darray->size; + /* cycle thru the provided array and see if this refers to + * tcp/udp-based resources - there is no required ordering + * of the keys, so just have to do a search */ + for (n=0; n < nreqs; n++) { + if (0 == strncasecmp(requests[n].key, PMIX_ALLOC_NETWORK_TYPE, PMIX_MAX_KEYLEN)) { + /* check for bozo error */ + if (PMIX_STRING != requests[n].value.type || + NULL == requests[n].value.data.string) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } + type = requests[n].value.data.string; + } else if (0 == strncasecmp(requests[n].key, PMIX_ALLOC_NETWORK_PLANE, PMIX_MAX_KEYLEN)) { + /* check for bozo error */ + if (PMIX_STRING != requests[n].value.type || + NULL == requests[n].value.data.string) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } + plane = requests[n].value.data.string; + } else if (0 == strncasecmp(requests[n].key, PMIX_ALLOC_NETWORK_ENDPTS, PMIX_MAX_KEYLEN)) { + PMIX_VALUE_GET_NUMBER(rc, &requests[n].value, ports_per_node, int); + if (PMIX_SUCCESS != rc) { + return rc; + } + } else if (0 == strncmp(requests[n].key, PMIX_ALLOC_NETWORK_ID, PMIX_MAX_KEYLEN)) { + /* check for bozo error */ + if (PMIX_STRING != requests[n].value.type || + NULL == requests[n].value.data.string) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } + idkey = requests[n].value.data.string; + } else if (0 == strncasecmp(requests[n].key, PMIX_ALLOC_NETWORK_SEC_KEY, PMIX_MAX_KEYLEN)) { + seckey = PMIX_INFO_TRUE(&requests[n]); + } + } + + /* we at least require an attribute key for the response */ + if (NULL == idkey) { + return PMIX_ERR_BAD_PARAM; + } + + /* must include the idkey */ + kv = PMIX_NEW(pmix_kval_t); + if (NULL == kv) { + return PMIX_ERR_NOMEM; + } + kv->key = strdup(PMIX_ALLOC_NETWORK_ID); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + return PMIX_ERR_NOMEM; + } + kv->value->type = PMIX_STRING; + kv->value->data.string = strdup(idkey); + pmix_list_append(&mylist, &kv->super); + + /* note that they might not provide + * the network type (letting it fall to a default component + * based on priority), and they are not required to provide + * a plane. In addition, they are allowed to simply request + * a network security key without asking for endpts */ + + if (NULL != type) { + /* if it is tcp or udp, then this is something we should process */ + if (0 == strcasecmp(type, "tcp")) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp:allocate allocating TCP ports for nspace %s", + nptr->nspace); + /* do we have static tcp ports? */ + avail = NULL; + PMIX_LIST_FOREACH(aptr, &available, tcp_available_ports_t) { + if (0 == strcmp(aptr->type, "tcp")) { + /* if they specified a plane, then require it */ + if (NULL != plane && (NULL == aptr->plane || 0 != strcmp(aptr->plane, plane))) { + continue; + } + avail = aptr; + break; + } + } + /* nope - they asked for something that we cannot do */ + if (NULL == avail) { + return PMIX_ERR_NOT_AVAILABLE; + } + /* setup to track the assignment */ + trk = PMIX_NEW(tcp_port_tracker_t); + if (NULL == trk) { + return PMIX_ERR_NOMEM; + } + trk->nspace = strdup(nptr->nspace); + PMIX_RETAIN(avail); + trk->src = avail; + pmix_list_append(&allocations, &trk->super); + rc = process_request(nptr, idkey, ports_per_node, trk, &mylist); + if (PMIX_SUCCESS != rc) { + /* return the allocated ports */ + pmix_list_remove_item(&allocations, &trk->super); + PMIX_RELEASE(trk); + return rc; + } + allocated = true; + + } else if (0 == strcasecmp(requests[n].value.data.string, "udp")) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp:allocate allocating UDP ports for nspace %s", + nptr->nspace); + /* do we have static udp ports? */ + avail = NULL; + PMIX_LIST_FOREACH(aptr, &available, tcp_available_ports_t) { + if (0 == strcmp(aptr->type, "udp")) { + /* if they specified a plane, then require it */ + if (NULL != plane && (NULL == aptr->plane || 0 != strcmp(aptr->plane, plane))) { + continue; + } + avail = aptr; + break; + } + } + /* nope - they asked for something that we cannot do */ + if (NULL == avail) { + return PMIX_ERR_NOT_AVAILABLE; + } + /* setup to track the assignment */ + trk = PMIX_NEW(tcp_port_tracker_t); + if (NULL == trk) { + return PMIX_ERR_NOMEM; + } + trk->nspace = strdup(nptr->nspace); + PMIX_RETAIN(avail); + trk->src = avail; + pmix_list_append(&allocations, &trk->super); + rc = process_request(nptr, idkey, ports_per_node, trk, &mylist); + if (PMIX_SUCCESS != rc) { + /* return the allocated ports */ + pmix_list_remove_item(&allocations, &trk->super); + PMIX_RELEASE(trk); + return rc; + } + allocated = true; + } else { + /* unsupported type */ + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp:allocate unsupported type %s for nspace %s", + type, nptr->nspace); + return PMIX_ERR_TAKE_NEXT_OPTION; + } + + } else { + if (NULL != plane) { + /* if they didn't specify a type, but they did specify a plane, we can + * see if that is a plane we recognize */ + PMIX_LIST_FOREACH(aptr, &available, tcp_available_ports_t) { + if (0 != strcmp(aptr->plane, plane)) { + continue; + } + /* setup to track the assignment */ + trk = PMIX_NEW(tcp_port_tracker_t); + if (NULL == trk) { + return PMIX_ERR_NOMEM; + } + trk->nspace = strdup(nptr->nspace); + PMIX_RETAIN(aptr); + trk->src = aptr; + pmix_list_append(&allocations, &trk->super); + rc = process_request(nptr, idkey, ports_per_node, trk, &mylist); + if (PMIX_SUCCESS != rc) { + /* return the allocated ports */ + pmix_list_remove_item(&allocations, &trk->super); + PMIX_RELEASE(trk); + return rc; + } + allocated = true; + break; + } + } else { + /* if they didn't specify either type or plane, then we got here because + * nobody of a higher priority could act as a default transport - so try + * to provide something here, starting by looking at any provided setting */ + if (NULL != mca_pnet_tcp_component.default_request) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp:allocate allocating default ports %s for nspace %s", + mca_pnet_tcp_component.default_request, nptr->nspace); + reqs = pmix_argv_split(mca_pnet_tcp_component.default_request, ';'); + for (n=0; NULL != reqs[n]; n++) { + /* if there is no colon, then it is just + * a number of ports to use */ + type = NULL; + plane = NULL; + if (NULL == (cptr = strrchr(reqs[n], ':'))) { + avail = (tcp_available_ports_t*)pmix_list_get_first(&available); + } else { + *cptr = '\0'; + ++cptr; + ports_per_node = strtoul(cptr, NULL, 10); + /* look for the plane */ + cptr -= 2; + if (NULL != (cptr = strrchr(cptr, ':'))) { + *cptr = '\0'; + ++cptr; + plane = cptr; + } + type = reqs[n]; + avail = NULL; + PMIX_LIST_FOREACH(aptr, &available, tcp_available_ports_t) { + if (0 == strcmp(aptr->type, type)) { + /* if they specified a plane, then require it */ + if (NULL != plane && (NULL == aptr->plane || 0 != strcmp(aptr->plane, plane))) { + continue; + } + avail = aptr; + break; + } + } + /* if we didn't find it, that isn't an error - just ignore */ + if (NULL == avail) { + continue; + } + } + /* setup to track the assignment */ + trk = PMIX_NEW(tcp_port_tracker_t); + if (NULL == trk) { + pmix_argv_free(reqs); + return PMIX_ERR_NOMEM; + } + trk->nspace = strdup(nptr->nspace); + PMIX_RETAIN(avail); + trk->src = avail; + pmix_list_append(&allocations, &trk->super); + rc = process_request(nptr, idkey, ports_per_node, trk, &mylist); + if (PMIX_SUCCESS != rc) { + /* return the allocated ports */ + pmix_list_remove_item(&allocations, &trk->super); + PMIX_RELEASE(trk); + return rc; + } + allocated = true; + } + } else { + avail = (tcp_available_ports_t*)pmix_list_get_first(&available); + if (NULL != avail) { + /* setup to track the assignment */ + trk = PMIX_NEW(tcp_port_tracker_t); + if (NULL == trk) { + return PMIX_ERR_NOMEM; + } + trk->nspace = strdup(nptr->nspace); + PMIX_RETAIN(avail); + trk->src = avail; + pmix_list_append(&allocations, &trk->super); + rc = process_request(nptr, idkey, ports_per_node, trk, &mylist); + if (PMIX_SUCCESS != rc) { + /* return the allocated ports */ + pmix_list_remove_item(&allocations, &trk->super); + PMIX_RELEASE(trk); + } else { + allocated = true; + } + } + } + } + if (!allocated) { + /* nope - we cannot help */ + return PMIX_ERR_TAKE_NEXT_OPTION; + } + } + + if (seckey) { + generate_key(unique_key); + kv = PMIX_NEW(pmix_kval_t); + if (NULL == kv) { + return PMIX_ERR_NOMEM; + } + kv->key = strdup(PMIX_ALLOC_NETWORK_SEC_KEY); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + return PMIX_ERR_NOMEM; + } + kv->value->type = PMIX_BYTE_OBJECT; + kv->value->data.bo.bytes = (char*)malloc(2 * sizeof(uint64_t)); + if (NULL == kv->value->data.bo.bytes) { + PMIX_RELEASE(kv); + return PMIX_ERR_NOMEM; + } + memcpy(kv->value->data.bo.bytes, unique_key, 2 * sizeof(uint64_t)); + kv->value->data.bo.size = 2 * sizeof(uint64_t); + pmix_list_append(&mylist, &kv->super); + } + + + n = pmix_list_get_size(&mylist); + if (0 < n) { + PMIX_CONSTRUCT(&buf, pmix_buffer_t); + /* pack the number of kvals for ease on the remote end */ + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, &n, 1, PMIX_SIZE); + /* cycle across the list and pack the kvals */ + while (NULL != (kv = (pmix_kval_t*)pmix_list_remove_first(&mylist))) { + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, kv, 1, PMIX_KVAL); + PMIX_RELEASE(kv); + if (PMIX_SUCCESS != rc) { + PMIX_DESTRUCT(&buf); + PMIX_LIST_DESTRUCT(&mylist); + return rc; + } + } + PMIX_LIST_DESTRUCT(&mylist); + kv = PMIX_NEW(pmix_kval_t); + kv->key = strdup(PMIX_TCP_SETUP_APP_KEY); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + PMIX_DESTRUCT(&buf); + return PMIX_ERR_NOMEM; + } + kv->value->type = PMIX_BYTE_OBJECT; + PMIX_UNLOAD_BUFFER(&buf, kv->value->data.bo.bytes, kv->value->data.bo.size); + PMIX_DESTRUCT(&buf); + pmix_list_append(ilist, &kv->super); + } + + /* if we got here, then we processed this specific request, so + * indicate that by returning success */ + return PMIX_SUCCESS; +} + +/* upon receipt of the launch message, each daemon adds the + * static address assignments to the job-level info cache + * for that job */ +static pmix_status_t setup_local_network(pmix_nspace_t *nptr, + pmix_info_t info[], + size_t ninfo) +{ + size_t n, m, nkvals; + pmix_buffer_t bkt; + int32_t cnt; + pmix_kval_t *kv; + pmix_status_t rc; + pmix_info_t *jinfo, stinfo; + char *idkey = NULL; + + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp:setup_local_network"); + + if (NULL != info) { + for (n=0; n < ninfo; n++) { + /* look for my key */ + if (0 == strncmp(info[n].key, PMIX_TCP_SETUP_APP_KEY, PMIX_MAX_KEYLEN)) { + /* this macro NULLs and zero's the incoming bo */ + PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt, + info[n].value.data.bo.bytes, + info[n].value.data.bo.size); + /* unpack the number of kvals */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, &nkvals, &cnt, PMIX_SIZE); + /* setup the info array */ + PMIX_INFO_CREATE(jinfo, nkvals); + /* cycle thru the blob and extract the kvals */ + kv = PMIX_NEW(pmix_kval_t); + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, kv, &cnt, PMIX_KVAL); + m = 0; + while (PMIX_SUCCESS == rc) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "recvd KEY %s %s", kv->key, + (PMIX_STRING == kv->value->type) ? kv->value->data.string : "NON-STRING"); + /* xfer the value to the info */ + (void)strncpy(jinfo[m].key, kv->key, PMIX_MAX_KEYLEN); + PMIX_BFROPS_VALUE_XFER(rc, pmix_globals.mypeer, + &jinfo[m].value, kv->value); + /* if this is the ID key, save it */ + if (NULL == idkey && + 0 == strncmp(kv->key, PMIX_ALLOC_NETWORK_ID, PMIX_MAX_KEYLEN)) { + idkey = strdup(kv->value->data.string); + } + ++m; + PMIX_RELEASE(kv); + kv = PMIX_NEW(pmix_kval_t); + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, kv, &cnt, PMIX_KVAL); + } + /* restore the incoming data */ + info[n].value.data.bo.bytes = bkt.base_ptr; + info[n].value.data.bo.size = bkt.bytes_used; + bkt.base_ptr = NULL; + bkt.bytes_used = 0; + + /* if they didn't include a network ID, then this is an error */ + if (NULL == idkey) { + PMIX_INFO_FREE(jinfo, nkvals); + return PMIX_ERR_BAD_PARAM; + } + /* the data gets stored as a pmix_data_array_t on the provided key */ + PMIX_INFO_CONSTRUCT(&stinfo); + (void)strncpy(stinfo.key, idkey, PMIX_MAX_KEYLEN); + stinfo.value.type = PMIX_DATA_ARRAY; + PMIX_DATA_ARRAY_CREATE(stinfo.value.data.darray, nkvals, PMIX_INFO); + stinfo.value.data.darray->array = jinfo; + + /* cache the info on the job */ + PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, + &stinfo, 1); + PMIX_INFO_DESTRUCT(&stinfo); + } + } + } + if (NULL != idkey) { + free(idkey); + } + return PMIX_SUCCESS; +} + +static pmix_status_t setup_fork(pmix_nspace_t *nptr, + const pmix_proc_t *peer, char ***env) +{ + return PMIX_SUCCESS; +} + +/* when a local client finalizes, the server gives us a chance + * to do any required local cleanup for that peer. We don't + * have anything we need to do */ +static void child_finalized(pmix_proc_t *peer) +{ + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp child finalized"); +} + +/* when all local clients for a given job finalize, the server + * provides an opportunity for the local network to cleanup + * any resources consumed locally by the clients of that job. + * We don't have anything we need to do */ +static void local_app_finalized(pmix_nspace_t *nptr) +{ + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp app finalized"); +} + +/* when the job completes, the scheduler calls the "deregister nspace" + * PMix function, which in turn calls my TCP component to release the + * assignments for that job. The addresses are marked as "available" + * for reuse on the next job. */ +static void deregister_nspace(pmix_nspace_t *nptr) +{ + tcp_port_tracker_t *trk; + + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp deregister nspace %s", nptr->nspace); + + /* if we are not the "gateway", then there is nothing + * for us to do */ + if (!PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { + return; + } + + /* find this tracker */ + PMIX_LIST_FOREACH(trk, &allocations, tcp_port_tracker_t) { + if (0 == strcmp(nptr->nspace, trk->nspace)) { + pmix_list_remove_item(&allocations, &trk->super); + PMIX_RELEASE(trk); + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp released tracker for nspace %s", nptr->nspace); + return; + } + } +} + +static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, + pmix_inventory_cbfunc_t cbfunc, void *cbdata) +{ + pmix_inventory_rollup_t *cd = (pmix_inventory_rollup_t*)cbdata; + char *prefix, myhost[PMIX_MAXHOSTNAMELEN]; + char myconnhost[PMIX_MAXHOSTNAMELEN]; + char name[32], uri[2048]; + struct sockaddr_storage my_ss; + char *foo; + pmix_buffer_t bucket, pbkt; + int i; + pmix_status_t rc; + bool found = false; + pmix_byte_object_t pbo; + pmix_kval_t *kv; + + /* setup the bucket - we will pass the results as a blob */ + PMIX_CONSTRUCT(&bucket, pmix_buffer_t); + /* add our hostname */ + gethostname(myhost, sizeof(myhost)); + foo = &myhost[0]; + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &bucket, &foo, 1, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_DESTRUCT(&bucket); + return rc; + } + + /* look at all available interfaces */ + for (i = pmix_ifbegin(); i >= 0; i = pmix_ifnext(i)) { + if (PMIX_SUCCESS != pmix_ifindextoaddr(i, (struct sockaddr*)&my_ss, sizeof(my_ss))) { + pmix_output (0, "ptl_tcp: problems getting address for index %i (kernel index %i)\n", + i, pmix_ifindextokindex(i)); + continue; + } + /* ignore non-ip4/6 interfaces */ + if (AF_INET != my_ss.ss_family && + AF_INET6 != my_ss.ss_family) { + continue; + } + /* get the name for diagnostic purposes */ + pmix_ifindextoname(i, name, sizeof(name)); + + /* ignore any virtual interfaces */ + if (0 == strncmp(name, "vir", 3)) { + continue; + } + /* ignore the loopback device */ + if (pmix_ifisloopback(i)) { + continue; + } + if (AF_INET == my_ss.ss_family) { + prefix = "tcp4://"; + inet_ntop(AF_INET, &((struct sockaddr_in*) &my_ss)->sin_addr, + myconnhost, PMIX_MAXHOSTNAMELEN); + } else if (AF_INET6 == my_ss.ss_family) { + prefix = "tcp6://"; + inet_ntop(AF_INET6, &((struct sockaddr_in6*) &my_ss)->sin6_addr, + myconnhost, PMIX_MAXHOSTNAMELEN); + } + (void)snprintf(uri, 2048, "%s%s", prefix, myconnhost); + pmix_output_verbose(2, pmix_pnet_base_framework. framework_output, + "TCP INVENTORY ADDING: %s %s", name, uri); + found = true; + /* pack the name of the device */ + PMIX_CONSTRUCT(&pbkt, pmix_buffer_t); + foo = &name[0]; + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &foo, 1, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_DESTRUCT(&pbkt); + PMIX_DESTRUCT(&bucket); + return rc; + } + /* pack the address */ + foo = &uri[0]; + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &foo, 1, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_DESTRUCT(&pbkt); + PMIX_DESTRUCT(&bucket); + return rc; + } + /* extract the resulting blob - this is a device unit */ + PMIX_UNLOAD_BUFFER(&pbkt, pbo.bytes, pbo.size); + /* now load that into the blob */ + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &bucket, &pbo, 1, PMIX_BYTE_OBJECT); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_BYTE_OBJECT_DESTRUCT(&pbo); + PMIX_DESTRUCT(&bucket); + return rc; + } + } + /* if we have anything to report, then package it up for transfer */ + if (!found) { + PMIX_DESTRUCT(&bucket); + return PMIX_ERR_TAKE_NEXT_OPTION; + } + /* extract the resulting blob */ + PMIX_UNLOAD_BUFFER(&bucket, pbo.bytes, pbo.size); + kv = PMIX_NEW(pmix_kval_t); + kv->key = strdup(PMIX_TCP_INVENTORY_KEY); + PMIX_VALUE_CREATE(kv->value, 1); + pmix_value_load(kv->value, &pbo, PMIX_BYTE_OBJECT); + PMIX_BYTE_OBJECT_DESTRUCT(&pbo); + pmix_list_append(&cd->payload, &kv->super); + + return PMIX_SUCCESS; +} + +static pmix_status_t process_request(pmix_nspace_t *nptr, + char *idkey, int ports_per_node, + tcp_port_tracker_t *trk, + pmix_list_t *ilist) +{ + char **plist; + pmix_kval_t *kv; + size_t m; + int p, ppn; + tcp_available_ports_t *avail = trk->src; + + kv = PMIX_NEW(pmix_kval_t); + if (NULL == kv) { + return PMIX_ERR_NOMEM; + } + kv->key = strdup(idkey); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + return PMIX_ERR_NOMEM; + } + kv->value->type = PMIX_STRING; + kv->value->data.string = NULL; + if (0 == ports_per_node) { + /* find the maxprocs on the nodes in this nspace and + * allocate that number of resources */ + return PMIX_ERR_NOT_SUPPORTED; + } else { + ppn = ports_per_node; + } + + /* assemble the list of ports */ + p = 0; + plist = NULL; + for (m=0; p < ppn && m < avail->nports; m++) { + if (NULL != avail->ports[m]) { + pmix_argv_append_nosize(&trk->ports, avail->ports[m]); + pmix_argv_append_nosize(&plist, avail->ports[m]); + free(avail->ports[m]); + avail->ports[m] = NULL; + ++p; + } + } + /* if we couldn't find enough, then that's an error */ + if (p < ppn) { + PMIX_RELEASE(kv); + /* the caller will release trk, and that will return + * any allocated ports back to the available list */ + return PMIX_ERR_OUT_OF_RESOURCE; + } + /* pass the value */ + kv->value->data.string = pmix_argv_join(plist, ','); + pmix_argv_free(plist); + pmix_list_append(ilist, &kv->super); + + /* track where it came from */ + kv = PMIX_NEW(pmix_kval_t); + if (NULL == kv) { + return PMIX_ERR_NOMEM; + } + kv->key = strdup(idkey); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + return PMIX_ERR_NOMEM; + } + kv->value->type = PMIX_STRING; + kv->value->data.string = strdup(trk->src->type); + pmix_list_append(ilist, &kv->super); + if (NULL != trk->src->plane) { + kv = PMIX_NEW(pmix_kval_t); + if (NULL == kv) { + return PMIX_ERR_NOMEM; + } + kv->key = strdup(idkey); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + return PMIX_ERR_NOMEM; + } + kv->value->type = PMIX_STRING; + kv->value->data.string = strdup(trk->src->plane); + pmix_list_append(ilist, &kv->super); + } + return PMIX_SUCCESS; +} + +static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, + pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_buffer_t bkt, pbkt; + size_t n; + int32_t cnt; + char *hostname, *device, *address; + pmix_byte_object_t pbo; + pmix_pnet_node_t *nd, *ndptr; + pmix_pnet_resource_t *lt, *lst; + tcp_available_ports_t *prts; + tcp_device_t *res; + pmix_status_t rc; + + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp deliver inventory"); + + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_TCP_INVENTORY_KEY, PMIX_MAX_KEYLEN)) { + /* this is our inventory in the form of a blob */ + PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt, + info[n].value.data.bo.bytes, + info[n].value.data.bo.size); + /* first is the host this came from */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, &hostname, &cnt, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + /* must _not_ destruct bkt as we don't + * own the bytes! */ + return rc; + } + /* do we already have this node? */ + nd = NULL; + PMIX_LIST_FOREACH(ndptr, &pmix_pnet_globals.nodes, pmix_pnet_node_t) { + if (0 == strcmp(hostname, ndptr->name)) { + nd = ndptr; + break; + } + } + if (NULL == nd) { + nd = PMIX_NEW(pmix_pnet_node_t); + nd->name = strdup(hostname); + pmix_list_append(&pmix_pnet_globals.nodes, &nd->super); + } + /* does this node already have a TCP entry? */ + lst = NULL; + PMIX_LIST_FOREACH(lt, &nd->resources, pmix_pnet_resource_t) { + if (0 == strcmp(lt->name, "tcp")) { + lst = lt; + break; + } + } + if (NULL == lst) { + lst = PMIX_NEW(pmix_pnet_resource_t); + lst->name = strdup("tcp"); + pmix_list_append(&nd->resources, &lst->super); + } + /* this is a list of ports and devices */ + prts = PMIX_NEW(tcp_available_ports_t); + pmix_list_append(&lst->resources, &prts->super); + /* cycle across any provided interfaces */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, &pbo, &cnt, PMIX_BYTE_OBJECT); + while (PMIX_SUCCESS == rc) { + /* load the byte object for unpacking */ + PMIX_LOAD_BUFFER(pmix_globals.mypeer, &pbkt, pbo.bytes, pbo.size); + /* unpack the name of the device */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &pbkt, &device, &cnt, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_DATA_BUFFER_DESTRUCT(&pbkt); + /* must _not_ destruct bkt as we don't + * own the bytes! */ + return rc; + } + /* unpack the address */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &pbkt, &address, &cnt, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_DATA_BUFFER_DESTRUCT(&pbkt); + /* must _not_ destruct bkt as we don't + * own the bytes! */ + return rc; + } + /* store this on the node */ + res = PMIX_NEW(tcp_device_t); + res->device = device; + res->address = address; + pmix_list_append(&prts->devices, &res->super); + PMIX_DATA_BUFFER_DESTRUCT(&pbkt); + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, &pbo, &cnt, PMIX_BYTE_OBJECT); + } + PMIX_DATA_BUFFER_DESTRUCT(&bkt); + } + } + + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.h b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.h new file mode 100644 index 00000000000..63ffd878712 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_PNET_OPA_H +#define PMIX_PNET_OPA_H + +#include + + +#include "src/mca/pnet/pnet.h" + +BEGIN_C_DECLS + +typedef struct { + pmix_pnet_base_component_t super; + char *static_ports; + char *default_request; + char **include; + char **exclude; +} pmix_pnet_tcp_component_t; + +/* the component must be visible data for the linker to find it */ +PMIX_EXPORT extern pmix_pnet_tcp_component_t mca_pnet_tcp_component; +extern pmix_pnet_module_t pmix_tcp_module; + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp_component.c new file mode 100644 index 00000000000..64dc93c2409 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp_component.c @@ -0,0 +1,133 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include +#include "pmix_common.h" + +#include "src/util/argv.h" +#include "src/mca/pnet/pnet.h" +#include "pnet_tcp.h" + +static pmix_status_t component_register(void); +static pmix_status_t component_open(void); +static pmix_status_t component_close(void); +static pmix_status_t component_query(pmix_mca_base_module_t **module, + int *priority); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +pmix_pnet_tcp_component_t mca_pnet_tcp_component = { + .super = { + .base = { + PMIX_PNET_BASE_VERSION_1_0_0, + + /* Component name and version */ + .pmix_mca_component_name = "tcp", + PMIX_MCA_BASE_MAKE_VERSION(component, + PMIX_MAJOR_VERSION, + PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + + /* Component open and close functions */ + .pmix_mca_register_component_params = component_register, + .pmix_mca_open_component = component_open, + .pmix_mca_close_component = component_close, + .pmix_mca_query_component = component_query, + }, + .data = { + /* The component is checkpoint ready */ + PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT + } + }, + .static_ports = NULL, + .default_request = NULL, + .include = NULL, + .exclude = NULL +}; + +static char *includeparam; +static char *excludeparam; + +static pmix_status_t component_register(void) +{ + pmix_mca_base_component_t *component = &mca_pnet_tcp_component.super.base; + + mca_pnet_tcp_component.static_ports = NULL; + (void)pmix_mca_base_component_var_register(component, "static_ports", + "Static ports for procs, expressed as a semi-colon delimited " + "list of type:(optional)plane:Comma-delimited list of ranges (e.g., " + "\"tcp:10.10.10.0/24:32000-32100,33000;udp:40000,40005\")", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_pnet_tcp_component.static_ports); + + (void)pmix_mca_base_component_var_register(component, "default_network_allocation", + "Semi-colon delimited list of (optional)type:(optional)plane:Comma-delimited list of ranges " + "(e.g., \"udp:10.10.10.0/24:3\", or \"5\" if the choice of " + "type and plane isn't critical)", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_pnet_tcp_component.default_request); + + includeparam = NULL; + (void)pmix_mca_base_component_var_register(component, "include_envars", + "Comma-delimited list of envars to harvest (\'*\' and \'?\' supported)", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_LOCAL, + &includeparam); + if (NULL != includeparam) { + mca_pnet_tcp_component.include = pmix_argv_split(includeparam, ','); + } + + excludeparam = NULL; + (void)pmix_mca_base_component_var_register(component, "exclude_envars", + "Comma-delimited list of envars to exclude (\'*\' and \'?\' supported)", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_LOCAL, + &excludeparam); + if (NULL != excludeparam) { + mca_pnet_tcp_component.exclude = pmix_argv_split(excludeparam, ','); + } + + return PMIX_SUCCESS; +} + +static pmix_status_t component_open(void) +{ + return PMIX_SUCCESS; +} + + +static pmix_status_t component_query(pmix_mca_base_module_t **module, + int *priority) +{ + *priority = 5; + *module = (pmix_mca_base_module_t *)&pmix_tcp_module; + return PMIX_SUCCESS; +} + + +static pmix_status_t component_close(void) +{ + + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/Makefile.am new file mode 100644 index 00000000000..3faf68a32c2 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/Makefile.am @@ -0,0 +1,52 @@ +# -*- makefile -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2017 Research Organization for Information Science +# and Technology (RIST). All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +headers = pnet_test.h +sources = \ + pnet_test_component.c \ + pnet_test.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_pmix_pnet_test_DSO +lib = +lib_sources = +component = mca_pnet_test.la +component_sources = $(headers) $(sources) +else +lib = libmca_pnet_test.la +lib_sources = $(headers) $(sources) +component = +component_sources = +endif + +mcacomponentdir = $(pmixlibdir) +mcacomponent_LTLIBRARIES = $(component) +mca_pnet_test_la_SOURCES = $(component_sources) +mca_pnet_test_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(lib) +libmca_pnet_test_la_SOURCES = $(lib_sources) +libmca_pnet_test_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test.c new file mode 100644 index 00000000000..a8808b43e6e --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test.c @@ -0,0 +1,485 @@ +/* + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#ifdef HAVE_FCNTL_H +#include +#endif +#include + +#include + +#include "src/mca/base/pmix_mca_base_var.h" +#include "src/include/pmix_socket_errno.h" +#include "src/include/pmix_globals.h" +#include "src/class/pmix_list.h" +#include "src/util/alfg.h" +#include "src/util/argv.h" +#include "src/util/error.h" +#include "src/util/name_fns.h" +#include "src/util/output.h" +#include "src/util/pmix_environ.h" +#include "src/mca/preg/preg.h" + +#include "src/mca/pnet/pnet.h" +#include "src/mca/pnet/base/base.h" +#include "pnet_test.h" + +static pmix_status_t test_init(void); +static void test_finalize(void); +static pmix_status_t allocate(pmix_nspace_t *nptr, + pmix_info_t *info, + pmix_list_t *ilist); +static pmix_status_t setup_local_network(pmix_nspace_t *nptr, + pmix_info_t info[], + size_t ninfo); +static pmix_status_t setup_fork(pmix_nspace_t *nptr, + const pmix_proc_t *proc, + char ***env); +static void child_finalized(pmix_proc_t *peer); +static void local_app_finalized(pmix_nspace_t *nptr); +static void deregister_nspace(pmix_nspace_t *nptr); +static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, + pmix_inventory_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, + pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +pmix_pnet_module_t pmix_test_module = { + .name = "test", + .init = test_init, + .finalize = test_finalize, + .allocate = allocate, + .setup_local_network = setup_local_network, + .setup_fork = setup_fork, + .child_finalized = child_finalized, + .local_app_finalized = local_app_finalized, + .deregister_nspace = deregister_nspace, + .collect_inventory = collect_inventory, + .deliver_inventory = deliver_inventory +}; + +static pmix_status_t test_init(void) +{ + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet: test init"); + return PMIX_SUCCESS; +} + +static void test_finalize(void) +{ + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet: test finalize"); +} + +/* NOTE: if there is any binary data to be transferred, then + * this function MUST pack it for transport as the host will + * not know how to do so */ +static pmix_status_t allocate(pmix_nspace_t *nptr, + pmix_info_t *info, + pmix_list_t *ilist) +{ + pmix_kval_t *kv; + bool seckey = false; + pmix_list_t mylist; + size_t n, nreqs=0; + pmix_info_t *requests = NULL; + char *idkey = NULL; + uint64_t unique_key = 12345; + pmix_buffer_t buf; + pmix_status_t rc; + pmix_pnet_job_t *jptr, *job; + pmix_pnet_node_t *nd; + pmix_pnet_local_procs_t *lptr, *lp; + + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:test:allocate for nspace %s key %s", + nptr->nspace, info->key); + + /* if I am not the gateway, then ignore this call - should never + * happen, but check to be safe */ + if (!PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { + return PMIX_SUCCESS; + } + + if (NULL == info) { + return PMIX_ERR_TAKE_NEXT_OPTION; + } + /* check directives to see if a crypto key and/or + * network resource allocations requested */ + PMIX_CONSTRUCT(&mylist, pmix_list_t); + if (0 == strncmp(info->key, PMIX_SETUP_APP_ENVARS, PMIX_MAX_KEYLEN) || + 0 == strncmp(info->key, PMIX_SETUP_APP_ALL, PMIX_MAX_KEYLEN)) { + kv = PMIX_NEW(pmix_kval_t); + if (NULL == kv) { + return PMIX_ERR_NOMEM; + } + kv->key = strdup(PMIX_SET_ENVAR); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + return PMIX_ERR_NOMEM; + } + kv->value->type = PMIX_ENVAR; + PMIX_ENVAR_LOAD(&kv->value->data.envar, "PMIX_TEST_ENVAR", "1", ':'); + pmix_list_append(ilist, &kv->super); + return PMIX_SUCCESS; + } else if (0 != strncmp(info->key, PMIX_ALLOC_NETWORK_ID, PMIX_MAX_KEYLEN)) { + return PMIX_SUCCESS; + } + + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:test:allocate alloc_network for nspace %s", + nptr->nspace); + + /* this info key includes an array of pmix_info_t, each providing + * a key (that is to be used as the key for the allocated ports) and + * a number of ports to allocate for that key */ + if (PMIX_DATA_ARRAY != info->value.type || + NULL == info->value.data.darray || + PMIX_INFO != info->value.data.darray->type || + NULL == info->value.data.darray->array) { + /* just process something for test */ + goto process; + } + requests = (pmix_info_t*)info->value.data.darray->array; + nreqs = info->value.data.darray->size; + /* cycle thru the provided array and get the ID key */ + for (n=0; n < nreqs; n++) { + if (0 == strncmp(requests[n].key, PMIX_ALLOC_NETWORK_ID, PMIX_MAX_KEYLEN)) { + /* check for bozo error */ + if (PMIX_STRING != requests[n].value.type || + NULL == requests[n].value.data.string) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } + idkey = requests[n].value.data.string; + } else if (0 == strncasecmp(requests[n].key, PMIX_ALLOC_NETWORK_SEC_KEY, PMIX_MAX_KEYLEN)) { + seckey = PMIX_INFO_TRUE(&requests[n]); + } + } + + process: + /* if they didn't give us a test key, just create one */ + if (NULL == idkey) { + idkey = "TESTKEY"; + } + + /* must include the idkey */ + kv = PMIX_NEW(pmix_kval_t); + if (NULL == kv) { + return PMIX_ERR_NOMEM; + } + kv->key = strdup(PMIX_ALLOC_NETWORK_ID); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + return PMIX_ERR_NOMEM; + } + kv->value->type = PMIX_STRING; + kv->value->data.string = strdup(idkey); + pmix_list_append(&mylist, &kv->super); + + if (seckey) { + kv = PMIX_NEW(pmix_kval_t); + if (NULL == kv) { + return PMIX_ERR_NOMEM; + } + kv->key = strdup(PMIX_ALLOC_NETWORK_SEC_KEY); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + return PMIX_ERR_NOMEM; + } + kv->value->type = PMIX_BYTE_OBJECT; + kv->value->data.bo.bytes = (char*)malloc(sizeof(uint64_t)); + if (NULL == kv->value->data.bo.bytes) { + PMIX_RELEASE(kv); + return PMIX_ERR_NOMEM; + } + memcpy(kv->value->data.bo.bytes, &unique_key, sizeof(uint64_t)); + kv->value->data.bo.size = sizeof(uint64_t); + pmix_list_append(&mylist, &kv->super); + } + + /* find the info on this job, if available */ + job = NULL; + PMIX_LIST_FOREACH(jptr, &pmix_pnet_globals.jobs, pmix_pnet_job_t) { + if (0 == strcmp(jptr->nspace, nptr->nspace)) { + job = jptr; + break; + } + } + if (NULL != job) { + pmix_output(0, "ALLOCATE RESOURCES FOR JOB %s", job->nspace); + for (n=0; (int)n < job->nodes.size; n++) { + if (NULL == (nd = (pmix_pnet_node_t*)pmix_pointer_array_get_item(&job->nodes, n))) { + continue; + } + lp = NULL; + PMIX_LIST_FOREACH(lptr, &nd->local_jobs, pmix_pnet_local_procs_t) { + if (0 == strcmp(job->nspace, lptr->nspace)) { + lp = lptr; + break; + } + } + if (NULL == lp) { + pmix_output(0, "\t NODE %s 0 RANKS", nd->name); + } else { + pmix_output(0, "\tNODE %s %d RANKS", nd->name, (int)lp->np); + } + } + } + + n = pmix_list_get_size(&mylist); + if (0 < n) { + PMIX_CONSTRUCT(&buf, pmix_buffer_t); + /* pack the number of kvals for ease on the remote end */ + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, &n, 1, PMIX_SIZE); + /* cycle across the list and pack the kvals */ + while (NULL != (kv = (pmix_kval_t*)pmix_list_remove_first(&mylist))) { + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, kv, 1, PMIX_KVAL); + PMIX_RELEASE(kv); + if (PMIX_SUCCESS != rc) { + PMIX_DESTRUCT(&buf); + PMIX_LIST_DESTRUCT(&mylist); + return rc; + } + } + PMIX_LIST_DESTRUCT(&mylist); + kv = PMIX_NEW(pmix_kval_t); + kv->key = strdup("pmix-pnet-test-blob"); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + PMIX_DESTRUCT(&buf); + return PMIX_ERR_NOMEM; + } + kv->value->type = PMIX_BYTE_OBJECT; + PMIX_UNLOAD_BUFFER(&buf, kv->value->data.bo.bytes, kv->value->data.bo.size); + PMIX_DESTRUCT(&buf); + pmix_list_append(ilist, &kv->super); + } + + return PMIX_SUCCESS; +} + +static pmix_status_t setup_local_network(pmix_nspace_t *nptr, + pmix_info_t info[], + size_t ninfo) +{ + size_t n, m, nkvals; + char *nodestring, **nodes; + pmix_proc_t *procs; + size_t nprocs; + pmix_buffer_t bkt; + int32_t cnt; + pmix_kval_t *kv; + pmix_status_t rc; + pmix_info_t *jinfo, stinfo; + char *idkey = NULL; + + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:test:setup_local_network"); + + /* get the list of nodes in this job - returns a regex */ + pmix_output(0, "pnet:test setup_local_network NSPACE %s", (NULL == nptr) ? "NULL" : nptr->nspace); + if (NULL == nptr) { + return PMIX_SUCCESS; + } + pmix_preg.resolve_nodes(nptr->nspace, &nodestring); + if (NULL == nodestring) { + return PMIX_SUCCESS; + } + pmix_preg.parse_nodes(nodestring, &nodes); // get an argv array of node names + pmix_output(0, "pnet:test setup_local_network NODES %s", (NULL == nodes) ? "NULL" : "NON-NULL"); + if (NULL == nodes) { + free(nodestring); + return PMIX_SUCCESS; + } + for (n=0; NULL != nodes[n]; n++) { + pmix_output(0, "pnet:test setup_local_network NODE: %s", nodes[n]); + } + + for (n=0; NULL != nodes[n]; n++) { + /* get an array of pmix_proc_t containing the names of the procs on that node */ + pmix_preg.resolve_peers(nodes[n], nptr->nspace, &procs, &nprocs); + if (NULL == procs) { + continue; + } + for (m=0; m < nprocs; m++) { + pmix_output(0, "pnet:test setup_local_network NODE %s: peer %s:%d", nodes[n], procs[m].nspace, procs[m].rank); + } + /* do stuff */ + free(procs); + } + + if (NULL != info) { + for (n=0; n < ninfo; n++) { + /* look for my key */ + if (0 == strncmp(info[n].key, "pmix-pnet-test-blob", PMIX_MAX_KEYLEN)) { + /* this macro NULLs and zero's the incoming bo */ + PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt, + info[n].value.data.bo.bytes, + info[n].value.data.bo.size); + /* unpack the number of kvals */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, &nkvals, &cnt, PMIX_SIZE); + /* setup the info array */ + PMIX_INFO_CREATE(jinfo, nkvals); + /* cycle thru the blob and extract the kvals */ + kv = PMIX_NEW(pmix_kval_t); + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, kv, &cnt, PMIX_KVAL); + m = 0; + while (PMIX_SUCCESS == rc) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "recvd KEY %s %s", kv->key, + (PMIX_STRING == kv->value->type) ? kv->value->data.string : "NON-STRING"); + /* xfer the value to the info */ + (void)strncpy(jinfo[m].key, kv->key, PMIX_MAX_KEYLEN); + PMIX_BFROPS_VALUE_XFER(rc, pmix_globals.mypeer, + &jinfo[m].value, kv->value); + /* if this is the ID key, save it */ + if (NULL == idkey && + 0 == strncmp(kv->key, PMIX_ALLOC_NETWORK_ID, PMIX_MAX_KEYLEN)) { + idkey = strdup(kv->value->data.string); + } + ++m; + PMIX_RELEASE(kv); + kv = PMIX_NEW(pmix_kval_t); + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, kv, &cnt, PMIX_KVAL); + } + /* restore the incoming data */ + info[n].value.data.bo.bytes = bkt.base_ptr; + info[n].value.data.bo.size = bkt.bytes_used; + bkt.base_ptr = NULL; + bkt.bytes_used = 0; + + /* if they didn't include a network ID, then this is an error */ + if (NULL == idkey) { + PMIX_INFO_FREE(jinfo, nkvals); + return PMIX_ERR_BAD_PARAM; + } + /* the data gets stored as a pmix_data_array_t on the provided key */ + PMIX_INFO_CONSTRUCT(&stinfo); + (void)strncpy(stinfo.key, idkey, PMIX_MAX_KEYLEN); + stinfo.value.type = PMIX_DATA_ARRAY; + PMIX_DATA_ARRAY_CREATE(stinfo.value.data.darray, nkvals, PMIX_INFO); + stinfo.value.data.darray->array = jinfo; + + /* cache the info on the job */ + PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, + &stinfo, 1); + PMIX_INFO_DESTRUCT(&stinfo); + } + } + } + if (NULL != idkey) { + free(idkey); + } + return PMIX_SUCCESS; +} + +static pmix_status_t setup_fork(pmix_nspace_t *nptr, + const pmix_proc_t *proc, + char ***env) +{ + pmix_cb_t cb; + pmix_status_t rc; + pmix_kval_t *kv; + uint16_t localrank; + + PMIX_CONSTRUCT(&cb, pmix_cb_t); + + cb.key = strdup(PMIX_LOCAL_RANK); + /* this data isn't going anywhere, so we don't require a copy */ + cb.copy = false; + /* scope is irrelevant as the info we seek must be local */ + cb.scope = PMIX_SCOPE_UNDEF; + /* ask for the value for the given proc */ + cb.proc = (pmix_proc_t*)proc; + + PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb); + if (PMIX_SUCCESS != rc) { + if (PMIX_ERR_INVALID_NAMESPACE != rc) { + PMIX_ERROR_LOG(rc); + } + PMIX_DESTRUCT(&cb); + return rc; + } + /* should just be the one value on the list */ + if (1 != pmix_list_get_size(&cb.kvs)) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + PMIX_DESTRUCT(&cb); + return PMIX_ERR_BAD_PARAM; + } + kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs); + if (PMIX_UINT16 != kv->value->type) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + PMIX_DESTRUCT(&cb); + return PMIX_ERR_BAD_PARAM; + } + localrank = kv->value->data.uint16; + + pmix_output(0, "pnet:test LOCAL RANK FOR PROC %s: %d", PMIX_NAME_PRINT(proc), (int)localrank); + + PMIX_DESTRUCT(&cb); + return PMIX_SUCCESS; +} + +static void child_finalized(pmix_proc_t *peer) +{ + pmix_output(0, "pnet:test CHILD %s:%d FINALIZED", + peer->nspace, peer->rank); +} + +static void local_app_finalized(pmix_nspace_t *nptr) +{ + pmix_output(0, "pnet:test NSPACE %s LOCALLY FINALIZED", nptr->nspace); +} + +static void deregister_nspace(pmix_nspace_t *nptr) +{ + pmix_output(0, "pnet:test DEREGISTER NSPACE %s", nptr->nspace); +} + +static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, + pmix_inventory_cbfunc_t cbfunc, void *cbdata) +{ + pmix_output(0, "pnet:test COLLECT INVENTORY"); + return PMIX_ERR_NOT_SUPPORTED; +} + +static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, + pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:test deliver inventory"); + + return PMIX_ERR_NOT_SUPPORTED; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test.h b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test.h new file mode 100644 index 00000000000..8601bc355b6 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_PNET_test_H +#define PMIX_PNET_test_H + +#include + + +#include "src/mca/pnet/pnet.h" + +BEGIN_C_DECLS + +typedef struct { + pmix_pnet_base_component_t super; + char **include; + char **exclude; +} pmix_pnet_test_component_t; + +/* the component must be visible data for the linker to find it */ +PMIX_EXPORT extern pmix_pnet_test_component_t mca_pnet_test_component; +extern pmix_pnet_module_t pmix_test_module; + +/* define a key for any blob we need to send in a launch msg */ +#define PMIX_PNET_TEST_BLOB "pmix.pnet.test.blob" + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test_component.c new file mode 100644 index 00000000000..06e360163fc --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test_component.c @@ -0,0 +1,101 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include +#include "pmix_common.h" + +#include "src/util/argv.h" +#include "src/mca/pnet/pnet.h" +#include "pnet_test.h" + +static pmix_status_t component_open(void); +static pmix_status_t component_close(void); +static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +pmix_pnet_test_component_t mca_pnet_test_component = { + .super = { + .base = { + PMIX_PNET_BASE_VERSION_1_0_0, + + /* Component name and version */ + .pmix_mca_component_name = "test", + PMIX_MCA_BASE_MAKE_VERSION(component, + PMIX_MAJOR_VERSION, + PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + + /* Component open and close functions */ + .pmix_mca_open_component = component_open, + .pmix_mca_close_component = component_close, + .pmix_mca_query_component = component_query, + }, + .data = { + /* The component is checkpoint ready */ + PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT + } + }, + .include = NULL, + .exclude = NULL +}; + +static pmix_status_t component_open(void) +{ + int index; + const pmix_mca_base_var_storage_t *value=NULL; + + /* we only allow ourselves to be considered IF the user + * specifically requested so */ + if (0 > (index = pmix_mca_base_var_find("pmix", "pnet", NULL, NULL))) { + return PMIX_ERROR; + } + pmix_mca_base_var_get_value(index, &value, NULL, NULL); + if (NULL != value && NULL != value->stringval && '\0' != value->stringval[0]) { + if (NULL != strstr(value->stringval, "test")) { + return PMIX_SUCCESS; + } + } + return PMIX_ERROR; +} + + +static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority) +{ + *priority = 0; + *module = (pmix_mca_base_module_t *)&pmix_test_module; + return PMIX_SUCCESS; +} + + +static pmix_status_t component_close(void) +{ + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/preg/native/preg_native.c b/opal/mca/pmix/pmix3x/pmix/src/mca/preg/native/preg_native.c index 89a3846120d..1a98766f8e7 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/preg/native/preg_native.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/preg/native/preg_native.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -501,27 +501,32 @@ static pmix_status_t parse_procs(const char *regexp, return rc; } -static void _resolve_peers(int sd, short args, void *cbdata) +static pmix_status_t resolve_peers(const char *nodename, + const char *nspace, + pmix_proc_t **procs, size_t *nprocs) { - pmix_cb_t *cb = (pmix_cb_t*)cbdata; + pmix_cb_t cb; pmix_status_t rc; pmix_kval_t *kv; pmix_proc_t proc; char **ptr; pmix_info_t *info; - pmix_proc_t *procs; - size_t ninfo, nprocs, n, j; + pmix_proc_t *p=NULL; + size_t ninfo, np=0, n, j; + PMIX_CONSTRUCT(&cb, pmix_cb_t); + + cb.key = strdup(nodename); /* this data isn't going anywhere, so we don't require a copy */ - cb->copy = false; + cb.copy = false; /* scope is irrelevant as the info we seek must be local */ - cb->scope = PMIX_SCOPE_UNDEF; + cb.scope = PMIX_SCOPE_UNDEF; /* let the proc point to the nspace */ - (void)strncpy(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN); + (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; - cb->proc = &proc; + cb.proc = &proc; - PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, cb); + PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, &cb); if (PMIX_SUCCESS != rc) { if (PMIX_ERR_INVALID_NAMESPACE != rc) { PMIX_ERROR_LOG(rc); @@ -529,12 +534,12 @@ static void _resolve_peers(int sd, short args, void *cbdata) goto complete; } /* should just be the one value on the list */ - if (1 != pmix_list_get_size(&cb->kvs)) { + if (1 != pmix_list_get_size(&cb.kvs)) { PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); rc = PMIX_ERR_BAD_PARAM; goto complete; } - kv = (pmix_kval_t*)pmix_list_get_first(&cb->kvs); + kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs); /* the hostname used as a key with wildcard rank will return * a pmix_data_array_t of pmix_info_t structs */ if (NULL == kv->value || @@ -552,184 +557,103 @@ static void _resolve_peers(int sd, short args, void *cbdata) if (0 == strncmp(info[n].key, PMIX_LOCAL_PEERS, PMIX_MAX_KEYLEN)) { /* split the string */ ptr = pmix_argv_split(info[n].value.data.string, ','); - nprocs = pmix_argv_count(ptr); - PMIX_PROC_CREATE(procs, nprocs); - if (NULL == procs) { + np = pmix_argv_count(ptr); + PMIX_PROC_CREATE(p, np); + if (NULL == p) { rc = PMIX_ERR_NOMEM; pmix_argv_free(ptr); goto complete; } - for (j=0; j < nprocs; j++) { - (void)strncpy(procs[j].nspace, cb->pname.nspace, PMIX_MAX_NSLEN); - procs[j].rank = strtoul(ptr[j], NULL, 10); + for (j=0; j < np; j++) { + (void)strncpy(p[j].nspace, nspace, PMIX_MAX_NSLEN); + p[j].rank = strtoul(ptr[j], NULL, 10); } - cb->procs = procs; - cb->nprocs = nprocs; rc = PMIX_SUCCESS; pmix_argv_free(ptr); - goto complete; + break; } } complete: - cb->status = rc; - if (NULL != cb->info) { - PMIX_INFO_FREE(cb->info, cb->ninfo); + if (NULL != cb.info) { + PMIX_INFO_FREE(cb.info, cb.ninfo); } - cb->pstatus = rc; - /* post the data so the receiving thread can acquire it */ - PMIX_POST_OBJECT(cb); - PMIX_WAKEUP_THREAD(&cb->lock); - return; -} - -static pmix_status_t resolve_peers(const char *nodename, - const char *nspace, - pmix_proc_t **procs, size_t *nprocs) -{ - pmix_cb_t *cb; - pmix_status_t rc; - pmix_proc_t proc; - - cb = PMIX_NEW(pmix_cb_t); - cb->key = (char*)nodename; - cb->pname.nspace = strdup(nspace); - - PMIX_THREADSHIFT(cb, _resolve_peers); - - /* wait for the result */ - PMIX_WAIT_THREAD(&cb->lock); - - /* if the nspace wasn't found, then we need to - * ask the server for that info */ - if (PMIX_ERR_INVALID_NAMESPACE == cb->status) { - (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); - proc.rank = PMIX_RANK_WILDCARD; - /* any key will suffice as it will bring down - * the entire data blob */ - rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, NULL); - if (PMIX_SUCCESS != rc) { - PMIX_RELEASE(cb); - return rc; - } - /* retry the fetch */ - cb->lock.active = true; - PMIX_THREADSHIFT(cb, _resolve_peers); - PMIX_WAIT_THREAD(&cb->lock); + if (NULL != cb.key) { + free(cb.key); + cb.key = NULL; } - *procs = cb->procs; - *nprocs = cb->nprocs; + PMIX_DESTRUCT(&cb); + *procs = p; + *nprocs = np; - rc = cb->status; - PMIX_RELEASE(cb); return rc; } -static void _resolve_nodes(int sd, short args, void *cbdata) +static pmix_status_t resolve_nodes(const char *nspace, + char **nodelist) { - pmix_cb_t *cb = (pmix_cb_t*)cbdata; + pmix_cb_t cb; pmix_status_t rc; pmix_kval_t *kv; pmix_proc_t proc; - /* create a pmix_info_t so we can pass the nspace - * into the fetch as a qualifier */ - PMIX_INFO_CREATE(cb->info, 1); - if (NULL == cb->info) { - cb->status = PMIX_ERR_NOMEM; - PMIX_POST_OBJECT(cb); - PMIX_WAKEUP_THREAD(&cb->lock); - return; - } - cb->ninfo = 1; - PMIX_INFO_LOAD(&cb->info[0], PMIX_NSPACE, cb->pname.nspace, PMIX_STRING); - /* tell the GDS what we want */ - cb->key = PMIX_NODE_LIST; - /* this data isn't going anywhere, so we don't require a copy */ - cb->copy = false; - /* scope is irrelevant as the info we seek must be local */ - cb->scope = PMIX_SCOPE_UNDEF; - /* put the nspace in the proc field */ - (void)strncpy(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN); - /* the info will be associated with PMIX_RANK_WILDCARD */ - proc.rank = PMIX_RANK_WILDCARD; - cb->proc = &proc; - - PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, cb); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - goto complete; - } - /* should just be the one value on the list */ - if (1 != pmix_list_get_size(&cb->kvs)) { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); - rc = PMIX_ERR_BAD_PARAM; - goto complete; - } - kv = (pmix_kval_t*)pmix_list_get_first(&cb->kvs); - /* the PMIX_NODE_LIST key is supposed to return a comma-delimited - * string of nodes in this - check that it did */ - if (NULL == kv->value || - PMIX_STRING != kv->value->type) { - PMIX_ERROR_LOG(PMIX_ERR_DATA_VALUE_NOT_FOUND); - rc = PMIX_ERR_DATA_VALUE_NOT_FOUND; - goto complete; - } - /* return the string */ - if (NULL != kv->value->data.string) { - cb->key = strdup(kv->value->data.string); - } - - complete: - cb->status = rc; - if (NULL != cb->info) { - PMIX_INFO_FREE(cb->info, cb->ninfo); - } - /* post the data so the receiving thread can acquire it */ - PMIX_POST_OBJECT(cb); - PMIX_WAKEUP_THREAD(&cb->lock); - return; -} + PMIX_CONSTRUCT(&cb, pmix_cb_t); -static pmix_status_t resolve_nodes(const char *nspace, - char **nodelist) -{ - pmix_cb_t *cb; - pmix_status_t rc; - pmix_proc_t proc; + /* setup default answer */ + *nodelist = NULL; - cb = PMIX_NEW(pmix_cb_t); - cb->pname.nspace = (char*)nspace; - - PMIX_THREADSHIFT(cb, _resolve_nodes); - - /* wait for the result */ - PMIX_WAIT_THREAD(&cb->lock); - - /* if the nspace wasn't found, then we need to - * ask the server for that info */ - if (PMIX_ERR_INVALID_NAMESPACE == cb->status) { - (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); - proc.rank = PMIX_RANK_WILDCARD; - /* any key will suffice as it will bring down - * the entire data blob */ - rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, NULL); - if (PMIX_SUCCESS != rc) { - PMIX_RELEASE(cb); - return rc; - } - /* retry the fetch */ - cb->lock.active = true; - PMIX_THREADSHIFT(cb, _resolve_nodes); - PMIX_WAIT_THREAD(&cb->lock); + /* create a pmix_info_t so we can pass the nspace + * into the fetch as a qualifier */ + PMIX_INFO_CREATE(cb.info, 1); + if (NULL == cb.info) { + PMIX_DESTRUCT(&cb); + return PMIX_ERR_NOMEM; } - /* the string we want is in the key field */ - *nodelist = cb->key; + cb.ninfo = 1; + PMIX_INFO_LOAD(&cb.info[0], PMIX_NSPACE, nspace, PMIX_STRING); - rc = cb->status; - PMIX_RELEASE(cb); - return rc; + /* tell the GDS what we want */ + cb.key = PMIX_NODE_MAP; + /* this data isn't going anywhere, so we don't require a copy */ + cb.copy = false; + /* scope is irrelevant as the info we seek must be local */ + cb.scope = PMIX_SCOPE_UNDEF; + /* put the nspace in the proc field */ + (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + /* the info will be associated with PMIX_RANK_WILDCARD */ + proc.rank = PMIX_RANK_WILDCARD; + cb.proc = &proc; + PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, &cb); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + /* should just be the one value on the list */ + if (1 != pmix_list_get_size(&cb.kvs)) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + rc = PMIX_ERR_BAD_PARAM; + goto complete; + } + kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs); + /* the PMIX_NODE_MAP key is supposed to return + * a regex string - check that it did */ + if (NULL == kv->value || + PMIX_STRING != kv->value->type) { + PMIX_ERROR_LOG(PMIX_ERR_DATA_VALUE_NOT_FOUND); + rc = PMIX_ERR_DATA_VALUE_NOT_FOUND; + goto complete; + } + /* return the string */ + if (NULL != kv->value->data.string) { + *nodelist = strdup(kv->value->data.string); + } + + complete: + if (NULL != cb.info) { + PMIX_INFO_FREE(cb.info, cb.ninfo); + } + return rc; } static pmix_status_t pmix_regex_extract_nodes(char *regexp, char ***names) diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/base.h b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/base.h index 47215a1e198..718a377fc5c 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/base.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/base.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -100,7 +100,7 @@ PMIX_EXPORT pmix_status_t pmix_ptl_base_cancel_recv(struct pmix_peer_t *peer, PMIX_EXPORT pmix_status_t pmix_ptl_base_start_listening(pmix_info_t *info, size_t ninfo); PMIX_EXPORT void pmix_ptl_base_stop_listening(void); - +PMIX_EXPORT pmix_status_t pmix_ptl_base_setup_fork(const pmix_proc_t *proc, char ***env); /* base support functions */ PMIX_EXPORT void pmix_ptl_base_send(int sd, short args, void *cbdata); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c index 12bd8652e61..5a59300533b 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c @@ -80,7 +80,8 @@ void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err) } CLOSE_THE_SOCKET(peer->sd); - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { /* if I am a server, then we need to ensure that * we properly account for the loss of this client * from any local collectives in which it was @@ -109,8 +110,8 @@ void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err) trk->modexcbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, NULL, 0, trk, NULL, NULL); } } else if (PMIX_CONNECTNB_CMD == trk->type) { - if (NULL != trk->cnct_cbfunc) { - trk->cnct_cbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, NULL, PMIX_RANK_WILDCARD, trk); + if (NULL != trk->op_cbfunc) { + trk->op_cbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, trk); } } else if (PMIX_DISCONNECTNB_CMD == trk->type) { if (NULL != trk->op_cbfunc) { diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_stubs.c b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_stubs.c index 67e0259b5e2..f2334e21a11 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_stubs.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_stubs.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -30,6 +30,26 @@ #include "src/mca/ptl/base/base.h" +pmix_status_t pmix_ptl_base_setup_fork(const pmix_proc_t *proc, char ***env) +{ + pmix_ptl_base_active_t *active; + pmix_status_t rc; + + if (!pmix_ptl_globals.initialized) { + return PMIX_ERR_INIT; + } + + PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) { + if (NULL != active->component->setup_fork) { + rc = active->component->setup_fork(proc, env); + if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_AVAILABLE != rc) { + return rc; + } + } + } + return PMIX_SUCCESS; +} + pmix_status_t pmix_ptl_base_set_notification_cbfunc(pmix_ptl_cbfunc_t cbfunc) { pmix_ptl_posted_recv_t *req; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl.h b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl.h index 157f45f580c..01f849b8055 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -171,6 +171,10 @@ extern pmix_status_t pmix_ptl_base_connect_to_peer(struct pmix_peer_t* peer, typedef pmix_status_t (*pmix_ptl_base_setup_listener_fn_t)(pmix_info_t info[], size_t ninfo, bool *need_listener); +/* define a component-level API for obtaining any envars that are to + * be passed to client procs upon fork */ +typedef pmix_status_t (*pmix_ptl_base_setup_fork_fn_t)(const pmix_proc_t *proc, char ***env); + /* * the standard component data structure */ @@ -180,6 +184,8 @@ struct pmix_ptl_base_component_t { int priority; char* uri; pmix_ptl_base_setup_listener_fn_t setup_listener; + pmix_ptl_base_setup_fork_fn_t setup_fork; + }; typedef struct pmix_ptl_base_component_t pmix_ptl_base_component_t; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl_types.h b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl_types.h index ffb15baaada..0008bb48d39 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl_types.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl_types.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -63,14 +63,20 @@ struct pmix_ptl_module_t; /* define a process type */ typedef uint16_t pmix_proc_type_t; -#define PMIX_PROC_UNDEF 0x0000 -#define PMIX_PROC_CLIENT 0x0001 -#define PMIX_PROC_SERVER 0x0002 -#define PMIX_PROC_TOOL 0x0004 -#define PMIX_PROC_V1 0x0008 -#define PMIX_PROC_V20 0x0010 -#define PMIX_PROC_V21 0x0020 -#define PMIX_PROC_V3 0x0040 +#define PMIX_PROC_UNDEF 0x0000 +#define PMIX_PROC_CLIENT 0x0001 +#define PMIX_PROC_SERVER 0x0002 +#define PMIX_PROC_TOOL 0x0004 +#define PMIX_PROC_V1 0x0008 +#define PMIX_PROC_V20 0x0010 +#define PMIX_PROC_V21 0x0020 +#define PMIX_PROC_V3 0x0040 +#define PMIX_PROC_LAUNCHER_ACT 0x1000 +#define PMIX_PROC_LAUNCHER (PMIX_PROC_TOOL | PMIX_PROC_SERVER | PMIX_PROC_LAUNCHER_ACT) +#define PMIX_PROC_CLIENT_TOOL_ACT 0x2000 +#define PMIX_PROC_CLIENT_TOOL (PMIX_PROC_TOOL | PMIX_PROC_CLIENT | PMIX_PROC_CLIENT_TOOL_ACT) +#define PMIX_PROC_GATEWAY_ACT 0x4000 +#define PMIX_PROC_GATEWAY (PMIX_PROC_SERVER | PMIX_PROC_GATEWAY_ACT) /* defins some convenience macros for testing proc type */ #define PMIX_PROC_IS_CLIENT(p) (PMIX_PROC_CLIENT & (p)->proc_type) @@ -80,6 +86,9 @@ typedef uint16_t pmix_proc_type_t; #define PMIX_PROC_IS_V20(p) (PMIX_PROC_V20 & (p)->proc_type) #define PMIX_PROC_IS_V21(p) (PMIX_PROC_V21 & (p)->proc_type) #define PMIX_PROC_IS_V3(p) (PMIX_PROC_V3 & (p)->proc_type) +#define PMIX_PROC_IS_LAUNCHER(p) (PMIX_PROC_LAUNCHER_ACT & (p)->proc_type) +#define PMIX_PROC_IS_CLIENT_TOOL(p) (PMIX_PROC_CLIENT_TOOL_ACT & (p)->proc_type) +#define PMIX_PROC_IS_GATEWAY(p) (PMIX_PROC_GATEWAY_ACT & (p)->proc_type) /**** MESSAGING STRUCTURES ****/ diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.c b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.c index fe2a4cff80b..429089a64f2 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.c @@ -51,6 +51,7 @@ #include "src/util/argv.h" #include "src/util/error.h" #include "src/util/os_path.h" +#include "src/util/show_help.h" #include "src/mca/bfrops/base/base.h" #include "src/mca/ptl/base/base.h" @@ -108,7 +109,7 @@ static pmix_status_t parse_uri_file(char *filename, char **uri, char **nspace, pmix_rank_t *rank); -static pmix_status_t try_connect(int *sd); +static pmix_status_t try_connect(char *uri, int *sd); static pmix_status_t df_search(char *dirname, char *prefix, int *sd, char **nspace, pmix_rank_t *rank); @@ -116,15 +117,16 @@ static pmix_status_t df_search(char *dirname, char *prefix, static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, pmix_info_t *info, size_t ninfo) { - char *evar, **uri, *suri; + char *evar, **uri, *suri = NULL, *suri2 = NULL; char *filename, *nspace=NULL; pmix_rank_t rank = PMIX_RANK_WILDCARD; - char *p, *p2; + char *p, *p2, *server_nspace = NULL; int sd, rc; size_t n; char myhost[PMIX_MAXHOSTNAMELEN]; bool system_level = false; bool system_level_only = false; + bool reconnect = false; pid_t pid = 0; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, @@ -137,7 +139,17 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, /* if I am a client, then we need to look for the appropriate * connection info in the environment */ if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { - if (NULL != (evar = getenv("PMIX_SERVER_URI21"))) { + if (NULL != (evar = getenv("PMIX_SERVER_URI3"))) { + /* we are talking to a v3 server */ + pmix_client_globals.myserver->proc_type = PMIX_PROC_SERVER | PMIX_PROC_V3; + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "V3 SERVER DETECTED"); + /* must use the v3 bfrops module */ + pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module("v3"); + if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) { + return PMIX_ERR_INIT; + } + } else if (NULL != (evar = getenv("PMIX_SERVER_URI21"))) { /* we are talking to a v2.1 server */ pmix_client_globals.myserver->proc_type = PMIX_PROC_SERVER | PMIX_PROC_V21; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, @@ -163,7 +175,7 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, } /* the server will be using the same bfrops as us */ pmix_client_globals.myserver->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops; - /* mark that we are using the V2 protocol */ + /* mark that we are using the V2 (i.e., tcp) protocol */ pmix_globals.mypeer->protocol = PMIX_PROTOCOL_V2; /* the URI consists of the following elements: @@ -189,106 +201,144 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, nspace = strdup(p); rank = strtoull(p2, NULL, 10); - /* save the URI, but do not overwrite what we may have received from - * the info-key directives */ - if (NULL == mca_ptl_tcp_component.super.uri) { - mca_ptl_tcp_component.super.uri = strdup(uri[1]); - } - pmix_argv_free(uri); - pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:tcp:client attempt connect to %s", - mca_ptl_tcp_component.super.uri); + "ptl:tcp:client attempt connect to %s", uri[1]); /* go ahead and try to connect */ - if (PMIX_SUCCESS != (rc = try_connect(&sd))) { + if (PMIX_SUCCESS != (rc = try_connect(uri[1], &sd))) { free(nspace); + pmix_argv_free(uri); return rc; } + pmix_argv_free(uri); goto complete; } /* get here if we are a tool - check any provided directives * to see where they want us to connect to */ + suri = NULL; if (NULL != info) { for (n=0; n < ninfo; n++) { if (0 == strcmp(info[n].key, PMIX_CONNECT_TO_SYSTEM)) { system_level_only = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strcmp(info[n].key, PMIX_CONNECT_SYSTEM_FIRST)) { + } else if (0 == strncmp(info[n].key, PMIX_CONNECT_SYSTEM_FIRST, PMIX_MAX_KEYLEN)) { /* try the system-level */ system_level = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strcmp(info[n].key, PMIX_SERVER_PIDINFO)) { + } else if (0 == strncmp(info[n].key, PMIX_SERVER_PIDINFO, PMIX_MAX_KEYLEN)) { pid = info[n].value.data.pid; - } else if (0 == strcmp(info[n].key, PMIX_SERVER_URI)) { - if (NULL == mca_ptl_tcp_component.super.uri) { - free(mca_ptl_tcp_component.super.uri); + } else if (0 == strncmp(info[n].key, PMIX_SERVER_NSPACE, PMIX_MAX_KEYLEN)) { + if (NULL != server_nspace) { + /* they included it more than once */ + if (0 == strcmp(server_nspace, info[n].value.data.string)) { + /* same value, so ignore it */ + continue; + } + /* otherwise, we don't know which one to use */ + free(server_nspace); + if (NULL != suri) { + free(suri); + } + return PMIX_ERR_BAD_PARAM; } - mca_ptl_tcp_component.super.uri = strdup(info[n].value.data.string); - } else if (0 == strcmp(info[n].key, PMIX_CONNECT_RETRY_DELAY)) { + server_nspace = strdup(info[n].value.data.string); + } else if (0 == strncmp(info[n].key, PMIX_SERVER_URI, PMIX_MAX_KEYLEN)) { + if (NULL != suri) { + /* they included it more than once */ + if (0 == strcmp(suri, info[n].value.data.string)) { + /* same value, so ignore it */ + continue; + } + /* otherwise, we don't know which one to use */ + free(suri); + if (NULL != server_nspace) { + free(server_nspace); + } + return PMIX_ERR_BAD_PARAM; + } + suri = strdup(info[n].value.data.string); + } else if (0 == strncmp(info[n].key, PMIX_CONNECT_RETRY_DELAY, PMIX_MAX_KEYLEN)) { mca_ptl_tcp_component.wait_to_connect = info[n].value.data.uint32; - } else if (0 == strcmp(info[n].key, PMIX_CONNECT_MAX_RETRIES)) { + } else if (0 == strncmp(info[n].key, PMIX_CONNECT_MAX_RETRIES, PMIX_MAX_KEYLEN)) { mca_ptl_tcp_component.max_retries = info[n].value.data.uint32; + } else if (0 == strncmp(info[n].key, PMIX_RECONNECT_SERVER, PMIX_MAX_KEYLEN)) { + reconnect = true; } } } + if (NULL == suri && !reconnect && NULL != mca_ptl_tcp_component.super.uri) { + suri = strdup(mca_ptl_tcp_component.super.uri); + } + /* mark that we are using the V2 protocol */ pmix_globals.mypeer->protocol = PMIX_PROTOCOL_V2; gethostname(myhost, sizeof(myhost)); /* if we were given a URI via MCA param, then look no further */ - if (NULL != mca_ptl_tcp_component.super.uri) { + if (NULL != suri) { + if (NULL != server_nspace) { + free(server_nspace); + server_nspace = NULL; + } /* if the string starts with "file:", then they are pointing * us to a file we need to read to get the URI itself */ - if (0 == strncmp(mca_ptl_tcp_component.super.uri, "file:", 5)) { + if (0 == strncmp(suri, "file:", 5)) { pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:tcp:tool getting connection info from %s", - mca_ptl_tcp_component.super.uri); + "ptl:tcp:tool getting connection info from %s", suri); nspace = NULL; - rc = parse_uri_file(&mca_ptl_tcp_component.super.uri[5], &suri, &nspace, &rank); + rc = parse_uri_file(&suri[5], &suri2, &nspace, &rank); if (PMIX_SUCCESS != rc) { + free(suri); return PMIX_ERR_UNREACH; } - free(mca_ptl_tcp_component.super.uri); - mca_ptl_tcp_component.super.uri = suri; + free(suri); + suri = suri2; } else { /* we need to extract the nspace/rank of the server from the string */ - p = strchr(mca_ptl_tcp_component.super.uri, ';'); + p = strchr(suri, ';'); if (NULL == p) { + free(suri); return PMIX_ERR_BAD_PARAM; } *p = '\0'; p++; - suri = strdup(p); // save the uri portion + suri2 = strdup(p); // save the uri portion /* the '.' in the first part of the original string separates * nspace from rank */ - p = strchr(mca_ptl_tcp_component.super.uri, '.'); + p = strchr(suri, '.'); if (NULL == p) { + free(suri2); free(suri); return PMIX_ERR_BAD_PARAM; } *p = '\0'; p++; - nspace = strdup(mca_ptl_tcp_component.super.uri); + nspace = strdup(suri); rank = strtoull(p, NULL, 10); + free(suri); + suri = suri2; /* now update the URI */ - free(mca_ptl_tcp_component.super.uri); - mca_ptl_tcp_component.super.uri = suri; } pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:tcp:tool attempt connect using given URI %s", - mca_ptl_tcp_component.super.uri); + "ptl:tcp:tool attempt connect using given URI %s", suri); /* go ahead and try to connect */ - if (PMIX_SUCCESS != (rc = try_connect(&sd))) { + if (PMIX_SUCCESS != (rc = try_connect(suri, &sd))) { if (NULL != nspace) { free(nspace); } + free(suri); return rc; } + free(suri); + suri = NULL; goto complete; } /* if they gave us a pid, then look for it */ if (0 != pid) { + if (NULL != server_nspace) { + free(server_nspace); + server_nspace = NULL; + } if (0 > asprintf(&filename, "pmix.%s.tool.%d", myhost, pid)) { return PMIX_ERR_NOMEM; } @@ -310,6 +360,31 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, return PMIX_ERR_UNREACH; } + /* if they gave us an nspace, then look for it */ + if (NULL != server_nspace) { + if (0 > asprintf(&filename, "pmix.%s.tool.%s", myhost, server_nspace)) { + free(server_nspace); + return PMIX_ERR_NOMEM; + } + free(server_nspace); + server_nspace = NULL; + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:tcp:tool searching for given session server %s", + filename); + nspace = NULL; + rc = df_search(mca_ptl_tcp_component.system_tmpdir, + filename, &sd, &nspace, &rank); + free(filename); + if (PMIX_SUCCESS == rc) { + goto complete; + } + if (NULL != nspace) { + free(nspace); + } + /* since they gave us a specific nspace and we couldn't + * connect to it, return an error */ + return PMIX_ERR_UNREACH; + } /* if they asked for system-level, we start there */ if (system_level || system_level_only) { @@ -323,12 +398,11 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, rc = parse_uri_file(filename, &suri, &nspace, &rank); free(filename); if (PMIX_SUCCESS == rc) { - mca_ptl_tcp_component.super.uri = suri; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:tcp:tool attempt connect to system server at %s", - mca_ptl_tcp_component.super.uri); + "ptl:tcp:tool attempt connect to system server at %s", suri); /* go ahead and try to connect */ - if (PMIX_SUCCESS == try_connect(&sd)) { + if (PMIX_SUCCESS == try_connect(suri, &sd)) { + /* don't free nspace - we will use it below */ goto complete; } free(nspace); @@ -382,22 +456,27 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, pmix_globals.connected = true; pmix_client_globals.myserver->sd = sd; - /* setup the server info */ - if (NULL == pmix_client_globals.myserver->info) { - pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); - } - if (NULL == pmix_client_globals.myserver->nptr) { - pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); - } - if (NULL == pmix_client_globals.myserver->nptr->nspace) { + /* tools setup their server info in try_connect because they + * utilize a broader handshake */ + if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + /* setup the server info */ + if (NULL == pmix_client_globals.myserver->info) { + pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); + } + if (NULL == pmix_client_globals.myserver->nptr) { + pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); + } + if (NULL != pmix_client_globals.myserver->nptr->nspace) { + free(pmix_client_globals.myserver->nptr->nspace); + } pmix_client_globals.myserver->nptr->nspace = nspace; - } else { - free(nspace); - } - if (NULL == pmix_client_globals.myserver->info->pname.nspace) { + + if (NULL != pmix_client_globals.myserver->info->pname.nspace) { + free(pmix_client_globals.myserver->info->pname.nspace); + } pmix_client_globals.myserver->info->pname.nspace = strdup(pmix_client_globals.myserver->nptr->nspace); + pmix_client_globals.myserver->info->pname.rank = rank; } - pmix_client_globals.myserver->info->pname.rank = rank; pmix_ptl_base_set_nonblocking(sd); @@ -419,6 +498,9 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, pmix_ptl_base_send_handler, pmix_client_globals.myserver); pmix_client_globals.myserver->send_ev_active = false; + if (NULL != suri) { + free(suri); + } return PMIX_SUCCESS; } @@ -481,6 +563,7 @@ static pmix_status_t parse_uri_file(char *filename, pmix_event_t ev; struct timeval tv; int retries; + int major; fp = fopen(filename, "r"); if (NULL == fp) { @@ -530,24 +613,27 @@ static pmix_status_t parse_uri_file(char *filename, p2 = pmix_getline(fp); if (NULL == p2) { pmix_client_globals.myserver->proc_type = PMIX_PROC_SERVER | PMIX_PROC_V20; - pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "V20 SERVER DETECTED"); - pmix_client_globals.myserver->protocol = PMIX_PROTOCOL_V2; - } else if (0 == strncmp(p2, "v2.1", strlen("v2.1")) || - 0 == strncmp(p2, "2.1", strlen("2.1"))) { - pmix_client_globals.myserver->proc_type = PMIX_PROC_SERVER | PMIX_PROC_V21; pmix_client_globals.myserver->protocol = PMIX_PROTOCOL_V2; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "V21 SERVER DETECTED"); - } else if (0 == strncmp(p2, "3", strlen("3")) || - 0 == strncmp(p2, "v3", strlen("v3"))) { - pmix_client_globals.myserver->proc_type = PMIX_PROC_SERVER | PMIX_PROC_V3; - pmix_client_globals.myserver->protocol = PMIX_PROTOCOL_V2; - pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "V3 SERVER DETECTED"); + "V20 SERVER DETECTED"); } else { - pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "UNKNOWN SERVER VERSION DETECTED: %s", p2); + /* convert the version to a number */ + if ('v' == p2[0]) { + major = strtoul(&p2[1], NULL, 10); + } else { + major = strtoul(p2, NULL, 10); + } + if (2 == major) { + pmix_client_globals.myserver->proc_type = PMIX_PROC_SERVER | PMIX_PROC_V21; + pmix_client_globals.myserver->protocol = PMIX_PROTOCOL_V2; + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "V21 SERVER DETECTED"); + } else if (3 <= major) { + pmix_client_globals.myserver->proc_type = PMIX_PROC_SERVER | PMIX_PROC_V3; + pmix_client_globals.myserver->protocol = PMIX_PROTOCOL_V2; + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "V3 SERVER DETECTED"); + } } if (NULL != p2) { free(p2); @@ -581,7 +667,7 @@ static pmix_status_t parse_uri_file(char *filename, return PMIX_SUCCESS; } -static pmix_status_t try_connect(int *sd) +static pmix_status_t try_connect(char *uri, int *sd) { char *p, *p2, *host; struct sockaddr_in *in; @@ -591,17 +677,16 @@ static pmix_status_t try_connect(int *sd) bool retried = false; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "pmix:tcp try connect to %s", - mca_ptl_tcp_component.super.uri); + "pmix:tcp try connect to %s", uri); /* mark that we are the active module for this server */ pmix_client_globals.myserver->nptr->compat.ptl = &pmix_ptl_tcp_module; /* setup the path to the daemon rendezvous point */ memset(&mca_ptl_tcp_component.connection, 0, sizeof(struct sockaddr_storage)); - if (0 == strncmp(mca_ptl_tcp_component.super.uri, "tcp4", 4)) { + if (0 == strncmp(uri, "tcp4", 4)) { /* need to skip the tcp4: part */ - p = strdup(&mca_ptl_tcp_component.super.uri[7]); + p = strdup(&uri[7]); if (NULL == p) { PMIX_ERROR_LOG(PMIX_ERR_NOMEM); return PMIX_ERR_NOMEM; @@ -630,7 +715,7 @@ static pmix_status_t try_connect(int *sd) len = sizeof(struct sockaddr_in); } else { /* need to skip the tcp6: part */ - p = strdup(&mca_ptl_tcp_component.super.uri[7]); + p = strdup(&uri[7]); if (NULL == p) { PMIX_ERROR_LOG(PMIX_ERR_NOMEM); return PMIX_ERR_NOMEM; @@ -708,12 +793,14 @@ static pmix_status_t send_connect_ack(int sd) uid_t euid; gid_t egid; uint32_t u32; + bool self_defined = false; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "pmix:tcp SEND CONNECT ACK"); /* if we are a server, then we shouldn't be here */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { return PMIX_ERR_NOT_SUPPORTED; } @@ -731,7 +818,6 @@ static pmix_status_t send_connect_ack(int sd) PMIX_PSEC_CREATE_CRED(rc, pmix_globals.mypeer, NULL, 0, NULL, 0, &cred); if (PMIX_SUCCESS != rc) { - pmix_output(0, "OUCH: %d", __LINE__); return rc; } @@ -741,11 +827,31 @@ static pmix_status_t send_connect_ack(int sd) if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { flag = 0; /* reserve space for our nspace and rank info */ - sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(int); - } else { + sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t); + } else if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + flag = 2; + /* add space for our uid/gid for ACL purposes */ + sdsize += 2*sizeof(uint32_t); + /* if we already have an identifier, we need to pass it */ + if (0 < strlen(pmix_globals.myid.nspace) && + PMIX_RANK_INVALID != pmix_globals.myid.rank) { + sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t) + 1; + self_defined = true; + } else { + ++sdsize; // need space for the flag indicating if have id + } + } else { // must be a simple tool flag = 1; /* add space for our uid/gid for ACL purposes */ sdsize += 2*sizeof(uint32_t); + /* if we self-defined an identifier, we need to pass it */ + if (0 < strlen(pmix_globals.myid.nspace) && + PMIX_RANK_INVALID != pmix_globals.myid.rank) { + sdsize += 1 + strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t); + self_defined = true; + } else { + ++sdsize; // need space for the flag indicating if have id + } } /* add the name of our active sec module - we selected it @@ -838,6 +944,25 @@ static pmix_status_t send_connect_ack(int sd) memcpy(msg+csize, gds, strlen(gds)); csize += strlen(gds)+1; + /* if we are not a client and self-defined an identifier, we need to pass it */ + if (!PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + if (self_defined) { + flag = 1; + memcpy(msg+csize, &flag, 1); + ++csize; + memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace)); + csize += strlen(pmix_globals.myid.nspace)+1; + /* again, need to convert */ + u32 = htonl((uint32_t)pmix_globals.myid.rank); + memcpy(msg+csize, &u32, sizeof(uint32_t)); + csize += sizeof(uint32_t); + } else { + flag = 0; + memcpy(msg+csize, &flag, 1); + ++csize; + } + } + /* send the entire message across */ if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) { free(msg); @@ -921,12 +1046,23 @@ static pmix_status_t recv_connect_ack(int sd) return reply; } /* recv our nspace */ - rc = pmix_ptl_base_recv_blocking(sd, (char*)&pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1); + rc = pmix_ptl_base_recv_blocking(sd, nspace, PMIX_MAX_NSLEN+1); if (PMIX_SUCCESS != rc) { return rc; } - /* our rank is always zero */ - pmix_globals.myid.rank = 0; + /* if we already have our nspace, then just verify it matches */ + if (0 < strlen(pmix_globals.myid.nspace)) { + if (0 != strncmp(pmix_globals.myid.nspace, nspace, PMIX_MAX_NSLEN)) { + return PMIX_ERR_INIT; + } + } else { + (void)strncpy(pmix_globals.myid.nspace, nspace, PMIX_MAX_NSLEN); + } + /* if we already have a rank, then leave it alone */ + if (PMIX_RANK_INVALID == pmix_globals.myid.rank) { + /* our rank is always zero */ + pmix_globals.myid.rank = 0; + } /* get the server's nspace and rank so we can send to it */ if (NULL == pmix_client_globals.myserver->info) { @@ -1026,14 +1162,10 @@ static pmix_status_t df_search(char *dirname, char *prefix, "pmix:tcp: reading file %s", newdir); rc = parse_uri_file(newdir, &suri, &nsp, &rk); if (PMIX_SUCCESS == rc) { - if (NULL != mca_ptl_tcp_component.super.uri) { - free(mca_ptl_tcp_component.super.uri); - } - mca_ptl_tcp_component.super.uri = suri; /* go ahead and try to connect */ pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "pmix:tcp: attempting to connect to %s", suri); - if (PMIX_SUCCESS == try_connect(sd)) { + if (PMIX_SUCCESS == try_connect(suri, sd)) { (*nspace) = nsp; *rank = rk; closedir(cur_dirp); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.h b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.h index 6f7db225200..dd92a893818 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.h @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -46,6 +46,7 @@ typedef struct { bool disable_ipv6_family; struct sockaddr_storage connection; char *session_filename; + char *nspace_filename; char *system_filename; int wait_to_connect; int max_retries; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c index 863bd3b5d35..a880faa9c8d 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c @@ -13,7 +13,7 @@ * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2017 Research Organization for Information Science + * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -62,6 +62,7 @@ #include "src/util/pif.h" #include "src/util/show_help.h" #include "src/util/strnlen.h" +#include "src/common/pmix_iof.h" #include "src/server/pmix_server_ops.h" #include "src/mca/bfrops/base/base.h" #include "src/mca/gds/base/base.h" @@ -76,7 +77,7 @@ static int component_register(void); static int component_query(pmix_mca_base_module_t **module, int *priority); static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, bool *need_listener); - +static pmix_status_t setup_fork(const pmix_proc_t *proc, char ***env); /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it @@ -101,7 +102,8 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, }, .priority = 30, .uri = NULL, - .setup_listener = setup_listener + .setup_listener = setup_listener, + .setup_fork = setup_fork }, .session_tmpdir = NULL, .system_tmpdir = NULL, @@ -112,6 +114,7 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, .disable_ipv4_family = false, .disable_ipv6_family = true, .session_filename = NULL, + .nspace_filename = NULL, .system_filename = NULL, .wait_to_connect = 4, .max_retries = 2, @@ -230,8 +233,12 @@ static pmix_status_t component_open(void) /* check for environ-based directives * on system tmpdir to use */ - if (NULL != (tdir = getenv("PMIX_SERVER_TMPDIR"))) { - mca_ptl_tcp_component.session_tmpdir = strdup(tdir); + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + mca_ptl_tcp_component.session_tmpdir = strdup(pmix_server_globals.tmpdir); + } else { + if (NULL != (tdir = getenv("PMIX_SERVER_TMPDIR"))) { + mca_ptl_tcp_component.session_tmpdir = strdup(tdir); + } } if (NULL != (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) { @@ -268,12 +275,21 @@ pmix_status_t component_close(void) if (NULL != mca_ptl_tcp_component.session_filename) { unlink(mca_ptl_tcp_component.session_filename); } + if (NULL != mca_ptl_tcp_component.nspace_filename) { + unlink(mca_ptl_tcp_component.nspace_filename); + } if (NULL != urifile) { /* remove the file */ unlink(urifile); free(urifile); urifile = NULL; } + if (NULL != mca_ptl_tcp_component.session_tmpdir) { + free(mca_ptl_tcp_component.session_tmpdir); + } + if (NULL != mca_ptl_tcp_component.system_tmpdir) { + free(mca_ptl_tcp_component.system_tmpdir); + } return PMIX_SUCCESS; } @@ -283,6 +299,25 @@ static int component_query(pmix_mca_base_module_t **module, int *priority) return PMIX_SUCCESS; } +static pmix_status_t setup_fork(const pmix_proc_t *proc, char ***env) +{ + char *evar; + + if (0 > asprintf(&evar, "PMIX_SERVER_TMPDIR=%s", mca_ptl_tcp_component.session_tmpdir)) { + return PMIX_ERR_NOMEM; + } + pmix_argv_append_nosize(env, evar); + free(evar); + + if (0 > asprintf(&evar, "PMIX_SYSTEM_TMPDIR=%s", mca_ptl_tcp_component.system_tmpdir)) { + return PMIX_ERR_NOMEM; + } + pmix_argv_append_nosize(env, evar); + free(evar); + + return PMIX_SUCCESS; +} + /* if we are the server, then we need to discover the available * interfaces, filter them thru any given directives, and select * the one we will listen on for connection requests. This will @@ -504,7 +539,7 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, } lt = PMIX_NEW(pmix_listener_t); - lt->varname = strdup("PMIX_SERVER_URI2:PMIX_SERVER_URI21"); + lt->varname = strdup("PMIX_SERVER_URI3:PMIX_SERVER_URI2:PMIX_SERVER_URI21"); lt->protocol = PMIX_PROTOCOL_V2; lt->ptl = (struct pmix_ptl_module_t*)&pmix_ptl_tcp_module; lt->cbfunc = connection_handler; @@ -516,6 +551,7 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, printf("%s:%d socket() failed\n", __FILE__, __LINE__); goto sockerror; } + /* set reusing ports flag */ if (setsockopt (lt->socket, SOL_SOCKET, SO_REUSEADDR, (const char *)&flags, sizeof(flags)) < 0) { pmix_output(0, "ptl:tcp:create_listen: unable to set the " @@ -656,6 +692,7 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, FILE *fp; pid_t mypid; + /* first output to a file based on pid */ mypid = getpid(); if (0 > asprintf(&mca_ptl_tcp_component.session_filename, "%s/pmix.%s.tool.%d", mca_ptl_tcp_component.session_tmpdir, myhost, mypid)) { @@ -684,10 +721,63 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, if (0 != chmod(mca_ptl_tcp_component.session_filename, S_IRUSR | S_IWUSR | S_IRGRP)) { PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); CLOSE_THE_SOCKET(lt->socket); - free(mca_ptl_tcp_component.system_filename); - mca_ptl_tcp_component.system_filename = NULL; + free(mca_ptl_tcp_component.session_filename); + mca_ptl_tcp_component.session_filename = NULL; goto sockerror; } + + /* now output it into a file based on my nspace */ + + if (0 > asprintf(&mca_ptl_tcp_component.nspace_filename, "%s/pmix.%s.tool.%s", + mca_ptl_tcp_component.session_tmpdir, myhost, pmix_globals.myid.nspace)) { + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "WRITING TOOL FILE %s", + mca_ptl_tcp_component.nspace_filename); + fp = fopen(mca_ptl_tcp_component.nspace_filename, "w"); + if (NULL == fp) { + pmix_output(0, "Impossible to open the file %s in write mode\n", mca_ptl_tcp_component.nspace_filename); + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.nspace_filename); + mca_ptl_tcp_component.nspace_filename = NULL; + goto sockerror; + } + + /* output my URI */ + fprintf(fp, "%s\n", lt->uri); + /* add a flag that indicates we accept v2.1 protocols */ + fprintf(fp, "%s\n", PMIX_VERSION); + fclose(fp); + /* set the file mode */ + if (0 != chmod(mca_ptl_tcp_component.nspace_filename, S_IRUSR | S_IWUSR | S_IRGRP)) { + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.nspace_filename); + mca_ptl_tcp_component.nspace_filename = NULL; + goto sockerror; + } + } + /* if we are a tool and connected, then register any rendezvous files for cleanup */ + if (PMIX_PROC_IS_TOOL(pmix_globals.mypeer) && pmix_globals.connected) { + char **clnup = NULL, *cptr = NULL; + pmix_info_t dir; + if (NULL != mca_ptl_tcp_component.nspace_filename) { + pmix_argv_append_nosize(&clnup, mca_ptl_tcp_component.nspace_filename); + } + if (NULL != mca_ptl_tcp_component.session_filename) { + pmix_argv_append_nosize(&clnup, mca_ptl_tcp_component.session_filename); + } + if (NULL != clnup) { + cptr = pmix_argv_join(clnup, ','); + pmix_argv_free(clnup); + PMIX_INFO_LOAD(&dir, PMIX_REGISTER_CLEANUP, cptr, PMIX_STRING); + free(cptr); + PMIx_Job_control_nb(&pmix_globals.myid, 1, &dir, 1, NULL, NULL); + PMIX_INFO_DESTRUCT(&dir); + } } /* we need listener thread support */ @@ -930,6 +1020,7 @@ static void connection_handler(int sd, short args, void *cbdata) if (0 == flag) { /* they must be a client, so get their nspace/rank */ + proc_type = PMIX_PROC_CLIENT; PMIX_STRNLEN(msglen, mg, cnt); if (msglen < cnt) { nspace = mg; @@ -956,6 +1047,33 @@ static void connection_handler(int sd, short args, void *cbdata) } } else if (1 == flag) { /* they are a tool */ + proc_type = PMIX_PROC_TOOL; + /* extract the uid/gid */ + if (sizeof(uint32_t) <= cnt) { + memcpy(&u32, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + pnd->uid = ntohl(u32); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + if (sizeof(uint32_t) <= cnt) { + memcpy(&u32, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + pnd->gid = ntohl(u32); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + } else if (2 == flag) { + /* they are a launcher */ + proc_type = PMIX_PROC_LAUNCHER; /* extract the uid/gid */ if (sizeof(uint32_t) <= cnt) { memcpy(&u32, mg, sizeof(uint32_t)); @@ -1002,15 +1120,17 @@ static void connection_handler(int sd, short args, void *cbdata) if (0 == strncmp(version, "2.0", 3)) { /* the 2.0 release handshake ends with the version string */ - proc_type = PMIX_PROC_V20; + proc_type = proc_type | PMIX_PROC_V20; bfrops = "v20"; bftype = pmix_bfrops_globals.default_type; // we can't know any better gds = NULL; } else { - if (0 == strncmp(version, "2.1", 3)) { - proc_type = PMIX_PROC_V21; - } else if (0 == strncmp(version, "3", 1)) { - proc_type = PMIX_PROC_V3; + int major; + major = strtoul(version, NULL, 10); + if (2 == major) { + proc_type = proc_type | PMIX_PROC_V21; + } else if (3 <= major) { + proc_type = proc_type | PMIX_PROC_V3; } else { free(msg); rc = PMIX_ERR_NOT_SUPPORTED; @@ -1059,38 +1179,79 @@ static void connection_handler(int sd, short args, void *cbdata) } /* see if this is a tool connection request */ - if (1 == flag) { + if (0 != flag) { /* does the server support tool connections? */ if (NULL == pmix_host_server.tool_connected) { /* send an error reply to the client */ rc = PMIX_ERR_NOT_SUPPORTED; goto error; } - /* setup the info array to pass the relevant info - * to the server - starting with the version, if present */ - n = 0; - PMIX_STRNLEN(msglen, mg, cnt); - if (msglen < cnt) { - pnd->ninfo = 4; - PMIX_INFO_CREATE(pnd->info, pnd->ninfo); - (void)strncpy(pnd->info[n].key, PMIX_VERSION_INFO, PMIX_MAX_KEYLEN); - pnd->info[n].value.type = PMIX_STRING; - pnd->info[n].value.data.string = strdup(mg); - ++n; + + if (PMIX_PROC_V3 & proc_type) { + /* the caller will have provided a flag indicating + * whether or not they have an assigned nspace/rank */ + if (cnt < 1) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + memcpy(&flag, mg, 1); + ++mg; + --cnt; + if (flag) { + PMIX_STRNLEN(msglen, mg, cnt); + if (msglen < cnt) { + nspace = mg; + mg += strlen(nspace) + 1; + cnt -= strlen(nspace) + 1; + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + if (sizeof(pmix_rank_t) <= cnt) { + /* have to convert this to host order */ + memcpy(&u32, mg, sizeof(uint32_t)); + rank = ntohl(u32); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + pnd->ninfo = 5; + } else { + pnd->ninfo = 3; + } } else { pnd->ninfo = 3; - PMIX_INFO_CREATE(pnd->info, pnd->ninfo); } + + /* setup the info array to pass the relevant info + * to the server */ + n = 0; + PMIX_INFO_CREATE(pnd->info, pnd->ninfo); + /* provide the version */ + PMIX_INFO_LOAD(&pnd->info[n], PMIX_VERSION_INFO, version, PMIX_STRING); + ++n; /* provide the user id */ - (void)strncpy(pnd->info[n].key, PMIX_USERID, PMIX_MAX_KEYLEN); - pnd->info[n].value.type = PMIX_UINT32; - pnd->info[n].value.data.uint32 = pnd->uid; + PMIX_INFO_LOAD(&pnd->info[n], PMIX_USERID, &pnd->uid, PMIX_UINT32); ++n; /* and the group id */ - (void)strncpy(pnd->info[n].key, PMIX_GRPID, PMIX_MAX_KEYLEN); - pnd->info[n].value.type = PMIX_UINT32; - pnd->info[n].value.data.uint32 = pnd->gid; + PMIX_INFO_LOAD(&pnd->info[n], PMIX_GRPID, &pnd->gid, PMIX_UINT32); ++n; + /* if we have it, pass along our ID */ + if (flag) { + PMIX_INFO_LOAD(&pnd->info[n], PMIX_NSPACE, nspace, PMIX_STRING); + ++n; + PMIX_INFO_LOAD(&pnd->info[n], PMIX_RANK, &rank, PMIX_PROC_RANK); + ++n; + } /* pass along the proc_type */ pnd->proc_type = proc_type; /* pass along the bfrop, buffer_type, and sec fields so @@ -1106,7 +1267,8 @@ static void connection_handler(int sd, short args, void *cbdata) /* release the msg */ free(msg); /* request an nspace for this requestor - it will - * automatically be assigned rank=0 */ + * automatically be assigned rank=0 if the rank + * isn't already known */ pmix_host_server.tool_connected(pnd->info, pnd->ninfo, cnct_cbfunc, pnd); return; } @@ -1156,7 +1318,7 @@ static void connection_handler(int sd, short args, void *cbdata) return; } /* mark that this peer is a client of the given type */ - peer->proc_type = PMIX_PROC_CLIENT | proc_type; + peer->proc_type = proc_type; /* save the protocol */ peer->protocol = pnd->protocol; /* add in the nspace pointer */ @@ -1328,6 +1490,7 @@ static void process_cbfunc(int sd, short args, void *cbdata) uint32_t u32; pmix_info_t ginfo; pmix_byte_object_t cred; + pmix_iof_req_t *req; /* acquire the object */ PMIX_ACQUIRE_OBJECT(cd); @@ -1414,7 +1577,7 @@ static void process_cbfunc(int sd, short args, void *cbdata) return; } /* mark the peer proc type */ - peer->proc_type = PMIX_PROC_TOOL | pnd->proc_type; + peer->proc_type = pnd->proc_type; /* save the protocol */ peer->protocol = pnd->protocol; /* add in the nspace pointer */ @@ -1476,6 +1639,22 @@ static void process_cbfunc(int sd, short args, void *cbdata) nptr->version_stored = true; } + /* automatically setup to forward output to the tool */ + req = PMIX_NEW(pmix_iof_req_t); + if (NULL == req) { + PMIX_RELEASE(peer); + pmix_list_remove_item(&pmix_server_globals.nspaces, &nptr->super); + PMIX_RELEASE(nptr); // will release the info object + CLOSE_THE_SOCKET(pnd->sd); + goto done; + } + PMIX_RETAIN(peer); + req->peer = peer; + req->pname.nspace = strdup(pmix_globals.myid.nspace); + req->pname.rank = pmix_globals.myid.rank; + req->channels = PMIX_FWD_STDOUT_CHANNEL | PMIX_FWD_STDERR_CHANNEL | PMIX_FWD_STDDIAG_CHANNEL; + pmix_list_append(&pmix_globals.iof_requests, &req->super); + /* validate the connection */ cred.bytes = pnd->cred; cred.size = pnd->len; @@ -1503,6 +1682,7 @@ static void process_cbfunc(int sd, short args, void *cbdata) /* probably cannot send an error reply if we are out of memory */ return; } + info->peerid = peer->index; /* start the events for this tool */ pmix_event_assign(&peer->recv_event, pmix_globals.evbase, peer->sd, diff --git a/opal/mca/pmix/pmix3x/pmix/src/runtime/help-pmix-runtime.txt b/opal/mca/pmix/pmix3x/pmix/src/runtime/help-pmix-runtime.txt index c1890837bf2..9e17710a6b0 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/runtime/help-pmix-runtime.txt +++ b/opal/mca/pmix/pmix3x/pmix/src/runtime/help-pmix-runtime.txt @@ -12,7 +12,7 @@ # All rights reserved. # Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. # Copyright (c) 2014 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2017 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -67,3 +67,23 @@ A received msg header indicates a size that is too large: If you believe this msg is legitimate, please increase the max msg size via the ptl_base_max_msg_size parameter. +# +[tool:no-server] +A call was made to PMIx_tool_connect_to_server, but no information +was given as to which server the tool should be connected. Accepted +attributes include: + + - PMIX_CONNECT_TO_SYSTEM: connect solely to the system server + + - PMIX_CONNECT_SYSTEM_FIRST: a request to use the system server first, + if existing, and then look for the server specified in a different + attribute + + - PMIX_SERVER_URI: connect to the server at the given URI + + - PMIX_SERVER_NSPACE: connect to the server of a given nspace + + - PMIX_SERVER_PIDINFO: connect to a server embedded in the process with + the given pid + +Please correct your program and try again. diff --git a/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_finalize.c b/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_finalize.c index bdb364749bc..bdfe4ebc416 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_finalize.c +++ b/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_finalize.c @@ -13,7 +13,7 @@ * Copyright (c) 2010-2015 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2016 Research Organization for Information Science + * Copyright (c) 2016-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -37,6 +37,7 @@ #include "src/mca/gds/base/base.h" #include "src/mca/pif/base/base.h" #include "src/mca/pinstalldirs/base/base.h" +#include "src/mca/plog/base/base.h" #include "src/mca/pnet/base/base.h" #include "src/mca/preg/base/base.h" #include "src/mca/psec/base/base.h" @@ -49,12 +50,6 @@ extern int pmix_initialized; extern bool pmix_init_called; -static void __pmix_attribute_destructor__ pmix_cleanup (void) -{ - /* finalize the class/object system */ - pmix_class_finalize(); -} - void pmix_rte_finalize(void) { if( --pmix_initialized != 0 ) { @@ -66,6 +61,9 @@ void pmix_rte_finalize(void) } + /* close plog */ + (void)pmix_mca_base_framework_close(&pmix_plog_base_framework); + /* close preg */ (void)pmix_mca_base_framework_close(&pmix_preg_base_framework); @@ -75,9 +73,6 @@ void pmix_rte_finalize(void) /* close the security framework */ (void)pmix_mca_base_framework_close(&pmix_psec_base_framework); - /* close the pnet framework */ - (void)pmix_mca_base_framework_close(&pmix_pnet_base_framework); - /* close bfrops */ (void)pmix_mca_base_framework_close(&pmix_bfrops_base_framework); @@ -109,6 +104,12 @@ void pmix_rte_finalize(void) PMIX_RELEASE(pmix_globals.mypeer); PMIX_DESTRUCT(&pmix_globals.events); PMIX_LIST_DESTRUCT(&pmix_globals.cached_events); + { + pmix_notify_caddy_t *cd; + while (NULL != (cd=(pmix_notify_caddy_t *)pmix_ring_buffer_pop(&pmix_globals.notifications))) { + PMIX_RELEASE(cd); + } + } PMIX_DESTRUCT(&pmix_globals.notifications); PMIX_LIST_DESTRUCT(&pmix_globals.iof_requests); @@ -117,9 +118,4 @@ void pmix_rte_finalize(void) (void)pmix_progress_thread_stop(NULL); } - -#if PMIX_NO_LIB_DESTRUCTOR - pmix_cleanup(); -#endif - } diff --git a/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_init.c b/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_init.c index bddc958f3b4..5f8e95455a0 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_init.c +++ b/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_init.c @@ -45,6 +45,7 @@ #include "src/mca/gds/base/base.h" #include "src/mca/pif/base/base.h" #include "src/mca/pinstalldirs/base/base.h" +#include "src/mca/plog/base/base.h" #include "src/mca/pnet/base/base.h" #include "src/mca/psec/base/base.h" #include "src/mca/preg/base/base.h" @@ -151,7 +152,8 @@ int pmix_rte_init(pmix_proc_type_t type, } /* setup the globals structure */ - memset(&pmix_globals.myid, 0, sizeof(pmix_proc_t)); + memset(&pmix_globals.myid.nspace, 0, PMIX_MAX_NSLEN+1); + pmix_globals.myid.rank = PMIX_RANK_INVALID; PMIX_CONSTRUCT(&pmix_globals.events, pmix_events_t); pmix_globals.event_window.tv_sec = pmix_event_caching_window; pmix_globals.event_window.tv_usec = 0; @@ -248,21 +250,21 @@ int pmix_rte_init(pmix_proc_type_t type, * time of connection to that peer */ /* open the bfrops and select the active plugins */ - if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_bfrops_base_framework, 0)) ) { + if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_bfrops_base_framework, 0)) ) { error = "pmix_bfrops_base_open"; goto return_error; } - if( PMIX_SUCCESS != (ret = pmix_bfrop_base_select()) ) { + if (PMIX_SUCCESS != (ret = pmix_bfrop_base_select()) ) { error = "pmix_bfrops_base_select"; goto return_error; } /* open the ptl and select the active plugins */ - if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_ptl_base_framework, 0)) ) { + if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_ptl_base_framework, 0)) ) { error = "pmix_ptl_base_open"; goto return_error; } - if( PMIX_SUCCESS != (ret = pmix_ptl_base_select()) ) { + if (PMIX_SUCCESS != (ret = pmix_ptl_base_select()) ) { error = "pmix_ptl_base_select"; goto return_error; } @@ -283,11 +285,11 @@ int pmix_rte_init(pmix_proc_type_t type, } /* open the gds and select the active plugins */ - if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_gds_base_framework, 0)) ) { + if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_gds_base_framework, 0)) ) { error = "pmix_gds_base_open"; goto return_error; } - if( PMIX_SUCCESS != (ret = pmix_gds_base_select(info, ninfo)) ) { + if (PMIX_SUCCESS != (ret = pmix_gds_base_select(info, ninfo)) ) { error = "pmix_gds_base_select"; goto return_error; } @@ -298,23 +300,23 @@ int pmix_rte_init(pmix_proc_type_t type, return ret; } - /* open the pnet and select the active modules for this environment */ - if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_pnet_base_framework, 0))) { - error = "pmix_pnet_base_open"; + /* open the preg and select the active plugins */ + if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_preg_base_framework, 0)) ) { + error = "pmix_preg_base_open"; goto return_error; } - if (PMIX_SUCCESS != (ret = pmix_pnet_base_select())) { - error = "pmix_pnet_base_select"; + if (PMIX_SUCCESS != (ret = pmix_preg_base_select()) ) { + error = "pmix_preg_base_select"; goto return_error; } - /* open the preg and select the active plugins */ - if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_preg_base_framework, 0)) ) { - error = "pmix_preg_base_open"; + /* open the plog and select the active plugins */ + if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_plog_base_framework, 0)) ) { + error = "pmix_plog_base_open"; goto return_error; } - if( PMIX_SUCCESS != (ret = pmix_preg_base_select()) ) { - error = "pmix_preg_base_select"; + if (PMIX_SUCCESS != (ret = pmix_plog_base_select()) ) { + error = "pmix_plog_base_select"; goto return_error; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c index 58316d0d6b0..9151cdbd792 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c +++ b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c @@ -60,13 +60,14 @@ #include "src/mca/base/base.h" #include "src/mca/base/pmix_mca_base_var.h" #include "src/mca/pinstalldirs/base/base.h" -#include "src/mca/pnet/pnet.h" +#include "src/mca/pnet/base/base.h" #include "src/runtime/pmix_progress_threads.h" #include "src/runtime/pmix_rte.h" #include "src/mca/bfrops/base/base.h" #include "src/mca/gds/base/base.h" #include "src/mca/preg/preg.h" #include "src/mca/ptl/base/base.h" +#include "src/hwloc/hwloc-internal.h" /* the server also needs access to client operations * as it can, and often does, behave as a client */ @@ -84,10 +85,6 @@ static char *gds_mode = NULL; static pid_t mypid; // local functions for connection support -static void server_message_handler(struct pmix_peer_t *pr, - pmix_ptl_hdr_t *hdr, - pmix_buffer_t *buf, void *cbdata); - static void iof_eviction_cbfunc(struct pmix_hotel_t *hotel, int room_num, void *occupant) @@ -96,38 +93,9 @@ static void iof_eviction_cbfunc(struct pmix_hotel_t *hotel, PMIX_RELEASE(cache); } -PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, - pmix_info_t info[], size_t ninfo) +pmix_status_t pmix_server_initialize(void) { - pmix_ptl_posted_recv_t *req; pmix_status_t rc; - size_t n, m; - pmix_kval_t *kv; - bool protect, nspace_given = false, rank_given = false; - pmix_info_t ginfo; - char *protected[] = { - PMIX_USERID, - PMIX_GRPID, - PMIX_SOCKET_MODE, - PMIX_SERVER_TOOL_SUPPORT, - PMIX_SERVER_SYSTEM_SUPPORT, - NULL - }; - char *evar; - pmix_rank_info_t *rinfo; - - PMIX_ACQUIRE_THREAD(&pmix_global_lock); - - pmix_output_verbose(2, pmix_server_globals.base_output, - "pmix:server init called"); - - /* setup the runtime - this init's the globals, - * opens and initializes the required frameworks */ - if (PMIX_SUCCESS != (rc = pmix_rte_init(PMIX_PROC_SERVER, info, ninfo, NULL))) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } /* setup the server-specific globals */ PMIX_CONSTRUCT(&pmix_server_globals.clients, pmix_pointer_array_t); @@ -202,10 +170,74 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, pmix_server_globals.base_verbose); } + return PMIX_SUCCESS; +} + +PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, + pmix_info_t info[], size_t ninfo) +{ + pmix_ptl_posted_recv_t *req; + pmix_status_t rc; + size_t n, m; + pmix_kval_t *kv; + bool protect, nspace_given = false, rank_given = false; + bool topology_req = false; + pmix_info_t ginfo; + char *protected[] = { + PMIX_USERID, + PMIX_GRPID, + PMIX_SOCKET_MODE, + PMIX_SERVER_TOOL_SUPPORT, + PMIX_SERVER_SYSTEM_SUPPORT, + PMIX_SERVER_GATEWAY, + NULL + }; + char *evar; + pmix_rank_info_t *rinfo; + pmix_proc_type_t ptype = PMIX_PROC_SERVER; + + PMIX_ACQUIRE_THREAD(&pmix_global_lock); + + pmix_output_verbose(2, pmix_server_globals.base_output, + "pmix:server init called"); + /* setup the function pointers */ - memset(&pmix_host_server, 0, sizeof(pmix_server_module_t)); pmix_host_server = *module; + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_SERVER_GATEWAY, PMIX_MAX_KEYLEN)) { + if (PMIX_INFO_TRUE(&info[n])) { + ptype |= PMIX_PROC_GATEWAY; + } + } else if (0 == strncmp(info[n].key, PMIX_SERVER_TMPDIR, PMIX_MAX_KEYLEN)) { + pmix_server_globals.tmpdir = strdup(info[n].value.data.string); + } + } + } + if (NULL == pmix_server_globals.tmpdir) { + if (NULL == (evar = getenv("PMIX_SERVER_TMPDIR"))) { + pmix_server_globals.tmpdir = strdup(pmix_tmp_directory()); + } else { + pmix_server_globals.tmpdir = strdup(evar); + } + } + + /* setup the runtime - this init's the globals, + * opens and initializes the required frameworks */ + if (PMIX_SUCCESS != (rc = pmix_rte_init(ptype, info, ninfo, NULL))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + + /* setup the server-specific globals */ + if (PMIX_SUCCESS != (rc = pmix_server_initialize())) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + /* assign our internal bfrops module */ pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(NULL); if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) { @@ -240,15 +272,13 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } + PMIX_INFO_DESTRUCT(&ginfo); - /* copy need parts over to the client_globals.myserver field + /* copy needed parts over to the client_globals.myserver field * so that calls into client-side functions will use our peer */ pmix_client_globals.myserver = PMIX_NEW(pmix_peer_t); PMIX_RETAIN(pmix_globals.mypeer->nptr); pmix_client_globals.myserver->nptr = pmix_globals.mypeer->nptr; - /* construct the global notification ring buffer */ - PMIX_CONSTRUCT(&pmix_globals.notifications, pmix_ring_buffer_t); - pmix_ring_buffer_init(&pmix_globals.notifications, 256); /* get our available security modules */ security_mode = pmix_psec_base_get_available_modules(); @@ -270,37 +300,41 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, if (0 == strncmp(info[n].key, PMIX_SERVER_NSPACE, PMIX_MAX_KEYLEN)) { (void)strncpy(pmix_globals.myid.nspace, info[n].value.data.string, PMIX_MAX_NSLEN); nspace_given = true; - continue; - } - if (0 == strncmp(info[n].key, PMIX_SERVER_RANK, PMIX_MAX_KEYLEN)) { + } else if (0 == strncmp(info[n].key, PMIX_SERVER_RANK, PMIX_MAX_KEYLEN)) { pmix_globals.myid.rank = info[n].value.data.rank; rank_given = true; - continue; - } - /* check the list of protected keys */ - protect = false; - for (m=0; NULL != protected[m]; m++) { - if (0 == strcmp(info[n].key, protected[m])) { - protect = true; - break; + } else if (0 == strncmp(info[n].key, PMIX_TOPOLOGY, PMIX_MAX_KEYLEN) || + 0 == strncmp(info[n].key, PMIX_TOPOLOGY_XML, PMIX_MAX_KEYLEN) || + 0 == strncmp(info[n].key, PMIX_TOPOLOGY_FILE, PMIX_MAX_KEYLEN) || + 0 == strncmp(info[n].key, PMIX_HWLOC_XML_V1, PMIX_MAX_KEYLEN) || + 0 == strncmp(info[n].key, PMIX_HWLOC_XML_V2, PMIX_MAX_KEYLEN)) { + topology_req = true; + } else { + /* check the list of protected keys */ + protect = false; + for (m=0; NULL != protected[m]; m++) { + if (0 == strcmp(info[n].key, protected[m])) { + protect = true; + break; + } } + if (protect) { + continue; + } + /* store and pass along to every client */ + kv = PMIX_NEW(pmix_kval_t); + kv->key = strdup(info[n].key); + PMIX_VALUE_CREATE(kv->value, 1); + PMIX_BFROPS_VALUE_XFER(rc, pmix_globals.mypeer, + kv->value, &info[n].value); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(kv); + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + pmix_list_append(&pmix_server_globals.gdata, &kv->super); } - if (protect) { - continue; - } - /* store and pass along to every client */ - kv = PMIX_NEW(pmix_kval_t); - kv->key = strdup(info[n].key); - PMIX_VALUE_CREATE(kv->value, 1); - PMIX_BFROPS_VALUE_XFER(rc, pmix_globals.mypeer, - kv->value, &info[n].value); - if (PMIX_SUCCESS != rc) { - PMIX_RELEASE(kv); - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - pmix_list_append(&pmix_server_globals.gdata, &kv->super); } } @@ -345,18 +379,45 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, PMIX_RETAIN(pmix_globals.mypeer->info); pmix_client_globals.myserver->info = pmix_globals.mypeer->info; + /* open the pnet framework and select the active modules for this environment */ + if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pnet_base_framework, 0))) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + if (PMIX_SUCCESS != (rc = pmix_pnet_base_select())) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + + /* if requested, setup the topology */ + if (topology_req) { + if (PMIX_SUCCESS != (rc = pmix_hwloc_get_topology(info, ninfo))) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + } + /* setup the wildcard recv for inbound messages from clients */ req = PMIX_NEW(pmix_ptl_posted_recv_t); req->tag = UINT32_MAX; - req->cbfunc = server_message_handler; + req->cbfunc = pmix_server_message_handler; /* add it to the end of the list of recvs */ pmix_list_append(&pmix_ptl_globals.posted_recvs, &req->super); + /* if we are a gateway, setup our IOF events */ + if (PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { + /* setup IOF */ + PMIX_IOF_SINK_DEFINE(&pmix_client_globals.iof_stdout, &pmix_globals.myid, + 1, PMIX_FWD_STDOUT_CHANNEL, pmix_iof_write_handler); + PMIX_IOF_SINK_DEFINE(&pmix_client_globals.iof_stderr, &pmix_globals.myid, + 2, PMIX_FWD_STDERR_CHANNEL, pmix_iof_write_handler); + } + /* start listening for connections */ if (PMIX_SUCCESS != pmix_ptl_base_start_listening(info, ninfo)) { pmix_show_help("help-pmix-server.txt", "listener-thread-start", true); - PMIx_server_finalize(); PMIX_RELEASE_THREAD(&pmix_global_lock); + PMIx_server_finalize(); return PMIX_ERR_INIT; } @@ -430,6 +491,8 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) } PMIX_LIST_DESTRUCT(&pmix_server_globals.nspaces); + pmix_hwloc_cleanup(); + if (NULL != security_mode) { free(security_mode); } @@ -445,12 +508,26 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) if (NULL != gds_mode) { free(gds_mode); } + if (NULL != pmix_server_globals.tmpdir) { + free(pmix_server_globals.tmpdir); + } + /* close the pnet framework */ + (void)pmix_mca_base_framework_close(&pmix_pnet_base_framework); + + + PMIX_RELEASE_THREAD(&pmix_global_lock); + PMIX_DESTRUCT_LOCK(&pmix_global_lock); + pmix_rte_finalize(); + if (NULL != pmix_globals.mypeer) { + PMIX_RELEASE(pmix_globals.mypeer); + } pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server finalize complete"); - PMIX_RELEASE_THREAD(&pmix_global_lock); - PMIX_DESTRUCT_LOCK(&pmix_global_lock); + + /* finalize the class/object system */ + pmix_class_finalize(); return PMIX_SUCCESS; } @@ -563,7 +640,13 @@ static void _deregister_nspace(int sd, short args, void *cbdata) "pmix:server _deregister_nspace %s", cd->proc.nspace); - /* see if we already have this nspace */ + /* release any job-level messaging resources */ + pmix_pnet.deregister_nspace(cd->proc.nspace); + + /* let our local storage clean up */ + PMIX_GDS_DEL_NSPACE(rc, cd->proc.nspace); + + /* release this nspace */ PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { if (0 == strcmp(tmp->nspace, cd->proc.nspace)) { pmix_list_remove_item(&pmix_server_globals.nspaces, &tmp->super); @@ -572,12 +655,6 @@ static void _deregister_nspace(int sd, short args, void *cbdata) } } - /* let our local storage clean up */ - PMIX_GDS_DEL_NSPACE(rc, cd->proc.nspace); - - /* release any job-level messaging resources */ - pmix_pnet.local_app_finalized(cd->proc.nspace); - /* release the caller */ if (NULL != cd->opcbfunc) { cd->opcbfunc(rc, cd->cbdata); @@ -755,9 +832,9 @@ void pmix_server_execute_collective(int sd, short args, void *cbdata) } else if (PMIX_CONNECTNB_CMD == trk->type) { pmix_host_server.connect(trk->pcs, trk->npcs, trk->info, trk->ninfo, - trk->cnct_cbfunc, trk); + trk->op_cbfunc, trk); } else if (PMIX_DISCONNECTNB_CMD == trk->type) { - pmix_host_server.disconnect(trk->pname.nspace, + pmix_host_server.disconnect(trk->pcs, trk->npcs, trk->info, trk->ninfo, trk->op_cbfunc, trk); } else { @@ -931,6 +1008,7 @@ static void _deregister_client(int sd, short args, void *cbdata) pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; pmix_rank_info_t *info; pmix_nspace_t *nptr, *tmp; + pmix_peer_t *peer; PMIX_ACQUIRE_OBJECT(cd); @@ -953,6 +1031,38 @@ static void _deregister_client(int sd, short args, void *cbdata) /* find and remove this client */ PMIX_LIST_FOREACH(info, &nptr->ranks, pmix_rank_info_t) { if (info->pname.rank == cd->proc.rank) { + /* if this client failed to call finalize, we still need + * to restore any allocations that were given to it */ + if (NULL == (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, info->peerid))) { + /* this peer never connected, and hence it won't finalize, + * so account for it here */ + nptr->nfinalized++; + /* even if they never connected, resources were allocated + * to them, so we need to ensure they are properly released */ + pmix_pnet.child_finalized(&cd->proc); + } else { + if (!peer->finalized) { + /* this peer connected to us, but is being deregistered + * without having finalized. This usually means an + * abnormal termination that was picked up by + * our host prior to our seeing the connection drop. + * It is also possible that we missed the dropped + * connection, so mark the peer as finalized so + * we don't duplicate account for it and take care + * of it here */ + peer->finalized = true; + nptr->nfinalized++; + } + /* resources may have been allocated to them, so + * ensure they get cleaned up - this isn't true + * for tools, so don't clean them up */ + if (!PMIX_PROC_IS_TOOL(peer)) { + pmix_pnet.child_finalized(&cd->proc); + } + } + if (nptr->nlocalprocs == nptr->nfinalized) { + pmix_pnet.local_app_finalized(nptr); + } pmix_list_remove_item(&nptr->ranks, &info->super); PMIX_RELEASE(info); break; @@ -1050,6 +1160,12 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_fork(const pmix_proc_t *proc, char * /* pass our available gds modules */ pmix_setenv("PMIX_GDS_MODULE", gds_mode, true, env); + /* get any PTL contribution such as tmpdir settings for session files */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_setup_fork(proc, env))) { + PMIX_ERROR_LOG(rc); + return rc; + } + /* get any network contribution */ if (PMIX_SUCCESS != (rc = pmix_pnet.setup_fork(proc, env))) { PMIX_ERROR_LOG(rc); @@ -1331,19 +1447,20 @@ static void _setup_op(pmix_status_t rc, void *cbdata) static void _setup_app(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; - pmix_buffer_t buffer; - pmix_byte_object_t blob; pmix_setup_caddy_t *fcd = NULL; pmix_status_t rc; pmix_list_t ilist; pmix_kval_t *kv; + size_t n; PMIX_ACQUIRE_OBJECT(cd); PMIX_CONSTRUCT(&ilist, pmix_list_t); /* pass to the network libraries */ - if (PMIX_SUCCESS != (rc = pmix_pnet.setup_app(cd->nspace, &ilist))) { + if (PMIX_SUCCESS != (rc = pmix_pnet.allocate(cd->nspace, + cd->info, cd->ninfo, + &ilist))) { goto depart; } @@ -1355,31 +1472,20 @@ static void _setup_app(int sd, short args, void *cbdata) goto depart; } - /* if anything came back, construct the blob */ - if (0 < pmix_list_get_size(&ilist)) { - PMIX_CONSTRUCT(&buffer, pmix_buffer_t); - PMIX_LIST_FOREACH(kv, &ilist, pmix_kval_t) { - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buffer, kv, 1, PMIX_KVAL); - if (PMIX_SUCCESS != rc) { - PMIX_DESTRUCT(&blob); - PMIX_RELEASE(fcd); - fcd = NULL; - goto depart; - } - } - PMIX_INFO_CREATE(fcd->info, 1); + /* if anything came back, construct an info array */ + if (0 < (fcd->ninfo = pmix_list_get_size(&ilist))) { + PMIX_INFO_CREATE(fcd->info, fcd->ninfo); if (NULL == fcd->info) { - PMIX_DESTRUCT(&blob); + rc = PMIX_ERR_NOMEM; PMIX_RELEASE(fcd); - fcd = NULL; goto depart; } - fcd->ninfo = 1; - PMIX_BYTE_OBJECT_CONSTRUCT(&blob); - PMIX_BYTE_OBJECT_LOAD(&blob, buffer.base_ptr, buffer.bytes_used); - PMIX_DESTRUCT(&buffer); - PMIX_INFO_LOAD(&fcd->info[0], PMIX_PNET_SETUP_APP, &blob, PMIX_BYTE_OBJECT); - PMIX_BYTE_OBJECT_DESTRUCT(&blob); + n = 0; + PMIX_LIST_FOREACH(kv, &ilist, pmix_kval_t) { + (void)strncpy(fcd->info[n].key, kv->key, PMIX_MAX_KEYLEN); + pmix_value_xfer(&fcd->info[n].value, kv->value); + ++n; + } } depart: @@ -1492,7 +1598,8 @@ static void _iofdeliver(int sd, short args, void *cbdata) int ignore; pmix_output_verbose(2, pmix_server_globals.iof_output, - "PMIX:SERVER delivering IOF"); + "PMIX:SERVER delivering IOF from %s on channel %0x", + PMIX_NAME_PRINT(cd->procs), cd->channels); /* cycle across our list of IOF requestors and see who wants * this channel from this source */ @@ -1501,11 +1608,20 @@ static void _iofdeliver(int sd, short args, void *cbdata) if (!(cd->channels & req->channels)) { continue; } - /* if the source matches the request, then forward this along */ + /* see if the source matches the request */ if (0 != strncmp(cd->procs->nspace, req->pname.nspace, PMIX_MAX_NSLEN) || (PMIX_RANK_WILDCARD != req->pname.rank && cd->procs->rank != req->pname.rank)) { continue; } + /* never forward back to the source! This can happen if the source + * is a launcher */ + if (NULL == req->peer->info || req->peer->finalized) { + continue; + } + if (0 == strncmp(cd->procs->nspace, req->peer->info->pname.nspace, PMIX_MAX_NSLEN) && + cd->procs->rank == req->peer->info->pname.rank) { + continue; + } found = true; /* setup the msg */ if (NULL == (msg = PMIX_NEW(pmix_buffer_t))) { @@ -1624,6 +1740,199 @@ pmix_status_t PMIx_server_IOF_deliver(const pmix_proc_t *source, return PMIX_SUCCESS; } +static void cirelease(void *cbdata) +{ + pmix_inventory_rollup_t *rollup = (pmix_inventory_rollup_t*)cbdata; + if (NULL != rollup->info) { + PMIX_INFO_FREE(rollup->info, rollup->ninfo); + } + PMIX_RELEASE(rollup); +} + +static void clct_complete(pmix_status_t status, + pmix_list_t *inventory, + void *cbdata) +{ + pmix_inventory_rollup_t *cd = (pmix_inventory_rollup_t*)cbdata; + pmix_kval_t *kv; + size_t n; + pmix_status_t rc; + + PMIX_ACQUIRE_THREAD(&cd->lock); + + /* collect the results */ + if (NULL != inventory) { + while (NULL != (kv = (pmix_kval_t*)pmix_list_remove_first(inventory))) { + pmix_list_append(&cd->payload, &kv->super); + } + } + if (PMIX_SUCCESS != status && PMIX_SUCCESS == cd->status) { + cd->status = status; + } + /* see if we are done */ + cd->replies++; + if (cd->replies == cd->requests) { + /* no need to continue tracking the input directives */ + cd->info = NULL; + cd->ninfo = 0; + if (NULL != cd->infocbfunc) { + /* convert the list to an array of pmix_info_t */ + cd->ninfo = pmix_list_get_size(&cd->payload); + if (0 < cd->ninfo) { + PMIX_INFO_CREATE(cd->info, cd->ninfo); + if (NULL == cd->info) { + cd->status = PMIX_ERR_NOMEM; + cd->ninfo = 0; + PMIX_RELEASE_THREAD(&cd->lock); + goto error; + } + /* transfer the results */ + n=0; + PMIX_LIST_FOREACH(kv, &cd->payload, pmix_kval_t) { + (void)strncpy(cd->info[n].key, kv->key, PMIX_MAX_KEYLEN); + rc = pmix_value_xfer(&cd->info[n].value, kv->value); + if (PMIX_SUCCESS != rc) { + PMIX_INFO_FREE(cd->info, cd->ninfo); + cd->status = rc; + break; + } + ++n; + } + } + /* now call the requestor back */ + PMIX_RELEASE_THREAD(&cd->lock); + cd->infocbfunc(cd->status, cd->info, cd->ninfo, cd->cbdata, cirelease, cd); + return; + } + } + /* continue to wait */ + PMIX_RELEASE_THREAD(&cd->lock); + return; + + error: + /* let them know */ + if (NULL != cd->infocbfunc) { + cd->infocbfunc(cd->status, NULL, 0, cd->cbdata, NULL, NULL); + } + PMIX_RELEASE(cd); + + +} +static void clct(int sd, short args, void *cbdata) +{ + pmix_inventory_rollup_t *cd = (pmix_inventory_rollup_t*)cbdata; + + + /* we only have one source at this time */ + cd->requests = 1; + + /* collect the pnet inventory */ + pmix_pnet.collect_inventory(cd->info, cd->ninfo, + clct_complete, cd); + + return; +} + +pmix_status_t PMIx_server_collect_inventory(pmix_info_t directives[], size_t ndirs, + pmix_info_cbfunc_t cbfunc, void *cbdata) +{ + pmix_inventory_rollup_t *cd; + + PMIX_ACQUIRE_THREAD(&pmix_global_lock); + if (pmix_globals.init_cntr <= 0) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_INIT; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + + /* need to threadshift this request */ + cd = PMIX_NEW(pmix_inventory_rollup_t); + if (NULL == cd) { + return PMIX_ERR_NOMEM; + } + cd->info = directives; + cd->ninfo = ndirs; + cd->infocbfunc = cbfunc; + cd->cbdata = cbdata; + PMIX_THREADSHIFT(cd, clct); + + return PMIX_SUCCESS; +} + +static void dlinv_complete(pmix_status_t status, void *cbdata) +{ + pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata; + + /* take the lock */ + PMIX_ACQUIRE_THREAD(&cd->lock); + + /* increment number of replies */ + cd->ndata++; // reuse field in shift_caddy + /* update status, if necessary */ + if (PMIX_SUCCESS != status && PMIX_SUCCESS == cd->status) { + cd->status = status; + } + if (cd->ncodes == cd->ndata) { + /* we are done - let the caller know */ + PMIX_RELEASE_THREAD(&cd->lock); + if (NULL != cd->cbfunc.opcbfn) { + cd->cbfunc.opcbfn(cd->status, cd->cbdata); + } + PMIX_RELEASE(cd); + return; + } + + PMIX_RELEASE_THREAD(&cd->lock); + return; +} + + +static void dlinv(int sd, short args, void *cbdata) +{ + pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata; + + /* only have one place to deliver inventory + * at this time */ + cd->ncodes = 1; // reuse field in shift_caddy + + pmix_pnet.deliver_inventory(cd->info, cd->ninfo, + cd->directives, cd->ndirs, + dlinv_complete, cd); + + return; + +} +pmix_status_t PMIx_server_deliver_inventory(pmix_info_t info[], size_t ninfo, + pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_shift_caddy_t *cd; + + PMIX_ACQUIRE_THREAD(&pmix_global_lock); + if (pmix_globals.init_cntr <= 0) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_INIT; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + + /* need to threadshift this request */ + cd = PMIX_NEW(pmix_shift_caddy_t); + if (NULL == cd) { + return PMIX_ERR_NOMEM; + } + cd->lock.active = false; + cd->info = info; + cd->ninfo = ninfo; + cd->directives = directives; + cd->ndirs = ndirs; + cd->cbfunc.opcbfn = cbfunc; + cd->cbdata = cbdata; + PMIX_THREADSHIFT(cd, dlinv); + + return PMIX_SUCCESS; + +} + /**** THE FOLLOWING CALLBACK FUNCTIONS ARE USED BY THE HOST SERVER **** **** THEY THEREFORE CAN OCCUR IN EITHER THE HOST SERVER'S THREAD **** **** CONTEXT, OR IN OUR OWN THREAD CONTEXT IF THE CALLBACK OCCURS **** @@ -1666,6 +1975,61 @@ static void op_cbfunc(pmix_status_t status, void *cbdata) PMIX_RELEASE(cd); } +static void connection_cleanup(int sd, short args, void *cbdata) +{ + pmix_server_caddy_t *cd = (pmix_server_caddy_t*)cbdata; + + pmix_ptl_base_lost_connection(cd->peer, PMIX_SUCCESS); + /* cleanup the caddy */ + PMIX_RELEASE(cd); +} + +static void op_cbfunc2(pmix_status_t status, void *cbdata) +{ + pmix_server_caddy_t *cd = (pmix_server_caddy_t*)cbdata; + pmix_buffer_t *reply; + pmix_status_t rc; + + /* no need to thread-shift here as no global data is + * being accessed */ + + /* setup the reply with the returned status */ + if (NULL == (reply = PMIX_NEW(pmix_buffer_t))) { + PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); + PMIX_RELEASE(cd); + return; + } + PMIX_BFROPS_PACK(rc, cd->peer, reply, &status, 1, PMIX_STATUS); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(reply); + PMIX_RELEASE(cd); + return; + } + + /* the function that created the server_caddy did a + * retain on the peer, so we don't have to worry about + * it still being present - send a copy to the originator */ + PMIX_PTL_SEND_ONEWAY(rc, cd->peer, reply, cd->hdr.tag); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(reply); + } + + /* ensure that we know the peer has finalized else we + * will generate an event - yes, it should have been + * done, but it is REALLY important that it be set */ + cd->peer->finalized = true; + /* cleanup any lingering references to this peer - note + * that we cannot call the lost_connection function + * directly as we need the connection to still + * exist for the message (queued above) to be + * sent. So we push this into an event, thus + * ensuring that it will "fire" after the message + * event has completed */ + PMIX_THREADSHIFT(cd, connection_cleanup); +} + static void _spcb(int sd, short args, void *cbdata) { pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata; @@ -1680,16 +2044,12 @@ static void _spcb(int sd, short args, void *cbdata) /* setup the reply with the returned status */ if (NULL == (reply = PMIX_NEW(pmix_buffer_t))) { PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); - PMIX_RELEASE(cd->cd); - PMIX_WAKEUP_THREAD(&cd->lock); - return; + goto cleanup; } PMIX_BFROPS_PACK(rc, cd->cd->peer, reply, &cd->status, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - PMIX_RELEASE(cd->cd); - PMIX_WAKEUP_THREAD(&cd->lock); - return; + goto cleanup; } if (PMIX_SUCCESS == cd->status) { /* pass back the name of the nspace */ @@ -1710,11 +2070,9 @@ static void _spcb(int sd, short args, void *cbdata) PMIX_BFROPS_PACK(rc, cd->cd->peer, reply, kv, 1, PMIX_KVAL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - PMIX_RELEASE(cd->cd); PMIX_RELEASE(reply); PMIX_DESTRUCT(&cb); - PMIX_WAKEUP_THREAD(&cd->lock); - return; + goto cleanup; } } PMIX_DESTRUCT(&cb); @@ -1725,6 +2083,8 @@ static void _spcb(int sd, short args, void *cbdata) * retain on the peer, so we don't have to worry about * it still being present - tell the originator the result */ PMIX_SERVER_QUEUE_REPLY(cd->cd->peer, cd->cd->hdr.tag, reply); + + cleanup: /* cleanup */ PMIX_RELEASE(cd->cd); PMIX_RELEASE(cd); @@ -2057,25 +2417,13 @@ static void _cnct(int sd, short args, void *cbdata) int i; pmix_server_caddy_t *cd; char **nspaces=NULL; - bool found, xchg; + bool found; pmix_proc_t proc; pmix_cb_t cb; pmix_kval_t *kptr; - pmix_nspace_t *nsptr = NULL; - size_t n; PMIX_ACQUIRE_OBJECT(scd); - /* see if this connect request was to return a new nspace/rank, or - * was just an exchange of info */ - xchg = false; - for (n=0; n < tracker->ninfo; n++) { - if (0 == strncmp(tracker->info[n].key, PMIX_CONNECT_XCHG_ONLY, PMIX_MAX_KEYLEN)) { - xchg = true; - break; - } - } - /* find the unique nspaces that are participating */ PMIX_LIST_FOREACH(cd, &tracker->local_cbs, pmix_server_caddy_t) { if (NULL == nspaces) { @@ -2111,27 +2459,6 @@ static void _cnct(int sd, short args, void *cbdata) goto cleanup; } if (PMIX_SUCCESS == scd->status) { - if (!xchg) { - if (NULL == nsptr) { - /* we have to track this nspace */ - nsptr = PMIX_NEW(pmix_nspace_t); - nsptr->nspace = strdup(scd->pname.nspace); - nsptr->all_registered = true; - /* we already counted the number of local procs */ - nsptr->nlocalprocs = pmix_list_get_size(&tracker->local_cbs); - pmix_list_append(&pmix_server_globals.nspaces, &nsptr->super); - } - /* if success, then provide the new nspace/rank */ - (void)strncpy(proc.nspace, scd->pname.nspace, PMIX_MAX_NSLEN); - proc.rank = scd->pname.rank; - PMIX_BFROPS_PACK(rc, cd->peer, reply, &proc, 1, PMIX_PROC); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(reply); - goto cleanup; - } - } - /* loop across all participating nspaces and include their * job-related info */ for (i=0; NULL != nspaces[i]; i++) { @@ -2224,16 +2551,13 @@ static void _cnct(int sd, short args, void *cbdata) PMIX_RELEASE(scd); } -static void cnct_cbfunc(pmix_status_t status, - char nspace[], int rank, - void *cbdata) +static void cnct_cbfunc(pmix_status_t status, void *cbdata) { pmix_server_trkr_t *tracker = (pmix_server_trkr_t*)cbdata; pmix_shift_caddy_t *scd; pmix_output_verbose(2, pmix_server_globals.base_output, - "server:cnct_cbfunc called with nspace %s", - (NULL == nspace) ? "NULL" : nspace); + "server:cnct_cbfunc called"); if (NULL == tracker) { /* nothing to do */ @@ -2247,10 +2571,6 @@ static void cnct_cbfunc(pmix_status_t status, return; } scd->status = status; - if (NULL != nspace) { - scd->pname.nspace = strdup(nspace); - } - scd->pname.rank = rank; scd->tracker = tracker; PMIX_THREADSHIFT(scd, _cnct); } @@ -2697,31 +3017,17 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, "recvd FINALIZE"); /* mark that this peer called finalize */ peer->finalized = true; - /* call the local server, if supported */ - if (NULL != pmix_host_server.client_finalized) { - PMIX_GDS_CADDY(cd, peer, tag); - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); - proc.rank = peer->info->pname.rank; - /* since the client is finalizing, remove them from any event - * registrations they may still have on our list */ - PMIX_LIST_FOREACH(reginfo, &pmix_server_globals.events, pmix_regevents_info_t) { - PMIX_LIST_FOREACH(prev, ®info->peers, pmix_peer_events_info_t) { - if (prev->peer == peer) { - pmix_list_remove_item(®info->peers, &prev->super); - PMIX_RELEASE(prev); - break; - } + peer->nptr->nfinalized++; + /* since the client is finalizing, remove them from any event + * registrations they may still have on our list */ + PMIX_LIST_FOREACH(reginfo, &pmix_server_globals.events, pmix_regevents_info_t) { + PMIX_LIST_FOREACH(prev, ®info->peers, pmix_peer_events_info_t) { + if (prev->peer == peer) { + pmix_list_remove_item(®info->peers, &prev->super); + PMIX_RELEASE(prev); + break; } } - /* now tell the host server */ - if (PMIX_SUCCESS != (rc = pmix_host_server.client_finalized(&proc, peer->info->server_object, - op_cbfunc, cd))) { - PMIX_RELEASE(cd); - } else { - /* don't reply to them ourselves - we will do so when the host - * server calls us back */ - return rc; - } } /* turn off the recv event - we shouldn't hear anything * more from this proc */ @@ -2729,9 +3035,39 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, pmix_event_del(&peer->recv_event); peer->recv_ev_active = false; } - /* let the network libraries cleanup */ - pmix_pnet.child_finalized(peer); - return rc; + PMIX_GDS_CADDY(cd, peer, tag); + /* call the local server, if supported */ + if (NULL != pmix_host_server.client_finalized) { + (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + proc.rank = peer->info->pname.rank; + /* now tell the host server */ + if (PMIX_SUCCESS == (rc = pmix_host_server.client_finalized(&proc, peer->info->server_object, + op_cbfunc2, cd))) { + /* don't reply to them ourselves - we will do so when the host + * server calls us back */ + return rc; + } + /* if the call doesn't succeed (e.g., they provided the stub + * but return NOT_SUPPORTED), then the callback function + * won't be called, but we still need to cleanup + * any lingering references to this peer and answer + * the client. Thus, we call the callback function ourselves + * in this case */ + op_cbfunc2(PMIX_SUCCESS, cd); + /* return SUCCESS as the cbfunc generated the return msg + * and released the cd object */ + return PMIX_SUCCESS; + } + /* if the host doesn't provide a client_finalized function, + * we still need to ensure that we cleanup any lingering + * references to this peer. We use the callback function + * here as well to ensure the client gets its required + * response and that we delay before cleaning up the + * connection*/ + op_cbfunc2(PMIX_SUCCESS, cd); + /* return SUCCESS as the cbfunc generated the return msg + * and released the cd object */ + return PMIX_SUCCESS; } @@ -2861,9 +3197,9 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, return PMIX_ERR_NOT_SUPPORTED; } -static void server_message_handler(struct pmix_peer_t *pr, - pmix_ptl_hdr_t *hdr, - pmix_buffer_t *buf, void *cbdata) +void pmix_server_message_handler(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, + pmix_buffer_t *buf, void *cbdata) { pmix_peer_t *peer = (pmix_peer_t*)pr; pmix_buffer_t *reply; diff --git a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_get.c b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_get.c index 70b26fb1f8d..8fba5f42248 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_get.c +++ b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_get.c @@ -690,12 +690,12 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, /* retrieve the data for the specific rank they are asking about */ if (PMIX_RANK_WILDCARD != rank) { if (!PMIX_PROC_IS_SERVER(peer) && !peer->commit_cnt) { - /* this condition works only for local requests, server does - * count commits for local ranks, and check this count when + /* this condition works only for local requests, server does + * count commits for local ranks, and check this count when * local request. - * if that request performs for remote rank on the remote + * if that request performs for remote rank on the remote * node (by direct modex) so `peer->commit_cnt` should be ignored, - * it is can not be counted for the remote side and this condition + * it is can not be counted for the remote side and this condition * does not matter for remote case */ return PMIX_ERR_NOT_FOUND; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.c b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.c index d75edc12515..570e1c82a42 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.c +++ b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.c @@ -51,6 +51,7 @@ #include "src/class/pmix_hotel.h" #include "src/class/pmix_list.h" #include "src/mca/bfrops/bfrops.h" +#include "src/mca/plog/plog.h" #include "src/util/argv.h" #include "src/util/error.h" #include "src/util/output.h" @@ -1057,6 +1058,12 @@ static void spcbfunc(pmix_status_t status, (PMIX_RANK_WILDCARD != req->pname.rank && occupant->procs->rank != req->pname.rank)) { continue; } + /* never forward back to the source! This can happen if the source + * is a launcher */ + if (0 == strncmp(occupant->procs->nspace, req->peer->info->pname.nspace, PMIX_MAX_NSLEN) && + occupant->procs->rank == req->peer->info->pname.rank) { + continue; + } /* setup the msg */ if (NULL == (msg = PMIX_NEW(pmix_buffer_t))) { PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); @@ -1096,6 +1103,7 @@ static void spcbfunc(pmix_status_t status, } } } + /* cleanup the caddy */ if (NULL != cd->info) { PMIX_INFO_FREE(cd->info, cd->ninfo); @@ -1122,7 +1130,7 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, bool stdout_found = false, stderr_found = false, stddiag_found = false; pmix_output_verbose(2, pmix_server_globals.spawn_output, - "recvd SPAWN"); + "recvd SPAWN from %s:%d", peer->info->pname.nspace, peer->info->pname.rank); if (NULL == pmix_host_server.spawn) { PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); @@ -1138,9 +1146,6 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, cd->peer = peer; cd->spcbfunc = cbfunc; cd->cbdata = cbdata; - /* setup the proc name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); - proc.rank = peer->info->pname.rank; /* unpack the number of job-level directives */ cnt=1; @@ -1153,11 +1158,6 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, /* always add one directive that indicates whether the requestor * is a tool or client */ cd->ninfo = ninfo + 1; - /* if it is a client, then we set the parent and - * "spawned" keys as well */ - if (!PMIX_PROC_IS_TOOL(peer)) { - cd->ninfo += 2; - } PMIX_INFO_CREATE(cd->info, cd->ninfo); if (NULL == cd->info) { rc = PMIX_ERR_NOMEM; @@ -1174,15 +1174,16 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, } /* run a quick check of the directives to see if any IOF * requests were included so we can set that up now - helps - * to catch any early output */ + * to catch any early output - and a request for notification + * of job termination so we can setup the event registration */ cd->channels = PMIX_FWD_NO_CHANNELS; for (n=0; n < cd->ninfo; n++) { if (0 == strncmp(cd->info[n].key, PMIX_FWD_STDIN, PMIX_MAX_KEYLEN)) { - stdout_found = true; if (PMIX_INFO_TRUE(&cd->info[n])) { cd->channels |= PMIX_FWD_STDIN_CHANNEL; } } else if (0 == strncmp(cd->info[n].key, PMIX_FWD_STDOUT, PMIX_MAX_KEYLEN)) { + stdout_found = true; if (PMIX_INFO_TRUE(&cd->info[n])) { cd->channels |= PMIX_FWD_STDOUT_CHANNEL; } @@ -1202,7 +1203,7 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, } /* add the directive to the end */ if (PMIX_PROC_IS_TOOL(peer)) { - PMIX_INFO_LOAD(&cd->info[cd->ninfo-1], PMIX_REQUESTOR_IS_TOOL, NULL, PMIX_BOOL); + PMIX_INFO_LOAD(&cd->info[ninfo], PMIX_REQUESTOR_IS_TOOL, NULL, PMIX_BOOL); /* if the requestor is a tool, we default to forwarding all * output IO channels */ if (!stdout_found) { @@ -1215,9 +1216,7 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, cd->channels |= PMIX_FWD_STDDIAG_CHANNEL; } } else { - PMIX_INFO_LOAD(&cd->info[cd->ninfo-3], PMIX_SPAWNED, NULL, PMIX_BOOL); - PMIX_INFO_LOAD(&cd->info[cd->ninfo-2], PMIX_PARENT_ID, &proc, PMIX_PROC); - PMIX_INFO_LOAD(&cd->info[cd->ninfo-1], PMIX_REQUESTOR_IS_CLIENT, NULL, PMIX_BOOL); + PMIX_INFO_LOAD(&cd->info[ninfo], PMIX_REQUESTOR_IS_CLIENT, NULL, PMIX_BOOL); } /* unpack the number of apps */ @@ -1242,6 +1241,8 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, } } /* call the local server */ + (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + proc.rank = peer->info->pname.rank; rc = pmix_host_server.spawn(&proc, cd->info, cd->ninfo, cd->apps, cd->napps, spcbfunc, cd); cleanup: @@ -1264,27 +1265,44 @@ pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, int32_t cnt; pmix_status_t rc; pmix_info_t *info = NULL; - size_t ninfo; + size_t nprocs, ninfo; pmix_server_trkr_t *trk; - char *nptr; - pmix_proc_t proc; - pmix_nspace_t *nsptr, *nspace; + pmix_proc_t *procs = NULL; if (NULL == pmix_host_server.disconnect) { return PMIX_ERR_NOT_SUPPORTED; } - /* unpack the nspace they want to be disconnected from */ + /* unpack the number of procs */ cnt = 1; - PMIX_BFROPS_UNPACK(rc, cd->peer, buf, &nptr, &cnt, PMIX_STRING); + PMIX_BFROPS_UNPACK(rc, cd->peer, buf, &nprocs, &cnt, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto cleanup; + } + /* there must be at least one proc - we do not allow the client + * to send us NULL proc as the server has no idea what to do + * with that situation. Instead, the client should at least send + * us their own namespace for the use-case where the connection + * spans all procs in that namespace */ + if (nprocs < 1) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + rc = PMIX_ERR_BAD_PARAM; + goto cleanup; + } + + /* unpack the procs */ + PMIX_PROC_CREATE(procs, nprocs); + if (NULL == procs) { + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + cnt = nprocs; + PMIX_BFROPS_UNPACK(rc, cd->peer, buf, procs, &cnt, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto cleanup; } - memset(proc.nspace, 0, PMIX_MAX_NSLEN+1); - (void)strncpy(proc.nspace, nptr, PMIX_MAX_NSLEN); - proc.rank = PMIX_RANK_WILDCARD; - free(nptr); /* unpack the number of provided info structs */ cnt = 1; @@ -1306,23 +1324,10 @@ pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, } } - /* we must already know about this nspace, so find its record */ - nspace = NULL; - PMIX_LIST_FOREACH(nsptr, &pmix_server_globals.nspaces, pmix_nspace_t) { - if (0 == strncmp(nsptr->nspace, proc.nspace, PMIX_MAX_NSLEN)) { - nspace = nsptr; - break; - } - } - if (NULL == nspace) { - PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND); - goto cleanup; - } - /* find/create the local tracker for this operation */ - if (NULL == (trk = get_tracker(&proc, 1, PMIX_DISCONNECTNB_CMD))) { + if (NULL == (trk = get_tracker(procs, nprocs, PMIX_DISCONNECTNB_CMD))) { /* we don't have this tracker yet, so get a new one */ - if (NULL == (trk = new_tracker(&proc, 1, PMIX_DISCONNECTNB_CMD))) { + if (NULL == (trk = new_tracker(procs, nprocs, PMIX_DISCONNECTNB_CMD))) { /* only if a bozo error occurs */ PMIX_ERROR_LOG(PMIX_ERROR); /* DO NOT HANG */ @@ -1332,11 +1337,8 @@ pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, rc = PMIX_ERROR; goto cleanup; } - trk->nlocal = nspace->nlocalprocs; trk->op_cbfunc = cbfunc; } - (void)strncpy(trk->pname.nspace, proc.nspace, PMIX_MAX_NSLEN); - trk->pname.rank = PMIX_RANK_WILDCARD; /* if the info keys have not been provided yet, pass * them along here */ @@ -1357,7 +1359,7 @@ pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, * across all participants has been completed */ if (trk->def_complete && pmix_list_get_size(&trk->local_cbs) == trk->nlocal) { - rc = pmix_host_server.disconnect(trk->pname.nspace, trk->info, trk->ninfo, cbfunc, trk); + rc = pmix_host_server.disconnect(trk->pcs, trk->npcs, trk->info, trk->ninfo, cbfunc, trk); } else { rc = PMIX_SUCCESS; } @@ -1377,8 +1379,8 @@ static void connect_timeout(int sd, short args, void *cbdata) "ALERT: connect timeout fired"); /* execute the provided callback function with the error */ - if (NULL != cd->trk->cnct_cbfunc) { - cd->trk->cnct_cbfunc(PMIX_ERR_TIMEOUT, NULL, PMIX_RANK_UNDEF, cd->trk); + if (NULL != cd->trk->op_cbfunc) { + cd->trk->op_cbfunc(PMIX_ERR_TIMEOUT, cd->trk); return; // the cbfunc will have cleaned up the tracker } cd->event_active = false; @@ -1389,7 +1391,7 @@ static void connect_timeout(int sd, short args, void *cbdata) pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, pmix_buffer_t *buf, - pmix_connect_cbfunc_t cbfunc) + pmix_op_cbfunc_t cbfunc) { int32_t cnt; pmix_status_t rc; @@ -1474,12 +1476,12 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, PMIX_ERROR_LOG(PMIX_ERROR); /* DO NOT HANG */ if (NULL != cbfunc) { - cbfunc(PMIX_ERROR, NULL, PMIX_RANK_UNDEF, cd); + cbfunc(PMIX_ERROR, cd); } rc = PMIX_ERROR; goto cleanup; } - trk->cnct_cbfunc = cbfunc; + trk->op_cbfunc = cbfunc; } /* if the info keys have not been provided yet, pass @@ -1545,9 +1547,12 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, bool found, matched; pmix_buffer_t *relay; pmix_cmd_t cmd = PMIX_NOTIFY_CMD; + pmix_proc_t *affected = NULL; + size_t naffected = 0; pmix_output_verbose(2, pmix_server_globals.event_output, - "recvd register events"); + "recvd register events for peer %s:%d", + peer->info->pname.nspace, peer->info->pname.rank); /* unpack the number of codes */ cnt=1; @@ -1593,11 +1598,28 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, } } - /* see if they asked for enviro events */ + /* check the directives */ for (n=0; n < ninfo; n++) { - if (0 == strcmp(info[n].key, PMIX_EVENT_ENVIRO_LEVEL)) { + if (0 == strncmp(info[n].key, PMIX_EVENT_ENVIRO_LEVEL, PMIX_MAX_KEYLEN)) { enviro_events = PMIX_INFO_TRUE(&info[n]); - break; + } else if (0 == strncmp(info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + if (NULL != affected) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + rc = PMIX_ERR_BAD_PARAM; + goto cleanup; + } + naffected = 1; + PMIX_PROC_CREATE(affected, naffected); + memcpy(affected, info[n].value.data.proc, sizeof(pmix_proc_t)); + } else if (0 == strncmp(info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) { + if (NULL != affected) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + rc = PMIX_ERR_BAD_PARAM; + goto cleanup; + } + naffected = info[n].value.data.darray->size; + PMIX_PROC_CREATE(affected, naffected); + memcpy(affected, info[n].value.data.darray->array, naffected * sizeof(pmix_proc_t)); } } @@ -1747,6 +1769,9 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, if (NULL != codes) { free(codes); } + if (NULL != affected) { + PMIX_PROC_FREE(affected, naffected); + } return rc; } @@ -1758,8 +1783,10 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, } found = false; if (NULL == codes) { - /* they registered a default event handler - always matches */ - found = true; + if (!cd->nondefault) { + /* they registered a default event handler - always matches */ + found = true; + } } else { for (k=0; k < ncodes; k++) { if (codes[k] == cd->status) { @@ -1768,77 +1795,87 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, } } } - if (found) { - /* if we were given specific targets, check if this is one */ - if (NULL != cd->targets) { - matched = false; - for (n=0; n < cd->ntargets; n++) { - if (0 != strncmp(peer->info->pname.nspace, cd->targets[n].nspace, PMIX_MAX_NSLEN)) { - continue; - } - /* if the source of the event is the same peer just registered, then ignore it - * as the event notification system will have already locally - * processed it */ - if (0 == strncmp(peer->info->pname.nspace, cd->source.nspace, PMIX_MAX_NSLEN) && - peer->info->pname.rank == cd->source.rank) { - continue; - } - if (PMIX_RANK_WILDCARD == cd->targets[n].rank || - peer->info->pname.rank == cd->targets[n].rank) { - matched = true; - break; - } + if (!found) { + continue; + } + /* if we were given specific targets, check if this is one */ + if (NULL != cd->targets) { + matched = false; + for (n=0; n < cd->ntargets; n++) { + if (0 != strncmp(peer->info->pname.nspace, cd->targets[n].nspace, PMIX_MAX_NSLEN)) { + continue; } - if (!matched) { - /* do not notify this one */ + /* if the source of the event is the same peer just registered, then ignore it + * as the event notification system will have already locally + * processed it */ + if (0 == strncmp(peer->info->pname.nspace, cd->source.nspace, PMIX_MAX_NSLEN) && + peer->info->pname.rank == cd->source.rank) { continue; } + if (PMIX_RANK_WILDCARD == cd->targets[n].rank || + peer->info->pname.rank == cd->targets[n].rank) { + matched = true; + break; + } } - /* all matches - notify */ - relay = PMIX_NEW(pmix_buffer_t); - if (NULL == relay) { - /* nothing we can do */ - PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - return PMIX_ERR_NOMEM; - } - /* pack the info data stored in the event */ - PMIX_BFROPS_PACK(rc, peer, relay, &cmd, 1, PMIX_COMMAND); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - break; - } - PMIX_BFROPS_PACK(rc, peer, relay, &cd->status, 1, PMIX_STATUS); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - break; - } - PMIX_BFROPS_PACK(rc, peer, relay, &cd->source, 1, PMIX_PROC); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - break; + if (!matched) { + /* do not notify this one */ + continue; } - PMIX_BFROPS_PACK(rc, peer, relay, &cd->ninfo, 1, PMIX_SIZE); + } + /* if they specified affected proc(s) they wanted to know about, check */ + if (!pmix_notify_check_affected(cd->affected, cd->naffected, + affected, naffected)) { + continue; + } + /* all matches - notify */ + relay = PMIX_NEW(pmix_buffer_t); + if (NULL == relay) { + /* nothing we can do */ + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + rc = PMIX_ERR_NOMEM; + break; + } + /* pack the info data stored in the event */ + PMIX_BFROPS_PACK(rc, peer, relay, &cmd, 1, PMIX_COMMAND); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + break; + } + PMIX_BFROPS_PACK(rc, peer, relay, &cd->status, 1, PMIX_STATUS); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + break; + } + PMIX_BFROPS_PACK(rc, peer, relay, &cd->source, 1, PMIX_PROC); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + break; + } + PMIX_BFROPS_PACK(rc, peer, relay, &cd->ninfo, 1, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + break; + } + if (0 < cd->ninfo) { + PMIX_BFROPS_PACK(rc, peer, relay, cd->info, cd->ninfo, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); break; } - if (0 < cd->ninfo) { - PMIX_BFROPS_PACK(rc, peer, relay, cd->info, cd->ninfo, PMIX_INFO); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - break; - } - } - PMIX_SERVER_QUEUE_REPLY(peer, 0, relay); } + PMIX_SERVER_QUEUE_REPLY(peer, 0, relay); } if (!enviro_events) { if (NULL != codes) { free(codes); } } + if (NULL != affected) { + PMIX_PROC_FREE(affected, naffected); + } - return PMIX_SUCCESS; + return rc; } void pmix_server_deregister_events(pmix_peer_t *peer, @@ -2048,6 +2085,7 @@ pmix_status_t pmix_server_query(pmix_peer_t *peer, static void logcbfn(pmix_status_t status, void *cbdata) { pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata; + if (NULL != cd->cbfunc.opcbfn) { cd->cbfunc.opcbfn(status, cd->cbdata); } @@ -2062,13 +2100,18 @@ pmix_status_t pmix_server_log(pmix_peer_t *peer, pmix_status_t rc; pmix_shift_caddy_t *cd; pmix_proc_t proc; + time_t timestamp; pmix_output_verbose(2, pmix_server_globals.base_output, "recvd log from client"); - if (NULL == pmix_host_server.log) { - return PMIX_ERR_NOT_SUPPORTED; - } + /* we need to deliver this to our internal log capability, + * which may upcall it to our host if it cannot process + * the request itself */ + + /* setup the requesting peer name */ + (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + proc.rank = peer->info->pname.rank; cd = PMIX_NEW(pmix_shift_caddy_t); if (NULL == cd) { @@ -2076,6 +2119,14 @@ pmix_status_t pmix_server_log(pmix_peer_t *peer, } cd->cbfunc.opcbfn = cbfunc; cd->cbdata = cbdata; + /* unpack the timestamp */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, peer, buf, ×tamp, &cnt, PMIX_TIME); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto exit; + } + /* unpack the number of data */ cnt = 1; PMIX_BFROPS_UNPACK(rc, peer, buf, &cd->ninfo, &cnt, PMIX_SIZE); @@ -2083,10 +2134,10 @@ pmix_status_t pmix_server_log(pmix_peer_t *peer, PMIX_ERROR_LOG(rc); goto exit; } + cnt = cd->ninfo; + PMIX_INFO_CREATE(cd->info, cd->ninfo); /* unpack the data */ - if (0 < cd->ninfo) { - PMIX_INFO_CREATE(cd->info, cd->ninfo); - cnt = cd->ninfo; + if (0 < cnt) { PMIX_BFROPS_UNPACK(rc, peer, buf, cd->info, &cnt, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); @@ -2100,10 +2151,24 @@ pmix_status_t pmix_server_log(pmix_peer_t *peer, PMIX_ERROR_LOG(rc); goto exit; } - /* unpack the directives */ - if (0 < cd->ndirs) { + cnt = cd->ndirs; + /* always add the source to the directives so we + * can tell downstream if this gets "upcalled" to + * our host for relay */ + cd->ndirs = cnt + 1; + /* if a timestamp was sent, then we add it to the directives */ + if (0 < timestamp) { + cd->ndirs++; PMIX_INFO_CREATE(cd->directives, cd->ndirs); - cnt = cd->ndirs; + PMIX_INFO_LOAD(&cd->directives[cnt], PMIX_LOG_SOURCE, &proc, PMIX_PROC); + PMIX_INFO_LOAD(&cd->directives[cnt+1], PMIX_LOG_TIMESTAMP, ×tamp, PMIX_TIME); + } else { + PMIX_INFO_CREATE(cd->directives, cd->ndirs); + PMIX_INFO_LOAD(&cd->directives[cnt], PMIX_LOG_SOURCE, &proc, PMIX_PROC); + } + + /* unpack the directives */ + if (0 < cnt) { PMIX_BFROPS_UNPACK(rc, peer, buf, cd->directives, &cnt, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); @@ -2111,15 +2176,11 @@ pmix_status_t pmix_server_log(pmix_peer_t *peer, } } - /* setup the requesting peer name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); - proc.rank = peer->info->pname.rank; - - /* ask the host to log the info */ - pmix_host_server.log(&proc, cd->info, cd->ninfo, - cd->directives, cd->ndirs, - logcbfn, cd); - return PMIX_SUCCESS; + /* pass it down */ + rc = pmix_plog.log(&proc, cd->info, cd->ninfo, + cd->directives, cd->ndirs, + logcbfn, cd); + return rc; exit: PMIX_RELEASE(cd); @@ -2193,21 +2254,31 @@ pmix_status_t pmix_server_alloc(pmix_peer_t *peer, return rc; } +typedef struct { + pmix_list_item_t super; + pmix_epilog_t *epi; +} pmix_srvr_epi_caddy_t; +static PMIX_CLASS_INSTANCE(pmix_srvr_epi_caddy_t, + pmix_list_item_t, + NULL, NULL); + pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, pmix_buffer_t *buf, pmix_info_cbfunc_t cbfunc, void *cbdata) { - int32_t cnt; + int32_t cnt, m; pmix_status_t rc; pmix_query_caddy_t *cd; + pmix_nspace_t *nptr, *tmp; + pmix_peer_t *pr; pmix_proc_t proc; size_t n; bool recurse = false, leave_topdir = false, duplicate; - pmix_list_t cachedirs, cachefiles; - pmix_epilog_t *epi = NULL; - pmix_cleanup_file_t *cf, *cf2; - pmix_cleanup_dir_t *cdir, *cdir2; + pmix_list_t cachedirs, cachefiles, ignorefiles, epicache; + pmix_srvr_epi_caddy_t *epicd = NULL; + pmix_cleanup_file_t *cf, *cf2, *cfptr; + pmix_cleanup_dir_t *cdir, *cdir2, *cdirptr; pmix_output_verbose(2, pmix_server_globals.base_output, "recvd job control request from client"); @@ -2240,14 +2311,52 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, } /* check targets to find proper place to put any epilog requests */ + PMIX_CONSTRUCT(&epicache, pmix_list_t); if (NULL == cd->targets) { - epi = &peer->nptr->epilog; - } else if (1 == cd->ntargets) { - if (0 == strncmp(cd->targets[0].nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN)) { - if (PMIX_RANK_WILDCARD == cd->targets[0].rank) { - epi = &peer->nptr->epilog; + epicd = PMIX_NEW(pmix_srvr_epi_caddy_t); + epicd->epi = &peer->nptr->epilog; + pmix_list_append(&epicache, &epicd->super); + } else { + for (n=0; n < cd->ntargets; n++) { + /* find the nspace of this proc */ + nptr = NULL; + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { + if (0 == strcmp(tmp->nspace, cd->targets[n].nspace)) { + nptr = tmp; + break; + } + } + if (NULL == nptr) { + nptr = PMIX_NEW(pmix_nspace_t); + if (NULL == nptr) { + rc = PMIX_ERR_NOMEM; + goto exit; + } + nptr->nspace = strdup(cd->targets[n].nspace); + pmix_list_append(&pmix_server_globals.nspaces, &nptr->super); + } + /* if the rank is wildcard, then we use the epilog for the nspace */ + if (PMIX_RANK_WILDCARD == cd->targets[n].rank) { + epicd = PMIX_NEW(pmix_srvr_epi_caddy_t); + epicd->epi = &nptr->epilog; + pmix_list_append(&epicache, &epicd->super); } else { - epi = &peer->epilog; + /* we need to find the precise peer - we can only + * do cleanup for a local client */ + for (m=0; m < pmix_server_globals.clients.size; m++) { + if (NULL == (pr = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, m))) { + continue; + } + if (0 != strncmp(pr->info->pname.nspace, cd->targets[n].nspace, PMIX_MAX_NSLEN)) { + continue; + } + if (pr->info->pname.rank == cd->targets[n].rank) { + epicd = PMIX_NEW(pmix_srvr_epi_caddy_t); + epicd->epi = &pr->epilog; + pmix_list_append(&epicache, &epicd->super); + break; + } + } } } } @@ -2274,16 +2383,12 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, * that request ourselves */ PMIX_CONSTRUCT(&cachedirs, pmix_list_t); PMIX_CONSTRUCT(&cachefiles, pmix_list_t); + PMIX_CONSTRUCT(&ignorefiles, pmix_list_t); + cnt = 0; // track how many infos are cleanup related for (n=0; n < cd->ninfo; n++) { if (0 == strncmp(cd->info[n].key, PMIX_REGISTER_CLEANUP, PMIX_MAX_KEYLEN)) { ++cnt; - /* see if we allow epilog requests */ - if (NULL == epi) { - /* return an error */ - rc = PMIX_ERR_BAD_PARAM; - goto exit; - } if (PMIX_STRING != cd->info[n].value.type || NULL == cd->info[n].value.data.string) { /* return an error */ @@ -2300,12 +2405,6 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, pmix_list_append(&cachefiles, &cf->super); } else if (0 == strncmp(cd->info[n].key, PMIX_REGISTER_CLEANUP_DIR, PMIX_MAX_KEYLEN)) { ++cnt; - /* see if we allow epilog requests */ - if (NULL == epi) { - /* return an error */ - rc = PMIX_ERR_BAD_PARAM; - goto exit; - } if (PMIX_STRING != cd->info[n].value.type || NULL == cd->info[n].value.data.string) { /* return an error */ @@ -2321,12 +2420,6 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, cdir->path = strdup(cd->info[n].value.data.string); pmix_list_append(&cachedirs, &cdir->super); } else if (0 == strncmp(cd->info[n].key, PMIX_CLEANUP_RECURSIVE, PMIX_MAX_KEYLEN)) { - /* see if we allow epilog requests */ - if (NULL == epi) { - /* return an error */ - rc = PMIX_ERR_BAD_PARAM; - goto exit; - } recurse = PMIX_INFO_TRUE(&cd->info[n]); ++cnt; } else if (0 == strncmp(cd->info[n].key, PMIX_CLEANUP_IGNORE, PMIX_MAX_KEYLEN)) { @@ -2336,104 +2429,111 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, rc = PMIX_ERR_BAD_PARAM; goto exit; } - /* see if we allow epilog requests */ - if (NULL == epi) { + cf = PMIX_NEW(pmix_cleanup_file_t); + if (NULL == cf) { /* return an error */ - rc = PMIX_ERR_BAD_PARAM; + rc = PMIX_ERR_NOMEM; goto exit; } - /* scan the list of ignores for any duplicate */ - duplicate = false; - PMIX_LIST_FOREACH(cf, &epi->ignores, pmix_cleanup_file_t) { - if (0 == strcmp(cf->path, cd->info[n].value.data.string)) { - /* we can drop this request */ - duplicate = true; - break; - } - } - if (!duplicate) { - cf = PMIX_NEW(pmix_cleanup_file_t); - if (NULL == cf) { - /* return an error */ - rc = PMIX_ERR_NOMEM; - goto exit; - } - cf->path = strdup(cd->info[n].value.data.string); - pmix_list_append(&epi->ignores, &cf->super); - } + cf->path = strdup(cd->info[n].value.data.string); + pmix_list_append(&ignorefiles, &cf->super); ++cnt; } else if (0 == strncmp(cd->info[n].key, PMIX_CLEANUP_LEAVE_TOPDIR, PMIX_MAX_KEYLEN)) { - /* see if we allow epilog requests */ - if (NULL == epi) { - /* return an error */ - rc = PMIX_ERR_BAD_PARAM; - goto exit; - } leave_topdir = PMIX_INFO_TRUE(&cd->info[n]); ++cnt; } } if (0 < cnt) { - while (NULL != (cdir = (pmix_cleanup_dir_t*)pmix_list_remove_first(&cachedirs))) { - /* scan the existing list of directories for any duplicate */ - PMIX_LIST_FOREACH(cdir2, &epi->cleanup_dirs, pmix_cleanup_dir_t) { - if (0 == strcmp(cdir2->path, cdir->path)) { - /* duplicate - check for difference in flags per RFC - * precedence rules */ - if (!cdir->recurse && recurse) { - cdir->recurse = recurse; - } - if (!cdir->leave_topdir && leave_topdir) { - cdir->leave_topdir = leave_topdir; + /* handle any ignore directives first */ + PMIX_LIST_FOREACH(cf, &ignorefiles, pmix_cleanup_file_t) { + PMIX_LIST_FOREACH(epicd, &epicache, pmix_srvr_epi_caddy_t) { + /* scan the existing list of files for any duplicate */ + duplicate = false; + PMIX_LIST_FOREACH(cf2, &epicd->epi->cleanup_files, pmix_cleanup_file_t) { + if (0 == strcmp(cf2->path, cf->path)) { + duplicate = true; + break; } - PMIX_RELEASE(cdir); - cdir = NULL; - break; + } + if (!duplicate) { + /* append it to the end of the list */ + cfptr = PMIX_NEW(pmix_cleanup_file_t); + cfptr->path = strdup(cf->path); + pmix_list_append(&epicd->epi->ignores, &cf->super); } } - if (NULL != cdir) { - /* check for conflict with ignore */ - PMIX_LIST_FOREACH(cf, &epi->ignores, pmix_cleanup_file_t) { - if (0 == strcmp(cf->path, cdir->path)) { - /* return an error */ - rc = PMIX_ERR_CONFLICTING_CLEANUP_DIRECTIVES; - PMIX_LIST_DESTRUCT(&cachedirs); - PMIX_LIST_DESTRUCT(&cachefiles); - goto exit; + } + PMIX_LIST_DESTRUCT(&ignorefiles); + /* now look at the directories */ + PMIX_LIST_FOREACH(cdir, &cachedirs, pmix_cleanup_dir_t) { + PMIX_LIST_FOREACH(epicd, &epicache, pmix_srvr_epi_caddy_t) { + /* scan the existing list of directories for any duplicate */ + duplicate = false; + PMIX_LIST_FOREACH(cdir2, &epicd->epi->cleanup_dirs, pmix_cleanup_dir_t) { + if (0 == strcmp(cdir2->path, cdir->path)) { + /* duplicate - check for difference in flags per RFC + * precedence rules */ + if (!cdir->recurse && recurse) { + cdir->recurse = recurse; + } + if (!cdir->leave_topdir && leave_topdir) { + cdir->leave_topdir = leave_topdir; + } + duplicate = true; + break; + } + } + if (!duplicate) { + /* check for conflict with ignore */ + PMIX_LIST_FOREACH(cf, &epicd->epi->ignores, pmix_cleanup_file_t) { + if (0 == strcmp(cf->path, cdir->path)) { + /* return an error */ + rc = PMIX_ERR_CONFLICTING_CLEANUP_DIRECTIVES; + PMIX_LIST_DESTRUCT(&cachedirs); + PMIX_LIST_DESTRUCT(&cachefiles); + PMIX_LIST_DESTRUCT(&epicache); + goto exit; + } } + /* append it to the end of the list */ + cdirptr = PMIX_NEW(pmix_cleanup_dir_t); + cdirptr->path = strdup(cdir->path); + cdirptr->recurse = recurse; + cdirptr->leave_topdir = leave_topdir; + pmix_list_append(&epicd->epi->cleanup_dirs, &cdirptr->super); } - cdir->recurse = recurse; - cdir->leave_topdir = leave_topdir; - /* just append it to the end of the list */ - pmix_list_append(&epi->cleanup_dirs, &cdir->super); } } - PMIX_DESTRUCT(&cachedirs); - while (NULL != (cf = (pmix_cleanup_file_t*)pmix_list_remove_first(&cachefiles))) { - /* scan the existing list of files for any duplicate */ - PMIX_LIST_FOREACH(cf2, &epi->cleanup_files, pmix_cleanup_file_t) { - if (0 == strcmp(cf2->path, cf->path)) { - PMIX_RELEASE(cf); - cf = NULL; - break; + PMIX_LIST_DESTRUCT(&cachedirs); + PMIX_LIST_FOREACH(cf, &cachefiles, pmix_cleanup_file_t) { + PMIX_LIST_FOREACH(epicd, &epicache, pmix_srvr_epi_caddy_t) { + /* scan the existing list of files for any duplicate */ + duplicate = false; + PMIX_LIST_FOREACH(cf2, &epicd->epi->cleanup_files, pmix_cleanup_file_t) { + if (0 == strcmp(cf2->path, cf->path)) { + duplicate = true; + break; + } } - } - if (NULL != cf) { - /* check for conflict with ignore */ - PMIX_LIST_FOREACH(cf2, &epi->ignores, pmix_cleanup_file_t) { - if (0 == strcmp(cf->path, cf2->path)) { - /* return an error */ - rc = PMIX_ERR_CONFLICTING_CLEANUP_DIRECTIVES; - PMIX_LIST_DESTRUCT(&cachedirs); - PMIX_LIST_DESTRUCT(&cachefiles); - goto exit; + if (!duplicate) { + /* check for conflict with ignore */ + PMIX_LIST_FOREACH(cf2, &epicd->epi->ignores, pmix_cleanup_file_t) { + if (0 == strcmp(cf->path, cf2->path)) { + /* return an error */ + rc = PMIX_ERR_CONFLICTING_CLEANUP_DIRECTIVES; + PMIX_LIST_DESTRUCT(&cachedirs); + PMIX_LIST_DESTRUCT(&cachefiles); + goto exit; + } } + /* append it to the end of the list */ + cfptr = PMIX_NEW(pmix_cleanup_file_t); + cfptr->path = strdup(cf->path); + pmix_list_append(&epicd->epi->cleanup_files, &cfptr->super); } - /* just append it to the end of the list */ - pmix_list_append(&epi->cleanup_files, &cf->super); } } - PMIX_DESTRUCT(&cachefiles); + PMIX_LIST_DESTRUCT(&cachefiles); if (cnt == (int)cd->ninfo) { /* nothing more to do */ if (NULL != cbfunc) { @@ -2969,7 +3069,7 @@ static void tcon(pmix_server_trkr_t *t) t->collect_type = PMIX_COLLECT_INVALID; t->modexcbfunc = NULL; t->op_cbfunc = NULL; - t->cnct_cbfunc = NULL; + t->hybrid = false; } static void tdes(pmix_server_trkr_t *t) { @@ -3152,3 +3252,27 @@ static void regdes(pmix_regevents_info_t *p) PMIX_CLASS_INSTANCE(pmix_regevents_info_t, pmix_list_item_t, regcon, regdes); + +static void ilcon(pmix_inventory_rollup_t *p) +{ + PMIX_CONSTRUCT_LOCK(&p->lock); + p->lock.active = false; + p->status = PMIX_SUCCESS; + p->requests = 0; + p->replies = 0; + PMIX_CONSTRUCT(&p->payload, pmix_list_t); + p->info = NULL; + p->ninfo = 0; + p->cbfunc = NULL; + p->infocbfunc = NULL; + p->opcbfunc = NULL; + p->cbdata = NULL; +} +static void ildes(pmix_inventory_rollup_t *p) +{ + PMIX_DESTRUCT_LOCK(&p->lock); + PMIX_LIST_DESTRUCT(&p->payload); +} +PMIX_CLASS_INSTANCE(pmix_inventory_rollup_t, + pmix_object_t, + ilcon, ildes); diff --git a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.h b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.h index 3913553053b..e90137c90cc 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.h +++ b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.h @@ -66,6 +66,29 @@ typedef struct { } pmix_setup_caddy_t; PMIX_CLASS_DECLARATION(pmix_setup_caddy_t); +/* define a callback function returning inventory */ +typedef void (*pmix_inventory_cbfunc_t)(pmix_status_t status, + pmix_list_t *inventory, + void *cbdata); + +/* define an object for rolling up the inventory*/ +typedef struct { + pmix_object_t super; + pmix_lock_t lock; + pmix_event_t ev; + pmix_status_t status; + int requests; + int replies; + pmix_list_t payload; // list of pmix_kval_t containing the replies + pmix_info_t *info; + size_t ninfo; + pmix_inventory_cbfunc_t cbfunc; + pmix_info_cbfunc_t infocbfunc; + pmix_op_cbfunc_t opcbfunc; + void *cbdata; +} pmix_inventory_rollup_t; +PMIX_CLASS_DECLARATION(pmix_inventory_rollup_t); + typedef struct { pmix_list_item_t super; pmix_setup_caddy_t *cd; @@ -117,6 +140,7 @@ typedef struct { pmix_list_t events; // list of pmix_regevents_info_t registered events pmix_hotel_t iof; // IO to be forwarded to clients bool tool_connections_allowed; + char *tmpdir; // temporary directory for this server // verbosity for server get operations int get_output; int get_verbose; @@ -211,7 +235,7 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, pmix_buffer_t *buf, - pmix_connect_cbfunc_t cbfunc); + pmix_op_cbfunc_t cbfunc); pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, pmix_buffer_t *buf, @@ -282,6 +306,12 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, void *cbdata); void pmix_server_execute_collective(int sd, short args, void *cbdata); +pmix_status_t pmix_server_initialize(void); + +void pmix_server_message_handler(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, + pmix_buffer_t *buf, void *cbdata); + PMIX_EXPORT extern pmix_server_module_t pmix_host_server; PMIX_EXPORT extern pmix_server_globals_t pmix_server_globals; diff --git a/opal/mca/pmix/pmix3x/pmix/src/threads/threads.h b/opal/mca/pmix/pmix3x/pmix/src/threads/threads.h index d66e594ead6..84c8c151502 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/threads/threads.h +++ b/opal/mca/pmix/pmix3x/pmix/src/threads/threads.h @@ -13,7 +13,7 @@ * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2015-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -65,6 +65,7 @@ typedef pthread_cond_t pmix_condition_t; #define PMIX_CONDITION_STATIC_INIT PTHREAD_COND_INITIALIZER typedef struct { + pmix_status_t status; pmix_mutex_t mutex; pmix_condition_t cond; volatile bool active; diff --git a/opal/mca/pmix/pmix3x/pmix/src/tool/pmix_tool.c b/opal/mca/pmix/pmix3x/pmix/src/tool/pmix_tool.c index 8ac864ab620..b2caf9b4662 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tool/pmix_tool.c +++ b/opal/mca/pmix/pmix3x/pmix/src/tool/pmix_tool.c @@ -63,6 +63,8 @@ #include "src/util/error.h" #include "src/util/hash.h" #include "src/util/output.h" +#include "src/util/pmix_environ.h" +#include "src/util/show_help.h" #include "src/runtime/pmix_progress_threads.h" #include "src/runtime/pmix_rte.h" #include "src/mca/bfrops/base/base.h" @@ -71,6 +73,7 @@ #include "src/mca/psec/psec.h" #include "src/include/pmix_globals.h" #include "src/common/pmix_iof.h" +#include "src/server/pmix_server_ops.h" #define PMIX_MAX_RETRIES 10 @@ -146,9 +149,9 @@ static void pmix_tool_notify_recv(struct pmix_peer_t *peer, goto error; } - /* we always leave space for a callback object */ - chain->ninfo = ninfo + 1; - PMIX_INFO_CREATE(chain->info, chain->ninfo); + /* we always leave space for event hdlr name and a callback object */ + chain->nallocated = ninfo + 2; + PMIX_INFO_CREATE(chain->info, chain->nallocated); if (NULL == chain->info) { PMIX_ERROR_LOG(PMIX_ERR_NOMEM); PMIX_RELEASE(chain); @@ -156,6 +159,7 @@ static void pmix_tool_notify_recv(struct pmix_peer_t *peer, } if (0 < ninfo) { + chain->ninfo = ninfo; cnt = ninfo; PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver, buf, chain->info, &cnt, PMIX_INFO); @@ -164,25 +168,40 @@ static void pmix_tool_notify_recv(struct pmix_peer_t *peer, PMIX_RELEASE(chain); goto error; } - /* check for non-default flag */ + /* check for directives */ for (cnt=0; cnt < (int)ninfo; cnt++) { if (0 == strncmp(chain->info[cnt].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) { chain->nondefault = PMIX_INFO_TRUE(&chain->info[cnt]); - break; + } else if (0 == strncmp(chain->info[cnt].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + PMIX_PROC_CREATE(chain->affected, 1); + if (NULL == chain->affected) { + PMIX_RELEASE(chain); + goto error; + } + chain->naffected = 1; + memcpy(chain->affected, chain->info[cnt].value.data.proc, sizeof(pmix_proc_t)); + } else if (0 == strncmp(chain->info[cnt].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) { + chain->naffected = chain->info[cnt].value.data.darray->size; + PMIX_PROC_CREATE(chain->affected, chain->naffected); + if (NULL == chain->affected) { + chain->naffected = 0; + PMIX_RELEASE(chain); + goto error; + } + memcpy(chain->affected, chain->info[cnt].value.data.darray->array, chain->naffected * sizeof(pmix_proc_t)); } } } - /* now put the callback object tag in the last element */ - PMIX_INFO_LOAD(&chain->info[ninfo], PMIX_EVENT_RETURN_OBJECT, NULL, PMIX_POINTER); pmix_output_verbose(2, pmix_client_globals.event_output, - "[%s:%d] pmix:tool_notify_recv - processing event %d, calling errhandler", - pmix_globals.myid.nspace, pmix_globals.myid.rank, chain->status); + "[%s:%d] pmix:tool_notify_recv - processing event %s from source %s:%d, calling errhandler", + pmix_globals.myid.nspace, pmix_globals.myid.rank, PMIx_Error_string(chain->status), + chain->source.nspace, chain->source.rank); pmix_invoke_local_event_hdlr(chain); return; - error: + error: /* we always need to return */ pmix_output_verbose(2, pmix_client_globals.event_output, "pmix:tool_notify_recv - unpack error status =%d, calling def errhandler", rc); @@ -191,7 +210,6 @@ static void pmix_tool_notify_recv(struct pmix_peer_t *peer, pmix_invoke_local_event_hdlr(chain); } - static void tool_iof_handler(struct pmix_peer_t *pr, pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) @@ -230,23 +248,52 @@ static void tool_iof_handler(struct pmix_peer_t *pr, return; } if (NULL != bo.bytes && 0 < bo.size) { - if (channel & PMIX_FWD_STDOUT_CHANNEL) { - pmix_iof_write_output(&source, channel, &bo, &pmix_client_globals.iof_stdout.wev); - } else { - pmix_iof_write_output(&source, channel, &bo, &pmix_client_globals.iof_stderr.wev); - } + pmix_iof_write_output(&source, channel, &bo, NULL); } PMIX_BYTE_OBJECT_DESTRUCT(&bo); } +/* callback to receive job info */ +static void job_data(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, + pmix_buffer_t *buf, void *cbdata) +{ + pmix_status_t rc; + char *nspace; + int32_t cnt = 1; + pmix_cb_t *cb = (pmix_cb_t*)cbdata; + + /* unpack the nspace - should be same as our own */ + PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver, + buf, &nspace, &cnt, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + cb->status = PMIX_ERROR; + PMIX_POST_OBJECT(cb); + PMIX_WAKEUP_THREAD(&cb->lock); + return; + } + + /* decode it */ + PMIX_GDS_STORE_JOB_INFO(cb->status, + pmix_client_globals.myserver, + nspace, buf); + cb->status = PMIX_SUCCESS; + PMIX_POST_OBJECT(cb); + PMIX_WAKEUP_THREAD(&cb->lock); +} + PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, pmix_info_t info[], size_t ninfo) { pmix_kval_t *kptr; pmix_status_t rc; char hostname[PMIX_MAX_NSLEN]; - bool found, do_not_connect = false; + char *evar, *nspace; + pmix_rank_t rank; + bool gdsfound, do_not_connect = false; bool nspace_given = false; + bool nspace_in_enviro = false; bool rank_given = false; bool fwd_stdin = false; pmix_info_t ginfo; @@ -254,6 +301,10 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, pmix_ptl_posted_recv_t *rcv; pmix_proc_t wildcard; int fd; + pmix_proc_type_t ptype; + pmix_cb_t cb; + pmix_buffer_t *req; + pmix_cmd_t cmd = PMIX_REQ_CMD; PMIX_ACQUIRE_THREAD(&pmix_global_lock); @@ -265,7 +316,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, /* since we have been called before, the nspace and * rank should be known. So return them here if * requested */ - if (NULL != proc) { + if (NULL != proc) { (void)strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); proc->rank = pmix_globals.myid.rank; } @@ -274,23 +325,107 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, return PMIX_SUCCESS; } - /* if we were given an nspace in the environment, then we - * must have been spawned by a PMIx server - so even though - * we technically will operate as a tool, we are actually - * a "client" of the PMIx server and should connect that way */ - if (NULL != getenv("PMIX_NAMESPACE")) { + /* parse the input directives */ + gdsfound = false; + ptype = PMIX_PROC_TOOL; + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_GDS_MODULE, PMIX_MAX_KEYLEN)) { + PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, info[n].value.data.string, PMIX_STRING); + gdsfound = true; + } else if (0 == strncmp(info[n].key, PMIX_TOOL_DO_NOT_CONNECT, PMIX_MAX_KEYLEN)) { + do_not_connect = PMIX_INFO_TRUE(&info[n]); + } else if (0 == strncmp(info[n].key, PMIX_TOOL_NSPACE, PMIX_MAX_KEYLEN)) { + nspace = strdup(info[n].value.data.string); + nspace_given = true; + } else if (0 == strncmp(info[n].key, PMIX_TOOL_RANK, PMIX_MAX_KEYLEN)) { + rank = info[n].value.data.rank; + rank_given = true; + } else if (0 == strncmp(info[n].key, PMIX_FWD_STDIN, PMIX_MAX_KEYLEN)) { + /* they want us to forward our stdin to someone */ + fwd_stdin = true; + } else if (0 == strncmp(info[n].key, PMIX_LAUNCHER, PMIX_MAX_KEYLEN)) { + ptype = PMIX_PROC_LAUNCHER; + } + } + } + if ((nspace_given && !rank_given) || + (!nspace_given && rank_given)) { + /* can't have one and not the other */ + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIx_Init(proc, info, ninfo); + return PMIX_ERR_BAD_PARAM; + } + + /* if we were not passed an nspace in the info keys, + * check to see if we were given one in the env - this + * will be the case when we are launched by a PMIx-enabled + * daemon */ + if (!nspace_given) { + if (NULL != (evar = getenv("PMIX_NAMESPACE"))) { + nspace = strdup(evar); + nspace_in_enviro = true; + } + } + /* also look for the rank - it normally is zero, but if we + * were launched, then it might have been as part of a + * multi-process tool */ + if (!rank_given) { + if (NULL != (evar = getenv("PMIX_RANK"))) { + rank = strtol(evar, NULL, 10); + if (!nspace_in_enviro) { + /* this is an error - we can't have one and not + * the other */ + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_BAD_PARAM; + } + /* flag that this tool is also a client */ + ptype |= PMIX_PROC_CLIENT_TOOL; + } else if (nspace_in_enviro) { + /* this is an error - we can't have one and not + * the other */ + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_BAD_PARAM; + } + } + + /* if we are a launcher, then we also need to act as a server, + * so setup the server-related structures here */ + if (PMIX_PROC_LAUNCHER == ptype) { + if (PMIX_SUCCESS != (rc = pmix_server_initialize())) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + /* setup the function pointers */ + memset(&pmix_host_server, 0, sizeof(pmix_server_module_t)); + /* setup our tmpdir */ + if (NULL == pmix_server_globals.tmpdir) { + if (NULL == (evar = getenv("PMIX_SERVER_TMPDIR"))) { + pmix_server_globals.tmpdir = strdup(pmix_tmp_directory()); + } else { + pmix_server_globals.tmpdir = strdup(evar); + } + } } /* setup the runtime - this init's the globals, * opens and initializes the required frameworks */ - if (PMIX_SUCCESS != (rc = pmix_rte_init(PMIX_PROC_TOOL, info, ninfo, + if (PMIX_SUCCESS != (rc = pmix_rte_init(ptype, info, ninfo, pmix_tool_notify_recv))) { PMIX_ERROR_LOG(rc); PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } + /* if we were given a name, then set it now */ + if (nspace_given || nspace_in_enviro) { + (void)strncpy(pmix_globals.myid.nspace, nspace, PMIX_MAX_NSLEN); + free(nspace); + pmix_globals.myid.rank = rank; + } + /* setup the IO Forwarding recv */ rcv = PMIX_NEW(pmix_ptl_posted_recv_t); rcv->tag = PMIX_PTL_TAG_IOF; @@ -299,31 +434,60 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, pmix_list_append(&pmix_ptl_globals.posted_recvs, &rcv->super); + /* setup the globals */ PMIX_CONSTRUCT(&pmix_client_globals.pending_requests, pmix_list_t); PMIX_CONSTRUCT(&pmix_client_globals.peers, pmix_pointer_array_t); pmix_pointer_array_init(&pmix_client_globals.peers, 1, INT_MAX, 1); pmix_client_globals.myserver = PMIX_NEW(pmix_peer_t); + if (NULL == pmix_client_globals.myserver) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_NOMEM; + } pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); + if (NULL == pmix_client_globals.myserver->nptr) { + PMIX_RELEASE(pmix_client_globals.myserver); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_NOMEM; + } + pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); + if (NULL == pmix_client_globals.myserver->info) { + PMIX_RELEASE(pmix_client_globals.myserver); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_NOMEM; + } pmix_output_verbose(2, pmix_globals.debug_output, "pmix: init called"); - /* select our bfrops compat module */ - pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(NULL); - if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) { - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_INIT; + if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + /* if we are a client, then we need to pickup the + * rest of the envar-based server assignments */ + pmix_globals.pindex = -1; + /* setup a rank_info object for us */ + pmix_globals.mypeer->info = PMIX_NEW(pmix_rank_info_t); + if (NULL == pmix_globals.mypeer->info) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_NOMEM; + } + pmix_globals.mypeer->info->pname.nspace = strdup(pmix_globals.myid.nspace); + pmix_globals.mypeer->info->pname.rank = pmix_globals.myid.rank; + /* our bfrops module will be set when we connect to the server */ + } else { + /* select our bfrops compat module */ + pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(NULL); + if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_INIT; + } + /* the server will be using the same */ + pmix_client_globals.myserver->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops; } - /* the server will be using the same */ - pmix_client_globals.myserver->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops; - /* set the buffer type */ - pmix_globals.mypeer->nptr->compat.type = pmix_bfrops_globals.default_type; - /* the server will be using the same */ - pmix_client_globals.myserver->nptr->compat.type = pmix_globals.mypeer->nptr->compat.type; - - /* select our psec compat module */ - pmix_globals.mypeer->nptr->compat.psec = pmix_psec_base_assign_module(NULL); + /* select our psec compat module - the selection may be based + * on the corresponding envars that should have been passed + * to us at launch */ + evar = getenv("PMIX_SECURITY_MODE"); + pmix_globals.mypeer->nptr->compat.psec = pmix_psec_base_assign_module(evar); if (NULL == pmix_globals.mypeer->nptr->compat.psec) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; @@ -331,38 +495,25 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, /* the server will be using the same */ pmix_client_globals.myserver->nptr->compat.psec = pmix_globals.mypeer->nptr->compat.psec; - /* select the gds compat module */ - pmix_client_globals.myserver->nptr->compat.gds = pmix_gds_base_assign_module(NULL, 0); - if (NULL == pmix_client_globals.myserver->nptr->compat.gds) { - PMIX_INFO_DESTRUCT(&ginfo); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_INIT; + /* set the buffer type - the selection will be based + * on the corresponding envars that should have been passed + * to us at launch */ + evar = getenv("PMIX_BFROP_BUFFER_TYPE"); + if (NULL == evar) { + /* just set to our default */ + pmix_globals.mypeer->nptr->compat.type = pmix_bfrops_globals.default_type; + } else if (0 == strcmp(evar, "PMIX_BFROP_BUFFER_FULLY_DESC")) { + pmix_globals.mypeer->nptr->compat.type = PMIX_BFROP_BUFFER_FULLY_DESC; + } else { + pmix_globals.mypeer->nptr->compat.type = PMIX_BFROP_BUFFER_NON_DESC; } + /* the server will be using the same */ + pmix_client_globals.myserver->nptr->compat.type = pmix_globals.mypeer->nptr->compat.type; - /* now select a GDS module for our own internal use - the user may + /* select a GDS module for our own internal use - the user may * have passed down a directive for this purpose. If they did, then * use it. Otherwise, we want the "hash" module */ - found = false; - if (NULL != info) { - for (n=0; n < ninfo; n++) { - if (0 == strncmp(info[n].key, PMIX_GDS_MODULE, PMIX_MAX_KEYLEN)) { - PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, info[n].value.data.string, PMIX_STRING); - found = true; - } else if (0 == strncmp(info[n].key, PMIX_TOOL_DO_NOT_CONNECT, PMIX_MAX_KEYLEN)) { - do_not_connect = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strncmp(info[n].key, PMIX_TOOL_NSPACE, PMIX_MAX_KEYLEN)) { - (void)strncpy(pmix_globals.myid.nspace, info[n].value.data.string, PMIX_MAX_NSLEN); - nspace_given = true; - } else if (0 == strncmp(info[n].key, PMIX_TOOL_RANK, PMIX_MAX_KEYLEN)) { - pmix_globals.myid.rank = info[n].value.data.rank; - rank_given = true; - } else if (0 == strncmp(info[n].key, PMIX_FWD_STDIN, PMIX_MAX_KEYLEN)) { - /* they want us to forward our stdin to someone */ - fwd_stdin = true; - } - } - } - if (!found) { + if (!gdsfound) { PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, "hash", PMIX_STRING); } pmix_globals.mypeer->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1); @@ -372,6 +523,22 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, return PMIX_ERR_INIT; } PMIX_INFO_DESTRUCT(&ginfo); + /* select the gds compat module we will use to interact with + * our server- the selection will be based + * on the corresponding envars that should have been passed + * to us at launch */ + evar = getenv("PMIX_GDS_MODULE"); + if (NULL != evar) { + PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, evar, PMIX_STRING); + pmix_client_globals.myserver->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1); + PMIX_INFO_DESTRUCT(&ginfo); + } else { + pmix_client_globals.myserver->nptr->compat.gds = pmix_gds_base_assign_module(NULL, 0); + } + if (NULL == pmix_client_globals.myserver->nptr->compat.gds) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_INIT; + } if (do_not_connect) { /* ensure we mark that we are not connected */ @@ -382,19 +549,23 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, return PMIX_ERR_INIT; } } else { - /* connect to the server - returns job info if successful */ + /* connect to the server */ rc = pmix_ptl_base_connect_to_peer((struct pmix_peer_t*)pmix_client_globals.myserver, info, ninfo); if (PMIX_SUCCESS != rc){ PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } } - /* Success, so copy the nspace and rank to the proc struct they gave us */ - (void)strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); - proc->rank = pmix_globals.myid.rank; + if (!nspace_given) { + /* Success, so copy the nspace and rank to the proc struct they gave us */ + (void)strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + } + if (!rank_given) { + proc->rank = pmix_globals.myid.rank; + } /* and into our own peer object */ if (NULL == pmix_globals.mypeer->nptr->nspace) { - pmix_globals.mypeer->nptr->nspace = strdup(proc->nspace); + pmix_globals.mypeer->nptr->nspace = strdup(pmix_globals.myid.nspace); } /* setup a rank_info object for us */ pmix_globals.mypeer->info = PMIX_NEW(pmix_rank_info_t); @@ -402,8 +573,40 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOMEM; } - pmix_globals.mypeer->info->pname.nspace = strdup(proc->nspace); - pmix_globals.mypeer->info->pname.rank = proc->rank; + pmix_globals.mypeer->info->pname.nspace = strdup(pmix_globals.myid.nspace); + pmix_globals.mypeer->info->pname.rank = pmix_globals.myid.rank; + + /* if we are acting as a client, then send a request for our + * job info - we do this as a non-blocking + * transaction because some systems cannot handle very large + * blocking operations and error out if we try them. */ + if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + req = PMIX_NEW(pmix_buffer_t); + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + req, &cmd, 1, PMIX_COMMAND); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(req); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + /* send to the server */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, + req, job_data, (void*)&cb); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + /* wait for the data to return */ + PMIX_WAIT_THREAD(&cb.lock); + rc = cb.status; + PMIX_DESTRUCT(&cb); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + } /* setup IOF */ PMIX_IOF_SINK_DEFINE(&pmix_client_globals.iof_stdout, &pmix_globals.myid, @@ -460,9 +663,9 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, PMIX_IOF_READ_ACTIVATE(&stdinev); } } else { - /* if we are not looking at a tty, just setup a read event - * and activate it - */ + /* if we are not looking at a tty, just setup a read event + * and activate it + */ PMIX_CONSTRUCT(&stdinev, pmix_iof_read_event_t); stdinev.fd = fd; stdinev.always_readable = pmix_iof_fd_always_ready(fd); @@ -480,319 +683,337 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, PMIX_IOF_READ_ACTIVATE(&stdinev); } } + /* increment our init reference counter */ pmix_globals.init_cntr++; - - /* now finish the initialization by filling our local - * datastore with typical job-related info. No point - * in having the server generate these as we are - * obviously a singleton, and so the values are well-known */ - (void)strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); - wildcard.rank = pmix_globals.myid.rank; - - /* the jobid is just our nspace */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_JOBID); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_STRING; - kptr->value->data.string = strdup(pmix_globals.myid.nspace); - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &wildcard, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* our rank */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_RANK); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_INT; - kptr->value->data.integer = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* nproc offset */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_NPROC_OFFSET); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &wildcard, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* node size */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_NODE_SIZE); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 1; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &wildcard, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* local peers */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_LOCAL_PEERS); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_STRING; - kptr->value->data.string = strdup("0"); - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &wildcard, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* local leader */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_LOCALLDR); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &wildcard, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* universe size */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_UNIV_SIZE); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 1; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &wildcard, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* job size - we are our very own job, so we have no peers */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_JOB_SIZE); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 1; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &wildcard, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* local size - only us in our job */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_LOCAL_SIZE); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 1; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &wildcard, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* max procs - since we are a self-started tool, there is no - * allocation within which we can grow ourselves */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_MAX_PROCS); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 1; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &wildcard, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* app number */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_APPNUM); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* app leader */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_APPLDR); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* app rank */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_APP_RANK); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* global rank */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_GLOBAL_RANK); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* local rank - we are alone in our job */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_LOCAL_RANK); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* we cannot know the node rank as we don't know what - * other processes are executing on this node - so - * we'll add that info to the server-tool handshake - * and load it from there */ - - /* hostname */ - gethostname(hostname, PMIX_MAX_NSLEN); - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_HOSTNAME); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_STRING; - kptr->value->data.string = strdup(hostname); - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* we cannot know the RM's nodeid for this host, so - * we'll add that info to the server-tool handshake - * and load it from there */ - - /* the nodemap is simply our hostname as there is no - * regex to generate */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_NODE_MAP); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_STRING; - kptr->value->data.string = strdup(hostname); - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &wildcard, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; + if (!PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + /* now finish the initialization by filling our local + * datastore with typical job-related info. No point + * in having the server generate these as we are + * obviously a singleton, and so the values are well-known */ + (void)strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + wildcard.rank = pmix_globals.myid.rank; + + /* the jobid is just our nspace */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_JOBID); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_STRING; + kptr->value->data.string = strdup(pmix_globals.myid.nspace); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* our rank */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_RANK); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_INT; + kptr->value->data.integer = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* nproc offset */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_NPROC_OFFSET); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* node size */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_NODE_SIZE); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 1; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* local peers */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_LOCAL_PEERS); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_STRING; + kptr->value->data.string = strdup("0"); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* local leader */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_LOCALLDR); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* universe size */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_UNIV_SIZE); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 1; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* job size - we are our very own job, so we have no peers */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_JOB_SIZE); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 1; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* local size - only us in our job */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_LOCAL_SIZE); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 1; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* max procs - since we are a self-started tool, there is no + * allocation within which we can grow ourselves */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_MAX_PROCS); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 1; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* app number */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_APPNUM); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* app leader */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_APPLDR); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* app rank */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_APP_RANK); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* global rank */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_GLOBAL_RANK); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* local rank - we are alone in our job */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_LOCAL_RANK); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT16; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* we cannot know the node rank as we don't know what + * other processes are executing on this node - so + * we'll add that info to the server-tool handshake + * and load it from there */ + + /* hostname */ + gethostname(hostname, PMIX_MAX_NSLEN); + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_HOSTNAME); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_STRING; + kptr->value->data.string = strdup(hostname); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* we cannot know the RM's nodeid for this host, so + * we'll add that info to the server-tool handshake + * and load it from there */ + + /* the nodemap is simply our hostname as there is no + * regex to generate */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_NODE_MAP); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_STRING; + kptr->value->data.string = strdup(hostname); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* likewise, the proc map is just our rank as we are + * the only proc in this job */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_PROC_MAP); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_STRING; + kptr->value->data.string = strdup("0"); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting } - PMIX_RELEASE(kptr); // maintain accounting - - /* likewise, the proc map is just our rank as we are - * the only proc in this job */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_PROC_MAP); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_STRING; - kptr->value->data.string = strdup("0"); - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &wildcard, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; + PMIX_RELEASE_THREAD(&pmix_global_lock); + + /* if we are acting as a server, then start listening */ + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + /* setup the wildcard recv for inbound messages from clients */ + rcv = PMIX_NEW(pmix_ptl_posted_recv_t); + rcv->tag = UINT32_MAX; + rcv->cbfunc = pmix_server_message_handler; + /* add it to the end of the list of recvs */ + pmix_list_append(&pmix_ptl_globals.posted_recvs, &rcv->super); + + /* start listening for connections */ + if (PMIX_SUCCESS != pmix_ptl_base_start_listening(info, ninfo)) { + pmix_show_help("help-pmix-server.txt", "listener-thread-start", true); + return PMIX_ERR_INIT; + } } - PMIX_RELEASE(kptr); // maintain accounting - PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } @@ -841,6 +1062,7 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) struct timeval tv = {2, 0}; int n; pmix_peer_t *peer; + pmix_setup_caddy_t *cd; PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (1 != pmix_globals.init_cntr) { @@ -849,11 +1071,6 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) return PMIX_SUCCESS; } pmix_globals.init_cntr = 0; - /* if we are not connected, then we are done */ - if (!pmix_globals.connected) { - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_SUCCESS; - } PMIX_RELEASE_THREAD(&pmix_global_lock); pmix_output_verbose(2, pmix_globals.debug_output, @@ -865,47 +1082,49 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) PMIX_DESTRUCT(&pmix_client_globals.iof_stdout); PMIX_DESTRUCT(&pmix_client_globals.iof_stderr); - /* setup a cmd message to notify the PMIx - * server that we are normally terminating */ - msg = PMIX_NEW(pmix_buffer_t); - /* pack the cmd */ - PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, - msg, &cmd, 1, PMIX_COMMAND); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - return rc; - } - + /* if we are connected, then disconnect */ + if (pmix_globals.connected) { + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:tool sending finalize sync to server"); + + /* setup a cmd message to notify the PMIx + * server that we are normally terminating */ + msg = PMIX_NEW(pmix_buffer_t); + /* pack the cmd */ + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + msg, &cmd, 1, PMIX_COMMAND); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + return rc; + } + /* setup a timer to protect ourselves should the server be unable + * to answer for some reason */ + PMIX_CONSTRUCT_LOCK(&tev.lock); + pmix_event_assign(&tev.ev, pmix_globals.evbase, -1, 0, + fin_timeout, &tev); + tev.active = true; + PMIX_POST_OBJECT(&tev); + pmix_event_add(&tev.ev, &tv); + PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg, + finwait_cbfunc, (void*)&tev); + if (PMIX_SUCCESS != rc) { + if (tev.active) { + pmix_event_del(&tev.ev); + } + return rc; + } - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:tool sending finalize sync to server"); - - /* setup a timer to protect ourselves should the server be unable - * to answer for some reason */ - PMIX_CONSTRUCT_LOCK(&tev.lock); - pmix_event_assign(&tev.ev, pmix_globals.evbase, -1, 0, - fin_timeout, &tev); - tev.active = true; - PMIX_POST_OBJECT(&tev); - pmix_event_add(&tev.ev, &tv); - PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg, - finwait_cbfunc, (void*)&tev); - if (PMIX_SUCCESS != rc) { + /* wait for the ack to return */ + PMIX_WAIT_THREAD(&tev.lock); + PMIX_DESTRUCT_LOCK(&tev.lock); if (tev.active) { pmix_event_del(&tev.ev); } - return rc; - } + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:tool finalize sync received"); - /* wait for the ack to return */ - PMIX_WAIT_THREAD(&tev.lock); - PMIX_DESTRUCT_LOCK(&tev.lock); - if (tev.active) { - pmix_event_del(&tev.ev); } - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:tool finalize sync received"); if (!pmix_globals.external_evbase) { /* stop the progress thread, but leave the event base @@ -923,8 +1142,111 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) } } + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + pmix_ptl_base_stop_listening(); + + /* cleanout any IOF */ + for (n=0; n < PMIX_IOF_HOTEL_SIZE; n++) { + pmix_hotel_checkout_and_return_occupant(&pmix_server_globals.iof, n, (void**)&cd); + if (NULL != cd) { + PMIX_RELEASE(cd); + } + } + PMIX_DESTRUCT(&pmix_server_globals.iof); + for (n=0; n < pmix_server_globals.clients.size; n++) { + if (NULL != (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, n))) { + PMIX_RELEASE(peer); + } + } + PMIX_DESTRUCT(&pmix_server_globals.clients); + PMIX_LIST_DESTRUCT(&pmix_server_globals.collectives); + PMIX_LIST_DESTRUCT(&pmix_server_globals.remote_pnd); + PMIX_LIST_DESTRUCT(&pmix_server_globals.local_reqs); + PMIX_LIST_DESTRUCT(&pmix_server_globals.gdata); + PMIX_LIST_DESTRUCT(&pmix_server_globals.events); + PMIX_LIST_DESTRUCT(&pmix_server_globals.nspaces); + } + /* shutdown services */ pmix_rte_finalize(); + if (NULL != pmix_globals.mypeer) { + PMIX_RELEASE(pmix_globals.mypeer); + } + /* finalize the class/object system */ + pmix_class_finalize(); return PMIX_SUCCESS; } + +pmix_status_t PMIx_tool_connect_to_server(pmix_proc_t *proc, + pmix_info_t info[], size_t ninfo) +{ + pmix_buffer_t *msg; + pmix_cmd_t cmd = PMIX_FINALIZE_CMD; + pmix_status_t rc; + pmix_tool_timeout_t tev; + struct timeval tv = {2, 0}; + + PMIX_ACQUIRE_THREAD(&pmix_global_lock); + if (pmix_globals.init_cntr <= 0) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_INIT; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + + /* check for bozo error - we do this + * prior to breaking any existing connection to avoid + * becoming stranded due to an incorrect function call */ + if (NULL == info || 0 == ninfo) { + pmix_show_help("help-pmix-runtime.txt", "tool:no-server", true); + return PMIX_ERR_BAD_PARAM; + } + + /* break any existing connection */ + if (pmix_globals.connected) { + /* gracefully terminate this connection */ + msg = PMIX_NEW(pmix_buffer_t); + /* pack the cmd */ + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + msg, &cmd, 1, PMIX_COMMAND); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + return rc; + } + + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:tool:reconnect sending finalize sync to server"); + + /* setup a timer to protect ourselves should the server be unable + * to answer for some reason */ + PMIX_CONSTRUCT_LOCK(&tev.lock); + pmix_event_assign(&tev.ev, pmix_globals.evbase, -1, 0, + fin_timeout, &tev); + tev.active = true; + PMIX_POST_OBJECT(&tev); + pmix_event_add(&tev.ev, &tv); + PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg, + finwait_cbfunc, (void*)&tev); + if (PMIX_SUCCESS != rc) { + if (tev.active) { + pmix_event_del(&tev.ev); + } + return rc; + } + + /* wait for the ack to return */ + PMIX_WAIT_THREAD(&tev.lock); + PMIX_DESTRUCT_LOCK(&tev.lock); + if (tev.active) { + pmix_event_del(&tev.ev); + } + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:tool:reconnect finalize sync received"); + } + + /* now ask the ptl to establish connection to the new server */ + rc = pmix_ptl_base_connect_to_peer((struct pmix_peer_t*)pmix_client_globals.myserver, info, ninfo); + return rc; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/Makefile.include b/opal/mca/pmix/pmix3x/pmix/src/tools/Makefile.include index 46e25c03ca7..58bfa148fd4 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/Makefile.include +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/Makefile.include @@ -13,7 +13,7 @@ # Copyright (c) 2006-2008 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights # reserved. -# Copyright (c) 2014-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2014-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -26,12 +26,12 @@ SUBDIRS += \ tools/pevent \ - tools/pinfo \ + tools/pmix_info \ tools/plookup \ tools/pps DIST_SUBDIRS += \ tools/pevent \ - tools/pinfo \ + tools/pmix_info \ tools/plookup \ tools/pps diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/Makefile.am similarity index 90% rename from opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/Makefile.am rename to opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/Makefile.am index 47225029d35..8fe2640c7ae 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2008-2014 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. -# Copyright (c) 2017 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -38,17 +38,17 @@ AM_CFLAGS = \ if PMIX_INSTALL_BINARIES -bin_PROGRAMS = pinfo +bin_PROGRAMS = pmix_info -dist_pmixdata_DATA = help-pinfo.txt +dist_pmixdata_DATA = help-pmix-info.txt endif # PMIX_INSTALL_BINARIES -pinfo_SOURCES = \ +pmix_info_SOURCES = \ pinfo.h \ support.h \ - pinfo.c \ + pmix_info.c \ support.c -pinfo_LDADD = \ +pmix_info_LDADD = \ $(top_builddir)/src/libpmix.la diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/help-pinfo.txt b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/help-pmix-info.txt similarity index 96% rename from opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/help-pinfo.txt rename to opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/help-pmix-info.txt index 864a1489067..88cfa7d6d6a 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/help-pinfo.txt +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/help-pmix-info.txt @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2012 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2017 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/pinfo.h b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/pinfo.h similarity index 97% rename from opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/pinfo.h rename to opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/pinfo.h index 28cb1207419..560703dbf38 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/pinfo.h +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/pinfo.h @@ -12,7 +12,7 @@ * Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/pinfo.c b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/pmix_info.c similarity index 99% rename from opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/pinfo.c rename to opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/pmix_info.c index 1374a5ac701..972cf1c7c5c 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/pinfo.c +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/pmix_info.c @@ -13,7 +13,7 @@ * Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010-2016 Los Alamos National Security, LLC. * All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/support.c b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/support.c similarity index 99% rename from opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/support.c rename to opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/support.c index 4896f81a059..66a06106f7b 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/support.c +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/support.c @@ -14,7 +14,7 @@ * Copyright (c) 2010-2016 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2011-2012 University of Houston. All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/support.h b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/support.h similarity index 98% rename from opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/support.h rename to opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/support.h index f31d03a4d81..0e1a3be1536 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/pinfo/support.h +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/support.h @@ -3,7 +3,7 @@ * All rights reserved. * Copyright (c) 2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. - * Copyright (c) 2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/error.c b/opal/mca/pmix/pmix3x/pmix/src/util/error.c index ed2e230a387..452582407b2 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/error.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/error.c @@ -11,7 +11,7 @@ * All rights reserved. * Copyright (c) 2007-2012 Los Alamos National Security, LLC. * All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -38,143 +38,212 @@ PMIX_EXPORT const char* PMIx_Error_string(pmix_status_t errnum) { switch(errnum) { - case PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER: - return "UNPACK-PAST-END"; - case PMIX_ERR_LOST_CONNECTION_TO_SERVER: - return "LOST_CONNECTION_TO_SERVER"; - case PMIX_ERR_NOT_SUPPORTED: - return "NOT-SUPPORTED"; - case PMIX_ERR_NOT_FOUND: - return "NOT-FOUND"; - case PMIX_ERR_SERVER_NOT_AVAIL: - return "SERVER-NOT-AVAIL"; - case PMIX_ERR_INVALID_NAMESPACE: - return "INVALID-NAMESPACE"; - case PMIX_ERR_INVALID_SIZE: - return "INVALID-SIZE"; - case PMIX_ERR_INVALID_KEYVALP: - return "INVALID-KEYVAL"; - case PMIX_ERR_INVALID_NUM_PARSED: - return "INVALID-NUM-PARSED"; - case PMIX_ERR_TAKE_NEXT_OPTION: - return "TAKE-NEXT-OPTION"; - case PMIX_ERR_INVALID_ARGS: - return "INVALID-ARGS"; - case PMIX_ERR_INVALID_NUM_ARGS: - return "INVALID-NUM-ARGS"; - case PMIX_ERR_INVALID_LENGTH: - return "INVALID-LENGTH"; - case PMIX_ERR_INVALID_VAL_LENGTH: - return "INVALID-VAL-LENGTH"; - case PMIX_ERR_INVALID_VAL: - return "INVALID-VAL"; - case PMIX_ERR_INVALID_KEY_LENGTH: - return "INVALID-KEY-LENGTH"; - case PMIX_ERR_INVALID_KEY: - return "INVALID-KEY"; - case PMIX_ERR_INVALID_ARG: - return "INVALID-ARG"; - case PMIX_ERR_NOMEM: - return "NO-MEM"; - case PMIX_ERR_INIT: - return "INIT"; + case PMIX_SUCCESS: + return "SUCCESS"; + case PMIX_ERROR: + return "ERROR"; + case PMIX_ERR_SILENT: + return "SILENT_ERROR"; + + + case PMIX_ERR_DEBUGGER_RELEASE: + return "DEBUGGER-RELEASE"; - case PMIX_ERR_DATA_VALUE_NOT_FOUND: - return "DATA-VALUE-NOT-FOUND"; - case PMIX_ERR_OUT_OF_RESOURCE: - return "OUT-OF-RESOURCE"; - case PMIX_ERR_BAD_PARAM: - return "BAD-PARAM"; - case PMIX_ERR_IN_ERRNO: - return "ERR-IN-ERRNO"; - case PMIX_ERR_UNREACH: - return "UNREACHABLE"; - case PMIX_ERR_TIMEOUT: - return "TIMEOUT"; - case PMIX_ERR_NO_PERMISSIONS: - return "NO-PERMISSIONS"; - case PMIX_ERR_PACK_MISMATCH: - return "PACK-MISMATCH"; - case PMIX_ERR_PACK_FAILURE: - return "PACK-FAILURE"; - case PMIX_ERR_UNPACK_FAILURE: - return "UNPACK-FAILURE"; - case PMIX_ERR_UNPACK_INADEQUATE_SPACE: - return "UNPACK-INADEQUATE-SPACE"; - case PMIX_ERR_TYPE_MISMATCH: - return "TYPE-MISMATCH"; - case PMIX_ERR_PROC_ENTRY_NOT_FOUND: - return "PROC-ENTRY-NOT-FOUND"; - case PMIX_ERR_UNKNOWN_DATA_TYPE: - return "UNKNOWN-DATA-TYPE"; - case PMIX_ERR_WOULD_BLOCK: - return "WOULD-BLOCK"; - case PMIX_ERR_READY_FOR_HANDSHAKE: - return "READY-FOR-HANDSHAKE"; - case PMIX_ERR_HANDSHAKE_FAILED: - return "HANDSHAKE-FAILED"; - case PMIX_ERR_INVALID_CRED: - return "INVALID-CREDENTIAL"; - case PMIX_EXISTS: - return "EXISTS"; - case PMIX_ERR_SERVER_FAILED_REQUEST: - return "SERVER FAILED REQUEST"; - case PMIX_ERR_PROC_MIGRATE: - return "PROC-MIGRATE"; - case PMIX_ERR_PROC_CHECKPOINT: - return "PROC-CHECKPOINT-ERROR"; case PMIX_ERR_PROC_RESTART: return "PROC_RESTART"; - case PMIX_ERR_PROC_ABORTING: - return "PROC-ABORTING"; - case PMIX_ERR_PROC_REQUESTED_ABORT: - return "PROC-ABORT-REQUESTED"; + case PMIX_ERR_PROC_CHECKPOINT: + return "PROC-CHECKPOINT-ERROR"; + case PMIX_ERR_PROC_MIGRATE: + return "PROC-MIGRATE"; + + case PMIX_ERR_PROC_ABORTED: return "PROC-ABORTED"; - case PMIX_ERR_DEBUGGER_RELEASE: - return "DEBUGGER-RELEASE"; - case PMIX_ERR_SILENT: - return "SILENT_ERROR"; - case PMIX_ERROR: - return "ERROR"; - case PMIX_ERR_NOT_AVAILABLE: - return "PMIX_ERR_NOT_AVAILABLE"; - case PMIX_ERR_FATAL: - return "PMIX_ERR_FATAL"; - case PMIX_ERR_VALUE_OUT_OF_BOUNDS: - return "PMIX_ERR_VALUE_OUT_OF_BOUNDS"; - case PMIX_ERR_NETWORK_NOT_PARSEABLE: - return "PMIX_ERR_NETWORK_NOT_PARSEABLE"; - case PMIX_ERR_FILE_OPEN_FAILURE: - return "PMIX_ERR_FILE_OPEN_FAILURE"; - case PMIX_ERR_FILE_READ_FAILURE: - return "PMIX_ERR_FILE_READ_FAILURE"; - case PMIX_ERR_PERM: - return "PMIX_ERR_PERM"; - case PMIX_ERR_JOB_TERMINATED: - return "PMIX_ERR_JOB_TERMINATED"; - case PMIX_MAX_ERR_CONSTANT: - return "PMIX_ERR_WILDCARD"; + case PMIX_ERR_PROC_REQUESTED_ABORT: + return "PROC-ABORT-REQUESTED"; + case PMIX_ERR_PROC_ABORTING: + return "PROC-ABORTING"; + + + case PMIX_ERR_SERVER_FAILED_REQUEST: + return "SERVER FAILED REQUEST"; + case PMIX_EXISTS: + return "EXISTS"; + case PMIX_ERR_INVALID_CRED: + return "INVALID-CREDENTIAL"; + case PMIX_ERR_HANDSHAKE_FAILED: + return "HANDSHAKE-FAILED"; + case PMIX_ERR_READY_FOR_HANDSHAKE: + return "READY-FOR-HANDSHAKE"; + case PMIX_ERR_WOULD_BLOCK: + return "WOULD-BLOCK"; + case PMIX_ERR_UNKNOWN_DATA_TYPE: + return "UNKNOWN-DATA-TYPE"; + case PMIX_ERR_PROC_ENTRY_NOT_FOUND: + return "PROC-ENTRY-NOT-FOUND"; + case PMIX_ERR_TYPE_MISMATCH: + return "TYPE-MISMATCH"; + case PMIX_ERR_UNPACK_INADEQUATE_SPACE: + return "UNPACK-INADEQUATE-SPACE"; + case PMIX_ERR_UNPACK_FAILURE: + return "UNPACK-FAILURE"; + case PMIX_ERR_PACK_FAILURE: + return "PACK-FAILURE"; + case PMIX_ERR_PACK_MISMATCH: + return "PACK-MISMATCH"; + case PMIX_ERR_NO_PERMISSIONS: + return "NO-PERMISSIONS"; + case PMIX_ERR_TIMEOUT: + return "TIMEOUT"; + case PMIX_ERR_UNREACH: + return "UNREACHABLE"; + case PMIX_ERR_IN_ERRNO: + return "ERR-IN-ERRNO"; + case PMIX_ERR_BAD_PARAM: + return "BAD-PARAM"; + case PMIX_ERR_RESOURCE_BUSY: + return "RESOURCE-BUSY"; + case PMIX_ERR_OUT_OF_RESOURCE: + return "OUT-OF-RESOURCE"; + case PMIX_ERR_DATA_VALUE_NOT_FOUND: + return "DATA-VALUE-NOT-FOUND"; + case PMIX_ERR_INIT: + return "INIT"; + case PMIX_ERR_NOMEM: + return "NO-MEM"; + case PMIX_ERR_INVALID_ARG: + return "INVALID-ARG"; + case PMIX_ERR_INVALID_KEY: + return "INVALID-KEY"; + case PMIX_ERR_INVALID_KEY_LENGTH: + return "INVALID-KEY-LENGTH"; + case PMIX_ERR_INVALID_VAL: + return "INVALID-VAL"; + case PMIX_ERR_INVALID_VAL_LENGTH: + return "INVALID-VAL-LENGTH"; + case PMIX_ERR_INVALID_LENGTH: + return "INVALID-LENGTH"; + case PMIX_ERR_INVALID_NUM_ARGS: + return "INVALID-NUM-ARGS"; + case PMIX_ERR_INVALID_ARGS: + return "INVALID-ARGS"; + case PMIX_ERR_INVALID_NUM_PARSED: + return "INVALID-NUM-PARSED"; + case PMIX_ERR_INVALID_KEYVALP: + return "INVALID-KEYVAL"; + case PMIX_ERR_INVALID_SIZE: + return "INVALID-SIZE"; + case PMIX_ERR_INVALID_NAMESPACE: + return "INVALID-NAMESPACE"; + case PMIX_ERR_SERVER_NOT_AVAIL: + return "SERVER-NOT-AVAIL"; + case PMIX_ERR_NOT_FOUND: + return "NOT-FOUND"; + case PMIX_ERR_NOT_SUPPORTED: + return "NOT-SUPPORTED"; + case PMIX_ERR_NOT_IMPLEMENTED: + return "NOT-IMPLEMENTED"; + case PMIX_ERR_COMM_FAILURE: + return "COMM-FAILURE"; + case PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER: + return "UNPACK-PAST-END"; + case PMIX_ERR_CONFLICTING_CLEANUP_DIRECTIVES: + return "PMIX CONFLICTING CLEANUP DIRECTIVES"; + + + case PMIX_ERR_LOST_CONNECTION_TO_SERVER: + return "LOST_CONNECTION_TO_SERVER"; + case PMIX_ERR_LOST_PEER_CONNECTION: + return "LOST-PEER-CONNECTION"; + case PMIX_ERR_LOST_CONNECTION_TO_CLIENT: + return "LOST-CONNECTION-TO-CLIENT"; + + + case PMIX_QUERY_PARTIAL_SUCCESS: + return "QUERY-PARTIAL-SUCCESS"; + + case PMIX_NOTIFY_ALLOC_COMPLETE: return "PMIX ALLOC OPERATION COMPLETE"; + + case PMIX_JCTRL_CHECKPOINT: return "PMIX JOB CONTROL CHECKPOINT"; + case PMIX_JCTRL_CHECKPOINT_COMPLETE: + return "PMIX JOB CONTROL CHECKPOINT COMPLETE"; case PMIX_JCTRL_PREEMPT_ALERT: return "PMIX PRE-EMPTION ALERT"; + + case PMIX_MONITOR_HEARTBEAT_ALERT: return "PMIX HEARTBEAT ALERT"; case PMIX_MONITOR_FILE_ALERT: return "PMIX FILE MONITOR ALERT"; + + case PMIX_ERR_EVENT_REGISTRATION: + return "EVENT-REGISTRATION"; + case PMIX_ERR_JOB_TERMINATED: + return "PMIX_ERR_JOB_TERMINATED"; + case PMIX_ERR_UPDATE_ENDPOINTS: + return "UPDATE-ENDPOINTS"; case PMIX_MODEL_DECLARED: return "PMIX MODEL DECLARED"; + case PMIX_GDS_ACTION_COMPLETE: + return "GDS-ACTION-COMPLETE"; + case PMIX_PROC_HAS_CONNECTED: + return "PROC-HAS-CONNECTED"; + case PMIX_CONNECT_REQUESTED: + return "CONNECT-REQUESTED"; + case PMIX_LAUNCH_DIRECTIVE: + return "LAUNCH-DIRECTIVE"; + case PMIX_LAUNCHER_READY: + return "LAUNCHER-READY"; + case PMIX_OPERATION_IN_PROGRESS: + return "OPERATION-IN-PROGRESS"; + + + case PMIX_ERR_NODE_DOWN: + return "NODE-DOWN"; + case PMIX_ERR_NODE_OFFLINE: + return "NODE-OFFLINE"; + + + case PMIX_EVENT_NO_ACTION_TAKEN: + return "EVENT-NO-ACTION-TAKEN"; + case PMIX_EVENT_PARTIAL_ACTION_TAKEN: + return "EVENT-PARTIAL-ACTION-TAKEN"; + case PMIX_EVENT_ACTION_DEFERRED: + return "EVENT-ACTION-DEFERRED"; + case PMIX_EVENT_ACTION_COMPLETE: + return "EVENT-ACTION-COMPLETE"; + + + case PMIX_ERR_NOT_AVAILABLE: + return "PMIX_ERR_NOT_AVAILABLE"; + case PMIX_ERR_FATAL: + return "PMIX_ERR_FATAL"; + case PMIX_ERR_VALUE_OUT_OF_BOUNDS: + return "PMIX_ERR_VALUE_OUT_OF_BOUNDS"; + case PMIX_ERR_PERM: + return "PMIX_ERR_PERM"; + case PMIX_ERR_NETWORK_NOT_PARSEABLE: + return "PMIX_ERR_NETWORK_NOT_PARSEABLE"; + case PMIX_ERR_FILE_OPEN_FAILURE: + return "PMIX_ERR_FILE_OPEN_FAILURE"; + case PMIX_ERR_FILE_READ_FAILURE: + return "PMIX_ERR_FILE_READ_FAILURE"; + case PMIX_ERR_TAKE_NEXT_OPTION: + return "TAKE-NEXT-OPTION"; case PMIX_ERR_TEMP_UNAVAILABLE: return "PMIX TEMPORARILY UNAVAILABLE"; - case PMIX_ERR_CONFLICTING_CLEANUP_DIRECTIVES: - return "PMIX CONFLICTING CLEANUP DIRECTIVES"; - case PMIX_SUCCESS: - return "SUCCESS"; + + + case PMIX_MAX_ERR_CONSTANT: + return "PMIX_ERR_WILDCARD"; + + default: return "ERROR STRING NOT FOUND"; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/error.h b/opal/mca/pmix/pmix3x/pmix/src/util/error.h index 12532b05a0c..e899e0368a1 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/error.h +++ b/opal/mca/pmix/pmix3x/pmix/src/util/error.h @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -33,7 +33,6 @@ #define PMIX_ERR_FATAL (PMIX_INTERNAL_ERR_BASE - 29) #define PMIX_ERR_VALUE_OUT_OF_BOUNDS (PMIX_INTERNAL_ERR_BASE - 30) #define PMIX_ERR_PERM (PMIX_INTERNAL_ERR_BASE - 31) -#define PMIX_ERR_OPERATION_IN_PROGRESS (PMIX_INTERNAL_ERR_BASE - 32) #define PMIX_ERR_NETWORK_NOT_PARSEABLE (PMIX_INTERNAL_ERR_BASE - 33) #define PMIX_ERR_FILE_OPEN_FAILURE (PMIX_INTERNAL_ERR_BASE - 34) #define PMIX_ERR_FILE_READ_FAILURE (PMIX_INTERNAL_ERR_BASE - 35) diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/hash.c b/opal/mca/pmix/pmix3x/pmix/src/util/hash.c index 76d6c70723e..1a0a95744ea 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/hash.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/hash.c @@ -7,7 +7,7 @@ * Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. @@ -282,7 +282,7 @@ pmix_status_t pmix_hash_remove_data(pmix_hash_table_t *table, /* if the rank is wildcard, we want to apply this to * all rank entries */ - if (PMIX_RANK_UNDEF == rank) { + if (PMIX_RANK_WILDCARD == rank) { rc = pmix_hash_table_get_first_key_uint64(table, &id, (void**)&proc_data, (void**)&node); while (PMIX_SUCCESS == rc) { @@ -302,6 +302,7 @@ pmix_status_t pmix_hash_remove_data(pmix_hash_table_t *table, rc = pmix_hash_table_get_next_key_uint64(table, &id, (void**)&proc_data, node, (void**)&node); } + return PMIX_SUCCESS; } /* lookup the specified proc */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/name_fns.c b/opal/mca/pmix/pmix3x/pmix/src/util/name_fns.c index 49c0f7e8a8f..14f19aef022 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/name_fns.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/name_fns.c @@ -13,7 +13,6 @@ * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2018 Cisco Systems, Inc. All rights reserved * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix3x/pmix/test/server_callbacks.c b/opal/mca/pmix/pmix3x/pmix/test/server_callbacks.c index 0983d38387a..783d45d6282 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/server_callbacks.c +++ b/opal/mca/pmix/pmix3x/pmix/test/server_callbacks.c @@ -310,22 +310,18 @@ static int numconnect = 0; pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, - pmix_connect_cbfunc_t cbfunc, void *cbdata) + pmix_op_cbfunc_t cbfunc, void *cbdata) { - char nspace[PMIX_MAX_NSLEN+1]; - - memset(nspace, 0, PMIX_MAX_NSLEN+1); - (void)snprintf(nspace, PMIX_MAX_NSLEN, "FOOBAR-%d", numconnect); if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, nspace, 1, cbdata); + cbfunc(PMIX_SUCCESS, cbdata); } numconnect++; return PMIX_SUCCESS; } -pmix_status_t disconnect_fn(const char nspace[], - const pmix_info_t info[], size_t ninfo, - pmix_op_cbfunc_t cbfunc, void *cbdata) +pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) { if (NULL != cbfunc) { cbfunc(PMIX_SUCCESS, cbdata); diff --git a/opal/mca/pmix/pmix3x/pmix/test/server_callbacks.h b/opal/mca/pmix/pmix3x/pmix/test/server_callbacks.h index 9248b5dfefe..8019c7a9955 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/server_callbacks.h +++ b/opal/mca/pmix/pmix3x/pmix/test/server_callbacks.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -46,8 +46,8 @@ pmix_status_t spawn_fn(const pmix_proc_t *proc, pmix_spawn_cbfunc_t cbfunc, void *cbdata); pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, - pmix_connect_cbfunc_t cbfunc, void *cbdata); -pmix_status_t disconnect_fn(const char nspace[], + pmix_op_cbfunc_t cbfunc, void *cbdata); +pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); pmix_status_t regevents_fn(pmix_status_t *codes, size_t ncodes, diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/Makefile.am b/opal/mca/pmix/pmix3x/pmix/test/simple/Makefile.am index 84795782328..3db10c3fd6b 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/simple/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/Makefile.am @@ -21,7 +21,11 @@ AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/include -I$(top_builddir)/include/pmix -noinst_PROGRAMS = simptest simpclient simppub simpdyn simpft simpdmodex test_pmix simptool simpdie simplegacy simptimeout +headers = simptest.h + +noinst_PROGRAMS = simptest simpclient simppub simpdyn simpft simpdmodex \ + test_pmix simptool simpdie simplegacy simptimeout \ + gwtest gwclient stability quietclient simptest_SOURCES = \ simptest.c @@ -88,3 +92,27 @@ simptimeout_SOURCES = \ simptimeout_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) simptimeout_LDADD = \ $(top_builddir)/src/libpmix.la + +gwtest_SOURCES = \ + gwtest.c +gwtest_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) +gwtest_LDADD = \ + $(top_builddir)/src/libpmix.la + +gwclient_SOURCES = \ + gwclient.c +gwclient_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) +gwclient_LDADD = \ + $(top_builddir)/src/libpmix.la + +stability_SOURCES = \ + stability.c +stability_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) +stability_LDADD = \ + $(top_builddir)/src/libpmix.la + +quietclient_SOURCES = \ + quietclient.c +quietclient_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) +quietclient_LDADD = \ + $(top_builddir)/src/libpmix.la diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/gwclient.c b/opal/mca/pmix/pmix3x/pmix/test/simple/gwclient.c new file mode 100644 index 00000000000..8fc3f087625 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/gwclient.c @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include +#include + +#include +#include +#include +#include + +#include "src/class/pmix_object.h" +#include "src/util/output.h" +#include "src/util/printf.h" + +static volatile bool completed = false; +static pmix_proc_t myproc; + +int main(int argc, char **argv) +{ + pmix_status_t rc; + pmix_value_t value; + pmix_value_t *val = &value; + uint64_t seckey[2]; + pmix_proc_t proc; + pmix_info_t *info; + size_t n, ninfo; + + /* init us */ + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + exit(rc); + } + pmix_output(0, "GWClient ns %s rank %d: Running", myproc.nspace, myproc.rank); + + /* look for network data */ + memset(&proc, 0, sizeof(pmix_proc_t)); + (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + + if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, "my.net.key", NULL, 0, &val)) || + PMIX_DATA_ARRAY != val->type || + NULL == val->data.darray || + NULL == val->data.darray->array) { + pmix_output(0, "Client ns %s rank %d: PMIx_Get my.net.key failed: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + exit(rc); + } + /* the returned value has all the network allocation info in a + * pmix_data_array_t */ + info = (pmix_info_t*)val->data.darray->array; + ninfo = val->data.darray->size; + pmix_output(0, "Client ns %s rank %d: got network assignment:", + myproc.nspace, myproc.rank); + for (n=0; n < ninfo; n++) { + if (PMIX_STRING == info[n].value.type) { + pmix_output(0, "\tKey: %s Value: %s", + info[n].key, info[n].value.data.string); + } else if (PMIX_BYTE_OBJECT == info[n].value.type) { + memcpy(seckey, info[n].value.data.bo.bytes, info[n].value.data.bo.size); + pmix_output(0, "\tKey: %s sec key: %ld.%ld", + info[n].key, (long int)seckey[0], (long int)seckey[1]); + } + } + PMIX_VALUE_RELEASE(val); + + /* finalize us */ + pmix_output(0, "Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank); + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { + fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + } else { + fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); + } + fflush(stderr); + return(rc); +} diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/gwtest.c b/opal/mca/pmix/pmix3x/pmix/test/simple/gwtest.c new file mode 100644 index 00000000000..0b7d376202f --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/gwtest.c @@ -0,0 +1,999 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include PMIX_EVENT_HEADER + +#include "src/class/pmix_list.h" +#include "src/util/pmix_environ.h" +#include "src/util/output.h" +#include "src/util/printf.h" +#include "src/util/argv.h" + +static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t abort_fn(const pmix_proc_t *proc, void *server_object, + int status, const char msg[], + pmix_proc_t procs[], size_t nprocs, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t fencenb_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + char *data, size_t ndata, + pmix_modex_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t dmodex_fn(const pmix_proc_t *proc, + const pmix_info_t info[], size_t ninfo, + pmix_modex_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t publish_fn(const pmix_proc_t *proc, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_lookup_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t unpublish_fn(const pmix_proc_t *proc, char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t spawn_fn(const pmix_proc_t *proc, + const pmix_info_t job_info[], size_t ninfo, + const pmix_app_t apps[], size_t napps, + pmix_spawn_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t register_event_fn(pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t deregister_events(pmix_status_t *codes, size_t ncodes, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t notify_event(pmix_status_t code, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t query_fn(pmix_proc_t *proct, + pmix_query_t *queries, size_t nqueries, + pmix_info_cbfunc_t cbfunc, + void *cbdata); +static void tool_connect_fn(pmix_info_t *info, size_t ninfo, + pmix_tool_connection_cbfunc_t cbfunc, + void *cbdata); +static void log_fn(const pmix_proc_t *client, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +static pmix_server_module_t mymodule = { + .client_connected = connected, + .client_finalized = finalized, + .abort = abort_fn, + .fence_nb = fencenb_fn, + .direct_modex = dmodex_fn, + .publish = publish_fn, + .lookup = lookup_fn, + .unpublish = unpublish_fn, + .spawn = spawn_fn, + .connect = connect_fn, + .disconnect = disconnect_fn, + .register_events = register_event_fn, + .deregister_events = deregister_events, + .notify_event = notify_event, + .query = query_fn, + .tool_connected = tool_connect_fn, + .log = log_fn +}; + +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond; + volatile bool active; + pmix_status_t status; +} mylock_t; + +#define DEBUG_CONSTRUCT_LOCK(l) \ + do { \ + pthread_mutex_init(&(l)->mutex, NULL); \ + pthread_cond_init(&(l)->cond, NULL); \ + (l)->active = true; \ + (l)->status = PMIX_SUCCESS; \ + } while(0) + +#define DEBUG_DESTRUCT_LOCK(l) \ + do { \ + pthread_mutex_destroy(&(l)->mutex); \ + pthread_cond_destroy(&(l)->cond); \ + } while(0) + +#define DEBUG_WAIT_THREAD(lck) \ + do { \ + pthread_mutex_lock(&(lck)->mutex); \ + while ((lck)->active) { \ + pthread_cond_wait(&(lck)->cond, &(lck)->mutex); \ + } \ + pthread_mutex_unlock(&(lck)->mutex); \ + } while(0) + +#define DEBUG_WAKEUP_THREAD(lck) \ + do { \ + pthread_mutex_lock(&(lck)->mutex); \ + (lck)->active = false; \ + pthread_cond_broadcast(&(lck)->cond); \ + pthread_mutex_unlock(&(lck)->mutex); \ + } while(0) + +typedef struct { + pmix_list_item_t super; + pmix_pdata_t pdata; +} pmix_locdat_t; +PMIX_CLASS_INSTANCE(pmix_locdat_t, + pmix_list_item_t, + NULL, NULL); + +typedef struct { + pmix_object_t super; + mylock_t lock; + pmix_proc_t caller; + pmix_info_t *info; + size_t ninfo; + pmix_op_cbfunc_t cbfunc; + pmix_spawn_cbfunc_t spcbfunc; + void *cbdata; +} myxfer_t; +static void xfcon(myxfer_t *p) +{ + DEBUG_CONSTRUCT_LOCK(&p->lock); + p->info = NULL; + p->ninfo = 0; + p->cbfunc = NULL; + p->spcbfunc = NULL; + p->cbdata = NULL; +} +static void xfdes(myxfer_t *p) +{ + DEBUG_DESTRUCT_LOCK(&p->lock); + if (NULL != p->info) { + PMIX_INFO_FREE(p->info, p->ninfo); + } +} +PMIX_CLASS_INSTANCE(myxfer_t, + pmix_object_t, + xfcon, xfdes); + +typedef struct { + pmix_list_item_t super; + int exit_code; + pid_t pid; +} wait_tracker_t; +PMIX_CLASS_INSTANCE(wait_tracker_t, + pmix_list_item_t, + NULL, NULL); + +static volatile int wakeup; +static int exit_code = 0; +static pmix_list_t pubdata; +static pmix_event_t handler; +static pmix_list_t children; +static bool istimeouttest = false; + +static void set_namespace(int nprocs, char *ranks, char *nspace, + pmix_op_cbfunc_t cbfunc, myxfer_t *x); +static void errhandler(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata); +static void wait_signal_callback(int fd, short event, void *arg); +static void errhandler_reg_callbk (pmix_status_t status, + size_t errhandler_ref, + void *cbdata); + +static void opcbfunc(pmix_status_t status, void *cbdata) +{ + myxfer_t *x = (myxfer_t*)cbdata; + + /* release the caller, if necessary */ + if (NULL != x->cbfunc) { + x->cbfunc(PMIX_SUCCESS, x->cbdata); + } + DEBUG_WAKEUP_THREAD(&x->lock); +} + +static void sacbfunc(pmix_status_t status, + pmix_info_t info[], size_t ninfo, + void *provided_cbdata, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + myxfer_t *x = (myxfer_t*)provided_cbdata; + size_t n; + + if (NULL != info) { + x->ninfo = ninfo; + PMIX_INFO_CREATE(x->info, x->ninfo); + for (n=0; n < ninfo; n++) { + /* copy the data across */ + PMIX_INFO_XFER(&x->info[n], &info[n]); + } + } + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + DEBUG_WAKEUP_THREAD(&x->lock); +} + +int main(int argc, char **argv) +{ + char **client_env=NULL; + char **client_argv=NULL; + char *tmp, **atmp, *executable=NULL; + int rc, nprocs=1, n; + uid_t myuid; + gid_t mygid; + pid_t pid; + myxfer_t *x; + pmix_proc_t proc; + wait_tracker_t *child; + pmix_info_t *info; + size_t ninfo; + mylock_t mylock; + pmix_data_array_t *darray; + pmix_info_t *iarray; + + /* smoke test */ + if (PMIX_SUCCESS != 0) { + fprintf(stderr, "ERROR IN COMPUTING CONSTANTS: PMIX_SUCCESS = %d\n", PMIX_SUCCESS); + exit(1); + } + + fprintf(stderr, "GW[%d]: Testing version %s\n", (int)getpid(), PMIx_Get_version()); + + /* see if we were passed the number of procs to run or + * the executable to use */ + for (n=1; n < argc; n++) { + if (0 == strcmp("-n", argv[n]) && + NULL != argv[n+1]) { + nprocs = strtol(argv[n+1], NULL, 10); + ++n; // step over the argument + } else if (0 == strcmp("-h", argv[n])) { + /* print the options and exit */ + fprintf(stderr, "usage: simptest \n"); + fprintf(stderr, " -n N Number of clients to run\n"); + exit(0); + } + } + executable = strdup("./gwclient"); + + /* setup the server library and tell it to support tool connections */ + ninfo = 3; + PMIX_INFO_CREATE(info, ninfo); + PMIX_INFO_LOAD(&info[0], PMIX_SERVER_TOOL_SUPPORT, NULL, PMIX_BOOL); + PMIX_INFO_LOAD(&info[1], PMIX_USOCK_DISABLE, NULL, PMIX_BOOL); + PMIX_INFO_LOAD(&info[2], PMIX_SERVER_GATEWAY, NULL, PMIX_BOOL); + if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, ninfo))) { + fprintf(stderr, "Init failed with error %d\n", rc); + return rc; + } + PMIX_INFO_FREE(info, ninfo); + + /* register the default errhandler */ + DEBUG_CONSTRUCT_LOCK(&mylock); + ninfo = 1; + PMIX_INFO_CREATE(info, ninfo); + PMIX_INFO_LOAD(&info[0], PMIX_EVENT_HDLR_NAME, "GWTEST-DEFAULT", PMIX_STRING); + PMIx_Register_event_handler(NULL, 0, info, ninfo, + errhandler, errhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + PMIX_INFO_FREE(info, ninfo); + if (PMIX_SUCCESS != mylock.status) { + exit(mylock.status); + } + DEBUG_DESTRUCT_LOCK(&mylock); + + /* setup the pub data, in case it is used */ + PMIX_CONSTRUCT(&pubdata, pmix_list_t); + + /* setup to see sigchld on the forked tests */ + PMIX_CONSTRUCT(&children, pmix_list_t); + event_assign(&handler, pmix_globals.evbase, SIGCHLD, + EV_SIGNAL|EV_PERSIST,wait_signal_callback, &handler); + event_add(&handler, NULL); + + /* we have a single namespace for all clients */ + atmp = NULL; + for (n=0; n < nprocs; n++) { + asprintf(&tmp, "%d", n); + pmix_argv_append_nosize(&atmp, tmp); + free(tmp); + } + tmp = pmix_argv_join(atmp, ','); + pmix_argv_free(atmp); + x = PMIX_NEW(myxfer_t); + set_namespace(nprocs, tmp, "foobar", opcbfunc, x); + + /* set common argv and env */ + client_env = pmix_argv_copy(environ); + pmix_argv_prepend_nosize(&client_argv, executable); + + wakeup = nprocs; + myuid = getuid(); + mygid = getgid(); + + /* if the nspace registration hasn't completed yet, + * wait for it here */ + DEBUG_WAIT_THREAD(&x->lock); + free(tmp); + PMIX_RELEASE(x); + + /* collect the launch blob */ + x = PMIX_NEW(myxfer_t); + ninfo = 1; + PMIX_INFO_CREATE(info, ninfo); + /* the 2nd info is going to carry our network allocation request + * consisting of: + * + * PMIX_ALLOC_NETWORK_ID - caller-provided key for resulting allocation + * PMIX_ALLOC_NETWORK_TYPE - type of network whose resources we want + * PMIX_ALLOC_NETWORK_ENDPTS - number of endpoints from that network + */ + darray = (pmix_data_array_t*)malloc(sizeof(pmix_data_array_t)); + darray->type = PMIX_INFO; + darray->size = 4; + PMIX_INFO_CREATE(darray->array, darray->size); + iarray = (pmix_info_t*)darray->array; + PMIX_INFO_LOAD(&iarray[0], PMIX_ALLOC_NETWORK_ID, "my.net.key", PMIX_STRING); + PMIX_INFO_LOAD(&iarray[1], PMIX_ALLOC_NETWORK_TYPE, "tcp", PMIX_STRING); + PMIX_INFO_LOAD(&iarray[2], PMIX_ALLOC_NETWORK_ENDPTS, &nprocs, PMIX_SIZE); + PMIX_INFO_LOAD(&iarray[3], PMIX_ALLOC_NETWORK_SEC_KEY, NULL, PMIX_BOOL); + /* now load the array */ + PMIX_INFO_LOAD(&info[0], PMIX_ALLOC_NETWORK, darray, PMIX_DATA_ARRAY); + + if (PMIX_SUCCESS != (rc = PMIx_server_setup_application("foobar", info, ninfo, sacbfunc, (void*)x))) { + return rc; + } + DEBUG_WAIT_THREAD(&x->lock); + DEBUG_DESTRUCT_LOCK(&x->lock); + PMIX_INFO_FREE(info, ninfo); + + /* pass any returned data down */ + DEBUG_CONSTRUCT_LOCK(&x->lock); + if (PMIX_SUCCESS != (rc = PMIx_server_setup_local_support("foobar", x->info, x->ninfo, opcbfunc, x))) { + return rc; + } + DEBUG_WAIT_THREAD(&x->lock); + PMIX_RELEASE(x); + + /* fork/exec the test */ + (void)strncpy(proc.nspace, "foobar", PMIX_MAX_NSLEN); + for (n = 0; n < nprocs; n++) { + proc.rank = n; + x = PMIX_NEW(myxfer_t); + if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid, + NULL, opcbfunc, x))) { + fprintf(stderr, "Server register client failed with error %d\n", rc); + PMIx_server_finalize(); + return rc; + } + /* don't fork/exec the client until we know it is registered + * so we avoid a potential race condition in the server */ + DEBUG_WAIT_THREAD(&x->lock); + PMIX_RELEASE(x); + if (PMIX_SUCCESS != (rc = PMIx_server_setup_fork(&proc, &client_env))) {//n + fprintf(stderr, "Server fork setup failed with error %d\n", rc); + PMIx_server_finalize(); + return rc; + } + pid = fork(); + if (pid < 0) { + fprintf(stderr, "Fork failed\n"); + PMIx_server_finalize(); + return -1; + } + child = PMIX_NEW(wait_tracker_t); + child->pid = pid; + pmix_list_append(&children, &child->super); + + if (pid == 0) { + execve(executable, client_argv, client_env); + /* Does not return */ + exit(0); + } + } + free(executable); + pmix_argv_free(client_argv); + pmix_argv_free(client_env); + + /* hang around until the client(s) finalize */ + while (0 < wakeup) { + struct timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = 100000; + nanosleep(&ts, NULL); + } + + /* see if anyone exited with non-zero status */ + n=0; + PMIX_LIST_FOREACH(child, &children, wait_tracker_t) { + if (0 != child->exit_code) { + fprintf(stderr, "Child %d exited with status %d - test FAILED\n", n, child->exit_code); + goto done; + } + ++n; + } + + /* deregister the nspace */ + x = PMIX_NEW(myxfer_t); + PMIx_server_deregister_nspace("foobar", opcbfunc, (void*)x); + DEBUG_WAIT_THREAD(&x->lock); + PMIX_RELEASE(x); + + done: + /* deregister the event handlers */ + PMIx_Deregister_event_handler(0, NULL, NULL); + + /* release any pub data */ + PMIX_LIST_DESTRUCT(&pubdata); + + /* release the child tracker */ + PMIX_LIST_DESTRUCT(&children); + + /* finalize the server library */ + if (PMIX_SUCCESS != (rc = PMIx_server_finalize())) { + fprintf(stderr, "Finalize failed with error %d\n", rc); + exit_code = rc; + } + + if (0 == exit_code) { + fprintf(stderr, "Test finished OK!\n"); + } else { + fprintf(stderr, "TEST FAILED WITH ERROR %d\n", exit_code); + } + + return exit_code; +} + +static void set_namespace(int nprocs, char *ranks, char *nspace, + pmix_op_cbfunc_t cbfunc, myxfer_t *x) +{ + char *regex, *ppn; + char hostname[PMIX_MAXHOSTNAMELEN]; + int n; + pmix_data_array_t *darray; + pmix_info_t *info; + pmix_rank_t rank; + uint16_t lr; + + gethostname(hostname, sizeof(hostname)); + x->ninfo = 7 + nprocs; + + PMIX_INFO_CREATE(x->info, x->ninfo); + (void)strncpy(x->info[0].key, PMIX_UNIV_SIZE, PMIX_MAX_KEYLEN); + x->info[0].value.type = PMIX_UINT32; + x->info[0].value.data.uint32 = nprocs; + + (void)strncpy(x->info[1].key, PMIX_SPAWNED, PMIX_MAX_KEYLEN); + x->info[1].value.type = PMIX_UINT32; + x->info[1].value.data.uint32 = 0; + + (void)strncpy(x->info[2].key, PMIX_LOCAL_SIZE, PMIX_MAX_KEYLEN); + x->info[2].value.type = PMIX_UINT32; + x->info[2].value.data.uint32 = nprocs; + + (void)strncpy(x->info[3].key, PMIX_LOCAL_PEERS, PMIX_MAX_KEYLEN); + x->info[3].value.type = PMIX_STRING; + x->info[3].value.data.string = strdup(ranks); + + PMIx_generate_regex(hostname, ®ex); + (void)strncpy(x->info[4].key, PMIX_NODE_MAP, PMIX_MAX_KEYLEN); + x->info[4].value.type = PMIX_STRING; + x->info[4].value.data.string = regex; + + PMIx_generate_ppn(ranks, &ppn); + (void)strncpy(x->info[5].key, PMIX_PROC_MAP, PMIX_MAX_KEYLEN); + x->info[5].value.type = PMIX_STRING; + x->info[5].value.data.string = ppn; + + (void)strncpy(x->info[6].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN); + x->info[6].value.type = PMIX_UINT32; + x->info[6].value.data.uint32 = nprocs; + + for (n=0; n < nprocs; n++) { + (void)strncpy(x->info[7+n].key, PMIX_PROC_DATA, PMIX_MAX_KEYLEN); + x->info[7+n].value.type = PMIX_DATA_ARRAY; + darray = (pmix_data_array_t*)malloc(sizeof(pmix_data_array_t)); + darray->size = 2; + darray->type = PMIX_INFO; + PMIX_INFO_CREATE(darray->array, 2); + info = (pmix_info_t*)darray->array; + rank = n; + PMIX_INFO_LOAD(&info[0], PMIX_RANK, &rank, PMIX_PROC_RANK); + lr = n; + PMIX_INFO_LOAD(&info[1], PMIX_LOCAL_RANK, &lr, PMIX_UINT16); + x->info[7+n].value.data.darray = darray; + } + + PMIx_server_register_nspace(nspace, nprocs, x->info, x->ninfo, + cbfunc, x); +} + +static void errhandler(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) +{ + pmix_output(0, "SERVER: ERRHANDLER CALLED WITH STATUS %d", status); +} + +static void errhandler_reg_callbk (pmix_status_t status, + size_t errhandler_ref, + void *cbdata) +{ + mylock_t *lock = (mylock_t*)cbdata; + + pmix_output(0, "SERVER: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu", + status, (unsigned long)errhandler_ref); + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); +} + +static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + return PMIX_SUCCESS; +} +static pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_output(0, "SERVER: FINALIZED %s:%d WAKEUP %d", + proc->nspace, proc->rank, wakeup); + /* ensure we call the cbfunc so the proc can exit! */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + return PMIX_SUCCESS; +} + +static void abcbfunc(pmix_status_t status, void *cbdata) +{ + myxfer_t *x = (myxfer_t*)cbdata; + + /* be sure to release the caller */ + if (NULL != x->cbfunc) { + x->cbfunc(status, x->cbdata); + } + PMIX_RELEASE(x); +} + +static pmix_status_t abort_fn(const pmix_proc_t *proc, + void *server_object, + int status, const char msg[], + pmix_proc_t procs[], size_t nprocs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_status_t rc; + myxfer_t *x; + + if (NULL != procs) { + pmix_output(0, "SERVER: ABORT on %s:%d", procs[0].nspace, procs[0].rank); + } else { + pmix_output(0, "SERVER: ABORT OF ALL PROCS IN NSPACE %s", proc->nspace); + } + + /* instead of aborting the specified procs, notify them + * (if they have registered their errhandler) */ + + /* use the myxfer_t object to ensure we release + * the caller when notification has been queued */ + x = PMIX_NEW(myxfer_t); + (void)strncpy(x->caller.nspace, proc->nspace, PMIX_MAX_NSLEN); + x->caller.rank = proc->rank; + + PMIX_INFO_CREATE(x->info, 2); + (void)strncpy(x->info[0].key, "DARTH", PMIX_MAX_KEYLEN); + x->info[0].value.type = PMIX_INT8; + x->info[0].value.data.int8 = 12; + (void)strncpy(x->info[1].key, "VADER", PMIX_MAX_KEYLEN); + x->info[1].value.type = PMIX_DOUBLE; + x->info[1].value.data.dval = 12.34; + x->cbfunc = cbfunc; + x->cbdata = cbdata; + + if (PMIX_SUCCESS != (rc = PMIx_Notify_event(status, &x->caller, + PMIX_RANGE_NAMESPACE, + x->info, 2, + abcbfunc, x))) { + pmix_output(0, "SERVER: FAILED NOTIFY ERROR %d", (int)rc); + } + + return PMIX_SUCCESS; +} + + +static pmix_status_t fencenb_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + char *data, size_t ndata, + pmix_modex_cbfunc_t cbfunc, void *cbdata) +{ + pmix_output(0, "SERVER: FENCENB"); + /* pass the provided data back to each participating proc */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, data, ndata, cbdata, NULL, NULL); + } + return PMIX_SUCCESS; +} + + +static pmix_status_t dmodex_fn(const pmix_proc_t *proc, + const pmix_info_t info[], size_t ninfo, + pmix_modex_cbfunc_t cbfunc, void *cbdata) +{ + pmix_output(0, "SERVER: DMODEX"); + + /* if this is a timeout test, then do nothing */ + if (istimeouttest) { + return PMIX_SUCCESS; + } + + /* we don't have any data for remote procs as this + * test only runs one server - so report accordingly */ + if (NULL != cbfunc) { + cbfunc(PMIX_ERR_NOT_FOUND, NULL, 0, cbdata, NULL, NULL); + } + return PMIX_SUCCESS; +} + + +static pmix_status_t publish_fn(const pmix_proc_t *proc, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_locdat_t *p; + size_t n; + + pmix_output(0, "SERVER: PUBLISH"); + + for (n=0; n < ninfo; n++) { + p = PMIX_NEW(pmix_locdat_t); + (void)strncpy(p->pdata.proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + p->pdata.proc.rank = proc->rank; + (void)strncpy(p->pdata.key, info[n].key, PMIX_MAX_KEYLEN); + pmix_value_xfer(&p->pdata.value, (pmix_value_t*)&info[n].value); + pmix_list_append(&pubdata, &p->super); + } + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + return PMIX_SUCCESS; +} + + +static pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_lookup_cbfunc_t cbfunc, void *cbdata) +{ + pmix_locdat_t *p, *p2; + pmix_list_t results; + size_t i, n; + pmix_pdata_t *pd = NULL; + pmix_status_t ret = PMIX_ERR_NOT_FOUND; + + pmix_output(0, "SERVER: LOOKUP"); + + PMIX_CONSTRUCT(&results, pmix_list_t); + + for (n=0; NULL != keys[n]; n++) { + PMIX_LIST_FOREACH(p, &pubdata, pmix_locdat_t) { + if (0 == strncmp(keys[n], p->pdata.key, PMIX_MAX_KEYLEN)) { + p2 = PMIX_NEW(pmix_locdat_t); + (void)strncpy(p2->pdata.proc.nspace, p->pdata.proc.nspace, PMIX_MAX_NSLEN); + p2->pdata.proc.rank = p->pdata.proc.rank; + (void)strncpy(p2->pdata.key, p->pdata.key, PMIX_MAX_KEYLEN); + pmix_value_xfer(&p2->pdata.value, &p->pdata.value); + pmix_list_append(&results, &p2->super); + break; + } + } + } + if (0 < (n = pmix_list_get_size(&results))) { + ret = PMIX_SUCCESS; + PMIX_PDATA_CREATE(pd, n); + for (i=0; i < n; i++) { + p = (pmix_locdat_t*)pmix_list_remove_first(&results); + if (p) { + (void)strncpy(pd[i].proc.nspace, p->pdata.proc.nspace, PMIX_MAX_NSLEN); + pd[i].proc.rank = p->pdata.proc.rank; + (void)strncpy(pd[i].key, p->pdata.key, PMIX_MAX_KEYLEN); + pmix_value_xfer(&pd[i].value, &p->pdata.value); + } + } + } + PMIX_LIST_DESTRUCT(&results); + if (NULL != cbfunc) { + cbfunc(ret, pd, n, cbdata); + } + if (0 < n) { + PMIX_PDATA_FREE(pd, n); + } + return PMIX_SUCCESS; +} + + +static pmix_status_t unpublish_fn(const pmix_proc_t *proc, char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_locdat_t *p, *p2; + size_t n; + + pmix_output(0, "SERVER: UNPUBLISH"); + + for (n=0; NULL != keys[n]; n++) { + PMIX_LIST_FOREACH_SAFE(p, p2, &pubdata, pmix_locdat_t) { + if (0 == strncmp(keys[n], p->pdata.key, PMIX_MAX_KEYLEN)) { + pmix_list_remove_item(&pubdata, &p->super); + PMIX_RELEASE(p); + break; + } + } + } + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + return PMIX_SUCCESS; +} + +static void spcbfunc(pmix_status_t status, void *cbdata) +{ + myxfer_t *x = (myxfer_t*)cbdata; + + if (NULL != x->spcbfunc) { + x->spcbfunc(PMIX_SUCCESS, "DYNSPACE", x->cbdata); + } +} + +static pmix_status_t spawn_fn(const pmix_proc_t *proc, + const pmix_info_t job_info[], size_t ninfo, + const pmix_app_t apps[], size_t napps, + pmix_spawn_cbfunc_t cbfunc, void *cbdata) +{ + myxfer_t *x; + size_t n; + pmix_proc_t *pptr; + bool spawned; + + pmix_output(0, "SERVER: SPAWN"); + + /* check the job info for parent and spawned keys */ + for (n=0; n < ninfo; n++) { + if (0 == strncmp(job_info[n].key, PMIX_PARENT_ID, PMIX_MAX_KEYLEN)) { + pptr = job_info[n].value.data.proc; + pmix_output(0, "SPAWN: Parent ID %s:%d", pptr->nspace, pptr->rank); + } else if (0 == strncmp(job_info[n].key, PMIX_SPAWNED, PMIX_MAX_KEYLEN)) { + spawned = PMIX_INFO_TRUE(&job_info[n]); + pmix_output(0, "SPAWN: Spawned %s", spawned ? "TRUE" : "FALSE"); + } + } + + /* in practice, we would pass this request to the local + * resource manager for launch, and then have that server + * execute our callback function. For now, we will fake + * the spawn and just pretend */ + + /* must register the nspace for the new procs before + * we return to the caller */ + x = PMIX_NEW(myxfer_t); + x->spcbfunc = cbfunc; + x->cbdata = cbdata; + + set_namespace(2, "0,1", "DYNSPACE", spcbfunc, x); + + return PMIX_SUCCESS; +} + +static int numconnects = 0; + +static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_output(0, "SERVER: CONNECT"); + + /* in practice, we would pass this request to the local + * resource manager for handling */ + + numconnects++; + + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + + return PMIX_SUCCESS; +} + + +static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_output(0, "SERVER: DISCONNECT"); + + /* in practice, we would pass this request to the local + * resource manager for handling */ + + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + + return PMIX_SUCCESS; +} + +static pmix_status_t register_event_fn(pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + return PMIX_SUCCESS; +} + +static pmix_status_t deregister_events(pmix_status_t *codes, size_t ncodes, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + return PMIX_SUCCESS; +} + +static pmix_status_t notify_event(pmix_status_t code, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + return PMIX_SUCCESS; +} + +typedef struct query_data_t { + pmix_info_t *data; + size_t ndata; +} query_data_t; + +static pmix_status_t query_fn(pmix_proc_t *proct, + pmix_query_t *queries, size_t nqueries, + pmix_info_cbfunc_t cbfunc, + void *cbdata) +{ + size_t n; + pmix_info_t *info; + + pmix_output(0, "SERVER: QUERY"); + + if (NULL == cbfunc) { + return PMIX_ERROR; + } + /* keep this simple */ + PMIX_INFO_CREATE(info, nqueries); + for (n=0; n < nqueries; n++) { + pmix_output(0, "\tKey: %s", queries[n].keys[0]); + (void)strncpy(info[n].key, queries[n].keys[0], PMIX_MAX_KEYLEN); + info[n].value.type = PMIX_STRING; + if (0 > asprintf(&info[n].value.data.string, "%d", (int)n)) { + return PMIX_ERROR; + } + } + cbfunc(PMIX_SUCCESS, info, nqueries, cbdata, NULL, NULL); + return PMIX_SUCCESS; +} + +static void tool_connect_fn(pmix_info_t *info, size_t ninfo, + pmix_tool_connection_cbfunc_t cbfunc, + void *cbdata) +{ + pmix_proc_t proc; + + pmix_output(0, "SERVER: TOOL CONNECT"); + + /* just pass back an arbitrary nspace */ + (void)strncpy(proc.nspace, "TOOL", PMIX_MAX_NSLEN); + proc.rank = 0; + + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, &proc, cbdata); + } +} + +static void log_fn(const pmix_proc_t *client, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_output(0, "SERVER: LOG"); + + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } +} + +static void wait_signal_callback(int fd, short event, void *arg) +{ + pmix_event_t *sig = (pmix_event_t*) arg; + int status; + pid_t pid; + wait_tracker_t *t2; + + if (SIGCHLD != event_get_signal(sig)) { + return; + } + + /* we can have multiple children leave but only get one + * sigchild callback, so reap all the waitpids until we + * don't get anything valid back */ + while (1) { + pid = waitpid(-1, &status, WNOHANG); + if (-1 == pid && EINTR == errno) { + /* try it again */ + continue; + } + /* if we got garbage, then nothing we can do */ + if (pid <= 0) { + return; + } + + /* we are already in an event, so it is safe to access the list */ + PMIX_LIST_FOREACH(t2, &children, wait_tracker_t) { + if (pid == t2->pid) { + t2->exit_code = status; + /* found it! */ + if (0 != status && 0 == exit_code) { + exit_code = status; + } + --wakeup; + break; + } + } + } +} diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/quietclient.c b/opal/mca/pmix/pmix3x/pmix/test/simple/quietclient.c new file mode 100644 index 00000000000..428ba4e341b --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/quietclient.c @@ -0,0 +1,288 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include +#include + +#include +#include +#include +#include + +#include "src/class/pmix_object.h" +#include "src/util/output.h" +#include "src/util/printf.h" + +#define MAXCNT 1 + +static volatile bool completed = false; +static pmix_proc_t myproc; + +static void notification_fn(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) +{ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } + completed = true; +} + +static void errhandler_reg_callbk(pmix_status_t status, + size_t errhandler_ref, + void *cbdata) +{ + volatile bool *active = (volatile bool*)cbdata; + + *active = false; +} + +/* this is an event notification function that we explicitly request + * be called when the PMIX_MODEL_DECLARED notification is issued. + * We could catch it in the general event notification function and test + * the status to see if the status matched, but it often is simpler + * to declare a use-specific notification callback point. In this case, + * we are asking to know whenever a model is declared as a means + * of testing server self-notification */ +static void model_callback(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) +{ + /* we must NOT tell the event handler state machine that we + * are the last step as that will prevent it from notifying + * anyone else that might be listening for declarations */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } +} + +/* event handler registration is done asynchronously */ +static void model_registration_callback(pmix_status_t status, + size_t evhandler_ref, + void *cbdata) +{ + volatile int *active = (volatile int*)cbdata; + + *active = false; +} + +int main(int argc, char **argv) +{ + int rc; + pmix_value_t value; + pmix_value_t *val = &value; + char *tmp; + pmix_proc_t proc; + uint32_t nprocs, n; + int cnt, j; + volatile bool active; + pmix_info_t *iptr; + size_t ninfo; + pmix_status_t code; + + /* init us and declare we are a test programming model */ + PMIX_INFO_CREATE(iptr, 2); + PMIX_INFO_LOAD(&iptr[0], PMIX_PROGRAMMING_MODEL, "TEST", PMIX_STRING); + PMIX_INFO_LOAD(&iptr[1], PMIX_MODEL_LIBRARY_NAME, "PMIX", PMIX_STRING); + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, iptr, 2))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + exit(rc); + } + PMIX_INFO_FREE(iptr, 2); + + /* test something */ + (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_JOB_SIZE, NULL, 0, &val))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Get failed: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + exit(rc); + } + PMIX_VALUE_RELEASE(val); + + /* register a handler specifically for when models declare */ + active = true; + ninfo = 1; + PMIX_INFO_CREATE(iptr, ninfo); + PMIX_INFO_LOAD(&iptr[0], PMIX_EVENT_HDLR_NAME, "SIMPCLIENT-MODEL", PMIX_STRING); + code = PMIX_MODEL_DECLARED; + PMIx_Register_event_handler(&code, 1, iptr, ninfo, + model_callback, model_registration_callback, (void*)&active); + while (active) { + usleep(10); + } + PMIX_INFO_FREE(iptr, ninfo); + + /* register our errhandler */ + active = true; + PMIx_Register_event_handler(NULL, 0, NULL, 0, + notification_fn, errhandler_reg_callbk, (void*)&active); + while (active) { + usleep(10); + } + + /* get our universe size */ + (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &val))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Get universe size failed: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + goto done; + } + nprocs = val->data.uint32; + PMIX_VALUE_RELEASE(val); + + /* put a few values */ + (void)asprintf(&tmp, "%s-%d-internal", myproc.nspace, myproc.rank); + value.type = PMIX_UINT32; + value.data.uint32 = 1234; + if (PMIX_SUCCESS != (rc = PMIx_Store_internal(&myproc, tmp, &value))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Store_internal failed: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + goto done; + } + + for (cnt=0; cnt < MAXCNT; cnt++) { + (void)asprintf(&tmp, "%s-%d-local-%d", myproc.nspace, myproc.rank, cnt); + value.type = PMIX_UINT64; + value.data.uint64 = 1234; + if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_LOCAL, tmp, &value))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Put internal failed: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + goto done; + } + + (void)asprintf(&tmp, "%s-%d-remote-%d", myproc.nspace, myproc.rank, cnt); + value.type = PMIX_STRING; + value.data.string = "1234"; + if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_REMOTE, tmp, &value))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Put internal failed: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + goto done; + } + + if (PMIX_SUCCESS != (rc = PMIx_Commit())) { + pmix_output(0, "Client ns %s rank %d cnt %d: PMIx_Commit failed: %s", + myproc.nspace, myproc.rank, cnt, PMIx_Error_string(rc)); + goto done; + } + + /* call fence to ensure the data is received */ + PMIX_PROC_CONSTRUCT(&proc); + (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, NULL, 0))) { + pmix_output(0, "Client ns %s rank %d cnt %d: PMIx_Fence failed: %s", + myproc.nspace, myproc.rank, cnt, PMIx_Error_string(rc)); + goto done; + } + + /* check the returned data */ + (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); + for (j=0; j <= cnt; j++) { + for (n=0; n < nprocs; n++) { + proc.rank = n; + (void)asprintf(&tmp, "%s-%d-local-%d", myproc.nspace, n, j); + if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, tmp, NULL, 0, &val))) { + pmix_output(0, "Client ns %s rank %d cnt %d: PMIx_Get %s failed: %s", + myproc.nspace, myproc.rank, j, tmp, PMIx_Error_string(rc)); + continue; + } + if (NULL == val) { + pmix_output(0, "Client ns %s rank %d: NULL value returned", + myproc.nspace, myproc.rank); + break; + } + if (PMIX_UINT64 != val->type) { + pmix_output(0, "Client ns %s rank %d cnt %d: PMIx_Get %s returned wrong type: %d", myproc.nspace, myproc.rank, j, tmp, val->type); + PMIX_VALUE_RELEASE(val); + free(tmp); + continue; + } + if (1234 != val->data.uint64) { + pmix_output(0, "Client ns %s rank %d cnt %d: PMIx_Get %s returned wrong value: %d", myproc.nspace, myproc.rank, j, tmp, (int)val->data.uint64); + PMIX_VALUE_RELEASE(val); + free(tmp); + continue; + } + PMIX_VALUE_RELEASE(val); + free(tmp); + + if (n != myproc.rank) { + (void)asprintf(&tmp, "%s-%d-remote-%d", proc.nspace, n, j); + if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, tmp, NULL, 0, &val))) { + /* this data should _not_ be found as we are on the same node + * and the data was "put" with a PMIX_REMOTE scope */ + continue; + } + pmix_output(0, "Client ns %s rank %d cnt %d: PMIx_Get %s returned remote data for a local proc", + myproc.nspace, myproc.rank, j, tmp); + PMIX_VALUE_RELEASE(val); + free(tmp); + } + } + } + } + + /* now get the data blob for myself */ + if (PMIX_SUCCESS == (rc = PMIx_Get(&myproc, NULL, NULL, 0, &val))) { + if (PMIX_DATA_ARRAY != val->type) { + pmix_output(0, "Client ns %s rank %d did not return an array for its internal modex blob", + myproc.nspace, myproc.rank); + PMIX_VALUE_RELEASE(val); + } else if (PMIX_INFO != val->data.darray->type) { + pmix_output(0, "Client ns %s rank %d returned an internal modex array of type %s instead of PMIX_INFO", + myproc.nspace, myproc.rank, PMIx_Data_type_string(val->data.darray->type)); + PMIX_VALUE_RELEASE(val); + } else if (0 == val->data.darray->size) { + pmix_output(0, "Client ns %s rank %d returned an internal modex array of zero length", + myproc.nspace, myproc.rank); + PMIX_VALUE_RELEASE(val); + } else { + PMIX_VALUE_RELEASE(val); + } + } else { + pmix_output(0, "Client ns %s rank %d internal modex blob FAILED with error %s(%d)", + myproc.nspace, myproc.rank, PMIx_Error_string(rc), rc); + } + + done: + /* finalize us */ + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { + fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + } + fflush(stderr); + return(rc); +} diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/simpclient.c b/opal/mca/pmix/pmix3x/pmix/test/simple/simpclient.c index 4bc66e5d9ea..ca277c5e28f 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/simple/simpclient.c +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/simpclient.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -320,13 +320,16 @@ int main(int argc, char **argv) /* log something */ PMIX_INFO_CONSTRUCT(&info); - (void)strncpy(info.key, "foobar", PMIX_MAX_KEYLEN); - info.value.type = PMIX_BOOL; - info.value.data.flag = true; + PMIX_INFO_LOAD(&info, PMIX_LOG_STDERR, "test log msg", PMIX_STRING); active = true; - PMIx_Log_nb(&info, 1, NULL, 0, opcbfunc, (void*)&active); - while (active) { - usleep(10); + rc = PMIx_Log_nb(&info, 1, NULL, 0, opcbfunc, (void*)&active); + if (PMIX_SUCCESS != rc) { + pmix_output(0, "Client ns %s rank %d - log_nb returned %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + } else { + while (active) { + usleep(10); + } } PMIX_INFO_DESTRUCT(&info); diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/simpdyn.c b/opal/mca/pmix/pmix3x/pmix/test/simple/simpdyn.c index 6d3ba10fb94..a20b8418a4f 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/simple/simpdyn.c +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/simpdyn.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -46,13 +46,12 @@ int main(int argc, char **argv) pmix_value_t *val = &value; pmix_proc_t proc; uint32_t nprocs; - char nsp2[PMIX_MAX_NSLEN+1], nsp3[PMIX_MAX_NSLEN+1]; + char nsp2[PMIX_MAX_NSLEN+1]; pmix_app_t *app; char hostname[PMIX_MAXHOSTNAMELEN]; pmix_proc_t *peers; size_t npeers, ntmp=0; char *nodelist; - pmix_rank_t newrank; gethostname(hostname, sizeof(hostname)); @@ -129,13 +128,13 @@ int main(int argc, char **argv) } /* just cycle the connect/disconnect functions */ - if (PMIX_SUCCESS != (rc = PMIx_Connect(&proc, 1, NULL, 0, nsp3, &newrank))) { + if (PMIX_SUCCESS != (rc = PMIx_Connect(&proc, 1, NULL, 0))) { pmix_output(0, "Client ns %s rank %d: PMIx_Connect failed: %d", myproc.nspace, myproc.rank, rc); goto done; } - pmix_output(0, "Client ns %s rank %d: PMIx_Connect succeeded - %s:%d", - myproc.nspace, myproc.rank, nsp3, newrank); - if (PMIX_SUCCESS != (rc = PMIx_Disconnect(nsp3, NULL, 0))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Connect succeeded", + myproc.nspace, myproc.rank); + if (PMIX_SUCCESS != (rc = PMIx_Disconnect(&proc, 1, NULL, 0))) { pmix_output(0, "Client ns %s rank %d: PMIx_Disonnect failed: %d", myproc.nspace, myproc.rank, rc); goto done; } diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/simptest.c b/opal/mca/pmix/pmix3x/pmix/test/simple/simptest.c index 8888aaee7a8..710ecbc135e 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/simple/simptest.c +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/simptest.c @@ -40,12 +40,18 @@ #include #include PMIX_EVENT_HEADER +#if PMIX_HAVE_HWLOC +#include +#endif + #include "src/class/pmix_list.h" #include "src/util/pmix_environ.h" #include "src/util/output.h" #include "src/util/printf.h" #include "src/util/argv.h" +#include "simptest.h" + static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, pmix_op_cbfunc_t cbfunc, void *cbdata); static pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, @@ -76,8 +82,8 @@ static pmix_status_t spawn_fn(const pmix_proc_t *proc, pmix_spawn_cbfunc_t cbfunc, void *cbdata); static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, - pmix_connect_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t disconnect_fn(const char nspace[], + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); static pmix_status_t register_event_fn(pmix_status_t *codes, size_t ncodes, @@ -132,25 +138,28 @@ PMIX_CLASS_INSTANCE(pmix_locdat_t, typedef struct { pmix_object_t super; - volatile bool active; + mylock_t lock; + pmix_event_t ev; pmix_proc_t caller; pmix_info_t *info; size_t ninfo; pmix_op_cbfunc_t cbfunc; pmix_spawn_cbfunc_t spcbfunc; + pmix_release_cbfunc_t relcbfunc; void *cbdata; } myxfer_t; static void xfcon(myxfer_t *p) { + DEBUG_CONSTRUCT_LOCK(&p->lock); p->info = NULL; p->ninfo = 0; - p->active = true; p->cbfunc = NULL; p->spcbfunc = NULL; p->cbdata = NULL; } static void xfdes(myxfer_t *p) { + DEBUG_DESTRUCT_LOCK(&p->lock); if (NULL != p->info) { PMIX_INFO_FREE(p->info, p->ninfo); } @@ -174,6 +183,7 @@ static pmix_list_t pubdata; static pmix_event_t handler; static pmix_list_t children; static bool istimeouttest = false; +static mylock_t globallock; static void set_namespace(int nprocs, char *ranks, char *nspace, pmix_op_cbfunc_t cbfunc, myxfer_t *x); @@ -197,7 +207,49 @@ static void opcbfunc(pmix_status_t status, void *cbdata) if (NULL != x->cbfunc) { x->cbfunc(PMIX_SUCCESS, x->cbdata); } - x->active = false; + DEBUG_WAKEUP_THREAD(&x->lock); +} + + +static void dlcbfunc(int sd, short flags, void *cbdata) +{ + myxfer_t *x = (myxfer_t*)cbdata; + + pmix_output(0, "INVENTORY READY FOR DELIVERY"); + + PMIx_server_deliver_inventory(x->info, x->ninfo, NULL, 0, opcbfunc, (void*)x); +} + +static void infocbfunc(pmix_status_t status, + pmix_info_t *info, size_t ninfo, + void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + mylock_t *lock = (mylock_t*)cbdata; + myxfer_t *x; + size_t n; + + pmix_output(0, "INVENTORY RECEIVED"); + + /* we don't have any place to send this, so for test + * purposes only, let's push it back down for processing. + * Note: it must be thread-shifted first as we are in + * the callback event thread of the underlying PMIx + * server */ + x = PMIX_NEW(myxfer_t); + x->ninfo = ninfo; + PMIX_INFO_CREATE(x->info, x->ninfo); + for (n=0; n < ninfo; n++) { + PMIX_INFO_XFER(&x->info[n], &info[n]); + } + PMIX_THREADSHIFT(x, dlcbfunc); + + if (NULL != release_fn) { + release_fn(release_cbdata); + } + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); } /* this is an event notification function that we explicitly request @@ -232,7 +284,7 @@ static void model_callback(size_t evhdlr_registration_id, if (NULL != cbfunc) { cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); } - wakeup = 0; + DEBUG_WAKEUP_THREAD(&globallock); } /* event handler registration is done asynchronously */ @@ -240,13 +292,14 @@ static void model_registration_callback(pmix_status_t status, size_t evhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; if (PMIX_SUCCESS != status) { fprintf(stderr, "simptest EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n", status, (unsigned long)evhandler_ref); } - *active = status; + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); } int main(int argc, char **argv) @@ -265,7 +318,11 @@ int main(int argc, char **argv) size_t ninfo; bool cross_version = false; bool usock = true; - volatile int active; + bool hwloc = false; +#if PMIX_HAVE_HWLOC + char *hwloc_file = NULL; +#endif + mylock_t mylock; pmix_status_t code; /* smoke test */ @@ -302,6 +359,21 @@ int main(int argc, char **argv) } else if (0 == strcmp("-u", argv[n])) { /* enable usock */ usock = false; +#if PMIX_HAVE_HWLOC + } else if (0 == strcmp("-hwloc", argv[n]) || + 0 == strcmp("--hwloc", argv[n])) { + /* test hwloc support */ + hwloc = true; + } else if (0 == strcmp("-hwloc-file", argv[n]) || + 0 == strcmp("--hwloc-file", argv[n])) { + if (NULL == argv[n+1]) { + fprintf(stderr, "The --hwloc-file option requires an argument\n"); + exit(1); + } + hwloc_file = strdup(argv[n+1]); + hwloc = true; + ++n; +#endif } else if (0 == strcmp("-h", argv[n])) { /* print the options and exit */ fprintf(stderr, "usage: simptest \n"); @@ -309,6 +381,8 @@ int main(int argc, char **argv) fprintf(stderr, " -e foo Name of the client executable to run (default: simpclient\n"); fprintf(stderr, " -x Test cross-version support\n"); fprintf(stderr, " -u Enable legacy usock support\n"); + fprintf(stderr, " -hwloc Test hwloc support\n"); + fprintf(stderr, " -hwloc-file FILE Use file to import topology\n"); exit(0); } } @@ -320,47 +394,78 @@ int main(int argc, char **argv) exit(1); } +#if !PMIX_HAVE_HWLOC + if (hwloc) { + fprintf(stderr, "PMIx was not configured with HWLOC support - cannot continue\n"); + exit(1); + } +#endif + /* setup the server library and tell it to support tool connections */ - ninfo = 2; +#if PMIX_HAVE_HWLOC + if (hwloc) { +#if HWLOC_API_VERSION < 0x20000 + ninfo = 4; +#else + ninfo = 5; +#endif + } else { + ninfo = 3; + } +#else + ninfo = 3; +#endif + PMIX_INFO_CREATE(info, ninfo); PMIX_INFO_LOAD(&info[0], PMIX_SERVER_TOOL_SUPPORT, NULL, PMIX_BOOL); PMIX_INFO_LOAD(&info[1], PMIX_USOCK_DISABLE, &usock, PMIX_BOOL); - if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, 2))) { + PMIX_INFO_LOAD(&info[2], PMIX_SERVER_GATEWAY, NULL, PMIX_BOOL); +#if PMIX_HAVE_HWLOC + if (hwloc) { + if (NULL != hwloc_file) { + PMIX_INFO_LOAD(&info[3], PMIX_TOPOLOGY_FILE, hwloc_file, PMIX_STRING); + } else { + PMIX_INFO_LOAD(&info[3], PMIX_TOPOLOGY, NULL, PMIX_STRING); + } +#if HWLOC_API_VERSION >= 0x20000 + PMIX_INFO_LOAD(&info[4], PMIX_HWLOC_SHARE_TOPO, NULL, PMIX_BOOL); +#endif + } +#endif + if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, ninfo))) { fprintf(stderr, "Init failed with error %d\n", rc); return rc; } PMIX_INFO_FREE(info, ninfo); /* register the default errhandler */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); ninfo = 1; PMIX_INFO_CREATE(info, ninfo); PMIX_INFO_LOAD(&info[0], PMIX_EVENT_HDLR_NAME, "SIMPTEST-DEFAULT", PMIX_STRING); PMIx_Register_event_handler(NULL, 0, info, ninfo, - errhandler, errhandler_reg_callbk, (void*)&active); - while (-1 == active) { - usleep(10); - } + errhandler, errhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); PMIX_INFO_FREE(info, ninfo); - if (0 != active) { - exit(active); + if (PMIX_SUCCESS != mylock.status) { + exit(mylock.status); } + DEBUG_DESTRUCT_LOCK(&mylock); /* register a handler specifically for when models declare */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); ninfo = 1; PMIX_INFO_CREATE(info, ninfo); PMIX_INFO_LOAD(&info[0], PMIX_EVENT_HDLR_NAME, "SIMPTEST-MODEL", PMIX_STRING); code = PMIX_MODEL_DECLARED; PMIx_Register_event_handler(&code, 1, info, ninfo, - model_callback, model_registration_callback, (void*)&active); - while (-1 == active) { - usleep(10); - } + model_callback, model_registration_callback, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); PMIX_INFO_FREE(info, ninfo); - if (0 != active) { - exit(active); + if (PMIX_SUCCESS != mylock.status) { + exit(mylock.status); } + DEBUG_DESTRUCT_LOCK(&mylock); /* setup the pub data, in case it is used */ PMIX_CONSTRUCT(&pubdata, pmix_list_t); @@ -391,9 +496,24 @@ int main(int argc, char **argv) myuid = getuid(); mygid = getgid(); + /* collect our inventory */ + DEBUG_CONSTRUCT_LOCK(&mylock); + fprintf(stderr, "Collecting inventory\n"); + if (PMIX_SUCCESS != (rc = PMIx_server_collect_inventory(NULL, 0, infocbfunc, (void*)&mylock))) { + fprintf(stderr, "Collect inventory failed: %d\n", rc); + DEBUG_DESTRUCT_LOCK(&mylock); + goto done; + } + DEBUG_WAIT_THREAD(&mylock); + fprintf(stderr, "Inventory collected: %d\n", mylock.status); + if (PMIX_SUCCESS != mylock.status) { + exit(mylock.status); + } + DEBUG_DESTRUCT_LOCK(&mylock); + /* if the nspace registration hasn't completed yet, * wait for it here */ - PMIX_WAIT_FOR_COMPLETION(x->active); + DEBUG_WAIT_THREAD(&x->lock); free(tmp); PMIX_RELEASE(x); @@ -427,7 +547,7 @@ int main(int argc, char **argv) } /* don't fork/exec the client until we know it is registered * so we avoid a potential race condition in the server */ - PMIX_WAIT_FOR_COMPLETION(x->active); + DEBUG_WAIT_THREAD(&x->lock); PMIX_RELEASE(x); pid = fork(); if (pid < 0) { @@ -473,13 +593,12 @@ int main(int argc, char **argv) PMIX_INFO_LOAD(&info[1], PMIX_MODEL_LIBRARY_NAME, "test", PMIX_STRING); /* mark that it is not to go to any default handlers */ PMIX_INFO_LOAD(&info[2], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL); - wakeup = -1; + DEBUG_CONSTRUCT_LOCK(&globallock); PMIx_Notify_event(PMIX_MODEL_DECLARED, &pmix_globals.myid, PMIX_RANGE_PROC_LOCAL, info, ninfo, NULL, NULL); - while (-1 == wakeup) { - usleep(10); - } + DEBUG_WAIT_THREAD(&globallock); + DEBUG_DESTRUCT_LOCK(&globallock); PMIX_INFO_FREE(info, ninfo); done: @@ -566,11 +685,12 @@ static void errhandler_reg_callbk (pmix_status_t status, size_t errhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; pmix_output(0, "SERVER: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu", status, (unsigned long)errhandler_ref); - *active = status; + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); } static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, @@ -834,26 +954,24 @@ static int numconnects = 0; static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, - pmix_connect_cbfunc_t cbfunc, void *cbdata) + pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_output(0, "SERVER: CONNECT"); - char nspace[PMIX_MAX_NSLEN+1]; /* in practice, we would pass this request to the local * resource manager for handling */ - (void)snprintf(nspace, PMIX_MAX_NSLEN, "NEW%d", numconnects); numconnects++; if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, nspace, 0, cbdata); + cbfunc(PMIX_SUCCESS, cbdata); } return PMIX_SUCCESS; } -static pmix_status_t disconnect_fn(const char nspace[], +static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/simptimeout.c b/opal/mca/pmix/pmix3x/pmix/test/simple/simptimeout.c index 7d390633568..f5454029887 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/simple/simptimeout.c +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/simptimeout.c @@ -72,7 +72,7 @@ int main(int argc, char **argv) int rc; pmix_value_t value; pmix_value_t *val = &value; - pmix_proc_t proc, newproc; + pmix_proc_t proc; uint32_t nprocs, n; volatile bool active; pmix_info_t info; @@ -133,7 +133,7 @@ int main(int argc, char **argv) /* check timeout on connect */ pmix_output(0, "TEST CONNECT TIMEOUT"); - if (PMIX_ERR_TIMEOUT != (rc = PMIx_Connect(&proc, 1, &info, 1, newproc.nspace, &newproc.rank))) { + if (PMIX_ERR_TIMEOUT != (rc = PMIx_Connect(&proc, 1, &info, 1))) { pmix_output(0, "Client ns %s rank %d: PMIx_Connect did not timeout: %s", myproc.nspace, myproc.rank, PMIx_Error_string(rc)); goto done; diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/stability.c b/opal/mca/pmix/pmix3x/pmix/test/simple/stability.c new file mode 100644 index 00000000000..d67b6f84109 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/stability.c @@ -0,0 +1,886 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include PMIX_EVENT_HEADER + +#if PMIX_HAVE_HWLOC +#include +#endif + +#include "src/class/pmix_list.h" +#include "src/util/pmix_environ.h" +#include "src/util/output.h" +#include "src/util/printf.h" +#include "src/util/argv.h" + +#include "simptest.h" + +static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t abort_fn(const pmix_proc_t *proc, void *server_object, + int status, const char msg[], + pmix_proc_t procs[], size_t nprocs, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t fencenb_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + char *data, size_t ndata, + pmix_modex_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t dmodex_fn(const pmix_proc_t *proc, + const pmix_info_t info[], size_t ninfo, + pmix_modex_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t publish_fn(const pmix_proc_t *proc, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_lookup_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t unpublish_fn(const pmix_proc_t *proc, char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t spawn_fn(const pmix_proc_t *proc, + const pmix_info_t job_info[], size_t ninfo, + const pmix_app_t apps[], size_t napps, + pmix_spawn_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t register_event_fn(pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t deregister_events(pmix_status_t *codes, size_t ncodes, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t notify_event(pmix_status_t code, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t query_fn(pmix_proc_t *proct, + pmix_query_t *queries, size_t nqueries, + pmix_info_cbfunc_t cbfunc, + void *cbdata); +static void tool_connect_fn(pmix_info_t *info, size_t ninfo, + pmix_tool_connection_cbfunc_t cbfunc, + void *cbdata); +static void log_fn(const pmix_proc_t *client, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata); + +static pmix_server_module_t mymodule = { + .client_connected = connected, + .client_finalized = finalized, + .abort = abort_fn, + .fence_nb = fencenb_fn, + .direct_modex = dmodex_fn, + .publish = publish_fn, + .lookup = lookup_fn, + .unpublish = unpublish_fn, + .spawn = spawn_fn, + .connect = connect_fn, + .disconnect = disconnect_fn, + .register_events = register_event_fn, + .deregister_events = deregister_events, + .notify_event = notify_event, + .query = query_fn, + .tool_connected = tool_connect_fn, + .log = log_fn +}; + +typedef struct { + pmix_list_item_t super; + pmix_pdata_t pdata; +} pmix_locdat_t; +PMIX_CLASS_INSTANCE(pmix_locdat_t, + pmix_list_item_t, + NULL, NULL); + +typedef struct { + pmix_object_t super; + mylock_t lock; + pmix_event_t ev; + pmix_proc_t caller; + pmix_info_t *info; + size_t ninfo; + pmix_op_cbfunc_t cbfunc; + pmix_spawn_cbfunc_t spcbfunc; + pmix_release_cbfunc_t relcbfunc; + void *cbdata; +} myxfer_t; +static void xfcon(myxfer_t *p) +{ + DEBUG_CONSTRUCT_LOCK(&p->lock); + p->info = NULL; + p->ninfo = 0; + p->cbfunc = NULL; + p->spcbfunc = NULL; + p->cbdata = NULL; +} +static void xfdes(myxfer_t *p) +{ + DEBUG_DESTRUCT_LOCK(&p->lock); + if (NULL != p->info) { + PMIX_INFO_FREE(p->info, p->ninfo); + } +} +PMIX_CLASS_INSTANCE(myxfer_t, + pmix_object_t, + xfcon, xfdes); + +typedef struct { + pmix_list_item_t super; + int exit_code; + pid_t pid; +} wait_tracker_t; +PMIX_CLASS_INSTANCE(wait_tracker_t, + pmix_list_item_t, + NULL, NULL); + +static volatile int wakeup; +static int exit_code = 0; +static pmix_list_t pubdata; +static pmix_event_t handler; +static pmix_list_t children; +static bool istimeouttest = false; + +static void set_namespace(int nprocs, char *ranks, char *nspace, + pmix_op_cbfunc_t cbfunc, myxfer_t *x); +static void errhandler(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata); +static void wait_signal_callback(int fd, short event, void *arg); +static void errhandler_reg_callbk (pmix_status_t status, + size_t errhandler_ref, + void *cbdata); + +static void opcbfunc(pmix_status_t status, void *cbdata) +{ + myxfer_t *x = (myxfer_t*)cbdata; + + /* release the caller, if necessary */ + if (NULL != x->cbfunc) { + x->cbfunc(PMIX_SUCCESS, x->cbdata); + } + DEBUG_WAKEUP_THREAD(&x->lock); +} + +int main(int argc, char **argv) +{ + char **client_env=NULL; + char **client_argv=NULL; + char *tmp, **atmp, *executable=NULL, *nspace; + int rc, nprocs=1, n, k; + uid_t myuid; + gid_t mygid; + pid_t pid; + myxfer_t *x; + pmix_proc_t proc; + wait_tracker_t *child; + pmix_info_t *info; + size_t ninfo; + mylock_t mylock; + int ncycles=1, m, delay=0; + + /* smoke test */ + if (PMIX_SUCCESS != 0) { + fprintf(stderr, "ERROR IN COMPUTING CONSTANTS: PMIX_SUCCESS = %d\n", PMIX_SUCCESS); + exit(1); + } + + fprintf(stderr, "Testing version %s\n", PMIx_Get_version()); + + /* see if we were passed the number of procs to run or + * the executable to use */ + for (n=1; n < argc; n++) { + if (0 == strcmp("-n", argv[n]) && + NULL != argv[n+1]) { + nprocs = strtol(argv[n+1], NULL, 10); + ++n; // step over the argument + } else if (0 == strcmp("-e", argv[n]) && + NULL != argv[n+1]) { + executable = strdup(argv[n+1]); + /* check for timeout test */ + if (NULL != strstr(executable, "quietclient")) { + istimeouttest = true; + } + for (k=n+2; NULL != argv[k]; k++) { + pmix_argv_append_nosize(&client_argv, argv[k]); + } + n += k; + } else if ((0 == strcmp("-reps", argv[n]) || + 0 == strcmp("--reps", argv[n])) && + NULL != argv[n+1]) { + ncycles = strtol(argv[n+1], NULL, 10); + } else if ((0 == strcmp("-sleep", argv[n]) || + 0 == strcmp("--sleep", argv[n])) && + NULL != argv[n+1]) { + delay = strtol(argv[n+1], NULL, 10); + } else if (0 == strcmp("-h", argv[n])) { + /* print the options and exit */ + fprintf(stderr, "usage: simptest \n"); + fprintf(stderr, " -n N Number of clients to run\n"); + fprintf(stderr, " -e foo Name of the client executable to run (default: simpclient\n"); + fprintf(stderr, " -reps N Cycle for N repetitions"); + exit(0); + } + } + if (NULL == executable) { + executable = strdup("./quietclient"); + } + /* setup the server library and tell it to support tool connections */ + ninfo = 3; + + PMIX_INFO_CREATE(info, ninfo); + PMIX_INFO_LOAD(&info[0], PMIX_SERVER_TOOL_SUPPORT, NULL, PMIX_BOOL); + PMIX_INFO_LOAD(&info[1], PMIX_USOCK_DISABLE, NULL, PMIX_BOOL); + PMIX_INFO_LOAD(&info[2], PMIX_SERVER_GATEWAY, NULL, PMIX_BOOL); + if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, ninfo))) { + fprintf(stderr, "Init failed with error %d\n", rc); + return rc; + } + PMIX_INFO_FREE(info, ninfo); + + /* register the default errhandler */ + DEBUG_CONSTRUCT_LOCK(&mylock); + ninfo = 1; + PMIX_INFO_CREATE(info, ninfo); + PMIX_INFO_LOAD(&info[0], PMIX_EVENT_HDLR_NAME, "SIMPTEST-DEFAULT", PMIX_STRING); + PMIx_Register_event_handler(NULL, 0, info, ninfo, + errhandler, errhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + PMIX_INFO_FREE(info, ninfo); + if (PMIX_SUCCESS != mylock.status) { + exit(mylock.status); + } + DEBUG_DESTRUCT_LOCK(&mylock); + + /* setup the pub data, in case it is used */ + PMIX_CONSTRUCT(&pubdata, pmix_list_t); + + /* setup to see sigchld on the forked tests */ + PMIX_CONSTRUCT(&children, pmix_list_t); + event_assign(&handler, pmix_globals.evbase, SIGCHLD, + EV_SIGNAL|EV_PERSIST,wait_signal_callback, &handler); + event_add(&handler, NULL); + + for (m=0; m < ncycles; m++) { + fprintf(stderr, "Running cycle %d\n", m); + /* we have a single namespace for all clients */ + atmp = NULL; + for (n=0; n < nprocs; n++) { + asprintf(&tmp, "%d", n); + pmix_argv_append_nosize(&atmp, tmp); + free(tmp); + } + tmp = pmix_argv_join(atmp, ','); + pmix_argv_free(atmp); + asprintf(&nspace, "foobar%d", m); + (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + x = PMIX_NEW(myxfer_t); + set_namespace(nprocs, tmp, nspace, opcbfunc, x); + + + /* set common argv and env */ + client_env = pmix_argv_copy(environ); + pmix_argv_prepend_nosize(&client_argv, executable); + + wakeup = nprocs; + myuid = getuid(); + mygid = getgid(); + + /* if the nspace registration hasn't completed yet, + * wait for it here */ + DEBUG_WAIT_THREAD(&x->lock); + free(tmp); + free(nspace); + PMIX_RELEASE(x); + + /* fork/exec the test */ + for (n = 0; n < nprocs; n++) { + proc.rank = n; + if (PMIX_SUCCESS != (rc = PMIx_server_setup_fork(&proc, &client_env))) { + fprintf(stderr, "Server fork setup failed with error %d\n", rc); + PMIx_server_finalize(); + return rc; + } + x = PMIX_NEW(myxfer_t); + if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid, + NULL, opcbfunc, x))) { + fprintf(stderr, "Server register client failed with error %d\n", rc); + PMIx_server_finalize(); + return rc; + } + /* don't fork/exec the client until we know it is registered + * so we avoid a potential race condition in the server */ + DEBUG_WAIT_THREAD(&x->lock); + PMIX_RELEASE(x); + pid = fork(); + if (pid < 0) { + fprintf(stderr, "Fork failed\n"); + PMIx_server_finalize(); + return -1; + } + child = PMIX_NEW(wait_tracker_t); + child->pid = pid; + pmix_list_append(&children, &child->super); + + if (pid == 0) { + execve(executable, client_argv, client_env); + /* Does not return */ + exit(0); + } + } + pmix_argv_free(client_argv); + client_argv = NULL; + pmix_argv_free(client_env); + client_env = NULL; + + /* hang around until the client(s) finalize */ + while (0 < wakeup) { + struct timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = 100000; + nanosleep(&ts, NULL); + } + + /* see if anyone exited with non-zero status */ + n=0; + PMIX_LIST_FOREACH(child, &children, wait_tracker_t) { + if (0 != child->exit_code) { + fprintf(stderr, "Child %d exited with status %d - test FAILED\n", n, child->exit_code); + goto done; + } + ++n; + } + + /* deregister the clients */ + for (n = 0; n < nprocs; n++) { + proc.rank = n; + x = PMIX_NEW(myxfer_t); + PMIx_server_deregister_client(&proc, opcbfunc, x); + DEBUG_WAIT_THREAD(&x->lock); + PMIX_RELEASE(x); + } + /* deregister the nspace */ + x = PMIX_NEW(myxfer_t); + PMIx_server_deregister_nspace(proc.nspace, opcbfunc, x); + DEBUG_WAIT_THREAD(&x->lock); + PMIX_RELEASE(x); + + PMIX_LIST_DESTRUCT(&children); + PMIX_CONSTRUCT(&children, pmix_list_t); + + sleep(delay); + } + + done: + /* deregister the event handlers */ + PMIx_Deregister_event_handler(0, NULL, NULL); + + /* release any pub data */ + PMIX_LIST_DESTRUCT(&pubdata); + + free(executable); + + /* finalize the server library */ + if (PMIX_SUCCESS != (rc = PMIx_server_finalize())) { + fprintf(stderr, "Finalize failed with error %d\n", rc); + exit_code = rc; + } + + if (0 == exit_code) { + fprintf(stderr, "Test finished OK!\n"); + } else { + fprintf(stderr, "TEST FAILED WITH ERROR %d\n", exit_code); + } + + return exit_code; +} + +static void set_namespace(int nprocs, char *ranks, char *nspace, + pmix_op_cbfunc_t cbfunc, myxfer_t *x) +{ + char *regex, *ppn; + char hostname[PMIX_MAXHOSTNAMELEN]; + + gethostname(hostname, sizeof(hostname)); + x->ninfo = 7; + + PMIX_INFO_CREATE(x->info, x->ninfo); + (void)strncpy(x->info[0].key, PMIX_UNIV_SIZE, PMIX_MAX_KEYLEN); + x->info[0].value.type = PMIX_UINT32; + x->info[0].value.data.uint32 = nprocs; + + (void)strncpy(x->info[1].key, PMIX_SPAWNED, PMIX_MAX_KEYLEN); + x->info[1].value.type = PMIX_UINT32; + x->info[1].value.data.uint32 = 0; + + (void)strncpy(x->info[2].key, PMIX_LOCAL_SIZE, PMIX_MAX_KEYLEN); + x->info[2].value.type = PMIX_UINT32; + x->info[2].value.data.uint32 = nprocs; + + (void)strncpy(x->info[3].key, PMIX_LOCAL_PEERS, PMIX_MAX_KEYLEN); + x->info[3].value.type = PMIX_STRING; + x->info[3].value.data.string = strdup(ranks); + + PMIx_generate_regex(hostname, ®ex); + (void)strncpy(x->info[4].key, PMIX_NODE_MAP, PMIX_MAX_KEYLEN); + x->info[4].value.type = PMIX_STRING; + x->info[4].value.data.string = regex; + + PMIx_generate_ppn(ranks, &ppn); + (void)strncpy(x->info[5].key, PMIX_PROC_MAP, PMIX_MAX_KEYLEN); + x->info[5].value.type = PMIX_STRING; + x->info[5].value.data.string = ppn; + + (void)strncpy(x->info[6].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN); + x->info[6].value.type = PMIX_UINT32; + x->info[6].value.data.uint32 = nprocs; + + PMIx_server_register_nspace(nspace, nprocs, x->info, x->ninfo, + cbfunc, x); +} + +static void errhandler(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) +{ + return; +} + +static void errhandler_reg_callbk (pmix_status_t status, + size_t errhandler_ref, + void *cbdata) +{ + mylock_t *lock = (mylock_t*)cbdata; + + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); +} + +static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + return PMIX_SUCCESS; +} +static pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + /* ensure we call the cbfunc so the proc can exit! */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + return PMIX_SUCCESS; +} + +static void abcbfunc(pmix_status_t status, void *cbdata) +{ + myxfer_t *x = (myxfer_t*)cbdata; + + /* be sure to release the caller */ + if (NULL != x->cbfunc) { + x->cbfunc(status, x->cbdata); + } + PMIX_RELEASE(x); +} + +static pmix_status_t abort_fn(const pmix_proc_t *proc, + void *server_object, + int status, const char msg[], + pmix_proc_t procs[], size_t nprocs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_status_t rc; + myxfer_t *x; + + /* instead of aborting the specified procs, notify them + * (if they have registered their errhandler) */ + + /* use the myxfer_t object to ensure we release + * the caller when notification has been queued */ + x = PMIX_NEW(myxfer_t); + (void)strncpy(x->caller.nspace, proc->nspace, PMIX_MAX_NSLEN); + x->caller.rank = proc->rank; + + PMIX_INFO_CREATE(x->info, 2); + (void)strncpy(x->info[0].key, "DARTH", PMIX_MAX_KEYLEN); + x->info[0].value.type = PMIX_INT8; + x->info[0].value.data.int8 = 12; + (void)strncpy(x->info[1].key, "VADER", PMIX_MAX_KEYLEN); + x->info[1].value.type = PMIX_DOUBLE; + x->info[1].value.data.dval = 12.34; + x->cbfunc = cbfunc; + x->cbdata = cbdata; + + if (PMIX_SUCCESS != (rc = PMIx_Notify_event(status, &x->caller, + PMIX_RANGE_NAMESPACE, + x->info, 2, + abcbfunc, x))) { + pmix_output(0, "SERVER: FAILED NOTIFY ERROR %d", (int)rc); + } + + return PMIX_SUCCESS; +} + + +static pmix_status_t fencenb_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + char *data, size_t ndata, + pmix_modex_cbfunc_t cbfunc, void *cbdata) +{ + /* pass the provided data back to each participating proc */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, data, ndata, cbdata, free, data); + } + return PMIX_SUCCESS; +} + + +static pmix_status_t dmodex_fn(const pmix_proc_t *proc, + const pmix_info_t info[], size_t ninfo, + pmix_modex_cbfunc_t cbfunc, void *cbdata) +{ + /* if this is a timeout test, then do nothing */ + if (istimeouttest) { + return PMIX_SUCCESS; + } + + /* we don't have any data for remote procs as this + * test only runs one server - so report accordingly */ + if (NULL != cbfunc) { + cbfunc(PMIX_ERR_NOT_FOUND, NULL, 0, cbdata, NULL, NULL); + } + return PMIX_SUCCESS; +} + + +static pmix_status_t publish_fn(const pmix_proc_t *proc, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_locdat_t *p; + size_t n; + + for (n=0; n < ninfo; n++) { + p = PMIX_NEW(pmix_locdat_t); + (void)strncpy(p->pdata.proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + p->pdata.proc.rank = proc->rank; + (void)strncpy(p->pdata.key, info[n].key, PMIX_MAX_KEYLEN); + pmix_value_xfer(&p->pdata.value, (pmix_value_t*)&info[n].value); + pmix_list_append(&pubdata, &p->super); + } + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + return PMIX_SUCCESS; +} + + +static pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_lookup_cbfunc_t cbfunc, void *cbdata) +{ + pmix_locdat_t *p, *p2; + pmix_list_t results; + size_t i, n; + pmix_pdata_t *pd = NULL; + pmix_status_t ret = PMIX_ERR_NOT_FOUND; + + PMIX_CONSTRUCT(&results, pmix_list_t); + + for (n=0; NULL != keys[n]; n++) { + PMIX_LIST_FOREACH(p, &pubdata, pmix_locdat_t) { + if (0 == strncmp(keys[n], p->pdata.key, PMIX_MAX_KEYLEN)) { + p2 = PMIX_NEW(pmix_locdat_t); + (void)strncpy(p2->pdata.proc.nspace, p->pdata.proc.nspace, PMIX_MAX_NSLEN); + p2->pdata.proc.rank = p->pdata.proc.rank; + (void)strncpy(p2->pdata.key, p->pdata.key, PMIX_MAX_KEYLEN); + pmix_value_xfer(&p2->pdata.value, &p->pdata.value); + pmix_list_append(&results, &p2->super); + break; + } + } + } + if (0 < (n = pmix_list_get_size(&results))) { + ret = PMIX_SUCCESS; + PMIX_PDATA_CREATE(pd, n); + for (i=0; i < n; i++) { + p = (pmix_locdat_t*)pmix_list_remove_first(&results); + if (p) { + (void)strncpy(pd[i].proc.nspace, p->pdata.proc.nspace, PMIX_MAX_NSLEN); + pd[i].proc.rank = p->pdata.proc.rank; + (void)strncpy(pd[i].key, p->pdata.key, PMIX_MAX_KEYLEN); + pmix_value_xfer(&pd[i].value, &p->pdata.value); + } + } + } + PMIX_LIST_DESTRUCT(&results); + if (NULL != cbfunc) { + cbfunc(ret, pd, n, cbdata); + } + if (0 < n) { + PMIX_PDATA_FREE(pd, n); + } + return PMIX_SUCCESS; +} + + +static pmix_status_t unpublish_fn(const pmix_proc_t *proc, char **keys, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + pmix_locdat_t *p, *p2; + size_t n; + + for (n=0; NULL != keys[n]; n++) { + PMIX_LIST_FOREACH_SAFE(p, p2, &pubdata, pmix_locdat_t) { + if (0 == strncmp(keys[n], p->pdata.key, PMIX_MAX_KEYLEN)) { + pmix_list_remove_item(&pubdata, &p->super); + PMIX_RELEASE(p); + break; + } + } + } + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + return PMIX_SUCCESS; +} + +static void spcbfunc(pmix_status_t status, void *cbdata) +{ + myxfer_t *x = (myxfer_t*)cbdata; + + if (NULL != x->spcbfunc) { + x->spcbfunc(PMIX_SUCCESS, "DYNSPACE", x->cbdata); + } +} + +static pmix_status_t spawn_fn(const pmix_proc_t *proc, + const pmix_info_t job_info[], size_t ninfo, + const pmix_app_t apps[], size_t napps, + pmix_spawn_cbfunc_t cbfunc, void *cbdata) +{ + myxfer_t *x; + + /* in practice, we would pass this request to the local + * resource manager for launch, and then have that server + * execute our callback function. For now, we will fake + * the spawn and just pretend */ + + /* must register the nspace for the new procs before + * we return to the caller */ + x = PMIX_NEW(myxfer_t); + x->spcbfunc = cbfunc; + x->cbdata = cbdata; + + set_namespace(2, "0,1", "DYNSPACE", spcbfunc, x); + + return PMIX_SUCCESS; +} + +static int numconnects = 0; + +static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + + /* in practice, we would pass this request to the local + * resource manager for handling */ + + numconnects++; + + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + + return PMIX_SUCCESS; +} + + +static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + + /* in practice, we would pass this request to the local + * resource manager for handling */ + + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + + return PMIX_SUCCESS; +} + +static pmix_status_t register_event_fn(pmix_status_t *codes, size_t ncodes, + const pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + return PMIX_SUCCESS; +} + +static pmix_status_t deregister_events(pmix_status_t *codes, size_t ncodes, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + return PMIX_SUCCESS; +} + +static pmix_status_t notify_event(pmix_status_t code, + const pmix_proc_t *source, + pmix_data_range_t range, + pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + return PMIX_SUCCESS; +} + +typedef struct query_data_t { + pmix_info_t *data; + size_t ndata; +} query_data_t; + +static pmix_status_t query_fn(pmix_proc_t *proct, + pmix_query_t *queries, size_t nqueries, + pmix_info_cbfunc_t cbfunc, + void *cbdata) +{ + size_t n; + pmix_info_t *info; + + if (NULL == cbfunc) { + return PMIX_ERROR; + } + /* keep this simple */ + PMIX_INFO_CREATE(info, nqueries); + for (n=0; n < nqueries; n++) { + (void)strncpy(info[n].key, queries[n].keys[0], PMIX_MAX_KEYLEN); + info[n].value.type = PMIX_STRING; + if (0 > asprintf(&info[n].value.data.string, "%d", (int)n)) { + return PMIX_ERROR; + } + } + cbfunc(PMIX_SUCCESS, info, nqueries, cbdata, NULL, NULL); + return PMIX_SUCCESS; +} + +static void tool_connect_fn(pmix_info_t *info, size_t ninfo, + pmix_tool_connection_cbfunc_t cbfunc, + void *cbdata) +{ + pmix_proc_t proc; + + /* just pass back an arbitrary nspace */ + (void)strncpy(proc.nspace, "TOOL", PMIX_MAX_NSLEN); + proc.rank = 0; + + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, &proc, cbdata); + } +} + +static void log_fn(const pmix_proc_t *client, + const pmix_info_t data[], size_t ndata, + const pmix_info_t directives[], size_t ndirs, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } +} + +static void wait_signal_callback(int fd, short event, void *arg) +{ + pmix_event_t *sig = (pmix_event_t*) arg; + int status; + pid_t pid; + wait_tracker_t *t2; + + if (SIGCHLD != event_get_signal(sig)) { + return; + } + + /* we can have multiple children leave but only get one + * sigchild callback, so reap all the waitpids until we + * don't get anything valid back */ + while (1) { + pid = waitpid(-1, &status, WNOHANG); + if (-1 == pid && EINTR == errno) { + /* try it again */ + continue; + } + /* if we got garbage, then nothing we can do */ + if (pid <= 0) { + return; + } + + /* we are already in an event, so it is safe to access the list */ + PMIX_LIST_FOREACH(t2, &children, wait_tracker_t) { + if (pid == t2->pid) { + t2->exit_code = status; + /* found it! */ + if (0 != status && 0 == exit_code) { + exit_code = status; + } + --wakeup; + break; + } + } + } +} diff --git a/opal/mca/pmix/pmix3x/pmix/test/test_cd.c b/opal/mca/pmix/pmix3x/pmix/test/test_cd.c index 45134ff17fe..f1a800202e2 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/test_cd.c +++ b/opal/mca/pmix/pmix3x/pmix/test/test_cd.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -25,16 +25,10 @@ static void cd_cb(pmix_status_t status, void *cbdata) cb->in_progress = 0; } -static void cnct_cb(pmix_status_t status, - char nspace[], int rank, - void *cbdata) +static void cnct_cb(pmix_status_t status, void *cbdata) { cd_cbdata *cb = (cd_cbdata*)cbdata; - if (NULL != nspace) { - (void)strncpy(cb->pname.nspace, nspace, PMIX_MAX_NSLEN); - } - cb->pname.rank = rank; cb->status = status; cb->in_progress = 0; } @@ -43,21 +37,19 @@ int test_connect_disconnect(char *my_nspace, int my_rank) { int rc; pmix_proc_t proc; - char nspace[PMIX_MAX_NSLEN+1]; - pmix_rank_t newrank; cd_cbdata cbdata; (void)strncpy(proc.nspace, my_nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; - rc = PMIx_Connect(&proc, 1, NULL, 0, nspace, &newrank); + rc = PMIx_Connect(&proc, 1, NULL, 0); if (PMIX_SUCCESS != rc) { TEST_ERROR(("%s:%d: Connect blocking test failed.", my_nspace, my_rank)); return PMIX_ERROR; } - TEST_VERBOSE(("%s:%d: Connect blocking test succeded to nspace %s.", my_nspace, my_rank, nspace)); + TEST_VERBOSE(("%s:%d: Connect blocking test succeded", my_nspace, my_rank)); - rc = PMIx_Disconnect(nspace, NULL, 0); + rc = PMIx_Disconnect(&proc, 1, NULL, 0); if (PMIX_SUCCESS != rc) { TEST_ERROR(("%s:%d: Disconnect blocking test failed.", my_nspace, my_rank)); return PMIX_ERROR; @@ -77,7 +69,7 @@ int test_connect_disconnect(char *my_nspace, int my_rank) TEST_VERBOSE(("%s:%d: Connect non-blocking test succeded.", my_nspace, my_rank)); cbdata.in_progress = 1; - rc = PMIx_Disconnect_nb(nspace, NULL, 0, cd_cb, &cbdata); + rc = PMIx_Disconnect_nb(&proc, 1, NULL, 0, cd_cb, &cbdata); if (PMIX_SUCCESS == rc) { PMIX_WAIT_FOR_COMPLETION(cbdata.in_progress); rc = cbdata.status; diff --git a/opal/mca/pmix/pmix3x/pmix/test/test_common.h b/opal/mca/pmix/pmix3x/pmix/test/test_common.h index 0c2837553ae..acc49d1bcef 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/test_common.h +++ b/opal/mca/pmix/pmix3x/pmix/test/test_common.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2015 Research Organization for Information Science @@ -33,6 +33,14 @@ #define TEST_NAMESPACE "smoky_nspace" #define TEST_CREDENTIAL "dummy" +#define PMIX_WAIT_FOR_COMPLETION(m) \ + do { \ + while ((m)) { \ + usleep(10); \ + } \ + } while(0) + + /* WARNING: pmix_test_output_prepare is currently not threadsafe! * fix it once needed! */ @@ -48,7 +56,7 @@ extern FILE *file; fflush(file); \ } -// Write output wightout adding anything to it. +// Write output without adding anything to it. // Need for automate tests to receive "OK" string #define TEST_OUTPUT_CLEAR(x) { \ fprintf(file, "%s", pmix_test_output_prepare x ); \ diff --git a/opal/mca/pmix/pmix3x/pmix/test/test_fence.c b/opal/mca/pmix/pmix3x/pmix/test/test_fence.c index 909edcffa3f..9ad4cf786df 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/test_fence.c +++ b/opal/mca/pmix/pmix3x/pmix/test/test_fence.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015-2017 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -63,7 +63,7 @@ int test_fence(test_params params, char *my_nspace, pmix_rank_t my_rank) pmix_proc_t *pcs; bool participate; int fence_num = 0; - char sval[50]; + char sval[500]; int put_ind; if (NULL != params.noise) { @@ -98,7 +98,7 @@ int test_fence(test_params params, char *my_nspace, pmix_rank_t my_rank) /*run fence test on this range */ /* first put value (my_ns, my_rank) with key based on fence_num to split results of different fences*/ put_ind = 0; - (void)snprintf(sval, 50, "%d:%s:%d", fence_num, my_nspace, my_rank); + (void)snprintf(sval, 500, "%d:%s:%d", fence_num, my_nspace, my_rank); PUT(string, sval, PMIX_GLOBAL, fence_num, put_ind++, params.use_same_keys); if (PMIX_SUCCESS != rc) { TEST_ERROR(("%s:%d: PMIx_Put failed: %d", my_nspace, my_rank, rc)); @@ -186,7 +186,7 @@ int test_fence(test_params params, char *my_nspace, pmix_rank_t my_rank) /* get data from all participating in this fence clients */ PMIX_LIST_FOREACH(p, desc->participants, participant_t) { put_ind = 0; - snprintf(sval, 50, "%d:%s:%d", fence_num, p->proc.nspace, p->proc.rank); + snprintf(sval, 500, "%d:%s:%d", fence_num, p->proc.nspace, p->proc.rank); GET(string, sval, p->proc.nspace, p->proc.rank, fence_num, put_ind++, params.use_same_keys, 1, 0); if (PMIX_SUCCESS != rc) { TEST_ERROR(("%s:%d: PMIx_Get failed (%d) from %s:%d", my_nspace, my_rank, rc, p->proc.nspace, p->proc.rank)); diff --git a/opal/mca/pmix/pmix3x/pmix/test/test_resolve_peers.c b/opal/mca/pmix/pmix3x/pmix/test/test_resolve_peers.c index 76005b6797f..c88e4ac879e 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/test_resolve_peers.c +++ b/opal/mca/pmix/pmix3x/pmix/test/test_resolve_peers.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -61,7 +61,6 @@ int test_resolve_peers(char *my_nspace, int my_rank, test_params params) int ns_num; char nspace[PMIX_MAX_NSLEN+1]; pmix_proc_t procs[2]; - pmix_proc_t pname; /* first resolve peers from the own namespace. */ rc = resolve_nspace(my_nspace, params, my_nspace, my_rank); @@ -99,11 +98,11 @@ int test_resolve_peers(char *my_nspace, int my_rank, test_params params) procs[1].rank = PMIX_RANK_WILDCARD; /* make a connection between processes from own namespace and processes from this namespace. */ - rc = PMIx_Connect(procs, 2, NULL, 0, pname.nspace, &pname.rank); + rc = PMIx_Connect(procs, 2, NULL, 0); if (PMIX_SUCCESS == rc) { - TEST_VERBOSE(("%s:%d: Connect to %s succeeded %s.", my_nspace, my_rank, nspace)); + TEST_VERBOSE(("%s:%d: Connect to %s succeeded.", my_nspace, my_rank, nspace)); } else { - TEST_ERROR(("%s:%d: Connect to %s failed %s.", my_nspace, my_rank, nspace)); + TEST_ERROR(("%s:%d: Connect to %s failed.", my_nspace, my_rank, nspace)); return PMIX_ERROR; } @@ -112,12 +111,12 @@ int test_resolve_peers(char *my_nspace, int my_rank, test_params params) if (PMIX_SUCCESS == rc) { TEST_VERBOSE(("%s:%d: Resolve peers succeeded for ns %s\n", my_nspace, my_rank, nspace)); } else { - PMIx_Disconnect(pname.nspace, NULL, 0); + PMIx_Disconnect(procs, 2, NULL, 0); break; } /* disconnect from the processes of this namespace. */ - rc = PMIx_Disconnect(pname.nspace, NULL, 0); + rc = PMIx_Disconnect(procs, 2, NULL, 0); if (PMIX_SUCCESS == rc) { TEST_VERBOSE(("%s:%d: Disconnect from %s succeeded %s.", my_nspace, my_rank, nspace)); } else { diff --git a/opal/mca/pmix/pmix3x/pmix3x.c b/opal/mca/pmix/pmix3x/pmix3x.c index 29e9062a50c..4177ddb14fd 100644 --- a/opal/mca/pmix/pmix3x/pmix3x.c +++ b/opal/mca/pmix/pmix3x/pmix3x.c @@ -1773,7 +1773,6 @@ static void ocadcon(pmix3x_opalcaddy_t *p) p->odmdxfunc = NULL; p->infocbfunc = NULL; p->toolcbfunc = NULL; - p->cnctcbfunc = NULL; p->ocbdata = NULL; } static void ocaddes(pmix3x_opalcaddy_t *p) diff --git a/opal/mca/pmix/pmix3x/pmix3x.h b/opal/mca/pmix/pmix3x/pmix3x.h index b3cf4de76c4..3579ed03a3c 100644 --- a/opal/mca/pmix/pmix3x/pmix3x.h +++ b/opal/mca/pmix/pmix3x/pmix3x.h @@ -126,7 +126,6 @@ typedef struct { pmix_spawn_cbfunc_t spwncbfunc; pmix_info_cbfunc_t infocbfunc; pmix_tool_connection_cbfunc_t toolcbfunc; - pmix_connect_cbfunc_t cnctcbfunc; void *cbdata; opal_pmix_release_cbfunc_t odmdxfunc; void *ocbdata; diff --git a/opal/mca/pmix/pmix3x/pmix3x_client.c b/opal/mca/pmix/pmix3x/pmix3x_client.c index 4ccf21c519a..61fc6825782 100644 --- a/opal/mca/pmix/pmix3x/pmix3x_client.c +++ b/opal/mca/pmix/pmix3x/pmix3x_client.c @@ -1305,9 +1305,6 @@ int pmix3x_connect(opal_list_t *procs) pmix_status_t ret; char *nsptr; size_t n; - char nspace[PMIX_MAX_NSLEN+1]; - pmix_rank_t rank; - pmix_info_t *info; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "pmix3x:client connect"); @@ -1339,28 +1336,12 @@ int pmix3x_connect(opal_list_t *procs) } OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); - /* tell PMIx that we don't want a return of an nspace */ - PMIX_INFO_CREATE(info, 1); - PMIX_INFO_LOAD(&info[0], PMIX_CONNECT_XCHG_ONLY, NULL, PMIX_BOOL); - - ret = PMIx_Connect(p, nprocs, info, 1, nspace, &rank); + ret = PMIx_Connect(p, nprocs, NULL, 0); PMIX_PROC_FREE(p, nprocs); - PMIX_INFO_FREE(info, 1); return pmix3x_convert_rc(ret); } -static void cnctcbfunc(pmix_status_t status, - char nspace[], int rank, - void *cbdata) -{ - pmix3x_opcaddy_t *op = (pmix3x_opcaddy_t*)cbdata; - if (NULL != op->opcbfunc) { - op->opcbfunc(status, op->cbdata); - } - OBJ_RELEASE(op); -} - int pmix3x_connectnb(opal_list_t *procs, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) @@ -1407,7 +1388,7 @@ int pmix3x_connectnb(opal_list_t *procs, } OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); - ret = PMIx_Connect_nb(op->procs, op->nprocs, NULL, 0, cnctcbfunc, op); + ret = PMIx_Connect_nb(op->procs, op->nprocs, NULL, 0, opcbfunc, op); if (PMIX_SUCCESS != ret) { OBJ_RELEASE(op); } @@ -1416,9 +1397,10 @@ int pmix3x_connectnb(opal_list_t *procs, int pmix3x_disconnect(opal_list_t *procs) { - size_t nprocs; + size_t nprocs, n; opal_namelist_t *ptr; pmix_status_t ret=PMIX_SUCCESS; + pmix_proc_t *p; char *nsptr; opal_output_verbose(1, opal_pmix_base_framework.framework_output, @@ -1435,17 +1417,24 @@ int pmix3x_disconnect(opal_list_t *procs) return OPAL_ERR_NOT_INITIALIZED; } + /* convert the list of procs to an array + * of pmix_proc_t */ + PMIX_PROC_CREATE(p, nprocs); + n=0; OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { if (NULL == (nsptr = pmix3x_convert_jobid(ptr->name.jobid))) { + PMIX_PROC_FREE(p, nprocs); OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); return OPAL_ERR_NOT_FOUND; } - OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); - ret = PMIx_Disconnect(nsptr, NULL, 0); - OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + (void)strncpy(p[n].nspace, nsptr, PMIX_MAX_NSLEN); + p[n].rank = pmix3x_convert_opalrank(ptr->name.vpid); + ++n; } OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); + ret = PMIx_Disconnect(p, nprocs, NULL, 0); + PMIX_PROC_FREE(p, nprocs); return pmix3x_convert_rc(ret); } @@ -1454,9 +1443,11 @@ int pmix3x_disconnectnb(opal_list_t *procs, opal_pmix_op_cbfunc_t cbfunc, void *cbdata) { + pmix3x_opcaddy_t *op; opal_namelist_t *ptr; - pmix_status_t ret=PMIX_SUCCESS; + pmix_status_t ret; char *nsptr; + size_t n; opal_output_verbose(1, opal_pmix_base_framework.framework_output, "pmix3x:client disconnect NB"); @@ -1472,22 +1463,31 @@ int pmix3x_disconnectnb(opal_list_t *procs, return OPAL_ERR_NOT_INITIALIZED; } + /* create the caddy */ + op = OBJ_NEW(pmix3x_opcaddy_t); + op->opcbfunc = cbfunc; + op->cbdata = cbdata; + op->nprocs = opal_list_get_size(procs); + + /* convert the list of procs to an array + * of pmix_proc_t */ + PMIX_PROC_CREATE(op->procs, op->nprocs); + n=0; OPAL_LIST_FOREACH(ptr, procs, opal_namelist_t) { if (NULL == (nsptr = pmix3x_convert_jobid(ptr->name.jobid))) { + OBJ_RELEASE(op); OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); return OPAL_ERR_NOT_FOUND; } - OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); - ret = PMIx_Disconnect(nsptr, NULL, 0); - if (PMIX_SUCCESS != ret) { - OPAL_ERROR_LOG(pmix3x_convert_rc(ret)); - } - OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock); + (void)strncpy(op->procs[n].nspace, nsptr, PMIX_MAX_NSLEN); + op->procs[n].rank = pmix3x_convert_opalrank(ptr->name.vpid); + ++n; } OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock); - if (NULL != cbfunc) { - cbfunc(pmix3x_convert_rc(ret), cbdata); + ret = PMIx_Disconnect_nb(op->procs, op->nprocs, NULL, 0, opcbfunc, op); + if (PMIX_SUCCESS != ret) { + OBJ_RELEASE(op); } return pmix3x_convert_rc(ret); } diff --git a/opal/mca/pmix/pmix3x/pmix3x_server_north.c b/opal/mca/pmix/pmix3x/pmix3x_server_north.c index 75261c7c461..9aae776fdea 100644 --- a/opal/mca/pmix/pmix3x/pmix3x_server_north.c +++ b/opal/mca/pmix/pmix3x/pmix3x_server_north.c @@ -76,8 +76,8 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *proc, pmix_spawn_cbfunc_t cbfunc, void *cbdata); static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, - pmix_connect_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t server_disconnect_fn(const char nspace[], + pmix_op_cbfunc_t cbfunc, void *cbdata); +static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); static pmix_status_t server_register_events(pmix_status_t *codes, size_t ncodes, @@ -711,20 +711,9 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *p, } -static void cnopcbfunc(int status, void *cbdata) -{ - pmix3x_opalcaddy_t *opalcaddy = (pmix3x_opalcaddy_t*)cbdata; - - if (NULL != opalcaddy->cnctcbfunc) { - opalcaddy->cnctcbfunc(pmix3x_convert_opalrc(status), - "1234", 0, opalcaddy->cbdata); - } - OBJ_RELEASE(opalcaddy); -} - static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, - pmix_connect_cbfunc_t cbfunc, void *cbdata) + pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; pmix3x_opalcaddy_t *opalcaddy; @@ -738,7 +727,7 @@ static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, /* setup the caddy */ opalcaddy = OBJ_NEW(pmix3x_opalcaddy_t); - opalcaddy->cnctcbfunc = cbfunc; + opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; /* convert the array of pmix_proc_t to the list of procs */ @@ -764,7 +753,7 @@ static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, } /* pass it up */ - rc = host_module->connect(&opalcaddy->procs, &opalcaddy->info, cnopcbfunc, opalcaddy); + rc = host_module->connect(&opalcaddy->procs, &opalcaddy->info, opal_opcbfunc, opalcaddy); if (OPAL_SUCCESS != rc) { OBJ_RELEASE(opalcaddy); } @@ -773,7 +762,7 @@ static pmix_status_t server_connect_fn(const pmix_proc_t procs[], size_t nprocs, } -static pmix_status_t server_disconnect_fn(const char nspace[], +static pmix_status_t server_disconnect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { @@ -792,14 +781,16 @@ static pmix_status_t server_disconnect_fn(const char nspace[], opalcaddy->opcbfunc = cbfunc; opalcaddy->cbdata = cbdata; - /* convert the nspace */ - nm = OBJ_NEW(opal_namelist_t); - opal_list_append(&opalcaddy->procs, &nm->super); - if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, nspace))) { - OBJ_RELEASE(opalcaddy); - return pmix3x_convert_opalrc(rc); + /* convert the array of pmix_proc_t to the list of procs */ + for (n=0; n < nprocs; n++) { + nm = OBJ_NEW(opal_namelist_t); + opal_list_append(&opalcaddy->procs, &nm->super); + if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&nm->name.jobid, procs[n].nspace))) { + OBJ_RELEASE(opalcaddy); + return pmix3x_convert_opalrc(rc); + } + nm->name.vpid = pmix3x_convert_rank(procs[n].rank); } - nm->name.vpid = OPAL_VPID_WILDCARD; /* convert the info */ for (n=0; n < ninfo; n++) {