From 6a4c0a604970d0bd0bd3fcfbb24f44e4b763629c Mon Sep 17 00:00:00 2001 From: Ralph Castain Date: Wed, 26 Jun 2019 11:36:30 -0700 Subject: [PATCH] Update to PMIx v2.2.3 pre-release candidate Will update to official release after passing MTT Signed-off-by: Ralph Castain --- opal/mca/pmix/pmix2x/pmix/INSTALL | 12 +- opal/mca/pmix/pmix2x/pmix/LICENSE | 8 +- opal/mca/pmix/pmix2x/pmix/Makefile.am | 9 +- opal/mca/pmix/pmix2x/pmix/NEWS | 78 +- opal/mca/pmix/pmix2x/pmix/README | 2 +- opal/mca/pmix/pmix2x/pmix/VERSION | 20 +- .../mca/pmix/pmix2x/pmix/config/distscript.sh | 8 +- opal/mca/pmix/pmix2x/pmix/config/pmix.m4 | 175 +- .../config/pmix_check_compiler_version.m4 | 44 +- .../pmix2x/pmix/config/pmix_check_lock.m4 | 4 +- .../pmix/config/pmix_check_os_flavors.m4 | 9 +- .../pmix2x/pmix/config/pmix_check_package.m4 | 97 +- .../pmix2x/pmix/config/pmix_config_asm.m4 | 579 +++-- .../pmix/config/pmix_config_pthreads.m4 | 353 +-- .../pmix2x/pmix/config/pmix_config_threads.m4 | 5 +- .../pmix/pmix2x/pmix/config/pmix_functions.m4 | 18 +- opal/mca/pmix/pmix2x/pmix/config/pmix_mca.m4 | 4 +- .../pmix2x/pmix/config/pmix_search_libs.m4 | 37 +- .../pmix2x/pmix/config/pmix_set_mca_prefix.m4 | 41 + .../pmix/pmix2x/pmix/config/pmix_setup_cc.m4 | 15 +- .../pmix/pmix2x/pmix/config/pmix_setup_cli.m4 | 40 + .../pmix2x/pmix/config/pmix_setup_libevent.m4 | 43 +- .../pmix2x/pmix/config/pmix_setup_zlib.m4 | 30 +- .../pmix/pmix2x/pmix/config/pmix_summary.m4 | 79 + opal/mca/pmix/pmix2x/pmix/configure.ac | 45 +- .../pmix2x/pmix/contrib/make_dist_tarball | 16 +- opal/mca/pmix/pmix2x/pmix/contrib/pmix.spec | 46 +- .../mca/pmix/pmix2x/pmix/examples/Makefile.am | 26 +- opal/mca/pmix/pmix2x/pmix/examples/alloc.c | 133 +- opal/mca/pmix/pmix2x/pmix/examples/client.c | 216 +- opal/mca/pmix/pmix2x/pmix/examples/client2.c | 28 +- opal/mca/pmix/pmix2x/pmix/examples/debugger.c | 260 +-- .../mca/pmix/pmix2x/pmix/examples/debuggerd.c | 9 +- opal/mca/pmix/pmix2x/pmix/examples/dmodex.c | 26 +- opal/mca/pmix/pmix2x/pmix/examples/dynamic.c | 7 +- opal/mca/pmix/pmix2x/pmix/examples/examples.h | 124 + opal/mca/pmix/pmix2x/pmix/examples/fault.c | 87 +- opal/mca/pmix/pmix2x/pmix/examples/jctrl.c | 71 +- opal/mca/pmix/pmix2x/pmix/examples/pub.c | 6 +- opal/mca/pmix/pmix2x/pmix/examples/server.c | 10 +- opal/mca/pmix/pmix2x/pmix/examples/tool.c | 86 +- opal/mca/pmix/pmix2x/pmix/include/Makefile.am | 5 +- opal/mca/pmix/pmix2x/pmix/include/pmix.h | 15 +- .../pmix/pmix2x/pmix/include/pmix_common.h.in | 567 +++-- .../pmix/pmix2x/pmix/include/pmix_extend.h | 140 ++ .../pmix/pmix2x/pmix/include/pmix_server.h | 10 +- .../pmix2x/pmix/include/pmix_version.h.in | 7 +- opal/mca/pmix/pmix2x/pmix/man/Makefile.am | 60 - opal/mca/pmix/pmix2x/pmix/man/README | 186 -- .../pmix/pmix2x/pmix/man/man3/pmix_abort.3 | 62 - .../pmix/pmix2x/pmix/man/man3/pmix_commit.3 | 35 - .../pmix/pmix2x/pmix/man/man3/pmix_finalize.3 | 45 - .../mca/pmix/pmix2x/pmix/man/man3/pmix_init.3 | 52 - .../pmix2x/pmix/man/man3/pmix_initialized.3 | 30 - opal/mca/pmix/pmix2x/pmix/man/man3/pmix_put.3 | 60 - opal/mca/pmix/pmix2x/pmix/man/man7/pmix.7 | 35 - .../pmix2x/pmix/man/man7/pmix_constants.7 | 79 - opal/mca/pmix/pmix2x/pmix/src/Makefile.am | 2 +- .../pmix/src/atomics/sys/Makefile.include | 5 +- .../pmix/src/atomics/sys/architecture.h | 3 +- .../pmix2x/pmix/src/atomics/sys/arm/atomic.h | 20 +- .../pmix/src/atomics/sys/arm64/atomic.h | 30 +- .../pmix/pmix2x/pmix/src/atomics/sys/atomic.h | 146 +- .../pmix2x/pmix/src/atomics/sys/atomic_impl.h | 96 +- .../pmix2x/pmix/src/atomics/sys/atomic_stdc.h | 269 +++ .../pmix/pmix2x/pmix/src/atomics/sys/cma.h | 10 +- .../pmix/src/atomics/sys/gcc_builtin/atomic.h | 46 +- .../pmix2x/pmix/src/atomics/sys/ia32/atomic.h | 20 +- .../pmix/src/atomics/sys/powerpc/atomic.h | 56 +- .../pmix/src/atomics/sys/sparcv9/atomic.h | 26 +- .../src/atomics/sys/sync_builtin/atomic.h | 30 +- .../pmix/pmix2x/pmix/src/atomics/sys/timer.h | 2 +- .../pmix/src/atomics/sys/x86_64/atomic.h | 38 +- .../pmix/src/atomics/sys/x86_64/timer.h | 2 +- .../pmix2x/pmix/src/class/Makefile.include | 4 +- .../pmix/pmix2x/pmix/src/class/pmix_bitmap.c | 412 ++++ .../pmix/pmix2x/pmix/src/class/pmix_bitmap.h | 259 +++ .../pmix/pmix2x/pmix/src/class/pmix_hotel.c | 11 +- .../pmix/pmix2x/pmix/src/class/pmix_hotel.h | 36 +- .../pmix/pmix2x/pmix/src/class/pmix_list.h | 4 +- .../pmix/pmix2x/pmix/src/class/pmix_object.h | 4 +- opal/mca/pmix/pmix2x/pmix/src/client/pmi1.c | 20 +- opal/mca/pmix/pmix2x/pmix/src/client/pmi2.c | 32 +- .../pmix/pmix2x/pmix/src/client/pmix_client.c | 292 ++- .../pmix/src/client/pmix_client_connect.c | 8 +- .../pmix/src/client/pmix_client_fence.c | 4 +- .../pmix2x/pmix/src/client/pmix_client_get.c | 128 +- .../pmix2x/pmix/src/client/pmix_client_ops.h | 23 +- .../pmix2x/pmix/src/client/pmix_client_pub.c | 4 +- .../pmix/src/client/pmix_client_spawn.c | 43 +- .../pmix2x/pmix/src/common/Makefile.include | 4 +- .../pmix2x/pmix/src/common/pmix_control.c | 13 +- .../pmix/pmix2x/pmix/src/common/pmix_data.c | 12 +- .../pmix/pmix2x/pmix/src/common/pmix_log.c | 2 +- .../pmix/pmix2x/pmix/src/common/pmix_query.c | 255 +- .../pmix2x/pmix/src/common/pmix_strings.c | 14 +- .../pmix/pmix2x/pmix/src/event/pmix_event.h | 137 +- .../pmix/src/event/pmix_event_notification.c | 349 +-- .../pmix/src/event/pmix_event_registration.c | 225 +- .../pmix2x/pmix/src/include/Makefile.include | 7 +- .../pmix/src/include/pmix_config_bottom.h | 15 +- .../pmix2x/pmix/src/include/pmix_globals.c | 302 ++- .../pmix2x/pmix/src/include/pmix_globals.h | 175 +- .../pmix/src/include/pmix_portable_platform.h | 401 ++++ .../pmix2x/pmix/src/include/pmix_stdatomic.h | 67 + .../pmix2x/pmix/src/include/pmix_stdint.h | 5 +- opal/mca/pmix/pmix2x/pmix/src/include/types.h | 7 +- .../pmix/pmix2x/pmix/src/mca/base/Makefile.am | 4 +- opal/mca/pmix/pmix2x/pmix/src/mca/base/base.h | 102 +- ...lp-mca-base.txt => help-pmix-mca-base.txt} | 12 +- ...help-mca-var.txt => help-pmix-mca-var.txt} | 1 + .../src/mca/base/pmix_mca_base_cmd_line.c | 241 ++ .../mca/base/pmix_mca_base_component_find.c | 6 +- .../base/pmix_mca_base_component_repository.c | 51 +- .../base/pmix_mca_base_component_repository.h | 33 +- .../src/mca/base/pmix_mca_base_framework.c | 4 +- .../src/mca/base/pmix_mca_base_framework.h | 4 +- .../pmix/src/mca/base/pmix_mca_base_open.c | 11 +- .../pmix/src/mca/base/pmix_mca_base_var.c | 45 +- .../src/mca/base/pmix_mca_base_var_group.h | 38 +- .../pmix2x/pmix/src/mca/bfrops/base/base.h | 21 +- .../src/mca/bfrops/base/bfrop_base_copy.c | 110 +- .../pmix/src/mca/bfrops/base/bfrop_base_fns.c | 463 +--- .../src/mca/bfrops/base/bfrop_base_frame.c | 6 +- .../src/mca/bfrops/base/bfrop_base_pack.c | 74 +- .../src/mca/bfrops/base/bfrop_base_print.c | 49 +- .../src/mca/bfrops/base/bfrop_base_unpack.c | 107 +- .../pmix/pmix2x/pmix/src/mca/bfrops/bfrops.h | 11 +- .../pmix/src/mca/bfrops/v12/Makefile.am | 5 +- .../pmix2x/pmix/src/mca/bfrops/v12/copy.c | 33 +- .../pmix2x/pmix/src/mca/bfrops/v12/internal.h | 4 +- .../pmix2x/pmix/src/mca/bfrops/v12/pack.c | 7 +- .../pmix2x/pmix/src/mca/bfrops/v12/unpack.c | 17 +- .../pmix/src/mca/bfrops/v20/Makefile.am | 5 +- .../pmix2x/pmix/src/mca/bfrops/v20/copy.c | 978 ++++---- .../pmix2x/pmix/src/mca/bfrops/v20/internal.h | 4 +- .../pmix2x/pmix/src/mca/bfrops/v20/pack.c | 17 +- .../pmix2x/pmix/src/mca/bfrops/v20/print.c | 10 +- .../pmix2x/pmix/src/mca/bfrops/v20/unpack.c | 21 +- .../pmix/src/mca/bfrops/v21/Makefile.am | 5 +- .../pmix/src/mca/bfrops/v21/bfrop_pmix21.c | 232 +- .../pmix2x/pmix/src/mca/common/Makefile.am | 26 + .../pmix/src/mca/common/dstore/Makefile.am | 59 + .../pmix/src/mca/common/dstore/configure.m4 | 18 + .../dstore/dstore_base.c} | 2059 +++++++---------- .../pmix/src/mca/common/dstore/dstore_base.h | 139 ++ .../src/mca/common/dstore/dstore_common.h | 80 + .../pmix/src/mca/common/dstore/dstore_file.h | 161 ++ .../src/mca/common/dstore/dstore_segment.c | 302 +++ .../src/mca/common/dstore/dstore_segment.h | 70 + .../pmix/pmix2x/pmix/src/mca/gds/base/base.h | 15 +- .../pmix/src/mca/gds/base/gds_base_fns.c | 104 +- .../pmix2x/pmix/src/mca/gds/ds12/Makefile.am | 27 +- .../pmix/src/mca/gds/ds12/gds_ds12_base.c | 170 ++ .../pmix/src/mca/gds/ds12/gds_ds12_base.h | 21 + ...store_component.c => gds_ds12_component.c} | 13 +- .../pmix/src/mca/gds/ds12/gds_ds12_file.c | 177 ++ .../pmix/src/mca/gds/ds12/gds_ds12_file.h | 21 + .../pmix/src/mca/gds/ds12/gds_ds12_lock.c | 26 + .../pmix/src/mca/gds/ds12/gds_ds12_lock.h | 30 + .../src/mca/gds/ds12/gds_ds12_lock_fcntl.c | 233 ++ .../src/mca/gds/ds12/gds_ds12_lock_pthread.c | 274 +++ .../pmix/src/mca/gds/ds12/gds_ds20_file.c | 167 ++ .../pmix2x/pmix/src/mca/gds/ds12/gds_dstore.h | 129 -- .../pmix2x/pmix/src/mca/gds/ds21/Makefile.am | 65 + .../pmix/src/mca/gds/ds21/gds_ds21_base.c | 176 ++ .../pmix/src/mca/gds/ds21/gds_ds21_base.h | 21 + .../src/mca/gds/ds21/gds_ds21_component.c | 93 + .../pmix/src/mca/gds/ds21/gds_ds21_file.c | 189 ++ .../pmix/src/mca/gds/ds21/gds_ds21_file.h | 20 + .../pmix/src/mca/gds/ds21/gds_ds21_lock.c | 26 + .../pmix/src/mca/gds/ds21/gds_ds21_lock.h | 31 + .../src/mca/gds/ds21/gds_ds21_lock_pthread.c | 439 ++++ opal/mca/pmix/pmix2x/pmix/src/mca/gds/gds.h | 37 +- .../pmix2x/pmix/src/mca/gds/hash/Makefile.am | 5 +- .../pmix2x/pmix/src/mca/gds/hash/gds_hash.c | 309 ++- .../pmix/pmix2x/pmix/src/mca/pdl/base/base.h | 4 +- .../pmix/pmix2x/pmix/src/mca/pdl/configure.m4 | 8 +- .../pmix/src/mca/pif/bsdx_ipv4/pif_bsdx.c | 19 +- .../src/mca/pif/bsdx_ipv6/pif_bsdx_ipv6.c | 19 +- .../src/mca/pif/linux_ipv6/pif_linux_ipv6.c | 23 +- opal/mca/pmix/pmix2x/pmix/src/mca/pif/pif.h | 18 +- .../pmix/src/mca/pif/posix_ipv4/pif_posix.c | 23 +- .../mca/pif/solaris_ipv6/pif_solaris_ipv6.c | 8 +- .../pmix/src/mca/pinstalldirs/base/base.h | 4 +- .../src/mca/pinstalldirs/config/Makefile.am | 4 +- .../pmix/src/mca/pinstalldirs/pinstalldirs.h | 4 +- .../pmix/pmix2x/pmix/src/mca/pnet/base/base.h | 2 +- .../pmix/src/mca/pnet/base/pnet_base_fns.c | 5 +- .../pmix/src/mca/pnet/base/pnet_base_frame.c | 2 +- .../pmix2x/pmix/src/mca/pnet/opa/Makefile.am | 5 +- .../pmix2x/pmix/src/mca/pnet/opa/configure.m4 | 70 +- .../pmix2x/pmix/src/mca/pnet/opa/pnet_opa.c | 2 +- .../pmix2x/pmix/src/mca/pnet/opa/pnet_opa.h | 2 +- .../src/mca/pnet/opa/pnet_opa_component.c | 2 +- opal/mca/pmix/pmix2x/pmix/src/mca/pnet/pnet.h | 2 +- .../pmix/src/mca/preg/base/preg_base_frame.c | 4 +- .../pmix/src/mca/preg/native/Makefile.am | 5 +- .../pmix/src/mca/preg/native/preg_native.c | 270 +-- .../pmix2x/pmix/src/mca/preg/preg_types.h | 4 +- .../src/mca/psec/dummy_handshake/Makefile.am | 56 + .../dummy_handshake/psec_dummy_handshake.c | 170 ++ .../dummy_handshake/psec_dummy_handshake.h | 29 + .../psec_dummy_handshake_component.c | 73 + .../pmix/src/mca/psec/munge/Makefile.am | 5 +- .../pmix/src/mca/psec/munge/psec_munge.c | 2 +- .../pmix/src/mca/psec/native/Makefile.am | 5 +- .../pmix/src/mca/psec/native/psec_native.c | 8 +- .../pmix2x/pmix/src/mca/psec/none/Makefile.am | 5 +- .../pmix2x/pmix/src/mca/psec/none/psec_none.c | 2 +- .../src/mca/psec/none/psec_none_component.c | 2 +- opal/mca/pmix/pmix2x/pmix/src/mca/psec/psec.h | 33 +- .../pmix/src/mca/psensor/file/Makefile.am | 5 +- .../pmix/src/mca/psensor/file/psensor_file.c | 4 +- .../pmix/src/mca/psensor/file/psensor_file.h | 2 +- .../src/mca/psensor/heartbeat/Makefile.am | 5 +- .../mca/psensor/heartbeat/psensor_heartbeat.c | 4 +- .../src/mca/pshmem/base/pshmem_base_frame.c | 5 +- .../pmix/src/mca/pshmem/mmap/Makefile.am | 4 + .../pmix/src/mca/pshmem/mmap/pshmem_mmap.c | 4 +- .../pmix/pmix2x/pmix/src/mca/ptl/base/base.h | 4 +- .../pmix/src/mca/ptl/base/ptl_base_frame.c | 6 +- .../pmix/src/mca/ptl/base/ptl_base_sendrecv.c | 180 +- .../pmix/src/mca/ptl/base/ptl_base_stubs.c | 22 +- opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl.h | 30 +- .../pmix/pmix2x/pmix/src/mca/ptl/ptl_types.h | 104 +- .../pmix2x/pmix/src/mca/ptl/tcp/Makefile.am | 5 +- .../pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c | 611 +++-- .../pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.h | 8 +- .../pmix/src/mca/ptl/tcp/ptl_tcp_component.c | 710 ++++-- .../pmix2x/pmix/src/mca/ptl/usock/Makefile.am | 5 +- .../pmix2x/pmix/src/mca/ptl/usock/ptl_usock.c | 25 +- .../src/mca/ptl/usock/ptl_usock_component.c | 68 +- .../pmix/src/runtime/help-pmix-runtime.txt | 35 +- .../pmix2x/pmix/src/runtime/pmix_finalize.c | 33 +- .../pmix/pmix2x/pmix/src/runtime/pmix_init.c | 109 +- .../pmix2x/pmix/src/runtime/pmix_params.c | 128 +- .../pmix/src/runtime/pmix_progress_threads.c | 6 +- .../pmix/pmix2x/pmix/src/server/pmix_server.c | 1355 ++++++++--- .../pmix2x/pmix/src/server/pmix_server_get.c | 250 +- .../pmix2x/pmix/src/server/pmix_server_ops.c | 1144 ++++++--- .../pmix2x/pmix/src/server/pmix_server_ops.h | 109 +- .../pmix/pmix2x/pmix/src/threads/mutex_unix.h | 10 +- .../pmix2x/pmix/src/threads/thread_usage.h | 21 +- .../pmix/pmix2x/pmix/src/threads/threads.h | 3 +- opal/mca/pmix/pmix2x/pmix/src/threads/tsd.h | 72 +- .../pmix/pmix2x/pmix/src/threads/wait_sync.h | 4 +- .../mca/pmix/pmix2x/pmix/src/tool/pmix_tool.c | 1106 +++++---- .../pmix2x/pmix/src/util/Makefile.include | 7 +- opal/mca/pmix/pmix2x/pmix/src/util/argv.c | 6 +- opal/mca/pmix/pmix2x/pmix/src/util/basename.c | 8 +- opal/mca/pmix/pmix2x/pmix/src/util/cmd_line.c | 1406 +++++++++++ opal/mca/pmix/pmix2x/pmix/src/util/cmd_line.h | 622 +++++ opal/mca/pmix/pmix2x/pmix/src/util/error.c | 289 ++- opal/mca/pmix/pmix2x/pmix/src/util/error.h | 3 +- opal/mca/pmix/pmix2x/pmix/src/util/fd.c | 36 +- opal/mca/pmix/pmix2x/pmix/src/util/fd.h | 33 +- opal/mca/pmix/pmix2x/pmix/src/util/hash.c | 9 +- .../pmix/pmix2x/pmix/src/util/keyval_parse.c | 6 +- .../pmix/pmix2x/pmix/src/util/keyval_parse.h | 14 +- opal/mca/pmix/pmix2x/pmix/src/util/name_fns.c | 154 ++ opal/mca/pmix/pmix2x/pmix/src/util/name_fns.h | 47 + opal/mca/pmix/pmix2x/pmix/src/util/net.c | 20 +- opal/mca/pmix/pmix2x/pmix/src/util/os_path.c | 4 +- opal/mca/pmix/pmix2x/pmix/src/util/os_path.h | 9 +- opal/mca/pmix/pmix2x/pmix/src/util/output.c | 4 +- opal/mca/pmix/pmix2x/pmix/src/util/path.c | 3 +- opal/mca/pmix/pmix2x/pmix/src/util/pif.c | 36 +- .../mca/pmix/pmix2x/pmix/src/util/show_help.c | 16 +- .../mca/pmix/pmix2x/pmix/src/util/show_help.h | 15 +- opal/mca/pmix/pmix2x/pmix/src/util/tsd.h | 172 -- opal/mca/pmix/pmix2x/pmix/test/Makefile.am | 2 +- opal/mca/pmix/pmix2x/pmix/test/cli_stages.c | 25 +- opal/mca/pmix/pmix2x/pmix/test/cli_stages.h | 13 +- opal/mca/pmix/pmix2x/pmix/test/pmix_client.c | 2 +- opal/mca/pmix/pmix2x/pmix/test/pmix_regex.c | 2 +- opal/mca/pmix/pmix2x/pmix/test/pmix_test.c | 11 +- .../pmix/pmix2x/pmix/test/server_callbacks.c | 26 +- .../pmix/pmix2x/pmix/test/server_callbacks.h | 2 +- .../pmix/pmix2x/pmix/test/simple/Makefile.am | 35 +- .../pmix/pmix2x/pmix/test/simple/simpclient.c | 27 +- .../pmix/pmix2x/pmix/test/simple/simpdie.c | 8 +- .../pmix/pmix2x/pmix/test/simple/simpdmodex.c | 36 +- .../pmix/pmix2x/pmix/test/simple/simpdyn.c | 5 +- .../pmix/pmix2x/pmix/test/simple/simptest.c | 327 ++- .../pmix/pmix2x/pmix/test/simple/simptest.h | 53 - .../pmix2x/pmix/test/simple/simptimeout.c | 168 ++ .../pmix/pmix2x/pmix/test/simple/simptool.c | 3 +- .../pmix/pmix2x/pmix/test/simple/stability.c | 2 +- .../pmix/pmix2x/pmix/test/simple/test_pmix.c | 105 +- opal/mca/pmix/pmix2x/pmix/test/test_cd.c | 2 +- opal/mca/pmix/pmix2x/pmix/test/test_cd.h | 2 +- opal/mca/pmix/pmix2x/pmix/test/test_common.c | 6 +- opal/mca/pmix/pmix2x/pmix/test/test_common.h | 13 +- opal/mca/pmix/pmix2x/pmix/test/test_fence.c | 19 +- .../pmix2x/pmix/test/test_resolve_peers.c | 6 +- opal/mca/pmix/pmix2x/pmix/test/utils.c | 30 +- opal/mca/pmix/pmix2x/pmix/test/utils.h | 6 +- 298 files changed, 19382 insertions(+), 8400 deletions(-) create mode 100644 opal/mca/pmix/pmix2x/pmix/config/pmix_set_mca_prefix.m4 create mode 100644 opal/mca/pmix/pmix2x/pmix/config/pmix_setup_cli.m4 create mode 100644 opal/mca/pmix/pmix2x/pmix/config/pmix_summary.m4 create mode 100644 opal/mca/pmix/pmix2x/pmix/examples/examples.h create mode 100644 opal/mca/pmix/pmix2x/pmix/include/pmix_extend.h delete mode 100644 opal/mca/pmix/pmix2x/pmix/man/Makefile.am delete mode 100644 opal/mca/pmix/pmix2x/pmix/man/README delete mode 100644 opal/mca/pmix/pmix2x/pmix/man/man3/pmix_abort.3 delete mode 100644 opal/mca/pmix/pmix2x/pmix/man/man3/pmix_commit.3 delete mode 100644 opal/mca/pmix/pmix2x/pmix/man/man3/pmix_finalize.3 delete mode 100644 opal/mca/pmix/pmix2x/pmix/man/man3/pmix_init.3 delete mode 100644 opal/mca/pmix/pmix2x/pmix/man/man3/pmix_initialized.3 delete mode 100644 opal/mca/pmix/pmix2x/pmix/man/man3/pmix_put.3 delete mode 100644 opal/mca/pmix/pmix2x/pmix/man/man7/pmix.7 delete mode 100644 opal/mca/pmix/pmix2x/pmix/man/man7/pmix_constants.7 create mode 100644 opal/mca/pmix/pmix2x/pmix/src/atomics/sys/atomic_stdc.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/class/pmix_bitmap.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/class/pmix_bitmap.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/include/pmix_portable_platform.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/include/pmix_stdatomic.h rename opal/mca/pmix/pmix2x/pmix/src/mca/base/{help-mca-base.txt => help-pmix-mca-base.txt} (86%) rename opal/mca/pmix/pmix2x/pmix/src/mca/base/{help-mca-var.txt => help-pmix-mca-var.txt} (98%) create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_cmd_line.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/common/Makefile.am create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/Makefile.am create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/configure.m4 rename opal/mca/pmix/pmix2x/pmix/src/mca/{gds/ds12/gds_dstore.c => common/dstore/dstore_base.c} (55%) create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_base.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_common.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_file.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_segment.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_segment.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_base.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_base.h rename opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/{gds_dstore_component.c => gds_ds12_component.c} (90%) create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_file.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_file.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock_fcntl.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock_pthread.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds20_file.c delete mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_dstore.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/Makefile.am create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_base.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_base.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_component.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_file.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_file.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock_pthread.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/Makefile.am create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake_component.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/util/cmd_line.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/util/cmd_line.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/util/name_fns.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/util/name_fns.h delete mode 100644 opal/mca/pmix/pmix2x/pmix/src/util/tsd.h delete mode 100644 opal/mca/pmix/pmix2x/pmix/test/simple/simptest.h create mode 100644 opal/mca/pmix/pmix2x/pmix/test/simple/simptimeout.c diff --git a/opal/mca/pmix/pmix2x/pmix/INSTALL b/opal/mca/pmix/pmix2x/pmix/INSTALL index 08fdfe641ff..f7bac5fd197 100644 --- a/opal/mca/pmix/pmix2x/pmix/INSTALL +++ b/opal/mca/pmix/pmix2x/pmix/INSTALL @@ -9,7 +9,7 @@ Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, Copyright (c) 2004-2005 The Regents of the University of California. All rights reserved. Copyright (c) 2008-2015 Cisco Systems, Inc. All rights reserved. -Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +Copyright (c) 2013-2019 Intel, Inc. All rights reserved. $COPYRIGHT$ Additional copyrights may follow @@ -22,9 +22,9 @@ For More Information This file is a *very* short overview of building and installing the PMIx library. Much more information is available in the -FAQ section on the PMIx web site: +How-To section on the PMIx web site: - http://pmix.github.io/pmix/faq + https://pmix.org/support/how-to/ Developer Builds @@ -56,6 +56,12 @@ shell$ ./configure --prefix=/where/to/install [...lots of output...] shell$ make all install +NOTE: this version of PMIx requires the Libevent package to build +and operate. Any version of Libevent greater than or equal to +2.0.21 is acceptable. You must point configure at the libevent +installation using the --with-libevent= option if it is in +a non-standard location. + If you need special access to install, then you can execute "make all" as a user with write permissions in the build tree, and a separate "make install" as a user with write permissions to the diff --git a/opal/mca/pmix/pmix2x/pmix/LICENSE b/opal/mca/pmix/pmix2x/pmix/LICENSE index 06f1248793d..9f9a1b943ba 100644 --- a/opal/mca/pmix/pmix2x/pmix/LICENSE +++ b/opal/mca/pmix/pmix2x/pmix/LICENSE @@ -26,7 +26,7 @@ Copyright (c) 2006-2010 Sun Microsystems, Inc. All rights reserved. Copyright (c) 2006-2010 The University of Houston. All rights reserved. Copyright (c) 2006-2009 Myricom, Inc. All rights reserved. Copyright (c) 2007-2008 UT-Battelle, LLC. All rights reserved. -Copyright (c) 2007-2018 IBM Corporation. All rights reserved. +Copyright (c) 2007-2019 IBM Corporation. All rights reserved. Copyright (c) 1998-2005 Forschungszentrum Juelich, Juelich Supercomputing Centre, Federal Republic of Germany Copyright (c) 2005-2008 ZIH, TU Dresden, Federal Republic of Germany @@ -36,7 +36,7 @@ Copyright (c) 2008-2009 Institut National de Recherche en Informatique. All rights reserved. Copyright (c) 2007 Lawrence Livermore National Security, LLC. All rights reserved. -Copyright (c) 2007-2018 Mellanox Technologies. All rights reserved. +Copyright (c) 2007-2019 Mellanox Technologies. All rights reserved. Copyright (c) 2006-2010 QLogic Corporation. All rights reserved. Copyright (c) 2008-2010 Oak Ridge National Labs. All rights reserved. Copyright (c) 2006-2010 Oracle and/or its affiliates. All rights reserved. @@ -45,8 +45,10 @@ Copyright (c) 2010 ARM ltd. All rights reserved. Copyright (c) 2010-2011 Alex Brick . All rights reserved. Copyright (c) 2012 The University of Wisconsin-La Crosse. All rights reserved. -Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +Copyright (c) 2013-2019 Intel, Inc. All rights reserved. Copyright (c) 2011-2014 NVIDIA Corporation. All rights reserved. +Copyright (c) 2019 Amazon.com, Inc. or its affiliates. All Rights + reserved. $COPYRIGHT$ diff --git a/opal/mca/pmix/pmix2x/pmix/Makefile.am b/opal/mca/pmix/pmix2x/pmix/Makefile.am index 1c70624e913..47e21332730 100644 --- a/opal/mca/pmix/pmix2x/pmix/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/Makefile.am @@ -11,7 +11,9 @@ # All rights reserved. # Copyright (c) 2006-2016 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. +# Copyright (c) 2019 Amazon.com, Inc. or its affiliates. All Rights +# reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -25,6 +27,7 @@ ACLOCAL_AMFLAGS = -I ./config SUBDIRS = config contrib include src etc +AM_DISTCHECK_CONFIGURE_FLAGS = --disable-dlopen headers = sources = @@ -37,10 +40,6 @@ dist_pmixdata_DATA = if ! PMIX_EMBEDDED_MODE dist_pmixdata_DATA += contrib/pmix-valgrind.supp -if PMIX_HAVE_PANDOC -SUBDIRS += man -endif - endif if PMIX_TESTS_EXAMPLES diff --git a/opal/mca/pmix/pmix2x/pmix/NEWS b/opal/mca/pmix/pmix2x/pmix/NEWS index 12f61cccfd7..ad547b6a4a0 100644 --- a/opal/mca/pmix/pmix2x/pmix/NEWS +++ b/opal/mca/pmix/pmix2x/pmix/NEWS @@ -1,5 +1,5 @@ -Copyright (c) 2015-2018 Intel, Inc. All rights reserved. -Copyright (c) 2017 IBM Corporation. All rights reserved. +Copyright (c) 2015-2019 Intel, Inc. All rights reserved. +Copyright (c) 2017-2018 IBM Corporation. All rights reserved. $COPYRIGHT$ Additional copyrights may follow @@ -21,6 +21,80 @@ example, a bug might be fixed in the master, and then moved to the current release as well as the "stable" bug fix release branch. +2.2.3 -- TBD +---------------------- +- PR #1162: Fix race condition when clients fail while in a PMIx + collective operation +- PR #1163: Fix a compiler warning in atomics on POWER arch +- PR #1165: Add BuildRequires: libevent-devel to spec file +- PR #1180: Remove dependency on C++ compiler for thread detection +- PR #1180: Add detection for Flex when building in non-tarball situations +- PR #1182: configury: use PMIX_ENABLE_DLOPEN_SUPPORT instead of enable_dlopen +- PR #1188: Link libpmix.so to MCA component libraries +- PR #1190: Ensure any cached notifications arrive after reg completes +- PR #1194: Ensure any cached notifications arrive after registration completes +- PR #1209: Update configure logic for clock_gettime +- PR #1213/#1217/#1221: Add configure option "--enable-nonglobal-dlopen" + If the MCA component libraries should link back to libpmix.so +- PR #1231: SPEC: Allow splitting PMIx in pmix and pmix-libpmi packages +- PR #1253: Preserve user-provided CFLAGS to protect autotools +- PR #1267: Correct dmodex example +- PR #1275: IPv6 IF Read: Buffer Overflow +- PR #1295: Fix comment: IPv6 IF_NAMESIZE +- PR #1296: Allow setup_fork to proceed even if gdds and pnet don't contribute +- PR #1296: Allow servers to pass NULL module +- PR #1297: Provide internal blocking ability to the register/deregister fns +- PR #1298: Add dummy handshake component to psec framework for testing +- PR #1303: Allow jobs to not specify proc-level info +- PR #1304: Provide proc data in cases where host does not +- PR #1305: Add some more values that can be computed +- PR #1308: Add missing tool rendezvous file +- PR #1309: Fix potential integer overflow in regex +- PR #1311: Work around memory bug in older gcc compilers +- PR #1329: Add -fPIC to static builds +- PR #1334: Cache only -W CFLAG entries to fix 32-bit builds + + +2.2.2 -- 24 Jan 2019 +---------------------- + - Fix a bug in macro identifying system events + + +2.2.1 -- 18 Jan 2019 +---------------------- + - Fix a bug in registration of default event handlers + that somehow slipped thru testing + + +2.2.0 -- 17 Jan 2019 +---------------------- +**** THIS RELEASE MARKS THE STARTING POINT FOR FULL COMPLIANCE +**** WITH THE PMIX v2.2 STANDARD. ALL API BEHAVIORS AND ATTRIBUTE +**** DEFINITIONS MEET THE v2.2 STANDARD SPECIFICATIONS. + - Add a new, faster dstore GDS component 'ds21' + - Performance optimizations for the dstore GDS components. + - Plug miscellaneous memory leaks + - Silence an unnecessary warning message when checking connection + to a non-supporting server + - Ensure lost-connection events get delivered to default event + handlers + - Correctly handle cache refresh for queries + - Protect against race conditions between host and internal library + when dealing with async requests + - Cleanup tool operations and add support for connections to + remote servers. + - Automatically store the server URI for easy access by client + - Provide MCA parameter to control TCP connect retry/timeout + - Update event notification system to properly evict oldest events + when more space is needed + - Fix a number of error paths + - Handle setsockopt(SO_RCVTIMEO) not being supported + - Pass the tool's command line to the server so it can be returned + in queries + - Add support for C11 atomics + - Fix a bug when registering default event handlers + + 2.1.4 -- 18 Sep 2018 ---------------------- - Updated configury to silence warnings on older compilers diff --git a/opal/mca/pmix/pmix2x/pmix/README b/opal/mca/pmix/pmix2x/pmix/README index 22b0ec55bb8..90c5168b5dc 100644 --- a/opal/mca/pmix/pmix2x/pmix/README +++ b/opal/mca/pmix/pmix2x/pmix/README @@ -15,7 +15,7 @@ Copyright (c) 2007 Myricom, Inc. All rights reserved. Copyright (c) 2008 IBM Corporation. All rights reserved. Copyright (c) 2010 Oak Ridge National Labs. All rights reserved. Copyright (c) 2011 University of Houston. All rights reserved. -Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +Copyright (c) 2013-2019 Intel, Inc. All rights reserved. $COPYRIGHT$ Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/VERSION b/opal/mca/pmix/pmix2x/pmix/VERSION index bc222742467..fbb55b172c3 100644 --- a/opal/mca/pmix/pmix2x/pmix/VERSION +++ b/opal/mca/pmix/pmix2x/pmix/VERSION @@ -4,6 +4,7 @@ # Copyright (c) 2013 Mellanox Technologies, Inc. # All rights reserved. # Copyright (c) 2014-2016 Intel, Inc. All rights reserved. +# Copyright (c) 2018 IBM Corporation. All rights reserved. # This is the VERSION file for PMIx, describing the precise # version of PMIx in this distribution. The various components of @@ -14,8 +15,8 @@ # ... major=2 -minor=1 -release=4 +minor=2 +release=3 # greek is used for alpha or beta release tags. If it is non-empty, # it will be appended to the version number. It does not have to be @@ -23,14 +24,14 @@ release=4 # The only requirement is that it must be entirely printable ASCII # characters and have no white space. -greek= +greek=rc1 # If repo_rev is empty, then the repository version number will be # obtained during "make dist" via the "git describe --tags --always" # command, or with the date (if "git describe" fails) in the form of # "date". -repo_rev=git1d605654 +repo_rev=git8aac6645 # If tarball_version is not empty, it is used as the version string in # the tarball filename, regardless of all other versions listed in @@ -44,7 +45,7 @@ tarball_version= # The date when this release was created -date="Sep 18, 2018" +date="Jun 26, 2019" # The shared library version of each of PMIx's public libraries. # These versions are maintained in accordance with the "Library @@ -75,6 +76,13 @@ date="Sep 18, 2018" # Version numbers are described in the Libtool current:revision:age # format. -libpmix_so_version=3:14:1 +libpmix_so_version=3:22:1 libpmi_so_version=1:0:0 libpmi2_so_version=1:0:0 + +# "Common" components install standalone libraries that are run-time +# # linked by one or more components. So they need to be versioned as +# # well. Yuck; this somewhat breaks the +# # components-don't-affect-the-build-system abstraction. +# +libmca_common_dstore_so_version=1:0:0 diff --git a/opal/mca/pmix/pmix2x/pmix/config/distscript.sh b/opal/mca/pmix/pmix2x/pmix/config/distscript.sh index de41d2ba7b6..e5c948f15f1 100755 --- a/opal/mca/pmix/pmix2x/pmix/config/distscript.sh +++ b/opal/mca/pmix/pmix2x/pmix/config/distscript.sh @@ -11,11 +11,11 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2015 Research Organization for Information Science -# and Technology (RIST). All rights reserved. +# Copyright (c) 2015-2019 Research Organization for Information Science +# and Technology (RIST). All rights reserved. # Copyright (c) 2015 Los Alamos National Security, LLC. All rights # reserved. -# Copyright (c) 2017 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -42,7 +42,7 @@ fi # Otherwise, use what configure told us, at the cost of allowing one # or two corner cases in (but otherwise VPATH builds won't work). repo_rev=$PMIX_REPO_REV -if test -d .git ; then +if test -e .git ; then repo_rev=$(config/pmix_get_version.sh VERSION --repo-rev) fi diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix.m4 index 86c1fd6e9ad..ec9109c1aad 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix.m4 @@ -17,9 +17,9 @@ dnl Copyright (c) 2009 Los Alamos National Security, LLC. All rights dnl reserved. dnl Copyright (c) 2009-2011 Oak Ridge National Labs. All rights reserved. dnl Copyright (c) 2011-2013 NVIDIA Corporation. All rights reserved. -dnl Copyright (c) 2013-2017 Intel, Inc. All rights reserved. -dnl Copyright (c) 2015-2017 Research Organization for Information Science -dnl and Technology (RIST). All rights reserved. +dnl Copyright (c) 2013-2019 Intel, Inc. All rights reserved. +dnl Copyright (c) 2015-2019 Research Organization for Information Science +dnl and Technology (RIST). All rights reserved. dnl Copyright (c) 2016 Mellanox Technologies, Inc. dnl All rights reserved. dnl @@ -120,11 +120,31 @@ AC_DEFUN([PMIX_SETUP_CORE],[ pmixmajor=${PMIX_MAJOR_VERSION}L pmixminor=${PMIX_MINOR_VERSION}L pmixrelease=${PMIX_RELEASE_VERSION}L + pmixnumeric=$(printf 0x%4.4x%2.2x%2.2x $PMIX_MAJOR_VERSION $PMIX_MINOR_VERSION $PMIX_RELEASE_VERSION) AC_SUBST(pmixmajor) AC_SUBST(pmixminor) AC_SUBST(pmixrelease) + AC_SUBST(pmixnumeric) AC_CONFIG_FILES(pmix_config_prefix[include/pmix_version.h]) + PMIX_GREEK_VERSION="`$PMIX_top_srcdir/config/pmix_get_version.sh $PMIX_top_srcdir/VERSION --greek`" + if test "$?" != "0"; then + AC_MSG_ERROR([Cannot continue]) + fi + AC_SUBST(PMIX_GREEK_VERSION) + + PMIX_REPO_REV="`$PMIX_top_srcdir/config/pmix_get_version.sh $PMIX_top_srcdir/VERSION --repo-rev`" + if test "$?" != "0"; then + AC_MSG_ERROR([Cannot continue]) + fi + AC_SUBST(PMIX_REPO_REV) + + PMIX_RELEASE_DATE="`$PMIX_top_srcdir/config/pmix_get_version.sh $PMIX_top_srcdir/VERSION --release-date`" + if test "$?" != "0"; then + AC_MSG_ERROR([Cannot continue]) + fi + AC_SUBST(PMIX_RELEASE_DATE) + # Debug mode? AC_MSG_CHECKING([if want pmix maintainer support]) pmix_debug= @@ -171,12 +191,51 @@ AC_DEFUN([PMIX_SETUP_CORE],[ [Link the output PMIx library to this extra lib (used in embedded mode)])) AC_MSG_CHECKING([for extra lib]) AS_IF([test ! -z "$with_pmix_extra_lib"], - [AC_MSG_RESULT([$with_pmix_extra_lib]) - PMIX_EXTRA_LIB=$with_pmix_extra_lib], + [AS_IF([test "$with_pmix_extra_lib" == "yes" || test "$with_pmix_extra_lib" == "no"], + [AC_MSG_RESULT([ERROR]) + AC_MSG_WARN([Invalid value for --with-extra-pmix-lib:]) + AC_MSG_WARN([ $with_pmix_extra_lib]) + AC_MSG_WARN([Must be path name of the library to add]) + AC_MSG_ERROR([Cannot continue])], + [AC_MSG_RESULT([$with_pmix_extra_lib]) + PMIX_EXTRA_LIB=$with_pmix_extra_lib])], [AC_MSG_RESULT([no]) PMIX_EXTRA_LIB=]) AC_SUBST(PMIX_EXTRA_LIB) + # Add any extra libtool lib? + AC_ARG_WITH([pmix-extra-ltlib], + AC_HELP_STRING([--with-pmix-extra-ltlib=LIB], + [Link any embedded components/tools that require it to the provided libtool lib (used in embedded mode)])) + AC_MSG_CHECKING([for extra ltlib]) + AS_IF([test ! -z "$with_pmix_extra_ltlib"], + [AS_IF([test "$with_pmix_extra_ltlib" == "yes" || test "$with_pmix_extra_ltlib" == "no"], + [AC_MSG_RESULT([ERROR]) + AC_MSG_WARN([Invalid value for --with-pmix-extra-ltlib:]) + AC_MSG_WARN([ $with_pmix_extra_ltlib]) + AC_MSG_WARN([Must be path name of the library to add]) + AC_MSG_ERROR([Cannot continue])], + [AC_MSG_RESULT([$with_pmix_extra_ltlib]) + PMIX_EXTRA_LTLIB=$with_pmix_extra_ltlib])], + [AC_MSG_RESULT([no]) + PMIX_EXTRA_LTLIB=]) + AC_SUBST(PMIX_EXTRA_LTLIB) + + # + # Package/brand string + # + AC_MSG_CHECKING([if want package/brand string]) + AC_ARG_WITH([pmix-package-string], + [AC_HELP_STRING([--with-pmix-package-string=STRING], + [Use a branding string throughout PMIx])]) + if test "$with_pmix_package_string" = "" || test "$with_pmix_package_string" = "no"; then + with_package_string="PMIx $PMIX_CONFIGURE_USER@$PMIX_CONFIGURE_HOST Distribution" + fi + AC_DEFINE_UNQUOTED([PMIX_PACKAGE_STRING], ["$with_package_string"], + [package/branding string for PMIx]) + AC_MSG_RESULT([$with_package_string]) + + # GCC specifics. if test "x$GCC" = "xyes"; then PMIX_GCC_CFLAGS="-Wall -Wmissing-prototypes -Wundef" @@ -358,7 +417,8 @@ AC_DEFUN([PMIX_SETUP_CORE],[ crt_externs.h signal.h \ ioLib.h sockLib.h hostLib.h limits.h \ sys/statfs.h sys/statvfs.h \ - netdb.h ucred.h zlib.h]) + netdb.h ucred.h zlib.h sys/auxv.h \ + sys/sysctl.h]) AC_CHECK_HEADERS([sys/mount.h], [], [], [AC_INCLUDES_DEFAULT @@ -587,6 +647,11 @@ AC_DEFUN([PMIX_SETUP_CORE],[ pmix_show_title "Library and Function tests" + # Darwin doesn't need -lutil, as it's something other than this -lutil. + PMIX_SEARCH_LIBS_CORE([openpty], [util]) + + PMIX_SEARCH_LIBS_CORE([gethostbyname], [nsl]) + PMIX_SEARCH_LIBS_CORE([socket], [socket]) # IRIX and CentOS have dirname in -lgen, usually in libc @@ -595,7 +660,10 @@ AC_DEFUN([PMIX_SETUP_CORE],[ # Darwin doesn't need -lm, as it's a symlink to libSystem.dylib PMIX_SEARCH_LIBS_CORE([ceil], [m]) - AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf strsignal socketpair strncpy_s usleep statfs statvfs getpeereid getpeerucred strnlen posix_fallocate]) + # -lrt might be needed for clock_gettime + PMIX_SEARCH_LIBS_CORE([clock_gettime], [rt]) + + AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf strsignal socketpair strncpy_s usleep statfs statvfs getpeereid getpeerucred strnlen posix_fallocate tcgetpgrp]) # On some hosts, htonl is a define, so the AC_CHECK_FUNC will get # confused. On others, it's in the standard library, but stubbed with @@ -622,10 +690,6 @@ AC_DEFUN([PMIX_SETUP_CORE],[ [AC_DEFINE_UNQUOTED([HAVE_UNIX_BYTESWAP], [1], [whether unix byteswap routines -- htonl, htons, nothl, ntohs -- are available])]) - # check pandoc separately so we can setup an AM_CONDITIONAL off it - AC_CHECK_PROG([pmix_have_pandoc], [pandoc], [yes], [no]) - AM_CONDITIONAL([PMIX_HAVE_PANDOC], [test "x$pmix_have_pandoc" = "xyes"]) - # # Make sure we can copy va_lists (need check declared, not linkable) # @@ -661,8 +725,6 @@ AC_DEFUN([PMIX_SETUP_CORE],[ CFLAGS="$CFLAGS $THREAD_CFLAGS" CPPFLAGS="$CPPFLAGS $THREAD_CPPFLAGS" - CXXFLAGS="$CXXFLAGS $THREAD_CXXFLAGS" - CXXCPPFLAGS="$CXXCPPFLAGS $THREAD_CXXCPPFLAGS" LDFLAGS="$LDFLAGS $THREAD_LDFLAGS" LIBS="$LIBS $THREAD_LIBS" @@ -672,10 +734,10 @@ AC_DEFUN([PMIX_SETUP_CORE],[ AC_PROG_LN_S + # Check for some common system programs that we need AC_PROG_GREP AC_PROG_EGREP - ################################## # Visibility ################################## @@ -817,6 +879,10 @@ AC_DEFUN([PMIX_DEFINE_ARGS],[ [Whether build should attempt to use dlopen (or similar) to dynamically load components. (default: enabled)])]) + AS_IF([test "$enable_dlopen" = "unknown"], + [AC_MSG_WARN([enable_dlopen variable has been overwritten by configure]) + AC_MSG_WARN([This is an internal error that should be reported to PMIx developers]) + AC_MSG_ERROR([Cannot continue])]) AS_IF([test "$enable_dlopen" = "no"], [enable_mca_dso="no" enable_mca_static="yes" @@ -832,7 +898,7 @@ AC_DEFUN([PMIX_DEFINE_ARGS],[ AC_ARG_ENABLE([embedded-mode], [AC_HELP_STRING([--enable-embedded-mode], [Using --enable-embedded-mode causes PMIx to skip a few configure checks and install nothing. It should only be used when building PMIx within the scope of a larger package.])]) - AS_IF([test ! -z "$enable_embedded_mode" && test "$enable_embedded_mode" = "yes"], + AS_IF([test "$enable_embedded_mode" = "yes"], [pmix_mode=embedded pmix_install_primary_headers=no AC_MSG_RESULT([yes])], @@ -844,8 +910,16 @@ AC_DEFUN([PMIX_DEFINE_ARGS],[ # Is this a developer copy? # -if test -d .git; then +if test -e $PMIX_TOP_SRCDIR/.git; then PMIX_DEVEL=1 + # check for Flex + AC_PROG_LEX + if test "x$LEX" != xflex; then + AC_MSG_WARN([PMIx requires Flex to build from non-tarball sources,]) + AC_MSG_WARN([but Flex was not found. Please install Flex into]) + AC_MSG_WARN([your path and try again]) + AC_MSG_ERROR([Cannot continue]) + fi else PMIX_DEVEL=0 fi @@ -896,7 +970,6 @@ fi #################### Early development override #################### if test "$WANT_DEBUG" = "0"; then CFLAGS="-DNDEBUG $CFLAGS" - CXXFLAGS="-DNDEBUG $CXXFLAGS" fi AC_DEFINE_UNQUOTED(PMIX_ENABLE_DEBUG, $WANT_DEBUG, [Whether we want developer-level debugging code or not]) @@ -1033,20 +1106,6 @@ AC_DEFINE_UNQUOTED([PMIX_ENABLE_TIMING], [$WANT_PMIX_TIMING], [Whether we want developer-level timing support or not]) # -# Install header files -# -AC_MSG_CHECKING([if want to head developer-level header files]) -AC_ARG_WITH(devel-headers, - AC_HELP_STRING([--with-devel-headers], - [also install developer-level header files (only for internal PMIx developers, default: disabled)])) -if test "$with_devel_headers" = "yes"; then - AC_MSG_RESULT([yes]) - WANT_INSTALL_HEADERS=1 -else - AC_MSG_RESULT([no]) - WANT_INSTALL_HEADERS=0 -fi - # # Install backward compatibility support for PMI-1 and PMI-2 # @@ -1063,6 +1122,59 @@ else fi AM_CONDITIONAL([WANT_INSTALL_HEADERS], [test $WANT_INSTALL_HEADERS -eq 1]) + +# +# Do we want to install binaries? +# +AC_MSG_CHECKING([if want to disable binaries]) +AC_ARG_ENABLE(pmix-binaries, + AC_HELP_STRING([--enable-pmix-binaries], + [enable PMIx tools])) +if test "$enable_pmix_binaries" = "no"; then + AC_MSG_RESULT([no]) + WANT_PMIX_BINARIES=0 +else + AC_MSG_RESULT([yes]) + WANT_PMIX_BINARIES=1 +fi + +AM_CONDITIONAL([PMIX_INSTALL_BINARIES], [test $WANT_PMIX_BINARIES -eq 1]) + +# +# psec/dummy_handshake +# + +AC_MSG_CHECKING([if want build psec/dummy_handshake]) +AC_ARG_ENABLE(dummy-handshake, + AC_HELP_STRING([--enable-dummy-handshake], + [Enables psec dummy component intended to check the PTL handshake scenario (default: disabled)])) +if test "$enable_dummy_handshake" != "yes"; then + AC_MSG_RESULT([no]) + eval "DISABLE_psec_dummy_handshake=1" +else + AC_MSG_RESULT([yes]) + eval "DISABLE_psec_dummy_handshake=0" +fi +AM_CONDITIONAL(MCA_BUILD_PSEC_DUMMY_HANDSHAKE, test "$DISABLE_psec_dummy_handshake" = "0") + +# see if they want to disable non-RTLD_GLOBAL dlopen +AC_MSG_CHECKING([if want to support dlopen of non-global namespaces]) +AC_ARG_ENABLE([nonglobal-dlopen], + AC_HELP_STRING([--enable-nonglobal-dlopen], + [enable non-global dlopen (default: enabled)])) +if test "$enable_nonglobal_dlopen" == "no"; then + AC_MSG_RESULT([no]) + pmix_need_libpmix=0 +else + AC_MSG_RESULT([yes]) + pmix_need_libpmix=1 +fi + +# if someone enables embedded mode but doesn't want to install the +# devel headers, then default nonglobal-dlopen to false +AS_IF([test -z "$enable_nonglobal_dlopen" && test "x$pmix_mode" = "xembedded" && test $WANT_INSTALL_HEADERS -eq 0 && test $pmix_need_libpmix -eq 1], + [pmix_need_libpmix=0]) + ])dnl # This must be a standalone routine so that it can be called both by @@ -1078,6 +1190,7 @@ AC_DEFUN([PMIX_DO_AM_CONDITIONALS],[ AM_CONDITIONAL([WANT_PRIMARY_HEADERS], [test "x$pmix_install_primary_headers" = "xyes"]) AM_CONDITIONAL(WANT_INSTALL_HEADERS, test "$WANT_INSTALL_HEADERS" = 1) AM_CONDITIONAL(WANT_PMI_BACKWARD, test "$WANT_PMI_BACKWARD" = 1) + AM_CONDITIONAL(NEED_LIBPMIX, [test "$pmix_need_libpmix" = "1"]) ]) pmix_did_am_conditionals=yes ])dnl diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_check_compiler_version.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_compiler_version.m4 index 78343b9126d..f918d7b590b 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix_check_compiler_version.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_compiler_version.m4 @@ -1,7 +1,7 @@ dnl -*- shell-script -*- dnl dnl Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. -dnl Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +dnl Copyright (c) 2013-2019 Intel, Inc. All rights reserved. dnl dnl $COPYRIGHT$ dnl @@ -22,7 +22,7 @@ AC_DEFUN([PMIX_CHECK_COMPILER_VERSION_ID], PMIX_CHECK_COMPILER(FAMILYID) PMIX_CHECK_COMPILER_STRINGIFY(FAMILYNAME) PMIX_CHECK_COMPILER(VERSION) - PMIX_CHECK_COMPILER_STRINGIFY(VERSION_STR) + PMIX_CHECK_COMPILER_STRING(VERSION_STR) ])dnl @@ -31,10 +31,11 @@ AC_DEFUN([PMIX_CHECK_COMPILER], [ AC_CACHE_CHECK([for compiler $lower], pmix_cv_compiler_[$1], [ CPPFLAGS_orig=$CPPFLAGS - CPPFLAGS="-I${top_pmix_srcdir}/src/include $CPPFLAGS" + CPPFLAGS="-I${top_srcdir}/src/include $CPPFLAGS" AC_TRY_RUN([ #include #include +#include "pmix_portable_platform.h" int main (int argc, char * argv[]) { @@ -57,16 +58,51 @@ int main (int argc, char * argv[]) [The compiler $lower which PMIx was built with]) ])dnl +AC_DEFUN([PMIX_CHECK_COMPILER_STRING], [ + lower=m4_tolower($1) + AC_CACHE_CHECK([for compiler $lower], pmix_cv_compiler_[$1], + [ + CPPFLAGS_orig=$CPPFLAGS + CPPFLAGS="-I${top_srcdir}/src/include $CPPFLAGS" + AC_TRY_RUN([ +#include +#include +#include "pmix_portable_platform.h" + +int main (int argc, char * argv[]) +{ + FILE * f; + f=fopen("conftestval", "w"); + if (!f) exit(1); + fprintf (f, "%s", PLATFORM_COMPILER_$1); + return 0; +} + ], [ + eval pmix_cv_compiler_$1=`cat conftestval`; + ], [ + eval pmix_cv_compiler_$1=UNKNOWN + ], [ + eval pmix_cv_compiler_$1=UNKNOWN + ]) + CPPFLAGS=$CPPFLAGS_orig + ]) + AC_DEFINE_UNQUOTED([PMIX_BUILD_PLATFORM_COMPILER_$1], $pmix_cv_compiler_[$1], + [The compiler $lower which PMIx was built with]) +])dnl + + + AC_DEFUN([PMIX_CHECK_COMPILER_STRINGIFY], [ lower=m4_tolower($1) AC_CACHE_CHECK([for compiler $lower], pmix_cv_compiler_[$1], [ CPPFLAGS_orig=$CPPFLAGS - CPPFLAGS="-I${top_pmix_srcdir}/src/include $CPPFLAGS" + CPPFLAGS="-I${top_srcdir}/src/include $CPPFLAGS" AC_TRY_RUN([ #include #include +#include "pmix_portable_platform.h" int main (int argc, char * argv[]) { diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_check_lock.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_lock.m4 index 69f184506c4..3a3e28d21ef 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix_check_lock.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_lock.m4 @@ -5,7 +5,7 @@ dnl All rights reserved. dnl Copyright (c) 2017 IBM Corporation. All rights reserved. dnl Copyright (c) 2017 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. -dnl Copyright (c) 2017 Intel, Inc. All rights reserved. +dnl Copyright (c) 2017-2019 Intel, Inc. All rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow @@ -57,4 +57,6 @@ AC_DEFUN([PMIX_CHECK_DSTOR_LOCK],[ fi fi LIBS="$orig_libs" + AM_CONDITIONAL([HAVE_DSTORE_PTHREAD_LOCK], [test "$_x_ac_pthread_lock_found" = "1"]) + AM_CONDITIONAL([HAVE_DSTORE_FCNTL_LOCK], [test "$_x_ac_fcntl_lock_found" = "1"]) ]) diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_check_os_flavors.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_os_flavors.m4 index 5c333ff5111..fa08cf906c5 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix_check_os_flavors.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_os_flavors.m4 @@ -1,7 +1,7 @@ dnl -*- shell-script -*- dnl dnl Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. -dnl Copyright (c) 2014-2016 Intel, Inc. All rights reserved. +dnl Copyright (c) 2014-2019 Intel, Inc. All rights reserved. dnl Copyright (c) 2014 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. dnl @@ -57,6 +57,13 @@ AC_DEFUN([PMIX_CHECK_OS_FLAVORS], [$pmix_have_solaris], [Whether or not we have solaris]) + AS_IF([test "$pmix_found_apple" = "yes"], + [pmix_have_apple=1], + [pmix_have_apple=0]) + AC_DEFINE_UNQUOTED([PMIX_HAVE_APPLE], + [$pmix_have_apple], + [Whether or not we have apple]) + # check for sockaddr_in (a good sign we have TCP) AC_CHECK_HEADERS([netdb.h netinet/in.h netinet/tcp.h]) AC_CHECK_TYPES([struct sockaddr_in], diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_check_package.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_package.m4 index babb542cde1..93b646add76 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix_check_package.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_package.m4 @@ -12,7 +12,7 @@ # All rights reserved. # Copyright (c) 2012-2015 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012 Oracle and/or its affiliates. All rights reserved. -# Copyright (c) 2014 Intel, Inc. All rights reserved. +# Copyright (c) 2014-2019 Intel, Inc. All rights reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. # $COPYRIGHT$ @@ -36,9 +36,14 @@ AC_DEFUN([_PMIX_CHECK_PACKAGE_HEADER], [ # of header includes without killing off the cache variable and trying # again... unset pmix_Header - pmix_check_package_header_happy="no" - AS_IF([test "$3" = "/usr" || test "$3" = "/usr/local"], + + # get rid of the trailing slash(es) + hdir_prefix=$(echo $3 | sed -e 'sX/*$XXg') + + AS_IF([test "$hdir_prefix" = "" || \ + test "$hdir_prefix" = "/usr" || \ + test "$hdir_prefix" = "/usr/local"], [ # try as is... AC_VERBOSE([looking for header without includes]) AC_CHECK_HEADERS([$2], [pmix_check_package_header_happy="yes"], []) @@ -47,12 +52,21 @@ AC_DEFUN([_PMIX_CHECK_PACKAGE_HEADER], [ unset pmix_Header])]) AS_IF([test "$pmix_check_package_header_happy" = "no"], - [AS_IF([test "$3" != ""], - [$1_CPPFLAGS="$$1_CPPFLAGS -I$3/include" - CPPFLAGS="$CPPFLAGS -I$3/include"]) - AC_CHECK_HEADERS([$2], [pmix_check_package_header_happy="yes"], [], [$6]) - AS_IF([test "$pmix_check_package_header_happy" = "yes"], [$4], [$5])], - [$4]) + [AS_IF([test "$hdir_prefix" != ""], + [$1_CPPFLAGS="$$1_CPPFLAGS -I$hdir_prefix" + CPPFLAGS="$CPPFLAGS -I$hdir_prefix" + AC_VERBOSE([looking for header in $hdir_prefix]) + AC_CHECK_HEADERS([$2], [pmix_check_package_header_happy="yes"], [], [$6]) + AS_IF([test "$pmix_check_package_header_happy" = "no"], + [unset pmix_Header + $1_CPPFLAGS="$$1_CPPFLAGS -I$hdir_prefix/include" + CPPFLAGS="$CPPFLAGS -I$hdir_prefix/include" + AC_VERBOSE([looking for header in $hdir_prefix/include]) + AC_CHECK_HEADERS([$2], [pmix_check_package_header_happy="yes"], [], [$6])])])]) + + AS_IF([test "$pmix_check_package_header_happy" = "yes"], + [$4], [$5]) + unset pmix_check_package_header_happy AS_VAR_POPDEF([pmix_Header])dnl @@ -74,10 +88,14 @@ AC_DEFUN([_PMIX_CHECK_PACKAGE_LIB], [ # see comment above unset pmix_Lib pmix_check_package_lib_happy="no" - AS_IF([test "$6" != ""], - [ # libdir was specified - search only there - $1_LDFLAGS="$$1_LDFLAGS -L$6" - LDFLAGS="$LDFLAGS -L$6" + + # get rid of the trailing slash(es) + libdir_prefix=$(echo $6 | sed -e 'sX/*$XXg') + + AS_IF([test "$libdir_prefix" != ""], + [# libdir was specified - search only there + $1_LDFLAGS="$$1_LDFLAGS -L$libdir_prefix" + LDFLAGS="$LDFLAGS -L$libdir_prefix" AC_SEARCH_LIBS([$3], [$2], [pmix_check_package_lib_happy="yes"], [pmix_check_package_lib_happy="no"], [$4]) @@ -86,24 +104,30 @@ AC_DEFUN([_PMIX_CHECK_PACKAGE_LIB], [ $1_LDFLAGS="$pmix_check_package_$1_orig_LDFLAGS" unset pmix_Lib])], [ # libdir was not specified - go through search path - pmix_check_package_libdir="$5" - AS_IF([test "$pmix_check_package_libdir" = "" || test "$pmix_check_package_libdir" = "/usr" || test "$pmix_check_package_libdir" = "/usr/local"], - [ # try as is... - AC_VERBOSE([looking for library without search path]) - AC_SEARCH_LIBS([$3], [$2], - [pmix_check_package_lib_happy="yes"], - [pmix_check_package_lib_happy="no"], [$4]) - AS_IF([test "$pmix_check_package_lib_happy" = "no"], - [ # no go on the as is.. see what happens later... - LDFLAGS="$pmix_check_package_$1_save_LDFLAGS" - $1_LDFLAGS="$pmix_check_package_$1_orig_LDFLAGS" - unset pmix_Lib])]) + # get rid of the trailing slash(es) + libdir_prefix=$(echo $5 | sed -e 'sX/*$XXg') + + # first try standard locations as otherwise our + # searches with libdir_prefix locations might come + # back positive and unnecessarily add an LDFLAG + AC_VERBOSE([looking for library without search path]) + AC_SEARCH_LIBS([$3], [$2], + [pmix_check_package_lib_happy="yes"], + [pmix_check_package_lib_happy="no"], [$4]) + AS_IF([test "$pmix_check_package_lib_happy" = "no"], + [ # no go on the as is.. see what happens later... + LDFLAGS="$pmix_check_package_$1_save_LDFLAGS" + $1_LDFLAGS="$pmix_check_package_$1_orig_LDFLAGS" + unset pmix_Lib]) AS_IF([test "$pmix_check_package_lib_happy" = "no"], - [AS_IF([test "$pmix_check_package_libdir" != ""], - [$1_LDFLAGS="$$1_LDFLAGS -L$pmix_check_package_libdir/lib" - LDFLAGS="$LDFLAGS -L$pmix_check_package_libdir/lib" - AC_VERBOSE([looking for library in lib]) + # if we didn't find it, check the libdir_prefix/lib64 directory + [AS_IF([test "$libdir_prefix" != "" && \ + test "$libdir_prefix" != "/usr" && \ + test "$libdir_prefix" != "/usr/local"], + [$1_LDFLAGS="$$1_LDFLAGS -L$libdir_prefix/lib64" + LDFLAGS="$LDFLAGS -L$libdir_prefix/lib64" + AC_VERBOSE([looking for library in $libdir_prefix/lib64]) AC_SEARCH_LIBS([$3], [$2], [pmix_check_package_lib_happy="yes"], [pmix_check_package_lib_happy="no"], [$4]) @@ -114,10 +138,13 @@ AC_DEFUN([_PMIX_CHECK_PACKAGE_LIB], [ unset pmix_Lib])])]) AS_IF([test "$pmix_check_package_lib_happy" = "no"], - [AS_IF([test "$pmix_check_package_libdir" != ""], - [$1_LDFLAGS="$$1_LDFLAGS -L$pmix_check_package_libdir/lib64" - LDFLAGS="$LDFLAGS -L$pmix_check_package_libdir/lib64" - AC_VERBOSE([looking for library in lib64]) + # if we still haven't found it, check the libdir_prefix/lib directory + [AS_IF([test "$libdir_prefix" != "" && \ + test "$libdir_prefix" != "/usr" && \ + test "$libdir_prefix" != "/usr/local"], + [$1_LDFLAGS="$$1_LDFLAGS -L$libdir_prefix/lib" + LDFLAGS="$LDFLAGS -L$libdir_prefix/lib" + AC_VERBOSE([looking for library in $libdir_prefix/lib]) AC_SEARCH_LIBS([$3], [$2], [pmix_check_package_lib_happy="yes"], [pmix_check_package_lib_happy="no"], [$4]) @@ -125,7 +152,9 @@ AC_DEFUN([_PMIX_CHECK_PACKAGE_LIB], [ [ # no go on the as is.. see what happens later... LDFLAGS="$pmix_check_package_$1_save_LDFLAGS" $1_LDFLAGS="$pmix_check_package_$1_orig_LDFLAGS" - unset pmix_Lib])])])]) + unset pmix_Lib])])]) + ]) + AS_IF([test "$pmix_check_package_lib_happy" = "yes"], [$1_LIBS="-l$2 $4" diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_config_asm.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_config_asm.m4 index 062440499a6..a46b73ad4d4 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix_config_asm.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_config_asm.m4 @@ -2,22 +2,22 @@ dnl dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana dnl University Research and Technology dnl Corporation. All rights reserved. -dnl Copyright (c) 2004-2015 The University of Tennessee and The University +dnl Copyright (c) 2004-2018 The University of Tennessee and The University dnl of Tennessee Research Foundation. All rights dnl reserved. dnl Copyright (c) 2004-2006 High Performance Computing Center Stuttgart, dnl University of Stuttgart. All rights reserved. dnl Copyright (c) 2004-2005 The Regents of the University of California. dnl All rights reserved. -dnl Copyright (c) 2008-2015 Cisco Systems, Inc. All rights reserved. +dnl Copyright (c) 2008-2018 Cisco Systems, Inc. All rights reserved. dnl Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. dnl Copyright (c) 2015-2017 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. -dnl Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights +dnl Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights dnl reserved. dnl Copyright (c) 2017 Amazon.com, Inc. or its affiliates. All Rights dnl reserved. -dnl Copyright (c) 2017 Intel, Inc. All rights reserved. +dnl Copyright (c) 2018-2019 Intel, Inc. All rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow @@ -25,65 +25,262 @@ dnl dnl $HEADER$ dnl +dnl This is a C test to see if 128-bit __atomic_compare_exchange_n() +dnl actually works (e.g., it compiles and links successfully on +dnl ARM64+clang, but returns incorrect answers as of August 2018). +AC_DEFUN([PMIX_ATOMIC_COMPARE_EXCHANGE_N_TEST_SOURCE],[[ +#include +#include +#include -AC_DEFUN([PMIX_CHECK_SYNC_BUILTIN_CSWAP_INT128], [ +typedef union { + uint64_t fake@<:@2@:>@; + __int128 real; +} pmix128; + +static void test1(void) +{ + // As of Aug 2018, we could not figure out a way to assign 128-bit + // constants -- the compilers would not accept it. So use a fake + // union to assign 2 uin64_t's to make a single __int128. + pmix128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + pmix128 expected = { .fake = { 0x11EEDDCCBBAA0099, 0x88776655443322FF }}; + pmix128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = __atomic_compare_exchange_n(&ptr.real, &expected.real, + desired.real, true, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + if ( !(r == false && ptr.real == expected.real)) { + exit(1); + } +} + +static void test2(void) +{ + pmix128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + pmix128 expected = ptr; + pmix128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = __atomic_compare_exchange_n(&ptr.real, &expected.real, + desired.real, true, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + if (!(r == true && ptr.real == desired.real)) { + exit(2); + } +} + +int main(int argc, char** argv) +{ + test1(); + test2(); + return 0; +} +]]) + +dnl ------------------------------------------------------------------ + +dnl This is a C test to see if 128-bit __sync_bool_compare_and_swap() +dnl actually works (e.g., it compiles and links successfully on +dnl ARM64+clang, but returns incorrect answers as of August 2018). +AC_DEFUN([PMIX_SYNC_BOOL_COMPARE_AND_SWAP_TEST_SOURCE],[[ +#include +#include +#include + +typedef union { + uint64_t fake@<:@2@:>@; + __int128 real; +} pmix128; + +static void test1(void) +{ + // As of Aug 2018, we could not figure out a way to assign 128-bit + // constants -- the compilers would not accept it. So use a fake + // union to assign 2 uin64_t's to make a single __int128. + pmix128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + pmix128 oldval = { .fake = { 0x11EEDDCCBBAA0099, 0x88776655443322FF }}; + pmix128 newval = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = __sync_bool_compare_and_swap(&ptr.real, oldval.real, newval.real); + if (!(r == false && ptr.real != newval.real)) { + exit(1); + } +} + +static void test2(void) +{ + pmix128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + pmix128 oldval = ptr; + pmix128 newval = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = __sync_bool_compare_and_swap(&ptr.real, oldval.real, newval.real); + if (!(r == true && ptr.real == newval.real)) { + exit(2); + } +} + +int main(int argc, char** argv) +{ + test1(); + test2(); + return 0; +} +]]) - PMIX_VAR_SCOPE_PUSH([sync_bool_compare_and_swap_128_result CFLAGS_save]) +dnl This is a C test to see if 128-bit __atomic_compare_exchange_n() +dnl actually works (e.g., it compiles and links successfully on +dnl ARM64+clang, but returns incorrect answers as of August 2018). +AC_DEFUN([PMIX_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE],[[ +#include +#include +#include +#include - AC_ARG_ENABLE([cross-cmpset128],[AC_HELP_STRING([--enable-cross-cmpset128], - [enable the use of the __sync builtin atomic compare-and-swap 128 when cross compiling])]) +typedef union { + uint64_t fake@<:@2@:>@; + _Atomic __int128 real; +} pmix128; - sync_bool_compare_and_swap_128_result=0 +static void test1(void) +{ + // As of Aug 2018, we could not figure out a way to assign 128-bit + // constants -- the compilers would not accept it. So use a fake + // union to assign 2 uin64_t's to make a single __int128. + pmix128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + pmix128 expected = { .fake = { 0x11EEDDCCBBAA0099, 0x88776655443322FF }}; + pmix128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real, + desired.real, true, + atomic_relaxed, atomic_relaxed); + if ( !(r == false && ptr.real == expected.real)) { + exit(1); + } +} - if test ! "$enable_cross_cmpset128" = "yes" ; then - AC_MSG_CHECKING([for processor support of __sync builtin atomic compare-and-swap on 128-bit values]) +static void test2(void) +{ + pmix128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + pmix128 expected = ptr; + pmix128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real, + desired.real, true, + atomic_relaxed, atomic_relaxed); + if (!(r == true && ptr.real == desired.real)) { + exit(2); + } +} - AC_RUN_IFELSE([AC_LANG_PROGRAM([], [__int128 x = 0; __sync_bool_compare_and_swap (&x, 0, 1);])], - [AC_MSG_RESULT([yes]) - sync_bool_compare_and_swap_128_result=1], - [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([no (cross compiling)])]) +int main(int argc, char** argv) +{ + test1(); + test2(); + return 0; +} +]]) - if test $sync_bool_compare_and_swap_128_result = 0 ; then - CFLAGS_save=$CFLAGS - CFLAGS="$CFLAGS -mcx16" +dnl ------------------------------------------------------------------ - AC_MSG_CHECKING([for __sync builtin atomic compare-and-swap on 128-bit values with -mcx16 flag]) - AC_RUN_IFELSE([AC_LANG_PROGRAM([], [__int128 x = 0; __sync_bool_compare_and_swap (&x, 0, 1);])], - [AC_MSG_RESULT([yes]) - sync_bool_compare_and_swap_128_result=1 - CFLAGS_save="$CFLAGS"], - [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([no (cross compiling)])]) +dnl +dnl Check to see if a specific function is linkable. +dnl +dnl Check with: +dnl 1. No compiler/linker flags. +dnl 2. CFLAGS += -mcx16 +dnl 3. LIBS += -latomic +dnl 4. Finally, if it links ok with any of #1, #2, or #3, actually try +dnl to run the test code (if we're not cross-compiling) and verify +dnl that it actually gives us the correct result. +dnl +dnl Note that we unfortunately can't use AC SEARCH_LIBS because its +dnl check incorrectly fails (because these functions are special compiler +dnl intrinsics -- SEARCH_LIBS tries with "check FUNC()", which the +dnl compiler complains doesn't match the internal prototype). So we have +dnl to use our own LINK_IFELSE tests. Indeed, since these functions are +dnl so special, we actually need a valid source code that calls the +dnl functions with correct arguments, etc. It's not enough, for example, +dnl to do the usual "try to set a function pointer to the symbol" trick to +dnl determine if these functions are available, because the compiler may +dnl not implement these as actual symbols. So just try to link a real +dnl test code. +dnl +dnl $1: function name to print +dnl $2: program to test +dnl $3: action if any of 1, 2, or 3 succeeds +dnl #4: action if all of 1, 2, and 3 fail +dnl +AC_DEFUN([PMIX_ASM_CHECK_ATOMIC_FUNC],[ + PMIX_VAR_SCOPE_PUSH([pmix_asm_check_func_happy pmix_asm_check_func_CFLAGS_save pmix_asm_check_func_LIBS_save]) + + pmix_asm_check_func_CFLAGS_save=$CFLAGS + pmix_asm_check_func_LIBS_save=$LIBS + + dnl Check with no compiler/linker flags + AC_MSG_CHECKING([for $1]) + AC_LINK_IFELSE([$2], + [pmix_asm_check_func_happy=1 + AC_MSG_RESULT([yes])], + [pmix_asm_check_func_happy=0 + AC_MSG_RESULT([no])]) + + dnl If that didn't work, try again with CFLAGS+=mcx16 + AS_IF([test $pmix_asm_check_func_happy -eq 0], + [AC_MSG_CHECKING([for $1 with -mcx16]) + CFLAGS="$CFLAGS -mcx16" + AC_LINK_IFELSE([$2], + [pmix_asm_check_func_happy=1 + AC_MSG_RESULT([yes])], + [pmix_asm_check_func_happy=0 + CFLAGS=$pmix_asm_check_func_CFLAGS_save + AC_MSG_RESULT([no])]) + ]) + + dnl If that didn't work, try again with LIBS+=-latomic + AS_IF([test $pmix_asm_check_func_happy -eq 0], + [AC_MSG_CHECKING([for $1 with -latomic]) + LIBS="$LIBS -latomic" + AC_LINK_IFELSE([$2], + [pmix_asm_check_func_happy=1 + AC_MSG_RESULT([yes])], + [pmix_asm_check_func_happy=0 + LIBS=$pmix_asm_check_func_LIBS_save + AC_MSG_RESULT([no])]) + ]) + + dnl If we have it, try it and make sure it gives a correct result. + dnl As of Aug 2018, we know that it links but does *not* work on clang + dnl 6 on ARM64. + AS_IF([test $pmix_asm_check_func_happy -eq 1], + [AC_MSG_CHECKING([if $1() gives correct results]) + AC_RUN_IFELSE([$2], + [AC_MSG_RESULT([yes])], + [pmix_asm_check_func_happy=0 + AC_MSG_RESULT([no])], + [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])]) + ]) - CFLAGS=$CFLAGS_save - fi - else - AC_MSG_CHECKING([for compiler support of __sync builtin atomic compare-and-swap on 128-bit values]) + dnl If we were unsuccessful, restore CFLAGS/LIBS + AS_IF([test $pmix_asm_check_func_happy -eq 0], + [CFLAGS=$pmix_asm_check_func_CFLAGS_save + LIBS=$pmix_asm_check_func_LIBS_save]) - # Check if the compiler supports the __sync builtin - AC_TRY_LINK([], [__int128 x = 0; __sync_bool_compare_and_swap (&x, 0, 1);], - [AC_MSG_RESULT([yes]) - sync_bool_compare_and_swap_128_result=1], - [AC_MSG_RESULT([no])]) + dnl Run the user actions + AS_IF([test $pmix_asm_check_func_happy -eq 1], [$3], [$4]) - if test $sync_bool_compare_and_swap_128_result = 0 ; then - CFLAGS_save=$CFLAGS - CFLAGS="$CFLAGS -mcx16" + PMIX_VAR_SCOPE_POP +]) + +dnl ------------------------------------------------------------------ - AC_MSG_CHECKING([for __sync builtin atomic compare-and-swap on 128-bit values with -mcx16 flag]) - AC_TRY_LINK([], [__int128 x = 0; __sync_bool_compare_and_swap (&x, 0, 1);], - [AC_MSG_RESULT([yes]) - sync_bool_compare_and_swap_128_result=1 - CFLAGS_save="$CFLAGS"], - [AC_MSG_RESULT([no])]) +AC_DEFUN([PMIX_CHECK_SYNC_BUILTIN_CSWAP_INT128], [ + PMIX_VAR_SCOPE_PUSH([sync_bool_compare_and_swap_128_result]) - CFLAGS=$CFLAGS_save - fi - fi + # Do we have __sync_bool_compare_and_swap? + # Use a special macro because we need to check with a few different + # CFLAGS/LIBS. + PMIX_ASM_CHECK_ATOMIC_FUNC([__sync_bool_compare_and_swap], + [AC_LANG_SOURCE(PMIX_SYNC_BOOL_COMPARE_AND_SWAP_TEST_SOURCE)], + [sync_bool_compare_and_swap_128_result=1], + [sync_bool_compare_and_swap_128_result=0]) - AC_DEFINE_UNQUOTED([PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128], [$sync_bool_compare_and_swap_128_result], - [Whether the __sync builtin atomic compare and swap supports 128-bit values]) + AC_DEFINE_UNQUOTED([PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128], + [$sync_bool_compare_and_swap_128_result], + [Whether the __sync builtin atomic compare and swap supports 128-bit values]) PMIX_VAR_SCOPE_POP ]) @@ -112,7 +309,7 @@ __sync_add_and_fetch(&tmp, 1);], pmix_asm_sync_have_64bit=0]) AC_DEFINE_UNQUOTED([PMIX_ASM_SYNC_HAVE_64BIT],[$pmix_asm_sync_have_64bit], - [Whether 64-bit is supported by the __sync builtin atomics]) + [Whether 64-bit is supported by the __sync builtin atomics]) # Check for 128-bit support PMIX_CHECK_SYNC_BUILTIN_CSWAP_INT128 @@ -120,73 +317,110 @@ __sync_add_and_fetch(&tmp, 1);], AC_DEFUN([PMIX_CHECK_GCC_BUILTIN_CSWAP_INT128], [ + PMIX_VAR_SCOPE_PUSH([atomic_compare_exchange_n_128_result atomic_compare_exchange_n_128_CFLAGS_save atomic_compare_exchange_n_128_LIBS_save]) + + atomic_compare_exchange_n_128_CFLAGS_save=$CFLAGS + atomic_compare_exchange_n_128_LIBS_save=$LIBS + + # Do we have __sync_bool_compare_and_swap? + # Use a special macro because we need to check with a few different + # CFLAGS/LIBS. + PMIX_ASM_CHECK_ATOMIC_FUNC([__atomic_compare_exchange_n], + [AC_LANG_SOURCE(PMIX_ATOMIC_COMPARE_EXCHANGE_N_TEST_SOURCE)], + [atomic_compare_exchange_n_128_result=1], + [atomic_compare_exchange_n_128_result=0]) + + # If we have it and it works, check to make sure it is always lock + # free. + AS_IF([test $atomic_compare_exchange_n_128_result -eq 1], + [AC_MSG_CHECKING([if __int128 atomic compare-and-swap is always lock-free]) + AC_RUN_IFELSE([AC_LANG_PROGRAM([], [if (!__atomic_always_lock_free(16, 0)) { return 1; }])], + [AC_MSG_RESULT([yes])], + [atomic_compare_exchange_n_128_result=0 + # If this test fails, need to reset CFLAGS/LIBS (the + # above tests atomically set CFLAGS/LIBS or not; this + # test is running after the fact, so we have to undo + # the side-effects of setting CFLAGS/LIBS if the above + # tests passed). + CFLAGS=$atomic_compare_exchange_n_128_CFLAGS_save + LIBS=$atomic_compare_exchange_n_128_LIBS_save + AC_MSG_RESULT([no])], + [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])]) + ]) + + AC_DEFINE_UNQUOTED([PMIX_HAVE_GCC_BUILTIN_CSWAP_INT128], + [$atomic_compare_exchange_n_128_result], + [Whether the __atomic builtin atomic compare swap is both supported and lock-free on 128-bit values]) + + dnl If we could not find decent support for 128-bits __atomic let's + dnl try the GCC _sync + AS_IF([test $atomic_compare_exchange_n_128_result -eq 0], + [PMIX_CHECK_SYNC_BUILTIN_CSWAP_INT128]) - PMIX_VAR_SCOPE_PUSH([atomic_compare_exchange_n_128_result CFLAGS_save]) - - AC_ARG_ENABLE([cross-cmpset128],[AC_HELP_STRING([--enable-cross-cmpset128], - [enable the use of the __sync builtin atomic compare-and-swap 128 when cross compiling])]) - - atomic_compare_exchange_n_128_result=0 - - if test ! "$enable_cross_cmpset128" = "yes" ; then - AC_MSG_CHECKING([for processor support of __atomic builtin atomic compare-and-swap on 128-bit values]) - - AC_RUN_IFELSE([AC_LANG_PROGRAM([], [__int128 x = 0, y = 0; __atomic_compare_exchange_n (&x, &y, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);])], - [AC_MSG_RESULT([yes]) - atomic_compare_exchange_n_128_result=1], - [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([no (cross compiling)])]) + PMIX_VAR_SCOPE_POP +]) - if test $atomic_compare_exchange_n_128_result = 0 ; then - CFLAGS_save=$CFLAGS - CFLAGS="$CFLAGS -mcx16" +AC_DEFUN([PMIX_CHECK_GCC_ATOMIC_BUILTINS], [ + AC_MSG_CHECKING([for __atomic builtin atomics]) - AC_MSG_CHECKING([for __atomic builtin atomic compare-and-swap on 128-bit values with -mcx16 flag]) - AC_RUN_IFELSE([AC_LANG_PROGRAM([], [__int128 x = 0, y = 0; __atomic_compare_exchange_n (&x, &y, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);])], - [AC_MSG_RESULT([yes]) - atomic_compare_exchange_n_128_result=1 - CFLAGS_save="$CFLAGS"], - [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([no (cross compiling)])]) + AC_TRY_LINK([ +#include +uint32_t tmp, old = 0; +uint64_t tmp64, old64 = 0;], [ +__atomic_thread_fence(__ATOMIC_SEQ_CST); +__atomic_compare_exchange_n(&tmp, &old, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); +__atomic_add_fetch(&tmp, 1, __ATOMIC_RELAXED); +__atomic_compare_exchange_n(&tmp64, &old64, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); +__atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);], + [AC_MSG_RESULT([yes]) + $1], + [AC_MSG_RESULT([no]) + $2]) - CFLAGS=$CFLAGS_save - fi + # Check for 128-bit support + PMIX_CHECK_GCC_BUILTIN_CSWAP_INT128 +]) - if test $atomic_compare_exchange_n_128_result = 1 ; then - AC_MSG_CHECKING([if __int128 atomic compare-and-swap is always lock-free]) - AC_RUN_IFELSE([AC_LANG_PROGRAM([], [if (!__atomic_always_lock_free(16, 0)) { return 1; }])], +AC_DEFUN([PMIX_CHECK_C11_CSWAP_INT128], [ + PMIX_VAR_SCOPE_PUSH([atomic_compare_exchange_result atomic_compare_exchange_CFLAGS_save atomic_compare_exchange_LIBS_save]) + + atomic_compare_exchange_CFLAGS_save=$CFLAGS + atomic_compare_exchange_LIBS_save=$LIBS + + # Do we have C11 atomics on 128-bit integers? + # Use a special macro because we need to check with a few different + # CFLAGS/LIBS. + PMIX_ASM_CHECK_ATOMIC_FUNC([atomic_compare_exchange_strong_16], + [AC_LANG_SOURCE(PMIX_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE)], + [atomic_compare_exchange_result=1], + [atomic_compare_exchange_result=0]) + + # If we have it and it works, check to make sure it is always lock + # free. + AS_IF([test $atomic_compare_exchange_result -eq 1], + [AC_MSG_CHECKING([if C11 __int128 atomic compare-and-swap is always lock-free]) + AC_RUN_IFELSE([AC_LANG_PROGRAM([#include ], [_Atomic __int128_t x; if (!atomic_is_lock_free(&x)) { return 1; }])], [AC_MSG_RESULT([yes])], - [AC_MSG_RESULT([no]) - PMIX_CHECK_SYNC_BUILTIN_CSWAP_INT128 - atomic_compare_exchange_n_128_result=0], - [AC_MSG_RESULT([no (cross compiling)])]) - fi - else - AC_MSG_CHECKING([for compiler support of __atomic builtin atomic compare-and-swap on 128-bit values]) - - # Check if the compiler supports the __atomic builtin - AC_TRY_LINK([], [__int128 x = 0, y = 0; __atomic_compare_exchange_n (&x, &y, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);], - [AC_MSG_RESULT([yes]) - atomic_compare_exchange_n_128_result=1], - [AC_MSG_RESULT([no])]) - - if test $atomic_compare_exchange_n_128_result = 0 ; then - CFLAGS_save=$CFLAGS - CFLAGS="$CFLAGS -mcx16" - - AC_MSG_CHECKING([for __atomic builtin atomic compare-and-swap on 128-bit values with -mcx16 flag]) - AC_TRY_LINK([], [__int128 x = 0, y = 0; __atomic_compare_exchange_n (&x, &y, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);], - [AC_MSG_RESULT([yes]) - atomic_compare_exchange_n_128_result=1 - CFLAGS_save="$CFLAGS"], - [AC_MSG_RESULT([no])]) - - CFLAGS=$CFLAGS_save - fi - fi - - AC_DEFINE_UNQUOTED([PMIX_HAVE_GCC_BUILTIN_CSWAP_INT128], [$atomic_compare_exchange_n_128_result], - [Whether the __atomic builtin atomic compare and swap is lock-free on 128-bit values]) + [atomic_compare_exchange_result=0 + # If this test fails, need to reset CFLAGS/LIBS (the + # above tests atomically set CFLAGS/LIBS or not; this + # test is running after the fact, so we have to undo + # the side-effects of setting CFLAGS/LIBS if the above + # tests passed). + CFLAGS=$atomic_compare_exchange_CFLAGS_save + LIBS=$atomic_compare_exchange_LIBS_save + AC_MSG_RESULT([no])], + [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])]) + ]) + + AC_DEFINE_UNQUOTED([PMIX_HAVE_C11_CSWAP_INT128], + [$atomic_compare_exchange_result], + [Whether C11 atomic compare swap is both supported and lock-free on 128-bit values]) + + dnl If we could not find decent support for 128-bits atomic let's + dnl try the GCC _sync + AS_IF([test $atomic_compare_exchange_result -eq 0], + [PMIX_CHECK_SYNC_BUILTIN_CSWAP_INT128]) PMIX_VAR_SCOPE_POP ]) @@ -533,7 +767,7 @@ dnl PMIX_CHECK_ASM_TYPE dnl dnl Sets PMIX_ASM_TYPE to the prefix for the function type to dnl set a symbol's type as function (needed on ELF for shared -dnl libaries). If no .type directive is needed, sets PMIX_ASM_TYPE +dnl libraries). If no .type directive is needed, sets PMIX_ASM_TYPE dnl to an empty string dnl dnl We look for @ \# % @@ -727,7 +961,7 @@ AC_DEFUN([PMIX_CHECK_SPARCV8PLUS],[ AC_MSG_CHECKING([if have Sparc v8+/v9 support]) sparc_result=0 PMIX_TRY_ASSEMBLE([$pmix_cv_asm_text - casa [%o0] 0x80, %o1, %o2], + casa [%o0] 0x80, %o1, %o2], [sparc_result=1], [sparc_result=0]) if test "$sparc_result" = "1" ; then @@ -746,35 +980,8 @@ dnl dnl PMIX_CHECK_CMPXCHG16B dnl dnl ################################################################# -AC_DEFUN([PMIX_CHECK_CMPXCHG16B],[ - PMIX_VAR_SCOPE_PUSH([cmpxchg16b_result]) - - AC_ARG_ENABLE([cross-cmpxchg16b],[AC_HELP_STRING([--enable-cross-cmpxchg16b], - [enable the use of the cmpxchg16b instruction when cross compiling])]) - - if test ! "$enable_cross_cmpxchg16b" = "yes" ; then - AC_MSG_CHECKING([if processor supports x86_64 16-byte compare-and-exchange]) - AC_RUN_IFELSE([AC_LANG_PROGRAM([[unsigned char tmp[16];]],[[ - __asm__ __volatile__ ("lock cmpxchg16b (%%rsi)" : : "S" (tmp) : "memory", "cc");]])], - [AC_MSG_RESULT([yes]) - cmpxchg16b_result=1], - [AC_MSG_RESULT([no]) - cmpxchg16b_result=0], - [AC_MSG_RESULT([no (cross-compiling)]) - cmpxchg16b_result=0]) - else - AC_MSG_CHECKING([if assembler supports x86_64 16-byte compare-and-exchange]) - - PMIX_TRY_ASSEMBLE([$pmix_cv_asm_text - cmpxchg16b 0], - [AC_MSG_RESULT([yes]) - cmpxchg16b_result=1], - [AC_MSG_RESULT([no]) - cmpxchg16b_result=0]) - fi - if test "$cmpxchg16b_result" = 1; then - AC_MSG_CHECKING([if compiler correctly handles volatile 128bits]) - AC_RUN_IFELSE([AC_LANG_PROGRAM([#include +AC_DEFUN([PMIX_CMPXCHG16B_TEST_SOURCE],[[ +#include #include union pmix_counted_pointer_t { @@ -788,8 +995,10 @@ union pmix_counted_pointer_t { int128_t value; #endif }; -typedef union pmix_counted_pointer_t pmix_counted_pointer_t;], - [volatile pmix_counted_pointer_t a; +typedef union pmix_counted_pointer_t pmix_counted_pointer_t; + +int main(int argc, char* argv) { + volatile pmix_counted_pointer_t a; pmix_counted_pointer_t b; a.data.counter = 0; @@ -814,12 +1023,28 @@ typedef union pmix_counted_pointer_t pmix_counted_pointer_t;], return (a.value != b.value); #else return 0; -#endif])], - [AC_MSG_RESULT([yes])], - [AC_MSG_RESULT([no]) - cmpxchg16b_result=0], - [AC_MSG_RESULT([untested, assuming ok])]) - fi +#endif +} +]]) + +AC_DEFUN([PMIX_CHECK_CMPXCHG16B],[ + PMIX_VAR_SCOPE_PUSH([cmpxchg16b_result]) + + PMIX_ASM_CHECK_ATOMIC_FUNC([cmpxchg16b], + [AC_LANG_PROGRAM([[unsigned char tmp[16];]], + [[__asm__ __volatile__ ("lock cmpxchg16b (%%rsi)" : : "S" (tmp) : "memory", "cc");]])], + [cmpxchg16b_result=1], + [cmpxchg16b_result=0]) + # If we have it, make sure it works. + AS_IF([test $cmpxchg16b_result -eq 1], + [AC_MSG_CHECKING([if cmpxchg16b_result works]) + AC_RUN_IFELSE([AC_LANG_SOURCE(PMIX_CMPXCHG16B_TEST_SOURCE)], + [AC_MSG_RESULT([yes])], + [cmpxchg16b_result=0 + AC_MSG_RESULT([no])], + [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])]) + ]) + AC_DEFINE_UNQUOTED([PMIX_HAVE_CMPXCHG16B], [$cmpxchg16b_result], [Whether the processor supports the cmpxchg16b instruction]) PMIX_VAR_SCOPE_POP @@ -832,7 +1057,7 @@ dnl dnl Check if the compiler is capable of doing GCC-style inline dnl assembly. Some compilers emit a warning and ignore the inline dnl assembly (xlc on OS X) and compile without error. Therefore, -dnl the test attempts to run the emited code to check that the +dnl the test attempts to run the emitted code to check that the dnl assembly is actually run. To run this test, one argument to dnl the macro must be an assembly instruction in gcc format to move dnl the value 0 into the register containing the variable ret. @@ -885,7 +1110,7 @@ return ret; if test "$asm_result" = "yes" ; then PMIX_C_GCC_INLINE_ASSEMBLY=1 - pmix_cv_asm_inline_supported="yes" + pmix_cv_asm_inline_supported="yes" else PMIX_C_GCC_INLINE_ASSEMBLY=0 fi @@ -912,18 +1137,30 @@ AC_DEFUN([PMIX_CONFIG_ASM],[ AC_REQUIRE([PMIX_SETUP_CC]) AC_REQUIRE([AM_PROG_AS]) + AC_ARG_ENABLE([c11-atomics],[AC_HELP_STRING([--enable-c11-atomics], + [Enable use of C11 atomics if available (default: enabled)])]) + AC_ARG_ENABLE([builtin-atomics], [AC_HELP_STRING([--enable-builtin-atomics], - [Enable use of __sync builtin atomics (default: enabled)])], - [], [enable_builtin_atomics="yes"]) + [Enable use of __sync builtin atomics (default: disabled)])]) - pmix_cv_asm_builtin="BUILTIN_NO" - AS_IF([test "$pmix_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"], - [PMIX_CHECK_GCC_ATOMIC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_GCC"], [])]) - AS_IF([test "$pmix_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"], - [PMIX_CHECK_SYNC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_SYNC"], [])]) - AS_IF([test "$pmix_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], - [AC_MSG_WARN([__sync builtin atomics requested but not found - proceeding with inline atomics])]) + PMIX_CHECK_C11_CSWAP_INT128 + + if test "x$enable_c11_atomics" != "xno" && test "$pmix_cv_c11_supported" = "yes" ; then + pmix_cv_asm_builtin="BUILTIN_C11" + PMIX_CHECK_C11_CSWAP_INT128 + elif test "x$enable_c11_atomics" = "xyes"; then + AC_MSG_WARN([C11 atomics were requested but are not supported]) + AC_MSG_ERROR([Cannot continue]) + else + pmix_cv_asm_builtin="BUILTIN_NO" + AS_IF([test "$pmix_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], + [PMIX_CHECK_GCC_ATOMIC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_GCC"], [])]) + AS_IF([test "$pmix_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], + [PMIX_CHECK_SYNC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_SYNC"], [])]) + AS_IF([test "$pmix_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], + [AC_MSG_ERROR([__sync builtin atomics requested but not found.])]) + fi PMIX_CHECK_ASM_PROC PMIX_CHECK_ASM_TEXT @@ -960,9 +1197,9 @@ AC_DEFUN([PMIX_CONFIG_ASM],[ ia64-*) pmix_cv_asm_arch="IA64" PMIX_CHECK_SYNC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_SYNC"], - [AC_MSG_ERROR([No atomic primitives available for $host])]) + [AC_MSG_ERROR([No atomic primitives available for $host])]) ;; - aarch64*) + aarch64*) pmix_cv_asm_arch="ARM64" PMIX_ASM_SUPPORT_64BIT=1 PMIX_ASM_ARM_VERSION=8 @@ -994,7 +1231,7 @@ AC_DEFUN([PMIX_CONFIG_ASM],[ # uses Linux kernel helpers for some atomic operations pmix_cv_asm_arch="ARM" PMIX_CHECK_SYNC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_SYNC"], - [AC_MSG_ERROR([No atomic primitives available for $host])]) + [AC_MSG_ERROR([No atomic primitives available for $host])]) ;; mips-*|mips64*) @@ -1002,7 +1239,7 @@ AC_DEFUN([PMIX_CONFIG_ASM],[ # a MIPS III machine (r4000 and later) pmix_cv_asm_arch="MIPS" PMIX_CHECK_SYNC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_SYNC"], - [AC_MSG_ERROR([No atomic primitives available for $host])]) + [AC_MSG_ERROR([No atomic primitives available for $host])]) ;; powerpc-*|powerpc64-*|powerpcle-*|powerpc64le-*|rs6000-*|ppc-*) @@ -1070,11 +1307,11 @@ AC_MSG_ERROR([Can not continue.]) ;; esac - if test "x$PMIX_ASM_SUPPORT_64BIT" = "x1" && test "$pmix_cv_asm_builtin" = "BUILTIN_SYNC" && - test "$pmix_asm_sync_have_64bit" = "0" ; then - # __sync builtins exist but do not implement 64-bit support. Fall back on inline asm. - pmix_cv_asm_builtin="BUILTIN_NO" - fi + if test "x$PMIX_ASM_SUPPORT_64BIT" = "x1" && test "$pmix_cv_asm_builtin" = "BUILTIN_SYNC" && + test "$pmix_asm_sync_have_64bit" = "0" ; then + # __sync builtins exist but do not implement 64-bit support. Fall back on inline asm. + pmix_cv_asm_builtin="BUILTIN_NO" + fi if test "$pmix_cv_asm_builtin" = "BUILTIN_SYNC" || test "$pmix_cv_asm_builtin" = "BUILTIN_GCC" ; then AC_DEFINE([PMIX_C_GCC_INLINE_ASSEMBLY], [1], @@ -1097,7 +1334,7 @@ AC_MSG_ERROR([Can not continue.]) ;; esac - pmix_cv_asm_inline_supported="no" + pmix_cv_asm_inline_supported="no" # now that we know our architecture, try to inline assemble PMIX_CHECK_INLINE_C_GCC([$PMIX_GCC_INLINE_ASSIGN]) diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_config_pthreads.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_config_pthreads.m4 index 2e2f1fd8f97..b23f66ebb01 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix_config_pthreads.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_config_pthreads.m4 @@ -10,7 +10,7 @@ dnl University of Stuttgart. All rights reserved. dnl Copyright (c) 2004-2005 The Regents of the University of California. dnl All rights reserved. dnl Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. -dnl Copyright (c) 2014-2017 Intel, Inc. All rights reserved. +dnl Copyright (c) 2014-2019 Intel, Inc. All rights reserved. dnl Copyright (c) 2014-2016 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. dnl $COPYRIGHT$ @@ -71,104 +71,6 @@ int main(int argc, char* argv[]) # END: PMIX_INTL_PTHREAD_TRY_LINK ])dnl - -AC_DEFUN([PMIX_INTL_PTHREAD_TRY_LINK_FORTRAN], [ -# BEGIN: PMIX_INTL_PTHREAD_TRY_LINK_FORTRAN -# -# Make sure that we can run a small application in Fortran, with -# pthreads living in a C object file - -# Fortran module -cat > conftestf.f < conftest.c < -#include -#include -$pmix_conftest_h - -#ifdef __cplusplus -extern "C" { -#endif -int i = 3; -pthread_t me, newthread; - -void cleanup_routine(void *foo); -void *thread_main(void *foo); -void pthreadtest_f(void); - -void cleanup_routine(void *foo) { i = 4; } -void *thread_main(void *foo) { i = 2; return (void*) &i; } - -void pthreadtest_f(void) -{ - pthread_attr_t attr; - - me = pthread_self(); - pthread_atfork(NULL, NULL, NULL); - pthread_attr_init(&attr); - pthread_cleanup_push(cleanup_routine, 0); - pthread_create(&newthread, &attr, thread_main, 0); - pthread_join(newthread, 0); - pthread_cleanup_pop(0); -} - -void pthreadtest(void) -{ pthreadtest_f(); } - -void pthreadtest_(void) -{ pthreadtest_f(); } - -void pthreadtest__(void) -{ pthreadtest_f(); } - -void PTHREADTEST(void) -{ pthreadtest_f(); } - -#ifdef __cplusplus -} -#endif -EOF - -# Try the compile -PMIX_LOG_COMMAND( - [$CC $CFLAGS -I. -c conftest.c], - PMIX_LOG_COMMAND( - [$FC $FCFLAGS conftestf.f conftest.o -o conftest $LDFLAGS $LIBS], - [HAPPY=1], - [HAPPY=0]), - [HAPPY=0]) - -if test "$HAPPY" = "1"; then - $1 -else - PMIX_LOG_MSG([here is the C program:], 1) - PMIX_LOG_FILE([conftest.c]) - if test -f conftest.h; then - PMIX_LOG_MSG([here is contest.h:], 1) - PMIX_LOG_FILE([conftest.h]) - fi - PMIX_LOG_MSG([here is the fortran program:], 1) - PMIX_LOG_FILE([conftestf.f]) - $2 -fi - -unset HAPPY pmix_conftest_h -rm -rf conftest* -# END: PMIX_INTL_PTHREAD_TRY_LINK_FORTRAN -])dnl - - # ******************************************************************** # # Try to compile thread support without any special flags @@ -194,48 +96,6 @@ fi ])dnl -AC_DEFUN([PMIX_INTL_POSIX_THREADS_PLAIN_CXX], [ -# -# C++ compiler -# -if test "$pmix_pthread_cxx_success" = "0"; then - AC_MSG_CHECKING([if C++ compiler and POSIX threads work as is]) - - AC_LANG_PUSH(C++) - PMIX_INTL_PTHREAD_TRY_LINK(pmix_pthread_cxx_success=1, - pmix_pthread_cxx_success=0) - AC_LANG_POP(C++) - if test "$pmix_pthread_cxx_success" = "1"; then - AC_MSG_RESULT([yes]) - else - AC_MSG_RESULT([no]) - fi -fi -])dnl - - -AC_DEFUN([PMIX_INTL_POSIX_THREADS_PLAIN_FC], [ -# -# Fortran compiler -# -if test "$pmix_pthread_fortran_success" = "0" && \ - test "$OMPI_TRY_FORTRAN_BINDINGS" -gt "$OMPI_FORTRAN_NO_BINDINGS" && \ - test $ompi_fortran_happy -eq 1; then - AC_MSG_CHECKING([if Fortran compiler and POSIX threads work as is]) - - AC_LANG_PUSH(C) - PMIX_INTL_PTHREAD_TRY_LINK_FORTRAN(pmix_pthread_fortran_success=1, - pmix_pthread_fortran_success=0) - AC_LANG_POP(C) - if test "$pmix_pthread_fortran_success" = "1"; then - AC_MSG_RESULT([yes]) - else - AC_MSG_RESULT([no]) - fi -fi -])dnl - - AC_DEFUN([PMIX_INTL_POSIX_THREADS_PLAIN], [ # BEGIN: PMIX_INTL_POSIX_THREADS_PLAIN # @@ -246,19 +106,10 @@ AC_DEFUN([PMIX_INTL_POSIX_THREADS_PLAIN], [ # why take chances? # -# Only run C++ and Fortran if those compilers already configured AC_PROVIDE_IFELSE([AC_PROG_CC], [PMIX_INTL_POSIX_THREADS_PLAIN_C], [pmix_pthread_c_success=1]) -AC_PROVIDE_IFELSE([AC_PROG_CXX], - [PMIX_INTL_POSIX_THREADS_PLAIN_CXX], - [pmix_pthread_cxx_success=1]) - -AC_PROVIDE_IFELSE([AC_PROG_FC], - [PMIX_INTL_POSIX_THREADS_PLAIN_FC], - [pmix_pthread_fortran_success=1]) - # End: PMIX_INTL_POSIX_THREADS_PLAIN ])dnl @@ -294,60 +145,6 @@ fi ]) -AC_DEFUN([PMIX_INTL_POSIX_THREADS_SPECIAL_FLAGS_CXX], [ -# -# C++ compiler -# -if test "$pmix_pthread_cxx_success" = "0"; then - for pf in $pflags; do - AC_MSG_CHECKING([if C++ compiler and POSIX threads work with $pf]) - CXXFLAGS="$orig_CXXFLAGS $pf" - AC_LANG_PUSH(C++) - PMIX_INTL_PTHREAD_TRY_LINK(pmix_pthread_cxx_success=1, - pmix_pthread_cxx_success=0) - AC_LANG_POP(C++) - if test "$pmix_pthread_cxx_success" = "1"; then - PTHREAD_CXXFLAGS="$pf" - AC_MSG_RESULT([yes]) - break - else - PTHREAD_CXXFLAGS= - CXXFLAGS="$orig_CXXFLAGS" - AC_MSG_RESULT([no]) - fi - done -fi -]) - - -AC_DEFUN([PMIX_INTL_POSIX_THREADS_SPECIAL_FLAGS_FC], [ -# -# Fortran compiler -# -if test "$pmix_pthread_fortran_success" = "0" && \ - test "$OMPI_TRY_FORTRAN_BINDINGS" -gt "$OMPI_FORTRAN_NO_BINDINGS" && \ - test $ompi_fortran_happy -eq 1; then - for pf in $pflags; do - AC_MSG_CHECKING([if Fortran compiler and POSIX threads work with $pf]) - FCFLAGS="$orig_FCFLAGS $pf" - AC_LANG_PUSH(C) - PMIX_INTL_PTHREAD_TRY_LINK_FORTRAN(pmix_pthread_fortran_success=1, - pmix_pthread_fortran_success=0) - AC_LANG_POP(C) - if test "$pmix_pthread_fortran_success" = "1"; then - PTHREAD_FCFLAGS="$pf" - AC_MSG_RESULT([yes]) - break - else - PTHREAD_FCFLAGS= - FCFLAGS="$orig_FCFLAGS" - AC_MSG_RESULT([no]) - fi - done -fi -]) - - AC_DEFUN([PMIX_INTL_POSIX_THREADS_SPECIAL_FLAGS],[ # Begin: PMIX_INTL_POSIX_THREADS_SPECIAL_FLAGS # @@ -374,19 +171,10 @@ case "${host_cpu}-${host_os}" in ;; esac -# Only run C++ and Fortran if those compilers already configured AC_PROVIDE_IFELSE([AC_PROG_CC], [PMIX_INTL_POSIX_THREADS_SPECIAL_FLAGS_C], [pmix_pthread_c_success=1]) -AC_PROVIDE_IFELSE([AC_PROG_CXX], - [PMIX_INTL_POSIX_THREADS_SPECIAL_FLAGS_CXX], - [pmix_pthread_cxx_success=1]) - -AC_PROVIDE_IFELSE([AC_PROG_FC], - [PMIX_INTL_POSIX_THREADS_SPECIAL_FLAGS_FC], - [pmix_pthread_fortran_success=1]) - # End: PMIX_INTL_POSIX_THREADS_SPECIAL_FLAGS ])dnl @@ -435,121 +223,6 @@ if test "$pmix_pthread_c_success" = "0"; then fi ])dnl - -AC_DEFUN([PMIX_INTL_POSIX_THREADS_LIBS_CXX],[ -# -# C++ compiler -# -if test "$pmix_pthread_cxx_success" = "0"; then - if test ! "$pmix_pthread_c_success" = "0" && test ! "$PTHREAD_LIBS" = "" ; then - AC_MSG_CHECKING([if C++ compiler and POSIX threads work with $PTHREAD_LIBS]) - case "${host_cpu}-${host-_os}" in - *-aix* | *-freebsd*) - if test "`echo $CXXCPPFLAGS | $GREP 'D_THREAD_SAFE'`" = ""; then - PTHREAD_CXXCPPFLAGS="-D_THREAD_SAFE" - CXXCPPFLAGS="$CXXCPPFLAGS $PTHREAD_CXXCPPFLAGS" - fi - ;; - *) - if test "`echo $CXXCPPFLAGS | $GREP 'D_REENTRANT'`" = ""; then - PTHREAD_CXXCPPFLAGS="-D_REENTRANT" - CXXCPPFLAGS="$CXXCPPFLAGS $PTHREAD_CXXCPPFLAGS" - fi - ;; - esac - LIBS="$orig_LIBS $PTHREAD_LIBS" - AC_LANG_PUSH(C++) - PMIX_INTL_PTHREAD_TRY_LINK(pmix_pthread_cxx_success=1, - pmix_pthread_cxx_success=0) - AC_LANG_POP(C++) - if test "$pmix_pthread_cxx_success" = "1"; then - AC_MSG_RESULT([yes]) - else - CXXCPPFLAGS="$orig_CXXCPPFLAGS" - LIBS="$orig_LIBS" - AC_MSG_RESULT([no]) - AC_MSG_ERROR([Can not find working threads configuration. aborting]) - fi - else - for pl in $plibs; do - AC_MSG_CHECKING([if C++ compiler and POSIX threads work with $pl]) - case "${host_cpu}-${host-_os}" in - *-aix* | *-freebsd*) - if test "`echo $CXXCPPFLAGS | $GREP 'D_THREAD_SAFE'`" = ""; then - PTHREAD_CXXCPPFLAGS="-D_THREAD_SAFE" - CXXCPPFLAGS="$CXXCPPFLAGS $PTHREAD_CXXCPPFLAGS" - fi - ;; - *) - if test "`echo $CXXCPPFLAGS | $GREP 'D_REENTRANT'`" = ""; then - PTHREAD_CXXCPPFLAGS="-D_REENTRANT" - CXXCPPFLAGS="$CXXCPPFLAGS $PTHREAD_CXXCPPFLAGS" - fi - ;; - esac - LIBS="$orig_LIBS $pl" - AC_LANG_PUSH(C++) - PMIX_INTL_PTHREAD_TRY_LINK(pmix_pthread_cxx_success=1, - pmix_pthread_cxx_success=0) - AC_LANG_POP(C++) - if test "$pmix_pthread_cxx_success" = "1"; then - PTHREAD_LIBS="$pl" - AC_MSG_RESULT([yes]) - else - PTHREAD_CXXCPPFLAGS= - CXXCPPFLAGS="$orig_CXXCPPFLAGS" - LIBS="$orig_LIBS" - AC_MSG_RESULT([no]) - fi - done - fi -fi -])dnl - - -AC_DEFUN([PMIX_INTL_POSIX_THREADS_LIBS_FC],[ -# -# Fortran compiler -# -if test "$pmix_pthread_fortran_success" = "0" && \ - test "$OMPI_TRY_FORTRAN_BINDINGS" -gt "$OMPI_FORTRAN_NO_BINDINGS" && \ - test $ompi_fortran_happy -eq 1; then - if test ! "$pmix_pthread_c_success" = "0" && test ! "$PTHREAD_LIBS" = "" ; then - AC_MSG_CHECKING([if Fortran compiler and POSIX threads work with $PTHREAD_LIBS]) - LIBS="$orig_LIBS $PTHREAD_LIBS" - AC_LANG_PUSH(C) - PMIX_INTL_PTHREAD_TRY_LINK_FORTRAN(pmix_pthread_fortran_success=1, - pmix_pthread_fortran_success=0) - AC_LANG_POP(C) - if test "$pmix_pthread_fortran_success" = "1"; then - AC_MSG_RESULT([yes]) - else - LIBS="$orig_LIBS" - AC_MSG_RESULT([no]) - AC_MSG_ERROR([Can not find working threads configuration. aborting]) - fi - else - for pl in $plibs; do - AC_MSG_CHECKING([if Fortran compiler and POSIX threads work with $pl]) - LIBS="$orig_LIBS $pl" - AC_LANG_PUSH(C) - PMIX_INTL_PTHREAD_TRY_LINK_FORTRAN(pmix_pthread_fortran_success=1, - pmix_pthread_fortran_success=0) - AC_LANG_POP(C) - if test "$pmix_pthread_fortran_success" = "1"; then - PTHREAD_LIBS="$pl" - AC_MSG_RESULT([yes]) - break - else - LIBS="$orig_LIBS" - AC_MSG_RESULT([no]) - fi - done - fi -fi -])dnl - - AC_DEFUN([PMIX_INTL_POSIX_THREADS_LIBS],[ # Begin: PMIX_INTL_POSIX_THREADS_LIBS # @@ -563,19 +236,10 @@ AC_DEFUN([PMIX_INTL_POSIX_THREADS_LIBS],[ # libpthread: The usual place (like we can define usual!) plibs="-lpthreads -llthread -lpthread" -# Only run C++ and Fortran if those compilers already configured AC_PROVIDE_IFELSE([AC_PROG_CC], [PMIX_INTL_POSIX_THREADS_LIBS_C], [pmix_pthread_c_success=1]) -AC_PROVIDE_IFELSE([AC_PROG_CXX], - [PMIX_INTL_POSIX_THREADS_LIBS_CXX], - [pmix_pthread_cxx_success=1]) - -AC_PROVIDE_IFELSE([AC_PROG_FC], - [PMIX_INTL_POSIX_THREADS_LIBS_FC], - [pmix_pthread_fortran_success=1]) - # End: PMIX_INTL_POSIX_THREADS_LIBS] )dnl @@ -589,21 +253,14 @@ AC_DEFUN([PMIX_CONFIG_POSIX_THREADS],[ AC_REQUIRE([AC_PROG_GREP]) pmix_pthread_c_success=0 -pmix_pthread_cxx_success=0 orig_CFLAGS="$CFLAGS" -orig_FCFLAGS="$FCFLAGS" -orig_CXXFLAGS="$CXXFLAGS" orig_CPPFLAGS="$CPPFLAGS" -orig_CXXCPPFLAGS="$CXXCPPFLAGS" orig_LDFLAGS="$LDFLAGS" orig_LIBS="$LIBS" PTHREAD_CFLAGS= -PTHREAD_FCFLAGS= -PTHREAD_CXXFLAGS= PTHREAD_CPPFLAGS= -PTHREAD_CXXCPPFLAGS= PTHREAD_LDFLAGS= PTHREAD_LIBS= @@ -648,15 +305,11 @@ AC_DEFINE_UNQUOTED([PMIX_HAVE_PTHREAD_MUTEX_ERRORCHECK], [$defval], [If PTHREADS implementation supports PTHREAD_MUTEX_ERRORCHECK]) CFLAGS="$orig_CFLAGS" -FCFLAGS="$orig_FCFLAGS" -CXXFLAGS="$orig_CXXFLAGS" CPPFLAGS="$orig_CPPFLAGS" -CXXCPPFLAGS="$orig_CXXCPPFLAGS" LDFLAGS="$orig_LDFLAGS" LIBS="$orig_LIBS" -if test "$pmix_pthread_c_success" = "1" && \ - test "$pmix_pthread_cxx_success" = "1"; then +if test "$pmix_pthread_c_success" = "1"; then internal_useless=1 $1 else @@ -664,6 +317,6 @@ else $2 fi -unset pmix_pthread_c_success pmix_pthread_fortran_success pmix_pthread_cxx_success +unset pmix_pthread_c_success unset internal_useless ])dnl diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_config_threads.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_config_threads.m4 index 541e63f726c..050f8735577 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix_config_threads.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_config_threads.m4 @@ -11,7 +11,7 @@ dnl Copyright (c) 2004-2005 The Regents of the University of California. dnl All rights reserved. dnl Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. dnl Copyright (c) 2009-2011 Oak Ridge National Labs. All rights reserved. -dnl Copyright (c) 2014-2017 Intel, Inc. All rights reserved. +dnl Copyright (c) 2014-2019 Intel, Inc. All rights reserved. dnl Copyright (c) 2015 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. dnl $COPYRIGHT$ @@ -56,10 +56,7 @@ if test "$HAVE_POSIX_THREADS" = "0"; then fi THREAD_CFLAGS="$PTHREAD_CFLAGS" -THREAD_FCFLAGS="$PTHREAD_FCFLAGS" -THREAD_CXXFLAGS="$PTHREAD_CXXFLAGS" THREAD_CPPFLAGS="$PTHREAD_CPPFLAGS" -THREAD_CXXCPPFLAGS="$PTHREAD_CXXCPPFLAGS" THREAD_LDFLAGS="$PTHREAD_LDFLAGS" THREAD_LIBS="$PTHREAD_LIBS" diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_functions.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_functions.m4 index e0f3a93cb68..2ff22f11ec1 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix_functions.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_functions.m4 @@ -13,7 +13,7 @@ dnl All rights reserved. dnl Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. dnl Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. dnl Copyright (c) 2009-2016 Cisco Systems, Inc. All rights reserved. -dnl Copyright (c) 2013-2016 Intel, Inc. All rights reserved. +dnl Copyright (c) 2013-2019 Intel, Inc. All rights reserved. dnl Copyright (c) 2017 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. dnl @@ -102,21 +102,21 @@ PMIX_CONFIGURE_DATE="`date`" # Save these details so that they can be used in pmix_info later # AC_SUBST(PMIX_CONFIGURE_USER) +AC_DEFINE_UNQUOTED([PMIX_CONFIGURE_USER], "$PMIX_CONFIGURE_USER", + [User who built PMIx]) AC_SUBST(PMIX_CONFIGURE_HOST) -AC_SUBST(PMIX_CONFIGURE_DATE)])dnl +AC_DEFINE_UNQUOTED([PMIX_CONFIGURE_HOST], "$PMIX_CONFIGURE_HOST", + [Hostname where PMIx was built]) +AC_SUBST(PMIX_CONFIGURE_DATE) +AC_DEFINE_UNQUOTED([PMIX_CONFIGURE_DATE], "$PMIX_CONFIGURE_DATE", + [Date when PMIx was built]) +])dnl dnl ####################################################################### dnl ####################################################################### dnl ####################################################################### AC_DEFUN([PMIX_BASIC_SETUP],[ -# -# Save some stats about this build -# - -PMIX_CONFIGURE_USER="`whoami`" -PMIX_CONFIGURE_HOST="`(hostname || uname -n) 2> /dev/null | sed 1q`" -PMIX_CONFIGURE_DATE="`date`" # # Make automake clean emacs ~ files for "make clean" diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_mca.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_mca.m4 index 262e6f4a88e..2978c928b9a 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix_mca.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_mca.m4 @@ -11,7 +11,7 @@ dnl University of Stuttgart. All rights reserved. dnl Copyright (c) 2004-2005 The Regents of the University of California. dnl All rights reserved. dnl Copyright (c) 2010-2015 Cisco Systems, Inc. All rights reserved. -dnl Copyright (c) 2013-2016 Intel, Inc. All rights reserved. +dnl Copyright (c) 2013-2019 Intel, Inc. All rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow @@ -428,7 +428,7 @@ AC_DEFUN([MCA_CONFIGURE_FRAMEWORK],[ # Create the final .h file that will be included in the type's # top-level glue. This lists all the static components. We don't # need to do this for "common". - if test "$2" != "common"; then + if test "$1" != "common"; then cat > $outfile < becomes _ +# +# -------------------------------------------------------- +AC_DEFUN([PMIX_SET_MCA_PREFIX],[ + AS_IF([test "$pmix_mca_prefix_set" = "yes"], + [AC_MSG_WARN([PMIX mca prefix was already set!]) + AC_MSG_WARN([This is a configury programming error]) + AC_MSG_ERROR([Cannot continue])]) + + MCA_PREFIX=$1 + pmix_mca_prefix_set=yes + AC_DEFINE_UNQUOTED([PMIX_MCA_PREFIX], ["$MCA_PREFIX"], [MCA prefix string for envars]) +])dnl + +# +# Set the MCA cmd line identifier - i.e., change "-mca" to "-" +# +AC_DEFUN([PMIX_SET_MCA_CMD_LINE_ID],[ + AS_IF([test "$pmix_mca_cmd_id_set" = "yes"], + [AC_MSG_WARN([PMIX mca cmd line id was already set!]) + AC_MSG_WARN([This is a configury programming error]) + AC_MSG_ERROR([Cannot continue])]) + + MCA_CMD_LINE_ID=$1 + pmix_mca_cmd_id_set=yes + AC_DEFINE_UNQUOTED([PMIX_MCA_CMD_LINE_ID], ["$MCA_CMD_LINE_ID"], [MCA cmd line identifier]) +])dnl diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_cc.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_cc.m4 index f35c16197e7..a8956a1d22f 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_cc.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_cc.m4 @@ -14,9 +14,9 @@ dnl Copyright (c) 2007-2009 Sun Microsystems, Inc. All rights reserved. dnl Copyright (c) 2008-2015 Cisco Systems, Inc. All rights reserved. dnl Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights dnl reserved. -dnl Copyright (c) 2015 Research Organization for Information Science -dnl and Technology (RIST). All rights reserved. -dnl Copyright (c) 2018 Intel, Inc. All rights reserved. +dnl Copyright (c) 2015-2019 Research Organization for Information Science +dnl and Technology (RIST). All rights reserved. +dnl Copyright (c) 2018-2019 Intel, Inc. All rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow @@ -60,7 +60,12 @@ AC_DEFUN([PMIX_PROG_CC_C11_HELPER],[ PMIX_CC_HELPER([if $CC $1 supports C11 _Static_assert], [pmix_prog_cc_c11_helper__static_assert_available], [[#include ]],[[_Static_assert(sizeof(int64_t) == 8, "WTH");]]) - AS_IF([test $pmix_prog_cc_c11_helper__Thread_local_available -eq 1 && test $pmix_prog_cc_c11_helper_atomic_var_available -eq 1], + PMIX_CC_HELPER([if $CC $1 supports C11 atomic_fetch_xor_explicit], [pmix_prog_cc_c11_helper_atomic_fetch_xor_explicit_available], + [[#include +#include ]],[[_Atomic uint32_t a; uint32_t b; atomic_fetch_xor_explicit(&a, b, memory_order_relaxed);]]) + + + AS_IF([test $pmix_prog_cc_c11_helper__Thread_local_available -eq 1 && test $pmix_prog_cc_c11_helper_atomic_var_available -eq 1 && test $pmix_prog_cc_c11_helper_atomic_fetch_xor_explicit_available -eq 1], [$2], [$3]) @@ -128,7 +133,7 @@ AC_DEFUN([PMIX_SETUP_CC],[ AC_REQUIRE([_PMIX_PROG_CC]) AC_REQUIRE([AM_PROG_CC_C_O]) - PMIX_VAR_SCOPE_PUSH([pmix_prog_cc_c11_helper__Thread_local_available pmix_prog_cc_c11_helper_atomic_var_available pmix_prog_cc_c11_helper__Atomic_available pmix_prog_cc_c11_helper__static_assert_available pmix_prog_cc_c11_helper__Generic_available pmix_prog_cc__thread_available]) + PMIX_VAR_SCOPE_PUSH([pmix_prog_cc_c11_helper__Thread_local_available pmix_prog_cc_c11_helper_atomic_var_available pmix_prog_cc_c11_helper__Atomic_available pmix_prog_cc_c11_helper__static_assert_available pmix_prog_cc_c11_helper__Generic_available pmix_prog_cc__thread_available pmix_prog_cc_c11_helper_atomic_fetch_xor_explicit_available]) PMIX_PROG_CC_C11 diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_cli.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_cli.m4 new file mode 100644 index 00000000000..043c0c4d44f --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_cli.m4 @@ -0,0 +1,40 @@ +dnl -*- shell-script -*- +dnl +dnl Copyright (c) 2016 Research Organization for Information Science +dnl and Technology (RIST). All rights reserved. +dnl Copyright (c) 2017-2019 Intel, Inc. All rights reserved. +dnl Copyright (c) 2018 Cisco, Inc. All rights reserved. +dnl $COPYRIGHT$ +dnl +dnl Additional copyrights may follow +dnl +dnl $HEADER$ +dnl + +dnl PMIX_CAPTURE_CONFIGURE_CLI +dnl +dnl Capture configure command line and do the AC substitution +dnl +dnl Arguments: the variable in which command line will be captured +dnl +dnl Dependencies: None +dnl +AC_DEFUN([PMIX_CAPTURE_CONFIGURE_CLI],[ + # Capture configure command line do the AC substitution + PMIX_VAR_SCOPE_PUSH([sed_quote_subst arg quoted_arg]) + $1= + for arg in "$[]@"; do + sed_quote_subst='s/\(@<:@`"$\\@:>@\)/\\\1/g' + case "$arg" in + *@<:@\\\`\"\$[]@:>@*) + quoted_arg=\'`echo "$arg" | sed $sed_quote_subst`\' ;; + *) + quoted_arg="\'$arg\'" ;; + esac + + eval "$1=\$$1\\ \$quoted_arg" + done + AC_DEFINE_UNQUOTED([$1], ["$$1"], [Capture the configure cmd line]) + PMIX_VAR_SCOPE_POP + AC_SUBST($1) +]) diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_libevent.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_libevent.m4 index b8b013c5107..949af1cfd58 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_libevent.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_libevent.m4 @@ -2,7 +2,7 @@ # # Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. # $COPYRIGHT$ @@ -19,11 +19,7 @@ AC_DEFUN([PMIX_LIBEVENT_CONFIG],[ [AC_HELP_STRING([--with-libevent-header=HEADER], [The value that should be included in C files to include event.h])]) - AC_ARG_ENABLE([embedded-libevent], - [AC_HELP_STRING([--enable-embedded-libevent], - [Enable use of locally embedded libevent])]) - - AS_IF([test "$enable_embedded_libevent" = "yes"], + AS_IF([test "$pmix_mode" = "embedded"], [_PMIX_LIBEVENT_EMBEDDED_MODE], [_PMIX_LIBEVENT_EXTERNAL]) @@ -35,6 +31,8 @@ AC_DEFUN([PMIX_LIBEVENT_CONFIG],[ AC_DEFINE_UNQUOTED([PMIX_EVENT2_THREAD_HEADER], [$PMIX_EVENT2_THREAD_HEADER], [Location of event2/thread.h]) AC_MSG_RESULT([$PMIX_EVENT2_THREAD_HEADER]) + + PMIX_SUMMARY_ADD([[External Packages]],[[Libevent]], [pmix_libevent], [yes ($pmix_libevent_source)]) ]) AC_DEFUN([_PMIX_LIBEVENT_EMBEDDED_MODE],[ @@ -47,6 +45,7 @@ AC_DEFUN([_PMIX_LIBEVENT_EMBEDDED_MODE],[ [PMIX_EVENT_HEADER="$with_libevent_header" PMIX_EVENT2_THREAD_HEADER="$with_libevent_header"]) + pmix_libevent_source=embedded ]) AC_DEFUN([_PMIX_LIBEVENT_EXTERNAL],[ @@ -65,18 +64,22 @@ AC_DEFUN([_PMIX_LIBEVENT_EXTERNAL],[ [AC_HELP_STRING([--with-libevent-libdir=DIR], [Search for libevent libraries in DIR ])]) + # get rid of the trailing slash(es) + libevent_prefix=$(echo $with_libevent | sed -e 'sX/*$XXg') + libeventdir_prefix=$(echo $with_libevent_libdir | sed -e 'sX/*$XXg') + AC_MSG_CHECKING([for libevent in]) - if test ! -z "$with_libevent" && test "$with_libevent" != "yes"; then + if test ! -z "$libevent_prefix" && test "$libevent_prefix" != "yes"; then pmix_event_defaults=no - pmix_event_dir=$with_libevent - if test -d $with_libevent/lib; then - pmix_event_libdir=$with_libevent/lib - elif test -d $with_libevent/lib64; then - pmix_event_libdir=$with_libevent/lib64 - elif test -d $with_libevent; then - pmix_event_libdir=$with_libevent + pmix_event_dir=$libevent_prefix + if test -d $libevent_prefix/lib; then + pmix_event_libdir=$libevent_prefix/lib + elif test -d $libevent_prefix/lib64; then + pmix_event_libdir=$libevent_prefix/lib64 + elif test -d $libevent_prefix; then + pmix_event_libdir=$libevent_prefix else - AC_MSG_RESULT([Could not find $with_libevent/lib, $with_libevent/lib64, or $with_libevent]) + AC_MSG_RESULT([Could not find $libevent_prefix/lib, $libevent_prefix/lib64, or $libevent_prefix]) AC_MSG_ERROR([Can not continue]) fi AC_MSG_RESULT([$pmix_event_dir and $pmix_event_libdir]) @@ -90,13 +93,13 @@ AC_DEFUN([_PMIX_LIBEVENT_EXTERNAL],[ else AC_MSG_RESULT([not found]) AC_MSG_WARN([Could not find /usr/lib or /usr/lib64 - you may]) - AC_MSG_WARN([to specify --with-libevent-libdir=]) + AC_MSG_WARN([need to specify --with-libevent-libdir=]) AC_MSG_ERROR([Can not continue]) fi AC_MSG_RESULT([(default search paths)]) fi - AS_IF([test ! -z "$with_libevent_libdir" && "$with_libevent_libdir" != "yes"], - [pmix_event_libdir="$with_libevent_libdir"]) + AS_IF([test ! -z "$libeventdir_prefix" && "$libeventdir_prefix" != "yes"], + [pmix_event_libdir="$libeventdir_prefix"]) PMIX_CHECK_PACKAGE([pmix_libevent], [event.h], @@ -111,9 +114,8 @@ AC_DEFUN([_PMIX_LIBEVENT_EXTERNAL],[ AS_IF([test "$pmix_event_defaults" = "no"], [PMIX_FLAGS_APPEND_UNIQ(CPPFLAGS, $pmix_libevent_CPPFLAGS) - PMIX_FLAGS_APPEND_UNIQ(LIBS, $pmix_libevent_LIBS) PMIX_FLAGS_APPEND_UNIQ(LDFLAGS, $pmix_libevent_LDFLAGS)]) - + PMIX_FLAGS_APPEND_UNIQ(LIBS, $pmix_libevent_LIBS) # Ensure that this libevent has the symbol # "evthread_set_lock_callbacks", which will only exist if @@ -134,6 +136,7 @@ AC_DEFUN([_PMIX_LIBEVENT_EXTERNAL],[ # Set output variables PMIX_EVENT_HEADER="" PMIX_EVENT2_THREAD_HEADER="" + pmix_libevent_source=$pmix_event_dir PMIX_VAR_SCOPE_POP ])dnl diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_zlib.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_zlib.m4 index a67f1b70439..b7bb281cf94 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_zlib.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_setup_zlib.m4 @@ -2,7 +2,7 @@ # # Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -13,7 +13,7 @@ # MCA_zlib_CONFIG([action-if-found], [action-if-not-found]) # -------------------------------------------------------------------- AC_DEFUN([PMIX_ZLIB_CONFIG],[ - PMIX_VAR_SCOPE_PUSH([pmix_zlib_dir pmix_zlib_libdir]) + PMIX_VAR_SCOPE_PUSH([pmix_zlib_dir pmix_zlib_libdir pmix_zlib_standard_lib_location pmix_zlib_standard_header_location]) AC_ARG_WITH([zlib], [AC_HELP_STRING([--with-zlib=DIR], @@ -24,28 +24,32 @@ AC_DEFUN([PMIX_ZLIB_CONFIG],[ [Search for zlib libraries in DIR ])]) pmix_zlib_support=0 + if test "$with_zlib" != "no"; then AC_MSG_CHECKING([for zlib in]) if test ! -z "$with_zlib" && test "$with_zlib" != "yes"; then pmix_zlib_dir=$with_zlib pmix_zlib_standard_header_location=no - if test -d $with_zlib/lib; then - pmix_zlib_libdir=$with_zlib/lib - elif test -d $with_zlib/lib64; then - pmix_zlib_libdir=$with_zlib/lib64 - else - AC_MSG_RESULT([Could not find $with_zlib/lib or $with_zlib/lib64]) - AC_MSG_ERROR([Can not continue]) - fi - AC_MSG_RESULT([$pmix_zlib_dir and $pmix_zlib_libdir]) + pmix_zlib_standard_lib_location=no + AS_IF([test -z "$with_zlib_libdir" || test "$with_zlib_libdir" = "yes"], + [if test -d $with_zlib/lib; then + pmix_zlib_libdir=$with_zlib/lib + elif test -d $with_zlib/lib64; then + pmix_zlib_libdir=$with_zlib/lib64 + else + AC_MSG_RESULT([Could not find $with_zlib/lib or $with_zlib/lib64]) + AC_MSG_ERROR([Can not continue]) + fi + AC_MSG_RESULT([$pmix_zlib_dir and $pmix_zlib_libdir])], + [AC_MSG_RESULT([$with_zlib_libdir])]) else AC_MSG_RESULT([(default search paths)]) pmix_zlib_standard_header_location=yes + pmix_zlib_standard_lib_location=yes fi AS_IF([test ! -z "$with_zlib_libdir" && test "$with_zlib_libdir" != "yes"], [pmix_zlib_libdir="$with_zlib_libdir" - pmix_zlib_standard_lib_location=no], - [pmix_zlib_standard_lib_location=yes]) + pmix_zlib_standard_lib_location=no]) PMIX_CHECK_PACKAGE([pmix_zlib], [zlib.h], diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_summary.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_summary.m4 new file mode 100644 index 00000000000..9b9bc9024e1 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_summary.m4 @@ -0,0 +1,79 @@ +dnl -*- shell-script -*- +dnl +dnl Copyright (c) 2016 Los Alamos National Security, LLC. All rights +dnl reserved. +dnl Copyright (c) 2016-2018 Cisco Systems, Inc. All rights reserved +dnl Copyright (c) 2016 Research Organization for Information Science +dnl and Technology (RIST). All rights reserved. +dnl Copyright (c) 2018-2019 Intel, Inc. All rights reserved. +dnl $COPYRIGHT$ +dnl +dnl Additional copyrights may follow +dnl +dnl $HEADER$ +dnl +AC_DEFUN([PMIX_SUMMARY_ADD],[ + PMIX_VAR_SCOPE_PUSH([pmix_summary_section pmix_summary_line pmix_summary_section_current]) + + dnl need to replace spaces in the section name with somethis else. _ seems like a reasonable + dnl choice. if this changes remember to change PMIX_PRINT_SUMMARY as well. + pmix_summary_section=$(echo $1 | tr ' ' '_') + pmix_summary_line="$2: $4" + pmix_summary_section_current=$(eval echo \$pmix_summary_values_$pmix_summary_section) + + if test -z "$pmix_summary_section_current" ; then + if test -z "$pmix_summary_sections" ; then + pmix_summary_sections=$pmix_summary_section + else + pmix_summary_sections="$pmix_summary_sections $pmix_summary_section" + fi + eval pmix_summary_values_$pmix_summary_section=\"$pmix_summary_line\" + else + eval pmix_summary_values_$pmix_summary_section=\"$pmix_summary_section_current,$pmix_summary_line\" + fi + + PMIX_VAR_SCOPE_POP +]) + +AC_DEFUN([PMIX_SUMMARY_PRINT],[ + PMIX_VAR_SCOPE_PUSH([pmix_summary_section pmix_summary_section_name]) + cat <&1 && touch success) | tee auto.out + (./autogen.pl $autogen_args 2>&1 && touch success) | tee auto.out if test ! -f success; then echo "Autogen failed. Aborting" exit 1 diff --git a/opal/mca/pmix/pmix2x/pmix/contrib/pmix.spec b/opal/mca/pmix/pmix2x/pmix/contrib/pmix.spec index 6204686d5b9..0652e442570 100644 --- a/opal/mca/pmix/pmix2x/pmix/contrib/pmix.spec +++ b/opal/mca/pmix/pmix2x/pmix/contrib/pmix.spec @@ -12,7 +12,7 @@ # Copyright (c) 2006-2016 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2013 Mellanox Technologies, Inc. # All rights reserved. -# Copyright (c) 2015-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2015-2019 Intel, Inc. All rights reserved. # Copyright (c) 2015 Research Organization for Information Science # and Technology (RIST). All rights reserved. # $COPYRIGHT$ @@ -192,7 +192,7 @@ Summary: An extended/exascale implementation of PMI Name: %{?_name:%{_name}}%{!?_name:pmix} -Version: 2.1.4 +Version: 2.2.3rc1 Release: 1%{?dist} License: BSD Group: Development/Libraries @@ -204,6 +204,7 @@ Prefix: %{_prefix} Provides: pmix Provides: pmix = %{version} BuildRoot: /var/tmp/%{name}-%{version}-%{release}-root +BuildRequires: libevent-devel %if %{disable_auto_requires} AutoReq: no %endif @@ -230,6 +231,22 @@ scalability. This RPM contains all the tools necessary to compile and link against PMIx. +# if build_all_in_one_rpm = 0, build split packages +%if !%{build_all_in_one_rpm} +%package libpmi +Summary: PMI-1 and PMI-2 compatibility libraries +Requires: %{name}%{?_isa} = %{version}-%{release} +Conflicts: slurm-libpmi + +%description libpmi +The %{name}-libpmi package contains libpmi and libpmi2 libraries that provide +the respective APIs and a copy of the PMIx library – each API is translated +into its PMIx equivalent. This is especially targeted at apps/libs that are +hardcoded to dlopen “libpmi” or “libpmi2”. +This package conflicts sith slurm-libpmi, which provides its own, incompatible +versions of libpmi.so and libpmi2.so. +%endif + ############################################################################# # # Prepatory Section @@ -346,6 +363,10 @@ export CFLAGS CXXFLAGS FCFLAGS # We don't need that in an RPM. find $RPM_BUILD_ROOT -name config.log -exec rm -f {} \; +# If we build separate RPMs, then move the libpmi.* and libpmi2.* compat libs +# out of the way +find $RPM_BUILD_ROOT -name 'libpmi.' | xargs rm -f + # First, the [optional] modulefile %if %{install_modulefile} @@ -490,6 +511,19 @@ test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT %endif %doc README INSTALL LICENSE +# if building separate RPMs, split the compatibility libs +%if !%{build_all_in_one_rpm} +%exclude %{_libdir}/libpmi.* +%exclude %{_libdir}/libpmi2.* +%exclude %{_includedir}/pmi.* +%exclude %{_includedir}/pmi2.* + +%files libpmi +%{_libdir}/libpmi.* +%{_libdir}/libpmi2.* +%{_includedir}/pmi.* +%{_includedir}/pmi2.* +%endif ############################################################################# # @@ -497,6 +531,14 @@ test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT # ############################################################################# %changelog +* Tue Apr 30 2019 Kilian Cavalotti +- Enable multiple RPMs build to allow backward compatibility PMI-1 and PMI-2 + libs to be built separate. "rpmbuild --define 'build_all_in_one_rpm 0' ..." + will build separate pmix and pmix-libpmi RPMs. + +* Tue Oct 17 2017 Ralph Castain +- Add PMIx bin directory + * Thu Sep 21 2017 Ralph Castain - Add PMIx etc directory diff --git a/opal/mca/pmix/pmix2x/pmix/examples/Makefile.am b/opal/mca/pmix/pmix2x/pmix/examples/Makefile.am index d2e9597a9cd..ae8bfe05af1 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/examples/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -28,52 +28,52 @@ if !WANT_HIDDEN noinst_PROGRAMS += server endif -client_SOURCES = client.c +client_SOURCES = client.c examples.h client_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) client_LDADD = $(top_builddir)/src/libpmix.la -client2_SOURCES = client2.c +client2_SOURCES = client2.c examples.h client2_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) client2_LDADD = $(top_builddir)/src/libpmix.la -debugger_SOURCES = debugger.c +debugger_SOURCES = debugger.c examples.h debugger_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) debugger_LDADD = $(top_builddir)/src/libpmix.la -debuggerd_SOURCES = debuggerd.c +debuggerd_SOURCES = debuggerd.c examples.h debuggerd_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) debuggerd_LDADD = $(top_builddir)/src/libpmix.la -alloc_SOURCES = alloc.c +alloc_SOURCES = alloc.c examples.h alloc_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) alloc_LDADD = $(top_builddir)/src/libpmix.la -jctrl_SOURCES = jctrl.c +jctrl_SOURCES = jctrl.c examples.h jctrl_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) jctrl_LDADD = $(top_builddir)/src/libpmix.la -dmodex_SOURCES = dmodex.c +dmodex_SOURCES = dmodex.c examples.h dmodex_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) dmodex_LDADD = $(top_builddir)/src/libpmix.la -dynamic_SOURCES = dynamic.c +dynamic_SOURCES = dynamic.c examples.h dynamic_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) dynamic_LDADD = $(top_builddir)/src/libpmix.la -fault_SOURCES = fault.c +fault_SOURCES = fault.c examples.h fault_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) fault_LDADD = $(top_builddir)/src/libpmix.la -pub_SOURCES = pub.c +pub_SOURCES = pub.c examples.h pub_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) pub_LDADD = $(top_builddir)/src/libpmix.la -tool_SOURCES = tool.c +tool_SOURCES = tool.c examples.h tool_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) tool_LDADD = $(top_builddir)/src/libpmix.la if !WANT_HIDDEN -server_SOURCES = server.c +server_SOURCES = server.c examples.h server_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) server_LDADD = $(top_builddir)/src/libpmix.la endif diff --git a/opal/mca/pmix/pmix2x/pmix/examples/alloc.c b/opal/mca/pmix/pmix2x/pmix/examples/alloc.c index f0cdf43a0ea..6984c1e1e96 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/alloc.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/alloc.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -30,16 +30,7 @@ #include #include - -/* define a structure for collecting returned - * info from an allocation request */ -typedef struct { - volatile bool active; - pmix_info_t *info; - size_t ninfo; -} mydata_t; - -static volatile bool waiting_for_allocation = true; +#include "examples.h" /* this is a callback function for the PMIx_Query and * PMIx_Allocate APIs. The query will callback with a status indicating @@ -59,7 +50,7 @@ static void infocbfunc(pmix_status_t status, pmix_release_cbfunc_t release_fn, void *release_cbdata) { - mydata_t *mq = (mydata_t*)cbdata; + myquery_data_t *mq = (myquery_data_t*)cbdata; size_t n; fprintf(stderr, "Allocation request returned %s", PMIx_Error_string(status)); @@ -75,6 +66,9 @@ static void infocbfunc(pmix_status_t status, PMIX_INFO_XFER(&mq->info[n], &info[n]); } } + /* the status returned here indicates whether the requested + * information was found or not - preserve it */ + mq->lock.status = status; /* let the library release the data and cleanup from * the operation */ @@ -83,7 +77,7 @@ static void infocbfunc(pmix_status_t status, } /* release the block */ - mq->active = false; + DEBUG_WAKEUP_THREAD(&mq->lock); } /* this is an event notification function that we explicitly request @@ -100,12 +94,37 @@ static void release_fn(size_t evhdlr_registration_id, pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata) { + myrel_t *lock; + size_t n; + + /* find the return object */ + lock = NULL; + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { + lock = (myrel_t*)info[n].value.data.ptr; + break; + } + } + /* if the object wasn't returned, then that is an error */ + if (NULL == lock) { + fprintf(stderr, "LOCK WASN'T RETURNED IN RELEASE CALLBACK\n"); + /* let the event handler progress */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } + return; + } + /* tell the event handler state machine that we are the last step */ if (NULL != cbfunc) { cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata); } - /* flag that the allocation is complete so we can exit */ - waiting_for_allocation = false; + /* the status will be PMIX_ERR_ALLOC_COMPLETE since that is the code + * we registered to receive. The result of the allocation request is + * in the info array - for now, just assume success */ + lock->lock.status = PMIX_SUCCESS; + /* release the lock */ + DEBUG_WAKEUP_THREAD(&lock->lock); } /* event handler registration is done asynchronously because it @@ -114,18 +133,20 @@ static void release_fn(size_t evhdlr_registration_id, * the status of the request (success or an error), plus a numerical index * to the registered event. The index is used later on to deregister * an event handler - if we don't explicitly deregister it, then the - * PMIx server will do so when it see us exit */ + * PMIx server will do so when it sees us exit */ static void evhandler_reg_callbk(pmix_status_t status, size_t evhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; if (PMIX_SUCCESS != status) { fprintf(stderr, "EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n", status, (unsigned long)evhandler_ref); } - *active = status; + lock->status = status; + lock->evhandler_ref = evhandler_ref; + DEBUG_WAKEUP_THREAD(lock); } int main(int argc, char **argv) @@ -138,11 +159,12 @@ int main(int argc, char **argv) uint32_t nprocs; pmix_info_t *info; uint64_t nnodes = 12; - mydata_t mydata; + myquery_data_t mydata; pmix_query_t *query; char *myallocation = "MYALLOCATION"; - volatile int active; - pmix_status_t code = PMIX_NOTIFY_ALLOC_COMPLETE; + mylock_t mylock; + pmix_status_t code; + myrel_t myrel; /* init us */ if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { @@ -163,63 +185,64 @@ int main(int argc, char **argv) PMIX_VALUE_RELEASE(val); fprintf(stderr, "Client %s:%d universe size %d\n", myproc.nspace, myproc.rank, nprocs); - /* initialize the return info struct */ - mydata.info = NULL; - mydata.ninfo = 0; - if (0 == myproc.rank) { /* try to get an allocation */ - mydata.active = true; + DEBUG_CONSTRUCT_MYQUERY(&mydata); PMIX_INFO_CREATE(info, 2); PMIX_INFO_LOAD(&info[0], PMIX_ALLOC_NUM_NODES, &nnodes, PMIX_UINT64); PMIX_INFO_LOAD(&info[0], PMIX_ALLOC_ID, myallocation, PMIX_STRING); - if (PMIX_SUCCESS != (rc = PMIx_Allocation_request_nb(PMIX_ALLOC_NEW, info, 2, infocbfunc, NULL))) { + if (PMIX_SUCCESS != (rc = PMIx_Allocation_request_nb(PMIX_ALLOC_NEW, info, 2, infocbfunc, &mydata))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Allocation_request_nb failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; } - while (mydata.active) { - usleep(10); - } + DEBUG_WAIT_THREAD(&mydata.lock); PMIX_INFO_FREE(info, 2); - if (NULL != mydata.info) { - PMIX_INFO_FREE(mydata.info, mydata.ninfo); - } + fprintf(stderr, "Client ns %s rank %d: Allocation returned status: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(mydata.lock.status)); + DEBUG_DESTRUCT_MYQUERY(&mydata); + } else if (1 == myproc.rank) { - /* register a handler specifically for when the allocation - * operation completes */ - PMIX_INFO_CREATE(info, 1); + /* demonstrate a notification based approach - register a handler + * specifically for when the allocation operation completes */ + DEBUG_CONSTRUCT_MYREL(&myrel); + PMIX_INFO_CREATE(info, 2); PMIX_INFO_LOAD(&info[0], PMIX_ALLOC_ID, myallocation, PMIX_STRING); - active = -1; - PMIx_Register_event_handler(&code, 1, info, 1, - release_fn, evhandler_reg_callbk, (void*)&active); - while (-1 == active) { - usleep(10); - } - if (0 != active) { - exit(active); - } - PMIX_INFO_FREE(info, 1); + PMIX_INFO_LOAD(&info[1], PMIX_EVENT_RETURN_OBJECT, &myrel, PMIX_POINTER); + DEBUG_CONSTRUCT_LOCK(&mylock); + code = PMIX_NOTIFY_ALLOC_COMPLETE; + PMIx_Register_event_handler(&code, 1, info, 2, + release_fn, evhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + PMIX_INFO_FREE(info, 2); + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + /* now wait to hear that the request is complete */ - while (waiting_for_allocation) { - usleep(10); - } + DEBUG_WAIT_THREAD(&myrel.lock); + fprintf(stderr, "[%s:%d] Allocation returned status: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(myrel.lock.status)); + DEBUG_DESTRUCT_MYREL(&myrel); + } else { - /* I am not the root rank, so let me wait a little while and then - * query the status of the allocation request */ + /* demonstrate a query-based approach - wait a little while and ask to + * see if it was done */ usleep(10); + DEBUG_CONSTRUCT_MYQUERY(&mydata); + PMIX_QUERY_CREATE(query, 1); PMIX_ARGV_APPEND(rc, query[0].keys, PMIX_QUERY_ALLOC_STATUS); PMIX_INFO_CREATE(query[0].qualifiers, 1); PMIX_INFO_LOAD(&query[0].qualifiers[0], PMIX_ALLOC_ID, myallocation, PMIX_STRING); - mydata.active = true; + if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, 1, infocbfunc, (void*)&mydata))) { fprintf(stderr, "PMIx_Query_info failed: %d\n", rc); goto done; } - while (mydata.active) { - usleep(10); - } + DEBUG_WAIT_THREAD(&mydata.lock); PMIX_QUERY_FREE(query, 1); + fprintf(stderr, "[%s:%d] Allocation returned status: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(mydata.lock.status)); + DEBUG_DESTRUCT_MYQUERY(&mydata); } done: diff --git a/opal/mca/pmix/pmix2x/pmix/examples/client.c b/opal/mca/pmix/pmix2x/pmix/examples/client.c index 7e38608950c..519ef649d62 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/client.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/client.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -31,8 +31,8 @@ #include #include +#include "examples.h" -static volatile bool waiting_for_debugger = true; static pmix_proc_t myproc; /* this is the event notification function we pass down below @@ -66,10 +66,36 @@ static void release_fn(size_t evhdlr_registration_id, pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata) { + myrel_t *lock; + size_t n; + + /* find the return object */ + lock = NULL; + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { + lock = (myrel_t*)info[n].value.data.ptr; + break; + } + } + /* if the object wasn't returned, then that is an error */ + if (NULL == lock) { + fprintf(stderr, "LOCK WASN'T RETURNED IN RELEASE CALLBACK\n"); + /* let the event handler progress */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } + return; + } + + /* tell the event handler state machine that we are the last step */ if (NULL != cbfunc) { cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata); } - waiting_for_debugger = false; + /* the status will be PMIX_ERR_DEBUGGER_RELEASE since that is the code + * we registered to receive, so just return success */ + lock->lock.status = PMIX_SUCCESS; + /* release the lock */ + DEBUG_WAKEUP_THREAD(&lock->lock); } /* event handler registration is done asynchronously because it @@ -83,27 +109,34 @@ static void evhandler_reg_callbk(pmix_status_t status, size_t evhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; if (PMIX_SUCCESS != status) { fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n", myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref); } - *active = status; + lock->status = status; + lock->evhandler_ref = evhandler_ref; + DEBUG_WAKEUP_THREAD(lock); } int main(int argc, char **argv) { - int rc; + pmix_status_t rc; pmix_value_t value; pmix_value_t *val = &value; - char *tmp, *ptr, *p; + char *tmp; pmix_proc_t proc; - uint32_t nprocs, m, n, local_cnt, *localpeers; + uint32_t nprocs, n; pmix_info_t *info; - bool flag, local; - volatile int active; + bool flag; + mylock_t mylock; + myrel_t myrel; pmix_status_t dbg = PMIX_ERR_DEBUGGER_RELEASE; + pid_t pid; + + pid = getpid(); + fprintf(stderr, "Client %lu: Running\n", (unsigned long)pid); /* init us - note that the call to "init" includes the return of * any job-related info provided by the RM. This includes any @@ -114,20 +147,21 @@ int main(int argc, char **argv) fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, rc); exit(0); } - fprintf(stderr, "Client ns %s rank %d: Running\n", myproc.nspace, myproc.rank); + fprintf(stderr, "Client ns %s rank %d pid %lu: Running\n", myproc.nspace, myproc.rank, (unsigned long)pid); /* register our default event handler - again, this isn't strictly * required, but is generally good practice */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); PMIx_Register_event_handler(NULL, 0, NULL, 0, - notification_fn, evhandler_reg_callbk, (void*)&active); - while (-1 == active) { - sleep(1); - } - if (0 != active) { + notification_fn, evhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + + if (PMIX_SUCCESS != rc) { fprintf(stderr, "[%s:%d] Default handler registration failed\n", myproc.nspace, myproc.rank); - exit(active); + goto done; } /* job-related info is found in our nspace, assigned to the @@ -145,21 +179,24 @@ int main(int argc, char **argv) * debugger */ if (PMIX_SUCCESS == (rc = PMIx_Get(&proc, PMIX_DEBUG_WAIT_FOR_NOTIFY, NULL, 0, &val))) { /* register for debugger release */ - active = -1; - PMIx_Register_event_handler(&dbg, 1, NULL, 0, - release_fn, evhandler_reg_callbk, (void*)&active); + DEBUG_CONSTRUCT_LOCK(&mylock); + PMIX_INFO_CREATE(info, 1); + DEBUG_CONSTRUCT_MYREL(&myrel); + PMIX_INFO_LOAD(&info[0], PMIX_EVENT_RETURN_OBJECT, &myrel, PMIX_POINTER); + PMIx_Register_event_handler(&dbg, 1, info, 1, + release_fn, evhandler_reg_callbk, (void*)&mylock); /* wait for registration to complete */ - while (-1 == active) { - sleep(1); - } - if (0 != active) { + DEBUG_WAIT_THREAD(&mylock); + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + PMIX_INFO_FREE(info, 1); + if (PMIX_SUCCESS != rc) { fprintf(stderr, "[%s:%d] Debug handler registration failed\n", myproc.nspace, myproc.rank); - exit(active); + goto done; } /* wait for debugger release */ - while (waiting_for_debugger) { - sleep(1); - } + DEBUG_WAIT_THREAD(&myrel.lock); + DEBUG_DESTRUCT_MYREL(&myrel); } /* get our universe size */ @@ -196,7 +233,7 @@ int main(int argc, char **argv) value.type = PMIX_UINT64; value.data.uint64 = 1234; if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_LOCAL, tmp, &value))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Put local failed: %d\n", myproc.nspace, myproc.rank, rc); + fprintf(stderr, "Client ns %s rank %d: PMIx_Put internal failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; } free(tmp); @@ -207,7 +244,7 @@ int main(int argc, char **argv) value.type = PMIX_STRING; value.data.string = "1234"; if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_REMOTE, tmp, &value))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Put remote failed: %d\n", myproc.nspace, myproc.rank, rc); + fprintf(stderr, "Client ns %s rank %d: PMIx_Put internal failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; } free(tmp); @@ -230,96 +267,53 @@ int main(int argc, char **argv) } PMIX_INFO_FREE(info, 1); - /* get the number of local peers */ - if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_LOCAL_SIZE, NULL, 0, &val))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get PMIX_LOCAL_SIZE failed: %d", myproc.nspace, myproc.rank, rc); - goto done; - } - local_cnt = val->data.uint32; - PMIX_VALUE_RELEASE(val); - - /* create an array for the peers */ - localpeers = (uint32_t*)malloc(local_cnt * sizeof(int)); - - /* get the list of local peers */ - if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_LOCAL_PEERS, NULL, 0, &val))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get PMIX_LOCAL_PEERS failed: %d", myproc.nspace, myproc.rank, rc); - goto done; - } - ptr = strdup(val->data.string); - PMIX_VALUE_RELEASE(val); - - /* populate the peers array */ - p = strtok(ptr, ","); - localpeers[0] = strtoul(p, NULL, 10); - for (n=1; n < local_cnt; n++) { - p = strtok(NULL, ","); - localpeers[n] = strtoul(p, NULL, 10); - } - free(ptr); - /* check the returned data */ for (n=0; n < nprocs; n++) { - if (n == myproc.rank) { - continue; + if (0 > asprintf(&tmp, "%s-%d-local", myproc.nspace, myproc.rank)) { + exit(1); + } + if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, tmp, NULL, 0, &val))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, myproc.rank, tmp, rc); + goto done; + } + if (PMIX_UINT64 != val->type) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d\n", myproc.nspace, myproc.rank, tmp, val->type); + PMIX_VALUE_RELEASE(val); + free(tmp); + goto done; + } + if (1234 != val->data.uint64) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong value: %d\n", myproc.nspace, myproc.rank, tmp, (int)val->data.uint64); + PMIX_VALUE_RELEASE(val); + free(tmp); + goto done; + } + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned correct\n", myproc.nspace, myproc.rank, tmp); + PMIX_VALUE_RELEASE(val); + free(tmp); + if (0 > asprintf(&tmp, "%s-%d-remote", myproc.nspace, myproc.rank)) { + exit(1); } - proc.rank = n; - local = false; - for (m=0; m < local_cnt; m++) { - if (localpeers[m] == proc.rank) { - local = true; - break; - } + if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, tmp, NULL, 0, &val))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, myproc.rank, tmp, rc); + goto done; } - if (local) { - if (0 > asprintf(&tmp, "%s-%d-local", proc.nspace, proc.rank)) { - exit(1); - } - if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, tmp, NULL, 0, &val))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, myproc.rank, tmp, rc); - goto done; - } - if (PMIX_UINT64 != val->type) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d\n", myproc.nspace, myproc.rank, tmp, val->type); - PMIX_VALUE_RELEASE(val); - free(tmp); - goto done; - } - if (1234 != val->data.uint64) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong value: %d\n", myproc.nspace, myproc.rank, tmp, (int)val->data.uint64); - PMIX_VALUE_RELEASE(val); - free(tmp); - goto done; - } - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned correct\n", myproc.nspace, myproc.rank, tmp); + if (PMIX_STRING != val->type) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d\n", myproc.nspace, myproc.rank, tmp, val->type); PMIX_VALUE_RELEASE(val); free(tmp); - } else { - if (0 > asprintf(&tmp, "%s-%d-remote", proc.nspace, proc.rank)) { - exit(1); - } - if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, tmp, NULL, 0, &val))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, myproc.rank, tmp, rc); - goto done; - } - if (PMIX_STRING != val->type) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d\n", myproc.nspace, myproc.rank, tmp, val->type); - PMIX_VALUE_RELEASE(val); - free(tmp); - goto done; - } - if (0 != strcmp(val->data.string, "1234")) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong value: %s\n", myproc.nspace, myproc.rank, tmp, val->data.string); - PMIX_VALUE_RELEASE(val); - free(tmp); - goto done; - } - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned correct\n", myproc.nspace, myproc.rank, tmp); + goto done; + } + if (0 != strcmp(val->data.string, "1234")) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong value: %s\n", myproc.nspace, myproc.rank, tmp, val->data.string); PMIX_VALUE_RELEASE(val); free(tmp); + goto done; } + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned correct\n", myproc.nspace, myproc.rank, tmp); + PMIX_VALUE_RELEASE(val); + free(tmp); } - free(localpeers); done: /* finalize us */ diff --git a/opal/mca/pmix/pmix2x/pmix/examples/client2.c b/opal/mca/pmix/pmix2x/pmix/examples/client2.c index 9b7f7e6c0fe..4021c15ec7c 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/client2.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/client2.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -31,6 +31,7 @@ #include #include +#include "examples.h" static pmix_proc_t myproc; @@ -62,25 +63,27 @@ static void evhandler_reg_callbk(pmix_status_t status, size_t evhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; if (PMIX_SUCCESS != status) { fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n", myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref); } - *active = status; + lock->status = status; + lock->evhandler_ref = evhandler_ref; + DEBUG_WAKEUP_THREAD(lock); } int main(int argc, char **argv) { - int rc; + pmix_status_t rc; pmix_value_t value; pmix_value_t *val, *vptr; pmix_proc_t proc; uint32_t nprocs, n, k; pmix_info_t *info; bool flag; - volatile int active; + mylock_t mylock; pmix_data_array_t da, *dptr; /* init us - note that the call to "init" includes the return of @@ -97,15 +100,16 @@ int main(int argc, char **argv) /* register our default event handler - again, this isn't strictly * required, but is generally good practice */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); PMIx_Register_event_handler(NULL, 0, NULL, 0, - notification_fn, evhandler_reg_callbk, (void*)&active); - while (-1 == active) { - sleep(1); - } - if (0 != active) { + notification_fn, evhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + + if (PMIX_SUCCESS != rc) { fprintf(stderr, "[%s:%d] Default handler registration failed\n", myproc.nspace, myproc.rank); - exit(active); + goto done; } /* job-related info is found in our nspace, assigned to the diff --git a/opal/mca/pmix/pmix2x/pmix/examples/debugger.c b/opal/mca/pmix/pmix2x/pmix/examples/debugger.c index b84bd610937..e1a2b815f42 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/debugger.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/debugger.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -23,24 +23,63 @@ * */ +#define _GNU_SOURCE #include #include #include #include +#include #include +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond; + volatile bool active; + pmix_status_t status; +} mylock_t; + +#define DEBUG_CONSTRUCT_LOCK(l) \ + do { \ + pthread_mutex_init(&(l)->mutex, NULL); \ + pthread_cond_init(&(l)->cond, NULL); \ + (l)->active = true; \ + (l)->status = PMIX_SUCCESS; \ + } while(0) + +#define DEBUG_DESTRUCT_LOCK(l) \ + do { \ + pthread_mutex_destroy(&(l)->mutex); \ + pthread_cond_destroy(&(l)->cond); \ + } while(0) + +#define DEBUG_WAIT_THREAD(lck) \ + do { \ + pthread_mutex_lock(&(lck)->mutex); \ + while ((lck)->active) { \ + pthread_cond_wait(&(lck)->cond, &(lck)->mutex); \ + } \ + pthread_mutex_unlock(&(lck)->mutex); \ + } while(0) + +#define DEBUG_WAKEUP_THREAD(lck) \ + do { \ + pthread_mutex_lock(&(lck)->mutex); \ + (lck)->active = false; \ + pthread_cond_broadcast(&(lck)->cond); \ + pthread_mutex_unlock(&(lck)->mutex); \ + } while(0) /* define a structure for collecting returned * info from a query */ typedef struct { - volatile bool active; + mylock_t lock; pmix_info_t *info; size_t ninfo; } myquery_data_t; static int attach_to_running_job(char *nspace); -static bool waiting_for_debugger = true; +static mylock_t waiting_for_debugger; static pmix_proc_t myproc; /* this is a callback function for the PMIx_Query @@ -83,7 +122,7 @@ static void cbfunc(pmix_status_t status, } /* release the block */ - mq->active = false; + DEBUG_WAKEUP_THREAD(&mq->lock); } /* this is the event notification function we pass down below @@ -124,7 +163,7 @@ static void release_fn(size_t evhdlr_registration_id, cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata); } /* flag that the debugger is complete so we can exit */ - waiting_for_debugger = false; + DEBUG_WAKEUP_THREAD(&waiting_for_debugger); } /* event handler registration is done asynchronously because it @@ -138,31 +177,71 @@ static void evhandler_reg_callbk(pmix_status_t status, size_t evhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; if (PMIX_SUCCESS != status) { fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n", myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref); } - *active = status; + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); +} + +static pmix_status_t spawn_debugger(char *appspace) +{ + pmix_status_t rc; + pmix_info_t *dinfo; + pmix_app_t *debugger; + size_t dninfo; + char cwd[1024]; + char dspace[PMIX_MAX_NSLEN+1]; + + /* setup the debugger */ + PMIX_APP_CREATE(debugger, 1); + debugger[0].cmd = strdup("./debuggerd"); + PMIX_ARGV_APPEND(rc, debugger[0].argv, "./debuggerd"); + getcwd(cwd, 1024); // point us to our current directory + debugger[0].cwd = strdup(cwd); + /* provide directives so the daemons go where we want, and + * let the RM know these are debugger daemons */ + dninfo = 5; + PMIX_INFO_CREATE(dinfo, dninfo); + PMIX_INFO_LOAD(&dinfo[0], PMIX_MAPBY, "ppr:1:node", PMIX_STRING); // instruct the RM to launch one copy of the executable on each node + PMIX_INFO_LOAD(&dinfo[1], PMIX_DEBUGGER_DAEMONS, NULL, PMIX_BOOL); // these are debugger daemons + PMIX_INFO_LOAD(&dinfo[2], PMIX_DEBUG_JOB, appspace, PMIX_STRING); // the nspace being debugged + PMIX_INFO_LOAD(&dinfo[3], PMIX_NOTIFY_COMPLETION, NULL, PMIX_BOOL); // notify us when the debugger job completes + PMIX_INFO_LOAD(&dinfo[4], PMIX_DEBUG_WAITING_FOR_NOTIFY, NULL, PMIX_BOOL); // tell the daemon that the proc is waiting to be released + /* spawn the daemons */ + fprintf(stderr, "Debugger: spawning %s\n", debugger[0].cmd); + if (PMIX_SUCCESS != (rc = PMIx_Spawn(dinfo, dninfo, debugger, 1, dspace))) { + fprintf(stderr, "Debugger daemons failed to launch with error: %s\n", PMIx_Error_string(rc)); + } + fprintf(stderr, "SPAWNED DEBUGGERD\n"); + /* cleanup */ + PMIX_INFO_FREE(dinfo, dninfo); + PMIX_APP_FREE(debugger, 1); + + return rc; } +#define DBGR_LOOP_LIMIT 10 + int main(int argc, char **argv) { pmix_status_t rc; - pmix_info_t *info, *dinfo; - pmix_app_t *app, *debugger; - size_t ninfo, napps, dninfo; - char *tdir, *nspace = NULL; - char appspace[PMIX_MAX_NSLEN+1], dspace[PMIX_MAX_NSLEN+1]; + pmix_info_t *info; + pmix_app_t *app; + size_t ninfo, napps; + char *nspace = NULL; + char appspace[PMIX_MAX_NSLEN+1]; int i; pmix_query_t *query; size_t nq, n; myquery_data_t myquery_data; bool cospawn = false, stop_on_exec = false; char cwd[1024]; - volatile int active; pmix_status_t code = PMIX_ERR_JOB_TERMINATED; + mylock_t mylock; /* Process any arguments we were given */ for (i=1; i < argc; i++) { @@ -191,33 +270,15 @@ int main(int argc, char **argv) exit(1); } } + info = NULL; + ninfo = 0; - /* we need to provide some info to the PMIx tool library so - * it can find the server's contact info. The simplest way - * of doing this here is to look for an environmental variable - * that tells us where to look. The PMIx reference server only - * allows one instantiation of the server per user, so setting - * this up is something a user could do in their login script. - * The reference server is based on OpenMPI, and so the contact - * info will always be found at: - * - * $TMPDIR/ompi../dvm - * - * NOTE: we will eliminate this requirement in a future version - */ - - if (NULL == (tdir = getenv("PMIX_SERVER_TMPDIR"))) { - fprintf(stderr, "Tool usage requires that the PMIX_SERVER_TMPDIR envar\n"); - fprintf(stderr, "be set to point at the directory where the PMIx Reference\n"); - fprintf(stderr, "Server leaves its contact info file.\n"); - exit(1); - } - - /* init us - pass along the location of the contact file */ - ninfo = 1; - PMIX_INFO_CREATE(info, ninfo); - PMIX_INFO_LOAD(&info[0], PMIX_SERVER_TMPDIR, tdir, PMIX_STRING); + DEBUG_CONSTRUCT_LOCK(&waiting_for_debugger); + /* use the system connection first, if available */ + PMIX_INFO_CREATE(info, 1); + PMIX_INFO_LOAD(&info[0], PMIX_CONNECT_SYSTEM_FIRST, NULL, PMIX_BOOL); + /* init as a tool */ if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, info, ninfo))) { fprintf(stderr, "PMIx_tool_init failed: %d\n", rc); exit(rc); @@ -227,27 +288,19 @@ int main(int argc, char **argv) fprintf(stderr, "Tool ns %s rank %d: Running\n", myproc.nspace, myproc.rank); /* register a default event handler */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); PMIx_Register_event_handler(NULL, 0, NULL, 0, - notification_fn, evhandler_reg_callbk, (void*)&active); - while (-1 == active) { - usleep(10); - } - if (0 != active) { - exit(active); - } + notification_fn, evhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + DEBUG_DESTRUCT_LOCK(&mylock); /* register another handler specifically for when the debugger * job completes */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); PMIx_Register_event_handler(&code, 1, NULL, 0, - release_fn, evhandler_reg_callbk, (void*)&active); - while (-1 == active) { - usleep(10); - } - if (0 != active) { - exit(active); - } + release_fn, evhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + DEBUG_DESTRUCT_LOCK(&mylock); /* if we are attaching to a running job, then attach to it */ if (NULL != nspace) { @@ -270,18 +323,17 @@ int main(int argc, char **argv) PMIX_ARGV_APPEND(rc, query[0].keys, PMIX_QUERY_SPAWN_SUPPORT); PMIX_ARGV_APPEND(rc, query[0].keys, PMIX_QUERY_DEBUG_SUPPORT); /* setup the caddy to retrieve the data */ + DEBUG_CONSTRUCT_LOCK(&myquery_data.lock); myquery_data.info = NULL; myquery_data.ninfo = 0; - myquery_data.active = true; /* execute the query */ fprintf(stderr, "Debugger: querying capabilities\n"); if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&myquery_data))) { fprintf(stderr, "PMIx_Query_info failed: %d\n", rc); goto done; } - while (myquery_data.active) { - usleep(10); - } + DEBUG_WAIT_THREAD(&myquery_data.lock); + DEBUG_DESTRUCT_LOCK(&myquery_data.lock); /* we should have received back two info structs, one containing * a comma-delimited list of PMIx spawn attributes the RM supports, @@ -334,13 +386,11 @@ int main(int argc, char **argv) /* setup the executable */ app[0].cmd = strdup("client"); PMIX_ARGV_APPEND(rc, app[0].argv, "./client"); - if (NULL == getcwd(cwd, 1024)) { // point us to our current directory - goto done; - } + getcwd(cwd, 1024); // point us to our current directory app[0].cwd = strdup(cwd); app[0].maxprocs = 2; /* provide job-level directives so the apps do what the user requested */ - ninfo = 2; + ninfo = 4; PMIX_INFO_CREATE(info, ninfo); PMIX_INFO_LOAD(&info[0], PMIX_MAPBY, "slot", PMIX_STRING); // map by slot if (stop_on_exec) { @@ -348,113 +398,59 @@ int main(int argc, char **argv) } else { PMIX_INFO_LOAD(&info[1], PMIX_DEBUG_STOP_IN_INIT, NULL, PMIX_BOOL); // procs are to pause in PMIx_Init for debugger attach } + PMIX_INFO_LOAD(&info[2], PMIX_FWD_STDOUT, NULL, PMIX_BOOL); // forward stdout to me + PMIX_INFO_LOAD(&info[3], PMIX_FWD_STDERR, NULL, PMIX_BOOL); // forward stderr to me + /* spawn the job - the function will return when the app * has been launched */ fprintf(stderr, "Debugger: spawning %s\n", app[0].cmd); if (PMIX_SUCCESS != (rc = PMIx_Spawn(info, ninfo, app, napps, appspace))) { - fprintf(stderr, "Application failed to launch with error: %s\n", PMIx_Error_string(rc)); + fprintf(stderr, "Application failed to launch with error: %s(%d)\n", PMIx_Error_string(rc), rc); goto done; } PMIX_INFO_FREE(info, ninfo); PMIX_APP_FREE(app, napps); - /* setup the debugger */ - PMIX_APP_CREATE(debugger, 1); - debugger[0].cmd = strdup("./debuggerd"); - PMIX_ARGV_APPEND(rc, debugger[0].argv, "./debuggerd"); - debugger[0].cwd = strdup(cwd); - /* provide directives so the daemons go where we want, and - * let the RM know these are debugger daemons */ - dninfo = 5; - PMIX_INFO_CREATE(dinfo, dninfo); - PMIX_INFO_LOAD(&dinfo[0], PMIX_MAPBY, "ppr:1:node", PMIX_STRING); // instruct the RM to launch one copy of the executable on each node - PMIX_INFO_LOAD(&dinfo[1], PMIX_DEBUGGER_DAEMONS, NULL, PMIX_BOOL); // these are debugger daemons - PMIX_INFO_LOAD(&dinfo[2], PMIX_DEBUG_JOB, appspace, PMIX_STRING); // the nspace being debugged - PMIX_INFO_LOAD(&dinfo[3], PMIX_NOTIFY_COMPLETION, NULL, PMIX_BOOL); // notify us when the debugger job completes - PMIX_INFO_LOAD(&dinfo[4], PMIX_DEBUG_WAITING_FOR_NOTIFY, NULL, PMIX_BOOL); // tell the daemon that the proc is waiting to be released - /* spawn the daemons */ - fprintf(stderr, "Debugger: spawning %s\n", debugger[0].cmd); - if (PMIX_SUCCESS != (rc = PMIx_Spawn(dinfo, dninfo, debugger, 1, dspace))) { - fprintf(stderr, "Debugger daemons failed to launch with error: %s\n", PMIx_Error_string(rc)); + /* now launch the debugger daemons */ + if (PMIX_SUCCESS != (rc = spawn_debugger(appspace))) { goto done; } - - /* cleanup */ - PMIX_INFO_FREE(dinfo, dninfo); - PMIX_APP_FREE(debugger, 1); } /* this is where a debugger tool would wait until the debug operation is complete */ - while (waiting_for_debugger) { - sleep(1); - } + DEBUG_WAIT_THREAD(&waiting_for_debugger); } done: + DEBUG_DESTRUCT_LOCK(&waiting_for_debugger); PMIx_tool_finalize(); return(rc); } -typedef struct { - volatile bool active; - pmix_status_t status; - pmix_info_t *info; - size_t ninfo; -} mydbug_query_t; - - -static void infocbfunc(pmix_status_t status, - pmix_info_t *info, size_t ninfo, - void *cbdata, - pmix_release_cbfunc_t release_fn, - void *release_cbdata) -{ - mydbug_query_t *q = (mydbug_query_t*)cbdata; - size_t n; - - q->status = status; - q->info = NULL; - q->ninfo = ninfo; - if (0 < ninfo) { - PMIX_INFO_CREATE(q->info, q->ninfo); - for (n=0; n < ninfo; n++) { - PMIX_INFO_XFER(&q->info[n], &info[n]); - } - } - if (NULL != release_fn) { - release_fn(release_cbdata); - } - q->active = false; -} - static int attach_to_running_job(char *nspace) { pmix_status_t rc; + pmix_proc_t myproc; pmix_query_t *query; size_t nq; - mydbug_query_t *q; + myquery_data_t *q; /* query the active nspaces so we can verify that the * specified one exists */ nq = 1; PMIX_QUERY_CREATE(query, nq); - query[0].keys = (char**)malloc(2 * sizeof(char*)); - query[0].keys[0] = strdup(PMIX_QUERY_NAMESPACES); - query[0].keys[1] = NULL; - - q = (mydbug_query_t*)malloc(sizeof(mydbug_query_t)); - q->active = true; + PMIX_ARGV_APPEND(rc, query[0].keys, PMIX_QUERY_NAMESPACES); - if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, infocbfunc, (void*)q))) { + q = (myquery_data_t*)malloc(sizeof(myquery_data_t)); + DEBUG_CONSTRUCT_LOCK(&q->lock); + if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)q))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Query_info failed: %d\n", myproc.nspace, myproc.rank, rc); return -1; } - /* wait for a response */ - while (q->active) { - sleep(1); - } + DEBUG_WAIT_THREAD(&q->lock); + DEBUG_DESTRUCT_LOCK(&q->lock); if (NULL == q->info) { fprintf(stderr, "Query returned no info\n"); diff --git a/opal/mca/pmix/pmix2x/pmix/examples/debuggerd.c b/opal/mca/pmix/pmix2x/pmix/examples/debuggerd.c index 5924dca7171..00e125e79d5 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/debuggerd.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/debuggerd.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -133,6 +133,11 @@ int main(int argc, char **argv) size_t nq, n; myquery_data_t myquery_data; +fprintf(stderr, "I AM HERE\n"); +fflush(stderr); + sleep(10); + exit(0); + /* init us - since we were launched by the RM, our connection info * will have been provided at startup. */ if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, NULL, 0))) { @@ -210,7 +215,7 @@ int main(int argc, char **argv) n = 0; fprintf(stderr, "[%s:%u] Hanging around awhile, doing debugger magic\n", myproc.nspace, myproc.rank); while (n < 5) { - usleep(10); + usleep(1000); ++n; } diff --git a/opal/mca/pmix/pmix2x/pmix/examples/dmodex.c b/opal/mca/pmix/pmix2x/pmix/examples/dmodex.c index c093d5bc829..c7b906b1587 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/dmodex.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/dmodex.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -33,24 +33,19 @@ #include #include +#include "examples.h" static uint32_t nprocs; static pmix_proc_t myproc; static uint32_t getcount = 0; -#define WAIT_FOR_COMPLETION(a) \ - do { \ - while ((a)) { \ - usleep(10); \ - } \ - } while (0) - static void opcbfunc(pmix_status_t status, void *cbdata) { - bool *active = (bool*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; fprintf(stderr, "%s:%d completed fence_nb\n", myproc.nspace, myproc.rank); - *active = false; + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); } static void valcbfunc(pmix_status_t status, @@ -98,7 +93,7 @@ int main(int argc, char **argv) char *tmp; pmix_proc_t proc; uint32_t n, num_gets; - bool active; + mylock_t mylock; /* init us */ if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { @@ -170,9 +165,10 @@ int main(int argc, char **argv) PMIX_PROC_CONSTRUCT(&proc); (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; - active = true; - if (PMIX_SUCCESS != (rc = PMIx_Fence_nb(&proc, 1, NULL, 0, opcbfunc, &active))) { + DEBUG_CONSTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS != (rc = PMIx_Fence_nb(&proc, 1, NULL, 0, opcbfunc, &mylock))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d\n", myproc.nspace, myproc.rank, rc); + DEBUG_DESTRUCT_LOCK(&mylock); goto done; } @@ -182,7 +178,6 @@ int main(int argc, char **argv) if (0 > asprintf(&tmp, "%s-%d-local", myproc.nspace, n)) { exit(1); } - (void)strncpy(proc.nspace, tmp, PMIX_MAX_NSLEN); proc.rank = n; if (PMIX_SUCCESS != (rc = PMIx_Get_nb(&proc, tmp, NULL, 0, valcbfunc, tmp))) { @@ -193,7 +188,6 @@ int main(int argc, char **argv) if (0 > asprintf(&tmp, "%s-%d-remote", myproc.nspace, n)) { exit(1); } - (void)strncpy(proc.nspace, tmp, PMIX_MAX_NSLEN); if (PMIX_SUCCESS != (rc = PMIx_Get_nb(&proc, tmp, NULL, 0, valcbfunc, tmp))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, n, tmp, rc); @@ -203,7 +197,7 @@ int main(int argc, char **argv) } /* wait for the first fence to finish */ - WAIT_FOR_COMPLETION(active); + DEBUG_WAIT_THREAD(&mylock); /* wait for all my "get" calls to complete */ while (getcount < num_gets) { diff --git a/opal/mca/pmix/pmix2x/pmix/examples/dynamic.c b/opal/mca/pmix/pmix2x/pmix/examples/dynamic.c index 3d0942b2dc6..d7c0d3701ff 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/dynamic.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/dynamic.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -35,7 +35,7 @@ #include #include - +#include "examples.h" static pmix_proc_t myproc; @@ -139,7 +139,8 @@ int main(int argc, char **argv) fprintf(stderr, "Client ns %s rank %d: PMIx_Connect failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; } - fprintf(stderr, "Client ns %s rank %d: PMIx_Connect succeeded\n", myproc.nspace, myproc.rank); + fprintf(stderr, "Client ns %s rank %d: PMIx_Connect succeeded\n", + myproc.nspace, myproc.rank); if (PMIX_SUCCESS != (rc = PMIx_Disconnect(&proc, 1, NULL, 0))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Disonnect failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; diff --git a/opal/mca/pmix/pmix2x/pmix/examples/examples.h b/opal/mca/pmix/pmix2x/pmix/examples/examples.h new file mode 100644 index 00000000000..d2e5ab7696d --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/examples/examples.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include + +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond; + volatile bool active; + pmix_status_t status; + int count; + size_t evhandler_ref; +} mylock_t; + +#define DEBUG_CONSTRUCT_LOCK(l) \ + do { \ + pthread_mutex_init(&(l)->mutex, NULL); \ + pthread_cond_init(&(l)->cond, NULL); \ + (l)->active = true; \ + (l)->status = PMIX_SUCCESS; \ + (l)->count = 0; \ + (l)->evhandler_ref = 0; \ + } while(0) + +#define DEBUG_DESTRUCT_LOCK(l) \ + do { \ + pthread_mutex_destroy(&(l)->mutex); \ + pthread_cond_destroy(&(l)->cond); \ + } while(0) + +#define DEBUG_WAIT_THREAD(lck) \ + do { \ + pthread_mutex_lock(&(lck)->mutex); \ + while ((lck)->active) { \ + pthread_cond_wait(&(lck)->cond, &(lck)->mutex); \ + } \ + pthread_mutex_unlock(&(lck)->mutex); \ + } while(0) + +#define DEBUG_WAKEUP_THREAD(lck) \ + do { \ + pthread_mutex_lock(&(lck)->mutex); \ + (lck)->active = false; \ + pthread_cond_broadcast(&(lck)->cond); \ + pthread_mutex_unlock(&(lck)->mutex); \ + } while(0) + +/* define a structure for collecting returned + * info from a query */ +typedef struct { + mylock_t lock; + pmix_info_t *info; + size_t ninfo; +} myquery_data_t; + +#define DEBUG_CONSTRUCT_MYQUERY(q) \ + do { \ + DEBUG_CONSTRUCT_LOCK(&((q)->lock)); \ + (q)->info = NULL; \ + (q)->ninfo = 0; \ + } while(0) + +#define DEBUG_DESTRUCT_MYQUERY(q) \ + do { \ + DEBUG_DESTRUCT_LOCK(&((q)->lock)); \ + if (NULL != (q)->info) { \ + PMIX_INFO_FREE((q)->info, (q)->ninfo); \ + } \ + } while(0) + +/* define a structure for releasing when a given + * nspace terminates */ +typedef struct { + mylock_t lock; + char *nspace; + int exit_code; + bool exit_code_given; +} myrel_t; + + +#define DEBUG_CONSTRUCT_MYREL(r) \ + do { \ + DEBUG_CONSTRUCT_LOCK(&((r)->lock)); \ + (r)->nspace = NULL; \ + (r)->exit_code = 0; \ + (r)->exit_code_given = false; \ + } while(0) + +#define DEBUG_DESTRUCT_MYREL(r) \ + do { \ + DEBUG_DESTRUCT_LOCK(&((r)->lock)); \ + if (NULL != (r)->nspace) { \ + free((r)->nspace); \ + } \ + } while(0) diff --git a/opal/mca/pmix/pmix2x/pmix/examples/fault.c b/opal/mca/pmix/pmix2x/pmix/examples/fault.c index a197f589619..1694583a2fe 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/fault.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/fault.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -28,11 +28,12 @@ #include #include #include +#include #include +#include "examples.h" static pmix_proc_t myproc; -static bool completed; static void notification_fn(size_t evhdlr_registration_id, pmix_status_t status, @@ -42,22 +43,57 @@ static void notification_fn(size_t evhdlr_registration_id, pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata) { - fprintf(stderr, "Client %s:%d NOTIFIED with status %d\n", myproc.nspace, myproc.rank, status); - completed = true; + myrel_t *lock; + size_t n; + pmix_proc_t *affected = NULL; + + /* find our return object */ + lock = NULL; + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { + lock = (myrel_t*)info[n].value.data.ptr; + /* not every RM will provide an exit code, but check if one was given */ + } else if (0 == strncmp(info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + affected = info[n].value.data.proc; + } + } + /* if the object wasn't returned, then that is an error */ + if (NULL == lock) { + fprintf(stderr, "LOCK WASN'T RETURNED IN RELEASE CALLBACK\n"); + /* let the event handler progress */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } + return; + } + + /* tell the event handler state machine that we are the last step */ + if (NULL != cbfunc) { + cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata); + } + fprintf(stderr, "DEBUGGER DAEMON NOTIFIED TERMINATED - AFFECTED %s\n", + (NULL == affected) ? "NULL" : affected->nspace); + + DEBUG_WAKEUP_THREAD(&lock->lock); } static void op_callbk(pmix_status_t status, void *cbdata) { + mylock_t *lock = (mylock_t*)cbdata; fprintf(stderr, "Client %s:%d OP CALLBACK CALLED WITH STATUS %d\n", myproc.nspace, myproc.rank, status); + DEBUG_WAKEUP_THREAD(lock); } -static void errhandler_reg_callbk(pmix_status_t status, +static void evhandler_reg_callbk(pmix_status_t status, size_t errhandler_ref, void *cbdata) { + mylock_t *lock = (mylock_t*)cbdata; + fprintf(stderr, "Client %s:%d ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu\n", myproc.nspace, myproc.rank, status, (unsigned long)errhandler_ref); + DEBUG_WAKEUP_THREAD(lock); } int main(int argc, char **argv) @@ -67,6 +103,10 @@ int main(int argc, char **argv) pmix_value_t *val = &value; pmix_proc_t proc; uint32_t nprocs; + pmix_info_t *info; + mylock_t mylock; + myrel_t myrel; + pmix_status_t code[2] = {PMIX_ERR_PROC_ABORTED, PMIX_ERR_JOB_TERMINATED}; /* init us */ if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { @@ -87,11 +127,27 @@ int main(int argc, char **argv) nprocs = val->data.uint32; PMIX_VALUE_RELEASE(val); fprintf(stderr, "Client %s:%d universe size %d\n", myproc.nspace, myproc.rank, nprocs); - completed = false; - /* register our errhandler */ - PMIx_Register_event_handler(NULL, 0, NULL, 0, - notification_fn, errhandler_reg_callbk, NULL); + /* register another handler specifically for when the target + * job completes */ + DEBUG_CONSTRUCT_MYREL(&myrel); + PMIX_INFO_CREATE(info, 2); + PMIX_INFO_LOAD(&info[0], PMIX_EVENT_RETURN_OBJECT, &myrel, PMIX_POINTER); + /* only call me back when one of us terminates */ + PMIX_INFO_LOAD(&info[1], PMIX_NSPACE, myproc.nspace, PMIX_STRING); + + DEBUG_CONSTRUCT_LOCK(&mylock); + PMIx_Register_event_handler(code, 2, info, 2, + notification_fn, evhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + if (PMIX_SUCCESS != mylock.status) { + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + PMIX_INFO_FREE(info, 2); + goto done; + } + DEBUG_DESTRUCT_LOCK(&mylock); + PMIX_INFO_FREE(info, 2); /* call fence to sync */ PMIX_PROC_CONSTRUCT(&proc); @@ -109,17 +165,16 @@ int main(int argc, char **argv) exit(1); } /* everyone simply waits */ - while (!completed) { - struct timespec ts; - ts.tv_sec = 0; - ts.tv_nsec = 100000; - nanosleep(&ts, NULL); - } + DEBUG_WAIT_THREAD(&myrel.lock); + DEBUG_DESTRUCT_MYREL(&myrel); done: /* finalize us */ fprintf(stderr, "Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank); - PMIx_Deregister_event_handler(1, op_callbk, NULL); + DEBUG_CONSTRUCT_LOCK(&mylock); + PMIx_Deregister_event_handler(1, op_callbk, &mylock); + DEBUG_WAIT_THREAD(&mylock); + DEBUG_DESTRUCT_LOCK(&mylock); if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); diff --git a/opal/mca/pmix/pmix2x/pmix/examples/jctrl.c b/opal/mca/pmix/pmix2x/pmix/examples/jctrl.c index 5c1c1d1f73d..0dc244b3c62 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/jctrl.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/jctrl.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -32,6 +32,7 @@ #include #include +#include "examples.h" static pmix_proc_t myproc; @@ -63,13 +64,15 @@ static void evhandler_reg_callbk(pmix_status_t status, size_t evhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; if (PMIX_SUCCESS != status) { fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n", myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref); } - *active = status; + lock->status = status; + lock->evhandler_ref = evhandler_ref; + DEBUG_WAKEUP_THREAD(lock); } static void infocbfunc(pmix_status_t status, @@ -78,26 +81,27 @@ static void infocbfunc(pmix_status_t status, pmix_release_cbfunc_t release_fn, void *release_cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; /* release the caller */ if (NULL != release_fn) { release_fn(release_cbdata); } - *active = status; + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); } int main(int argc, char **argv) { - int rc; + pmix_status_t rc; pmix_value_t value; pmix_value_t *val = &value; pmix_proc_t proc; uint32_t nprocs, n; pmix_info_t *info, *iptr; bool flag; - volatile int active; + mylock_t mylock; pmix_data_array_t *dptr; /* init us - note that the call to "init" includes the return of @@ -111,15 +115,16 @@ int main(int argc, char **argv) /* register our default event handler - again, this isn't strictly * required, but is generally good practice */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); PMIx_Register_event_handler(NULL, 0, NULL, 0, - notification_fn, evhandler_reg_callbk, (void*)&active); - while (-1 == active) { - sleep(1); - } - if (0 != active) { + notification_fn, evhandler_reg_callbk, (void*)&mylock); + /* wait for registration to complete */ + DEBUG_WAIT_THREAD(&mylock); + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS != rc) { fprintf(stderr, "[%s:%d] Default handler registration failed\n", myproc.nspace, myproc.rank); - exit(active); + goto done; } /* job-related info is found in our nspace, assigned to the @@ -145,12 +150,8 @@ int main(int argc, char **argv) PMIX_INFO_LOAD(&info[0], PMIX_JOB_CTRL_PREEMPTIBLE, (void*)&flag, PMIX_BOOL); /* can't use "load" to load a pmix_data_array_t */ (void)strncpy(info[1].key, PMIX_JOB_CTRL_CHECKPOINT_METHOD, PMIX_MAX_KEYLEN); - info[1].value.type = PMIX_DATA_ARRAY; - dptr = (pmix_data_array_t*)malloc(sizeof(pmix_data_array_t)); - info[1].value.data.darray = dptr; - dptr->type = PMIX_INFO; - dptr->size = 2; - PMIX_INFO_CREATE(dptr->array, dptr->size); + PMIX_DATA_ARRAY_CREATE(info[1].value.data.darray, 2, PMIX_INFO); + dptr = info[1].value.data.darray; rc = SIGUSR2; iptr = (pmix_info_t*)dptr->array; PMIX_INFO_LOAD(&iptr[0], PMIX_JOB_CTRL_CHECKPOINT_SIGNAL, &rc, PMIX_INT); @@ -159,18 +160,19 @@ int main(int argc, char **argv) /* since this is informational and not a requested operation, the target parameter * doesn't mean anything and can be ignored */ - active = -1; - if (PMIX_SUCCESS != (rc = PMIx_Job_control_nb(NULL, 0, info, 2, infocbfunc, (void*)&active))) { + DEBUG_CONSTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS != (rc = PMIx_Job_control_nb(NULL, 0, info, 2, infocbfunc, (void*)&mylock))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Job_control_nb failed: %d\n", myproc.nspace, myproc.rank, rc); + DEBUG_DESTRUCT_LOCK(&mylock); goto done; } - while (-1 == active) { - sleep(1); - } + DEBUG_WAIT_THREAD(&mylock); PMIX_INFO_FREE(info, 2); - if (0 != active) { + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS != rc) { fprintf(stderr, "Client ns %s rank %d: PMIx_Job_control_nb failed: %d\n", myproc.nspace, myproc.rank, rc); - exit(active); + goto done; } /* now request that this process be monitored using heartbeats */ @@ -185,20 +187,21 @@ int main(int argc, char **argv) PMIX_INFO_LOAD(&info[2], PMIX_MONITOR_HEARTBEAT_DROPS, &n, PMIX_UINT32); /* make the request */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); if (PMIX_SUCCESS != (rc = PMIx_Process_monitor_nb(iptr, PMIX_MONITOR_HEARTBEAT_ALERT, - info, 3, infocbfunc, (void*)&active))) { + info, 3, infocbfunc, (void*)&mylock))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Process_monitor_nb failed: %d\n", myproc.nspace, myproc.rank, rc); + DEBUG_DESTRUCT_LOCK(&mylock); goto done; } - while (-1 == active) { - sleep(1); - } + DEBUG_WAIT_THREAD(&mylock); PMIX_INFO_FREE(iptr, 1); PMIX_INFO_FREE(info, 3); - if (0 != active) { + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS != rc) { fprintf(stderr, "Client ns %s rank %d: PMIx_Process_monitor_nb failed: %d\n", myproc.nspace, myproc.rank, rc); - exit(active); + goto done; } /* send a heartbeat */ diff --git a/opal/mca/pmix/pmix2x/pmix/examples/pub.c b/opal/mca/pmix/pmix2x/pmix/examples/pub.c index d7e02c90a11..8a97a30b016 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/pub.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/pub.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -69,6 +69,7 @@ int main(int argc, char **argv) /* publish something */ if (0 == myproc.rank) { + fprintf(stderr, "%s:%d publishing two keys\n", myproc.nspace, myproc.rank); PMIX_INFO_CREATE(info, 2); (void)strncpy(info[0].key, "FOOBAR", PMIX_MAX_KEYLEN); info[0].value.type = PMIX_UINT8; @@ -80,6 +81,7 @@ int main(int argc, char **argv) fprintf(stderr, "Client ns %s rank %d: PMIx_Publish failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; } + fprintf(stderr, "%s:%d publish complete\n", myproc.nspace, myproc.rank); PMIX_INFO_FREE(info, 2); } @@ -93,6 +95,7 @@ int main(int argc, char **argv) /* lookup something */ if (0 != myproc.rank) { PMIX_PDATA_CREATE(pdata, 1); + fprintf(stderr, "%s:%d looking up key FOOBAR\n", myproc.nspace, myproc.rank); (void)strncpy(pdata[0].key, "FOOBAR", PMIX_MAX_KEYLEN); if (PMIX_SUCCESS != (rc = PMIx_Lookup(pdata, 1, NULL, 0))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup failed: %d\n", myproc.nspace, myproc.rank, rc); @@ -136,6 +139,7 @@ int main(int argc, char **argv) keys[1] = "PANDA"; keys[2] = NULL; + fprintf(stderr, "%s:%d unpublishing two keys\n", myproc.nspace, myproc.rank); if (PMIX_SUCCESS != (rc = PMIx_Unpublish(keys, NULL, 0))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Unpublish failed: %d\n", myproc.nspace, myproc.rank, rc); free(keys); diff --git a/opal/mca/pmix/pmix2x/pmix/examples/server.c b/opal/mca/pmix/pmix2x/pmix/examples/server.c index 72db59447c5..eedaff5a25e 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/server.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/server.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -134,6 +134,14 @@ PMIX_CLASS_INSTANCE(pmix_locdat_t, pmix_list_item_t, NULL, NULL); +#define PMIX_WAIT_FOR_COMPLETION(a) \ + do { \ + while ((a)) { \ + usleep(10); \ + } \ + PMIX_ACQUIRE_OBJECT((a)); \ + } while (0) + typedef struct { pmix_object_t super; volatile bool active; diff --git a/opal/mca/pmix/pmix2x/pmix/examples/tool.c b/opal/mca/pmix/pmix2x/pmix/examples/tool.c index 6818e49c5a3..79675838cc5 100644 --- a/opal/mca/pmix/pmix2x/pmix/examples/tool.c +++ b/opal/mca/pmix/pmix2x/pmix/examples/tool.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -29,6 +29,7 @@ #include #include +#include "examples.h" static void cbfunc(pmix_status_t status, pmix_info_t *info, size_t ninfo, @@ -36,15 +37,28 @@ static void cbfunc(pmix_status_t status, pmix_release_cbfunc_t release_fn, void *release_cbdata) { - volatile bool *active = (volatile bool*)cbdata; + myquery_data_t *mq = (myquery_data_t*)cbdata; + size_t n; - /* do something with the returned info - it will be + mq->lock.status = status; + + /* save the returned info - it will be * released in the release_fn */ + if (0 < ninfo) { + PMIX_INFO_CREATE(mq->info, ninfo); + mq->ninfo = ninfo; + for (n=0; n < ninfo; n++) { + PMIX_INFO_XFER(&mq->info[n], &info[n]); + } + } + /* let the library release the data */ if (NULL != release_fn) { release_fn(release_cbdata); } - *active = false; + + /* release the block */ + DEBUG_WAKEUP_THREAD(&mq->lock); } int main(int argc, char **argv) @@ -52,32 +66,66 @@ int main(int argc, char **argv) pmix_status_t rc; pmix_proc_t myproc; pmix_query_t *query; - size_t nq; - volatile bool active; + size_t nq, ninfo = 0, n; + myquery_data_t mydata; + pmix_info_t *info = NULL; + char *server_uri = NULL; + char *nspaces; + + for (n=1; n < (size_t)argc; n++) { + if (0 == strcmp("-u", argv[n]) || 0 == strcmp("--url", argv[n])) { + if (NULL == argv[n+1]) { + fprintf(stderr, "Must provide URI argument to %s option\n", argv[n]); + exit(1); + } + server_uri = argv[n+1]; + } + } + + if (NULL != server_uri) { + ninfo = 1; + PMIX_INFO_CREATE(info, ninfo); + PMIX_INFO_LOAD(&info[0], PMIX_SERVER_URI, server_uri, PMIX_STRING); + fprintf(stderr, "Connecting to %s\n", server_uri); + } /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, NULL, 0))) { + if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, info, ninfo))) { fprintf(stderr, "PMIx_tool_init failed: %d\n", rc); exit(rc); } + if (NULL != info) { + PMIX_INFO_FREE(info, ninfo); + } - /* query something */ - nq = 2; + /* query the list of active nspaces */ + nq = 1; PMIX_QUERY_CREATE(query, nq); - query[0].keys = (char**)malloc(2 * sizeof(char*)); - query[0].keys[0] = strdup("foobar"); - query[0].keys[1] = NULL; - query[1].keys = (char**)malloc(2 * sizeof(char*)); - query[1].keys[0] = strdup("spastic"); - query[1].keys[1] = NULL; - active = true; - if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&active))) { + PMIX_ARGV_APPEND(rc, query[0].keys, PMIX_QUERY_NAMESPACES); + DEBUG_CONSTRUCT_MYQUERY(&mydata); + if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&mydata))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Query_info failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; } - while(active) { - usleep(10); + DEBUG_WAIT_THREAD(&mydata.lock); + /* find the response */ + if (PMIX_SUCCESS == mydata.lock.status) { + /* should be in the first key */ + if (PMIX_CHECK_KEY(&mydata.info[0], PMIX_QUERY_NAMESPACES)) { + nspaces = mydata.info[0].value.data.string; + fprintf(stderr, "ACTIVE NSPACES:\n"); + if (NULL == nspaces) { + fprintf(stderr, "\tNone\n"); + } else { + fprintf(stderr, "\t%s\n", nspaces); + } + } else { + fprintf(stderr, "Query returned wrong info key at first posn: %s\n", mydata.info[0].key); + } + } else { + fprintf(stderr, "Query returned error: %s\n", PMIx_Error_string(mydata.lock.status)); } + DEBUG_DESTRUCT_MYQUERY(&mydata); done: /* finalize us */ diff --git a/opal/mca/pmix/pmix2x/pmix/include/Makefile.am b/opal/mca/pmix/pmix2x/pmix/include/Makefile.am index 3b205f72c57..3402faafec7 100644 --- a/opal/mca/pmix/pmix2x/pmix/include/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/include/Makefile.am @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2015-2019 Intel, Inc. All rights reserved. # # $COPYRIGHT$ # @@ -14,7 +14,8 @@ if WANT_PRIMARY_HEADERS include_HEADERS = \ pmix.h \ pmix_server.h \ - pmix_tool.h + pmix_tool.h \ + pmix_extend.h if WANT_PMI_BACKWARD include_HEADERS += \ diff --git a/opal/mca/pmix/pmix2x/pmix/include/pmix.h b/opal/mca/pmix/pmix2x/pmix/include/pmix.h index eada00d5e32..68a681fcfde 100644 --- a/opal/mca/pmix/pmix2x/pmix/include/pmix.h +++ b/opal/mca/pmix/pmix2x/pmix/include/pmix.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * @@ -129,7 +129,7 @@ PMIX_EXPORT pmix_status_t PMIx_Abort(int status, const char msg[], * the information locally until _PMIx_Commit_ is called. The provided scope * value is passed to the local PMIx server, which will distribute the data * as directed. */ -PMIX_EXPORT pmix_status_t PMIx_Put(pmix_scope_t scope, const char key[], pmix_value_t *val); +PMIX_EXPORT pmix_status_t PMIx_Put(pmix_scope_t scope, const pmix_key_t key, pmix_value_t *val); /* Push all previously _PMIx_Put_ values to the local PMIx server. @@ -200,7 +200,7 @@ PMIX_EXPORT pmix_status_t PMIx_Fence_nb(const pmix_proc_t procs[], size_t nprocs * an error. The timeout parameter can help avoid "hangs" due to programming * errors that prevent the target proc from ever exposing its data. */ -PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], +PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const pmix_key_t key, const pmix_info_t info[], size_t ninfo, pmix_value_t **val); @@ -208,7 +208,7 @@ PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], * be executed once the specified data has been _PMIx_Put_ * by the identified process and retrieved by the local server. The info * array is used as described above for the blocking form of this call. */ -PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char key[], +PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const pmix_key_t key, const pmix_info_t info[], size_t ninfo, pmix_value_cbfunc_t cbfunc, void *cbdata); @@ -337,7 +337,7 @@ PMIX_EXPORT pmix_status_t PMIx_Unpublish_nb(char **keys, */ PMIX_EXPORT pmix_status_t PMIx_Spawn(const pmix_info_t job_info[], size_t ninfo, const pmix_app_t apps[], size_t napps, - char nspace[]); + pmix_nspace_t nspace); /* Non-blocking form of the _PMIx_Spawn_ function. The callback @@ -394,7 +394,8 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const pmix_proc_t ranges[], size_t * for releasing the array when done with it - the PMIX_PROC_FREE macro is * provided for this purpose. */ -PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename, const char *nspace, +PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename, + const pmix_nspace_t nspace, pmix_proc_t **procs, size_t *nprocs); @@ -402,7 +403,7 @@ PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename, const char *n * that nspace. The returned string will contain a comma-delimited list * of nodenames. The caller is responsible for releasing the string * when done with it */ -PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist); +PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const pmix_nspace_t nspace, char **nodelist); /* Query information about the system in general - can include * a list of active nspaces, network topology, etc. Also can be diff --git a/opal/mca/pmix/pmix2x/pmix/include/pmix_common.h.in b/opal/mca/pmix/pmix2x/pmix/include/pmix_common.h.in index b4f4325e5e1..bec5dc3ad57 100644 --- a/opal/mca/pmix/pmix2x/pmix/include/pmix_common.h.in +++ b/opal/mca/pmix/pmix2x/pmix/include/pmix_common.h.in @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -84,6 +84,10 @@ extern "C" { #define PMIX_MAX_NSLEN 255 #define PMIX_MAX_KEYLEN 511 +/* define abstract types for namespaces and keys */ +typedef char pmix_nspace_t[PMIX_MAX_NSLEN+1]; +typedef char pmix_key_t[PMIX_MAX_KEYLEN+1]; + /* define a type for rank values */ typedef uint32_t pmix_rank_t; @@ -99,10 +103,14 @@ typedef uint32_t pmix_rank_t; * data for the given key from every rank that posted * that key */ #define PMIX_RANK_WILDCARD UINT32_MAX-1 - /* other special rank values will be used to define * groups of ranks for use in collectives */ #define PMIX_RANK_LOCAL_NODE UINT32_MAX-2 // all ranks on local node +#define PMIX_RANK_LOCAL_PEERS UINT32_MAX-4 // all peers (i.e., all procs within the same nspace) on local node +/* define an invalid value */ +#define PMIX_RANK_INVALID UINT32_MAX-3 +/* define a boundary for valid ranks */ +#define PMIX_RANK_VALID UINT32_MAX-50 /* define a set of "standard" PMIx attributes that can @@ -128,7 +136,6 @@ typedef uint32_t pmix_rank_t; // client rendezvous points and contact info #define PMIX_SYSTEM_TMPDIR "pmix.sys.tmpdir" // (char*) temp directory for this system, where PMIx // server will place tool rendezvous points and contact info -#define PMIX_REGISTER_NODATA "pmix.reg.nodata" // (bool) Registration is for nspace only, do not copy job data #define PMIX_SERVER_ENABLE_MONITORING "pmix.srv.monitor" // (bool) Enable PMIx internal monitoring by server #define PMIX_SERVER_NSPACE "pmix.srv.nspace" // (char*) Name of the nspace to use for this server #define PMIX_SERVER_RANK "pmix.srv.rank" // (pmix_rank_t) Rank of this server @@ -229,6 +236,7 @@ typedef uint32_t pmix_rank_t; #define PMIX_LOCAL_SIZE "pmix.local.size" // (uint32_t) #procs in this job on this node #define PMIX_NODE_SIZE "pmix.node.size" // (uint32_t) #procs across all jobs on this node #define PMIX_MAX_PROCS "pmix.max.size" // (uint32_t) max #procs for this job +#define PMIX_NUM_SLOTS "pmix.num.slots" // (uint32_t) #slots allocated #define PMIX_NUM_NODES "pmix.num.nodes" // (uint32_t) #nodes in this nspace /* Memory info */ @@ -239,7 +247,6 @@ typedef uint32_t pmix_rank_t; /* topology info */ #define PMIX_NET_TOPO "pmix.ntopo" // (char*) xml-representation of network topology #define PMIX_LOCAL_TOPO "pmix.ltopo" // (char*) xml-representation of local node topology -#define PMIX_NODE_LIST "pmix.nlist" // (char*) comma-delimited list of nodes running procs for this job #define PMIX_TOPOLOGY "pmix.topo" // (hwloc_topology_t) pointer to the PMIx client's internal topology object #define PMIX_TOPOLOGY_SIGNATURE "pmix.toposig" // (char*) topology signature string #define PMIX_LOCALITY_STRING "pmix.locstr" // (char*) string describing a proc's location @@ -291,8 +298,6 @@ typedef uint32_t pmix_rank_t; /* event handler registration and notification info keys */ #define PMIX_EVENT_HDLR_NAME "pmix.evname" // (char*) string name identifying this handler -#define PMIX_EVENT_JOB_LEVEL "pmix.evjob" // (bool) register for job-specific events only -#define PMIX_EVENT_ENVIRO_LEVEL "pmix.evenv" // (bool) register for environment events only #define PMIX_EVENT_HDLR_FIRST "pmix.evfirst" // (bool) invoke this event handler before any other handlers #define PMIX_EVENT_HDLR_LAST "pmix.evlast" // (bool) invoke this event handler after all other handlers have been called #define PMIX_EVENT_HDLR_FIRST_IN_CATEGORY "pmix.evfirstcat" // (bool) invoke this event handler before any other handlers in this category @@ -310,6 +315,9 @@ typedef uint32_t pmix_rank_t; // registered it #define PMIX_EVENT_DO_NOT_CACHE "pmix.evnocache" // (bool) instruct the PMIx server not to cache the event #define PMIX_EVENT_SILENT_TERMINATION "pmix.evsilentterm" // (bool) do not generate an event when this job normally terminates +#define PMIX_EVENT_PROXY "pmix.evproxy" // (pmix_proc_t*) PMIx server that sourced the event +#define PMIX_EVENT_TEXT_MESSAGE "pmix.evtext" // (char*) text message suitable for output by recipient - e.g., describing + // the cause of the event /* fault tolerance-related events */ @@ -366,6 +374,8 @@ typedef uint32_t pmix_rank_t; /* query attributes */ +#define PMIX_QUERY_REFRESH_CACHE "pmix.qry.rfsh" // (bool) retrieve updated information from server + // to update local cache #define PMIX_QUERY_NAMESPACES "pmix.qry.ns" // (char*) request a comma-delimited list of active nspaces #define PMIX_QUERY_JOB_STATUS "pmix.qry.jst" // (pmix_status_t) status of a specified currently executing job #define PMIX_QUERY_QUEUE_LIST "pmix.qry.qlst" // (char*) request a comma-delimited list of scheduler queues @@ -387,6 +397,49 @@ typedef uint32_t pmix_rank_t; #define PMIX_TIME_REMAINING "pmix.time.remaining" // (char*) query number of seconds (uint32_t) remaining in allocation // for the specified nspace +/* information retrieval attributes */ +#define PMIX_SESSION_INFO "pmix.ssn.info" // (bool) Return information about the specified session. If information + // about a session other than the one containing the requesting + // process is desired, then the attribute array must contain a + // PMIX_SESSION_ID attribute identifying the desired target. +#define PMIX_JOB_INFO "pmix.job.info" // (bool) Return information about the specified job or namespace. If + // information about a job or namespace other than the one containing + // the requesting process is desired, then the attribute array must + // contain a PMIX_JOBID or PMIX_NSPACE attribute identifying the + // desired target. Similarly, if information is requested about a + // job or namespace in a session other than the one containing the + // requesting process, then an attribute identifying the target + // session must be provided. +#define PMIX_APP_INFO "pmix.app.info" // (bool) Return information about the specified application. If information + // about an application other than the one containing the requesting + // process is desired, then the attribute array must contain a + // PMIX_APPNUM attribute identifying the desired target. Similarly, + // if information is requested about an application in a job or session + // other than the one containing the requesting process, then attributes + // identifying the target job and/or session must be provided. +#define PMIX_NODE_INFO "pmix.node.info" // (bool) Return information about the specified node. If information about a + // node other than the one containing the requesting process is desired, + // then the attribute array must contain either the PMIX_NODEID or + // PMIX_HOSTNAME attribute identifying the desired target. + +/* information storage attributes */ +#define PMIX_SESSION_INFO_ARRAY "pmix.ssn.arr" // (pmix_data_array_t) Provide an array of pmix_info_t containing + // session-level information. The PMIX_SESSION_ID attribute is required + // to be included in the array. +#define PMIX_JOB_INFO_ARRAY "pmix.job.arr" // (pmix_data_array_t) Provide an array of pmix_info_t containing job-level + // information. Information is registered one job (aka namespace) at a time + // via the PMIx_server_register_nspace API. Thus, there is no requirement that + // the array contain either the PMIX_NSPACE or PMIX_JOBID attributes, though + // either or both of them may be included. +#define PMIX_APP_INFO_ARRAY "pmix.app.arr" // (pmix_data_array_t) Provide an array of pmix_info_t containing app-level + // information. The PMIX_NSPACE or PMIX_JOBID attributes of the job containing + // the appplication, plus its PMIX_APPNUM attribute, are required to be + // included in the array. +#define PMIX_NODE_INFO_ARRAY "pmix.node.arr" // (pmix_data_array_t) Provide an array of pmix_info_t containing node-level + // information. At a minimum, either the PMIX_NODEID or PMIX_HOSTNAME + // attribute is required to be included in the array, though both may be + // included. + /* log attributes */ #define PMIX_LOG_STDERR "pmix.log.stderr" // (char*) log string to stderr #define PMIX_LOG_STDOUT "pmix.log.stdout" // (char*) log string to stdout @@ -572,71 +625,63 @@ typedef int pmix_status_t; #define PMIX_ERR_COMM_FAILURE -49 #define PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER -50 // internal-only -/* define a starting point for v2.x error values */ -#define PMIX_ERR_V2X_BASE -100 - /* v2.x communication errors */ -#define PMIX_ERR_LOST_CONNECTION_TO_SERVER (PMIX_ERR_V2X_BASE - 1) -#define PMIX_ERR_LOST_PEER_CONNECTION (PMIX_ERR_V2X_BASE - 2) -#define PMIX_ERR_LOST_CONNECTION_TO_CLIENT (PMIX_ERR_V2X_BASE - 3) +#define PMIX_ERR_LOST_CONNECTION_TO_SERVER -101 +#define PMIX_ERR_LOST_PEER_CONNECTION -102 +#define PMIX_ERR_LOST_CONNECTION_TO_CLIENT -103 /* used by the query system */ -#define PMIX_QUERY_PARTIAL_SUCCESS (PMIX_ERR_V2X_BASE - 4) +#define PMIX_QUERY_PARTIAL_SUCCESS -104 /* request responses */ -#define PMIX_NOTIFY_ALLOC_COMPLETE (PMIX_ERR_V2X_BASE - 5) +#define PMIX_NOTIFY_ALLOC_COMPLETE -105 /* job control */ -#define PMIX_JCTRL_CHECKPOINT (PMIX_ERR_V2X_BASE - 6) // monitored by client to trigger checkpoint operation -#define PMIX_JCTRL_CHECKPOINT_COMPLETE (PMIX_ERR_V2X_BASE - 7) // sent by client and monitored by server to notify that requested +#define PMIX_JCTRL_CHECKPOINT -106 // monitored by client to trigger checkpoint operation +#define PMIX_JCTRL_CHECKPOINT_COMPLETE -107 // sent by client and monitored by server to notify that requested // checkpoint operation has completed -#define PMIX_JCTRL_PREEMPT_ALERT (PMIX_ERR_V2X_BASE - 8) // monitored by client to detect RM intends to preempt -/* monitoring */ -#define PMIX_MONITOR_HEARTBEAT_ALERT (PMIX_ERR_V2X_BASE - 9) -#define PMIX_MONITOR_FILE_ALERT (PMIX_ERR_V2X_BASE - 10) -#define PMIX_PROC_TERMINATED (PMIX_ERR_V2X_BASE - 11) +#define PMIX_JCTRL_PREEMPT_ALERT -108 // monitored by client to detect RM intends to preempt -/* define a starting point for operational error constants so - * we avoid renumbering when making additions */ -#define PMIX_ERR_OP_BASE PMIX_ERR_V2X_BASE-30 +/* monitoring */ +#define PMIX_MONITOR_HEARTBEAT_ALERT -109 +#define PMIX_MONITOR_FILE_ALERT -110 +#define PMIX_PROC_TERMINATED -111 +#define PMIX_ERR_INVALID_TERMINATION -112 /* operational */ -#define PMIX_ERR_EVENT_REGISTRATION (PMIX_ERR_OP_BASE - 14) -#define PMIX_ERR_JOB_TERMINATED (PMIX_ERR_OP_BASE - 15) -#define PMIX_ERR_UPDATE_ENDPOINTS (PMIX_ERR_OP_BASE - 16) -#define PMIX_MODEL_DECLARED (PMIX_ERR_OP_BASE - 17) -#define PMIX_GDS_ACTION_COMPLETE (PMIX_ERR_OP_BASE - 18) +#define PMIX_ERR_EVENT_REGISTRATION -144 +#define PMIX_ERR_JOB_TERMINATED -145 +#define PMIX_ERR_UPDATE_ENDPOINTS -146 +#define PMIX_MODEL_DECLARED -147 +#define PMIX_GDS_ACTION_COMPLETE -148 /* gap created by v3 definitions */ -#define PMIX_OPERATION_SUCCEEDED (PMIX_ERR_OP_BASE - 27) +#define PMIX_OPERATION_SUCCEEDED -157 +#define PMIX_ERR_INVALID_OPERATION -158 /* gap for group codes */ -/* define a starting point for system error constants so - * we avoid renumbering when making additions */ -#define PMIX_ERR_SYS_BASE PMIX_ERR_OP_BASE-100 - /* system failures */ -#define PMIX_ERR_NODE_DOWN (PMIX_ERR_SYS_BASE - 1) -#define PMIX_ERR_NODE_OFFLINE (PMIX_ERR_SYS_BASE - 2) - +#define PMIX_ERR_NODE_DOWN -231 +#define PMIX_ERR_NODE_OFFLINE -232 +#define PMIX_ERR_SYS_OTHER -330 -/* define a starting point for event handler error constants so - * we avoid renumbering when making additions */ -#define PMIX_ERR_EVHDLR_BASE PMIX_ERR_SYS_BASE-100 +/* define a macro for identifying system event values */ +#define PMIX_SYSTEM_EVENT(a) \ + ((a) <= PMIX_ERR_NODE_DOWN && PMIX_ERR_SYS_OTHER <= (a)) /* used by event handlers */ -#define PMIX_EVENT_NO_ACTION_TAKEN (PMIX_ERR_EVHDLR_BASE - 1) -#define PMIX_EVENT_PARTIAL_ACTION_TAKEN (PMIX_ERR_EVHDLR_BASE - 2) -#define PMIX_EVENT_ACTION_DEFERRED (PMIX_ERR_EVHDLR_BASE - 3) -#define PMIX_EVENT_ACTION_COMPLETE (PMIX_ERR_EVHDLR_BASE - 4) +#define PMIX_EVENT_NO_ACTION_TAKEN -331 +#define PMIX_EVENT_PARTIAL_ACTION_TAKEN -332 +#define PMIX_EVENT_ACTION_DEFERRED -333 +#define PMIX_EVENT_ACTION_COMPLETE -334 /* define a starting point for PMIx internal error codes * that are never exposed outside the library */ -#define PMIX_INTERNAL_ERR_BASE -1000 +#define PMIX_INTERNAL_ERR_BASE -1330 /* define a starting point for user-level defined error * constants - negative values larger than this are guaranteed * not to conflict with PMIx values. Definitions should always * be based on the PMIX_EXTERNAL_ERR_BASE constant and -not- a * specific value as the value of the constant may change */ -#define PMIX_EXTERNAL_ERR_BASE -2000 +#define PMIX_EXTERNAL_ERR_BASE PMIX_INTERNAL_ERR_BASE-2000 /**** PMIX DATA TYPES ****/ typedef uint16_t pmix_data_type_t; @@ -735,8 +780,12 @@ typedef uint8_t pmix_persistence_t; /* define a set of bit-mask flags for specifying behavior of * command directives via pmix_info_t arrays */ typedef uint32_t pmix_info_directives_t; -#define PMIX_INFO_REQD 0x0001 - +#define PMIX_INFO_REQD 0x00000001 +#define PMIX_INFO_ARRAY_END 0x00000002 // mark the end of an array created by PMIX_INFO_CREATE +/* the top 16-bits are reserved for internal use by + * implementers - these may be changed inside the + * PMIx library */ +#define PMIX_INFO_DIR_RESERVED 0xffff0000 /* define a set of directives for allocation requests */ typedef uint8_t pmix_alloc_directive_t; @@ -754,19 +803,6 @@ typedef uint8_t pmix_alloc_directive_t; #define PMIX_ALLOC_EXTERNAL 128 -/* declare a convenience macro for checking keys */ -#define PMIX_CHECK_KEY(a, b) \ - (0 == strncmp((a)->key, (b), PMIX_MAX_KEYLEN)) - -/* define a convenience macro for checking nspaces */ -#define PMIX_CHECK_NSPACE(a, b) \ - (0 == strncmp((a), (b), PMIX_MAX_NSLEN)) - -/* define a convenience macro for checking names */ -#define PMIX_CHECK_PROCID(a, b) \ - (PMIX_CHECK_NSPACE((a)->nspace, (b)->nspace) && ((a)->rank == (b)->rank || (PMIX_RANK_WILDCARD == (a)->rank || PMIX_RANK_WILDCARD == (b)->rank))) - - /**** PMIX BYTE OBJECT ****/ typedef struct pmix_byte_object { char *bytes; @@ -847,13 +883,33 @@ typedef struct pmix_data_buffer { do { \ if (NULL != (m)->base_ptr) { \ free((m)->base_ptr); \ + (m)->base_ptr = NULL; \ } \ + (m)->pack_ptr = NULL; \ + (m)->unpack_ptr = NULL; \ + (m)->bytes_allocated = 0; \ + (m)->bytes_used = 0; \ } while (0) +#define PMIX_DATA_BUFFER_LOAD(b, d, s) \ + do { \ + (b)->base_ptr = (char*)(d); \ + (b)->pack_ptr = (b)->base_ptr + (s); \ + (b)->unpack_ptr = (b)->base_ptr; \ + (b)->bytes_allocated = (s); \ + (b)->bytes_used = (s); \ + } while(0) + +#define PMIX_DATA_BUFFER_UNLOAD(b, d, s) \ + do { \ + (d) = (b)->base_ptr; \ + (s) = (b)->bytes_used; \ + (b)->base_ptr = NULL; \ + } while(0) /**** PMIX PROC OBJECT ****/ typedef struct pmix_proc { - char nspace[PMIX_MAX_NSLEN+1]; + pmix_nspace_t nspace; pmix_rank_t rank; } pmix_proc_t; #define PMIX_PROC_CREATE(m, n) \ @@ -881,6 +937,13 @@ typedef struct pmix_proc { } \ } while (0) +#define PMIX_PROC_LOAD(m, n, r) \ + do { \ + PMIX_PROC_CONSTRUCT((m)); \ + pmix_strncpy((m)->nspace, (n), PMIX_MAX_NSLEN); \ + (m)->rank = (r); \ + } while(0) + /**** PMIX PROC INFO STRUCT ****/ typedef struct pmix_proc_info { @@ -910,9 +973,11 @@ typedef struct pmix_proc_info { do { \ if (NULL != (m)->hostname) { \ free((m)->hostname); \ + (m)->hostname = NULL; \ } \ if (NULL != (m)->executable_name) { \ free((m)->executable_name); \ + (m)->executable_name = NULL; \ } \ } while(0) @@ -937,6 +1002,11 @@ typedef struct pmix_data_array { void *array; } pmix_data_array_t; +/**** THE PMIX_DATA_ARRAY SUPPORT MACROS ARE DEFINED ****/ +/**** DOWN BELOW (NEAR THE BOTTOM OF THE FILE) TO ****/ +/**** AVOID CIRCULAR DEPENDENCIES ****/ + + typedef struct pmix_info_array { size_t size; pmix_info_t *array; @@ -1006,6 +1076,7 @@ typedef struct pmix_value { do { \ PMIX_VALUE_DESTRUCT((m)); \ free((m)); \ + (m) = NULL; \ } while (0) /* initialize a single value struct */ @@ -1035,11 +1106,12 @@ typedef struct pmix_value { * want to expose the entire header here. For * consistency, we provide macro versions as well */ -void pmix_value_load(pmix_value_t *v, const void *data, pmix_data_type_t type); #define PMIX_VALUE_LOAD(v, d, t) \ pmix_value_load((v), (d), (t)) -pmix_status_t pmix_value_xfer(pmix_value_t *kv, pmix_value_t *src); +#define PMIX_VALUE_UNLOAD(r, k, d, s) \ + (r) = pmix_value_unload((k), (d), (s)) + #define PMIX_VALUE_XFER(r, v, s) \ do { \ if (NULL == (v)) { \ @@ -1054,18 +1126,65 @@ pmix_status_t pmix_value_xfer(pmix_value_t *kv, pmix_value_t *src); } \ } while(0) -pmix_status_t pmix_argv_append_nosize(char ***argv, const char *arg); -#define PMIX_ARGV_APPEND(r, a, b) \ - (r) = pmix_argv_append_nosize(&(a), (b)) +#define PMIX_VALUE_GET_NUMBER(s, m, n, t) \ + do { \ + (s) = PMIX_SUCCESS; \ + if (PMIX_SIZE == (m)->type) { \ + (n) = (t)((m)->data.size); \ + } else if (PMIX_INT == (m)->type) { \ + (n) = (t)((m)->data.integer); \ + } else if (PMIX_INT8 == (m)->type) { \ + (n) = (t)((m)->data.int8); \ + } else if (PMIX_INT16 == (m)->type) { \ + (n) = (t)((m)->data.int16); \ + } else if (PMIX_INT32 == (m)->type) { \ + (n) = (t)((m)->data.int32); \ + } else if (PMIX_INT64 == (m)->type) { \ + (n) = (t)((m)->data.int64); \ + } else if (PMIX_UINT == (m)->type) { \ + (n) = (t)((m)->data.uint); \ + } else if (PMIX_UINT8 == (m)->type) { \ + (n) = (t)((m)->data.uint8); \ + } else if (PMIX_UINT16 == (m)->type) { \ + (n) = (t)((m)->data.uint16); \ + } else if (PMIX_UINT32 == (m)->type) { \ + (n) = (t)((m)->data.uint32); \ + } else if (PMIX_UINT64 == (m)->type) { \ + (n) = (t)((m)->data.uint64); \ + } else if (PMIX_FLOAT == (m)->type) { \ + (n) = (t)((m)->data.fval); \ + } else if (PMIX_DOUBLE == (m)->type) { \ + (n) = (t)((m)->data.dval); \ + } else if (PMIX_PID == (m)->type) { \ + (n) = (t)((m)->data.pid); \ + } else { \ + (s) = PMIX_ERR_BAD_PARAM; \ + } \ + } while(0) -pmix_status_t pmix_setenv(const char *name, const char *value, - bool overwrite, char ***env); -#define PMIX_SETENV(r, a, b, c) \ - (r) = pmix_setenv((a), (b), true, (c)) +#define PMIX_VALUE_COMPRESSED_STRING_UNPACK(s) \ + do { \ + char *tmp; \ + /* if this is a compressed string, then uncompress it */ \ + if (PMIX_COMPRESSED_STRING == (s)->type) { \ + pmix_util_uncompress_string(&tmp, (uint8_t*)(s)->data.bo.bytes, \ + (s)->data.bo.size); \ + if (NULL == tmp) { \ + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); \ + rc = PMIX_ERR_NOMEM; \ + PMIX_VALUE_RELEASE(s); \ + val = NULL; \ + } else { \ + PMIX_VALUE_DESTRUCT(s); \ + (s)->data.string = tmp; \ + (s)->type = PMIX_STRING; \ + } \ + } \ + } while(0) /**** PMIX INFO STRUCT ****/ struct pmix_info_t { - char key[PMIX_MAX_KEYLEN+1]; // ensure room for the NULL terminator + pmix_key_t key; pmix_info_directives_t flags; // bit-mask of flags pmix_value_t value; }; @@ -1095,17 +1214,23 @@ struct pmix_info_t { PMIX_INFO_DESTRUCT(&((m)[_s])); \ } \ free((m)); \ + (m) = NULL; \ } \ } while (0) -#define PMIX_INFO_LOAD(m, k, v, t) \ - do { \ - (void)strncpy((m)->key, (k), PMIX_MAX_KEYLEN); \ - pmix_value_load(&((m)->value), (v), (t)); \ +#define PMIX_INFO_LOAD(m, k, v, t) \ + do { \ + if (NULL != (k)) { \ + pmix_strncpy((m)->key, (k), PMIX_MAX_KEYLEN); \ + } \ + (m)->flags = 0; \ + pmix_value_load(&((m)->value), (v), (t)); \ } while (0) #define PMIX_INFO_XFER(d, s) \ do { \ - (void)strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \ + if (NULL != (s)->key) { \ + pmix_strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \ + } \ (d)->flags = (s)->flags; \ pmix_value_xfer(&(d)->value, (pmix_value_t*)&(s)->value); \ } while(0) @@ -1113,30 +1238,14 @@ struct pmix_info_t { #define PMIX_INFO_REQUIRED(m) \ (m)->flags |= PMIX_INFO_REQD; #define PMIX_INFO_OPTIONAL(m) \ - (m)->flags &= ~PMIX_INFO_REQD; - -#define PMIX_INFO_UNLOAD(r, v, l) \ - do { \ - pmix_info_t *_info; \ - size_t _n, _ninfo; \ - pmix_kval_t *_kv; \ - _info = (pmix_info_t*)(v)->data.darray->array; \ - _ninfo = (v)->data.darray->size; \ - for (_n = 0; _n < _ninfo; _n++){ \ - _kv = PMIX_NEW(pmix_kval_t); \ - if (NULL == _kv) { \ - (r) = PMIX_ERR_NOMEM; \ - break; \ - } \ - _kv->key = strdup(_info[_n].key); \ - PMIX_VALUE_XFER((r), _kv->value, &_info[_n].value);\ - if (PMIX_SUCCESS != (r)) { \ - PMIX_RELEASE(_kv); \ - break; \ - } \ - pmix_list_append((l), &_kv->super); \ - } \ - } while(0) + (m)->flags &= ~PMIX_INFO_REQD + +#define PMIX_INFO_IS_REQUIRED(m) \ + (m)->flags & PMIX_INFO_REQD +#define PMIX_INFO_IS_OPTIONAL(m) \ + !((m)->flags & PMIX_INFO_REQD) + + /* define a special macro for checking if a boolean * info is true - when info structs are provided, a * type of PMIX_UNDEF is taken to imply a boolean "true" @@ -1149,7 +1258,7 @@ struct pmix_info_t { /**** PMIX LOOKUP RETURN STRUCT ****/ typedef struct pmix_pdata { pmix_proc_t proc; - char key[PMIX_MAX_KEYLEN+1]; // ensure room for the NULL terminator + pmix_key_t key; pmix_value_t value; } pmix_pdata_t; @@ -1163,6 +1272,7 @@ typedef struct pmix_pdata { do { \ PMIX_VALUE_DESTRUCT(&(m)->value); \ free((m)); \ + (m) = NULL; \ } while (0) #define PMIX_PDATA_CONSTRUCT(m) \ @@ -1184,6 +1294,7 @@ typedef struct pmix_pdata { PMIX_PDATA_DESTRUCT(&((m)[_s])); \ } \ free((m)); \ + (m) = NULL; \ } \ } while (0) @@ -1191,9 +1302,9 @@ typedef struct pmix_pdata { do { \ if (NULL != (m)) { \ memset((m), 0, sizeof(pmix_pdata_t)); \ - (void)strncpy((m)->proc.nspace, (p)->nspace, PMIX_MAX_NSLEN); \ + pmix_strncpy((m)->proc.nspace, (p)->nspace, PMIX_MAX_NSLEN); \ (m)->proc.rank = (p)->rank; \ - (void)strncpy((m)->key, (k), PMIX_MAX_KEYLEN); \ + pmix_strncpy((m)->key, (k), PMIX_MAX_KEYLEN); \ pmix_value_load(&((m)->value), (v), (t)); \ } \ } while (0) @@ -1202,9 +1313,9 @@ typedef struct pmix_pdata { do { \ if (NULL != (d)) { \ memset((d), 0, sizeof(pmix_pdata_t)); \ - (void)strncpy((d)->proc.nspace, (s)->proc.nspace, PMIX_MAX_NSLEN); \ + pmix_strncpy((d)->proc.nspace, (s)->proc.nspace, PMIX_MAX_NSLEN); \ (d)->proc.rank = (s)->proc.rank; \ - (void)strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \ + pmix_strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \ pmix_value_xfer(&((d)->value), &((s)->value)); \ } \ } while (0) @@ -1226,10 +1337,17 @@ typedef struct pmix_app { (m) = (pmix_app_t*)calloc((n), sizeof(pmix_app_t)); \ } while (0) +#define PMIX_APP_INFO_CREATE(m, n) \ + do { \ + (m)->ninfo = (n); \ + PMIX_INFO_CREATE((m)->info, (m)->ninfo); \ + } while(0) + #define PMIX_APP_RELEASE(m) \ do { \ PMIX_APP_DESTRUCT((m)); \ free((m)); \ + (m) = NULL; \ } while (0) #define PMIX_APP_CONSTRUCT(m) \ @@ -1242,27 +1360,32 @@ typedef struct pmix_app { size_t _ii; \ if (NULL != (m)->cmd) { \ free((m)->cmd); \ + (m)->cmd = NULL; \ } \ if (NULL != (m)->argv) { \ for (_ii=0; NULL != (m)->argv[_ii]; _ii++) { \ free((m)->argv[_ii]); \ } \ free((m)->argv); \ + (m)->argv = NULL; \ } \ if (NULL != (m)->env) { \ for (_ii=0; NULL != (m)->env[_ii]; _ii++) { \ free((m)->env[_ii]); \ } \ free((m)->env); \ + (m)->env = NULL; \ } \ if (NULL != (m)->cwd) { \ free((m)->cwd); \ + (m)->cwd = NULL; \ } \ if (NULL != (m)->info) { \ for (_ii=0; _ii < (m)->ninfo; _ii++) { \ PMIX_INFO_DESTRUCT(&(m)->info[_ii]); \ } \ free((m)->info); \ + (m)->info = NULL; \ } \ } while (0) @@ -1274,6 +1397,7 @@ typedef struct pmix_app { PMIX_APP_DESTRUCT(&((m)[_s])); \ } \ free((m)); \ + (m) = NULL; \ } \ } while (0) @@ -1290,10 +1414,17 @@ typedef struct pmix_query { (m) = (pmix_query_t*)calloc((n) , sizeof(pmix_query_t)); \ } while (0) +#define PMIX_QUERY_QUALIFIERS_CREATE(m, n) \ + do { \ + (m)->nqual = (n); \ + PMIX_INFO_CREATE((m)->qualifiers, (m)->nqual); \ + } while(0) + #define PMIX_QUERY_RELEASE(m) \ do { \ PMIX_QUERY_DESTRUCT((m)); \ free((m)); \ + (m) = NULL; \ } while (0) #define PMIX_QUERY_CONSTRUCT(m) \ @@ -1309,12 +1440,14 @@ typedef struct pmix_query { free((m)->keys[_ii]); \ } \ free((m)->keys); \ + (m)->keys = NULL; \ } \ if (NULL != (m)->qualifiers) { \ for (_ii=0; _ii < (m)->nqual; _ii++) { \ PMIX_INFO_DESTRUCT(&(m)->qualifiers[_ii]); \ } \ free((m)->qualifiers); \ + (m)->qualifiers = NULL; \ } \ } while (0) @@ -1326,6 +1459,7 @@ typedef struct pmix_query { PMIX_QUERY_DESTRUCT(&((m)[_s])); \ } \ free((m)); \ + (m) = NULL; \ } \ } while (0) @@ -1334,7 +1468,7 @@ typedef struct pmix_query { /**** PMIX MODEX STRUCT ****/ typedef struct pmix_modex_data { - char nspace[PMIX_MAX_NSLEN+1]; + pmix_nspace_t nspace; int rank; uint8_t *blob; size_t size; @@ -1349,6 +1483,7 @@ typedef struct pmix_modex_data { do { \ PMIX_MODEX_DESTRUCT((m)); \ free((m)); \ + (m) = NULL; \ } while (0) #define PMIX_MODEX_CONSTRUCT(m) \ @@ -1360,6 +1495,7 @@ typedef struct pmix_modex_data { do { \ if (NULL != (m)->blob) { \ free((m)->blob); \ + (m)->blob = NULL; \ } \ } while (0) @@ -1371,6 +1507,7 @@ typedef struct pmix_modex_data { PMIX_MODEX_DESTRUCT(&((m)[_s])); \ } \ free((m)); \ + (m) = NULL; \ } \ } while (0) @@ -1401,7 +1538,7 @@ typedef void (*pmix_modex_cbfunc_t)(pmix_status_t status, * released by the library upon return from the callback function, so * the receiver must copy it if it needs to be retained */ typedef void (*pmix_spawn_cbfunc_t)(pmix_status_t status, - char nspace[], void *cbdata); + pmix_nspace_t nspace, void *cbdata); /* define a callback for common operations that simply return * a status. Examples include the non-blocking versions of @@ -1603,7 +1740,7 @@ PMIX_EXPORT void PMIx_Deregister_event_handler(size_t evhdlr_ref, PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, const pmix_proc_t *source, pmix_data_range_t range, - pmix_info_t info[], size_t ninfo, + const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); /* Provide a string representation for several types of value. Note @@ -1635,7 +1772,7 @@ PMIX_EXPORT const char* PMIx_Get_version(void); * proc. This is data that has only internal scope - it will * never be "pushed" externally */ PMIX_EXPORT pmix_status_t PMIx_Store_internal(const pmix_proc_t *proc, - const char *key, pmix_value_t *val); + const pmix_key_t key, pmix_value_t *val); /** * Top-level interface function to pack one or more values into a @@ -1928,8 +2065,10 @@ PMIX_EXPORT pmix_status_t PMIx_Data_copy_payload(pmix_data_buffer_t *dest, #define PMIX_VAL_FREE(_v) \ PMIx_free_value_data(_v) -static inline void pmix_value_destruct(pmix_value_t * m) { - size_t _n; +static inline void pmix_darray_destruct(pmix_data_array_t *m); + +static inline void pmix_value_destruct(pmix_value_t * m) +{ if (PMIX_STRING == (m)->type) { if (NULL != (m)->data.string) { free((m)->data.string); @@ -1943,64 +2082,170 @@ static inline void pmix_value_destruct(pmix_value_t * m) { (m)->data.bo.size = 0; } } else if (PMIX_DATA_ARRAY == (m)->type) { - if (NULL != (m)->data.darray && NULL != (m)->data.darray->array) { - if (PMIX_STRING == (m)->data.darray->type) { - char **_str = (char**)(m)->data.darray->array; - for (_n=0; _n < (m)->data.darray->size; _n++) { - if (NULL != _str[_n]) { - free(_str[_n]); - } - } - } else if (PMIX_PROC_INFO == (m)->data.darray->type) { - pmix_proc_info_t *_info = - (pmix_proc_info_t*)(m)->data.darray->array; - for (_n=0; _n < (m)->data.darray->size; _n++) { - PMIX_PROC_INFO_DESTRUCT(&_info[_n]); - } - } else if (PMIX_INFO == (m)->data.darray->type) { - pmix_info_t *_info = - (pmix_info_t*)(m)->data.darray->array; - for (_n=0; _n < (m)->data.darray->size; _n++) { - pmix_value_destruct(&_info[_n].value); - } - } else if (PMIX_BYTE_OBJECT == (m)->data.darray->type) { - pmix_byte_object_t *_obj = - (pmix_byte_object_t*)(m)->data.darray->array; - for (_n=0; _n < (m)->data.darray->size; _n++) { - if (NULL != _obj[_n].bytes) { - free(_obj[_n].bytes); - } - } - } - free((m)->data.darray->array); - (m)->data.darray->array = NULL; - (m)->data.darray->size = 0; - } if (NULL != (m)->data.darray) { + pmix_darray_destruct((m)->data.darray); free((m)->data.darray); (m)->data.darray = NULL; } - /**** DEPRECATED ****/ - } else if (PMIX_INFO_ARRAY == (m)->type) { - pmix_info_t *_p = (pmix_info_t*)((m)->data.array->array); - for (_n=0; _n < (m)->data.array->size; _n++) { - if (PMIX_STRING == _p[_n].value.type) { - if (NULL != _p[_n].value.data.string) { - free(_p[_n].value.data.string); - } - } else if (PMIX_BYTE_OBJECT == _p[_n].value.type) { - if (NULL != _p[_n].value.data.bo.bytes) { - free(_p[_n].value.data.bo.bytes); - } - } else if (PMIX_PROC_INFO == _p[_n].value.type) { - PMIX_PROC_INFO_DESTRUCT(_p[_n].value.data.pinfo); + } else if (PMIX_PROC == (m)->type) { + PMIX_PROC_RELEASE((m)->data.proc); + } +} + +static inline void pmix_darray_destruct(pmix_data_array_t *m) +{ + if (NULL != m) { + if (PMIX_INFO == m->type) { + pmix_info_t *_info = (pmix_info_t*)m->array; + PMIX_INFO_FREE(_info, m->size); + } else if (PMIX_PROC == m->type) { + pmix_proc_t *_p = (pmix_proc_t*)m->array; + PMIX_PROC_FREE(_p, m->size); + } else if (PMIX_PROC_INFO == m->type) { + pmix_proc_info_t *_pi = (pmix_proc_info_t*)m->array; + PMIX_PROC_INFO_FREE(_pi, m->size); + } else if (PMIX_VALUE == m->type) { + pmix_value_t *_v = (pmix_value_t*)m->array; + PMIX_VALUE_FREE(_v, m->size); + } else if (PMIX_PDATA == m->type) { + pmix_pdata_t *_pd = (pmix_pdata_t*)m->array; + PMIX_PDATA_FREE(_pd, m->size); + } else if (PMIX_QUERY == m->type) { + pmix_query_t *_q = (pmix_query_t*)m->array; + PMIX_QUERY_FREE(_q, m->size); + } else if (PMIX_APP == m->type) { + pmix_app_t *_a = (pmix_app_t*)m->array; + PMIX_APP_FREE(_a, m->size); + } else if (PMIX_BYTE_OBJECT == m->type) { + pmix_byte_object_t *_b = (pmix_byte_object_t*)m->array; + PMIX_BYTE_OBJECT_FREE(_b, m->size); + } else if (PMIX_STRING == m->type) { + char **_s = (char**)m->array; + size_t _si; + for (_si=0; _si < m->size; _si++) { + free(_s[_si]); } + free(m->array); + m->array = NULL; + } else { + free(m->array); + } + } +} + +#define PMIX_DATA_ARRAY_CONSTRUCT(m, n, t) \ + do { \ + (m)->type = (t); \ + (m)->size = (n); \ + if (0 < (n)) { \ + if (PMIX_INFO == (t)) { \ + PMIX_INFO_CREATE((m)->array, (n)); \ + } else if (PMIX_PROC == (t)) { \ + PMIX_PROC_CREATE((m)->array, (n)); \ + } else if (PMIX_PROC_INFO == (t)) { \ + PMIX_PROC_INFO_CREATE((m)->array, (n)); \ + } else if (PMIX_VALUE == (t)) { \ + PMIX_VALUE_CREATE((m)->array, (n)); \ + } else if (PMIX_PDATA == (t)) { \ + PMIX_PDATA_CREATE((m)->array, (n)); \ + } else if (PMIX_QUERY == (t)) { \ + PMIX_QUERY_CREATE((m)->array, (n)); \ + } else if (PMIX_APP == (t)) { \ + PMIX_APP_CREATE((m)->array, (n)); \ + } else if (PMIX_BYTE_OBJECT == (t)) { \ + PMIX_BYTE_OBJECT_CREATE((m)->array, (n)); \ + } else if (PMIX_ALLOC_DIRECTIVE == (t) || \ + PMIX_PROC_STATE == (t) || \ + PMIX_PERSIST == (t) || \ + PMIX_SCOPE == (t) || \ + PMIX_DATA_RANGE == (t) || \ + PMIX_BYTE == (t) || \ + PMIX_INT8 == (t) || \ + PMIX_UINT8 == (t)) { \ + (m)->array = calloc((n), sizeof(int8_t)); \ + } else if (PMIX_STRING == (t)) { \ + (m)->array = calloc((n), sizeof(char*)); \ + } else if (PMIX_SIZE == (t)) { \ + (m)->array = calloc((n), sizeof(size_t)); \ + } else if (PMIX_PID == (t)) { \ + (m)->array = calloc((n), sizeof(pid_t)); \ + } else if (PMIX_INT == (t) || \ + PMIX_UINT == (t) || \ + PMIX_STATUS == (t)) { \ + (m)->array = calloc((n), sizeof(int)); \ + } else if (PMIX_DATA_TYPE == (t) || \ + PMIX_INT16 == (t) || \ + PMIX_UINT16 == (t)) { \ + (m)->array = calloc((n), sizeof(int16_t)); \ + } else if (PMIX_PROC_RANK == (t) || \ + PMIX_INFO_DIRECTIVES == (t) || \ + PMIX_INT32 == (t) || \ + PMIX_UINT32 == (t)) { \ + (m)->array = calloc((n), sizeof(int32_t)); \ + } else if (PMIX_INT64 == (t) || \ + PMIX_UINT64 == (t)) { \ + (m)->array = calloc((n), sizeof(int64_t)); \ + } else if (PMIX_FLOAT == (t)) { \ + (m)->array = calloc((n), sizeof(float)); \ + } else if (PMIX_DOUBLE == (t)) { \ + (m)->array = calloc((n), sizeof(double)); \ + } else if (PMIX_TIMEVAL == (t)) { \ + (m)->array = calloc((n), sizeof(struct timeval)); \ + } else if (PMIX_TIME == (t)) { \ + (m)->array = calloc((n), sizeof(time_t)); \ + } \ + } else { \ + (m)->array = NULL; \ + } \ + } while(0) +#define PMIX_DATA_ARRAY_CREATE(m, n, t) \ + do { \ + (m) = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); \ + PMIX_DATA_ARRAY_CONSTRUCT((m), (n), (t)); \ + } while(0) + +#define PMIX_DATA_ARRAY_DESTRUCT(m) pmix_darray_destruct(m) + +#define PMIX_DATA_ARRAY_FREE(m) \ + do { \ + if (NULL != (m)) { \ + PMIX_DATA_ARRAY_DESTRUCT(m); \ + free((m)); \ + (m) = NULL; \ + } \ + } while(0) + +/** + * Provide a safe version of strncpy that doesn't generate + * a ton of spurious warnings. Note that not every environment + * provides nice string functions, and we aren't concerned about + * max performance here + * + * @param dest Destination string. + * @param src Source string. + * @param len Size of the dest array - 1 + * + */ +static inline void pmix_strncpy(char *dest, const char *src, size_t len) +{ + size_t i, k; + char *new_dest = dest; + + /* use an algorithm that also protects against + * non-NULL-terminated src strings */ + for (i=0, k=0; i <= len; ++i, ++src, ++new_dest) { + ++k; + *new_dest = *src; + if ('\0' == *src) { + break; } - free(_p); - /********************/ } + dest[k-1] = '\0'; } + +#include + #if defined(c_plusplus) || defined(__cplusplus) } #endif diff --git a/opal/mca/pmix/pmix2x/pmix/include/pmix_extend.h b/opal/mca/pmix/pmix2x/pmix/include/pmix_extend.h new file mode 100644 index 00000000000..cf3547db50c --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/include/pmix_extend.h @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Artem Y. Polyakov . + * All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer listed + * in this license in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * The copyright holders provide no reassurances that the source code + * provided does not infringe any patent, copyright, or any other + * intellectual property rights of third parties. The copyright holders + * disclaim any liability to any recipient for claims brought against + * recipient by any third party for infringement of that parties + * intellectual property rights. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $HEADER$ + * + * PMIx provides a "function-shipping" approach to support for + * implementing the server-side of the protocol. This method allows + * resource managers to implement the server without being burdened + * with PMIx internal details. Accordingly, each PMIx API is mirrored + * here in a function call to be provided by the server. When a + * request is received from the client, the corresponding server function + * will be called with the information. + * + * Any functions not supported by the RM can be indicated by a NULL for + * the function pointer. Client calls to such functions will have a + * "not supported" error returned. + */ + +#ifndef PMIx_EXTEND_H +#define PMIx_EXTEND_H + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + +/* declare a convenience macro for checking keys */ +#define PMIX_CHECK_KEY(a, b) \ + (0 == strncmp((a)->key, (b), PMIX_MAX_KEYLEN)) + +#define PMIX_LOAD_KEY(a, b) \ + do { \ + memset((a), 0, PMIX_MAX_KEYLEN+1); \ + pmix_strncpy((a), (b), PMIX_MAX_KEYLEN); \ + }while(0) + +/* define a convenience macro for loading nspaces */ +#define PMIX_LOAD_NSPACE(a, b) \ + do { \ + memset((a), 0, PMIX_MAX_NSLEN+1); \ + pmix_strncpy((a), (b), PMIX_MAX_NSLEN); \ + }while(0) + +/* define a convenience macro for checking nspaces */ +#define PMIX_CHECK_NSPACE(a, b) \ + (0 == strncmp((a), (b), PMIX_MAX_NSLEN)) + +/* define a convenience macro for loading names */ +#define PMIX_LOAD_PROCID(a, b, c) \ + do { \ + PMIX_LOAD_NSPACE((a)->nspace, (b)); \ + (a)->rank = (c); \ + }while(0) + +/* define a convenience macro for checking names */ +#define PMIX_CHECK_PROCID(a, b) \ + (PMIX_CHECK_NSPACE((a)->nspace, (b)->nspace) && ((a)->rank == (b)->rank || (PMIX_RANK_WILDCARD == (a)->rank || PMIX_RANK_WILDCARD == (b)->rank))) + +#define PMIX_ARGV_APPEND(r, a, b) \ + (r) = pmix_argv_append_nosize(&(a), (b)) + +#define PMIX_SETENV(r, a, b, c) \ + (r) = pmix_setenv((a), (b), true, (c)) + +/* expose some functions that are resolved in the + * PMIx library, but part of a header that + * includes internal functions - we don't + * want to expose the entire header here. For + * consistency, we provide macro versions as well + */ +void pmix_value_load(pmix_value_t *v, const void *data, pmix_data_type_t type); + +pmix_status_t pmix_value_unload(pmix_value_t *kv, void **data, size_t *sz); + +pmix_status_t pmix_value_xfer(pmix_value_t *kv, const pmix_value_t *src); + +pmix_status_t pmix_argv_append_nosize(char ***argv, const char *arg); + +pmix_status_t pmix_argv_prepend_nosize(char ***argv, const char *arg); + +pmix_status_t pmix_argv_append_unique_nosize(char ***argv, const char *arg, bool overwrite); + +void pmix_argv_free(char **argv); + +char **pmix_argv_split(const char *src_string, int delimiter); + +int pmix_argv_count(char **argv); + +char *pmix_argv_join(char **argv, int delimiter); + +char **pmix_argv_copy(char **argv); + +pmix_status_t pmix_setenv(const char *name, const char *value, + bool overwrite, char ***env); + + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif diff --git a/opal/mca/pmix/pmix2x/pmix/include/pmix_server.h b/opal/mca/pmix/pmix2x/pmix/include/pmix_server.h index 137d9f04a2d..91a5debb3ab 100644 --- a/opal/mca/pmix/pmix2x/pmix/include/pmix_server.h +++ b/opal/mca/pmix/pmix2x/pmix/include/pmix_server.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2015 Research Organization for Information Science @@ -442,7 +442,7 @@ PMIX_EXPORT pmix_status_t PMIx_generate_ppn(const char *input, char **ppn); * for the PMIx server library to correctly handle collectives * as a collective operation call can occur before all the * procs have been started */ -PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const char nspace[], int nlocalprocs, +PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const pmix_nspace_t nspace, int nlocalprocs, pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); @@ -451,7 +451,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const char nspace[], int n * intended to support persistent PMIx servers by providing * an opportunity for the host RM to tell the PMIx server * library to release all memory for a completed job */ -PMIX_EXPORT void PMIx_server_deregister_nspace(const char nspace[], +PMIX_EXPORT void PMIx_server_deregister_nspace(const pmix_nspace_t nspace, pmix_op_cbfunc_t cbfunc, void *cbdata); /* Register a client process with the PMIx server library. The @@ -521,7 +521,7 @@ typedef void (*pmix_setup_application_cbfunc_t)(pmix_status_t status, * is defined as a non-blocking operation in case network * libraries need to perform some action before responding. The * returned env will be distributed along with the application */ -PMIX_EXPORT pmix_status_t PMIx_server_setup_application(const char nspace[], +PMIX_EXPORT pmix_status_t PMIx_server_setup_application(const pmix_nspace_t nspace, pmix_info_t info[], size_t ninfo, pmix_setup_application_cbfunc_t cbfunc, void *cbdata); @@ -530,7 +530,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_application(const char nspace[], * clients of a given application. For example, a network library * might need to setup the local driver for "instant on" addressing. */ -PMIX_EXPORT pmix_status_t PMIx_server_setup_local_support(const char nspace[], +PMIX_EXPORT pmix_status_t PMIx_server_setup_local_support(const pmix_nspace_t nspace, pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); diff --git a/opal/mca/pmix/pmix2x/pmix/include/pmix_version.h.in b/opal/mca/pmix/pmix2x/pmix/include/pmix_version.h.in index 202a77a85dd..af4a00cd5df 100644 --- a/opal/mca/pmix/pmix2x/pmix/include/pmix_version.h.in +++ b/opal/mca/pmix/pmix2x/pmix/include/pmix_version.h.in @@ -2,7 +2,9 @@ * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2018 IBM Corporation. All rights reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -19,6 +21,5 @@ #define PMIX_VERSION_MINOR @pmixminor@ #define PMIX_VERSION_RELEASE @pmixrelease@ -#define PMIX_NUMERIC_VERSION 0x00020103 - +#define PMIX_NUMERIC_VERSION @pmixnumeric@ #endif diff --git a/opal/mca/pmix/pmix2x/pmix/man/Makefile.am b/opal/mca/pmix/pmix2x/pmix/man/Makefile.am deleted file mode 100644 index 7c0f8bffe46..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/man/Makefile.am +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2006-2016 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -if !PMIX_EMBEDDED_MODE - -man_MANS = \ - man3/pmix_init.3 \ - man3/pmix_finalize.3 \ - man3/pmix_initialized.3 \ - man3/pmix_abort.3 \ - man3/pmix_put.3 \ - man3/pmix_commit.3 \ - man7/pmix.7 \ - man7/pmix_constants.7 - -EXTRA_DIST = $(man_MANS) - -man3/pmix_init.3: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_init.3.md; - -man3/pmix_finalize.3: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_finalize.3.md; - -man3/pmix_initialized.3: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_initialized.3.md; - -man3/pmix_abort.3: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_abort.3.md; - -man3/pmix_put.3: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_put.3.md; - -man3/pmix_commit.3: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_commit.3.md; - -man7/pmix.7: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix.7.md; - -man7/pmix_constants.7: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_constants.7.md; - -endif # !PMIX_EMBEDDED_MODE diff --git a/opal/mca/pmix/pmix2x/pmix/man/README b/opal/mca/pmix/pmix2x/pmix/man/README deleted file mode 100644 index 73c605cb7f0..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/man/README +++ /dev/null @@ -1,186 +0,0 @@ -This file describes how the developer side of man pages work in PMIx. - -The Definitive Source Of Truth man pages are the Markdown man pages in -this directory (i.e., the files ending in ..md. If you want to -edit man pages, you need to edit the ..md pages. Do NOT edit -the . nroff man pages directly; these files are automatically -generated -- you will lose any manual edits the next time those files -are generated. - -The Markdown web pages are rendered in two different ways: - -1. Nroff man pages. These man pages are put into the `master` branch - and later included in PMIx distribution tarballs. - -2. HTML. The http://open-mpi.github.io/pmix/ web site (which is - served by the Github web servers) automatically renders the content - of the `gh-pages` branch of the PMIx repo. - -Markdown syntax -=============== - -The definitive man pages are the Markdown man pages. To edit them, -you need to understand the syntax used in these files. - -The canonical reference for Markdown is here: - - http://daringfireball.net/projects/markdown/syntax - -Note, however, that the PMIx Markdown man pages are served via -the Github Pages web servers, which use a system called Jekyll to -render the Markdown into HTML (https://github.com/jekyll/jekyll). -As such, there are a few Jekyll annotations in the PMIx Markdown -pages (so that they can be served up properly from Github's web -servers). - -If you're familiar with Markdown, you should be ok. But there are a -small number differences and quirks with which you should be familiar: - -1. The first few lines of each file are a YAML header and include - directive for Jekyll. DO NOT REMOVE THIS HEADER (or the file will - not render to HTML properly when served up from Github's web - servers). Here's a sample YAML header from pmix.7.md: - ---- -layout: page -title: PMIx(7) -tagline: PMIx Programmer's Manual ---- -{% include JB/setup %} - - The whole block is needed, and it must be the first input in the - file. - -2. In Github-flavored Markdown, you may be used to using "fenced - blocks" for multi-line code blocks, like this: - -```c -void my_c_code(void) { - int i; - /* Hello, world */ -} -``` - - Such fenced blocks will not work in Jekyll. Instead, you must - delineate your code blocks with Jekyll delimiters: - -{% highlight c %} -void my_c_code(void) { - int i; - /* Hello, world */ -} -{% endhighlight %} - - This will result in a pretty code box in the rendered HTML output, - and it will be syntax highlighted for the C language. Leave the - "c" out of the first directive if your multi-line block is not C - code, and then it won't do C syntax highlighting. - -3. The PMIx man pages are full of 2-level lists of things. E.g., - lists of functions, and then in some of the functions, there is a - sub-list of flags that can be used with that function. - - The convention used in the PMIx man pages is to highlight a - word/phrase representing each list item. Then use a ":" to start - the next line that describes that item. For example: - -*PMIX_FLOAT* -: A single-precision floating point value (IEEE 754). - - This will make the token "PMIX_FLOAT" be highlighted in both - HTML and nroff output, and then the paragraph that comes after it - will be properly delimited and indented. - - To make a sub-list inside an item, use the same format, but prefix - the sub-list items with "-", like this: - -*scope* -: Flag that controls the visible scope of the data. - -- *PMIX_GLOBAL* -: Indicates that the data is to be visible to all applications executed - by this user. - -4. There may be a small number of places in the PMIx man pages where - there are unnumbered lists with deliberate line breaks. For - example: - -foo / bar -baz / goo -: Something really intelligent - - Note the first line is "foo / bar", and then there is - a deliberate line break, and then the second line is "baz / goo". - - To effect the deliberate line break, you have to put two blank - spaces after "bar". To show that graphically (showing "_" - for " "): - -foo / bar__ -baz / goo -: Something really intelligent - -5. The "SEE ALSO" items at the end of each man page are linked to - their corresponding man pages. Note that the links are made to - ".html" files -- *not* ".md" files. If you care, the reason is - because the Github web servers statically generate .html files from - the .md files when you git push to the gh-pages branch. Hence, the - man pages are actually served from static .html files on the Github - web servers. - - Also, since links are meaningless in nroff, they are effectively - ignored in the resulting nroff output. - -Workflow -======== - -The workflow is like this: - -1. Developer edits ..md files for new changes. - -2. In a perfect world, the developer makes perfect edits and pushes - the changes up to `master`. An automated cron job will eventually - notice the new pages, and do two things: - - 2a. Copy the modified Markdown pages to the `gh-master` branch (so - that they go live on the web site). - - 2b. Re-generate any relevant nroff man pages in `master`. - - The automated cron job actually does exist and does these things, - but it should only be relied upon once a developer is sure that - their changes to the Markdown man pages are correct. - -3. To check that the changes will render properly, developers should - do two things: - - 3a. Run "make nroff". This will convert all the Markdown man pages - into nroff man pages (in the man/ directory). Check to ensure - that your changes look appropriate in the rendered nroff - output. - - *CAUTION* The "pandoc" utility is used to generate the nroff - files from the Markdown source. Different versions of pandoc - will generate slightly different nroff output. Meaning: when - you run "make nroff", you might end up changing every nroff man - page, simply because your version of pandoc is different than - the last person who ran it. Please only check in your changes, - if possible. - - 3b. Check out the `gh-pages` branch from PMIx and copy any - modified Markdown pages into the "master/man" directory (i.e., - the directory for man pages from the master development - branch). - - Then run the "jekyll serve" command from the top-level - directory in `gh-pages`. This runs a local web server on your - computer and renders the Markdown files into HTML such that you - can point a browser to http://127.0.0.1:4000 and see the web - site. - - If you make any changes to files in the tree where "jekyll" is - running, Jekyll will notice the changes and automatically - re-generate the relevant HTML. Meaning: you can just refresh - the page from http://127.0.0.1:4000 in your browser and you'll - see your changes -- there's no need to restart Jekyll to force - it to notice new changes. diff --git a/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_abort.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_abort.3 deleted file mode 100644 index ea0690ae943..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_abort.3 +++ /dev/null @@ -1,62 +0,0 @@ -.TH "pmix_abort" "3" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx_Abort \- Abort the specified processes -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ - -pmix\\_status\\_t\ PMIx\\_Abort(int\ status,\ const\ char\ msg[], -\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ pmix\\_proc\\_t\ procs[],\ size_t\ nprocs); -\f[] -.fi -.SH ARGUMENTS -.PP -\f[I]status\f[] : Status value to be returned. -A value of zero is permitted by PMIx, but may not be returned by some -resource managers. -.PP -\f[I]msg\f[] : A string message to be displayed -.PP -\f[I]procs\f[] : An array of pmix_proc_t structures defining the -processes to be aborted. -A \f[I]NULL\f[] for the proc array indicates that all processes in the -caller\[aq]s nspace are to be aborted. -A wildcard value for the rank in any structure indicates that all -processes in that nspace are to be aborted. -.PP -\f[I]nprocs\f[] : Number of pmix_proc_t structures in the \f[I]procs\f[] -array -.SH DESCRIPTION -.PP -Request that the provided array of procs be aborted, returning the -provided \f[I]status\f[] and printing the provided message. -A \f[I]NULL\f[] for the proc array indicates that all processes in the -caller\[aq]s nspace are to be aborted. -.PP -The response to this request is somewhat dependent on the specific -resource manager and its configuration (e.g., some resource managers -will not abort the application if the provided \f[I]status\f[] is zero -unless specifically configured to do so), and thus lies outside the -control of PMIx itself. -However, the client will inform the RM of the request that the -application be aborted, regardless of the value of the provided -\f[I]status\f[]. -.PP -Passing a \f[I]NULL\f[] msg parameter is allowed. -Note that race conditions caused by multiple processes calling -PMIx_Abort are left to the server implementation to resolve with regard -to which status is returned and what messages (if any) are printed. -.SH RETURN VALUE -.PP -Returns PMIX_SUCCESS on success. -On error, a negative value corresponding to a PMIx errno is returned. -.SH ERRORS -.PP -PMIx errno values are defined in \f[C]pmix_common.h\f[]. -.SH NOTES -.SH SEE ALSO -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_commit.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_commit.3 deleted file mode 100644 index 31dd4fc2f76..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_commit.3 +++ /dev/null @@ -1,35 +0,0 @@ -.TH "pmix_commit" "3" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx_Commit \- Push all previously \f[I]PMIx_Put\f[] values to the local -PMIx server. -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ - -pmix\\_status\\_t\ PMIx_Commit(void); -\f[] -.fi -.SH ARGUMENTS -.PP -\f[I]none\f[] -.SH DESCRIPTION -.PP -This is an asynchronous operation \- the library will immediately return -to the caller while the data is transmitted to the local server in the -background -.SH RETURN VALUE -.PP -Returns PMIX_SUCCESS on success. -On error, a negative value corresponding to a PMIx errno is returned. -.SH ERRORS -.PP -PMIx errno values are defined in \f[C]pmix_common.h\f[]. -.SH NOTES -.SH SEE ALSO -.PP -\f[C]PMIx_Put\f[](3) -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_finalize.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_finalize.3 deleted file mode 100644 index 6b15282945b..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_finalize.3 +++ /dev/null @@ -1,45 +0,0 @@ -.TH "pmix_finalize" "3" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx_Finalize \- Finalize the PMIx Client -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ - -pmix\\_status\\_t\ PMIx\\_Finalize(const\ pmix\\_info\\_t\ info[],\ size_t\ ninfo); -\f[] -.fi -.SH ARGUMENTS -.PP -\f[I]info\f[] : An optional array of pmix_info_t structures -.PP -\f[I]ninfo\f[] : Number of pmix_info_t structures in the pmix_info_t -array -.SH DESCRIPTION -.PP -Finalize the PMIx client, closing the connection with the local PMIx -server and releasing all malloc\[aq]d memory. -.PP -The info array is used to pass user requests regarding the fence -operation. -This can include: -.IP "(a)" 4 -PMIX_EMBED_BARRIER \- By default, \f[I]PMIx_Finalize\f[] does not -include an internal barrier operation. -This attribute directs \f[I]PMIx_Finalize\f[] to execute a barrier as -part of the finalize operation. -.SH RETURN VALUE -.PP -Returns PMIX_SUCCESS on success. -On error, a negative value corresponding to a PMIx errno is returned. -.SH ERRORS -.PP -PMIx errno values are defined in \f[C]pmix_common.h\f[]. -.SH NOTES -.SH SEE ALSO -.PP -\f[C]PMIx_Init\f[](3) -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_init.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_init.3 deleted file mode 100644 index b988200b33b..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_init.3 +++ /dev/null @@ -1,52 +0,0 @@ -.TH "pmix_init" "3" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx_Init \- Initialize the PMIx Client -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ - -pmix\\_status\\_t\ PMIx_Init(pmix\\_proc\\_t\ *proc); -\f[] -.fi -.SH ARGUMENTS -.PP -\f[I]proc\f[] : Pointer to a pmix_proc_t object in which the -client\[aq]s namespace and rank are to be returned. -.SH DESCRIPTION -.PP -Initialize the PMIx client, returning the process identifier assigned to -this client\[aq]s application in the provided pmix_proc_t struct. -Passing a value of \f[I]NULL\f[] for this parameter is allowed if the -user wishes solely to initialize the PMIx system and does not require -return of the identifier at that time. -.PP -When called, the PMIx client will check for the required connection -information of the local PMIx server and will establish the connection. -If the information is not found, or the server connection fails, then an -appropriate error constant will be returned. -.PP -If successful, the function will return PMIX_SUCCESS and will fill the -provided structure with the server\-assigned namespace and rank of the -process within the application. -In addition, all startup information provided by the resource manager -will be made available to the client process via subsequent calls to -\f[I]PMIx_Get\f[]. -.PP -Note that the PMIx client library is referenced counted, and so multiple -calls to PMIx_Init are allowed. -Thus, one way to obtain the namespace and rank of the process is to -simply call PMIx_Init with a non\-NULL parameter. -.SH RETURN VALUE -.PP -Returns PMIX_SUCCESS on success. -On error, a negative value corresponding to a PMIx errno is returned. -.SH ERRORS -.PP -PMIx errno values are defined in \f[C]pmix_common.h\f[]. -.SH NOTES -.SH SEE ALSO -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_initialized.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_initialized.3 deleted file mode 100644 index 6e2170f35c4..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_initialized.3 +++ /dev/null @@ -1,30 +0,0 @@ -.TH "pmix_initialized" "3" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx_Initialized \- Check if \f[I]PMIx_Init\f[] has been called -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ - -int\ PMIx_Initialized(void); -\f[] -.fi -.SH ARGUMENTS -.PP -\f[I]none\f[] -.SH DESCRIPTION -.PP -Check to see if the PMIx Client library has been intialized -.SH RETURN VALUE -.PP -Returns \f[I]true\f[] if the PMIx Client has been initialized, and -\f[I]false\f[] if not. -.SH ERRORS -.SH NOTES -.SH SEE ALSO -.PP -\f[C]PMIx_Init\f[](3) -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_put.3 b/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_put.3 deleted file mode 100644 index e960583e404..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/man/man3/pmix_put.3 +++ /dev/null @@ -1,60 +0,0 @@ -.TH "pmix_put" "3" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx_Put \- Push a value into the client\[aq]s namespace -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ - -pmix\\_status\\_t\ PMIx\\_Init(pmix\\_scope\\_t\ scope,\ const\ char\ key[],\ pmix\\_value\\_t\ *val); -\f[] -.fi -.SH ARGUMENTS -.PP -\f[I]scope\f[] : Defines a scope for data "put" by PMI per the -following: -.IP "(a)" 4 -PMI_LOCAL \- the data is intended only for other application processes -on the same node. -Data marked in this way will not be included in data packages sent to -remote requestors -.IP "(b)" 4 -PMI_REMOTE \- the data is intended solely for application processes on -remote nodes. -Data marked in this way will not be shared with other processes on the -same node -.IP "(c)" 4 -PMI_GLOBAL \- the data is to be shared with all other requesting -processes, regardless of location -.PP -\f[I]key\f[] : String key identifying the information. -This can be either one of the PMIx defined attributes, or a -user\-defined value -.PP -\f[I]val\f[] : Pointer to a pmix_value_t structure containing the data -to be pushed along with the type of the provided data. -.SH DESCRIPTION -.PP -Push a value into the client\[aq]s namespace. -The client library will cache the information locally until -\f[I]PMIx_Commit\f[] is called. -The provided scope value is passed to the local PMIx server, which will -distribute the data as directed. -.SH RETURN VALUE -.PP -Returns PMIX_SUCCESS on success. -On error, a negative value corresponding to a PMIx errno is returned. -.SH ERRORS -.PP -PMIx errno values are defined in \f[C]pmix_common.h\f[]. -.SH NOTES -.PP -See \[aq]pmix_common.h\[aq] for definition of the pmix_value_t -structure. -.SH SEE ALSO -.PP -\f[C]PMIx_Constants\f[](7), \f[C]PMIx_Structures\f[](7) -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix2x/pmix/man/man7/pmix.7 b/opal/mca/pmix/pmix2x/pmix/man/man7/pmix.7 deleted file mode 100644 index d2e545af94f..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/man/man7/pmix.7 +++ /dev/null @@ -1,35 +0,0 @@ -.TH "pmix" "7" "2015\-10\-29" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -Process Management Interface \- Exascale -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ -\f[] -.fi -.SH OVERVIEW -.PP -The Process Management Interface (PMI) has been used for quite some time -as a means of exchanging wireup information needed for interprocess -communication. -Two versions (PMI\-1 and PMI\-2) have been released as part of the MPICH -effort. -While PMI\-2 demonstrates better scaling properties than its PMI\-1 -predecessor, attaining rapid launch and wireup of the roughly 1M -processes executing across 100k nodes expected for exascale operations -remains challenging. -.PP -PMI Exascale (PMIx) represents an attempt to resolve these questions by -providing an extended version of the PMI standard specifically designed -to support clusters up to and including exascale sizes. -The overall objective of the project is not to branch the existing -pseudo\-standard definitions \- in fact, PMIx fully supports both of the -existing PMI\-1 and PMI\-2 APIs \- but rather to (a) augment and extend -those APIs to eliminate some current restrictions that impact -scalability, and (b) provide a reference implementation of the -PMI\-server that demonstrates the desired level of scalability. -.SH SEE ALSO -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix2x/pmix/man/man7/pmix_constants.7 b/opal/mca/pmix/pmix2x/pmix/man/man7/pmix_constants.7 deleted file mode 100644 index bd6414cb7a2..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/man/man7/pmix_constants.7 +++ /dev/null @@ -1,79 +0,0 @@ -.TH "pmix_constants" "7" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx Constants -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ -\f[] -.fi -.SH OVERVIEW -.PP -PMIx relies on the following types of constants: -.PP -\f[I]Maximum Sizes\f[] : In order to minimize malloc performance -penalties, PMIx utilizes constant\-sized arrays wherever possible. -These constants provide the user with the maximum size of the various -array types. -.PP -\f[I]Attributes\f[] : . -.PP -\f[I]Errors\f[] : PMIx uses negative error constants, with 0 indicating -"success". -.SH MAXIMUM SIZES -.PP -The . -.PP -\f[I]PMIX_MAX_NSLEN\f[] : The maximum length of a namespace. -Note that any declaration of an array to hold a key string must include -one extra space for the terminating \f[I]NULL\f[]. -.PP -\f[I]PMIX_MAX_KEYLEN\f[] : Maximum length of the key string used in -structures such as the \f[I]pmix\f[]info_t_. -Note that any declaration of an array to hold a key string must include -one extra space for the terminating \f[I]NULL\f[]. -.SH ATTRIBUTES -.PP -Define a set of "standard" PMIx attributes that can be queried using the -PMIx_Get function. -Implementations (and users) are free to extend as desired \- thus, -functions calling PMIx_Get must be capable of handling the "not found" -condition. -Note that these are attributes of the system and the job as opposed to -values the application (or underlying programming library) might choose -to expose \- i.e., they are values provided by the resource manager as -opposed to the application. -Thus, these keys are RESERVED for use by PMIx, and users should avoid -defining any attribute starting with the keyword \f[I]PMIX\f[]. -.PP -A list of the current PMIx attributes, and the type of their associated -data value, is provided here. -.PP -\f[I]PMIX_ATTR_UNDEF (NULL)\f[] : Used to initialize an attribute field, -indicating that the attribute has not yet been assigned. -.PP -\f[I]PMIX_USERID (uint32_t)\f[] : . -.PP -\f[I]PMIX_GRPID (uint32_t)\f[] : An access domain represents a single -logical connection into a fabric. -It may map to a single physical or virtual NIC or a port. -An access domain defines the boundary across which fabric resources may -be associated. -Each access domain belongs to a single fabric domain. -.PP -\f[I]PMIX_CPUSET (char*)\f[] : . -.SH ERROR CONSTANTS -.PP -\&. -.PP -\f[I]PMIX_SUCCESS\f[] : Indicates that the operation was successful. -.PP -\f[I]PMIX_ERROR\f[] : A general error code \- an error occurred, but no -specific reason can be provided. -.SH SEE ALSO -.PP -\f[C]pmix\f[](7) -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix2x/pmix/src/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/Makefile.am index acf37ff2ff1..720dbb55599 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2006-2016 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/Makefile.include b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/Makefile.include index dbd384e3e5a..111c76808b5 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/Makefile.include +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/Makefile.include @@ -15,7 +15,7 @@ # reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. -# Copyright (c) 2017 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -30,7 +30,8 @@ headers += \ atomics/sys/atomic.h \ atomics/sys/atomic_impl.h \ atomics/sys/timer.h \ - atomics/sys/cma.h + atomics/sys/cma.h \ + atomics/sys/atomic_stdc.h include atomics/sys/x86_64/Makefile.include include atomics/sys/arm/Makefile.include diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/architecture.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/architecture.h index cb386f67b3c..3e9e8e5a756 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/architecture.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/architecture.h @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017 Research Organization for Information Science @@ -47,6 +47,7 @@ #define PMIX_BUILTIN_SYNC 0200 #define PMIX_BUILTIN_GCC 0202 #define PMIX_BUILTIN_NO 0203 +#define PMIX_BUILTIN_C11 0204 /* Formats */ #define PMIX_DEFAULT 1000 /* standard for given architecture */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/arm/atomic.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/arm/atomic.h index 1598547f34d..6c346088280 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/arm/atomic.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/arm/atomic.h @@ -12,9 +12,9 @@ * All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 ARM ltd. All rights reserved. - * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -110,7 +110,7 @@ void pmix_atomic_isync(void) #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 #define PMIX_HAVE_ATOMIC_MATH_32 1 -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -138,7 +138,7 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { bool rc; @@ -149,7 +149,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { pmix_atomic_wmb(); return pmix_atomic_compare_exchange_strong_32 (addr, oldval, newval); @@ -158,7 +158,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t #if (PMIX_ASM_SUPPORT_64BIT == 1) #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -189,7 +189,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { bool rc; @@ -200,7 +200,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { pmix_atomic_wmb(); return pmix_atomic_compare_exchange_strong_64 (addr, oldval, newval); @@ -210,7 +210,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define PMIX_HAVE_ATOMIC_ADD_32 1 -static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int inc) +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t* v, int inc) { int32_t t, old; int tmp; @@ -231,7 +231,7 @@ static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int inc) } #define PMIX_HAVE_ATOMIC_SUB_32 1 -static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t* v, int dec) +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t* v, int dec) { int32_t t, old; int tmp; diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/arm64/atomic.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/arm64/atomic.h index b3df2624313..b7df6458bfc 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/arm64/atomic.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/arm64/atomic.h @@ -12,9 +12,9 @@ * All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 ARM ltd. All rights reserved. - * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -83,7 +83,7 @@ static inline void pmix_atomic_isync (void) * *********************************************************************/ -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -103,7 +103,7 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add return ret; } -static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, int32_t newval) +static inline int32_t pmix_atomic_swap_32(pmix_atomic_int32_t *addr, int32_t newval) { int32_t ret, tmp; @@ -122,7 +122,7 @@ static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, int32_t newval atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -143,7 +143,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -165,7 +165,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t #define pmix_atomic_ll_32(addr, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + pmix_atomic_int32_t *_addr = (addr); \ int32_t _ret; \ \ __asm__ __volatile__ ("ldaxr %w0, [%1] \n" \ @@ -177,7 +177,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t #define pmix_atomic_sc_32(addr, newval, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + pmix_atomic_int32_t *_addr = (addr); \ int32_t _newval = (int32_t) newval; \ int _ret; \ \ @@ -189,7 +189,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t ret = (_ret == 0); \ } while (0) -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -210,7 +210,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add return ret; } -static inline int64_t pmix_atomic_swap_64 (volatile int64_t *addr, int64_t newval) +static inline int64_t pmix_atomic_swap_64 (pmix_atomic_int64_t *addr, int64_t newval) { int64_t ret; int tmp; @@ -230,7 +230,7 @@ static inline int64_t pmix_atomic_swap_64 (volatile int64_t *addr, int64_t newva atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -252,7 +252,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -275,7 +275,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define pmix_atomic_ll_64(addr, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + pmix_atomic_int64_t *_addr = (addr); \ int64_t _ret; \ \ __asm__ __volatile__ ("ldaxr %0, [%1] \n" \ @@ -287,7 +287,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define pmix_atomic_sc_64(addr, newval, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + pmix_atomic_int64_t *_addr = (addr); \ int64_t _newval = (int64_t) newval; \ int _ret; \ \ @@ -300,7 +300,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t } while (0) #define PMIX_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \ - static inline type pmix_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \ + static inline type pmix_atomic_fetch_ ## name ## _ ## bits (pmix_atomic_ ## type *addr, type value) \ { \ type newval, old; \ int32_t tmp; \ diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/atomic.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/atomic.h index f28ee1d35a2..fd5a113a888 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/atomic.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/atomic.h @@ -16,7 +16,7 @@ * reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -57,7 +57,13 @@ #include #include "src/atomics/sys/architecture.h" -#include "src/include/pmix_stdint.h" +#include "src/include/pmix_stdatomic.h" + +#if PMIX_ASSEMBLY_BUILTIN == PMIX_BUILTIN_C11 + +#include "atomic_stdc.h" + +#else /* !PMIX_C_HAVE__ATOMIC */ /* do some quick #define cleanup in cases where we are doing testing... */ @@ -93,7 +99,7 @@ BEGIN_C_DECLS */ struct pmix_atomic_lock_t { union { - volatile int32_t lock; /**< The lock address (an integer) */ + pmix_atomic_int32_t lock; /**< The lock address (an integer) */ volatile unsigned char sparc_lock; /**< The lock address on sparc */ char padding[sizeof(int)]; /**< Array for optional padding */ } u; @@ -148,6 +154,8 @@ enum { PMIX_ATOMIC_LOCK_LOCKED = 1 }; +#define PMIX_ATOMIC_LOCK_INIT {.u = {.lock = PMIX_ATOMIC_LOCK_UNLOCKED}} + /********************************************************************** * * Load the appropriate architecture files and set some reasonable @@ -351,19 +359,19 @@ void pmix_atomic_unlock(pmix_atomic_lock_t *lock); #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 static inline #endif -bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, +bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval); #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 static inline #endif -bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, +bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval); #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 static inline #endif -bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, +bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval); #endif @@ -376,19 +384,19 @@ bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 static inline #endif -bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, +bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval); #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 static inline #endif -bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, +bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval); #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 static inline #endif -bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, +bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval); #endif @@ -400,20 +408,20 @@ bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 -static inline int32_t pmix_atomic_add_fetch_32(volatile int32_t *addr, int delta); -static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t *addr, int delta); -static inline int32_t pmix_atomic_and_fetch_32(volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_fetch_and_32(volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_or_fetch_32(volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_fetch_or_32(volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_sub_fetch_32(volatile int32_t *addr, int delta); -static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t *addr, int delta); -static inline int32_t pmix_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_add_fetch_32(pmix_atomic_int32_t *addr, int delta); +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t *addr, int delta); +static inline int32_t pmix_atomic_and_fetch_32(pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_fetch_and_32(pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_or_fetch_32(pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_fetch_or_32(pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_xor_fetch_32(pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_fetch_xor_32(pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_sub_fetch_32(pmix_atomic_int32_t *addr, int delta); +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t *addr, int delta); +static inline int32_t pmix_atomic_min_fetch_32 (pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_fetch_min_32 (pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_max_fetch_32 (pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_fetch_max_32 (pmix_atomic_int32_t *addr, int32_t value); #endif /* PMIX_HAVE_ATOMIC_MATH_32 */ @@ -430,19 +438,19 @@ static inline int32_t pmix_atomic_fetch_max_32 (volatile int32_t *addr, int32_t #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MATH_64 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 -static inline int64_t pmix_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta); -static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta); -static inline int64_t pmix_atomic_and_fetch_64(volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_fetch_and_64(volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_or_fetch_64(volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_fetch_or_64(volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta); -static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta); -static inline int64_t pmix_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_add_fetch_64(pmix_atomic_int64_t *addr, int64_t delta); +static inline int64_t pmix_atomic_fetch_add_64(pmix_atomic_int64_t *addr, int64_t delta); +static inline int64_t pmix_atomic_and_fetch_64(pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_fetch_and_64(pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_or_fetch_64(pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_fetch_or_64(pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_fetch_xor_64(pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_sub_fetch_64(pmix_atomic_int64_t *addr, int64_t delta); +static inline int64_t pmix_atomic_fetch_sub_64(pmix_atomic_int64_t *addr, int64_t delta); +static inline int64_t pmix_atomic_min_fetch_64 (pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_fetch_min_64 (pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_max_fetch_64 (pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_fetch_max_64 (pmix_atomic_int64_t *addr, int64_t value); #endif /* PMIX_HAVE_ATOMIC_MATH_64 */ @@ -459,7 +467,7 @@ static inline int64_t pmix_atomic_fetch_max_64 (volatile int64_t *addr, int64_t */ #if defined(DOXYGEN) || PMIX_ENABLE_DEBUG static inline size_t -pmix_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta) +pmix_atomic_add_fetch_size_t(pmix_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) pmix_atomic_add_fetch_32((int32_t*) addr, delta); @@ -471,7 +479,7 @@ pmix_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta) } static inline size_t -pmix_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta) +pmix_atomic_fetch_add_size_t(pmix_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) pmix_atomic_fetch_add_32((int32_t*) addr, delta); @@ -483,7 +491,7 @@ pmix_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta) } static inline size_t -pmix_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta) +pmix_atomic_sub_fetch_size_t(pmix_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) pmix_atomic_sub_fetch_32((int32_t*) addr, delta); @@ -495,7 +503,7 @@ pmix_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta) } static inline size_t -pmix_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) +pmix_atomic_fetch_sub_size_t(pmix_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) pmix_atomic_fetch_sub_32((int32_t*) addr, delta); @@ -508,15 +516,15 @@ pmix_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) #else #if SIZEOF_SIZE_T == 4 -#define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_32((volatile int32_t *) addr, delta)) -#define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_32((volatile int32_t *) addr, delta)) -#define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_32((volatile int32_t *) addr, delta)) -#define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_32((volatile int32_t *) addr, delta)) +#define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_32((pmix_atomic_int32_t *) addr, delta)) +#define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_32((pmix_atomic_int32_t *) addr, delta)) +#define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_32((pmix_atomic_int32_t *) addr, delta)) +#define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_32((pmix_atomic_int32_t *) addr, delta)) #elif SIZEOF_SIZE_T == 8 -#define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_64((volatile int64_t *) addr, delta)) -#define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_64((volatile int64_t *) addr, delta)) -#define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_64((volatile int64_t *) addr, delta)) -#define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_64((volatile int64_t *) addr, delta)) +#define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_64((pmix_atomic_int64_t *) addr, delta)) +#define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_64((pmix_atomic_int64_t *) addr, delta)) +#define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_64((pmix_atomic_int64_t *) addr, delta)) +#define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_64((pmix_atomic_int64_t *) addr, delta)) #else #error "Unknown size_t size" #endif @@ -526,20 +534,20 @@ pmix_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) /* these are always done with inline functions, so always mark as static inline */ -static inline bool pmix_atomic_compare_exchange_strong_xx (volatile void *addr, void *oldval, +static inline bool pmix_atomic_compare_exchange_strong_xx (pmix_atomic_intptr_t *addr, intptr_t *oldval, int64_t newval, size_t length); -static inline bool pmix_atomic_compare_exchange_strong_acq_xx (volatile void *addr, void *oldval, +static inline bool pmix_atomic_compare_exchange_strong_acq_xx (pmix_atomic_intptr_t *addr, intptr_t *oldval, int64_t newval, size_t length); -static inline bool pmix_atomic_compare_exchange_strong_rel_xx (volatile void *addr, void *oldval, +static inline bool pmix_atomic_compare_exchange_strong_rel_xx (pmix_atomic_intptr_t *addr, intptr_t *oldval, int64_t newval, size_t length); -static inline bool pmix_atomic_compare_exchange_strong_ptr (volatile void* addr, void *oldval, - void *newval); -static inline bool pmix_atomic_compare_exchange_strong_acq_ptr (volatile void* addr, void *oldval, - void *newval); -static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (volatile void* addr, void *oldval, - void *newval); +static inline bool pmix_atomic_compare_exchange_strong_ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval, + intptr_t newval); +static inline bool pmix_atomic_compare_exchange_strong_acq_ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval, + intptr_t newval); +static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval, + intptr_t newval); /** * Atomic compare and set of generic type with relaxed semantics. This @@ -555,7 +563,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (volatile void* a * See pmix_atomic_compare_exchange_* for pseudo-code. */ #define pmix_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL ) \ - pmix_atomic_compare_exchange_strong_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ + pmix_atomic_compare_exchange_strong_xx( (pmix_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \ (intptr_t)(NEWVAL), sizeof(*(ADDR)) ) /** @@ -572,7 +580,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (volatile void* a * See pmix_atomic_compare_exchange_acq_* for pseudo-code. */ #define pmix_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL ) \ - pmix_atomic_compare_exchange_strong_acq_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ + pmix_atomic_compare_exchange_strong_acq_xx( (pmix_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \ (intptr_t)(NEWVAL), sizeof(*(ADDR)) ) /** @@ -589,7 +597,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (volatile void* a * See pmix_atomic_compare_exchange_rel_* for pseudo-code. */ #define pmix_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \ - pmix_atomic_compare_exchange_strong_rel_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ + pmix_atomic_compare_exchange_strong_rel_xx( (pmix_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \ (intptr_t)(NEWVAL), sizeof(*(ADDR)) ) @@ -597,15 +605,15 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (volatile void* a #if defined(DOXYGEN) || (PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64) -static inline void pmix_atomic_add_xx(volatile void* addr, +static inline void pmix_atomic_add_xx(pmix_atomic_intptr_t* addr, int32_t value, size_t length); -static inline void pmix_atomic_sub_xx(volatile void* addr, +static inline void pmix_atomic_sub_xx(pmix_atomic_intptr_t* addr, int32_t value, size_t length); -static inline intptr_t pmix_atomic_add_fetch_ptr( volatile void* addr, void* delta ); -static inline intptr_t pmix_atomic_fetch_add_ptr( volatile void* addr, void* delta ); -static inline intptr_t pmix_atomic_sub_fetch_ptr( volatile void* addr, void* delta ); -static inline intptr_t pmix_atomic_fetch_sub_ptr( volatile void* addr, void* delta ); +static inline intptr_t pmix_atomic_add_fetch_ptr( pmix_atomic_intptr_t* addr, void* delta ); +static inline intptr_t pmix_atomic_fetch_add_ptr( pmix_atomic_intptr_t* addr, void* delta ); +static inline intptr_t pmix_atomic_sub_fetch_ptr( pmix_atomic_intptr_t* addr, void* delta ); +static inline intptr_t pmix_atomic_fetch_sub_ptr( pmix_atomic_intptr_t* addr, void* delta ); /** * Atomically increment the content depending on the type. This @@ -618,7 +626,7 @@ static inline intptr_t pmix_atomic_fetch_sub_ptr( volatile void* addr, void* del * @param delta Value to add (converted to ). */ #define pmix_atomic_add( ADDR, VALUE ) \ - pmix_atomic_add_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ + pmix_atomic_add_xx( (pmix_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \ sizeof(*(ADDR)) ) /** @@ -632,7 +640,7 @@ static inline intptr_t pmix_atomic_fetch_sub_ptr( volatile void* addr, void* del * @param delta Value to substract (converted to ). */ #define pmix_atomic_sub( ADDR, VALUE ) \ - pmix_atomic_sub_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ + pmix_atomic_sub_xx( (pmix_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \ sizeof(*(ADDR)) ) #endif /* PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64 */ @@ -644,6 +652,8 @@ static inline intptr_t pmix_atomic_fetch_sub_ptr( volatile void* addr, void* del */ #include "src/atomics/sys/atomic_impl.h" +#endif /* !PMIX_C_HAVE__ATOMIC */ + END_C_DECLS #endif /* PMIX_SYS_ATOMIC_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/atomic_impl.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/atomic_impl.h index d3a3f40dd31..d03f83de283 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/atomic_impl.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/atomic_impl.h @@ -11,9 +11,9 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -41,7 +41,7 @@ #if PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #if !defined(PMIX_HAVE_ATOMIC_MIN_32) -static inline int32_t pmix_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_min_32 (pmix_atomic_int32_t *addr, int32_t value) { int32_t old = *addr; do { @@ -58,7 +58,7 @@ static inline int32_t pmix_atomic_fetch_min_32 (volatile int32_t *addr, int32_t #endif /* PMIX_HAVE_ATOMIC_MIN_32 */ #if !defined(PMIX_HAVE_ATOMIC_MAX_32) -static inline int32_t pmix_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_max_32 (pmix_atomic_int32_t *addr, int32_t value) { int32_t old = *addr; do { @@ -74,7 +74,7 @@ static inline int32_t pmix_atomic_fetch_max_32 (volatile int32_t *addr, int32_t #endif /* PMIX_HAVE_ATOMIC_MAX_32 */ #define PMIX_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operation, name) \ - static inline type pmix_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \ + static inline type pmix_atomic_fetch_ ## name ## _ ## bits (pmix_atomic_ ## type *addr, type value) \ { \ type oldval; \ do { \ @@ -86,7 +86,7 @@ static inline int32_t pmix_atomic_fetch_max_32 (volatile int32_t *addr, int32_t #if !defined(PMIX_HAVE_ATOMIC_SWAP_32) #define PMIX_HAVE_ATOMIC_SWAP_32 1 -static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, +static inline int32_t pmix_atomic_swap_32(pmix_atomic_int32_t *addr, int32_t newval) { int32_t old = *addr; @@ -139,7 +139,7 @@ PMIX_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, -, sub) #if PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #if !defined(PMIX_HAVE_ATOMIC_MIN_64) -static inline int64_t pmix_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_min_64 (pmix_atomic_int64_t *addr, int64_t value) { int64_t old = *addr; do { @@ -156,7 +156,7 @@ static inline int64_t pmix_atomic_fetch_min_64 (volatile int64_t *addr, int64_t #endif /* PMIX_HAVE_ATOMIC_MIN_64 */ #if !defined(PMIX_HAVE_ATOMIC_MAX_64) -static inline int64_t pmix_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_max_64 (pmix_atomic_int64_t *addr, int64_t value) { int64_t old = *addr; do { @@ -173,7 +173,7 @@ static inline int64_t pmix_atomic_fetch_max_64 (volatile int64_t *addr, int64_t #if !defined(PMIX_HAVE_ATOMIC_SWAP_64) #define PMIX_HAVE_ATOMIC_SWAP_64 1 -static inline int64_t pmix_atomic_swap_64(volatile int64_t *addr, +static inline int64_t pmix_atomic_swap_64(pmix_atomic_int64_t *addr, int64_t newval) { int64_t old = *addr; @@ -236,15 +236,15 @@ PMIX_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub) #if PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 && PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #define PMIX_ATOMIC_DEFINE_CMPXCG_XX(semantics) \ static inline bool \ - pmix_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \ + pmix_atomic_compare_exchange_strong ## semantics ## xx (pmix_atomic_intptr_t* addr, intptr_t *oldval, \ int64_t newval, const size_t length) \ { \ switch (length) { \ case 4: \ - return pmix_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \ + return pmix_atomic_compare_exchange_strong_32 ((pmix_atomic_int32_t *) addr, \ (int32_t *) oldval, (int32_t) newval); \ case 8: \ - return pmix_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, \ + return pmix_atomic_compare_exchange_strong_64 ((pmix_atomic_int64_t *) addr, \ (int64_t *) oldval, (int64_t) newval); \ } \ abort(); \ @@ -252,12 +252,12 @@ PMIX_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub) #elif PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #define PMIX_ATOMIC_DEFINE_CMPXCG_XX(semantics) \ static inline bool \ - pmix_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \ + pmix_atomic_compare_exchange_strong ## semantics ## xx (pmix_atomic_intptr_t* addr, intptr_t *oldval, \ int64_t newval, const size_t length) \ { \ switch (length) { \ case 4: \ - return pmix_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \ + return pmix_atomic_compare_exchange_strong_32 ((pmix_atomic_int32_t *) addr, \ (int32_t *) oldval, (int32_t) newval); \ } \ abort(); \ @@ -273,16 +273,16 @@ PMIX_ATOMIC_DEFINE_CMPXCG_XX(_rel_) #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #define PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \ static inline bool \ - pmix_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \ + pmix_atomic_compare_exchange_strong ## semantics ## ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \ { \ - return pmix_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \ + return pmix_atomic_compare_exchange_strong_32 ((pmix_atomic_int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \ } #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #define PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \ static inline bool \ - pmix_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \ + pmix_atomic_compare_exchange_strong ## semantics ## ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \ { \ - return pmix_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \ + return pmix_atomic_compare_exchange_strong_64 ((pmix_atomic_int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \ } #else #error "Can not define pmix_atomic_compare_exchange_strong_ptr with existing atomics" @@ -298,9 +298,9 @@ PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if (PMIX_HAVE_ATOMIC_SWAP_32 || PMIX_HAVE_ATOMIC_SWAP_64) #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_SWAP_32 -#define pmix_atomic_swap_ptr(addr, value) (void *) pmix_atomic_swap_32((int32_t *) addr, (int32_t) value) +#define pmix_atomic_swap_ptr(addr, value) (intptr_t) pmix_atomic_swap_32((pmix_atomic_int32_t *) addr, (int32_t) value) #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_SWAP_64 -#define pmix_atomic_swap_ptr(addr, value) (void *) pmix_atomic_swap_64((int64_t *) addr, (int64_t) value) +#define pmix_atomic_swap_ptr(addr, value) (intptr_t) pmix_atomic_swap_64((pmix_atomic_int64_t *) addr, (int64_t) value) #endif #endif /* (PMIX_HAVE_ATOMIC_SWAP_32 || PMIX_HAVE_ATOMIC_SWAP_64) */ @@ -309,15 +309,15 @@ PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_LLSC_32 -#define pmix_atomic_ll_ptr(addr, ret) pmix_atomic_ll_32((volatile int32_t *) (addr), ret) -#define pmix_atomic_sc_ptr(addr, value, ret) pmix_atomic_sc_32((volatile int32_t *) (addr), (intptr_t) (value), ret) +#define pmix_atomic_ll_ptr(addr, ret) pmix_atomic_ll_32((pmix_atomic_int32_t *) (addr), ret) +#define pmix_atomic_sc_ptr(addr, value, ret) pmix_atomic_sc_32((pmix_atomic_int32_t *) (addr), (intptr_t) (value), ret) #define PMIX_HAVE_ATOMIC_LLSC_PTR 1 #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_LLSC_64 -#define pmix_atomic_ll_ptr(addr, ret) pmix_atomic_ll_64((volatile int64_t *) (addr), ret) -#define pmix_atomic_sc_ptr(addr, value, ret) pmix_atomic_sc_64((volatile int64_t *) (addr), (intptr_t) (value), ret) +#define pmix_atomic_ll_ptr(addr, ret) pmix_atomic_ll_64((pmix_atomic_int64_t *) (addr), ret) +#define pmix_atomic_sc_ptr(addr, value, ret) pmix_atomic_sc_64((pmix_atomic_int64_t *) (addr), (intptr_t) (value), ret) #define PMIX_HAVE_ATOMIC_LLSC_PTR 1 @@ -332,18 +332,18 @@ PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64 static inline void - pmix_atomic_add_xx(volatile void* addr, int32_t value, size_t length) + pmix_atomic_add_xx(pmix_atomic_intptr_t* addr, int32_t value, size_t length) { switch( length ) { #if PMIX_HAVE_ATOMIC_ADD_32 case 4: - (void) pmix_atomic_fetch_add_32( (volatile int32_t*)addr, (int32_t)value ); + (void) pmix_atomic_fetch_add_32( (pmix_atomic_int32_t*)addr, (int32_t)value ); break; #endif /* PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */ #if PMIX_HAVE_ATOMIC_ADD_64 case 8: - (void) pmix_atomic_fetch_add_64( (volatile int64_t*)addr, (int64_t)value ); + (void) pmix_atomic_fetch_add_64( (pmix_atomic_int64_t*)addr, (int64_t)value ); break; #endif /* PMIX_HAVE_ATOMIC_ADD_64 */ default: @@ -355,18 +355,18 @@ static inline void static inline void -pmix_atomic_sub_xx(volatile void* addr, int32_t value, size_t length) +pmix_atomic_sub_xx(pmix_atomic_intptr_t* addr, int32_t value, size_t length) { switch( length ) { #if PMIX_HAVE_ATOMIC_SUB_32 case 4: - (void) pmix_atomic_fetch_sub_32( (volatile int32_t*)addr, (int32_t)value ); + (void) pmix_atomic_fetch_sub_32( (pmix_atomic_int32_t*)addr, (int32_t)value ); break; #endif /* PMIX_HAVE_ATOMIC_SUB_32 */ #if PMIX_HAVE_ATOMIC_SUB_64 case 8: - (void) pmix_atomic_fetch_sub_64( (volatile int64_t*)addr, (int64_t)value ); + (void) pmix_atomic_fetch_sub_64( (pmix_atomic_int64_t*)addr, (int64_t)value ); break; #endif /* PMIX_HAVE_ATOMIC_SUB_64 */ default: @@ -377,7 +377,7 @@ pmix_atomic_sub_xx(volatile void* addr, int32_t value, size_t length) } #define PMIX_ATOMIC_DEFINE_OP_FETCH(op, operation, type, ptr_type, suffix) \ - static inline type pmix_atomic_ ## op ## _fetch_ ## suffix (volatile ptr_type *addr, type value) \ + static inline type pmix_atomic_ ## op ## _fetch_ ## suffix (pmix_atomic_ ## ptr_type *addr, type value) \ { \ return pmix_atomic_fetch_ ## op ## _ ## suffix (addr, value) operation value; \ } @@ -388,13 +388,13 @@ PMIX_ATOMIC_DEFINE_OP_FETCH(or, |, int32_t, int32_t, 32) PMIX_ATOMIC_DEFINE_OP_FETCH(xor, ^, int32_t, int32_t, 32) PMIX_ATOMIC_DEFINE_OP_FETCH(sub, -, int32_t, int32_t, 32) -static inline int32_t pmix_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_min_fetch_32 (pmix_atomic_int32_t *addr, int32_t value) { int32_t old = pmix_atomic_fetch_min_32 (addr, value); return old <= value ? old : value; } -static inline int32_t pmix_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_max_fetch_32 (pmix_atomic_int32_t *addr, int32_t value) { int32_t old = pmix_atomic_fetch_max_32 (addr, value); return old >= value ? old : value; @@ -407,13 +407,13 @@ PMIX_ATOMIC_DEFINE_OP_FETCH(or, |, int64_t, int64_t, 64) PMIX_ATOMIC_DEFINE_OP_FETCH(xor, ^, int64_t, int64_t, 64) PMIX_ATOMIC_DEFINE_OP_FETCH(sub, -, int64_t, int64_t, 64) -static inline int64_t pmix_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_min_fetch_64 (pmix_atomic_int64_t *addr, int64_t value) { int64_t old = pmix_atomic_fetch_min_64 (addr, value); return old <= value ? old : value; } -static inline int64_t pmix_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_max_fetch_64 (pmix_atomic_int64_t *addr, int64_t value) { int64_t old = pmix_atomic_fetch_max_64 (addr, value); return old >= value ? old : value; @@ -421,52 +421,52 @@ static inline int64_t pmix_atomic_max_fetch_64 (volatile int64_t *addr, int64_t #endif -static inline intptr_t pmix_atomic_fetch_add_ptr( volatile void* addr, +static inline intptr_t pmix_atomic_fetch_add_ptr( pmix_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_ADD_32 - return pmix_atomic_fetch_add_32((int32_t*) addr, (unsigned long) delta); + return pmix_atomic_fetch_add_32((pmix_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_ADD_64 - return pmix_atomic_fetch_add_64((int64_t*) addr, (unsigned long) delta); + return pmix_atomic_fetch_add_64((pmix_atomic_int64_t*) addr, (unsigned long) delta); #else abort (); return 0; #endif } -static inline intptr_t pmix_atomic_add_fetch_ptr( volatile void* addr, +static inline intptr_t pmix_atomic_add_fetch_ptr( pmix_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_ADD_32 - return pmix_atomic_add_fetch_32((int32_t*) addr, (unsigned long) delta); + return pmix_atomic_add_fetch_32((pmix_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_ADD_64 - return pmix_atomic_add_fetch_64((int64_t*) addr, (unsigned long) delta); + return pmix_atomic_add_fetch_64((pmix_atomic_int64_t*) addr, (unsigned long) delta); #else abort (); return 0; #endif } -static inline intptr_t pmix_atomic_fetch_sub_ptr( volatile void* addr, +static inline intptr_t pmix_atomic_fetch_sub_ptr( pmix_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_SUB_32 - return pmix_atomic_fetch_sub_32((int32_t*) addr, (unsigned long) delta); + return pmix_atomic_fetch_sub_32((pmix_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_SUB_32 - return pmix_atomic_fetch_sub_64((int64_t*) addr, (unsigned long) delta); + return pmix_atomic_fetch_sub_64((pmix_atomic_int64_t*) addr, (unsigned long) delta); #else abort(); return 0; #endif } -static inline intptr_t pmix_atomic_sub_fetch_ptr( volatile void* addr, +static inline intptr_t pmix_atomic_sub_fetch_ptr( pmix_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_SUB_32 - return pmix_atomic_sub_fetch_32((int32_t*) addr, (unsigned long) delta); + return pmix_atomic_sub_fetch_32((pmix_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_SUB_32 - return pmix_atomic_sub_fetch_64((int64_t*) addr, (unsigned long) delta); + return pmix_atomic_sub_fetch_64((pmix_atomic_int64_t*) addr, (unsigned long) delta); #else abort(); return 0; @@ -496,7 +496,7 @@ static inline int pmix_atomic_trylock(pmix_atomic_lock_t *lock) { int32_t unlocked = PMIX_ATOMIC_LOCK_UNLOCKED; - bool ret = pmix_atomic_compare_exchange_strong_32 (&lock->u.lock, &unlocked, PMIX_ATOMIC_LOCK_LOCKED); + bool ret = pmix_atomic_compare_exchange_strong_acq_32 (&lock->u.lock, &unlocked, PMIX_ATOMIC_LOCK_LOCKED); return (ret == false) ? 1 : 0; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/atomic_stdc.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/atomic_stdc.h new file mode 100644 index 00000000000..5fc5b0a1326 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/atomic_stdc.h @@ -0,0 +1,269 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/* This file provides shims between the pmix atomics interface and the C11 atomics interface. It + * is intended as the first step in moving to using C11 atomics across the entire codebase. Once + * all officially supported compilers offer C11 atomic (GCC 4.9.0+, icc 2018+, pgi, xlc, etc) then + * this shim will go away and the codebase will be updated to use C11's atomic support + * directly. + * This shim contains some functions already present in atomic_impl.h because we do not include + * atomic_impl.h when using C11 atomics. It would require alot of #ifdefs to avoid duplicate + * definitions to be worthwhile. */ + +#if !defined(PMIX_ATOMIC_STDC_H) +#define PMIX_ATOMIC_STDC_H + +#include +#include +#include "src/include/pmix_stdint.h" + +#define PMIX_HAVE_ATOMIC_MEM_BARRIER 1 + +#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 +#define PMIX_HAVE_ATOMIC_SWAP_32 1 + +#define PMIX_HAVE_ATOMIC_MATH_32 1 +#define PMIX_HAVE_ATOMIC_ADD_32 1 +#define PMIX_HAVE_ATOMIC_AND_32 1 +#define PMIX_HAVE_ATOMIC_OR_32 1 +#define PMIX_HAVE_ATOMIC_XOR_32 1 +#define PMIX_HAVE_ATOMIC_SUB_32 1 + +#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 +#define PMIX_HAVE_ATOMIC_SWAP_64 1 + +#define PMIX_HAVE_ATOMIC_MATH_64 1 +#define PMIX_HAVE_ATOMIC_ADD_64 1 +#define PMIX_HAVE_ATOMIC_AND_64 1 +#define PMIX_HAVE_ATOMIC_OR_64 1 +#define PMIX_HAVE_ATOMIC_XOR_64 1 +#define PMIX_HAVE_ATOMIC_SUB_64 1 + +#define PMIX_HAVE_ATOMIC_LLSC_32 0 +#define PMIX_HAVE_ATOMIC_LLSC_64 0 +#define PMIX_HAVE_ATOMIC_LLSC_PTR 0 + +#define PMIX_HAVE_ATOMIC_MIN_32 1 +#define PMIX_HAVE_ATOMIC_MAX_32 1 + +#define PMIX_HAVE_ATOMIC_MIN_64 1 +#define PMIX_HAVE_ATOMIC_MAX_64 1 + +#define PMIX_HAVE_ATOMIC_SPINLOCKS 1 + +static inline void pmix_atomic_mb (void) +{ + atomic_thread_fence (memory_order_seq_cst); +} + +static inline void pmix_atomic_wmb (void) +{ + atomic_thread_fence (memory_order_release); +} + +static inline void pmix_atomic_rmb (void) +{ +#if PMIX_ASSEMBLY_ARCH == PMIX_X86_64 + /* work around a bug in older gcc versions (observed in gcc 6.x) + * where acquire seems to get treated as a no-op instead of being + * equivalent to __asm__ __volatile__("": : :"memory") on x86_64 */ + pmix_atomic_mb (); +#else + atomic_thread_fence (memory_order_acquire); +#endif +} + +#define pmix_atomic_compare_exchange_strong_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_acq_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_acq_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) + +#define pmix_atomic_compare_exchange_strong_rel_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_rel_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) + +#define pmix_atomic_compare_exchange_strong(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_relaxed, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_acq(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_acquire, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_rel(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_release, memory_order_relaxed) + +#define pmix_atomic_swap_32(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed) +#define pmix_atomic_swap_64(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed) +#define pmix_atomic_swap_ptr(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed) + +#define PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator) \ + static inline type pmix_atomic_fetch_ ## op ##_## bits (pmix_atomic_ ## type *addr, type value) \ + { \ + return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed); \ + } \ + \ + static inline type pmix_atomic_## op ## _fetch_ ## bits (pmix_atomic_ ## type *addr, type value) \ + { \ + return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed) operator value; \ + } + +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +) + +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 32, int32_t, -) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 64, int64_t, -) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -) + +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(or, 32, int32_t, |) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(or, 64, int64_t, |) + +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 32, int32_t, ^) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 64, int64_t, ^) + +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(and, 32, int32_t, &) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(and, 64, int64_t, &) + +#define pmix_atomic_add(addr, value) (void) atomic_fetch_add_explicit (addr, value, memory_order_relaxed) + +static inline int32_t pmix_atomic_fetch_min_32 (pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t old = *addr; + do { + if (old <= value) { + break; + } + } while (!pmix_atomic_compare_exchange_strong_32 (addr, &old, value)); + + return old; +} + +static inline int32_t pmix_atomic_fetch_max_32 (pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t old = *addr; + do { + if (old >= value) { + break; + } + } while (!pmix_atomic_compare_exchange_strong_32 (addr, &old, value)); + + return old; +} + +static inline int64_t pmix_atomic_fetch_min_64 (pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t old = *addr; + do { + if (old <= value) { + break; + } + } while (!pmix_atomic_compare_exchange_strong_64 (addr, &old, value)); + + return old; +} + +static inline int64_t pmix_atomic_fetch_max_64 (pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t old = *addr; + do { + if (old >= value) { + break; + } + } while (!pmix_atomic_compare_exchange_strong_64 (addr, &old, value)); + + return old; +} + +static inline int32_t pmix_atomic_min_fetch_32 (pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t old = pmix_atomic_fetch_min_32 (addr, value); + return old <= value ? old : value; +} + +static inline int32_t pmix_atomic_max_fetch_32 (pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t old = pmix_atomic_fetch_max_32 (addr, value); + return old >= value ? old : value; +} + +static inline int64_t pmix_atomic_min_fetch_64 (pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t old = pmix_atomic_fetch_min_64 (addr, value); + return old <= value ? old : value; +} + +static inline int64_t pmix_atomic_max_fetch_64 (pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t old = pmix_atomic_fetch_max_64 (addr, value); + return old >= value ? old : value; +} + +#define PMIX_ATOMIC_LOCK_UNLOCKED false +#define PMIX_ATOMIC_LOCK_LOCKED true + +#define PMIX_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT + +typedef atomic_flag pmix_atomic_lock_t; + +/* + * Lock initialization function. It set the lock to UNLOCKED. + */ +static inline void pmix_atomic_lock_init (pmix_atomic_lock_t *lock, bool value) +{ + atomic_flag_clear (lock); +} + + +static inline int pmix_atomic_trylock (pmix_atomic_lock_t *lock) +{ + return (int) atomic_flag_test_and_set (lock); +} + + +static inline void pmix_atomic_lock(pmix_atomic_lock_t *lock) +{ + while (pmix_atomic_trylock (lock)) { + } +} + + +static inline void pmix_atomic_unlock (pmix_atomic_lock_t *lock) +{ + atomic_flag_clear (lock); +} + + +#if PMIX_HAVE_C11_CSWAP_INT128 + +/* the C11 atomic compare-exchange is lock free so use it */ +#define pmix_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong + +#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 + +#elif PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128 + +/* fall back on the __sync builtin if available since it will emit the expected instruction on x86_64 (cmpxchng16b) */ +__pmix_attribute_always_inline__ +static inline bool pmix_atomic_compare_exchange_strong_128 (pmix_atomic_int128_t *addr, + pmix_int128_t *oldval, pmix_int128_t newval) +{ + pmix_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); + bool ret = prev == *oldval; + *oldval = prev; + return ret; +} + +#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 + +#else + +#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0 + +#endif + +#endif /* !defined(PMIX_ATOMIC_STDC_H) */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/cma.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/cma.h index 3eacce23351..eb57ed5921f 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/cma.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/cma.h @@ -4,7 +4,7 @@ * reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ */ @@ -85,13 +85,13 @@ #elif PMIX_ASSEMBLY_ARCH == PMIX_S390 -#define __NR_process_vm_readv 340 -#define __NR_process_vm_writev 341 +#define __NR_process_vm_readv 340 +#define __NR_process_vm_writev 341 #elif PMIX_ASSEMBLY_ARCH == PMIX_S390X -#define __NR_process_vm_readv 340 -#define __NR_process_vm_writev 341 +#define __NR_process_vm_readv 340 +#define __NR_process_vm_writev 341 #else #error "Unsupported architecture for process_vm_readv and process_vm_writev syscalls" diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/gcc_builtin/atomic.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/gcc_builtin/atomic.h index f3d1a03193b..06b9ef726e8 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/gcc_builtin/atomic.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/gcc_builtin/atomic.h @@ -11,13 +11,13 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * Copyright (c) 2018 Triad National Security, LLC. All rights * reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -86,103 +86,103 @@ static inline void pmix_atomic_wmb(void) /* * Suppress numerous (spurious ?) warnings from Oracle Studio compilers * see https://community.oracle.com/thread/3968347 - */ + */ #if defined(__SUNPRO_C) || defined(__SUNPRO_CC) #pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L) #endif -static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED); } -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline int32_t pmix_atomic_swap_32 (volatile int32_t *addr, int32_t newval) +static inline int32_t pmix_atomic_swap_32 (pmix_atomic_int32_t *addr, int32_t newval) { int32_t oldval; __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED); return oldval; } -static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta) +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t *addr, int32_t delta) { return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED); } -static inline int32_t pmix_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_and_32(pmix_atomic_int32_t *addr, int32_t value) { return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED); } -static inline int32_t pmix_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_or_32(pmix_atomic_int32_t *addr, int32_t value) { return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED); } -static inline int32_t pmix_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_xor_32(pmix_atomic_int32_t *addr, int32_t value) { return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED); } -static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta) +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t *addr, int32_t delta) { return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED); } -static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED); } -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline int64_t pmix_atomic_swap_64 (volatile int64_t *addr, int64_t newval) +static inline int64_t pmix_atomic_swap_64 (pmix_atomic_int64_t *addr, int64_t newval) { int64_t oldval; __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED); return oldval; } -static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta) +static inline int64_t pmix_atomic_fetch_add_64(pmix_atomic_int64_t *addr, int64_t delta) { return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED); } -static inline int64_t pmix_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_and_64(pmix_atomic_int64_t *addr, int64_t value) { return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED); } -static inline int64_t pmix_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_or_64(pmix_atomic_int64_t *addr, int64_t value) { return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED); } -static inline int64_t pmix_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_xor_64(pmix_atomic_int64_t *addr, int64_t value) { return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED); } -static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta) +static inline int64_t pmix_atomic_fetch_sub_64(pmix_atomic_int64_t *addr, int64_t delta) { return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED); } @@ -191,7 +191,7 @@ static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t d #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 -static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128_t *addr, +static inline bool pmix_atomic_compare_exchange_strong_128 (pmix_atomic_int128_t *addr, pmix_int128_t *oldval, pmix_int128_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, @@ -204,7 +204,7 @@ static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128 /* __atomic version is not lock-free so use legacy __sync version */ -static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128_t *addr, +static inline bool pmix_atomic_compare_exchange_strong_128 (pmix_atomic_pmix_int128_t *addr, pmix_int128_t *oldval, pmix_int128_t newval) { pmix_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/ia32/atomic.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/ia32/atomic.h index 4e8a6d81455..0eace3b5a27 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/ia32/atomic.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/ia32/atomic.h @@ -13,9 +13,9 @@ * Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -85,7 +85,7 @@ static inline void pmix_atomic_isync(void) *********************************************************************/ #if PMIX_GCC_INLINE_ASSEMBLY -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { unsigned char ret; __asm__ __volatile__ ( @@ -107,15 +107,15 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add #define PMIX_HAVE_ATOMIC_SWAP_32 1 -static inline int32_t pmix_atomic_swap_32( volatile int32_t *addr, - int32_t newval) +static inline int32_t pmix_atomic_swap_32( pmix_atomic_int32_t *addr, + int32_t newval) { int32_t oldval; __asm__ __volatile__("xchg %1, %0" : - "=r" (oldval), "=m" (*addr) : - "0" (newval), "m" (*addr) : - "memory"); + "=r" (oldval), "=m" (*addr) : + "0" (newval), "m" (*addr) : + "memory"); return oldval; } @@ -131,7 +131,7 @@ static inline int32_t pmix_atomic_swap_32( volatile int32_t *addr, * * Atomically adds @i to @v. */ -static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int i) +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t* v, int i) { int ret = i; __asm__ __volatile__( @@ -151,7 +151,7 @@ static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int i) * * Atomically subtracts @i from @v. */ -static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t* v, int i) +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t* v, int i) { int ret = -i; __asm__ __volatile__( diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/powerpc/atomic.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/powerpc/atomic.h index e5ce2663082..17134e11c16 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/powerpc/atomic.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/powerpc/atomic.h @@ -13,7 +13,7 @@ * Copyright (c) 2010-2017 IBM Corporation. All rights reserved. * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -99,27 +99,7 @@ void pmix_atomic_isync(void) ISYNC(); } -#elif PMIX_XLC_INLINE_ASSEMBLY /* end PMIX_GCC_INLINE_ASSEMBLY */ - -/* Yeah, I don't know who thought this was a reasonable syntax for - * inline assembly. Do these because they are used so often and they - * are fairly simple (aka: there is a tech pub on IBM's web site - * containing the right hex for the instructions). - */ - -#undef PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER -#define PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER 0 - -#pragma mc_func pmix_atomic_mb { "7c0004ac" } /* sync */ -#pragma reg_killed_by pmix_atomic_mb /* none */ - -#pragma mc_func pmix_atomic_rmb { "7c2004ac" } /* lwsync */ -#pragma reg_killed_by pmix_atomic_rmb /* none */ - -#pragma mc_func pmix_atomic_wmb { "7c2004ac" } /* lwsync */ -#pragma reg_killed_by pmix_atomic_wmb /* none */ - -#endif +#endif /* end PMIX_GCC_INLINE_ASSEMBLY */ /********************************************************************** * @@ -145,7 +125,7 @@ void pmix_atomic_isync(void) #define PMIX_ASM_VALUE64(x) x #endif -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev; bool ret; @@ -171,7 +151,7 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add * load the arguments to/from the stack. This sequence may cause the ll reservation to be cancelled. */ #define pmix_atomic_ll_32(addr, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + pmix_atomic_int32_t *_addr = (addr); \ int32_t _ret; \ __asm__ __volatile__ ("lwarx %0, 0, %1 \n\t" \ : "=&r" (_ret) \ @@ -182,7 +162,7 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add #define pmix_atomic_sc_32(addr, value, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + pmix_atomic_int32_t *_addr = (addr); \ int32_t _ret, _foo, _newval = (int32_t) value; \ \ __asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t" \ @@ -201,7 +181,7 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { bool rc; @@ -212,13 +192,13 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { pmix_atomic_wmb(); return pmix_atomic_compare_exchange_strong_32 (addr, oldval, newval); } -static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, int32_t newval) +static inline int32_t pmix_atomic_swap_32(pmix_atomic_int32_t *addr, int32_t newval) { int32_t ret; @@ -240,7 +220,7 @@ static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, int32_t newval #if PMIX_GCC_INLINE_ASSEMBLY #define PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr) \ -static inline int64_t pmix_atomic_fetch_ ## type ## _64(volatile int64_t* v, int64_t val) \ +static inline int64_t pmix_atomic_fetch_ ## type ## _64(pmix_atomic_int64_t* v, int64_t val) \ { \ int64_t t, old; \ \ @@ -262,7 +242,7 @@ PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or) PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor) PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf) -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; bool ret; @@ -285,7 +265,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add #define pmix_atomic_ll_64(addr, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + pmix_atomic_int64_t *_addr = (addr); \ int64_t _ret; \ __asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \ : "=&r" (_ret) \ @@ -296,8 +276,8 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add #define pmix_atomic_sc_64(addr, value, ret) \ do { \ - volatile int64_t *_addr = (addr); \ - int64_t _foo, _newval = (int64_t) value; \ + pmix_atomic_int64_t *_addr = (addr); \ + int64_t _newval = (int64_t) value; \ int32_t _ret; \ \ __asm__ __volatile__ (" stdcx. %2, 0, %1 \n\t" \ @@ -311,7 +291,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add ret = _ret; \ } while (0) -static inline int64_t pmix_atomic_swap_64(volatile int64_t *addr, int64_t newval) +static inline int64_t pmix_atomic_swap_64(pmix_atomic_int64_t *addr, int64_t newval) { int64_t ret; @@ -336,7 +316,7 @@ static inline int64_t pmix_atomic_swap_64(volatile int64_t *addr, int64_t newval #if PMIX_GCC_INLINE_ASSEMBLY -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int ret; @@ -383,7 +363,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { bool rc; @@ -394,7 +374,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { pmix_atomic_wmb(); return pmix_atomic_compare_exchange_strong_64 (addr, oldval, newval); @@ -402,7 +382,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \ -static inline int32_t pmix_atomic_fetch_ ## type ## _32(volatile int32_t* v, int val) \ +static inline int32_t pmix_atomic_fetch_ ## type ## _32(pmix_atomic_int32_t* v, int val) \ { \ int32_t t, old; \ \ diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/sparcv9/atomic.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/sparcv9/atomic.h index 45afeea54d5..82953a3fcd2 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/sparcv9/atomic.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/sparcv9/atomic.h @@ -13,9 +13,9 @@ * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -32,7 +32,7 @@ #define ASI_P "0x80" -#define MEMBAR(type) __asm__ __volatile__ ("membar " type : : : "memory") +#define MEPMIXMBAR(type) __asm__ __volatile__ ("membar " type : : : "memory") /********************************************************************** @@ -56,19 +56,19 @@ static inline void pmix_atomic_mb(void) { - MEMBAR("#LoadLoad | #LoadStore | #StoreStore | #StoreLoad"); + MEPMIXMBAR("#LoadLoad | #LoadStore | #StoreStore | #StoreLoad"); } static inline void pmix_atomic_rmb(void) { - MEMBAR("#LoadLoad"); + MEPMIXMBAR("#LoadLoad"); } static inline void pmix_atomic_wmb(void) { - MEMBAR("#StoreStore"); + MEPMIXMBAR("#StoreStore"); } static inline void pmix_atomic_isync(void) @@ -86,7 +86,7 @@ static inline void pmix_atomic_isync(void) *********************************************************************/ #if PMIX_GCC_INLINE_ASSEMBLY -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -108,7 +108,7 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add } -static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { bool rc; @@ -119,7 +119,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { pmix_atomic_wmb(); return pmix_atomic_compare_exchange_strong_32 (addr, oldval, newval); @@ -128,7 +128,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t #if PMIX_ASSEMBLY_ARCH == PMIX_SPARCV9_64 -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -150,7 +150,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add #else /* PMIX_ASSEMBLY_ARCH == PMIX_SPARCV9_64 */ -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -180,7 +180,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add #endif /* PMIX_ASSEMBLY_ARCH == PMIX_SPARCV9_64 */ -static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { bool rc; @@ -191,7 +191,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { pmix_atomic_wmb(); return pmix_atomic_compare_exchange_strong_64 (addr, oldval, newval); diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/sync_builtin/atomic.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/sync_builtin/atomic.h index 2c91411e194..981530df360 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/sync_builtin/atomic.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/sync_builtin/atomic.h @@ -11,11 +11,11 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -58,7 +58,7 @@ static inline void pmix_atomic_wmb(void) #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); bool ret = prev == *oldval; @@ -72,31 +72,31 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add #define PMIX_HAVE_ATOMIC_MATH_32 1 #define PMIX_HAVE_ATOMIC_ADD_32 1 -static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta) +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t *addr, int32_t delta) { return __sync_fetch_and_add(addr, delta); } #define PMIX_HAVE_ATOMIC_AND_32 1 -static inline int32_t pmix_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_and_32(pmix_atomic_int32_t *addr, int32_t value) { return __sync_fetch_and_and(addr, value); } #define PMIX_HAVE_ATOMIC_OR_32 1 -static inline int32_t pmix_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_or_32(pmix_atomic_int32_t *addr, int32_t value) { return __sync_fetch_and_or(addr, value); } #define PMIX_HAVE_ATOMIC_XOR_32 1 -static inline int32_t pmix_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_xor_32(pmix_atomic_int32_t *addr, int32_t value) { return __sync_fetch_and_xor(addr, value); } #define PMIX_HAVE_ATOMIC_SUB_32 1 -static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta) +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t *addr, int32_t delta) { return __sync_fetch_and_sub(addr, delta); } @@ -105,7 +105,7 @@ static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t *addr, int32_t d #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); bool ret = prev == *oldval; @@ -118,31 +118,31 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add #define PMIX_HAVE_ATOMIC_MATH_64 1 #define PMIX_HAVE_ATOMIC_ADD_64 1 -static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta) +static inline int64_t pmix_atomic_fetch_add_64(pmix_atomic_int64_t *addr, int64_t delta) { return __sync_fetch_and_add(addr, delta); } #define PMIX_HAVE_ATOMIC_AND_64 1 -static inline int64_t pmix_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_and_64(pmix_atomic_int64_t *addr, int64_t value) { return __sync_fetch_and_and(addr, value); } #define PMIX_HAVE_ATOMIC_OR_64 1 -static inline int64_t pmix_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_or_64(pmix_atomic_int64_t *addr, int64_t value) { return __sync_fetch_and_or(addr, value); } #define PMIX_HAVE_ATOMIC_XOR_64 1 -static inline int64_t pmix_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_xor_64(pmix_atomic_int64_t *addr, int64_t value) { return __sync_fetch_and_xor(addr, value); } #define PMIX_HAVE_ATOMIC_SUB_64 1 -static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta) +static inline int64_t pmix_atomic_fetch_sub_64(pmix_atomic_int64_t *addr, int64_t delta) { return __sync_fetch_and_sub(addr, delta); } @@ -150,7 +150,7 @@ static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t d #endif #if PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128 -static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128_t *addr, +static inline bool pmix_atomic_compare_exchange_strong_128 (pmix_atomic_int128_t *addr, pmix_int128_t *oldval, pmix_int128_t newval) { pmix_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/timer.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/timer.h index b2a562e4218..28008fc88a6 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/timer.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/timer.h @@ -13,7 +13,7 @@ * Copyright (c) 2016 Broadcom Limited. All rights reserved. * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/x86_64/atomic.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/x86_64/atomic.h index f2144eb0467..2b523622ce2 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/x86_64/atomic.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/x86_64/atomic.h @@ -11,11 +11,11 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. - * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -83,7 +83,7 @@ static inline void pmix_atomic_isync(void) *********************************************************************/ #if PMIX_GCC_INLINE_ASSEMBLY -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { unsigned char ret; __asm__ __volatile__ ( @@ -103,13 +103,13 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add #if PMIX_GCC_INLINE_ASSEMBLY -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { unsigned char ret; __asm__ __volatile__ ( SMPLOCK "cmpxchgq %3,%2 \n\t" "sete %0 \n\t" - : "=qm" (ret), "+a" (*oldval), "+m" (*((volatile long*)addr)) + : "=qm" (ret), "+a" (*oldval), "+m" (*((pmix_atomic_long_t *)addr)) : "q"(newval) : "memory", "cc" ); @@ -124,7 +124,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add #if PMIX_GCC_INLINE_ASSEMBLY && PMIX_HAVE_CMPXCHG16B && HAVE_PMIX_INT128_T -static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128_t *addr, pmix_int128_t *oldval, pmix_int128_t newval) +static inline bool pmix_atomic_compare_exchange_strong_128 (pmix_atomic_int128_t *addr, pmix_int128_t *oldval, pmix_int128_t newval) { unsigned char ret; @@ -151,15 +151,15 @@ static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128 #define PMIX_HAVE_ATOMIC_SWAP_64 1 -static inline int32_t pmix_atomic_swap_32( volatile int32_t *addr, - int32_t newval) +static inline int32_t pmix_atomic_swap_32( pmix_atomic_int32_t *addr, + int32_t newval) { int32_t oldval; __asm__ __volatile__("xchg %1, %0" : - "=r" (oldval), "+m" (*addr) : - "0" (newval) : - "memory"); + "=r" (oldval), "+m" (*addr) : + "0" (newval) : + "memory"); return oldval; } @@ -167,15 +167,15 @@ static inline int32_t pmix_atomic_swap_32( volatile int32_t *addr, #if PMIX_GCC_INLINE_ASSEMBLY -static inline int64_t pmix_atomic_swap_64( volatile int64_t *addr, +static inline int64_t pmix_atomic_swap_64( pmix_atomic_int64_t *addr, int64_t newval) { int64_t oldval; __asm__ __volatile__("xchgq %1, %0" : - "=r" (oldval), "+m" (*addr) : - "0" (newval) : - "memory"); + "=r" (oldval), "+m" (*addr) : + "0" (newval) : + "memory"); return oldval; } @@ -197,7 +197,7 @@ static inline int64_t pmix_atomic_swap_64( volatile int64_t *addr, * * Atomically adds @i to @v. */ -static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int i) +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t* v, int i) { int ret = i; __asm__ __volatile__( @@ -218,7 +218,7 @@ static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int i) * * Atomically adds @i to @v. */ -static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t* v, int64_t i) +static inline int64_t pmix_atomic_fetch_add_64(pmix_atomic_int64_t* v, int64_t i) { int64_t ret = i; __asm__ __volatile__( @@ -239,7 +239,7 @@ static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t* v, int64_t i) * * Atomically subtracts @i from @v. */ -static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t* v, int i) +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t* v, int i) { int ret = -i; __asm__ __volatile__( @@ -260,7 +260,7 @@ static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t* v, int i) * * Atomically subtracts @i from @v. */ -static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t* v, int64_t i) +static inline int64_t pmix_atomic_fetch_sub_64(pmix_atomic_int64_t* v, int64_t i) { int64_t ret = -i; __asm__ __volatile__( diff --git a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/x86_64/timer.h b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/x86_64/timer.h index 6b25f9aafee..907a4a1c269 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/x86_64/timer.h +++ b/opal/mca/pmix/pmix2x/pmix/src/atomics/sys/x86_64/timer.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2016 Los Alamos National Security, LLC. ALl rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/src/class/Makefile.include b/opal/mca/pmix/pmix2x/pmix/src/class/Makefile.include index 904995173d3..295d02718a0 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/class/Makefile.include +++ b/opal/mca/pmix/pmix2x/pmix/src/class/Makefile.include @@ -10,7 +10,7 @@ # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. # $COPYRIGHT$ # @@ -24,6 +24,7 @@ # Source code files headers += \ + class/pmix_bitmap.h \ class/pmix_object.h \ class/pmix_list.h \ class/pmix_pointer_array.h \ @@ -33,6 +34,7 @@ headers += \ class/pmix_value_array.h sources += \ + class/pmix_bitmap.c \ class/pmix_object.c \ class/pmix_list.c \ class/pmix_pointer_array.c \ diff --git a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_bitmap.c b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_bitmap.c new file mode 100644 index 00000000000..1dc7ab390a5 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_bitmap.c @@ -0,0 +1,412 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010-2012 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#include + +#include "pmix_common.h" +#include "src/class/pmix_bitmap.h" + +/* The number of bits in the underlying type of the bitmap field + * in the pmix_bitmap_t struct + */ +#define SIZE_OF_BASE_TYPE 64 + +static void pmix_bitmap_construct(pmix_bitmap_t *bm); +static void pmix_bitmap_destruct(pmix_bitmap_t *bm); + +PMIX_CLASS_INSTANCE(pmix_bitmap_t, pmix_object_t, + pmix_bitmap_construct, pmix_bitmap_destruct); + + +static void +pmix_bitmap_construct(pmix_bitmap_t *bm) +{ + bm->bitmap = NULL; + bm->array_size = 0; + bm->max_size = INT_MAX; +} + + +static void +pmix_bitmap_destruct(pmix_bitmap_t *bm) +{ + if (NULL != bm->bitmap) { + free(bm->bitmap); + bm->bitmap = NULL; + } +} + + +int pmix_bitmap_set_max_size (pmix_bitmap_t *bm, int max_size) +{ + if (NULL == bm) { + return PMIX_ERR_BAD_PARAM; + } + + /* + * Only if the caller wants to set the maximum size, + * we set it (in numbers of bits!), otherwise it is + * set to INT_MAX in the constructor. + */ + bm->max_size = (int)(((size_t)max_size + SIZE_OF_BASE_TYPE - 1) / SIZE_OF_BASE_TYPE); + + return PMIX_SUCCESS; +} + + +int +pmix_bitmap_init(pmix_bitmap_t *bm, int size) +{ + /* + * Only if the caller set the maximum size before initializing, + * we test here (in numbers of bits!) + * By default, the max size is INT_MAX, set in the constructor. + */ + if ((size <= 0) || (NULL == bm) || (size > bm->max_size)) { + return PMIX_ERR_BAD_PARAM; + } + + bm->array_size = (int)(((size_t)size + SIZE_OF_BASE_TYPE - 1) / SIZE_OF_BASE_TYPE); + if( NULL != bm->bitmap ) { + free(bm->bitmap); + if(bm->max_size < bm->array_size) + bm->max_size = bm->array_size; + } + bm->bitmap = (uint64_t*) malloc(bm->array_size * sizeof(uint64_t)); + if (NULL == bm->bitmap) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + + pmix_bitmap_clear_all_bits(bm); + return PMIX_SUCCESS; +} + + +int +pmix_bitmap_set_bit(pmix_bitmap_t *bm, int bit) +{ + int index, offset, new_size; + + if ((bit < 0) || (NULL == bm) || (bit > bm->max_size)) { + return PMIX_ERR_BAD_PARAM; + } + + index = bit / SIZE_OF_BASE_TYPE; + offset = bit % SIZE_OF_BASE_TYPE; + + if (index >= bm->array_size) { + + /* We need to allocate more space for the bitmap, since we are + out of range. We don't throw any error here, because this is + valid and we simply expand the bitmap */ + + new_size = index + 1; + if( new_size > bm->max_size ) + new_size = bm->max_size; + + /* New size is just a multiple of the original size to fit in + the index. */ + bm->bitmap = (uint64_t*)realloc(bm->bitmap, new_size*sizeof(uint64_t)); + if (NULL == bm->bitmap) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + + /* zero out the new elements */ + memset(&bm->bitmap[bm->array_size], 0, (new_size - bm->array_size) * sizeof(uint64_t)); + + /* Update the array_size */ + bm->array_size = new_size; + } + + /* Now set the bit */ + bm->bitmap[index] |= (1UL << offset); + + return PMIX_SUCCESS; +} + + +int +pmix_bitmap_clear_bit(pmix_bitmap_t *bm, int bit) +{ + int index, offset; + + if ((bit < 0) || NULL == bm || (bit >= (bm->array_size * SIZE_OF_BASE_TYPE))) { + return PMIX_ERR_BAD_PARAM; + } + + index = bit / SIZE_OF_BASE_TYPE; + offset = bit % SIZE_OF_BASE_TYPE; + + bm->bitmap[index] &= ~(1UL << offset); + return PMIX_SUCCESS; +} + + +bool +pmix_bitmap_is_set_bit(pmix_bitmap_t *bm, int bit) +{ + int index, offset; + + if ((bit < 0) || NULL == bm || (bit >= (bm->array_size * SIZE_OF_BASE_TYPE))) { + return false; + } + + index = bit / SIZE_OF_BASE_TYPE; + offset = bit % SIZE_OF_BASE_TYPE; + + if (0 != (bm->bitmap[index] & (1UL << offset))) { + return true; + } + + return false; +} + + +int +pmix_bitmap_clear_all_bits(pmix_bitmap_t *bm) +{ + if (NULL == bm) { + return PMIX_ERR_BAD_PARAM; + } + + memset(bm->bitmap, 0, bm->array_size * sizeof(uint64_t)); + return PMIX_SUCCESS; +} + + +int +pmix_bitmap_set_all_bits(pmix_bitmap_t *bm) +{ + if (NULL == bm) { + return PMIX_ERR_BAD_PARAM; + } + + memset(bm->bitmap, 0xff, bm->array_size * sizeof(uint64_t)); + + return PMIX_SUCCESS; +} + + +int +pmix_bitmap_find_and_set_first_unset_bit(pmix_bitmap_t *bm, int *position) +{ + int i = 0; + uint64_t temp, all_ones = 0xffffffffffffffffUL; + + if (NULL == bm) { + return PMIX_ERR_BAD_PARAM; + } + + /* Neglect all which don't have an unset bit */ + *position = 0; + while((i < bm->array_size) && (bm->bitmap[i] == all_ones)) { + ++i; + } + + if (i == bm->array_size) { + /* increase the bitmap size then */ + *position = bm->array_size * SIZE_OF_BASE_TYPE; + return pmix_bitmap_set_bit(bm, *position); + } + + /* This one has an unset bit, find its bit number */ + + temp = bm->bitmap[i]; + bm->bitmap[i] |= (bm->bitmap[i] + 1); /* Set the first zero bit */ + temp ^= bm->bitmap[i]; /* Compute the change: the first unset bit in the original number */ + while( !(temp & 0x1) ) { + ++(*position); + temp >>= 1; + } + + (*position) += i * SIZE_OF_BASE_TYPE; + return PMIX_SUCCESS; +} + +int pmix_bitmap_bitwise_and_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right) +{ + int i; + + /* + * Sanity check + */ + if( NULL == dest || NULL == right ) { + return PMIX_ERR_BAD_PARAM; + } + if( dest->array_size != right->array_size ) { + return PMIX_ERR_BAD_PARAM; + } + + /* + * Bitwise AND + */ + for(i = 0; i < dest->array_size; ++i) { + dest->bitmap[i] &= right->bitmap[i]; + } + + return PMIX_SUCCESS; +} + +int pmix_bitmap_bitwise_or_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right) +{ + int i; + + /* + * Sanity check + */ + if( NULL == dest || NULL == right ) { + return PMIX_ERR_BAD_PARAM; + } + if( dest->array_size != right->array_size ) { + return PMIX_ERR_BAD_PARAM; + } + + /* + * Bitwise OR + */ + for(i = 0; i < dest->array_size; ++i) { + dest->bitmap[i] |= right->bitmap[i]; + } + + return PMIX_SUCCESS; +} + +int pmix_bitmap_bitwise_xor_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right) +{ + int i; + + /* + * Sanity check + */ + if( NULL == dest || NULL == right ) { + return PMIX_ERR_BAD_PARAM; + } + if( dest->array_size != right->array_size ) { + return PMIX_ERR_BAD_PARAM; + } + + /* + * Bitwise XOR + */ + for(i = 0; i < dest->array_size; ++i) { + dest->bitmap[i] ^= right->bitmap[i]; + } + + return PMIX_SUCCESS; +} + +bool pmix_bitmap_are_different(pmix_bitmap_t *left, pmix_bitmap_t *right) +{ + int i; + + /* + * Sanity check + */ + if( NULL == left || NULL == right ) { + return PMIX_ERR_BAD_PARAM; + } + + if( pmix_bitmap_size(left) != pmix_bitmap_size(right) ) { + return true; + } + + /* + * Direct comparison + */ + for(i = 0; i < left->array_size; ++i) { + if( left->bitmap[i] != right->bitmap[i] ) { + return true; + } + } + + return false; +} + +char * pmix_bitmap_get_string(pmix_bitmap_t *bitmap) +{ + int i; + char *bitmap_str = NULL; + + if( NULL == bitmap) { + return NULL; + } + + bitmap_str = malloc(bitmap->array_size * SIZE_OF_BASE_TYPE + 1); + if (NULL == bitmap_str) { + return NULL; + } + bitmap_str[bitmap->array_size * SIZE_OF_BASE_TYPE] = '\0'; + + for( i = 0; i < (bitmap->array_size * SIZE_OF_BASE_TYPE); ++i) { + if( pmix_bitmap_is_set_bit(bitmap, i) ) { + bitmap_str[i] = 'X'; + } else { + bitmap_str[i] = '_'; + } + } + + return bitmap_str; +} + +int pmix_bitmap_num_unset_bits(pmix_bitmap_t *bm, int len) +{ + return (len - pmix_bitmap_num_set_bits(bm, len)); +} + +int pmix_bitmap_num_set_bits(pmix_bitmap_t *bm, int len) +{ + int i, cnt = 0; + uint64_t val; + +#if PMIX_ENABLE_DEBUG + if ((len < 0) || NULL == bm || (len >= (bm->array_size * SIZE_OF_BASE_TYPE))) { + return 0; + } +#endif + + for(i = 0; i < len; ++i) { + if( 0 == (val = bm->bitmap[i]) ) continue; + /* Peter Wegner in CACM 3 (1960), 322. This method goes through as many + * iterations as there are set bits. */ + for( ; val; cnt++ ) { + val &= val - 1; /* clear the least significant bit set */ + } + } + + return cnt; +} + +bool pmix_bitmap_is_clear(pmix_bitmap_t *bm) +{ + int i; + + for (i = 0; i < bm->array_size; ++i) { + if (0 != bm->bitmap[i]) { + return false; + } + } + return true; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_bitmap.h b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_bitmap.h new file mode 100644 index 00000000000..35f257e7d38 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_bitmap.h @@ -0,0 +1,259 @@ +/* -*- Mode: C; c-basic-offset:4 ; -*- */ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2014 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010-2012 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +/** @file + * + * A bitmap implementation. The bits start off with 0, so this bitmap + * has bits numbered as bit 0, bit 1, bit 2 and so on. This bitmap + * has auto-expansion capabilities, that is once the size is set + * during init, it can be automatically expanded by setting the bit + * beyond the current size. But note, this is allowed just when the + * bit is set -- so the valid functions are set_bit and + * find_and_set_bit. Other functions like clear, if passed a bit + * outside the initialized range will result in an error. + * + * To allow these bitmaps to track fortran handles (which MPI defines + * to be Fortran INTEGER), we offer a pmix_bitmap_set_max_size, so that + * the upper layer can ask to never have more than + * OMPI_FORTRAN_HANDLE_MAX, which is min(INT_MAX, fortran INTEGER max). + */ + +#ifndef PMIX_BITMAP_H +#define PMIX_BITMAP_H + +#include + +#include + +#include "src/class/pmix_object.h" + +BEGIN_C_DECLS + +struct pmix_bitmap_t { + pmix_object_t super; /**< Subclass of pmix_object_t */ + uint64_t *bitmap; /**< The actual bitmap array of characters */ + int array_size; /**< The actual array size that maintains the bitmap */ + int max_size; /**< The maximum size that this bitmap may grow (optional) */ +}; + +typedef struct pmix_bitmap_t pmix_bitmap_t; + +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_bitmap_t); + +/** + * Set the maximum size of the bitmap. + * May be reset any time, but HAS TO BE SET BEFORE pmix_bitmap_init! + * + * @param bitmap The input bitmap (IN) + * @param max_size The maximum size of the bitmap in terms of bits (IN) + * @return PMIX error code or success + * + */ +PMIX_EXPORT int pmix_bitmap_set_max_size (pmix_bitmap_t *bm, int max_size); + + +/** + * Initializes the bitmap and sets its size. This must be called + * before the bitmap can be actually used + * + * @param bitmap The input bitmap (IN) + * @param size The initial size of the bitmap in terms of bits (IN) + * @return PMIX error code or success + * + */ +PMIX_EXPORT int pmix_bitmap_init (pmix_bitmap_t *bm, int size); + + +/** + * Set a bit of the bitmap. If the bit asked for is beyond the current + * size of the bitmap, then the bitmap is extended to accomodate the + * bit + * + * @param bitmap The input bitmap (IN) + * @param bit The bit which is to be set (IN) + * @return PMIX error code or success + * + */ +PMIX_EXPORT int pmix_bitmap_set_bit(pmix_bitmap_t *bm, int bit); + + +/** + * Clear/unset a bit of the bitmap. If the bit is beyond the current + * size of the bitmap, an error is returned + * + * @param bitmap The input bitmap (IN) + * @param bit The bit which is to be cleared (IN) + * @return PMIX error code if the bit is out of range, else success + * + */ +PMIX_EXPORT int pmix_bitmap_clear_bit(pmix_bitmap_t *bm, int bit); + + +/** + * Find out if a bit is set in the bitmap + * + * @param bitmap The input bitmap (IN) + * @param bit The bit which is to be checked (IN) + * @return true if the bit is set + * false if the bit is not set OR the index + * is outside the bounds of the provided + * bitmap + * + */ +PMIX_EXPORT bool pmix_bitmap_is_set_bit(pmix_bitmap_t *bm, int bit); + + +/** + * Find the first clear bit in the bitmap and set it + * + * @param bitmap The input bitmap (IN) + * @param position Position of the first clear bit (OUT) + + * @return err PMIX_SUCCESS on success + */ +PMIX_EXPORT int pmix_bitmap_find_and_set_first_unset_bit(pmix_bitmap_t *bm, + int *position); + + +/** + * Clear all bits in the bitmap + * + * @param bitmap The input bitmap (IN) + * @return PMIX error code if bm is NULL + * + */ +PMIX_EXPORT int pmix_bitmap_clear_all_bits(pmix_bitmap_t *bm); + + +/** + * Set all bits in the bitmap + * @param bitmap The input bitmap (IN) + * @return PMIX error code if bm is NULL + * + */ +PMIX_EXPORT int pmix_bitmap_set_all_bits(pmix_bitmap_t *bm); + + +/** + * Gives the current size (number of bits) in the bitmap. This is the + * legal (accessible) number of bits + * + * @param bitmap The input bitmap (IN) + * @return PMIX error code if bm is NULL + * + */ +static inline int pmix_bitmap_size(pmix_bitmap_t *bm) +{ + return (NULL == bm) ? 0 : (bm->array_size * ((int) (sizeof(*bm->bitmap) * 8))); +} + + +/** + * Copy a bitmap + * + * @param dest Pointer to the destination bitmap + * @param src Pointer to the source bitmap + * @ return PMIX error code if something goes wrong + */ +static inline void pmix_bitmap_copy(pmix_bitmap_t *dest, pmix_bitmap_t *src) +{ + if( dest->array_size < src->array_size ) { + if( NULL != dest->bitmap) free(dest->bitmap); + dest->max_size = src->max_size; + dest->bitmap = (uint64_t*)malloc(src->array_size*sizeof(uint64_t)); + } + memcpy(dest->bitmap, src->bitmap, src->array_size * sizeof(uint64_t)); + dest->array_size = src->array_size; +} + +/** + * Bitwise AND operator (inplace) + * + * @param dest Pointer to the bitmap that should be modified + * @param right Point to the other bitmap in the operation + * @return PMIX error code if the length of the two bitmaps is not equal or one is NULL. + */ +PMIX_EXPORT int pmix_bitmap_bitwise_and_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right); + +/** + * Bitwise OR operator (inplace) + * + * @param dest Pointer to the bitmap that should be modified + * @param right Point to the other bitmap in the operation + * @return PMIX error code if the length of the two bitmaps is not equal or one is NULL. + */ +PMIX_EXPORT int pmix_bitmap_bitwise_or_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right); + +/** + * Bitwise XOR operator (inplace) + * + * @param dest Pointer to the bitmap that should be modified + * @param right Point to the other bitmap in the operation + * @return PMIX error code if the length of the two bitmaps is not equal or one is NULL. + */ +PMIX_EXPORT int pmix_bitmap_bitwise_xor_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right); + +/** + * If the bitmaps are different + * + * @param left Pointer to a bitmap + * @param right Pointer to another bitmap + * @return true if different, false if the same + */ +PMIX_EXPORT bool pmix_bitmap_are_different(pmix_bitmap_t *left, pmix_bitmap_t *right); + +/** + * Get a string representation of the bitmap. + * Useful for debugging. + * + * @param bitmap Point to the bitmap to represent + * @return Pointer to the string (caller must free if not NULL) + */ +PMIX_EXPORT char * pmix_bitmap_get_string(pmix_bitmap_t *bitmap); + +/** + * Return the number of 'unset' bits, upto the specified length + * + * @param bitmap Pointer to the bitmap + * @param len Number of bits to check + * @return Integer + */ +PMIX_EXPORT int pmix_bitmap_num_unset_bits(pmix_bitmap_t *bm, int len); + +/** + * Return the number of 'set' bits, upto the specified length + * + * @param bitmap Pointer to the bitmap + * @param len Number of bits to check + * @return Integer + */ +PMIX_EXPORT int pmix_bitmap_num_set_bits(pmix_bitmap_t *bm, int len); + +/** + * Check a bitmap to see if any bit is set + */ +PMIX_EXPORT bool pmix_bitmap_is_clear(pmix_bitmap_t *bm); + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.c b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.c index af1a55968fe..aec846774b8 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.c +++ b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.c @@ -2,7 +2,7 @@ /* * Copyright (c) 2012-2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -44,11 +44,10 @@ static void local_eviction_callback(int fd, short flags, void *arg) } -int pmix_hotel_init(pmix_hotel_t *h, int num_rooms, - pmix_event_base_t *evbase, - uint32_t eviction_timeout, - int eviction_event_priority, - pmix_hotel_eviction_callback_fn_t evict_callback_fn) +pmix_status_t pmix_hotel_init(pmix_hotel_t *h, int num_rooms, + pmix_event_base_t *evbase, + uint32_t eviction_timeout, + pmix_hotel_eviction_callback_fn_t evict_callback_fn) { int i; diff --git a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.h b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.h index 8eb40646108..883a2c5c6ce 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.h +++ b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_hotel.h @@ -2,7 +2,7 @@ /* * Copyright (c) 2012-2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -142,7 +142,6 @@ PMIX_CLASS_DECLARATION(pmix_hotel_t); * @param evbase Pointer to event base used for eviction timeout * @param eviction_timeout Max length of a stay at the hotel before * the eviction callback is invoked (in microseconds) - * @param eviction_event_priority Event lib priority for the eviction timeout * @param evict_callback_fn Callback function invoked if an occupant * does not check out before the eviction_timeout. * @@ -158,11 +157,10 @@ PMIX_CLASS_DECLARATION(pmix_hotel_t); * @return PMIX_SUCCESS if all initializations were succesful. Otherwise, * the error indicate what went wrong in the function. */ -PMIX_EXPORT int pmix_hotel_init(pmix_hotel_t *hotel, int num_rooms, - pmix_event_base_t *evbase, - uint32_t eviction_timeout, - int eviction_event_priority, - pmix_hotel_eviction_callback_fn_t evict_callback_fn); +PMIX_EXPORT pmix_status_t pmix_hotel_init(pmix_hotel_t *hotel, int num_rooms, + pmix_event_base_t *evbase, + uint32_t eviction_timeout, + pmix_hotel_eviction_callback_fn_t evict_callback_fn); /** * Check in an occupant to the hotel. @@ -184,14 +182,15 @@ PMIX_EXPORT int pmix_hotel_init(pmix_hotel_t *hotel, int num_rooms, * @return PMIX_ERR_TEMP_OUT_OF_RESOURCE is the hotel is full. Try * again later. */ -static inline int pmix_hotel_checkin(pmix_hotel_t *hotel, - void *occupant, - int *room_num) +static inline pmix_status_t pmix_hotel_checkin(pmix_hotel_t *hotel, + void *occupant, + int *room_num) { pmix_hotel_room_t *room; /* Do we have any rooms available? */ if (PMIX_UNLIKELY(hotel->last_unoccupied_room < 0)) { + *room_num = -1; return PMIX_ERR_OUT_OF_RESOURCE; } @@ -214,8 +213,8 @@ static inline int pmix_hotel_checkin(pmix_hotel_t *hotel, * caller *knows* that there is a room available. */ static inline void pmix_hotel_checkin_with_res(pmix_hotel_t *hotel, - void *occupant, - int *room_num) + void *occupant, + int *room_num) { pmix_hotel_room_t *room; @@ -249,6 +248,10 @@ static inline void pmix_hotel_checkout(pmix_hotel_t *hotel, int room_num) /* Bozo check */ assert(room_num < hotel->num_rooms); + if (0 > room_num) { + /* occupant wasn't checked in */ + return; + } /* If there's an occupant in the room, check them out */ room = &(hotel->rooms[room_num]); @@ -287,6 +290,11 @@ static inline void pmix_hotel_checkout_and_return_occupant(pmix_hotel_t *hotel, /* Bozo check */ assert(room_num < hotel->num_rooms); + if (0 > room_num) { + /* occupant wasn't checked in */ + *occupant = NULL; + return; + } /* If there's an occupant in the room, check them out */ room = &(hotel->rooms[room_num]); @@ -341,6 +349,10 @@ static inline void pmix_hotel_knock(pmix_hotel_t *hotel, int room_num, void **oc assert(room_num < hotel->num_rooms); *occupant = NULL; + if (0 > room_num) { + /* occupant wasn't checked in */ + return; + } /* If there's an occupant in the room, have them come to the door */ room = &(hotel->rooms[room_num]); diff --git a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_list.h b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_list.h index df3f6a2280d..d9eb890f2ab 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_list.h +++ b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_list.h @@ -13,7 +13,7 @@ * Copyright (c) 2007 Voltaire All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -109,7 +109,7 @@ struct pmix_list_item_t #if PMIX_ENABLE_DEBUG /** Atomic reference count for debugging */ - volatile int32_t pmix_list_item_refcount; + pmix_atomic_int32_t pmix_list_item_refcount; /** The list this item belong to */ volatile struct pmix_list_t* pmix_list_item_belong_to; #endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_object.h b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_object.h index 8ad7f90f105..331325c3da5 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/class/pmix_object.h +++ b/opal/mca/pmix/pmix2x/pmix/src/class/pmix_object.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -192,7 +192,7 @@ struct pmix_object_t { uint64_t obj_magic_id; #endif pmix_class_t *obj_class; /**< class descriptor */ - volatile int32_t obj_reference_count; /**< reference count */ + pmix_atomic_int32_t obj_reference_count; /**< reference count */ #if PMIX_ENABLE_DEBUG const char* cls_init_file_name; /**< In debug mode store the file where the object get contructed */ int cls_init_lineno; /**< In debug mode store the line number where the object get contructed */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmi1.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmi1.c index 1c8517ea0e7..ac7ce80d5df 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmi1.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmi1.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -75,7 +75,7 @@ PMIX_EXPORT int PMI_Init(int *spawned) *spawned = 0; } pmi_singleton = true; - (void)strncpy(myproc.nspace, "1234", PMIX_MAX_NSLEN); + pmix_strncpy(myproc.nspace, "1234", PMIX_MAX_NSLEN); myproc.rank = 0; pmi_init = 1; return PMI_SUCCESS; @@ -242,7 +242,7 @@ PMIX_EXPORT int PMI_KVS_Get( const char kvsname[], const char key[], char value[ proc.rank = PMIX_RANK_WILDCARD; if (PMIX_SUCCESS == PMIx_Get(&proc, PMIX_ANL_MAP, NULL, 0, &val) && (NULL != val) && (PMIX_STRING == val->type)) { - strncpy(value, val->data.string, length); + pmix_strncpy(value, val->data.string, length-1); PMIX_VALUE_FREE(val, 1); return PMI_SUCCESS; } else { @@ -259,7 +259,7 @@ PMIX_EXPORT int PMI_KVS_Get( const char kvsname[], const char key[], char value[ /* retrieve the data from PMIx - since we don't have a rank, * we indicate that by passing the UNDEF value */ - (void)strncpy(proc.nspace, kvsname, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, kvsname, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_UNDEF; rc = PMIx_Get(&proc, key, NULL, 0, &val); @@ -267,7 +267,7 @@ PMIX_EXPORT int PMI_KVS_Get( const char kvsname[], const char key[], char value[ if (PMIX_STRING != val->type) { rc = PMIX_ERROR; } else if (NULL != val->data.string) { - (void)strncpy(value, val->data.string, length); + pmix_strncpy(value, val->data.string, length-1); } PMIX_VALUE_RELEASE(val); } @@ -445,7 +445,7 @@ PMIX_EXPORT int PMI_Publish_name(const char service_name[], const char port[]) } /* pass the service/port */ - (void) strncpy(info.key, service_name, PMIX_MAX_KEYLEN); + pmix_strncpy(info.key, service_name, PMIX_MAX_KEYLEN); info.value.type = PMIX_STRING; info.value.data.string = (char*) port; @@ -497,7 +497,7 @@ PMIX_EXPORT int PMI_Lookup_name(const char service_name[], char port[]) PMIX_PDATA_CONSTRUCT(&pdata); /* pass the service */ - (void) strncpy(pdata.key, service_name, PMIX_MAX_KEYLEN); + pmix_strncpy(pdata.key, service_name, PMIX_MAX_KEYLEN); /* PMI-1 doesn't want the nspace back */ if (PMIX_SUCCESS != (rc = PMIx_Lookup(&pdata, 1, NULL, 0))) { @@ -514,7 +514,7 @@ PMIX_EXPORT int PMI_Lookup_name(const char service_name[], char port[]) * potential we could overrun it. As this feature * isn't widely supported in PMI-1, try being * conservative */ - (void) strncpy(port, pdata.value.data.string, PMIX_MAX_KEYLEN); + pmix_strncpy(port, pdata.value.data.string, PMIX_MAX_KEYLEN); PMIX_PDATA_DESTRUCT(&pdata); return PMIX_SUCCESS; @@ -535,7 +535,7 @@ PMIX_EXPORT int PMI_Get_id(char id_str[], int length) return PMI_ERR_INVALID_LENGTH; } - (void) strncpy(id_str, myproc.nspace, length); + pmix_strncpy(id_str, myproc.nspace, length-1); return PMI_SUCCESS; } @@ -742,7 +742,7 @@ PMIX_EXPORT int PMI_Spawn_multiple(int count, apps[i].info = (pmix_info_t*)malloc(apps[i].ninfo * sizeof(pmix_info_t)); /* copy the info objects */ for (j = 0; j < apps[i].ninfo; j++) { - (void)strncpy(apps[i].info[j].key, info_keyval_vectors[i][j].key, PMIX_MAX_KEYLEN); + pmix_strncpy(apps[i].info[j].key, info_keyval_vectors[i][j].key, PMIX_MAX_KEYLEN); apps[i].info[j].value.type = PMIX_STRING; apps[i].info[j].value.data.string = strdup(info_keyval_vectors[i][j].val); } diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmi2.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmi2.c index 67818e9a35d..4df81d03bf4 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmi2.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmi2.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -79,7 +79,7 @@ PMIX_EXPORT int PMI2_Init(int *spawned, int *size, int *rank, int *appnum) *appnum = 0; } pmi2_singleton = true; - (void)strncpy(myproc.nspace, "1234", PMIX_MAX_NSLEN); + pmix_strncpy(myproc.nspace, "1234", PMIX_MAX_NSLEN); myproc.rank = 0; pmi2_init = 1; return PMI2_SUCCESS; @@ -227,7 +227,7 @@ PMIX_EXPORT int PMI2_Job_Spawn(int count, const char * cmds[], apps[i].info = (pmix_info_t*)malloc(apps[i].ninfo * sizeof(pmix_info_t)); /* copy the info objects */ for (j=0; j < apps[i].ninfo; j++) { - (void)strncpy(apps[i].info[j].key, info_keyval_vectors[i][j].key, PMIX_MAX_KEYLEN); + pmix_strncpy(apps[i].info[j].key, info_keyval_vectors[i][j].key, PMIX_MAX_KEYLEN); apps[i].info[j].value.type = PMIX_STRING; apps[i].info[j].value.data.string = strdup(info_keyval_vectors[i][j].val); } @@ -271,7 +271,7 @@ PMIX_EXPORT int PMI2_Job_GetId(char jobid[], int jobid_size) if (NULL == jobid) { return PMI2_ERR_INVALID_ARGS; } - (void)strncpy(jobid, myproc.nspace, jobid_size); + pmix_strncpy(jobid, myproc.nspace, jobid_size-1); return PMI2_SUCCESS; } @@ -339,7 +339,7 @@ PMIX_EXPORT int PMI2_Job_Connect(const char jobid[], PMI2_Connect_comm_t *conn) } memset(proc.nspace, 0, sizeof(proc.nspace)); - (void)strncpy(proc.nspace, (jobid ? jobid : proc.nspace), sizeof(proc.nspace)-1); + pmix_strncpy(proc.nspace, (jobid ? jobid : proc.nspace), PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; rc = PMIx_Connect(&proc, 1, NULL, 0); return convert_err(rc); @@ -357,7 +357,7 @@ PMIX_EXPORT int PMI2_Job_Disconnect(const char jobid[]) } memset(proc.nspace, 0, sizeof(proc.nspace)); - (void)strncpy(proc.nspace, (jobid ? jobid : proc.nspace), sizeof(proc.nspace)-1); + pmix_strncpy(proc.nspace, (jobid ? jobid : proc.nspace), PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; rc = PMIx_Disconnect(&proc, 1, NULL, 0); return convert_err(rc); @@ -455,7 +455,7 @@ PMIX_EXPORT int PMI2_KVS_Get(const char *jobid, int src_pmi_id, pmix_output_verbose(3, pmix_globals.debug_output, "PMI2_KVS_Get: key=%s jobid=%s src_pmi_id=%d", key, (jobid ? jobid : "null"), src_pmi_id); - (void)strncpy(proc.nspace, (jobid ? jobid : myproc.nspace), PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, (jobid ? jobid : myproc.nspace), PMIX_MAX_NSLEN); if (src_pmi_id == PMI2_ID_NULL) { /* the rank is UNDEF */ proc.rank = PMIX_RANK_UNDEF; @@ -468,7 +468,7 @@ PMIX_EXPORT int PMI2_KVS_Get(const char *jobid, int src_pmi_id, if (PMIX_STRING != val->type) { rc = PMIX_ERROR; } else if (NULL != val->data.string) { - (void)strncpy(value, val->data.string, maxvalue); + pmix_strncpy(value, val->data.string, maxvalue-1); *vallen = strlen(val->data.string); } PMIX_VALUE_RELEASE(val); @@ -511,7 +511,7 @@ PMIX_EXPORT int PMI2_Info_GetNodeAttr(const char name[], if (PMIX_STRING != val->type) { rc = PMIX_ERROR; } else if (NULL != val->data.string) { - (void)strncpy(value, val->data.string, valuelen); + pmix_strncpy(value, val->data.string, valuelen-1); *found = 1; } PMIX_VALUE_RELEASE(val); @@ -586,7 +586,7 @@ PMIX_EXPORT int PMI2_Info_GetJobAttr(const char name[], char value[], int valuel proc.rank = PMIX_RANK_WILDCARD; if (PMIX_SUCCESS == PMIx_Get(&proc, PMIX_ANL_MAP, NULL, 0, &val) && (NULL != val) && (PMIX_STRING == val->type)) { - strncpy(value, val->data.string, valuelen); + pmix_strncpy(value, val->data.string, valuelen); PMIX_VALUE_FREE(val, 1); *found = 1; return PMI2_SUCCESS; @@ -610,7 +610,7 @@ PMIX_EXPORT int PMI2_Info_GetJobAttr(const char name[], char value[], int valuel if (PMIX_STRING != val->type) { rc = PMIX_ERROR; } else if (NULL != val->data.string) { - (void)strncpy(value, val->data.string, valuelen); + pmix_strncpy(value, val->data.string, valuelen-1); *found = 1; } PMIX_VALUE_RELEASE(val); @@ -648,14 +648,14 @@ PMIX_EXPORT int PMI2_Nameserv_publish(const char service_name[], } /* pass the service/port */ - (void)strncpy(info[0].key, service_name, PMIX_MAX_KEYLEN); + pmix_strncpy(info[0].key, service_name, PMIX_MAX_KEYLEN); info[0].value.type = PMIX_STRING; info[0].value.data.string = (char*)port; nvals = 1; /* if provided, add any other value */ if (NULL != info_ptr) { - (void)strncpy(info[1].key, info_ptr->key, PMIX_MAX_KEYLEN); + pmix_strncpy(info[1].key, info_ptr->key, PMIX_MAX_KEYLEN); info[1].value.type = PMIX_STRING; info[1].value.data.string = (char*)info_ptr->val; nvals = 2; @@ -689,12 +689,12 @@ PMIX_EXPORT int PMI2_Nameserv_lookup(const char service_name[], PMIX_PDATA_CONSTRUCT(&pdata[1]); /* pass the service */ - (void)strncpy(pdata[0].key, service_name, PMIX_MAX_KEYLEN); + pmix_strncpy(pdata[0].key, service_name, PMIX_MAX_KEYLEN); nvals = 1; /* if provided, add any other value */ if (NULL != info_ptr) { - (void)strncpy(pdata[1].key, info_ptr->key, PMIX_MAX_KEYLEN); + pmix_strncpy(pdata[1].key, info_ptr->key, PMIX_MAX_KEYLEN); pdata[1].value.type = PMIX_STRING; pdata[1].value.data.string = info_ptr->val; nvals = 2; @@ -716,7 +716,7 @@ PMIX_EXPORT int PMI2_Nameserv_lookup(const char service_name[], } /* return the port */ - (void)strncpy(port, pdata[0].value.data.string, portLen); + pmix_strncpy(port, pdata[0].value.data.string, portLen-1); PMIX_PDATA_DESTRUCT(&pdata[0]); if (NULL != info_ptr) { diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c index 2d951b14f74..5dcac88a4b3 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2014-2017 Research Organization for Information Science + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . * All rights reserved. @@ -53,7 +53,7 @@ #include PMIX_EVENT2_THREAD_HEADER static const char pmix_version_string[] = PMIX_VERSION; - +static pmix_status_t pmix_init_result = PMIX_ERR_INIT; #include "src/class/pmix_list.h" #include "src/event/pmix_event.h" @@ -61,6 +61,7 @@ static const char pmix_version_string[] = PMIX_VERSION; #include "src/util/compress.h" #include "src/util/error.h" #include "src/util/hash.h" +#include "src/util/name_fns.h" #include "src/util/output.h" #include "src/runtime/pmix_progress_threads.h" #include "src/runtime/pmix_rte.h" @@ -92,7 +93,7 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer, pmix_event_chain_t *chain; size_t ninfo; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.base_output, "pmix:client_notify_recv - processing event"); /* a zero-byte buffer indicates that this recv is being @@ -171,16 +172,16 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer, /* prep the chain for processing */ pmix_prep_event_chain(chain, chain->info, ninfo, false); - pmix_output_verbose(2, pmix_globals.debug_output, - "[%s:%d] pmix:client_notify_recv - processing event %d, calling errhandler", - pmix_globals.myid.nspace, pmix_globals.myid.rank, chain->status); + pmix_output_verbose(2, pmix_client_globals.base_output, + "[%s:%d] pmix:client_notify_recv - processing event %s, calling errhandler", + pmix_globals.myid.nspace, pmix_globals.myid.rank, PMIx_Error_string(chain->status)); pmix_invoke_local_event_hdlr(chain); return; error: /* we always need to return */ - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.base_output, "pmix:client_notify_recv - unpack error status =%d, calling def errhandler", rc); chain = PMIX_NEW(pmix_event_chain_t); if (NULL == chain) { @@ -201,7 +202,7 @@ static void wait_cbfunc(struct pmix_peer_t *pr, { pmix_lock_t *lock = (pmix_lock_t*)cbdata; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.base_output, "pmix:client wait_cbfunc received"); PMIX_WAKEUP_THREAD(lock); } @@ -231,6 +232,7 @@ static void job_data(struct pmix_peer_t *pr, PMIX_GDS_STORE_JOB_INFO(cb->status, pmix_client_globals.myserver, nspace, buf); + free(nspace); cb->status = PMIX_SUCCESS; PMIX_POST_OBJECT(cb); PMIX_WAKEUP_THREAD(&cb->lock); @@ -241,6 +243,18 @@ PMIX_EXPORT const char* PMIx_Get_version(void) return pmix_version_string; } +/* event handler registration callback */ +static void evhandler_reg_callbk(pmix_status_t status, + size_t evhandler_ref, + void *cbdata) +{ + pmix_lock_t *lock = (pmix_lock_t*)cbdata; + + lock->status = status; + PMIX_WAKEUP_THREAD(lock); +} + + static void notification_fn(size_t evhdlr_registration_id, pmix_status_t status, const pmix_proc_t *source, @@ -249,13 +263,41 @@ static void notification_fn(size_t evhdlr_registration_id, pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata) { - pmix_lock_t *reglock = (pmix_lock_t*)cbdata; + pmix_lock_t *lock=NULL; + char *name = NULL; + size_t n; + + pmix_output_verbose(2, pmix_client_globals.base_output, + "[%s:%d] DEBUGGER RELEASE RECVD", + pmix_globals.myid.nspace, pmix_globals.myid.rank); + if (NULL != info) { + lock = NULL; + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { + lock = (pmix_lock_t*)info[n].value.data.ptr; + } else if (0 == strncmp(info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) { + name = info[n].value.data.string; + } + } + /* if the object wasn't returned, then that is an error */ + if (NULL == lock) { + pmix_output_verbose(2, pmix_client_globals.base_output, + "event handler %s failed to return object", + (NULL == name) ? "NULL" : name); + /* let the event handler progress */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } + return; + } + } + if (NULL != lock) { + PMIX_WAKEUP_THREAD(lock); + } if (NULL != cbfunc) { cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata); } - PMIX_WAKEUP_THREAD(reglock); - } typedef struct { @@ -336,15 +378,14 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, { char *evar; pmix_status_t rc; - pmix_nspace_t *nsptr; pmix_cb_t cb; pmix_buffer_t *req; pmix_cmd_t cmd = PMIX_REQ_CMD; pmix_status_t code = PMIX_ERR_DEBUGGER_RELEASE; pmix_proc_t wildcard; - pmix_info_t ginfo; + pmix_info_t ginfo, evinfo[2]; pmix_value_t *val = NULL; - pmix_lock_t reglock; + pmix_lock_t reglock, releaselock; size_t n; bool found; @@ -356,7 +397,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, * rank should be known. So return them here if * requested */ if (NULL != proc) { - (void)strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); proc->rank = pmix_globals.myid.rank; } ++pmix_globals.init_cntr; @@ -367,10 +408,13 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, if (NULL != info) { _check_for_notify(info, ninfo); } - return PMIX_SUCCESS; + return pmix_init_result; } + ++pmix_globals.init_cntr; + /* if we don't see the required info, then we cannot init */ - if (NULL == getenv("PMIX_NAMESPACE")) { + if (NULL == (evar = getenv("PMIX_NAMESPACE"))) { + pmix_init_result = PMIX_ERR_INVALID_NAMESPACE; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INVALID_NAMESPACE; } @@ -380,6 +424,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, if (PMIX_SUCCESS != (rc = pmix_rte_init(PMIX_PROC_CLIENT, info, ninfo, pmix_client_notify_recv))) { PMIX_ERROR_LOG(rc); + pmix_init_result = rc; PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } @@ -390,50 +435,48 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, pmix_pointer_array_init(&pmix_client_globals.peers, 1, INT_MAX, 1); pmix_client_globals.myserver = PMIX_NEW(pmix_peer_t); if (NULL == pmix_client_globals.myserver) { + pmix_init_result = PMIX_ERR_NOMEM; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOMEM; } - pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); + pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t); if (NULL == pmix_client_globals.myserver->nptr) { PMIX_RELEASE(pmix_client_globals.myserver); + pmix_init_result = PMIX_ERR_NOMEM; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOMEM; } pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); if (NULL == pmix_client_globals.myserver->info) { PMIX_RELEASE(pmix_client_globals.myserver); + pmix_init_result = PMIX_ERR_NOMEM; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOMEM; } - /* construct the global notification ring buffer */ - PMIX_CONSTRUCT(&pmix_globals.notifications, pmix_ring_buffer_t); - pmix_ring_buffer_init(&pmix_globals.notifications, 256); - pmix_output_verbose(2, pmix_globals.debug_output, + /* setup the base verbosity */ + if (0 < pmix_client_globals.base_verbose) { + /* set default output */ + pmix_client_globals.base_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_client_globals.base_output, + pmix_client_globals.base_verbose); + } + + pmix_output_verbose(2, pmix_client_globals.base_output, "pmix: init called"); /* we require our nspace */ - if (NULL == (evar = getenv("PMIX_NAMESPACE"))) { - /* let the caller know that the server isn't available yet */ - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_INVALID_NAMESPACE; - } if (NULL != proc) { - (void)strncpy(proc->nspace, evar, PMIX_MAX_NSLEN); - } - (void)strncpy(pmix_globals.myid.nspace, evar, PMIX_MAX_NSLEN); - /* create a pmix_nspace_t object for our peer */ - nsptr = PMIX_NEW(pmix_nspace_t); - if (NULL == nsptr){ - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_NOMEM; + pmix_strncpy(proc->nspace, evar, PMIX_MAX_NSLEN); } - nsptr->nspace = strdup(evar); - pmix_globals.mypeer->nptr = nsptr; + PMIX_LOAD_NSPACE(pmix_globals.myid.nspace, evar); + /* set the global pmix_namespace_t object for our peer */ + pmix_globals.mypeer->nptr->nspace = strdup(evar); /* we also require our rank */ if (NULL == (evar = getenv("PMIX_RANK"))) { /* let the caller know that the server isn't available yet */ + pmix_init_result = PMIX_ERR_DATA_VALUE_NOT_FOUND; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_DATA_VALUE_NOT_FOUND; } @@ -445,6 +488,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, /* setup a rank_info object for us */ pmix_globals.mypeer->info = PMIX_NEW(pmix_rank_info_t); if (NULL == pmix_globals.mypeer->info) { + pmix_init_result = PMIX_ERR_NOMEM; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOMEM; } @@ -457,6 +501,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, evar = getenv("PMIX_SECURITY_MODE"); pmix_globals.mypeer->nptr->compat.psec = pmix_psec_base_assign_module(evar); if (NULL == pmix_globals.mypeer->nptr->compat.psec) { + pmix_init_result = PMIX_ERR_INIT; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; } @@ -491,6 +536,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, pmix_client_globals.myserver->nptr->compat.gds = pmix_gds_base_assign_module(NULL, 0); } if (NULL == pmix_client_globals.myserver->nptr->compat.gds) { + pmix_init_result = PMIX_ERR_INIT; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; } @@ -513,6 +559,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, pmix_globals.mypeer->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1); if (NULL == pmix_globals.mypeer->nptr->compat.gds) { PMIX_INFO_DESTRUCT(&ginfo); + pmix_init_result = PMIX_ERR_INIT; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; } @@ -521,6 +568,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, /* connect to the server */ rc = pmix_ptl_base_connect_to_peer((struct pmix_peer_t*)pmix_client_globals.myserver, info, ninfo); if (PMIX_SUCCESS != rc) { + pmix_init_result = rc; PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } @@ -536,6 +584,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(req); + pmix_init_result = rc; PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } @@ -544,6 +593,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, req, job_data, (void*)&cb); if (PMIX_SUCCESS != rc) { + pmix_init_result = rc; PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } @@ -553,15 +603,16 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, PMIX_DESTRUCT(&cb); if (PMIX_SUCCESS == rc) { - pmix_globals.init_cntr++; + pmix_init_result = PMIX_SUCCESS; } else { + pmix_init_result = rc; PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } PMIX_RELEASE_THREAD(&pmix_global_lock); - /* lood for a debugger attach key */ - (void)strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + /* look for a debugger attach key */ + pmix_strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); wildcard.rank = PMIX_RANK_WILDCARD; PMIX_INFO_LOAD(&ginfo, PMIX_OPTIONAL, NULL, PMIX_BOOL); if (PMIX_SUCCESS == PMIx_Get(&wildcard, PMIX_DEBUG_STOP_IN_INIT, &ginfo, 1, &val)) { @@ -569,11 +620,22 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, /* if the value was found, then we need to wait for debugger attach here */ /* register for the debugger release notification */ PMIX_CONSTRUCT_LOCK(®lock); - PMIx_Register_event_handler(&code, 1, NULL, 0, - notification_fn, NULL, (void*)®lock); - /* wait for it to arrive */ + PMIX_CONSTRUCT_LOCK(&releaselock); + PMIX_INFO_LOAD(&evinfo[0], PMIX_EVENT_RETURN_OBJECT, &releaselock, PMIX_POINTER); + PMIX_INFO_LOAD(&evinfo[1], PMIX_EVENT_HDLR_NAME, "WAIT-FOR-DEBUGGER", PMIX_STRING); + pmix_output_verbose(2, pmix_client_globals.base_output, + "[%s:%d] WAITING IN INIT FOR DEBUGGER", + pmix_globals.myid.nspace, pmix_globals.myid.rank); + PMIx_Register_event_handler(&code, 1, evinfo, 2, + notification_fn, evhandler_reg_callbk, (void*)®lock); + /* wait for registration to complete */ PMIX_WAIT_THREAD(®lock); PMIX_DESTRUCT_LOCK(®lock); + PMIX_INFO_DESTRUCT(&evinfo[0]); + PMIX_INFO_DESTRUCT(&evinfo[1]); + /* wait for release to arrive */ + PMIX_WAIT_THREAD(&releaselock); + PMIX_DESTRUCT_LOCK(&releaselock); } PMIX_INFO_DESTRUCT(&ginfo); @@ -581,7 +643,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, if (NULL != info) { _check_for_notify(info, ninfo); } - return PMIX_SUCCESS; } @@ -609,7 +670,7 @@ static void fin_timeout(int sd, short args, void *cbdata) pmix_client_timeout_t *tev; tev = (pmix_client_timeout_t*)cbdata; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.base_output, "pmix:client finwait timeout fired"); if (tev->active) { tev->active = false; @@ -624,7 +685,7 @@ static void finwait_cbfunc(struct pmix_peer_t *pr, pmix_client_timeout_t *tev; tev = (pmix_client_timeout_t*)cbdata; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.base_output, "pmix:client finwait_cbfunc received"); if (tev->active) { tev->active = false; @@ -651,7 +712,7 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo) } pmix_globals.init_cntr = 0; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.base_output, "%s:%d pmix:client finalize called", pmix_globals.myid.nspace, pmix_globals.myid.rank); @@ -689,7 +750,7 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo) } - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.base_output, "%s:%d pmix:client sending finalize sync to server", pmix_globals.myid.nspace, pmix_globals.myid.rank); @@ -716,7 +777,7 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo) pmix_event_del(&tev.ev); } - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.base_output, "%s:%d pmix:client finalize sync received", pmix_globals.myid.nspace, pmix_globals.myid.rank); } @@ -735,6 +796,7 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo) PMIX_RELEASE(peer); } } + PMIX_DESTRUCT(&pmix_client_globals.peers); if (0 <= pmix_client_globals.myserver->sd) { CLOSE_THE_SOCKET(pmix_client_globals.myserver->sd); @@ -745,8 +807,15 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo) pmix_rte_finalize(); + if (NULL != pmix_globals.mypeer) { + PMIX_RELEASE(pmix_globals.mypeer); + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + /* finalize the class/object system */ + pmix_class_finalize(); + return PMIX_SUCCESS; } @@ -758,7 +827,7 @@ PMIX_EXPORT pmix_status_t PMIx_Abort(int flag, const char msg[], pmix_status_t rc; pmix_lock_t reglock; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.base_output, "pmix:client abort called"); PMIX_ACQUIRE_THREAD(&pmix_global_lock); @@ -904,12 +973,14 @@ static void _putfn(int sd, short args, void *cbdata) PMIX_WAKEUP_THREAD(&cb->lock); } -PMIX_EXPORT pmix_status_t PMIx_Put(pmix_scope_t scope, const char key[], pmix_value_t *val) +PMIX_EXPORT pmix_status_t PMIx_Put(pmix_scope_t scope, + const pmix_key_t key, + pmix_value_t *val) { pmix_cb_t *cb; pmix_status_t rc; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.base_output, "pmix: executing put for key %s type %d", key, val->type); @@ -1100,11 +1171,26 @@ static void _commitfn(int sd, short args, void *cbdata) return rc; } +static void _resolve_peers(int sd, short args, void *cbdata) +{ + pmix_cb_t *cb = (pmix_cb_t*)cbdata; + + cb->status = pmix_preg.resolve_peers(cb->key, cb->pname.nspace, + &cb->procs, &cb->nprocs); + /* post the data so the receiving thread can acquire it */ + PMIX_POST_OBJECT(cb); + PMIX_WAKEUP_THREAD(&cb->lock); +} + /* need to thread-shift this request */ PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename, - const char *nspace, + const pmix_nspace_t nspace, pmix_proc_t **procs, size_t *nprocs) { + pmix_cb_t *cb; + pmix_status_t rc; + pmix_proc_t proc; + PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { PMIX_RELEASE_THREAD(&pmix_global_lock); @@ -1112,16 +1198,71 @@ PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename, } PMIX_RELEASE_THREAD(&pmix_global_lock); - /* set default */ - *procs = NULL; - *nprocs = 0; - return pmix_preg.resolve_peers(nodename, nspace, procs, nprocs); + cb = PMIX_NEW(pmix_cb_t); + cb->key = (char*)nodename; + cb->pname.nspace = strdup(nspace); + + PMIX_THREADSHIFT(cb, _resolve_peers); + + /* wait for the result */ + PMIX_WAIT_THREAD(&cb->lock); + + /* if the nspace wasn't found, then we need to + * ask the server for that info */ + if (PMIX_ERR_INVALID_NAMESPACE == cb->status) { + pmix_strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + /* any key will suffice as it will bring down + * the entire data blob */ + rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, NULL); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(cb); + return rc; + } + /* retry the fetch */ + cb->lock.active = true; + PMIX_THREADSHIFT(cb, _resolve_peers); + PMIX_WAIT_THREAD(&cb->lock); + } + *procs = cb->procs; + *nprocs = cb->nprocs; + + rc = cb->status; + PMIX_RELEASE(cb); + return rc; +} + +static void _resolve_nodes(int fd, short args, void *cbdata) +{ + pmix_cb_t *cb = (pmix_cb_t*)cbdata; + char *regex, **names; + + /* get a regular expression describing the PMIX_NODE_MAP */ + cb->status = pmix_preg.resolve_nodes(cb->pname.nspace, ®ex); + if (PMIX_SUCCESS == cb->status) { + /* parse it into an argv array of names */ + cb->status = pmix_preg.parse_nodes(regex, &names); + if (PMIX_SUCCESS == cb->status) { + /* assemble it into a comma-delimited list */ + cb->key = pmix_argv_join(names, ','); + pmix_argv_free(names); + } else { + free(regex); + } + } + /* post the data so the receiving thread can acquire it */ + PMIX_POST_OBJECT(cb); + PMIX_WAKEUP_THREAD(&cb->lock); } /* need to thread-shift this request */ -PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist) +PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const pmix_nspace_t nspace, char **nodelist) { + pmix_cb_t *cb; + pmix_status_t rc; + pmix_proc_t proc; + PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { PMIX_RELEASE_THREAD(&pmix_global_lock); @@ -1129,8 +1270,35 @@ PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist } PMIX_RELEASE_THREAD(&pmix_global_lock); - /* set default */ - *nodelist = NULL; + cb = PMIX_NEW(pmix_cb_t); + cb->pname.nspace = strdup(nspace); + + PMIX_THREADSHIFT(cb, _resolve_nodes); + + /* wait for the result */ + PMIX_WAIT_THREAD(&cb->lock); - return pmix_preg.resolve_nodes(nspace, nodelist); + /* if the nspace wasn't found, then we need to + * ask the server for that info */ + if (PMIX_ERR_INVALID_NAMESPACE == cb->status) { + pmix_strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + /* any key will suffice as it will bring down + * the entire data blob */ + rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, NULL); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(cb); + return rc; + } + /* retry the fetch */ + cb->lock.active = true; + PMIX_THREADSHIFT(cb, _resolve_nodes); + PMIX_WAIT_THREAD(&cb->lock); + } + /* the string we want is in the key field */ + *nodelist = cb->key; + + rc = cb->status; + PMIX_RELEASE(cb); + return rc; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_connect.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_connect.c index de54c00c67b..e652c98ed9c 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_connect.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_connect.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . @@ -72,7 +72,7 @@ PMIX_EXPORT pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs, PMIX_ACQUIRE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.connect_output, "pmix: connect called"); if (pmix_globals.init_cntr <= 0) { @@ -120,8 +120,8 @@ PMIX_EXPORT pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t npro PMIX_ACQUIRE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: connect called"); + pmix_output_verbose(2, pmix_client_globals.connect_output, + "pmix:connect_nb called"); if (pmix_globals.init_cntr <= 0) { PMIX_RELEASE_THREAD(&pmix_global_lock); diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_fence.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_fence.c index d3e182d584d..342305ea206 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_fence.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_fence.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . @@ -147,7 +147,7 @@ PMIX_EXPORT pmix_status_t PMIx_Fence_nb(const pmix_proc_t procs[], size_t nprocs /* if we are given a NULL proc, then the caller is referencing * all procs within our own nspace */ if (NULL == procs) { - (void)strncpy(rg.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(rg.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); rg.rank = PMIX_RANK_WILDCARD; rgs = &rg; nrg = 1; diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c index deaf1210180..ac3c950bb87 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c @@ -1,11 +1,11 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . * All rights reserved. - * Copyright (c) 2016 Mellanox Technologies, Inc. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ @@ -75,8 +75,15 @@ static void _getnb_cbfunc(struct pmix_peer_t *pr, static void _value_cbfunc(pmix_status_t status, pmix_value_t *kv, void *cbdata); +static pmix_status_t _getfn_fastpath(const pmix_proc_t *proc, const pmix_key_t key, + const pmix_info_t info[], size_t ninfo, + pmix_value_t **val); -PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], +static pmix_status_t process_values(pmix_value_t **v, pmix_cb_t *cb); + + +PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, + const pmix_key_t key, const pmix_info_t info[], size_t ninfo, pmix_value_t **val) { @@ -91,6 +98,17 @@ PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], } PMIX_RELEASE_THREAD(&pmix_global_lock); + pmix_output_verbose(2, pmix_client_globals.get_output, + "pmix:client get for %s:%d key %s", + (NULL == proc) ? "NULL" : proc->nspace, + (NULL == proc) ? PMIX_RANK_UNDEF : proc->rank, + (NULL == key) ? "NULL" : key); + + /* try to get data directly, without threadshift */ + if (PMIX_SUCCESS == (rc = _getfn_fastpath(proc, key, info, ninfo, val))) { + goto done; + } + /* create a callback object as we need to pass it to the * recv routine so we know which callback to use when * the return message is recvd */ @@ -105,16 +123,18 @@ PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], rc = cb->status; if (NULL != val) { *val = cb->value; + cb->value = NULL; } PMIX_RELEASE(cb); - pmix_output_verbose(2, pmix_globals.debug_output, + done: + pmix_output_verbose(2, pmix_client_globals.get_output, "pmix:client get completed"); return rc; } -PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char *key, +PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const pmix_key_t key, const pmix_info_t info[], size_t ninfo, pmix_value_cbfunc_t cbfunc, void *cbdata) { @@ -142,12 +162,16 @@ PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char *key, * Either case is supported. However, we don't currently * support the case where -both- values are NULL */ if (NULL == proc && NULL == key) { + pmix_output_verbose(2, pmix_client_globals.get_output, + "pmix: get_nb value error - both proc and key are NULL"); return PMIX_ERR_BAD_PARAM; } /* if the key is NULL, the rank cannot be WILDCARD as * we cannot return all info from every rank */ if (NULL != proc && PMIX_RANK_WILDCARD == proc->rank && NULL == key) { + pmix_output_verbose(2, pmix_client_globals.get_output, + "pmix: get_nb value error - WILDCARD rank and key is NULL"); return PMIX_ERR_BAD_PARAM; } @@ -168,7 +192,7 @@ PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char *key, rank = proc->rank; } - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.get_output, "pmix: get_nb value for proc %s:%u key %s", nm, rank, (NULL == key) ? "NULL" : key); @@ -273,7 +297,7 @@ static void _getnb_cbfunc(struct pmix_peer_t *pr, pmix_proc_t proc; pmix_kval_t *kv; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.get_output, "pmix: get_nb callback recvd"); if (NULL == cb) { @@ -283,7 +307,7 @@ static void _getnb_cbfunc(struct pmix_peer_t *pr, } /* cache the proc id */ - (void)strncpy(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN); proc.rank = cb->pname.rank; /* a zero-byte buffer indicates that this recv is being @@ -396,7 +420,7 @@ static pmix_status_t process_values(pmix_value_t **v, pmix_cb_t *cb) /* copy the list elements */ n=0; PMIX_LIST_FOREACH(kv, kvs, pmix_kval_t) { - (void)strncpy(info[n].key, kv->key, PMIX_MAX_KEYLEN); + pmix_strncpy(info[n].key, kv->key, PMIX_MAX_KEYLEN); pmix_value_xfer(&info[n].value, kv->value); ++n; } @@ -461,6 +485,55 @@ static void infocb(pmix_status_t status, } } +static pmix_status_t _getfn_fastpath(const pmix_proc_t *proc, const pmix_key_t key, + const pmix_info_t info[], size_t ninfo, + pmix_value_t **val) +{ + pmix_cb_t *cb = PMIX_NEW(pmix_cb_t); + pmix_status_t rc = PMIX_SUCCESS; + size_t n; + + /* scan the incoming directives */ + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_DATA_SCOPE, PMIX_MAX_KEYLEN)) { + cb->scope = info[n].value.data.scope; + break; + } + } + } + cb->proc = (pmix_proc_t*)proc; + cb->copy = true; + cb->key = (char*)key; + cb->info = (pmix_info_t*)info; + cb->ninfo = ninfo; + + PMIX_GDS_FETCH_IS_TSAFE(rc, pmix_globals.mypeer); + if (PMIX_SUCCESS == rc) { + PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, cb); + if (PMIX_SUCCESS == rc) { + goto done; + } + } + PMIX_GDS_FETCH_IS_TSAFE(rc, pmix_client_globals.myserver); + if (PMIX_SUCCESS == rc) { + PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, cb); + if (PMIX_SUCCESS == rc) { + goto done; + } + } + PMIX_RELEASE(cb); + return rc; + + done: + rc = process_values(val, cb); + if (NULL != *val) { + PMIX_VALUE_COMPRESSED_STRING_UNPACK(*val); + } + PMIX_RELEASE(cb); + return rc; +} + static void _getnbfn(int fd, short flags, void *cbdata) { pmix_cb_t *cb = (pmix_cb_t*)cbdata; @@ -469,7 +542,6 @@ static void _getnbfn(int fd, short flags, void *cbdata) pmix_value_t *val = NULL; pmix_status_t rc; size_t n; - char *tmp; pmix_proc_t proc; bool optional = false; bool immediate = false; @@ -479,13 +551,13 @@ static void _getnbfn(int fd, short flags, void *cbdata) /* cb was passed to us from another thread - acquire it */ PMIX_ACQUIRE_OBJECT(cb); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.get_output, "pmix: getnbfn value for proc %s:%u key %s", cb->pname.nspace, cb->pname.rank, (NULL == cb->key) ? "NULL" : cb->key); /* set the proc object identifier */ - (void)strncpy(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN); proc.rank = cb->pname.rank; /* scan the incoming directives */ @@ -517,9 +589,13 @@ static void _getnbfn(int fd, short flags, void *cbdata) cb->copy = true; PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, cb); if (PMIX_SUCCESS == rc) { + pmix_output_verbose(5, pmix_client_globals.get_output, + "pmix:client data found in internal storage"); rc = process_values(&val, cb); goto respond; } + pmix_output_verbose(5, pmix_client_globals.get_output, + "pmix:client data NOT found in internal storage"); /* if the key is NULL or starts with "pmix", then they are looking * for data that was provided by the server at startup */ @@ -530,6 +606,8 @@ static void _getnbfn(int fd, short flags, void *cbdata) cb->copy = true; PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, cb); if (PMIX_SUCCESS != rc) { + pmix_output_verbose(5, pmix_client_globals.get_output, + "pmix:client job-level data NOT found"); if (0 != strncmp(cb->pname.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN)) { /* we are asking about the job-level info from another * namespace. It seems that we don't have it - go and @@ -542,6 +620,8 @@ static void _getnbfn(int fd, short flags, void *cbdata) * job-level info. In some cases, a server may elect not * to provide info at init to save memory */ if (immediate) { + pmix_output_verbose(5, pmix_client_globals.get_output, + "pmix:client IMMEDIATE given - querying data"); /* the direct modex request doesn't pass a key as it * was intended to support non-job-level information. * So instead, we will use the PMIx_Query function @@ -560,11 +640,17 @@ static void _getnbfn(int fd, short flags, void *cbdata) return; } /* we should have had this info, so respond with the error */ + pmix_output_verbose(5, pmix_client_globals.get_output, + "pmix:client returning NOT FOUND error"); goto respond; } else { + pmix_output_verbose(5, pmix_client_globals.get_output, + "pmix:client NULL KEY - returning error"); goto respond; } } + pmix_output_verbose(5, pmix_client_globals.get_output, + "pmix:client job-level data NOT found"); rc = process_values(&val, cb); goto respond; } else { @@ -583,19 +669,7 @@ static void _getnbfn(int fd, short flags, void *cbdata) /* if a callback was provided, execute it */ if (NULL != cb->cbfunc.valuefn) { if (NULL != val) { - /* if this is a compressed string, then uncompress it */ - if (PMIX_COMPRESSED_STRING == val->type) { - pmix_util_uncompress_string(&tmp, (uint8_t*)val->data.bo.bytes, val->data.bo.size); - if (NULL == tmp) { - PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - rc = PMIX_ERR_NOMEM; - PMIX_VALUE_RELEASE(val); - val = NULL; - } else { - PMIX_VALUE_DESTRUCT(val); - PMIX_VAL_ASSIGN(val, string, tmp); - } - } + PMIX_VALUE_COMPRESSED_STRING_UNPACK(val); } cb->cbfunc.valuefn(rc, val, cb->cbdata); } @@ -619,7 +693,7 @@ static void _getnbfn(int fd, short flags, void *cbdata) * us to attempt to retrieve it from the server */ if (optional) { /* they don't want us to try and retrieve it */ - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.get_output, "PMIx_Get key=%s for rank = %u, namespace = %s was not found - request was optional", cb->key, cb->pname.rank, cb->pname.nspace); rc = PMIX_ERR_NOT_FOUND; @@ -647,7 +721,7 @@ static void _getnbfn(int fd, short flags, void *cbdata) goto respond; } - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.get_output, "%s:%d REQUESTING DATA FROM SERVER FOR %s:%d KEY %s", pmix_globals.myid.nspace, pmix_globals.myid.rank, cb->pname.nspace, cb->pname.rank, cb->key); diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_ops.h b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_ops.h index 4561b0a619a..d173978b405 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_ops.h +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_ops.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -24,6 +24,27 @@ typedef struct { pmix_peer_t *myserver; // messaging support to/from my server pmix_list_t pending_requests; // list of pmix_cb_t pending data requests pmix_pointer_array_t peers; // array of pmix_peer_t cached for data ops + // verbosity for client get operations + int get_output; + int get_verbose; + // verbosity for client connect operations + int connect_output; + int connect_verbose; + // verbosity for client fence operations + int fence_output; + int fence_verbose; + // verbosity for client pub operations + int pub_output; + int pub_verbose; + // verbosity for client spawn operations + int spawn_output; + int spawn_verbose; + // verbosity for client event operations + int event_output; + int event_verbose; + // verbosity for basic client functions + int base_output; + int base_verbose; } pmix_client_globals_t; PMIX_EXPORT extern pmix_client_globals_t pmix_client_globals; diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_pub.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_pub.c index cee3dcaf940..fed1b129e32 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_pub.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_pub.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . @@ -660,7 +660,7 @@ static void lookup_cbfunc(pmix_status_t status, pmix_pdata_t pdata[], size_t nda for (j=0; j < cb->nvals; j++) { if (0 == strcmp(pdata[i].key, tgt[j].key)) { /* transfer the publishing proc id */ - (void)strncpy(tgt[j].proc.nspace, pdata[i].proc.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(tgt[j].proc.nspace, pdata[i].proc.nspace, PMIX_MAX_NSLEN); tgt[j].proc.rank = pdata[i].proc.rank; /* transfer the value to the pmix_info_t */ PMIX_BFROPS_VALUE_XFER(cb->status, pmix_client_globals.myserver, &tgt[j].value, &pdata[i].value); diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_spawn.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_spawn.c index f04dcec34d1..fe12bbed90f 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_spawn.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_spawn.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . @@ -49,6 +49,7 @@ #include "src/class/pmix_list.h" #include "src/threads/threads.h" #include "src/mca/bfrops/bfrops.h" +#include "src/mca/pnet/base/base.h" #include "src/util/argv.h" #include "src/util/error.h" #include "src/util/output.h" @@ -64,7 +65,7 @@ static void spawn_cbfunc(pmix_status_t status, char nspace[], void *cbdata); PMIX_EXPORT pmix_status_t PMIx_Spawn(const pmix_info_t job_info[], size_t ninfo, const pmix_app_t apps[], size_t napps, - char nspace[]) + pmix_nspace_t nspace) { pmix_status_t rc; pmix_cb_t *cb; @@ -104,7 +105,7 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn(const pmix_info_t job_info[], size_t ninfo, PMIX_WAIT_THREAD(&cb->lock); rc = cb->status; if (NULL != nspace) { - (void)strncpy(nspace, cb->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(nspace, cb->pname.nspace, PMIX_MAX_NSLEN); } PMIX_RELEASE(cb); @@ -240,27 +241,25 @@ static void wait_cbfunc(struct pmix_peer_t *pr, PMIX_ERROR_LOG(rc); ret = rc; } - if (PMIX_SUCCESS == ret) { - /* unpack the namespace */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver, - buf, &n2, &cnt, PMIX_STRING); + /* unpack the namespace */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver, + buf, &n2, &cnt, PMIX_STRING); + if (PMIX_SUCCESS != rc && PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) { + PMIX_ERROR_LOG(rc); + ret = rc; + } + pmix_output_verbose(1, pmix_globals.debug_output, + "pmix:client recv '%s'", n2); + + if (NULL != n2) { + /* protect length */ + pmix_strncpy(nspace, n2, PMIX_MAX_NSLEN); + free(n2); + PMIX_GDS_STORE_JOB_INFO(rc, pmix_globals.mypeer, nspace, buf); + /* extract and process any job-related info for this nspace */ if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - ret = rc; - } - pmix_output_verbose(1, pmix_globals.debug_output, - "pmix:client recv '%s'", n2); - - if (NULL != n2) { - /* protect length */ - (void)strncpy(nspace, n2, PMIX_MAX_NSLEN); - free(n2); - PMIX_GDS_STORE_JOB_INFO(rc, pmix_globals.mypeer, nspace, buf); - /* extract and process any job-related info for this nspace */ - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - } } } diff --git a/opal/mca/pmix/pmix2x/pmix/src/common/Makefile.include b/opal/mca/pmix/pmix2x/pmix/src/common/Makefile.include index dda109eb699..c1dd7169de7 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/common/Makefile.include +++ b/opal/mca/pmix/pmix2x/pmix/src/common/Makefile.include @@ -1,7 +1,7 @@ # -*- makefile -*- # -# Copyright (c) 2015-2017 Intel, Inc. All rights reserved. -# Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2015-2019 Intel, Inc. All rights reserved. +# Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_control.c b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_control.c index 7dd8c7f4e8b..39691938cd1 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_control.c +++ b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_control.c @@ -1,9 +1,11 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -24,6 +26,7 @@ #include "src/threads/threads.h" #include "src/util/argv.h" #include "src/util/error.h" +#include "src/util/name_fns.h" #include "src/util/output.h" #include "src/mca/bfrops/bfrops.h" #include "src/mca/ptl/ptl.h" @@ -104,6 +107,8 @@ static void query_cbfunc(struct pmix_peer_t *peer, /* release the caller */ if (NULL != cd->cbfunc) { cd->cbfunc(results->status, results->info, results->ninfo, cd->cbdata, relcbfunc, results); + } else { + PMIX_RELEASE(results); } PMIX_RELEASE(cd); } @@ -129,7 +134,8 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_ /* if we are the server, then we just issue the request and * return the response */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { PMIX_RELEASE_THREAD(&pmix_global_lock); if (NULL == pmix_host_server.job_control) { /* nothing we can do */ @@ -246,7 +252,8 @@ PMIX_EXPORT pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pm /* if we are the server, then we just issue the request and * return the response */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { PMIX_RELEASE_THREAD(&pmix_global_lock); if (NULL == pmix_host_server.monitor) { /* nothing we can do */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_data.c b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_data.c index 37e076c8c12..fe7a8964395 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_data.c +++ b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_data.c @@ -11,7 +11,7 @@ * All rights reserved. * Copyright (c) 2007-2012 Los Alamos National Security, LLC. * All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -76,7 +76,7 @@ static pmix_peer_t* find_peer(const pmix_proc_t *proc) pmix_value_t *value; int i; - if (NULL == proc) { + if (NULL == proc ) { return pmix_globals.mypeer; } @@ -98,7 +98,7 @@ static pmix_peer_t* find_peer(const pmix_proc_t *proc) /* didn't find it, so try to get the library version of the target * from the host - the result will be cached, so we will only have * to retrieve it once */ - (void)strncpy(wildcard.nspace, proc->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(wildcard.nspace, proc->nspace, PMIX_MAX_NSLEN); wildcard.rank = PMIX_RANK_WILDCARD; if (PMIX_SUCCESS != (rc = PMIx_Get(&wildcard, PMIX_BFROPS_MODULE, NULL, 0, &value))) { /* couldn't get it - nothing we can do */ @@ -110,7 +110,7 @@ static pmix_peer_t* find_peer(const pmix_proc_t *proc) PMIX_RELEASE(value); return NULL; } - peer->nptr = PMIX_NEW(pmix_nspace_t); + peer->nptr = PMIX_NEW(pmix_namespace_t); if (NULL == peer->nptr) { PMIX_RELEASE(peer); PMIX_RELEASE(value); @@ -139,7 +139,7 @@ static pmix_peer_t* find_peer(const pmix_proc_t *proc) /* try to get the library version of this peer - the result will be * cached, so we will only have to retrieve it once */ - (void)strncpy(wildcard.nspace, proc->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(wildcard.nspace, proc->nspace, PMIX_MAX_NSLEN); wildcard.rank = PMIX_RANK_WILDCARD; if (PMIX_SUCCESS != (rc = PMIx_Get(&wildcard, PMIX_BFROPS_MODULE, NULL, 0, &value))) { /* couldn't get it - nothing we can do */ @@ -151,7 +151,7 @@ static pmix_peer_t* find_peer(const pmix_proc_t *proc) PMIX_RELEASE(value); return NULL; } - peer->nptr = PMIX_NEW(pmix_nspace_t); + peer->nptr = PMIX_NEW(pmix_namespace_t); if (NULL == peer->nptr) { PMIX_RELEASE(peer); PMIX_RELEASE(value); diff --git a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_log.c b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_log.c index b32c4cc5cf7..ab2d2a41f31 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_log.c +++ b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_log.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. diff --git a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_query.c b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_query.c index 5aa14a532a7..f5223b83b5d 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_query.c +++ b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_query.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -24,6 +24,7 @@ #include "src/threads/threads.h" #include "src/util/argv.h" #include "src/util/error.h" +#include "src/util/name_fns.h" #include "src/util/output.h" #include "src/mca/bfrops/bfrops.h" #include "src/mca/ptl/ptl.h" @@ -52,6 +53,8 @@ static void query_cbfunc(struct pmix_peer_t *peer, pmix_status_t rc; pmix_shift_caddy_t *results; int cnt; + size_t n; + pmix_kval_t *kv; pmix_output_verbose(2, pmix_globals.debug_output, "pmix:query cback from server"); @@ -63,6 +66,7 @@ static void query_cbfunc(struct pmix_peer_t *peer, PMIX_BFROPS_UNPACK(rc, peer, buf, &results->status, &cnt, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); + results->status = rc; goto complete; } if (PMIX_SUCCESS != results->status) { @@ -74,6 +78,7 @@ static void query_cbfunc(struct pmix_peer_t *peer, PMIX_BFROPS_UNPACK(rc, peer, buf, &results->ninfo, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); + results->status = rc; goto complete; } if (0 < results->ninfo) { @@ -82,8 +87,22 @@ static void query_cbfunc(struct pmix_peer_t *peer, PMIX_BFROPS_UNPACK(rc, peer, buf, results->info, &cnt, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); + results->status = rc; goto complete; } + /* locally cache the results */ + for (n=0; n < results->ninfo; n++) { + kv = PMIX_NEW(pmix_kval_t); + kv->key = strdup(results->info[n].key); + PMIX_VALUE_CREATE(kv->value, 1); + PMIX_BFROPS_VALUE_XFER(rc, pmix_globals.mypeer, + kv->value, &results->info[n].value); + + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, PMIX_INTERNAL, + kv); + PMIX_RELEASE(kv); // maintain accounting + } } complete: @@ -96,6 +115,22 @@ static void query_cbfunc(struct pmix_peer_t *peer, PMIX_RELEASE(cd); } +static void _local_relcb(void *cbdata) +{ + pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata; + PMIX_RELEASE(cd); +} + +static void _local_cbfunc(int sd, short args, void *cbdata) +{ + pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata; + if (NULL != cd->cbfunc) { + cd->cbfunc(cd->status, cd->info, cd->ninfo, cd->cbdata, _local_relcb, cd); + return; + } + PMIX_RELEASE(cd); +} + PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nqueries, pmix_info_cbfunc_t cbfunc, void *cbdata) @@ -104,6 +139,11 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque pmix_cmd_t cmd = PMIX_QUERY_CMD; pmix_buffer_t *msg; pmix_status_t rc; + pmix_cb_t cb; + size_t n, p; + pmix_list_t results; + pmix_kval_t *kv, *kvnxt; + pmix_proc_t proc; PMIX_ACQUIRE_THREAD(&pmix_global_lock); @@ -115,67 +155,175 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque return PMIX_ERR_INIT; } - /* if we aren't connected, don't attempt to send */ - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !pmix_globals.connected) { + if (0 == nqueries || NULL == queries) { PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_UNREACH; + return PMIX_ERR_BAD_PARAM; } - PMIX_RELEASE_THREAD(&pmix_global_lock); - if (0 == nqueries || NULL == queries) { - return PMIX_ERR_BAD_PARAM; + /* setup the list of local results */ + PMIX_CONSTRUCT(&results, pmix_list_t); + + /* check the directives to see if they want us to refresh + * the local cached results - if we wanted to optimize this + * more, we would check each query and allow those that don't + * want to be refreshed to be executed locally, and those that + * did would be sent to the host. However, for now we simply + * */ + memset(proc.nspace, 0, PMIX_MAX_NSLEN+1); + proc.rank = PMIX_RANK_INVALID; + for (n=0; n < nqueries; n++) { + for (p=0; p < queries[n].nqual; p++) { + if (PMIX_CHECK_KEY(&queries[n].qualifiers[p], PMIX_QUERY_REFRESH_CACHE)) { + if (PMIX_INFO_TRUE(&queries[n].qualifiers[p])) { + PMIX_LIST_DESTRUCT(&results); + goto query; + } + } else if (PMIX_CHECK_KEY(&queries[n].qualifiers[p], PMIX_PROCID)) { + PMIX_LOAD_NSPACE(proc.nspace, queries[n].qualifiers[p].value.data.proc->nspace); + proc.rank = queries[n].qualifiers[p].value.data.proc->rank; + } else if (PMIX_CHECK_KEY(&queries[n].qualifiers[p], PMIX_NSPACE)) { + PMIX_LOAD_NSPACE(proc.nspace, queries[n].qualifiers[p].value.data.string); + } else if (PMIX_CHECK_KEY(&queries[n].qualifiers[p], PMIX_RANK)) { + proc.rank = queries[n].qualifiers[p].value.data.rank; + } else if (PMIX_CHECK_KEY(&queries[n].qualifiers[p], PMIX_HOSTNAME)) { + if (0 != strcmp(queries[n].qualifiers[p].value.data.string, pmix_globals.hostname)) { + /* asking about a different host, so ask for the info */ + PMIX_LIST_DESTRUCT(&results); + goto query; + } + } + } + /* we get here if a refresh isn't required - first try a local + * "get" on the data to see if we already have it */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + cb.copy = false; + /* set the proc */ + if (PMIX_RANK_INVALID == proc.rank && + 0 == strlen(proc.nspace)) { + /* use our id */ + cb.proc = &pmix_globals.myid; + } else { + if (0 == strlen(proc.nspace)) { + /* use our nspace */ + PMIX_LOAD_NSPACE(cb.proc->nspace, pmix_globals.myid.nspace); + } + if (PMIX_RANK_INVALID == proc.rank) { + /* user the wildcard rank */ + proc.rank = PMIX_RANK_WILDCARD; + } + cb.proc = &proc; + } + for (p=0; NULL != queries[n].keys[p]; p++) { + cb.key = queries[n].keys[p]; + PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb); + if (PMIX_SUCCESS != rc) { + /* needs to be passed to the host */ + PMIX_LIST_DESTRUCT(&results); + PMIX_DESTRUCT(&cb); + goto query; + } + /* need to retain this result */ + PMIX_LIST_FOREACH_SAFE(kv, kvnxt, &cb.kvs, pmix_kval_t) { + pmix_list_remove_item(&cb.kvs, &kv->super); + pmix_list_append(&results, &kv->super); + } + PMIX_DESTRUCT(&cb); + } } + /* if we get here, then all queries were completely locally + * resolved, so construct the results for return */ + cd = PMIX_NEW(pmix_query_caddy_t); + cd->cbfunc = cbfunc; + cd->cbdata = cbdata; + cd->status = PMIX_SUCCESS; + cd->ninfo = pmix_list_get_size(&results); + PMIX_INFO_CREATE(cd->info, cd->ninfo); + n = 0; + PMIX_LIST_FOREACH_SAFE(kv, kvnxt, &results, pmix_kval_t) { + PMIX_LOAD_KEY(cd->info[n].key, kv->key); + rc = pmix_value_xfer(&cd->info[n].value, kv->value); + if (PMIX_SUCCESS != rc) { + cd->status = rc; + PMIX_INFO_FREE(cd->info, cd->ninfo); + break; + } + ++n; + } + /* done with the list of results */ + PMIX_LIST_DESTRUCT(&results); + /* we need to thread-shift as we are not allowed to + * execute the callback function prior to returning + * from the API */ + PMIX_THREADSHIFT(cd, _local_cbfunc); + /* regardless of the result of the query, we return + * PMIX_SUCCESS here to indicate that the operation + * was accepted for processing */ + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_SUCCESS; + + + query: /* if we are the server, then we just issue the query and * return the response */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + PMIX_RELEASE_THREAD(&pmix_global_lock); if (NULL == pmix_host_server.query) { /* nothing we can do */ return PMIX_ERR_NOT_SUPPORTED; } pmix_output_verbose(2, pmix_globals.debug_output, "pmix:query handed to RM"); - pmix_host_server.query(&pmix_globals.myid, - queries, nqueries, - cbfunc, cbdata); - rc = PMIX_SUCCESS; - } else { - /* if we are a client, then relay this request to the server */ - cd = PMIX_NEW(pmix_query_caddy_t); - cd->cbfunc = cbfunc; - cd->cbdata = cbdata; - msg = PMIX_NEW(pmix_buffer_t); - PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, - msg, &cmd, 1, PMIX_COMMAND); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - PMIX_RELEASE(cd); - return rc; - } - PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, - msg, &nqueries, 1, PMIX_SIZE); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - PMIX_RELEASE(cd); - return rc; - } - PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, - msg, queries, nqueries, PMIX_QUERY); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - PMIX_RELEASE(cd); - return rc; - } - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:query sending to server"); - PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, - msg, query_cbfunc, (void*)cd); - if (PMIX_SUCCESS != rc) { - PMIX_RELEASE(cd); - } + rc = pmix_host_server.query(&pmix_globals.myid, + queries, nqueries, + cbfunc, cbdata); + return rc; + } + + /* if we aren't connected, don't attempt to send */ + if (!pmix_globals.connected) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_UNREACH; + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + + /* if we are a client, then relay this request to the server */ + cd = PMIX_NEW(pmix_query_caddy_t); + cd->cbfunc = cbfunc; + cd->cbdata = cbdata; + msg = PMIX_NEW(pmix_buffer_t); + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + msg, &cmd, 1, PMIX_COMMAND); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + PMIX_RELEASE(cd); + return rc; + } + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + msg, &nqueries, 1, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + PMIX_RELEASE(cd); + return rc; + } + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + msg, queries, nqueries, PMIX_QUERY); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + PMIX_RELEASE(cd); + return rc; + } + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:query sending to server"); + PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, + msg, query_cbfunc, (void*)cd); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(cd); } return rc; } @@ -192,13 +340,18 @@ PMIX_EXPORT pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t dire pmix_output_verbose(2, pmix_globals.debug_output, "pmix: allocate called"); + PMIX_ACQUIRE_THREAD(&pmix_global_lock); + if (pmix_globals.init_cntr <= 0) { + PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; } /* if we are the server, then we just issue the request and * return the response */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + PMIX_RELEASE_THREAD(&pmix_global_lock); if (NULL == pmix_host_server.allocate) { /* nothing we can do */ return PMIX_ERR_NOT_SUPPORTED; @@ -216,8 +369,10 @@ PMIX_EXPORT pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t dire /* if we aren't connected, don't attempt to send */ if (!pmix_globals.connected) { + PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_UNREACH; } + PMIX_RELEASE_THREAD(&pmix_global_lock); msg = PMIX_NEW(pmix_buffer_t); /* pack the cmd */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_strings.c b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_strings.c index 98e6609d8d6..5862c84cfd1 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_strings.c +++ b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_strings.c @@ -11,7 +11,9 @@ * All rights reserved. * Copyright (c) 2007-2012 Los Alamos National Security, LLC. * All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -139,6 +141,8 @@ PMIX_EXPORT const char* PMIx_Data_range_string(pmix_data_range_t range) return "AVAIL TO ANYONE WITH AUTHORIZATION"; case PMIX_RANGE_CUSTOM: return "AVAIL AS SPECIFIED IN DIRECTIVES"; + case PMIX_RANGE_PROC_LOCAL: + return "AVAIL ON LOCAL PROC ONLY"; default: return "UNKNOWN"; } @@ -206,6 +210,14 @@ PMIX_EXPORT const char* pmix_command_string(pmix_cmd_t cmd) return "DEREGISTER EVENT HANDLER"; case PMIX_QUERY_CMD: return "QUERY"; + case PMIX_LOG_CMD: + return "LOG"; + case PMIX_ALLOC_CMD: + return "ALLOCATE"; + case PMIX_JOB_CONTROL_CMD: + return "JOB CONTROL"; + case PMIX_MONITOR_CMD: + return "MONITOR"; default: return "UNKNOWN"; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event.h b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event.h index 1cd7d3fe719..c3d7f8a55e1 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event.h +++ b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event.h @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -172,63 +172,84 @@ pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status, void pmix_event_timeout_cb(int fd, short flags, void *arg); -#define PMIX_REPORT_EVENT(e, p, r, f) \ - do { \ - pmix_event_chain_t *ch, *cp; \ - size_t n, ninfo; \ - pmix_info_t *info; \ - pmix_proc_t proc; \ - \ - ch = NULL; \ - /* see if we already have this event cached */ \ - PMIX_LIST_FOREACH(cp, &pmix_globals.cached_events, pmix_event_chain_t) { \ - if (cp->status == (e)) { \ - ch = cp; \ - break; \ - } \ - } \ - if (NULL == ch) { \ - /* nope - need to add it */ \ - ch = PMIX_NEW(pmix_event_chain_t); \ - ch->status = (e); \ - ch->range = (r); \ - (void)strncpy(ch->source.nspace, \ - (p)->nptr->nspace, \ - PMIX_MAX_NSLEN); \ - ch->source.rank = (p)->info->pname.rank; \ - ch->ninfo = 0; \ - ch->nallocated = 2; \ - ch->final_cbfunc = (f); \ - ch->final_cbdata = ch; \ - PMIX_INFO_CREATE(ch->info, ch->nallocated); \ - /* cache it */ \ - pmix_list_append(&pmix_globals.cached_events, &ch->super); \ - ch->timer_active = true; \ - pmix_event_assign(&ch->ev, pmix_globals.evbase, -1, 0, \ - pmix_event_timeout_cb, ch); \ - PMIX_POST_OBJECT(ch); \ - pmix_event_add(&ch->ev, &pmix_globals.event_window); \ - } else { \ - /* add this peer to the array of sources */ \ - (void)strncpy(proc.nspace, (p)->nptr->nspace, PMIX_MAX_NSLEN); \ - proc.rank = (p)->info->pname.rank; \ - ninfo = ch->nallocated + 1; \ - PMIX_INFO_CREATE(info, ninfo); \ - /* must keep the hdlr name and return object at the end, so prepend */ \ - PMIX_INFO_LOAD(&info[0], PMIX_PROCID, \ - &proc, PMIX_PROC); \ - for (n=0; n < ch->ninfo; n++) { \ - PMIX_INFO_XFER(&info[n+1], &ch->info[n]); \ - } \ - PMIX_INFO_FREE(ch->info, ch->nallocated); \ - ch->nallocated = ninfo; \ - ch->info = info; \ - ch->ninfo = ninfo - 2; \ - /* reset the timer */ \ - pmix_event_del(&ch->ev); \ - PMIX_POST_OBJECT(ch); \ - pmix_event_add(&ch->ev, &pmix_globals.event_window); \ - } \ +#define PMIX_REPORT_EVENT(e, p, r, f) \ + do { \ + pmix_event_chain_t *ch, *cp; \ + size_t n, ninfo; \ + pmix_info_t *info; \ + pmix_proc_t proc; \ + \ + ch = NULL; \ + /* see if we already have this event cached */ \ + PMIX_LIST_FOREACH(cp, &pmix_globals.cached_events, pmix_event_chain_t) { \ + if (cp->status == (e)) { \ + ch = cp; \ + break; \ + } \ + } \ + if (NULL == ch) { \ + /* nope - need to add it */ \ + ch = PMIX_NEW(pmix_event_chain_t); \ + ch->status = (e); \ + ch->range = (r); \ + PMIX_LOAD_PROCID(&ch->source, (p)->nptr->nspace, \ + (p)->info->pname.rank); \ + PMIX_PROC_CREATE(ch->affected, 1); \ + ch->naffected = 1; \ + PMIX_LOAD_PROCID(ch->affected, (p)->nptr->nspace, \ + (p)->info->pname.rank); \ + /* if I'm a client or tool and this is my server, then we don't */ \ + /* set the targets - otherwise, we do */ \ + if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && \ + !PMIX_CHECK_PROCID(&pmix_client_globals.myserver->info->pname, \ + &(p)->info->pname)) { \ + PMIX_PROC_CREATE(ch->targets, 1); \ + ch->ntargets = 1; \ + PMIX_LOAD_PROCID(ch->targets, (p)->nptr->nspace, PMIX_RANK_WILDCARD); \ + } \ + /* if this is lost-connection-to-server, then we let it go to */ \ + /* the default event handler - otherwise, we don't */ \ + if (PMIX_ERR_LOST_CONNECTION_TO_SERVER != (e) && \ + PMIX_ERR_UNREACH != (e)) { \ + ch->ninfo = 1; \ + ch->nallocated = 3; \ + PMIX_INFO_CREATE(ch->info, ch->nallocated); \ + /* mark for non-default handlers only */ \ + PMIX_INFO_LOAD(&ch->info[0], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL); \ + } else { \ + ch->nallocated = 2; \ + PMIX_INFO_CREATE(ch->info, ch->nallocated); \ + } \ + ch->final_cbfunc = (f); \ + ch->final_cbdata = ch; \ + /* cache it */ \ + pmix_list_append(&pmix_globals.cached_events, &ch->super); \ + ch->timer_active = true; \ + pmix_event_assign(&ch->ev, pmix_globals.evbase, -1, 0, \ + pmix_event_timeout_cb, ch); \ + PMIX_POST_OBJECT(ch); \ + pmix_event_add(&ch->ev, &pmix_globals.event_window); \ + } else { \ + /* add this peer to the array of sources */ \ + pmix_strncpy(proc.nspace, (p)->nptr->nspace, PMIX_MAX_NSLEN); \ + proc.rank = (p)->info->pname.rank; \ + ninfo = ch->nallocated + 1; \ + PMIX_INFO_CREATE(info, ninfo); \ + /* must keep the hdlr name and return object at the end, so prepend */ \ + PMIX_INFO_LOAD(&info[0], PMIX_PROCID, \ + &proc, PMIX_PROC); \ + for (n=0; n < ch->ninfo; n++) { \ + PMIX_INFO_XFER(&info[n+1], &ch->info[n]); \ + } \ + PMIX_INFO_FREE(ch->info, ch->nallocated); \ + ch->nallocated = ninfo; \ + ch->info = info; \ + ch->ninfo = ninfo - 2; \ + /* reset the timer */ \ + pmix_event_del(&ch->ev); \ + PMIX_POST_OBJECT(ch); \ + pmix_event_add(&ch->ev, &pmix_globals.event_window); \ + } \ } while(0) diff --git a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c index 554bb333faa..574607ec4b0 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c +++ b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c @@ -1,8 +1,8 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2017 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * * $COPYRIGHT$ @@ -39,7 +39,7 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, const pmix_proc_t *source, pmix_data_range_t range, - pmix_info_t info[], size_t ninfo, + const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; @@ -51,17 +51,20 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, return PMIX_ERR_INIT; } - - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { PMIX_RELEASE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix_server_notify_event source = %s:%d event_status = %d", + + pmix_output_verbose(2, pmix_server_globals.event_output, + "pmix_server_notify_event source = %s:%d event_status = %s", (NULL == source) ? "UNKNOWN" : source->nspace, - (NULL == source) ? PMIX_RANK_WILDCARD : source->rank, status); + (NULL == source) ? PMIX_RANK_WILDCARD : source->rank, PMIx_Error_string(status)); + rc = pmix_server_notify_client_of_event(status, source, range, info, ninfo, cbfunc, cbdata); - if (PMIX_SUCCESS != rc) { + + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { PMIX_ERROR_LOG(rc); } return rc; @@ -73,7 +76,7 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, return PMIX_ERR_UNREACH; } PMIX_RELEASE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix_client_notify_event source = %s:%d event_status =%d", (NULL == source) ? pmix_globals.myid.nspace : source->nspace, (NULL == source) ? pmix_globals.myid.rank : source->rank, status); @@ -107,6 +110,49 @@ static void notify_event_cbfunc(struct pmix_peer_t *pr, pmix_ptl_hdr_t *hdr, PMIX_RELEASE(cb); } +static pmix_status_t notify_event_cache(pmix_notify_caddy_t *cd) +{ + pmix_status_t rc; + int j; + pmix_notify_caddy_t *pk; + int idx; + time_t etime; + + /* add to our cache */ + rc = pmix_hotel_checkin(&pmix_globals.notifications, cd, &cd->room); + /* if there wasn't room, then search for the longest tenured + * occupant and evict them */ + if (PMIX_SUCCESS != rc) { + etime = 0; + idx = -1; + for (j=0; j < pmix_globals.max_events; j++) { + pmix_hotel_knock(&pmix_globals.notifications, j, (void**)&pk); + if (NULL == pk) { + /* hey, there is room! */ + pmix_hotel_checkin_with_res(&pmix_globals.notifications, cd, &cd->room); + return PMIX_SUCCESS; + } + /* check the age */ + if (0 == j) { + etime = pk->ts; + idx = j; + } else { + if (difftime(pk->ts, etime) < 0) { + etime = pk->ts; + idx = j; + } + } + } + if (0 <= idx) { + /* we found the oldest occupant - evict it */ + pmix_hotel_checkout_and_return_occupant(&pmix_globals.notifications, idx, (void**)&pk); + PMIX_RELEASE(pk); + rc = pmix_hotel_checkin(&pmix_globals.notifications, cd, &cd->room); + } + } + return rc; +} + /* as a client, we pass the notification to our server */ static pmix_status_t notify_server_of_event(pmix_status_t status, const pmix_proc_t *source, @@ -120,11 +166,13 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, pmix_cb_t *cb; pmix_event_chain_t *chain; size_t n; - pmix_notify_caddy_t *cd, *rbout; + pmix_notify_caddy_t *cd; - pmix_output_verbose(2, pmix_globals.debug_output, - "client: notifying server %s:%d of status %s for range %s", + pmix_output_verbose(2, pmix_client_globals.event_output, + "[%s:%d] client: notifying server %s:%d of status %s for range %s", pmix_globals.myid.nspace, pmix_globals.myid.rank, + pmix_client_globals.myserver->info->pname.nspace, + pmix_client_globals.myserver->info->pname.rank, PMIx_Error_string(status), PMIx_Data_range_string(range)); if (PMIX_RANGE_PROC_LOCAL != range) { @@ -171,7 +219,7 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, /* setup for our own local callbacks */ chain = PMIX_NEW(pmix_event_chain_t); chain->status = status; - (void)strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); chain->source.rank = pmix_globals.myid.rank; /* we always leave space for event hdlr name and a callback object */ chain->nallocated = ninfo + 2; @@ -184,10 +232,10 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, cd = PMIX_NEW(pmix_notify_caddy_t); cd->status = status; if (NULL == source) { - (void)strncpy(cd->source.nspace, "UNDEF", PMIX_MAX_NSLEN); + pmix_strncpy(cd->source.nspace, "UNDEF", PMIX_MAX_NSLEN); cd->source.rank = PMIX_RANK_UNDEF; } else { - (void)strncpy(cd->source.nspace, source->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->source.nspace, source->nspace, PMIX_MAX_NSLEN); cd->source.rank = source->rank; } cd->range = range; @@ -199,28 +247,28 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, for (n=0; n < cd->ninfo; n++) { PMIX_INFO_XFER(&cd->info[n], &chain->info[n]); } - if (NULL != chain->targets) { - cd->ntargets = chain->ntargets; - PMIX_PROC_CREATE(cd->targets, cd->ntargets); - memcpy(cd->targets, chain->targets, cd->ntargets * sizeof(pmix_proc_t)); - } - if (NULL != chain->affected) { - cd->naffected = chain->naffected; - PMIX_PROC_CREATE(cd->affected, cd->naffected); - if (NULL == cd->affected) { - cd->naffected = 0; - rc = PMIX_ERR_NOMEM; - goto cleanup; - } - memcpy(cd->affected, chain->affected, cd->naffected * sizeof(pmix_proc_t)); + } + if (NULL != chain->targets) { + cd->ntargets = chain->ntargets; + PMIX_PROC_CREATE(cd->targets, cd->ntargets); + memcpy(cd->targets, chain->targets, cd->ntargets * sizeof(pmix_proc_t)); + } + if (NULL != chain->affected) { + cd->naffected = chain->naffected; + PMIX_PROC_CREATE(cd->affected, cd->naffected); + if (NULL == cd->affected) { + cd->naffected = 0; + rc = PMIX_ERR_NOMEM; + goto cleanup; } + memcpy(cd->affected, chain->affected, cd->naffected * sizeof(pmix_proc_t)); } - - /* add to our cache */ - rbout = pmix_ring_buffer_push(&pmix_globals.notifications, cd); - /* if an older event was bumped, release it */ - if (NULL != rbout) { - PMIX_RELEASE(rbout); + /* cache it */ + rc = notify_event_cache(cd); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(cd); + goto cleanup; } if (PMIX_RANGE_PROC_LOCAL != range && NULL != msg) { @@ -233,9 +281,11 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, cb->cbfunc.opfn = cbfunc; cb->cbdata = cbdata; /* send to the server */ - pmix_output_verbose(2, pmix_globals.debug_output, - "client: notifying server %s:%d - sending", - pmix_globals.myid.nspace, pmix_globals.myid.rank); + pmix_output_verbose(2, pmix_client_globals.event_output, + "[%s:%d] client: notifying server %s:%d - sending", + pmix_globals.myid.nspace, pmix_globals.myid.rank, + pmix_client_globals.myserver->info->pname.nspace, + pmix_client_globals.myserver->info->pname.rank); PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg, notify_event_cbfunc, cb); if (PMIX_SUCCESS != rc) { @@ -243,7 +293,7 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, PMIX_RELEASE(cb); goto cleanup; } - } else { + } else if (NULL != cbfunc) { cbfunc(PMIX_SUCCESS, cbdata); } @@ -253,7 +303,7 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, return PMIX_SUCCESS; cleanup: - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "client: notifying server - unable to send"); if (NULL != msg) { PMIX_RELEASE(msg); @@ -305,9 +355,9 @@ static void progress_local_event_hdlr(pmix_status_t status, /* save this handler's returned status */ if (NULL != chain->evhdlr->name) { - (void)strncpy(newinfo[cnt].key, chain->evhdlr->name, PMIX_MAX_KEYLEN); + pmix_strncpy(newinfo[cnt].key, chain->evhdlr->name, PMIX_MAX_KEYLEN); } else { - (void)strncpy(newinfo[cnt].key, "UNKNOWN", PMIX_MAX_KEYLEN); + pmix_strncpy(newinfo[cnt].key, "UNKNOWN", PMIX_MAX_KEYLEN); } newinfo[cnt].value.type = PMIX_STATUS; newinfo[cnt].value.data.status = status; @@ -569,7 +619,7 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) pmix_status_t rc = PMIX_SUCCESS; bool found; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "%s:%d invoke_local_event_hdlr for status %s", pmix_globals.myid.nspace, pmix_globals.myid.rank, PMIx_Error_string(chain->status)); @@ -709,6 +759,8 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) /* we still have to call their final callback */ if (NULL != chain->final_cbfunc) { chain->final_cbfunc(rc, chain->final_cbdata); + } else { + PMIX_RELEASE(chain); } return; @@ -730,7 +782,7 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) } /* invoke the handler */ - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "[%s:%d] INVOKING EVHDLR %s", __FILE__, __LINE__, (NULL == chain->evhdlr->name) ? "NULL" : chain->evhdlr->name); @@ -755,22 +807,24 @@ static void local_cbfunc(pmix_status_t status, void *cbdata) static void _notify_client_event(int sd, short args, void *cbdata) { pmix_notify_caddy_t *cd = (pmix_notify_caddy_t*)cbdata; - pmix_notify_caddy_t *rbout; pmix_regevents_info_t *reginfoptr; pmix_peer_events_info_t *pr; pmix_event_chain_t *chain; - size_t n; + size_t n, nleft; bool matched, holdcd; pmix_buffer_t *bfr; pmix_cmd_t cmd = PMIX_NOTIFY_CMD; pmix_status_t rc; pmix_list_t trk; pmix_namelist_t *nm; + pmix_namespace_t *nptr, *tmp; + pmix_range_trkr_t rngtrk; + pmix_proc_t proc; /* need to acquire the object from its originating thread */ PMIX_ACQUIRE_OBJECT(cd); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.event_output, "pmix_server: _notify_client_event notifying clients of event %s range %s type %s", PMIx_Error_string(cd->status), PMIx_Data_range_string(cd->range), @@ -795,11 +849,9 @@ static void _notify_client_event(int sd, short args, void *cbdata) * the message until all local procs have received it, or it ages to * the point where it gets pushed out by more recent events */ PMIX_RETAIN(cd); - rbout = pmix_ring_buffer_push(&pmix_globals.notifications, cd); - - /* if an older event was bumped, release it */ - if (NULL != rbout) { - PMIX_RELEASE(rbout); + rc = notify_event_cache(cd); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); } } @@ -807,7 +859,7 @@ static void _notify_client_event(int sd, short args, void *cbdata) * against our registrations */ chain = PMIX_NEW(pmix_event_chain_t); chain->status = cd->status; - (void)strncpy(chain->source.nspace, cd->source.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(chain->source.nspace, cd->source.nspace, PMIX_MAX_NSLEN); chain->source.rank = cd->source.rank; /* we always leave space for a callback object and * the evhandler name. */ @@ -816,29 +868,52 @@ static void _notify_client_event(int sd, short args, void *cbdata) /* prep the chain for processing */ pmix_prep_event_chain(chain, cd->info, cd->ninfo, true); - if (0 < cd->ninfo) { - /* copy setup to the cd object */ - cd->nondefault = chain->nondefault; - if (NULL != chain->targets) { - cd->ntargets = chain->ntargets; - PMIX_PROC_CREATE(cd->targets, cd->ntargets); - memcpy(cd->targets, chain->targets, cd->ntargets * sizeof(pmix_proc_t)); - } - if (NULL != chain->affected) { - cd->naffected = chain->naffected; - PMIX_PROC_CREATE(cd->affected, cd->naffected); - if (NULL == cd->affected) { - cd->naffected = 0; - /* notify the caller */ - if (NULL != cd->cbfunc) { - cd->cbfunc(PMIX_ERR_NOMEM, cd->cbdata); + /* copy setup to the cd object */ + cd->nondefault = chain->nondefault; + if (NULL != chain->targets) { + cd->ntargets = chain->ntargets; + PMIX_PROC_CREATE(cd->targets, cd->ntargets); + memcpy(cd->targets, chain->targets, cd->ntargets * sizeof(pmix_proc_t)); + /* compute the number of targets that need to be notified */ + nleft = 0; + for (n=0; n < cd->ntargets; n++) { + /* if this is a single proc, then increment by one */ + if (PMIX_RANK_VALID >= cd->targets[n].rank) { + ++nleft; + } else { + /* look up the nspace for this proc */ + nptr = NULL; + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { + if (PMIX_CHECK_NSPACE(tmp->nspace, cd->targets[n].nspace)) { + nptr = tmp; + break; + } } - PMIX_RELEASE(cd); - PMIX_RELEASE(chain); - return; + /* if we don't yet know it, then nothing to do */ + if (NULL == nptr) { + nleft = SIZE_MAX; + break; + } + /* might notify all local members */ + nleft += nptr->nlocalprocs; } - memcpy(cd->affected, chain->affected, cd->naffected * sizeof(pmix_proc_t)); } + cd->nleft = nleft; + } + if (NULL != chain->affected) { + cd->naffected = chain->naffected; + PMIX_PROC_CREATE(cd->affected, cd->naffected); + if (NULL == cd->affected) { + cd->naffected = 0; + /* notify the caller */ + if (NULL != cd->cbfunc) { + cd->cbfunc(PMIX_ERR_NOMEM, cd->cbdata); + } + PMIX_RELEASE(cd); + PMIX_RELEASE(chain); + return; + } + memcpy(cd->affected, chain->affected, cd->naffected * sizeof(pmix_proc_t)); } /* if they provided a PMIX_EVENT_CUSTOM_RANGE info object but @@ -855,10 +930,11 @@ static void _notify_client_event(int sd, short args, void *cbdata) return; } - holdcd = false; if (PMIX_RANGE_PROC_LOCAL != cd->range) { PMIX_CONSTRUCT(&trk, pmix_list_t); + rngtrk.procs = NULL; + rngtrk.nprocs = 0; /* cycle across our registered events and send the message to * any client who registered for it */ PMIX_LIST_FOREACH(reginfoptr, &pmix_server_globals.events, pmix_regevents_info_t) { @@ -882,21 +958,26 @@ static void _notify_client_event(int sd, short args, void *cbdata) if (matched) { continue; } - /* if we were given specific targets, check if this is one */ - if (NULL != cd->targets) { - matched = false; - for (n=0; n < cd->ntargets; n++) { - if (PMIX_CHECK_PROCID(&pr->peer->info->pname, &cd->targets[n])) { - matched = true; - break; - } - } - if (!matched) { - /* do not notify this one */ - continue; - } + /* check if the affected procs (if given) match those they + * wanted to know about */ + if (!pmix_notify_check_affected(cd->affected, cd->naffected, + pr->affected, pr->naffected)) { + continue; } - pmix_output_verbose(2, pmix_globals.debug_output, + /* check the range */ + if (NULL == cd->targets) { + rngtrk.procs = &cd->source; + rngtrk.nprocs = 1; + } else { + rngtrk.procs = cd->targets; + rngtrk.nprocs = cd->ntargets; + } + rngtrk.range = cd->range; + PMIX_LOAD_PROCID(&proc, pr->peer->info->pname.nspace, pr->peer->info->pname.rank); + if (!pmix_notify_check_range(&rngtrk, &proc)) { + continue; + } + pmix_output_verbose(2, pmix_server_globals.event_output, "pmix_server: notifying client %s:%u on status %s", pr->peer->info->pname.nspace, pr->peer->info->pname.rank, PMIx_Error_string(cd->status)); @@ -949,15 +1030,28 @@ static void _notify_client_event(int sd, short args, void *cbdata) continue; } } - PMIX_SERVER_QUEUE_REPLY(pr->peer, 0, bfr); + PMIX_SERVER_QUEUE_REPLY(rc, pr->peer, 0, bfr); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(bfr); + } + if (NULL != cd->targets && 0 < cd->nleft) { + /* track the number of targets we have left to notify */ + --cd->nleft; + /* if the event was cached and this is the last one, + * then evict this event from the cache */ + if (0 == cd->nleft) { + pmix_hotel_checkout(&pmix_globals.notifications, cd->room); + holdcd = false; + break; + } + } } } } PMIX_LIST_DESTRUCT(&trk); if (PMIX_RANGE_LOCAL != cd->range && PMIX_CHECK_PROCID(&cd->source, &pmix_globals.myid)) { /* if we are the source, then we need to post this upwards as - * well so the host RM can broadcast it as necessary - we rely - * on the host RM to _not_ deliver this back to us! */ + * well so the host RM can broadcast it as necessary */ if (NULL != pmix_host_server.notify_event) { /* mark that we sent it upstairs so we don't release * the caddy until we return from the host RM */ @@ -997,27 +1091,31 @@ pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status, pmix_notify_caddy_t *cd; size_t n; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.event_output, "pmix_server: notify client of event %s", PMIx_Error_string(status)); - /* check for prior processing */ - if (NULL != info && PMIX_CHECK_KEY(&info[ninfo], PMIX_SERVER_INTERNAL_NOTIFY)) { - return PMIX_OPERATION_SUCCEEDED; + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (PMIX_CHECK_KEY(&info[n], PMIX_EVENT_PROXY) && + PMIX_CHECK_PROCID(info[n].value.data.proc, &pmix_globals.myid)) { + return PMIX_OPERATION_SUCCEEDED; + } + } } cd = PMIX_NEW(pmix_notify_caddy_t); cd->status = status; if (NULL == source) { - (void)strncpy(cd->source.nspace, "UNDEF", PMIX_MAX_NSLEN); + pmix_strncpy(cd->source.nspace, "UNDEF", PMIX_MAX_NSLEN); cd->source.rank = PMIX_RANK_UNDEF; } else { - (void)strncpy(cd->source.nspace, source->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->source.nspace, source->nspace, PMIX_MAX_NSLEN); cd->source.rank = source->rank; } cd->range = range; /* have to copy the info to preserve it for future when cached */ - if (0 < ninfo) { + if (0 < ninfo && NULL != info) { cd->ninfo = ninfo; PMIX_INFO_CREATE(cd->info, cd->ninfo); /* need to copy the info */ @@ -1030,7 +1128,7 @@ pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status, cd->cbfunc = cbfunc; cd->cbdata = cbdata; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.event_output, "pmix_server_notify_event status =%d, source = %s:%d, ninfo =%lu", status, cd->source.nspace, cd->source.rank, ninfo); @@ -1052,37 +1150,34 @@ bool pmix_notify_check_range(pmix_range_trkr_t *rng, return true; } if (PMIX_RANGE_NAMESPACE == rng->range) { - if (0 == strncmp(pmix_globals.myid.nspace, proc->nspace, PMIX_MAX_NSLEN)) { - return true; + for (n=0; n < rng->nprocs; n++) { + if (PMIX_CHECK_NSPACE(rng->procs[n].nspace, proc->nspace)) { + return true; + } } return false; } if (PMIX_RANGE_PROC_LOCAL == rng->range) { - if (0 == strncmp(pmix_globals.myid.nspace, proc->nspace, PMIX_MAX_NSLEN) && - pmix_globals.myid.rank == proc->rank) { - return true; + for (n=0; n < rng->nprocs; n++) { + if (PMIX_CHECK_PROCID(&rng->procs[n], proc)) { + return true; + } } return false; } if (PMIX_RANGE_CUSTOM == rng->range) { - if (NULL != rng->procs) { - /* see if this proc was included */ - for (n=0; n < rng->nprocs; n++) { - if (0 != strncmp(rng->procs[n].nspace, proc->nspace, PMIX_MAX_NSLEN)) { - continue; - } - if (PMIX_RANK_WILDCARD == rng->procs[n].rank || - rng->procs[n].rank == proc->rank) { - return true; - } + /* see if this proc was included */ + for (n=0; n < rng->nprocs; n++) { + if (0 != strncmp(rng->procs[n].nspace, proc->nspace, PMIX_MAX_NSLEN)) { + continue; + } + if (PMIX_RANK_WILDCARD == rng->procs[n].rank || + rng->procs[n].rank == proc->rank) { + return true; } - /* if we get here, then this proc isn't in range */ - return false; - } else { - /* if they didn't give us a list, then assume - * everyone included */ - return true; } + /* if we get here, then this proc isn't in range */ + return false; } /* if it is anything else, then reject it */ @@ -1105,12 +1200,7 @@ bool pmix_notify_check_affected(pmix_proc_t *interested, size_t ninterested, /* check if the two overlap */ for (n=0; n < naffected; n++) { for (m=0; m < ninterested; m++) { - if (0 != strncmp(affected[n].nspace, interested[m].nspace, PMIX_MAX_NSLEN)) { - continue; - } - if (PMIX_RANK_WILDCARD == interested[m].rank || - PMIX_RANK_WILDCARD == affected[n].rank || - affected[n].rank == interested[m].rank) { + if (PMIX_CHECK_PROCID(&affected[n], &interested[m])) { return true; } } @@ -1133,7 +1223,8 @@ void pmix_event_timeout_cb(int fd, short flags, void *arg) pmix_list_remove_item(&pmix_globals.cached_events, &ch->super); /* process this event thru the regular channels */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { pmix_server_notify_client_of_event(ch->status, &ch->source, ch->range, ch->info, ch->ninfo, ch->final_cbfunc, ch->final_cbdata); diff --git a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c index b2f187c9166..28ad7990ea0 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c +++ b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2017 Research Organization for Information Science + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -40,6 +40,8 @@ size_t ncodes; pmix_info_t *info; size_t ninfo; + pmix_proc_t *affected; + size_t naffected; pmix_notification_fn_t evhdlr; pmix_evhdlr_reg_cbfunc_t evregcbfn; void *cbdata; @@ -55,12 +57,17 @@ static void rscon(pmix_rshift_caddy_t *p) p->ncodes = 0; p->info = NULL; p->ninfo = 0; + p->affected = NULL; + p->naffected = 0; p->evhdlr = NULL; p->evregcbfn = NULL; p->cbdata = NULL; } static void rsdes(pmix_rshift_caddy_t *p) { + if (0 < p->ncodes) { + free(p->codes); + } if (NULL != p->cd) { PMIX_RELEASE(p->cd); } @@ -71,6 +78,8 @@ PMIX_CLASS_INSTANCE(pmix_rshift_caddy_t, static void check_cached_events(pmix_rshift_caddy_t *cd); +/* catch the event registration response message from the + * server and process it */ static void regevents_cbfunc(struct pmix_peer_t *peer, pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { @@ -80,7 +89,7 @@ static void regevents_cbfunc(struct pmix_peer_t *peer, pmix_ptl_hdr_t *hdr, int cnt; size_t index = rb->index; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: regevents callback recvd"); /* unpack the status code */ @@ -88,8 +97,14 @@ static void regevents_cbfunc(struct pmix_peer_t *peer, pmix_ptl_hdr_t *hdr, PMIX_BFROPS_UNPACK(rc, peer, buf, &ret, &cnt, PMIX_STATUS); if ((PMIX_SUCCESS != rc) || (PMIX_SUCCESS != ret)) { - PMIX_ERROR_LOG(rc); - /* remove the err handler and call the error handler reg completion callback fn.*/ + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } else { + PMIX_ERROR_LOG(ret); + } + /* remove the err handler and call the error handler + * reg completion callback fn so the requestor + * doesn't hang */ if (NULL == rb->list) { if (NULL != rb->hdlr) { PMIX_RELEASE(rb->hdlr); @@ -230,7 +245,7 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer) pmix_active_code_t *active; pmix_status_t rc; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: _add_hdlr"); /* check to see if we have an active registration on these codes */ @@ -286,11 +301,7 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer) PMIX_INFO_CREATE(cd2->info, cd2->ninfo); n=0; PMIX_LIST_FOREACH(ixfer, xfer, pmix_info_caddy_t) { - (void)strncpy(cd2->info[n].key, ixfer->info[n].key, PMIX_MAX_KEYLEN); - PMIX_BFROPS_VALUE_LOAD(pmix_client_globals.myserver, - &cd2->info[n].value, - &ixfer->info[n].value.data, - ixfer->info[n].value.type); + PMIX_INFO_XFER(&cd2->info[n], ixfer->info); ++n; } } @@ -299,15 +310,16 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer) * type with our server, or if we have directives, then we need to notify * the server - however, don't do this for a v1 server as the event * notification system there doesn't work */ - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && pmix_globals.connected && + if ((!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) || PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) && + pmix_globals.connected && !PMIX_PROC_IS_V1(pmix_client_globals.myserver) && (need_register || 0 < pmix_list_get_size(xfer))) { - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: _add_hdlr sending to server"); /* send the directives to the server - we will ack this * registration upon return from there */ if (PMIX_SUCCESS != (rc = _send_to_server(cd2))) { - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: add_hdlr - pack send_to_server failed status=%d", rc); if (NULL != cd2->info) { PMIX_INFO_FREE(cd2->info, cd2->ninfo); @@ -320,20 +332,22 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer) /* if we are a server and are registering for events, then we only contact * our host if we want environmental events */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && cd->enviro && + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer) && cd->enviro && NULL != pmix_host_server.register_events) { - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: _add_hdlr registering with server"); - if (PMIX_SUCCESS != (rc = pmix_host_server.register_events(cd->codes, cd->ncodes, - cd2->info, cd2->ninfo, - reg_cbfunc, cd2))) { + rc = pmix_host_server.register_events(cd->codes, cd->ncodes, + cd2->info, cd2->ninfo, + reg_cbfunc, cd2); + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { if (NULL != cd2->info) { PMIX_INFO_FREE(cd2->info, cd2->ninfo); } PMIX_RELEASE(cd2); return rc; } - return PMIX_ERR_WOULD_BLOCK; + return PMIX_SUCCESS; } else { if (NULL != cd2->info) { PMIX_INFO_FREE(cd2->info, cd2->ninfo); @@ -346,19 +360,23 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer) static void check_cached_events(pmix_rshift_caddy_t *cd) { - size_t i, n; + size_t n; pmix_notify_caddy_t *ncd; bool found, matched; pmix_event_chain_t *chain; + int j; - for (i=0; i < (size_t)pmix_globals.notifications.size; i++) { - if (NULL == (ncd = (pmix_notify_caddy_t*)pmix_ring_buffer_poke(&pmix_globals.notifications, i))) { + for (j=0; j < pmix_globals.max_events; j++) { + pmix_hotel_knock(&pmix_globals.notifications, j, (void**)&ncd); + if (NULL == ncd) { continue; } found = false; if (NULL == cd->codes) { - /* they registered a default event handler - always matches */ - found = true; + if (!ncd->nondefault) { + /* they registered a default event handler - always matches */ + found = true; + } } else { for (n=0; n < cd->ncodes; n++) { if (cd->codes[n] == ncd->status) { @@ -367,49 +385,74 @@ static void check_cached_events(pmix_rshift_caddy_t *cd) } } } - if (found) { - /* if we were given specific targets, check if we are one */ - if (NULL != ncd->targets) { - matched = false; - for (n=0; n < ncd->ntargets; n++) { - if (0 != strncmp(pmix_globals.myid.nspace, ncd->targets[n].nspace, PMIX_MAX_NSLEN)) { - continue; - } - if (PMIX_RANK_WILDCARD == ncd->targets[n].rank || - pmix_globals.myid.rank == ncd->targets[n].rank) { - matched = true; - break; - } - } - if (!matched) { - /* do not notify this one */ - continue; + if (!found) { + continue; + } + /* if we were given specific targets, check if we are one */ + if (NULL != ncd->targets) { + matched = false; + for (n=0; n < ncd->ntargets; n++) { + if (PMIX_CHECK_PROCID(&pmix_globals.myid, &ncd->targets[n])) { + matched = true; + break; } } - /* all matches - notify */ - chain = PMIX_NEW(pmix_event_chain_t); - chain->status = ncd->status; - (void)strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); - chain->source.rank = pmix_globals.myid.rank; - /* we always leave space for event hdlr name and a callback object */ - chain->nallocated = ncd->ninfo + 2; - PMIX_INFO_CREATE(chain->info, chain->nallocated); - if (0 < cd->ninfo) { - chain->ninfo = ncd->ninfo; - /* need to copy the info */ - for (n=0; n < ncd->ninfo; n++) { - PMIX_INFO_XFER(&chain->info[n], &ncd->info[n]); - if (0 == strncmp(chain->info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) { - chain->nondefault = true; + if (!matched) { + /* do not notify this one */ + continue; + } + } + /* if they specified affected proc(s) they wanted to know about, check */ + if (!pmix_notify_check_affected(cd->affected, cd->naffected, + ncd->affected, ncd->naffected)) { + continue; + } + /* create the chain */ + chain = PMIX_NEW(pmix_event_chain_t); + chain->status = ncd->status; + pmix_strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + chain->source.rank = pmix_globals.myid.rank; + /* we always leave space for event hdlr name and a callback object */ + chain->nallocated = ncd->ninfo + 2; + PMIX_INFO_CREATE(chain->info, chain->nallocated); + if (0 < cd->ninfo) { + chain->ninfo = ncd->ninfo; + /* need to copy the info */ + for (n=0; n < ncd->ninfo; n++) { + PMIX_INFO_XFER(&chain->info[n], &ncd->info[n]); + if (0 == strncmp(ncd->info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) { + chain->nondefault = true; + } else if (0 == strncmp(ncd->info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + PMIX_PROC_CREATE(chain->affected, 1); + if (NULL == chain->affected) { + PMIX_RELEASE(chain); + return; } + chain->naffected = 1; + memcpy(chain->affected, ncd->info[n].value.data.proc, sizeof(pmix_proc_t)); + } else if (0 == strncmp(ncd->info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) { + chain->naffected = ncd->info[n].value.data.darray->size; + PMIX_PROC_CREATE(chain->affected, chain->naffected); + if (NULL == chain->affected) { + chain->naffected = 0; + PMIX_RELEASE(chain); + return; + } + memcpy(chain->affected, ncd->info[n].value.data.darray->array, chain->naffected * sizeof(pmix_proc_t)); } } - /* we don't want this chain to propagate, so indicate it - * should only be run as a single-shot */ - chain->endchain = true; - /* now notify any matching registered callbacks we have */ - pmix_invoke_local_event_hdlr(chain); } + /* check this event out of the cache since we + * are processing it */ + pmix_hotel_checkout(&pmix_globals.notifications, ncd->room); + /* release the storage */ + PMIX_RELEASE(ncd); + + /* we don't want this chain to propagate, so indicate it + * should only be run as a single-shot */ + chain->endchain = true; + /* now notify any matching registered callbacks we have */ + pmix_invoke_local_event_hdlr(chain); } } @@ -433,7 +476,7 @@ static void reg_event_hdlr(int sd, short args, void *cbdata) /* need to acquire the object from its originating thread */ PMIX_ACQUIRE_OBJECT(cd); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: register event_hdlr with %d infos", (int)cd->ninfo); PMIX_CONSTRUCT(&xfer, pmix_list_t); @@ -459,8 +502,6 @@ static void reg_event_hdlr(int sd, short args, void *cbdata) } } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) { name = cd->info[n].value.data.string; - } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_ENVIRO_LEVEL, PMIX_MAX_KEYLEN)) { - cd->enviro = PMIX_INFO_TRUE(&cd->info[n]); } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { cbobject = cd->info[n].value.data.ptr; } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_HDLR_FIRST_IN_CATEGORY, PMIX_MAX_KEYLEN)) { @@ -482,14 +523,37 @@ static void reg_event_hdlr(int sd, short args, void *cbdata) } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_CUSTOM_RANGE, PMIX_MAX_KEYLEN)) { parray = (pmix_proc_t*)cd->info[n].value.data.darray->array; nprocs = cd->info[n].value.data.darray->size; + } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + cd->affected = cd->info[n].value.data.proc; + cd->naffected = 1; + ixfer = PMIX_NEW(pmix_info_caddy_t); + ixfer->info = &cd->info[n]; + ixfer->ninfo = 1; + pmix_list_append(&xfer, &ixfer->super); + } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) { + cd->affected = (pmix_proc_t*)cd->info[n].value.data.darray->array; + cd->naffected = cd->info[n].value.data.darray->size; + ixfer = PMIX_NEW(pmix_info_caddy_t); + ixfer->info = &cd->info[n]; + ixfer->ninfo = 1; + pmix_list_append(&xfer, &ixfer->super); } else { ixfer = PMIX_NEW(pmix_info_caddy_t); ixfer->info = &cd->info[n]; + ixfer->ninfo = 1; pmix_list_append(&xfer, &ixfer->super); } } } + /* check the codes for system events */ + for (n=0; n < cd->ncodes; n++) { + if (PMIX_SYSTEM_EVENT(cd->codes[n])) { + cd->enviro = true; + break; + } + } + /* if they indicated this is to be the "first" or "last" event, then * first check to ensure they didn't already direct some * other event into the same cherished position */ @@ -525,6 +589,17 @@ static void reg_event_hdlr(int sd, short args, void *cbdata) } memcpy(evhdlr->rng.procs, parray, nprocs * sizeof(pmix_proc_t)); } + if (NULL != cd->affected && 0 < cd->naffected) { + evhdlr->naffected = cd->naffected; + PMIX_PROC_CREATE(evhdlr->affected, cd->naffected); + if (NULL == evhdlr->affected) { + index = UINT_MAX; + rc = PMIX_ERR_EVENT_REGISTRATION; + PMIX_RELEASE(evhdlr); + goto ack; + } + memcpy(evhdlr->affected, cd->affected, cd->naffected * sizeof(pmix_proc_t)); + } evhdlr->evhdlr = cd->evhdlr; evhdlr->cbobject = cbobject; if (NULL != cd->codes) { @@ -599,6 +674,17 @@ static void reg_event_hdlr(int sd, short args, void *cbdata) } memcpy(evhdlr->rng.procs, parray, nprocs * sizeof(pmix_proc_t)); } + if (NULL != cd->affected && 0 < cd->naffected) { + evhdlr->naffected = cd->naffected; + PMIX_PROC_CREATE(evhdlr->affected, cd->naffected); + if (NULL == evhdlr->affected) { + index = UINT_MAX; + rc = PMIX_ERR_EVENT_REGISTRATION; + PMIX_RELEASE(evhdlr); + goto ack; + } + memcpy(evhdlr->affected, cd->affected, cd->naffected * sizeof(pmix_proc_t)); + } evhdlr->evhdlr = cd->evhdlr; evhdlr->cbobject = cbobject; if (NULL == cd->codes) { @@ -752,7 +838,7 @@ static void reg_event_hdlr(int sd, short args, void *cbdata) cd->evregcbfn(rc, index, cd->cbdata); } - /* check if any matching notifications have been cached */ + /* check if any matching notifications have been locally cached */ check_cached_events(cd); if (NULL != cd->codes) { free(cd->codes); @@ -810,7 +896,7 @@ PMIX_EXPORT void PMIx_Register_event_handler(pmix_status_t codes[], size_t ncode cd->evregcbfn = cbfunc; cd->cbdata = cbdata; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix_register_event_hdlr shifting to progress thread"); PMIX_THREADSHIFT(cd, reg_event_hdlr); @@ -832,7 +918,8 @@ static void dereg_event_hdlr(int sd, short args, void *cbdata) /* if I am not the server, and I am connected, then I need * to notify the server to remove my registration */ - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && pmix_globals.connected) { + if ((!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) || PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) && + pmix_globals.connected) { msg = PMIX_NEW(pmix_buffer_t); PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, &cmd, 1, PMIX_COMMAND); @@ -1025,7 +1112,7 @@ PMIX_EXPORT void PMIx_Deregister_event_handler(size_t event_hdlr_ref, cd->cbdata = cbdata; cd->ref = event_hdlr_ref; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix_deregister_event_hdlr shifting to progress thread"); PMIX_THREADSHIFT(cd, dereg_event_hdlr); } diff --git a/opal/mca/pmix/pmix2x/pmix/src/include/Makefile.include b/opal/mca/pmix/pmix2x/pmix/src/include/Makefile.include index 140737f3524..44f6d02e0e5 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/include/Makefile.include +++ b/opal/mca/pmix/pmix2x/pmix/src/include/Makefile.include @@ -10,7 +10,7 @@ # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # Copyright (c) 2007-2016 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. @@ -39,7 +39,10 @@ headers += \ include/prefetch.h \ include/types.h \ include/pmix_config_top.h \ - include/pmix_config_bottom.h + include/pmix_config_bottom.h \ + include/pmix_portable_platform.h \ + include/frameworks.h \ + include/pmix_stdatomic.h endif ! PMIX_EMBEDDED_MODE diff --git a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_config_bottom.h b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_config_bottom.h index 328f6a0baba..a571e9964ba 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_config_bottom.h +++ b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_config_bottom.h @@ -13,7 +13,7 @@ * Copyright (c) 2009-2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013 Mellanox Technologies, Inc. * All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -337,6 +337,18 @@ # define __pmix_attribute_destructor__ #endif +#if PMIX_HAVE_ATTRIBUTE_OPTNONE +# define __pmix_attribute_optnone__ __attribute__((__optnone__)) +#else +# define __pmix_attribute_optnone__ +#endif + +#if PMIX_HAVE_ATTRIBUTE_EXTENSION +# define __pmix_attribute_extension__ __extension__ +#else +# define __pmix_attribute_extension__ +#endif + /* * Do we have ? */ @@ -561,4 +573,5 @@ typedef PMIX_PTRDIFF_TYPE ptrdiff_t; #undef HAVE_CONFIG_H #endif /* PMIX_BUILDING */ + #endif /* PMIX_CONFIG_BOTTOM_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c index 112a331ed84..dc945e7a0c0 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c +++ b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c @@ -1,9 +1,9 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Research Organization for Information Science - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -36,11 +36,25 @@ #endif #include #include PMIX_EVENT_HEADER +#if HAVE_SYS_STAT_H +#include +#endif /* HAVE_SYS_STAT_H */ +#ifdef HAVE_DIRENT_H +#include +#endif /* HAVE_DIRENT_H */ + +#include #include "src/mca/bfrops/bfrops_types.h" #include "src/class/pmix_hash_table.h" #include "src/class/pmix_list.h" #include "src/threads/threads.h" +#include "src/util/argv.h" +#include "src/util/os_path.h" + +static void dirpath_destroy(char *path, pmix_cleanup_dir_t *cd, + pmix_epilog_t *epi); +static bool dirpath_is_empty(const char *path); PMIX_EXPORT pmix_lock_t pmix_global_lock = { .mutex = PMIX_MUTEX_STATIC_INIT, @@ -52,7 +66,37 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_namelist_t, pmix_list_item_t, NULL, NULL); -static void nscon(pmix_nspace_t *p) +static void cfcon(pmix_cleanup_file_t *p) +{ + p->path = NULL; +} +static void cfdes(pmix_cleanup_file_t *p) +{ + if (NULL != p->path) { + free(p->path); + } +} +PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_cleanup_file_t, + pmix_list_item_t, + cfcon, cfdes); + +static void cdcon(pmix_cleanup_dir_t *p) +{ + p->path = NULL; + p->recurse = false; + p->leave_topdir = false; +} +static void cddes(pmix_cleanup_dir_t *p) +{ + if (NULL != p->path) { + free(p->path); + } +} +PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_cleanup_dir_t, + pmix_list_item_t, + cdcon, cddes); + +static void nscon(pmix_namespace_t *p) { p->nspace = NULL; p->nprocs = 0; @@ -61,10 +105,15 @@ static void nscon(pmix_nspace_t *p) p->version_stored = false; p->jobbkt = NULL; p->ndelivered = 0; + p->nfinalized = 0; PMIX_CONSTRUCT(&p->ranks, pmix_list_t); memset(&p->compat, 0, sizeof(p->compat)); + PMIX_CONSTRUCT(&p->epilog.cleanup_dirs, pmix_list_t); + PMIX_CONSTRUCT(&p->epilog.cleanup_files, pmix_list_t); + PMIX_CONSTRUCT(&p->epilog.ignores, pmix_list_t); + PMIX_CONSTRUCT(&p->setup_data, pmix_list_t); } -static void nsdes(pmix_nspace_t *p) +static void nsdes(pmix_namespace_t *p) { if (NULL != p->nspace) { free(p->nspace); @@ -73,8 +122,15 @@ static void nsdes(pmix_nspace_t *p) PMIX_RELEASE(p->jobbkt); } PMIX_LIST_DESTRUCT(&p->ranks); + /* perform any epilog */ + pmix_execute_epilog(&p->epilog); + /* cleanup the epilog */ + PMIX_LIST_DESTRUCT(&p->epilog.cleanup_dirs); + PMIX_LIST_DESTRUCT(&p->epilog.cleanup_files); + PMIX_LIST_DESTRUCT(&p->epilog.ignores); + PMIX_LIST_DESTRUCT(&p->setup_data); } -PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_nspace_t, +PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_namespace_t, pmix_list_item_t, nscon, nsdes); @@ -115,19 +171,24 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_rank_info_t, static void pcon(pmix_peer_t *p) { p->proc_type = PMIX_PROC_UNDEF; + p->protocol = PMIX_PROTOCOL_UNDEF; p->finalized = false; p->info = NULL; p->proc_cnt = 0; p->index = 0; p->sd = -1; - p->finalized = false; p->send_ev_active = false; p->recv_ev_active = false; PMIX_CONSTRUCT(&p->send_queue, pmix_list_t); p->send_msg = NULL; p->recv_msg = NULL; p->commit_cnt = 0; + PMIX_CONSTRUCT(&p->epilog.cleanup_dirs, pmix_list_t); + PMIX_CONSTRUCT(&p->epilog.cleanup_files, pmix_list_t); + PMIX_CONSTRUCT(&p->epilog.ignores, pmix_list_t); + } + static void pdes(pmix_peer_t *p) { if (0 <= p->sd) { @@ -151,11 +212,21 @@ static void pdes(pmix_peer_t *p) if (NULL != p->recv_msg) { PMIX_RELEASE(p->recv_msg); } + /* perform any epilog */ + pmix_execute_epilog(&p->epilog); + /* cleanup the epilog */ + PMIX_LIST_DESTRUCT(&p->epilog.cleanup_dirs); + PMIX_LIST_DESTRUCT(&p->epilog.cleanup_files); + PMIX_LIST_DESTRUCT(&p->epilog.ignores); + if (NULL != p->nptr) { + PMIX_RELEASE(p->nptr); + } } PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_peer_t, pmix_object_t, pcon, pdes); + static void scon(pmix_shift_caddy_t *p) { PMIX_CONSTRUCT_LOCK(&p->lock); @@ -243,6 +314,7 @@ static void qcon(pmix_query_caddy_t *p) p->ntargets = 0; p->info = NULL; p->ninfo = 0; + PMIX_BYTE_OBJECT_CONSTRUCT(&p->bo); p->cbfunc = NULL; p->valcbfunc = NULL; p->cbdata = NULL; @@ -251,7 +323,223 @@ static void qcon(pmix_query_caddy_t *p) static void qdes(pmix_query_caddy_t *p) { PMIX_DESTRUCT_LOCK(&p->lock); + PMIX_BYTE_OBJECT_DESTRUCT(&p->bo); + PMIX_PROC_FREE(p->targets, p->ntargets); + PMIX_INFO_FREE(p->info, p->ninfo); } PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_query_caddy_t, pmix_object_t, qcon, qdes); + +void pmix_execute_epilog(pmix_epilog_t *epi) +{ + pmix_cleanup_file_t *cf, *cfnext; + pmix_cleanup_dir_t *cd, *cdnext; + struct stat statbuf; + int rc; + char **tmp; + size_t n; + + /* start with any specified files */ + PMIX_LIST_FOREACH_SAFE(cf, cfnext, &epi->cleanup_files, pmix_cleanup_file_t) { + /* check the effective uid/gid of the file and ensure it + * matches that of the peer - we do this to provide at least + * some minimum level of protection */ + tmp = pmix_argv_split(cf->path, ','); + for (n=0; NULL != tmp[n]; n++) { + rc = stat(tmp[n], &statbuf); + if (0 != rc) { + pmix_output_verbose(10, pmix_globals.debug_output, + "File %s failed to stat: %d", tmp[n], rc); + continue; + } + if (statbuf.st_uid != epi->uid || + statbuf.st_gid != epi->gid) { + pmix_output_verbose(10, pmix_globals.debug_output, + "File %s uid/gid doesn't match: uid %lu(%lu) gid %lu(%lu)", + cf->path, + (unsigned long)statbuf.st_uid, (unsigned long)epi->uid, + (unsigned long)statbuf.st_gid, (unsigned long)epi->gid); + continue; + } + rc = unlink(tmp[n]); + if (0 != rc) { + pmix_output_verbose(10, pmix_globals.debug_output, + "File %s failed to unlink: %d", tmp[n], rc); + } + } + pmix_argv_free(tmp); + pmix_list_remove_item(&epi->cleanup_files, &cf->super); + PMIX_RELEASE(cf); + } + + /* now cleanup the directories */ + PMIX_LIST_FOREACH_SAFE(cd, cdnext, &epi->cleanup_dirs, pmix_cleanup_dir_t) { + /* check the effective uid/gid of the file and ensure it + * matches that of the peer - we do this to provide at least + * some minimum level of protection */ + tmp = pmix_argv_split(cd->path, ','); + for (n=0; NULL != tmp[n]; n++) { + rc = stat(tmp[n], &statbuf); + if (0 != rc) { + pmix_output_verbose(10, pmix_globals.debug_output, + "Directory %s failed to stat: %d", tmp[n], rc); + continue; + } + if (statbuf.st_uid != epi->uid || + statbuf.st_gid != epi->gid) { + pmix_output_verbose(10, pmix_globals.debug_output, + "Directory %s uid/gid doesn't match: uid %lu(%lu) gid %lu(%lu)", + cd->path, + (unsigned long)statbuf.st_uid, (unsigned long)epi->uid, + (unsigned long)statbuf.st_gid, (unsigned long)epi->gid); + continue; + } + if ((statbuf.st_mode & S_IRWXU) == S_IRWXU) { + dirpath_destroy(tmp[n], cd, epi); + } else { + pmix_output_verbose(10, pmix_globals.debug_output, + "Directory %s lacks permissions", tmp[n]); + } + } + pmix_argv_free(tmp); + pmix_list_remove_item(&epi->cleanup_dirs, &cd->super); + PMIX_RELEASE(cd); + } +} + +static void dirpath_destroy(char *path, pmix_cleanup_dir_t *cd, pmix_epilog_t *epi) +{ + int rc; + bool is_dir = false; + DIR *dp; + struct dirent *ep; + char *filenm; + struct stat buf; + pmix_cleanup_file_t *cf; + + if (NULL == path) { /* protect against error */ + return; + } + + /* if this path is it to be ignored, then do so */ + PMIX_LIST_FOREACH(cf, &epi->ignores, pmix_cleanup_file_t) { + if (0 == strcmp(cf->path, path)) { + return; + } + } + + /* Open up the directory */ + dp = opendir(path); + if (NULL == dp) { + return; + } + + while (NULL != (ep = readdir(dp))) { + /* skip: + * - . and .. + */ + if ((0 == strcmp(ep->d_name, ".")) || + (0 == strcmp(ep->d_name, ".."))) { + continue; + } + + /* Create a pathname. This is not always needed, but it makes + * for cleaner code just to create it here. Note that we are + * allocating memory here, so we need to free it later on. + */ + filenm = pmix_os_path(false, path, ep->d_name, NULL); + + /* if this path is to be ignored, then do so */ + PMIX_LIST_FOREACH(cf, &epi->ignores, pmix_cleanup_file_t) { + if (0 == strcmp(cf->path, filenm)) { + free(filenm); + filenm = NULL; + break; + } + } + if (NULL == filenm) { + continue; + } + + /* Check to see if it is a directory */ + is_dir = false; + + rc = stat(filenm, &buf); + if (0 > rc) { + /* Handle a race condition. filenm might have been deleted by an + * other process running on the same node. That typically occurs + * when one task is removing the job_session_dir and an other task + * is still removing its proc_session_dir. + */ + free(filenm); + continue; + } + /* if the uid/gid don't match, then leave it alone */ + if (buf.st_uid != epi->uid || + buf.st_gid != epi->gid) { + free(filenm); + continue; + } + + if (S_ISDIR(buf.st_mode)) { + is_dir = true; + } + + /* + * If not recursively decending, then if we find a directory then fail + * since we were not told to remove it. + */ + if (is_dir && !cd->recurse) { + /* continue removing files */ + free(filenm); + continue; + } + + /* Directories are recursively destroyed */ + if (is_dir && cd->recurse && ((buf.st_mode & S_IRWXU) == S_IRWXU)) { + dirpath_destroy(filenm, cd, epi); + free(filenm); + } else { + /* Files are removed right here */ + unlink(filenm); + free(filenm); + } + } + + /* Done with this directory */ + closedir(dp); + + /* If the directory is empty, then remove it unless we + * were told to leave it */ + if (0 == strcmp(path, cd->path) && cd->leave_topdir) { + return; + } + if (dirpath_is_empty(path)) { + rmdir(path); + } +} + +static bool dirpath_is_empty(const char *path ) +{ + DIR *dp; + struct dirent *ep; + + if (NULL != path) { /* protect against error */ + dp = opendir(path); + if (NULL != dp) { + while ((ep = readdir(dp))) { + if ((0 != strcmp(ep->d_name, ".")) && + (0 != strcmp(ep->d_name, ".."))) { + closedir(dp); + return false; + } + } + closedir(dp); + return true; + } + return false; + } + + return true; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h index fe63d3e8c42..d0048d69478 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h +++ b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,11 +31,12 @@ #endif #include PMIX_EVENT_HEADER +#include #include #include "src/class/pmix_hash_table.h" #include "src/class/pmix_list.h" -#include "src/class/pmix_ring_buffer.h" +#include "src/class/pmix_hotel.h" #include "src/event/pmix_event.h" #include "src/threads/threads.h" @@ -47,14 +48,20 @@ BEGIN_C_DECLS /* some limits */ -#define PMIX_MAX_CRED_SIZE 131072 // set max at 128kbytes #define PMIX_MAX_ERR_CONSTANT INT_MIN +#define PMIX_MAX_CRED_SIZE 131072 // set max at 128kbytes /* internal-only attributes */ #define PMIX_BFROPS_MODULE "pmix.bfrops.mod" // (char*) name of bfrops plugin in-use by a given nspace #define PMIX_PNET_SETUP_APP "pmix.pnet.setapp" // (pmix_byte_object_t) blob containing info to be given to // pnet framework on remote nodes +#define PMIX_INFO_OP_COMPLETE 0x80000000 +#define PMIX_INFO_OP_COMPLETED(m) \ + ((pmix_info_t*)(m))->flags |= PMIX_INFO_OP_COMPLETE +#define PMIX_INFO_OP_IS_COMPLETE(m) \ + ((m)->flags & PMIX_INFO_OP_COMPLETE) + /* define an internal-only process name that has * a dynamically-sized nspace field to save memory */ typedef struct { @@ -75,26 +82,26 @@ PMIX_CLASS_DECLARATION(pmix_namelist_t); typedef uint8_t pmix_cmd_t; /* define some commands */ -#define PMIX_REQ_CMD 0 -#define PMIX_ABORT_CMD 1 -#define PMIX_COMMIT_CMD 2 -#define PMIX_FENCENB_CMD 3 -#define PMIX_GETNB_CMD 4 -#define PMIX_FINALIZE_CMD 5 -#define PMIX_PUBLISHNB_CMD 6 -#define PMIX_LOOKUPNB_CMD 7 -#define PMIX_UNPUBLISHNB_CMD 8 -#define PMIX_SPAWNNB_CMD 9 -#define PMIX_CONNECTNB_CMD 10 -#define PMIX_DISCONNECTNB_CMD 11 -#define PMIX_NOTIFY_CMD 12 -#define PMIX_REGEVENTS_CMD 13 -#define PMIX_DEREGEVENTS_CMD 14 -#define PMIX_QUERY_CMD 15 -#define PMIX_LOG_CMD 16 -#define PMIX_ALLOC_CMD 17 -#define PMIX_JOB_CONTROL_CMD 18 -#define PMIX_MONITOR_CMD 19 +#define PMIX_REQ_CMD 0 +#define PMIX_ABORT_CMD 1 +#define PMIX_COMMIT_CMD 2 +#define PMIX_FENCENB_CMD 3 +#define PMIX_GETNB_CMD 4 +#define PMIX_FINALIZE_CMD 5 +#define PMIX_PUBLISHNB_CMD 6 +#define PMIX_LOOKUPNB_CMD 7 +#define PMIX_UNPUBLISHNB_CMD 8 +#define PMIX_SPAWNNB_CMD 9 +#define PMIX_CONNECTNB_CMD 10 +#define PMIX_DISCONNECTNB_CMD 11 +#define PMIX_NOTIFY_CMD 12 +#define PMIX_REGEVENTS_CMD 13 +#define PMIX_DEREGEVENTS_CMD 14 +#define PMIX_QUERY_CMD 15 +#define PMIX_LOG_CMD 16 +#define PMIX_ALLOC_CMD 17 +#define PMIX_JOB_CONTROL_CMD 18 +#define PMIX_MONITOR_CMD 19 /* provide a "pretty-print" function for cmds */ const char* pmix_command_string(pmix_cmd_t cmd); @@ -123,6 +130,29 @@ typedef struct pmix_personality_t { pmix_gds_base_module_t *gds; } pmix_personality_t; +/* define a set of structs for tracking post-termination cleanup */ +typedef struct pmix_epilog_t { + uid_t uid; + gid_t gid; + pmix_list_t cleanup_dirs; + pmix_list_t cleanup_files; + pmix_list_t ignores; +} pmix_epilog_t; + +typedef struct { + pmix_list_item_t super; + char *path; +} pmix_cleanup_file_t; +PMIX_CLASS_DECLARATION(pmix_cleanup_file_t); + +typedef struct { + pmix_list_item_t super; + char *path; + bool recurse; + bool leave_topdir; +} pmix_cleanup_dir_t; +PMIX_CLASS_DECLARATION(pmix_cleanup_dir_t); + /* objects used by servers for tracking active nspaces */ typedef struct { pmix_list_item_t super; @@ -133,20 +163,25 @@ typedef struct { bool version_stored; // the version string used by this nspace has been stored pmix_buffer_t *jobbkt; // packed version of jobinfo size_t ndelivered; // count of #local clients that have received the jobinfo + size_t nfinalized; // count of #local clients that have finalized pmix_list_t ranks; // list of pmix_rank_info_t for connection support of my clients /* all members of an nspace are required to have the * same personality, but it can differ between nspaces. * Since servers may support clients from multiple nspaces, * track their respective compatibility modules here */ pmix_personality_t compat; -} pmix_nspace_t; -PMIX_CLASS_DECLARATION(pmix_nspace_t); - -/* define a caddy for quickly creating a list of pmix_nspace_t + pmix_epilog_t epilog; // things to do upon termination of all local clients + // from this nspace + pmix_list_t setup_data; // list of pmix_kval_t containing info structs having blobs + // for setting up the local node for this nspace/application +} pmix_namespace_t; +PMIX_CLASS_DECLARATION(pmix_namespace_t); + +/* define a caddy for quickly creating a list of pmix_namespace_t * objects for local, dedicated purposes */ typedef struct { pmix_list_item_t super; - pmix_nspace_t *ns; + pmix_namespace_t *ns; } pmix_nspace_caddy_t; PMIX_CLASS_DECLARATION(pmix_nspace_caddy_t); @@ -162,6 +197,17 @@ typedef struct pmix_rank_info_t { } pmix_rank_info_t; PMIX_CLASS_DECLARATION(pmix_rank_info_t); + +/* define a very simple caddy for dealing with pmix_info_t + * objects when transferring portions of arrays */ +typedef struct { + pmix_list_item_t super; + pmix_info_t *info; + size_t ninfo; +} pmix_info_caddy_t; +PMIX_CLASS_DECLARATION(pmix_info_caddy_t); + + /* object for tracking peers - each peer can have multiple * connections. This can occur if the initial app executes * a fork/exec, and the child initiates its own connection @@ -169,9 +215,10 @@ PMIX_CLASS_DECLARATION(pmix_rank_info_t); * by the socket, not the process nspace/rank */ typedef struct pmix_peer_t { pmix_object_t super; - pmix_nspace_t *nptr; // point to the nspace object for this process + pmix_namespace_t *nptr; // point to the nspace object for this process pmix_rank_info_t *info; pmix_proc_type_t proc_type; + pmix_listener_protocol_t protocol; int proc_cnt; int index; // index into the local clients array on the server int sd; @@ -184,20 +231,12 @@ typedef struct pmix_peer_t { pmix_ptl_send_t *send_msg; /**< current send in progress */ pmix_ptl_recv_t *recv_msg; /**< current recv in progress */ int commit_cnt; + pmix_epilog_t epilog; /**< things to be performed upon + termination of this peer */ } pmix_peer_t; PMIX_CLASS_DECLARATION(pmix_peer_t); -/* define an object for moving a send - * request into the server's event base - * - instanced in pmix_server_ops.c */ -typedef struct { - pmix_list_item_t super; - pmix_ptl_hdr_t hdr; - pmix_peer_t *peer; -} pmix_server_caddy_t; -PMIX_CLASS_DECLARATION(pmix_server_caddy_t); - /* caddy for query requests */ typedef struct { pmix_object_t super; @@ -210,6 +249,7 @@ typedef struct { size_t ntargets; pmix_info_t *info; size_t ninfo; + pmix_byte_object_t bo; pmix_info_cbfunc_t cbfunc; pmix_value_cbfunc_t valcbfunc; pmix_release_cbfunc_t relcbfunc; @@ -221,7 +261,13 @@ PMIX_CLASS_DECLARATION(pmix_query_caddy_t); * - instanced in pmix_server_ops.c */ typedef struct { pmix_list_item_t super; + pmix_event_t ev; + bool event_active; + bool host_called; // tracker has been passed up to host + bool local; // operation is strictly local + char *id; // string identifier for the collective pmix_cmd_t type; + pmix_proc_t pname; bool hybrid; // true if participating procs are from more than one nspace pmix_proc_t *pcs; // copy of the original array of participants size_t npcs; // number of procs in the array @@ -237,9 +283,24 @@ typedef struct { pmix_collect_t collect_type; // whether or not data is to be returned at completion pmix_modex_cbfunc_t modexcbfunc; pmix_op_cbfunc_t op_cbfunc; + void *cbdata; } pmix_server_trkr_t; PMIX_CLASS_DECLARATION(pmix_server_trkr_t); +/* define an object for moving a send + * request into the server's event base and + * dealing with some request timeouts + * - instanced in pmix_server_ops.c */ +typedef struct { + pmix_list_item_t super; + pmix_event_t ev; + bool event_active; + pmix_server_trkr_t *trk; + pmix_ptl_hdr_t hdr; + pmix_peer_t *peer; +} pmix_server_caddy_t; +PMIX_CLASS_DECLARATION(pmix_server_caddy_t); + /**** THREAD-RELATED ****/ /* define a caddy for thread-shifting operations */ typedef struct { @@ -265,9 +326,9 @@ PMIX_CLASS_DECLARATION(pmix_server_trkr_t); bool enviro; union { pmix_release_cbfunc_t relfn; - pmix_evhdlr_reg_cbfunc_t evregcbfn; + pmix_evhdlr_reg_cbfunc_t hdlrregcbfn; pmix_op_cbfunc_t opcbfn; - pmix_evhdlr_reg_cbfunc_t errregcbfn; + pmix_modex_cbfunc_t modexcbfunc; } cbfunc; void *cbdata; size_t ref; @@ -290,7 +351,7 @@ typedef struct { pmix_value_cbfunc_t valuefn; pmix_lookup_cbfunc_t lookupfn; pmix_spawn_cbfunc_t spawnfn; - pmix_evhdlr_reg_cbfunc_t errregfn; + pmix_evhdlr_reg_cbfunc_t hdlrregfn; } cbfunc; size_t errhandler_ref; void *cbdata; @@ -309,14 +370,6 @@ typedef struct { } pmix_cb_t; PMIX_CLASS_DECLARATION(pmix_cb_t); -/* define a very simple caddy for dealing with pmix_info_t - * objects when transferring portions of arrays */ -typedef struct { - pmix_list_item_t super; - pmix_info_t *info; -} pmix_info_caddy_t; -PMIX_CLASS_DECLARATION(pmix_info_caddy_t); - #define PMIX_THREADSHIFT(r, c) \ do { \ pmix_event_assign(&((r)->ev), pmix_globals.evbase, \ @@ -326,18 +379,15 @@ PMIX_CLASS_DECLARATION(pmix_info_caddy_t); } while (0) -#define PMIX_WAIT_FOR_COMPLETION(a) \ - do { \ - while ((a)) { \ - usleep(10); \ - } \ - PMIX_ACQUIRE_OBJECT((a)); \ - } while (0) - typedef struct { pmix_object_t super; pmix_event_t ev; pmix_lock_t lock; + /* timestamp receipt of the notification so we + * can evict the oldest one if we get overwhelmed */ + time_t ts; + /* what room of the hotel they are in */ + int room; pmix_status_t status; pmix_proc_t source; pmix_data_range_t range; @@ -347,6 +397,7 @@ typedef struct { */ pmix_proc_t *targets; size_t ntargets; + size_t nleft; // number of targets left to be notified /* When generating a notification, the originator can * specify the range of procs affected by this event. * For example, when creating a JOB_TERMINATED event, @@ -382,6 +433,8 @@ typedef struct { pmix_peer_t *mypeer; // my own peer object uid_t uid; // my effective uid gid_t gid; // my effective gid + char *hostname; // my hostname + uint32_t nodeid; // my nodeid, if given int pindex; pmix_event_base_t *evbase; bool external_evbase; @@ -391,7 +444,9 @@ typedef struct { bool commits_pending; struct timeval event_window; pmix_list_t cached_events; // events waiting in the window prior to processing - pmix_ring_buffer_t notifications; // ring buffer of pending notifications + int max_events; // size of the notifications hotel + int event_eviction_time; // max time to cache notifications + pmix_hotel_t notifications; // hotel of pending notifications /* processes also need a place where they can store * their own internal data - e.g., data provided by * the user via the store_internal interface, as well @@ -401,6 +456,8 @@ typedef struct { pmix_gds_base_module_t *mygds; } pmix_globals_t; +/* provide access to a function to cleanup epilogs */ +PMIX_EXPORT void pmix_execute_epilog(pmix_epilog_t *ep); PMIX_EXPORT extern pmix_globals_t pmix_globals; PMIX_EXPORT extern pmix_lock_t pmix_global_lock; diff --git a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_portable_platform.h b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_portable_platform.h new file mode 100644 index 00000000000..5d7fa5e08d7 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_portable_platform.h @@ -0,0 +1,401 @@ +/* + * Header file with preprocessor magic to figure out, which compiler the user has been calling! + * + * This code is adapted from the file other/portable_platform.h of GASnet-1.14.0: + * - Ripping out the required parts. + * - Get rid of brackets as it messes up autoconf + * - Delete version tests for older PGI versions (#include "omp.h" not acceptabe) + * - Indent ('#' should be in column 0) + * + * External packages (i.e., romio) depend on top_build_dir/ompi/include, therefore + * although this is not changed in the configure process, this has to be set as + * a .in file... + * --------------------------------------------------------------------------- + */ +#ifndef PMIX_PORTABLE_PLATFORM_H +#define PMIX_PORTABLE_PLATFORM_H + +/* All files in this directory and all sub-directories (except where otherwise noted) + * are subject to the following licensing terms: + * + * --------------------------------------------------------------------------- + * "Copyright (c) 2000-2003 The Regents of the University of California. + * All rights reserved. + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without written agreement is + * hereby granted, provided that the above copyright notice and the following + * two paragraphs appear in all copies of this software. + * + * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT + * OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF + * CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS + * ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATION TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS." + * --------------------------------------------------------------------------- + * + * Please see the license.txt files within the gm-conduit, lapi-conduit and + * vapi-conduit directories for the licensing terms governing those + * contributed components. + * + * The authors/contributors of GASNet include: + * + * Dan Bonachea : + * General infrastructure & documentation + * mpi-conduit + * elan-conduit + * smp-conduit + * udp-conduit + * extended-ref + * template-conduit + * Christian Bell : gm-conduit, shmem-conduit + * Mike Welcome : lapi-conduit, portals-conduit + * Paul H. Hargrove : vapi-conduit, ibv-conduit + * Rajesh Nishtala : collectives, dcmf-conduit + * Parry Husbands (PJRHusbands@lbl.gov): lapi-conduit + * + * For more information about GASNet, visit our home page at: + * http://gasnet.cs.berkeley.edu/ + * Or send email to: + * + * + * Source code contributions (fixes, patches, extensions etc.) should be + * sent to to be reviewed for acceptance into the primary + * distribution. Contributions are most likely to be accepted if they + * are provided as public domain, or under a BSD-style license such as + * the one above. + * + */ +#ifndef _STRINGIFY +#define _STRINGIFY_HELPER(x) #x +#define _STRINGIFY(x) _STRINGIFY_HELPER(x) +#endif + +#if defined(__INTEL_COMPILER) +# define PLATFORM_COMPILER_FAMILYNAME INTEL +# define PLATFORM_COMPILER_FAMILYID 2 +# ifdef __cplusplus +# define PLATFORM_COMPILER_INTEL_CXX 1 +# else +# define PLATFORM_COMPILER_INTEL_C 1 +# endif +# define _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE 19700000 /* year 1970: predates most intel products :) */ +# ifdef __INTEL_COMPILER_BUILD_DATE +# define _PLATFORM_INTEL_COMPILER_BUILD_DATE __INTEL_COMPILER_BUILD_DATE +# else +# define _PLATFORM_INTEL_COMPILER_BUILD_DATE _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE +# endif + /* patch number is a decimal build date: YYYYMMDD */ +# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \ + (((((maj) * 10) | (min)) << 20) | \ + ((pat) < _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE ? \ + _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE : ((pat)-_PLATFORM_COMPILER_INTEL_MIN_BUILDDATE))) +# define PLATFORM_COMPILER_VERSION \ + PLATFORM_COMPILER_VERSION_INT(__INTEL_COMPILER/10, __INTEL_COMPILER/100, _PLATFORM_INTEL_COMPILER_BUILD_DATE) +# define PLATFORM_COMPILER_VERSION_STR \ + _STRINGIFY(__INTEL_COMPILER) "." _STRINGIFY(_PLATFORM_INTEL_COMPILER_BUILD_DATE) + +#elif defined(__PATHSCALE__) +# define PLATFORM_COMPILER_PATHSCALE 1 +# define PLATFORM_COMPILER_FAMILYNAME PATHSCALE +# define PLATFORM_COMPILER_FAMILYID 3 +# ifdef __cplusplus +# define PLATFORM_COMPILER_PATHSCALE_CXX 1 +# else +# define PLATFORM_COMPILER_PATHSCALE_C 1 +# endif +# define PLATFORM_COMPILER_VERSION \ + PLATFORM_COMPILER_VERSION_INT(__PATHCC__,__PATHCC_MINOR__,__PATHCC_PATCHLEVEL__) +# define PLATFORM_COMPILER_VERSION_STR __PATHSCALE__ + +#elif defined(__PGI) +# define PLATFORM_COMPILER_PGI 1 +# define PLATFORM_COMPILER_FAMILYNAME PGI +# define PLATFORM_COMPILER_FAMILYID 4 +# ifdef __cplusplus +# define PLATFORM_COMPILER_PGI_CXX 1 +# else +# define PLATFORM_COMPILER_PGI_C 1 +# endif +# if __PGIC__ == 99 + /* bug 2230: PGI versioning was broken for some platforms in 7.0 + no way to know exact version, but provide something slightly more accurate */ +# define PLATFORM_COMPILER_VERSION 0x070000 +# define PLATFORM_COMPILER_VERSION_STR "7.?-?" +# elif defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__) +# define PLATFORM_COMPILER_VERSION \ + PLATFORM_COMPILER_VERSION_INT(__PGIC__,__PGIC_MINOR__,__PGIC_PATCHLEVEL__) +# define PLATFORM_COMPILER_VERSION_STR \ + _STRINGIFY(__PGIC__) "." _STRINGIFY(__PGIC_MINOR__) "-" _STRINGIFY(__PGIC_PATCHLEVEL__) +# else + /* PGI before 6.1-4 lacks any version ID preprocessor macros - so use this filthy hack */ + /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + * We cannot do these within mpi.h.in, as we should not include ompi.h + * Hopefully, compilers with integrated preprocessors will not analyse code within the #if 0-block + * XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + */ +#if 0 +# ifdef PLATFORM_PGI_IS_ANCIENT + /* Include below might fail for ancient versions lacking this header, but testing shows it + works back to at least 5.1-3 (Nov 2003), and based on docs probably back to 3.2 (Sep 2000) */ +# define PLATFORM_COMPILER_VERSION 0 +# elif defined(__x86_64__) /* bug 1753 - 64-bit omp.h upgrade happenned in <6.0-8,6.1-1) */ +# include "omp.h" +# if defined(_PGOMP_H) + /* 6.1.1 or newer */ +# define PLATFORM_COMPILER_VERSION 0x060101 +# define PLATFORM_COMPILER_VERSION_STR ">=6.1-1" +# else + /* 6.0.8 or older */ +# define PLATFORM_COMPILER_VERSION 0 +# define PLATFORM_COMPILER_VERSION_STR "<=6.0-8" +# endif +# else /* 32-bit omp.h upgrade happenned in <5.2-4,6.0-8 */ +# include "omp.h" +# if defined(_PGOMP_H) + /* 6.0-8 or newer */ +# define PLATFORM_COMPILER_VERSION 0x060008 +# define PLATFORM_COMPILER_VERSION_STR ">=6.0-8" +# else + /* 5.2-4 or older */ +# define PLATFORM_COMPILER_VERSION 0 +# define PLATFORM_COMPILER_VERSION_STR "<=5.2-4" +# endif +# endif +#endif /* 0 */ + /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX */ +# endif + +#elif defined(__xlC__) +# define PLATFORM_COMPILER_XLC 1 +# define PLATFORM_COMPILER_FAMILYNAME XLC +# define PLATFORM_COMPILER_FAMILYID 5 +# ifdef __cplusplus +# define PLATFORM_COMPILER_XLC_CXX 1 +# else +# define PLATFORM_COMPILER_XLC_C 1 +# endif +# define PLATFORM_COMPILER_VERSION __xlC__ +# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \ + ( ((maj) << 8) | ((min) << 4) | (pat) ) + +#elif defined(__DECC) || defined(__DECCXX) +# define PLATFORM_COMPILER_COMPAQ 1 +# define PLATFORM_COMPILER_FAMILYNAME COMPAQ +# define PLATFORM_COMPILER_FAMILYID 6 +# ifdef __cplusplus +# define PLATFORM_COMPILER_COMPAQ_CXX 1 +# else +# define PLATFORM_COMPILER_COMPAQ_C 1 +# endif +# if defined(__DECC_VER) +# define PLATFORM_COMPILER_VERSION __DECC_VER +# elif defined(__DECCXX_VER) +# define PLATFORM_COMPILER_VERSION __DECCXX_VER +# endif + +# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \ + ( ((maj) * 10000000) + ((min) * 100000) + (90000) + (pat) ) + /* 90000 = official ver, 80000 = customer special ver, 60000 = field test ver */ + +#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# define PLATFORM_COMPILER_SUN 1 +# define PLATFORM_COMPILER_FAMILYNAME SUN +# define PLATFORM_COMPILER_FAMILYID 7 +# ifdef __cplusplus +# define PLATFORM_COMPILER_SUN_CXX 1 +# else +# define PLATFORM_COMPILER_SUN_C 1 +# endif +# if defined(__SUNPRO_C) && __SUNPRO_C > 0 +# define PLATFORM_COMPILER_VERSION __SUNPRO_C +# elif defined(__SUNPRO_CC) && __SUNPRO_CC > 0 +# define PLATFORM_COMPILER_VERSION __SUNPRO_CC +# endif +# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \ + ( ((maj) << 8) | ((min) << 4) | (pat) ) + +#elif defined(__HP_cc) || defined(__HP_aCC) +# define PLATFORM_COMPILER_HP 1 +# define PLATFORM_COMPILER_FAMILYNAME HP +# define PLATFORM_COMPILER_FAMILYID 8 +# ifdef __cplusplus +# define PLATFORM_COMPILER_HP_CXX 1 +# else +# define PLATFORM_COMPILER_HP_C 1 +# endif +# if defined(__HP_cc) && __HP_cc > 0 +# define PLATFORM_COMPILER_VERSION __HP_cc +# elif defined(__HP_aCC) && __HP_aCC > 0 +# define PLATFORM_COMPILER_VERSION __HP_aCC +# endif +# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \ + ( ((maj) << 16) | ((min) << 8) | (pat) ) + +#elif defined(_SGI_COMPILER_VERSION) || \ + (defined(_COMPILER_VERSION) && defined(__sgi) && !defined(__GNUC__)) /* 7.3.0 and earlier lack _SGI_COMPILER_VERSION */ +# define PLATFORM_COMPILER_SGI 1 +# define PLATFORM_COMPILER_FAMILYNAME SGI +# define PLATFORM_COMPILER_FAMILYID 9 +# ifdef __cplusplus +# define PLATFORM_COMPILER_SGI_CXX 1 +# else +# define PLATFORM_COMPILER_SGI_C 1 +# endif +# if defined(_SGI_COMPILER_VERSION) && _SGI_COMPILER_VERSION > 0 +# define PLATFORM_COMPILER_VERSION _SGI_COMPILER_VERSION +# elif defined(_COMPILER_VERSION) && _COMPILER_VERSION > 0 +# define PLATFORM_COMPILER_VERSION _COMPILER_VERSION +# endif +# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \ + ( ((maj) << 8) | ((min) << 4) | (pat) ) + +#elif defined(_CRAYC) +# define PLATFORM_COMPILER_CRAY 1 +# define PLATFORM_COMPILER_FAMILYNAME CRAY +# define PLATFORM_COMPILER_FAMILYID 10 +# ifdef __cplusplus +# define PLATFORM_COMPILER_CRAY_CXX 1 +# else +# define PLATFORM_COMPILER_CRAY_C 1 +# endif +# if defined(_RELEASE) && defined(_RELEASE_MINOR) /* X1 and XT */ +# define PLATFORM_COMPILER_VERSION \ + PLATFORM_COMPILER_VERSION_INT(_RELEASE,_RELEASE_MINOR,0) +# elif defined(_RELEASE) /* T3E */ +# define PLATFORM_COMPILER_VERSION \ + PLATFORM_COMPILER_VERSION_INT(_RELEASE,0,0) +# endif +# ifdef _RELEASE_STRING /* X1 and XT */ +# define PLATFORM_COMPILER_VERSION_STR _RELEASE_STRING +# endif + +#elif defined(__KCC) +# define PLATFORM_COMPILER_KAI 1 +# define PLATFORM_COMPILER_FAMILYNAME KAI +# define PLATFORM_COMPILER_FAMILYID 11 +# ifdef __cplusplus +# define PLATFORM_COMPILER_KAI_CXX 1 +# else +# define PLATFORM_COMPILER_KAI_C 1 +# endif + +#elif defined(__MTA__) +# define PLATFORM_COMPILER_MTA 1 +# define PLATFORM_COMPILER_FAMILYNAME MTA +# define PLATFORM_COMPILER_FAMILYID 12 +# ifdef __cplusplus +# define PLATFORM_COMPILER_MTA_CXX 1 +# else +# define PLATFORM_COMPILER_MTA_C 1 +# endif + +#elif defined(_SX) +# define PLATFORM_COMPILER_NECSX 1 +# define PLATFORM_COMPILER_FAMILYNAME NECSX +# define PLATFORM_COMPILER_FAMILYID 13 +# ifdef __cplusplus +# define PLATFORM_COMPILER_NECSX_CXX 1 +# else +# define PLATFORM_COMPILER_NECSX_C 1 +# endif + +#elif defined(_MSC_VER) +# define PLATFORM_COMPILER_MICROSOFT 1 +# define PLATFORM_COMPILER_FAMILYNAME MICROSOFT +# define PLATFORM_COMPILER_FAMILYID 14 +# ifdef __cplusplus +# define PLATFORM_COMPILER_MICROSOFT_CXX 1 +# else +# define PLATFORM_COMPILER_MICROSOFT_C 1 +# endif +# define PLATFORM_COMPILER_VERSION _MSC_VER + +#elif defined(__TINYC__) +# define PLATFORM_COMPILER_TINY 1 +# define PLATFORM_COMPILER_FAMILYNAME TINY +# define PLATFORM_COMPILER_FAMILYID 15 +# ifdef __cplusplus +# define PLATFORM_COMPILER_TINY_CXX 1 +# else +# define PLATFORM_COMPILER_TINY_C 1 +# endif + +#elif defined(__LCC__) +# define PLATFORM_COMPILER_LCC 1 +# define PLATFORM_COMPILER_FAMILYNAME LCC +# define PLATFORM_COMPILER_FAMILYID 16 +# ifdef __cplusplus +# define PLATFORM_COMPILER_LCC_CXX 1 +# else +# define PLATFORM_COMPILER_LCC_C 1 +# endif + +#else /* unknown compiler */ +# define PLATFORM_COMPILER_UNKNOWN 1 +#endif + +/* this stanza comes last, because many vendor compilers lie and claim + to be GNU C for compatibility reasons and/or because they share a frontend */ +#if defined(__GNUC__) +# undef PLATFORM_COMPILER_UNKNOWN +# ifndef PLATFORM_COMPILER_FAMILYID +# define PLATFORM_COMPILER_GNU 1 +# define PLATFORM_COMPILER_FAMILYNAME GNU +# define PLATFORM_COMPILER_FAMILYID 1 +# ifdef __cplusplus +# define PLATFORM_COMPILER_GNU_CXX 1 +# else +# define PLATFORM_COMPILER_GNU_C 1 +# endif +# if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) +# define PLATFORM_COMPILER_VERSION \ + PLATFORM_COMPILER_VERSION_INT(__GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__) +# elif defined(__GNUC_MINOR__) /* older versions of egcs lack __GNUC_PATCHLEVEL__ */ +# define PLATFORM_COMPILER_VERSION \ + PLATFORM_COMPILER_VERSION_INT(__GNUC__,__GNUC_MINOR__,0) +# else +# define PLATFORM_COMPILER_VERSION \ + PLATFORM_COMPILER_VERSION_INT(__GNUC__,0,0) +# endif +# define PLATFORM_COMPILER_VERSION_STR __PLATFORM_COMPILER_GNU_VERSION_STR +# else +# define _PLATFORM_COMPILER_GNU_VERSION_STR __PLATFORM_COMPILER_GNU_VERSION_STR +# endif + /* gather any advertised GNU version number info, even for non-gcc compilers */ +# if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) +# define __PLATFORM_COMPILER_GNU_VERSION_STR \ + _STRINGIFY(__GNUC__) "." _STRINGIFY(__GNUC_MINOR__) "." _STRINGIFY(__GNUC_PATCHLEVEL__) +# elif defined(__GNUC_MINOR__) +# define __PLATFORM_COMPILER_GNU_VERSION_STR \ + _STRINGIFY(__GNUC__) "." _STRINGIFY(__GNUC_MINOR__) ".?" +# else +# define __PLATFORM_COMPILER_GNU_VERSION_STR \ + _STRINGIFY(__GNUC__) ".?.?" +# endif +#elif defined(PLATFORM_COMPILER_UNKNOWN) /* unknown compiler */ +# define PLATFORM_COMPILER_FAMILYNAME UNKNOWN +# define PLATFORM_COMPILER_FAMILYID 0 +#endif + +/* Default Values */ +#ifndef PLATFORM_COMPILER_VERSION +# define PLATFORM_COMPILER_VERSION 0 /* don't know */ +#endif + +#ifndef PLATFORM_COMPILER_VERSION_STR +# define PLATFORM_COMPILER_VERSION_STR _STRINGIFY(PLATFORM_COMPILER_VERSION) +#endif + +#ifndef PLATFORM_COMPILER_VERSION_INT +# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \ + (((maj) << 16) | ((min) << 8) | (pat)) +#endif + + +#endif /* PMIX_PORTABLE_PLATFORM_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_stdatomic.h b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_stdatomic.h new file mode 100644 index 00000000000..8716ceec055 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_stdatomic.h @@ -0,0 +1,67 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#if !defined(PMIX_STDATOMIC_H) +#define PMIX_STDATOMIC_H + +#include "pmix_stdint.h" + +#if PMIX_ASSEMBLY_BUILTIN != PMIX_BUILTIN_C11 + +typedef volatile int pmix_atomic_int_t; +typedef volatile long pmix_atomic_long_t; + +typedef volatile int32_t pmix_atomic_int32_t; +typedef volatile uint32_t pmix_atomic_uint32_t; +typedef volatile int64_t pmix_atomic_int64_t; +typedef volatile uint64_t pmix_atomic_uint64_t; + +typedef volatile size_t pmix_atomic_size_t; +typedef volatile ssize_t pmix_atomic_ssize_t; +typedef volatile intptr_t pmix_atomic_intptr_t; +typedef volatile uintptr_t pmix_atomic_uintptr_t; + +#else /* PMIX_HAVE_C__ATOMIC */ + +#include + +typedef atomic_int pmix_atomic_int_t; +typedef atomic_long pmix_atomic_long_t; + +typedef _Atomic int32_t pmix_atomic_int32_t; +typedef _Atomic uint32_t pmix_atomic_uint32_t; +typedef _Atomic int64_t pmix_atomic_int64_t; +typedef _Atomic uint64_t pmix_atomic_uint64_t; + +typedef _Atomic size_t pmix_atomic_size_t; +typedef _Atomic ssize_t pmix_atomic_ssize_t; +typedef _Atomic intptr_t pmix_atomic_intptr_t; +typedef _Atomic uintptr_t pmix_atomic_uintptr_t; + +#endif /* PMIX_HAVE_C__ATOMIC */ + +#if HAVE_PMIX_INT128_T + +/* do not use C11 atomics for __int128 if they are not lock free */ +#if PMIX_HAVE_C11_CSWAP_INT128 + +typedef _Atomic pmix_int128_t pmix_atomic_int128_t; + +#else + +typedef volatile pmix_int128_t pmix_atomic_int128_t; + +#endif + +#endif + +#endif /* !defined(PMIX_STDATOMIC_H) */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_stdint.h b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_stdint.h index 28c3099ef37..9fe7ad0b20a 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_stdint.h +++ b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_stdint.h @@ -14,7 +14,7 @@ * reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -29,6 +29,8 @@ #ifndef PMIX_STDINT_H #define PMIX_STDINT_H 1 +#include "pmix_config.h" + /* * Include what we can and define what is missing. */ @@ -125,3 +127,4 @@ typedef unsigned long long uintptr_t; #endif #endif /* PMIX_STDINT_H */ + diff --git a/opal/mca/pmix/pmix2x/pmix/src/include/types.h b/opal/mca/pmix/pmix2x/pmix/src/include/types.h index 7c073ccf4f8..ef2ee1cbb2d 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/include/types.h +++ b/opal/mca/pmix/pmix2x/pmix/src/include/types.h @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -232,6 +232,8 @@ typedef struct event pmix_event_t; #define pmix_event_base_free(b) event_base_free(b) +#define pmix_event_free(x) event_free(x) + #define pmix_event_base_loopbreak(b) event_base_loopbreak(b) #define pmix_event_base_loopexit(b) event_base_loopexit(b, NULL) @@ -244,6 +246,8 @@ typedef struct event pmix_event_t; #define pmix_event_assign(x, b, fd, fg, cb, arg) event_assign((x), (b), (fd), (fg), (event_callback_fn) (cb), (arg)) +#define pmix_event_set(b, x, fd, fg, cb, arg) event_assign((x), (b), (fd), (fg), (event_callback_fn) (cb), (arg)) + #define pmix_event_add(ev, tv) event_add((ev), (tv)) #define pmix_event_del(ev) event_del((ev)) @@ -264,5 +268,6 @@ typedef struct event pmix_event_t; #define pmix_event_evtimer_del(x) pmix_event_del((x)) +#define pmix_event_signal_set(b, x, fd, cb, arg) event_assign((x), (b), (fd), EV_SIGNAL|EV_PERSIST, (event_callback_fn) (cb), (arg)) #endif /* PMIX_TYPES_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/base/Makefile.am index 948d687eed2..2881305fffa 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/Makefile.am @@ -10,6 +10,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2016 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2017-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -25,7 +26,7 @@ AM_CPPFLAGS = \ noinst_LTLIBRARIES = libpmix_mca_base.la -dist_pmixdata_DATA = help-mca-base.txt help-mca-var.txt +dist_pmixdata_DATA = help-pmix-mca-base.txt help-pmix-mca-var.txt # Source code files @@ -43,6 +44,7 @@ headers = \ libpmix_mca_base_la_SOURCES = \ $(headers) \ pmix_mca_base_close.c \ + pmix_mca_base_cmd_line.c \ pmix_mca_base_component_compare.c \ pmix_mca_base_component_find.c \ pmix_mca_base_component_repository.c \ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/base.h b/opal/mca/pmix/pmix2x/pmix/src/mca/base/base.h index 0554431d7b9..317253d57bb 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/base.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/base.h @@ -15,7 +15,7 @@ * reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -37,6 +37,7 @@ #include "src/mca/mca.h" #include "src/mca/base/pmix_mca_base_var.h" #include "src/mca/base/pmix_mca_base_framework.h" +#include "src/util/cmd_line.h" #include "src/util/output.h" BEGIN_C_DECLS @@ -49,7 +50,7 @@ struct pmix_mca_base_component_list_item_t { const pmix_mca_base_component_t *cli_component; }; typedef struct pmix_mca_base_component_list_item_t pmix_mca_base_component_list_item_t; -PMIX_CLASS_DECLARATION(pmix_mca_base_component_list_item_t); +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_mca_base_component_list_item_t); /* * Structure for making priority lists of components @@ -61,16 +62,17 @@ struct pmix_mca_base_component_priority_list_item_t { typedef struct pmix_mca_base_component_priority_list_item_t pmix_mca_base_component_priority_list_item_t; -PMIX_CLASS_DECLARATION(pmix_mca_base_component_priority_list_item_t); +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_mca_base_component_priority_list_item_t); /* * Public variables */ -extern char *pmix_mca_base_component_path; -extern bool pmix_mca_base_component_show_load_errors; -extern bool pmix_mca_base_component_disable_dlopen; -extern char *pmix_mca_base_system_default_path; -extern char *pmix_mca_base_user_default_path; +PMIX_EXPORT extern char *pmix_mca_base_component_path; +PMIX_EXPORT extern bool pmix_mca_base_component_show_load_errors; +PMIX_EXPORT extern bool pmix_mca_base_component_track_load_errors; +PMIX_EXPORT extern bool pmix_mca_base_component_disable_dlopen; +PMIX_EXPORT extern char *pmix_mca_base_system_default_path; +PMIX_EXPORT extern char *pmix_mca_base_user_default_path; /* * Standard verbosity levels @@ -114,7 +116,7 @@ enum { * invoked during the initialization stage and specifically * invoked in the special case of the *_info command. */ -int pmix_mca_base_open(void); +PMIX_EXPORT int pmix_mca_base_open(void); /** * Last function called in the MCA @@ -128,17 +130,17 @@ int pmix_mca_base_open(void); * It must be the last MCA function invoked. It is normally invoked * during the finalize stage. */ -int pmix_mca_base_close(void); +PMIX_EXPORT int pmix_mca_base_close(void); /** * A generic select function * */ -int pmix_mca_base_select(const char *type_name, int output_id, - pmix_list_t *components_available, - pmix_mca_base_module_t **best_module, - pmix_mca_base_component_t **best_component, - int *priority_out); +PMIX_EXPORT int pmix_mca_base_select(const char *type_name, int output_id, + pmix_list_t *components_available, + pmix_mca_base_module_t **best_module, + pmix_mca_base_component_t **best_component, + int *priority_out); /** * A function for component query functions to discover if they have @@ -148,32 +150,40 @@ int pmix_mca_base_select(const char *type_name, int output_id, * available for selection. * */ -int pmix_mca_base_is_component_required(pmix_list_t *components_available, - pmix_mca_base_component_t *component, - bool exclusive, - bool *is_required); +PMIX_EXPORT int pmix_mca_base_is_component_required(pmix_list_t *components_available, + pmix_mca_base_component_t *component, + bool exclusive, + bool *is_required); + +/* mca_base_cmd_line.c */ + +PMIX_EXPORT int pmix_mca_base_cmd_line_setup(pmix_cmd_line_t *cmd); +PMIX_EXPORT int pmix_mca_base_cmd_line_process_args(pmix_cmd_line_t *cmd, + char ***app_env, + char ***global_env); +PMIX_EXPORT void pmix_mca_base_cmd_line_wrap_args(char **args); /* pmix_mca_base_component_compare.c */ -int pmix_mca_base_component_compare_priority(pmix_mca_base_component_priority_list_item_t *a, - pmix_mca_base_component_priority_list_item_t *b); -int pmix_mca_base_component_compare(const pmix_mca_base_component_t *a, - const pmix_mca_base_component_t *b); -int pmix_mca_base_component_compatible(const pmix_mca_base_component_t *a, - const pmix_mca_base_component_t *b); -char * pmix_mca_base_component_to_string(const pmix_mca_base_component_t *a); +PMIX_EXPORT int pmix_mca_base_component_compare_priority(pmix_mca_base_component_priority_list_item_t *a, + pmix_mca_base_component_priority_list_item_t *b); +PMIX_EXPORT int pmix_mca_base_component_compare(const pmix_mca_base_component_t *a, + const pmix_mca_base_component_t *b); +PMIX_EXPORT int pmix_mca_base_component_compatible(const pmix_mca_base_component_t *a, + const pmix_mca_base_component_t *b); +PMIX_EXPORT char * pmix_mca_base_component_to_string(const pmix_mca_base_component_t *a); /* pmix_mca_base_component_find.c */ -int pmix_mca_base_component_find (const char *directory, pmix_mca_base_framework_t *framework, - bool ignore_requested, bool open_dso_components); +PMIX_EXPORT int pmix_mca_base_component_find (const char *directory, pmix_mca_base_framework_t *framework, + bool ignore_requested, bool open_dso_components); /** * Parse the requested component string and return an pmix_argv of the requested * (or not requested) components. */ -int pmix_mca_base_component_parse_requested (const char *requested, bool *include_mode, - char ***requested_component_names); +PMIX_EXPORT int pmix_mca_base_component_parse_requested (const char *requested, bool *include_mode, + char ***requested_component_names); /** * Filter a list of components based on a comma-delimted list of names and/or @@ -193,27 +203,27 @@ int pmix_mca_base_component_parse_requested (const char *requested, bool *includ * This function closes and releases any components that do not match the filter_name and * filter flags. */ -int pmix_mca_base_components_filter (pmix_mca_base_framework_t *framework, uint32_t filter_flags); +PMIX_EXPORT int pmix_mca_base_components_filter (pmix_mca_base_framework_t *framework, uint32_t filter_flags); /* Safely release some memory allocated by pmix_mca_base_component_find() (i.e., is safe to call even if you never called pmix_mca_base_component_find()). */ -int pmix_mca_base_component_find_finalize(void); +PMIX_EXPORT int pmix_mca_base_component_find_finalize(void); /* pmix_mca_base_components_register.c */ -int pmix_mca_base_framework_components_register (struct pmix_mca_base_framework_t *framework, - pmix_mca_base_register_flag_t flags); +PMIX_EXPORT int pmix_mca_base_framework_components_register (struct pmix_mca_base_framework_t *framework, + pmix_mca_base_register_flag_t flags); /* pmix_mca_base_components_open.c */ -int pmix_mca_base_framework_components_open (struct pmix_mca_base_framework_t *framework, - pmix_mca_base_open_flag_t flags); +PMIX_EXPORT int pmix_mca_base_framework_components_open (struct pmix_mca_base_framework_t *framework, + pmix_mca_base_open_flag_t flags); -int pmix_mca_base_components_open(const char *type_name, int output_id, - const pmix_mca_base_component_t **static_components, - pmix_list_t *components_available, - bool open_dso_components); +PMIX_EXPORT int pmix_mca_base_components_open(const char *type_name, int output_id, + const pmix_mca_base_component_t **static_components, + pmix_list_t *components_available, + bool open_dso_components); /* pmix_mca_base_components_close.c */ /** @@ -224,7 +234,7 @@ int pmix_mca_base_components_open(const char *type_name, int output_id, * * After calling this function the component may no longer be used. */ -void pmix_mca_base_component_close (const pmix_mca_base_component_t *component, int output_id); +PMIX_EXPORT void pmix_mca_base_component_close (const pmix_mca_base_component_t *component, int output_id); /** * Release a component without closing it. @@ -233,13 +243,13 @@ void pmix_mca_base_component_close (const pmix_mca_base_component_t *component, * * After calling this function the component may no longer be used. */ -void pmix_mca_base_component_unload (const pmix_mca_base_component_t *component, int output_id); +PMIX_EXPORT void pmix_mca_base_component_unload (const pmix_mca_base_component_t *component, int output_id); -int pmix_mca_base_components_close(int output_id, pmix_list_t *components_available, - const pmix_mca_base_component_t *skip); +PMIX_EXPORT int pmix_mca_base_components_close(int output_id, pmix_list_t *components_available, + const pmix_mca_base_component_t *skip); -int pmix_mca_base_framework_components_close (struct pmix_mca_base_framework_t *framework, - const pmix_mca_base_component_t *skip); +PMIX_EXPORT int pmix_mca_base_framework_components_close (struct pmix_mca_base_framework_t *framework, + const pmix_mca_base_component_t *skip); END_C_DECLS diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/help-mca-base.txt b/opal/mca/pmix/pmix2x/pmix/src/mca/base/help-pmix-mca-base.txt similarity index 86% rename from opal/mca/pmix/pmix2x/pmix/src/mca/base/help-mca-base.txt rename to opal/mca/pmix/pmix2x/pmix/src/mca/base/help-pmix-mca-base.txt index c0b82510767..16b8b86ada7 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/help-mca-base.txt +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/help-pmix-mca-base.txt @@ -10,7 +10,8 @@ # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. -# Copyright (c) 2008-2014 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2008-2019 Cisco Systems, Inc. All rights reserved +# Copyright (c) 2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -59,3 +60,12 @@ all components *except* a and b", while "c,d" specifies the inclusive behavior and means "use *only* components c and d." You cannot mix inclusive and exclusive behavior. +# +[failed to add component dir] +The pmix_mca_base_component_path MCA variable was used to add paths to +search for PMIX components. At least one directory failed to add +properly: + + %s + +Check to make sure that this directory exists, is readable, etc. diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/help-mca-var.txt b/opal/mca/pmix/pmix2x/pmix/src/mca/base/help-pmix-mca-var.txt similarity index 98% rename from opal/mca/pmix/pmix2x/pmix/src/mca/base/help-mca-var.txt rename to opal/mca/pmix/pmix2x/pmix/src/mca/base/help-pmix-mca-var.txt index b306c31ff90..37ac4447ec0 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/help-mca-var.txt +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/help-pmix-mca-var.txt @@ -13,6 +13,7 @@ # Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2013 Los Alamos National Security, LLC. All rights # reserved. +# Copyright (c) 2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_cmd_line.c b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_cmd_line.c new file mode 100644 index 00000000000..075304daa71 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_cmd_line.c @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2015 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "pmix_config.h" + +#include +#include + +#include "src/util/cmd_line.h" +#include "src/util/argv.h" +#include "src/util/pmix_environ.h" +#include "src/util/show_help.h" +#include "src/mca/base/base.h" +#include "pmix_common.h" + + +/* + * Private variables + */ + +/* + * Private functions + */ +static int process_arg(const char *param, const char *value, + char ***params, char ***values); +static void add_to_env(char **params, char **values, char ***env); + + +/* + * Add -mca to the possible command line options list + */ +int pmix_mca_base_cmd_line_setup(pmix_cmd_line_t *cmd) +{ + int ret = PMIX_SUCCESS; + + ret = pmix_cmd_line_make_opt3(cmd, '\0', PMIX_MCA_CMD_LINE_ID, PMIX_MCA_CMD_LINE_ID, 2, + "Pass context-specific MCA parameters; they are considered global if --g"PMIX_MCA_CMD_LINE_ID" is not used and only one context is specified (arg0 is the parameter name; arg1 is the parameter value)"); + if (PMIX_SUCCESS != ret) { + return ret; + } + + ret = pmix_cmd_line_make_opt3(cmd, '\0', "g"PMIX_MCA_CMD_LINE_ID, "g"PMIX_MCA_CMD_LINE_ID, 2, + "Pass global MCA parameters that are applicable to all contexts (arg0 is the parameter name; arg1 is the parameter value)"); + + if (PMIX_SUCCESS != ret) { + return ret; + } + + { + pmix_cmd_line_init_t entry = + {"mca_base_param_file_prefix", '\0', "am", NULL, 1, + NULL, PMIX_CMD_LINE_TYPE_STRING, + "Aggregate MCA parameter set file list", + PMIX_CMD_LINE_OTYPE_LAUNCH + }; + ret = pmix_cmd_line_make_opt_mca(cmd, entry); + if (PMIX_SUCCESS != ret) { + return ret; + } + } + + { + pmix_cmd_line_init_t entry = + {"mca_base_envar_file_prefix", '\0', "tune", NULL, 1, + NULL, PMIX_CMD_LINE_TYPE_STRING, + "Application profile options file list", + PMIX_CMD_LINE_OTYPE_DEBUG + }; + ret = pmix_cmd_line_make_opt_mca(cmd, entry); + if (PMIX_SUCCESS != ret) { + return ret; + } + } + + return ret; +} + + +/* + * Look for and handle any -mca options on the command line + */ +int pmix_mca_base_cmd_line_process_args(pmix_cmd_line_t *cmd, + char ***context_env, char ***global_env) +{ + int i, num_insts, rc; + char **params; + char **values; + + /* If no relevant parameters were given, just return */ + + if (!pmix_cmd_line_is_taken(cmd, PMIX_MCA_CMD_LINE_ID) && + !pmix_cmd_line_is_taken(cmd, "g"PMIX_MCA_CMD_LINE_ID)) { + return PMIX_SUCCESS; + } + + /* Handle app context-specific parameters */ + + num_insts = pmix_cmd_line_get_ninsts(cmd, PMIX_MCA_CMD_LINE_ID); + params = values = NULL; + for (i = 0; i < num_insts; ++i) { + if (PMIX_SUCCESS != (rc = process_arg(pmix_cmd_line_get_param(cmd, PMIX_MCA_CMD_LINE_ID, i, 0), + pmix_cmd_line_get_param(cmd, PMIX_MCA_CMD_LINE_ID, i, 1), + ¶ms, &values))) { + return rc; + } + } + if (NULL != params) { + add_to_env(params, values, context_env); + pmix_argv_free(params); + pmix_argv_free(values); + } + + /* Handle global parameters */ + + num_insts = pmix_cmd_line_get_ninsts(cmd, "g"PMIX_MCA_CMD_LINE_ID); + params = values = NULL; + for (i = 0; i < num_insts; ++i) { + if (PMIX_SUCCESS != (rc = process_arg(pmix_cmd_line_get_param(cmd, "g"PMIX_MCA_CMD_LINE_ID, i, 0), + pmix_cmd_line_get_param(cmd, "g"PMIX_MCA_CMD_LINE_ID, i, 1), + ¶ms, &values))) { + return rc; + } + } + if (NULL != params) { + add_to_env(params, values, global_env); + pmix_argv_free(params); + pmix_argv_free(values); + } + + /* All done */ + + return PMIX_SUCCESS; +} + + + +/* + * Process a single MCA argument. + */ +static int process_arg(const char *param, const char *value, + char ***params, char ***values) +{ + int i; + char *p1; + + /* check for quoted value */ + if ('\"' == value[0] && '\"' == value[strlen(value)-1]) { + p1 = strdup(&value[1]); + p1[strlen(p1)-1] = '\0'; + } else { + p1 = strdup(value); + } + + /* Look to see if we've already got an -mca argument for the same + param. Check against the list of MCA param's that we've + already saved arguments for - if found, return an error. */ + + for (i = 0; NULL != *params && NULL != (*params)[i]; ++i) { + if (0 == strcmp(param, (*params)[i])) { + /* cannot use show_help here as it may not get out prior + * to the process exiting */ + fprintf(stderr, + "---------------------------------------------------------------------------\n" + "The following MCA parameter has been listed multiple times on the\n" + "command line:\n\n" + " MCA param: %s\n\n" + "MCA parameters can only be listed once on a command line to ensure there\n" + "is no ambiguity as to its value. Please correct the situation and\n" + "try again.\n" + "---------------------------------------------------------------------------\n", + param); + free(p1); + return PMIX_ERROR; + } + } + + /* If we didn't already have an value for the same param, save + this one away */ + pmix_argv_append_nosize(params, param); + pmix_argv_append_nosize(values, p1); + free(p1); + + return PMIX_SUCCESS; +} + + +static void add_to_env(char **params, char **values, char ***env) +{ + int i; + char *name; + + /* Loop through all the args that we've gotten and make env + vars of the form PMIX_MCA_PREFIX*=value. */ + + for (i = 0; NULL != params && NULL != params[i]; ++i) { + (void) pmix_mca_base_var_env_name (params[i], &name); + pmix_setenv(name, values[i], true, env); + free(name); + } +} + +void pmix_mca_base_cmd_line_wrap_args(char **args) +{ + int i; + char *tstr; + + for (i=0; NULL != args && NULL != args[i]; i++) { + if (0 == strcmp(args[i], "-"PMIX_MCA_CMD_LINE_ID) || + 0 == strcmp(args[i], "--"PMIX_MCA_CMD_LINE_ID)) { + if (NULL == args[i+1] || NULL == args[i+2]) { + /* this should be impossible as the error would + * have been detected well before here, but just + * be safe */ + return; + } + i += 2; + if (0 > asprintf(&tstr, "\"%s\"", args[i])) { + return; + } + free(args[i]); + args[i] = tstr; + } + } +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_component_find.c b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_component_find.c index 981511ee5a5..c9cd23740ed 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_component_find.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_component_find.c @@ -16,7 +16,7 @@ * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -341,7 +341,7 @@ static int component_find_check (pmix_mca_base_framework_t *framework, char **re if (!found) { char h[MAXHOSTNAMELEN]; gethostname(h, sizeof(h)); - pmix_show_help("help-mca-base.txt", + pmix_show_help("help-pmix-mca-base.txt", "find-available:not-valid", true, h, framework->framework_name, requested_component_names[i]); return PMIX_ERR_NOT_FOUND; @@ -375,7 +375,7 @@ int pmix_mca_base_component_parse_requested (const char *requested, bool *includ /* Double check to ensure that the user did not specify the negate character anywhere else in the value. */ if (NULL != strstr (requested, negate)) { - pmix_show_help("help-mca-base.txt", + pmix_show_help("help-pmix-mca-base.txt", "framework-param:too-many-negates", true, requested_orig); return PMIX_ERROR; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_component_repository.c b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_component_repository.c index ccf730e6988..062b1cb75d6 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_component_repository.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_component_repository.c @@ -10,12 +10,12 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2008-2015 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2008-2019 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -43,6 +43,7 @@ #include "pmix_common.h" #include "src/class/pmix_hash_table.h" #include "src/util/basename.h" +#include "src/util/show_help.h" #if PMIX_HAVE_PDL_SUPPORT @@ -56,6 +57,26 @@ PMIX_CLASS_INSTANCE(pmix_mca_base_component_repository_item_t, pmix_list_item_t, #endif /* PMIX_HAVE_PDL_SUPPORT */ +static void clf_constructor(pmix_object_t *obj) +{ + pmix_mca_base_failed_component_t *cli = (pmix_mca_base_failed_component_t *) obj; + cli->comp = NULL; + cli->error_msg = NULL; +} + +static void clf_destructor(pmix_object_t *obj) +{ + pmix_mca_base_failed_component_t *cli = (pmix_mca_base_failed_component_t *) obj; + cli->comp = NULL; + if( NULL != cli->error_msg ) { + free(cli->error_msg); + cli->error_msg = NULL; + } +} + +PMIX_CLASS_INSTANCE(pmix_mca_base_failed_component_t, pmix_list_item_t, + clf_constructor, clf_destructor); + /* * Private variables @@ -144,12 +165,12 @@ static int process_repository_item (const char *filename, void *data) return PMIX_ERR_OUT_OF_RESOURCE; } - /* strncpy does not guarantee a \0 */ + /* pmix_strncpy does not guarantee a \0 */ ri->ri_type[PMIX_MCA_BASE_MAX_TYPE_NAME_LEN] = '\0'; - strncpy (ri->ri_type, type, PMIX_MCA_BASE_MAX_TYPE_NAME_LEN); + pmix_strncpy (ri->ri_type, type, PMIX_MCA_BASE_MAX_TYPE_NAME_LEN); ri->ri_name[PMIX_MCA_BASE_MAX_TYPE_NAME_LEN] = '\0'; - strncpy (ri->ri_name, name, PMIX_MCA_BASE_MAX_COMPONENT_NAME_LEN); + pmix_strncpy (ri->ri_name, name, PMIX_MCA_BASE_MAX_COMPONENT_NAME_LEN); pmix_list_append (component_list, &ri->super); @@ -200,8 +221,13 @@ int pmix_mca_base_component_repository_add (const char *path) dir = pmix_mca_base_system_default_path; } - if (0 != pmix_pdl_foreachfile(dir, process_repository_item, NULL)) { - break; + if (0 != pmix_pdl_foreachfile(dir, process_repository_item, NULL) && + !(0 == strcmp(dir, pmix_mca_base_system_default_path) || 0 == strcmp(dir, pmix_mca_base_user_default_path))) { + // It is not an error if a directory fails to add (e.g., + // if it doesn't exist). But we should warn about it as + // it is something related to "show_load_errors" + pmix_show_help("help-pmix-mca-base.txt", + "failed to add component dir", true, dir); } } while (NULL != (dir = strtok_r (NULL, sep, &ctx))); @@ -409,6 +435,17 @@ int pmix_mca_base_component_repository_open(pmix_mca_base_framework_t *framework } pmix_output_verbose(vl, 0, "pmix_mca_base_component_repository_open: unable to open %s: %s (ignored)", ri->ri_base, err_msg); + + if( pmix_mca_base_component_track_load_errors ) { + pmix_mca_base_failed_component_t *f_comp = PMIX_NEW(pmix_mca_base_failed_component_t); + f_comp->comp = ri; + if (0 > asprintf(&(f_comp->error_msg), "%s", err_msg)) { + PMIX_RELEASE(f_comp); + return PMIX_ERR_BAD_PARAM; + } + pmix_list_append(&framework->framework_failed_components, &f_comp->super); + } + return PMIX_ERR_BAD_PARAM; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_component_repository.h b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_component_repository.h index 38d6f464db9..7675e04efa1 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_component_repository.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_component_repository.h @@ -13,7 +13,7 @@ * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -59,7 +59,18 @@ struct pmix_mca_base_component_repository_item_t { }; typedef struct pmix_mca_base_component_repository_item_t pmix_mca_base_component_repository_item_t; -PMIX_CLASS_DECLARATION(pmix_mca_base_component_repository_item_t); +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_mca_base_component_repository_item_t); + +/* + * Structure to track information about why a component failed to load. + */ +struct pmix_mca_base_failed_component_t { + pmix_list_item_t super; + pmix_mca_base_component_repository_item_t *comp; + char *error_msg; +}; +typedef struct pmix_mca_base_failed_component_t pmix_mca_base_failed_component_t; +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_mca_base_failed_component_t); /** * @brief initialize the component repository @@ -70,14 +81,14 @@ PMIX_CLASS_DECLARATION(pmix_mca_base_component_repository_item_t); * pmix_mca_base_component_path MCA parameter. pmix_mca_base_open () is a * prerequisite call as it registers the pmix_mca_base_component_path parameter. */ -int pmix_mca_base_component_repository_init(void); +PMIX_EXPORT int pmix_mca_base_component_repository_init(void); /** * @brief add search path for dynamically loaded components * * @param[in] path delimited list of search paths to add */ -int pmix_mca_base_component_repository_add(const char *path); +PMIX_EXPORT int pmix_mca_base_component_repository_add(const char *path); /** @@ -89,13 +100,13 @@ int pmix_mca_base_component_repository_add(const char *path); * The list returned in {framework_components} is owned by the component * repository and CAN NOT be modified by the caller. */ -int pmix_mca_base_component_repository_get_components(pmix_mca_base_framework_t *framework, - pmix_list_t **framework_components); +PMIX_EXPORT int pmix_mca_base_component_repository_get_components(pmix_mca_base_framework_t *framework, + pmix_list_t **framework_components); /** * @brief finalize the mca component repository */ -void pmix_mca_base_component_repository_finalize(void); +PMIX_EXPORT void pmix_mca_base_component_repository_finalize(void); /** * @brief open the repository item and add it to the framework's component @@ -104,14 +115,14 @@ void pmix_mca_base_component_repository_finalize(void); * @param[in] framework framework that matches the component * @param[in] ri dynamic component to open */ -int pmix_mca_base_component_repository_open(pmix_mca_base_framework_t *framework, - pmix_mca_base_component_repository_item_t *ri); +PMIX_EXPORT int pmix_mca_base_component_repository_open(pmix_mca_base_framework_t *framework, + pmix_mca_base_component_repository_item_t *ri); /** * @brief Reduce the reference count of a component and dlclose it if necessary */ -void pmix_mca_base_component_repository_release(const pmix_mca_base_component_t *component); +PMIX_EXPORT void pmix_mca_base_component_repository_release(const pmix_mca_base_component_t *component); /** * @brief Increase the reference count of a component @@ -126,7 +137,7 @@ void pmix_mca_base_component_repository_release(const pmix_mca_base_component_t * @note all components are automatically unloaded by the * pmix_mca_base_component_repository_finalize() call. */ -int pmix_mca_base_component_repository_retain_component(const char *type, const char *name); +PMIX_EXPORT int pmix_mca_base_component_repository_retain_component(const char *type, const char *name); END_C_DECLS diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_framework.c b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_framework.c index 5812ebaf9b3..a931349e5d4 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_framework.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_framework.c @@ -3,7 +3,7 @@ * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -67,6 +67,7 @@ int pmix_mca_base_framework_register (struct pmix_mca_base_framework_t *framewor } PMIX_CONSTRUCT(&framework->framework_components, pmix_list_t); + PMIX_CONSTRUCT(&framework->framework_failed_components, pmix_list_t); if (framework->framework_flags & PMIX_MCA_BASE_FRAMEWORK_FLAG_NO_DSO) { flags |= PMIX_MCA_BASE_REGISTER_STATIC_ONLY; @@ -239,6 +240,7 @@ int pmix_mca_base_framework_close (struct pmix_mca_base_framework_t *framework) framework->framework_flags &= ~(PMIX_MCA_BASE_FRAMEWORK_FLAG_REGISTERED | PMIX_MCA_BASE_FRAMEWORK_FLAG_OPEN); PMIX_DESTRUCT(&framework->framework_components); + PMIX_LIST_DESTRUCT(&framework->framework_failed_components); framework_close_output (framework); diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_framework.h b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_framework.h index 3e4f3db9109..86f7de7e292 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_framework.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_framework.h @@ -2,7 +2,7 @@ /* * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -156,6 +156,8 @@ typedef struct pmix_mca_base_framework_t { /** List of selected components (filled in by pmix_mca_base_framework_register() or pmix_mca_base_framework_open() */ pmix_list_t framework_components; + /** List of components that failed to load */ + pmix_list_t framework_failed_components; } pmix_mca_base_framework_t; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_open.c b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_open.c index 77c9fdf1842..dd9b2cd08ee 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_open.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_open.c @@ -13,7 +13,7 @@ * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -49,6 +49,7 @@ int pmix_mca_base_opened = 0; char *pmix_mca_base_system_default_path = NULL; char *pmix_mca_base_user_default_path = NULL; bool pmix_mca_base_component_show_load_errors = (bool) PMIX_SHOW_LOAD_ERRORS_DEFAULT; +bool pmix_mca_base_component_track_load_errors = false; bool pmix_mca_base_component_disable_dlopen = false; static char *pmix_mca_base_verbose = NULL; @@ -119,6 +120,14 @@ int pmix_mca_base_open(void) (void) pmix_mca_base_var_register_synonym(var_id, "pmix", "mca", NULL, "component_show_load_errors", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); + pmix_mca_base_component_track_load_errors = false; + var_id = pmix_mca_base_var_register("pmix", "mca", "base", "component_track_load_errors", + "Whether to track errors for components that failed to load or not", + PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, + PMIX_INFO_LVL_9, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &pmix_mca_base_component_track_load_errors); + pmix_mca_base_component_disable_dlopen = false; var_id = pmix_mca_base_var_register("pmix", "mca", "base", "component_disable_dlopen", "Whether to attempt to disable opening dynamic components or not", diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c index bbc2e12365c..2ba3bf6f4f1 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c @@ -13,7 +13,7 @@ * Copyright (c) 2008-2015 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -330,7 +330,7 @@ static void process_env_list(char *env_list, char ***argv, char sep) if (NULL == (ptr = strchr(tokens[i], '='))) { value = getenv(tokens[i]); if (NULL == value) { - pmix_show_help("help-mca-var.txt", "incorrect-env-list-param", + pmix_show_help("help-pmix-mca-var.txt", "incorrect-env-list-param", true, tokens[i], env_list); break; } @@ -368,7 +368,7 @@ int pmix_mca_base_var_process_env_list(char ***argv) if (1 == strlen(pmix_mca_base_env_list_sep)) { sep = pmix_mca_base_env_list_sep[0]; } else { - pmix_show_help("help-mca-var.txt", "incorrect-env-list-sep", + pmix_show_help("help-pmix-mca-var.txt", "incorrect-env-list-sep", true, pmix_mca_base_env_list_sep); return PMIX_SUCCESS; } @@ -558,7 +558,7 @@ int pmix_mca_base_var_cache_files(bool rel_path_search) if (NULL != pmix_mca_base_var_file_prefix) { resolve_relative_paths(&pmix_mca_base_var_file_prefix, pmix_mca_base_param_file_path, rel_path_search, &pmix_mca_base_var_files, PMIX_ENV_SEP); } - read_files (pmix_mca_base_var_files, &pmix_mca_base_var_file_values, PMIX_ENV_SEP); + read_files (pmix_mca_base_var_files, &pmix_mca_base_var_file_values, ','); if (NULL != pmix_mca_base_envar_file_prefix) { resolve_relative_paths(&pmix_mca_base_envar_file_prefix, pmix_mca_base_param_file_path, rel_path_search, &pmix_mca_base_envar_files, ','); @@ -742,11 +742,11 @@ static int var_set_from_string (pmix_mca_base_var_t *var, char *src) if (var->mbv_enumerator) { char *valid_values; (void) var->mbv_enumerator->dump(var->mbv_enumerator, &valid_values); - pmix_show_help("help-mca-var.txt", "invalid-value-enum", + pmix_show_help("help-pmix-mca-var.txt", "invalid-value-enum", true, var->mbv_full_name, src, valid_values); free(valid_values); } else { - pmix_show_help("help-mca-var.txt", "invalid-value", + pmix_show_help("help-pmix-mca-var.txt", "invalid-value", true, var->mbv_full_name, src); } @@ -1229,7 +1229,7 @@ static int fixup_files(char **file_list, char * path, bool rel_path_search, char } if (NULL == tmp_file) { - pmix_show_help("help-mca-var.txt", "missing-param-file", + pmix_show_help("help-pmix-mca-var.txt", "missing-param-file", true, getpid(), files[i], msg_path); exit_status = PMIX_ERROR; break; @@ -1365,7 +1365,10 @@ static int register_variable (const char *project_name, const char *framework_na /* Initialize the array if it has never been initialized */ if (!pmix_mca_base_var_initialized) { - pmix_mca_base_var_init(); + ret = pmix_mca_base_var_init(); + if (PMIX_SUCCESS != ret) { + return ret; + } } /* See if this entry is already in the array */ @@ -1383,7 +1386,7 @@ static int register_variable (const char *project_name, const char *framework_na /* Read-only and constant variables can't be settable */ if (scope < PMIX_MCA_BASE_VAR_SCOPE_LOCAL || (flags & PMIX_MCA_BASE_VAR_FLAG_DEFAULT_ONLY)) { if ((flags & PMIX_MCA_BASE_VAR_FLAG_DEFAULT_ONLY) && (flags & PMIX_MCA_BASE_VAR_FLAG_SETTABLE)) { - pmix_show_help("help-mca-var.txt", "invalid-flag-combination", + pmix_show_help("help-pmix-mca-var.txt", "invalid-flag-combination", true, "PMIX_MCA_BASE_VAR_FLAG_DEFAULT_ONLY", "PMIX_MCA_BASE_VAR_FLAG_SETTABLE"); return PMIX_ERROR; } @@ -1473,7 +1476,7 @@ static int register_variable (const char *project_name, const char *framework_na if (0 != compare_strings(framework_name, group->group_framework) || 0 != compare_strings(component_name, group->group_component) || 0 != compare_strings(variable_name, var->mbv_variable_name)) { - pmix_show_help("help-mca-var.txt", "var-name-conflict", + pmix_show_help("help-pmix-mca-var.txt", "var-name-conflict", true, var->mbv_full_name, framework_name, component_name, variable_name, group->group_framework, group->group_component, @@ -1485,7 +1488,7 @@ static int register_variable (const char *project_name, const char *framework_na if (var->mbv_type != type) { #if PMIX_ENABLE_DEBUG - pmix_show_help("help-mca-var.txt", + pmix_show_help("help-pmix-mca-var.txt", "re-register-with-different-type", true, var->mbv_full_name); #endif @@ -1657,7 +1660,7 @@ static int var_set_from_env (pmix_mca_base_var_t *var, pmix_mca_base_var_t *orig /* we found an environment variable but this variable is default-only. print a warning. */ if (PMIX_VAR_IS_DEFAULT_ONLY(original[0])) { - pmix_show_help("help-mca-var.txt", "default-only-param-set", + pmix_show_help("help-pmix-mca-var.txt", "default-only-param-set", true, var_full_name); return PMIX_ERR_NOT_FOUND; @@ -1665,7 +1668,7 @@ static int var_set_from_env (pmix_mca_base_var_t *var, pmix_mca_base_var_t *orig if (PMIX_MCA_BASE_VAR_SOURCE_OVERRIDE == original->mbv_source) { if (!pmix_mca_base_var_suppress_override_warning) { - pmix_show_help("help-mca-var.txt", "overridden-param-set", + pmix_show_help("help-pmix-mca-var.txt", "overridden-param-set", true, var_full_name); } @@ -1696,16 +1699,16 @@ static int var_set_from_env (pmix_mca_base_var_t *var, pmix_mca_base_var_t *orig switch (var->mbv_source) { case PMIX_MCA_BASE_VAR_SOURCE_ENV: - pmix_show_help("help-mca-var.txt", "deprecated-mca-env", + pmix_show_help("help-pmix-mca-var.txt", "deprecated-mca-env", true, var_full_name, new_variable); break; case PMIX_MCA_BASE_VAR_SOURCE_COMMAND_LINE: - pmix_show_help("help-mca-var.txt", "deprecated-mca-cli", + pmix_show_help("help-pmix-mca-var.txt", "deprecated-mca-cli", true, var_full_name, new_variable); break; case PMIX_MCA_BASE_VAR_SOURCE_FILE: case PMIX_MCA_BASE_VAR_SOURCE_OVERRIDE: - pmix_show_help("help-mca-var.txt", "deprecated-mca-file", + pmix_show_help("help-pmix-mca-var.txt", "deprecated-mca-file", true, var_full_name, pmix_mca_base_var_source_file (var), new_variable); break; @@ -1744,14 +1747,14 @@ static int var_set_from_file (pmix_mca_base_var_t *var, pmix_mca_base_var_t *ori /* found it */ if (PMIX_VAR_IS_DEFAULT_ONLY(var[0])) { - pmix_show_help("help-mca-var.txt", "default-only-param-set", + pmix_show_help("help-pmix-mca-var.txt", "default-only-param-set", true, var_full_name); return PMIX_ERR_NOT_FOUND; } if (PMIX_MCA_BASE_VAR_FLAG_ENVIRONMENT_ONLY & original->mbv_flags) { - pmix_show_help("help-mca-var.txt", "environment-only-param", + pmix_show_help("help-pmix-mca-var.txt", "environment-only-param", true, var_full_name, fv->mbvfv_value, fv->mbvfv_file); @@ -1760,7 +1763,7 @@ static int var_set_from_file (pmix_mca_base_var_t *var, pmix_mca_base_var_t *ori if (PMIX_MCA_BASE_VAR_SOURCE_OVERRIDE == original->mbv_source) { if (!pmix_mca_base_var_suppress_override_warning) { - pmix_show_help("help-mca-var.txt", "overridden-param-set", + pmix_show_help("help-pmix-mca-var.txt", "overridden-param-set", true, var_full_name); } @@ -1774,7 +1777,7 @@ static int var_set_from_file (pmix_mca_base_var_t *var, pmix_mca_base_var_t *ori new_variable = original->mbv_full_name; } - pmix_show_help("help-mca-var.txt", "deprecated-mca-file", + pmix_show_help("help-pmix-mca-var.txt", "deprecated-mca-file", true, var_full_name, fv->mbvfv_file, new_variable); } @@ -2038,7 +2041,7 @@ int pmix_mca_base_var_check_exclusive (const char *project, str_b = source_name(var_b); /* Print it all out */ - pmix_show_help("help-mca-var.txt", + pmix_show_help("help-pmix-mca-var.txt", "mutually-exclusive-vars", true, var_a->mbv_full_name, str_a, var_b->mbv_full_name, diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var_group.h b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var_group.h index bd43c7840ae..96a2a15bbf3 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var_group.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var_group.h @@ -13,7 +13,7 @@ * Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -58,7 +58,7 @@ typedef struct pmix_mca_base_var_group_t pmix_mca_base_var_group_t; /** * Object declaration for pmix_mca_base_var_group_t */ -PMIX_CLASS_DECLARATION(pmix_mca_base_var_group_t); +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_mca_base_var_group_t); /** * Register an MCA variable group @@ -74,10 +74,10 @@ PMIX_CLASS_DECLARATION(pmix_mca_base_var_group_t); * Create an MCA variable group. If the group already exists * this call is equivalent to pmix_mca_base_ver_find_group(). */ -int pmix_mca_base_var_group_register(const char *project_name, - const char *framework_name, - const char *component_name, - const char *description); +PMIX_EXPORT int pmix_mca_base_var_group_register(const char *project_name, + const char *framework_name, + const char *component_name, + const char *description); /** * Register an MCA variable group for a component @@ -89,8 +89,8 @@ int pmix_mca_base_var_group_register(const char *project_name, * @retval index Unique group index * @return pmix error code on Error */ -int pmix_mca_base_var_group_component_register (const pmix_mca_base_component_t *component, - const char *description); +PMIX_EXPORT int pmix_mca_base_var_group_component_register (const pmix_mca_base_component_t *component, + const char *description); /** * Deregister an MCA param group @@ -100,7 +100,7 @@ int pmix_mca_base_var_group_component_register (const pmix_mca_base_component_t * * This call deregisters all associated variables and subgroups. */ -int pmix_mca_base_var_group_deregister (int group_index); +PMIX_EXPORT int pmix_mca_base_var_group_deregister (int group_index); /** * Find an MCA group @@ -112,9 +112,9 @@ int pmix_mca_base_var_group_deregister (int group_index); * @returns PMIX_SUCCESS if found * @returns PMIX_ERR_NOT_FOUND if not found */ -int pmix_mca_base_var_group_find (const char *project_name, - const char *framework_name, - const char *component_name); +PMIX_EXPORT int pmix_mca_base_var_group_find (const char *project_name, + const char *framework_name, + const char *component_name); /** * Find an MCA group by its full name @@ -125,7 +125,7 @@ int pmix_mca_base_var_group_find (const char *project_name, * @returns PMIX_SUCCESS if found * @returns PMIX_ERR_NOT_FOUND if not found */ -int pmix_mca_base_var_group_find_by_name (const char *full_name, int *index); +PMIX_EXPORT int pmix_mca_base_var_group_find_by_name (const char *full_name, int *index); /** * Get the group at a specified index @@ -139,8 +139,8 @@ int pmix_mca_base_var_group_find_by_name (const char *full_name, int *index); * The returned pointer belongs to the MCA variable system. Do not modify/release/retain * the pointer. */ -int pmix_mca_base_var_group_get (const int group_index, - const pmix_mca_base_var_group_t **group); +PMIX_EXPORT int pmix_mca_base_var_group_get (const int group_index, + const pmix_mca_base_var_group_t **group); /** * Set/unset a flags for all variables in a group. @@ -151,15 +151,15 @@ int pmix_mca_base_var_group_get (const int group_index, * * Set a flag for every variable in a group. See pmix_mca_base_var_set_flag() for more info. */ -int pmix_mca_base_var_group_set_var_flag (const int group_index, int flags, - bool set); +PMIX_EXPORT int pmix_mca_base_var_group_set_var_flag (const int group_index, int flags, + bool set); /** * Get the number of registered MCA groups * * @retval count Number of registered MCA groups */ -int pmix_mca_base_var_group_get_count (void); +PMIX_EXPORT int pmix_mca_base_var_group_get_count (void); /** * Get a relative timestamp for the MCA group system @@ -168,6 +168,6 @@ int pmix_mca_base_var_group_get_count (void); * * This value will change if groups or variables are either added or removed. */ -int pmix_mca_base_var_group_get_stamp (void); +PMIX_EXPORT int pmix_mca_base_var_group_get_stamp (void); #endif /* PMIX_MCA_BASE_VAR_GROUP_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/base.h b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/base.h index 0b08a5d0b18..4ff0bdfc95d 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/base.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/base.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -205,7 +205,6 @@ PMIX_EXPORT extern pmix_bfrops_globals_t pmix_bfrops_globals; free(tmpbuf); \ } while (0) - /** * Internal struct used for holding registered bfrop functions */ @@ -268,7 +267,7 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_stub_copy_payload(struct pmix_peer_t *peer pmix_buffer_t *src); PMIX_EXPORT pmix_status_t pmix_bfrops_stub_value_xfer(struct pmix_peer_t *peer, pmix_value_t *dest, - pmix_value_t *src); + const pmix_value_t *src); PMIX_EXPORT void pmix_bfrops_stub_value_load(struct pmix_peer_t *peer, pmix_value_t *v, void *data, pmix_data_type_t type); @@ -488,9 +487,6 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_string(char **dest, char *src, PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_value(pmix_value_t **dest, pmix_value_t *src, pmix_data_type_t type); -PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_array(pmix_info_array_t **dest, - pmix_info_array_t *src, - pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_proc(pmix_proc_t **dest, pmix_proc_t *src, pmix_data_type_t type); @@ -506,9 +502,6 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_buf(pmix_buffer_t **dest, PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_kval(pmix_kval_t **dest, pmix_kval_t *src, pmix_data_type_t type); -PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_modex(pmix_modex_data_t **dest, - pmix_modex_data_t *src, - pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrop_base_copy_persist(pmix_persistence_t **dest, pmix_persistence_t *src, pmix_data_type_t type); @@ -527,10 +520,6 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_darray(pmix_data_array_t **dest, PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_query(pmix_query_t **dest, pmix_query_t *src, pmix_data_type_t type); -/**** DEPRECATED ****/ -PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_array(pmix_info_array_t **dest, - pmix_info_array_t *src, - pmix_data_type_t type); /* * "Standard" print functions @@ -585,8 +574,6 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_status(char **output, char *pre PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_value(char **output, char *prefix, pmix_value_t *src, pmix_data_type_t type); -PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_array(char **output, char *prefix, - pmix_info_array_t *src, pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_proc(char **output, char *prefix, pmix_proc_t *src, pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_app(char **output, char *prefix, @@ -597,8 +584,6 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_buf(char **output, char *prefix pmix_buffer_t *src, pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_kval(char **output, char *prefix, pmix_kval_t *src, pmix_data_type_t type); -PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_modex(char **output, char *prefix, - pmix_modex_data_t *src, pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_persist(char **output, char *prefix, pmix_persistence_t *src, pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_bo(char **output, char *prefix, @@ -661,7 +646,7 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_value_unload(pmix_value_t *kv, size_t *sz); PMIX_EXPORT pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p, - pmix_value_t *src); + const pmix_value_t *src); PMIX_EXPORT pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p, pmix_value_t *p1); diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_copy.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_copy.c index 7b91c19baee..f81c051ba27 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_copy.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_copy.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -235,7 +235,7 @@ pmix_status_t pmix_bfrops_base_copy_info(pmix_info_t **dest, pmix_data_type_t type) { *dest = (pmix_info_t*)malloc(sizeof(pmix_info_t)); - (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); (*dest)->flags = src->flags; return pmix_bfrops_base_value_xfer(&(*dest)->value, &src->value); } @@ -266,7 +266,7 @@ pmix_status_t pmix_bfrops_base_copy_app(pmix_app_t **dest, (*dest)->ninfo = src->ninfo; (*dest)->info = (pmix_info_t*)malloc(src->ninfo * sizeof(pmix_info_t)); for (j=0; j < src->ninfo; j++) { - (void)strncpy((*dest)->info[j].key, src->info[j].key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->info[j].key, src->info[j].key, PMIX_MAX_KEYLEN); pmix_value_xfer(&(*dest)->info[j].value, &src->info[j].value); } return PMIX_SUCCESS; @@ -299,32 +299,11 @@ pmix_status_t pmix_bfrops_base_copy_proc(pmix_proc_t **dest, if (NULL == *dest) { return PMIX_ERR_OUT_OF_RESOURCE; } - (void)strncpy((*dest)->nspace, src->nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->nspace, src->nspace, PMIX_MAX_NSLEN); (*dest)->rank = src->rank; return PMIX_SUCCESS; } -pmix_status_t pmix_bfrops_base_copy_modex(pmix_modex_data_t **dest, - pmix_modex_data_t *src, - pmix_data_type_t type) -{ - *dest = (pmix_modex_data_t*)malloc(sizeof(pmix_modex_data_t)); - if (NULL == *dest) { - return PMIX_ERR_OUT_OF_RESOURCE; - } - (*dest)->blob = NULL; - (*dest)->size = 0; - if (NULL != src->blob) { - (*dest)->blob = (uint8_t*)malloc(src->size * sizeof(uint8_t)); - if (NULL == (*dest)->blob) { - return PMIX_ERR_OUT_OF_RESOURCE; - } - memcpy((*dest)->blob, src->blob, src->size * sizeof(uint8_t)); - (*dest)->size = src->size; - } - return PMIX_SUCCESS; -} - pmix_status_t pmix_bfrop_base_copy_persist(pmix_persistence_t **dest, pmix_persistence_t *src, pmix_data_type_t type) @@ -356,9 +335,9 @@ pmix_status_t pmix_bfrops_base_copy_pdata(pmix_pdata_t **dest, pmix_data_type_t type) { *dest = (pmix_pdata_t*)malloc(sizeof(pmix_pdata_t)); - (void)strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); (*dest)->proc.rank = src->proc.rank; - (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); return pmix_bfrops_base_value_xfer(&(*dest)->value, &src->value); } @@ -366,18 +345,23 @@ pmix_status_t pmix_bfrops_base_copy_pinfo(pmix_proc_info_t **dest, pmix_proc_info_t *src, pmix_data_type_t type) { - *dest = (pmix_proc_info_t*)malloc(sizeof(pmix_proc_info_t)); - (void)strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); - (*dest)->proc.rank = src->proc.rank; + pmix_proc_info_t *p; + + PMIX_PROC_INFO_CREATE(p, 1); + if (NULL == p) { + return PMIX_ERR_NOMEM; + } + memcpy(&p->proc, &src->proc, sizeof(pmix_proc_t)); if (NULL != src->hostname) { - (*dest)->hostname = strdup(src->hostname); + p->hostname = strdup(src->hostname); } if (NULL != src->executable_name) { - (*dest)->executable_name = strdup(src->executable_name); + p->executable_name = strdup(src->executable_name); } - (*dest)->pid = src->pid; - (*dest)->exit_code = src->exit_code; - (*dest)->state = src->state; + memcpy(&p->pid, &src->pid, sizeof(pid_t)); + memcpy(&p->exit_code, &src->exit_code, sizeof(int)); + memcpy(&p->state, &src->state, sizeof(pmix_proc_state_t)); + *dest = p; return PMIX_SUCCESS; } @@ -399,7 +383,6 @@ pmix_status_t pmix_bfrops_base_copy_darray(pmix_data_array_t **dest, pmix_buffer_t *pb, *sb; pmix_byte_object_t *pbo, *sbo; pmix_kval_t *pk, *sk; - pmix_modex_data_t *pm, *sm; pmix_proc_info_t *pi, *si; pmix_query_t *pq, *sq; @@ -617,7 +600,7 @@ pmix_status_t pmix_bfrops_base_copy_darray(pmix_data_array_t **dest, p1 = (pmix_info_t*)p->array; s1 = (pmix_info_t*)src->array; for (n=0; n < src->size; n++) { - PMIX_INFO_LOAD(&p1[n], s1[n].key, &s1[n].value.data.flag, s1[n].value.type); + PMIX_INFO_XFER(&p1[n], &s1[n]); } break; case PMIX_PDATA: @@ -629,7 +612,7 @@ pmix_status_t pmix_bfrops_base_copy_darray(pmix_data_array_t **dest, pd = (pmix_pdata_t*)p->array; sd = (pmix_pdata_t*)src->array; for (n=0; n < src->size; n++) { - PMIX_PDATA_LOAD(&pd[n], &sd[n].proc, sd[n].key, &sd[n].value.data.flag, sd[n].value.type); + PMIX_PDATA_XFER(&pd[n], &sd[n]); } break; case PMIX_BUFFER: @@ -692,31 +675,6 @@ pmix_status_t pmix_bfrops_base_copy_darray(pmix_data_array_t **dest, } } break; - case PMIX_MODEX: - PMIX_MODEX_CREATE(p->array, src->size); - if (NULL == p->array) { - free(p); - return PMIX_ERR_NOMEM; - } - pm = (pmix_modex_data_t*)p->array; - sm = (pmix_modex_data_t*)src->array; - for (n=0; n < src->size; n++) { - memcpy(&pm[n], &sm[n], sizeof(pmix_modex_data_t)); - if (NULL != sm[n].blob && 0 < sm[n].size) { - pm[n].blob = (uint8_t*)malloc(sm[n].size); - if (NULL == pm[n].blob) { - PMIX_MODEX_FREE(pm, src->size); - free(p); - return PMIX_ERR_NOMEM; - } - memcpy(pm[n].blob, sm[n].blob, sm[n].size); - pm[n].size = sm[n].size; - } else { - pm[n].blob = NULL; - pm[n].size = 0; - } - } - break; case PMIX_PERSIST: p->array = (pmix_persistence_t*)malloc(src->size * sizeof(pmix_persistence_t)); if (NULL == p->array) { @@ -850,29 +808,3 @@ pmix_status_t pmix_bfrops_base_copy_query(pmix_query_t **dest, } return PMIX_SUCCESS; } - -/**** DEPRECATED ****/ -pmix_status_t pmix_bfrops_base_copy_array(pmix_info_array_t **dest, - pmix_info_array_t *src, - pmix_data_type_t type) -{ - pmix_info_t *d1, *s1; - - *dest = (pmix_info_array_t*)malloc(sizeof(pmix_info_array_t)); - if (NULL == (*dest)) { - return PMIX_ERR_NOMEM; - } - (*dest)->size = src->size; - if (0 < src->size) { - (*dest)->array = (pmix_info_t*)malloc(src->size * sizeof(pmix_info_t)); - if (NULL == (*dest)->array) { - free(*dest); - return PMIX_ERR_NOMEM; - } - d1 = (pmix_info_t*)(*dest)->array; - s1 = (pmix_info_t*)src->array; - memcpy(d1, s1, src->size * sizeof(pmix_info_t)); - } - return PMIX_SUCCESS; -} -/*******************/ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_fns.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_fns.c index 5abf19c1e86..c862a08faef 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_fns.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_fns.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -38,8 +38,15 @@ PMIX_EXPORT void pmix_value_load(pmix_value_t *v, const void *data, pmix_bfrops_base_value_load(v, data, type); } +PMIX_EXPORT pmix_status_t pmix_value_unload(pmix_value_t *kv, + void **data, + size_t *sz) +{ + return pmix_bfrops_base_value_unload(kv, data, sz); +} + PMIX_EXPORT pmix_status_t pmix_value_xfer(pmix_value_t *dest, - pmix_value_t *src) + const pmix_value_t *src) { return pmix_bfrops_base_value_xfer(dest, src); } @@ -49,6 +56,8 @@ void pmix_bfrops_base_value_load(pmix_value_t *v, const void *data, { pmix_byte_object_t *bo; pmix_proc_info_t *pi; + pmix_data_array_t *darray; + pmix_status_t rc; v->type = type; if (NULL == data) { @@ -172,8 +181,16 @@ void pmix_bfrops_base_value_load(pmix_value_t *v, const void *data, memcpy(&(v->data.pinfo->exit_code), &pi->exit_code, sizeof(int)); break; case PMIX_POINTER: - memcpy(&(v->data.ptr), data, sizeof(void*)); + v->data.ptr = (void*)data; break; + case PMIX_DATA_ARRAY: + darray = (pmix_data_array_t*)data; + rc = pmix_bfrops_base_copy_darray(&v->data.darray, darray, PMIX_DATA_ARRAY); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + break; + default: /* silence warnings */ break; @@ -187,6 +204,7 @@ pmix_status_t pmix_bfrops_base_value_unload(pmix_value_t *kv, size_t *sz) { pmix_status_t rc; + pmix_data_array_t **darray; rc = PMIX_SUCCESS; if (NULL == data || @@ -301,9 +319,14 @@ pmix_status_t pmix_bfrops_base_value_unload(pmix_value_t *kv, *sz = sizeof(pmix_proc_state_t); break; case PMIX_POINTER: - memcpy(*data, &(kv->data.ptr), sizeof(void*)); + *data = (void*)kv->data.ptr; *sz = sizeof(void*); break; + case PMIX_DATA_ARRAY: + darray = (pmix_data_array_t**)data; + rc = pmix_bfrops_base_copy_darray(darray, kv->data.darray, PMIX_DATA_ARRAY); + *sz = sizeof(pmix_data_array_t); + break; default: /* silence warnings */ rc = PMIX_ERROR; @@ -398,11 +421,12 @@ pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p, } break; case PMIX_COMPRESSED_STRING: - if (p->data.bo.size != p1->data.bo.size) { - return false; + if (p->data.bo.size > p1->data.bo.size) { + return PMIX_VALUE2_GREATER; } else { - return true; + return PMIX_VALUE1_GREATER; } + break; case PMIX_STATUS: if (p->data.status == p1->data.status) { rc = PMIX_EQUAL; @@ -416,22 +440,8 @@ pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p, /* Xfer FUNCTIONS FOR GENERIC PMIX TYPES */ pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p, - pmix_value_t *src) + const pmix_value_t *src) { - size_t n, m; - pmix_status_t rc; - char **prarray, **strarray; - pmix_value_t *pv, *sv; - pmix_info_t *p1, *s1; - pmix_app_t *pa, *sa; - pmix_pdata_t *pd, *sd; - pmix_buffer_t *pb, *sb; - pmix_byte_object_t *pbo, *sbo; - pmix_kval_t *pk, *sk; - pmix_modex_data_t *pm, *sm; - pmix_proc_info_t *pi, *si; - pmix_query_t *pq, *sq; - /* copy the right field */ p->type = src->type; switch (src->type) { @@ -544,414 +554,15 @@ pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p, memcpy(&p->data.state, &src->data.state, sizeof(pmix_proc_state_t)); break; case PMIX_PROC_INFO: - PMIX_PROC_INFO_CREATE(p->data.pinfo, 1); - if (NULL != src->data.pinfo->hostname) { - p->data.pinfo->hostname = strdup(src->data.pinfo->hostname); - } - if (NULL != src->data.pinfo->executable_name) { - p->data.pinfo->executable_name = strdup(src->data.pinfo->executable_name); - } - memcpy(&p->data.pinfo->pid, &src->data.pinfo->pid, sizeof(pid_t)); - memcpy(&p->data.pinfo->exit_code, &src->data.pinfo->exit_code, sizeof(int)); - memcpy(&p->data.pinfo->state, &src->data.pinfo->state, sizeof(pmix_proc_state_t)); - break; + return pmix_bfrops_base_copy_pinfo(&p->data.pinfo, src->data.pinfo, PMIX_PROC_INFO); case PMIX_DATA_ARRAY: - p->data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); - p->data.darray->type = src->data.darray->type; - p->data.darray->size = src->data.darray->size; - if (0 == p->data.darray->size || NULL == src->data.darray->array) { - p->data.darray->array = NULL; - p->data.darray->size = 0; - break; - } - /* allocate space and do the copy */ - switch (src->data.darray->type) { - case PMIX_UINT8: - case PMIX_INT8: - case PMIX_BYTE: - p->data.darray->array = (char*)malloc(src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size); - break; - case PMIX_UINT16: - case PMIX_INT16: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint16_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint16_t)); - break; - case PMIX_UINT32: - case PMIX_INT32: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint32_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint32_t)); - break; - case PMIX_UINT64: - case PMIX_INT64: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint64_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint64_t)); - break; - case PMIX_BOOL: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(bool)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(bool)); - break; - case PMIX_SIZE: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(size_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(size_t)); - break; - case PMIX_PID: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(pid_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pid_t)); - break; - case PMIX_STRING: - p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - prarray = (char**)p->data.darray->array; - strarray = (char**)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != strarray[n]) { - prarray[n] = strdup(strarray[n]); - } - } - break; - case PMIX_INT: - case PMIX_UINT: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(int)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(int)); - break; - case PMIX_FLOAT: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(float)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(float)); - break; - case PMIX_DOUBLE: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(double)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(double)); - break; - case PMIX_TIMEVAL: - p->data.darray->array = (struct timeval*)malloc(src->data.darray->size * sizeof(struct timeval)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(struct timeval)); - break; - case PMIX_TIME: - p->data.darray->array = (time_t*)malloc(src->data.darray->size * sizeof(time_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(time_t)); - break; - case PMIX_STATUS: - p->data.darray->array = (pmix_status_t*)malloc(src->data.darray->size * sizeof(pmix_status_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_status_t)); - break; - case PMIX_VALUE: - PMIX_VALUE_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pv = (pmix_value_t*)p->data.darray->array; - sv = (pmix_value_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (PMIX_SUCCESS != (rc = pmix_value_xfer(&pv[n], &sv[n]))) { - PMIX_VALUE_FREE(pv, src->data.darray->size); - return rc; - } - } - break; - case PMIX_PROC: - PMIX_PROC_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_proc_t)); - break; - case PMIX_APP: - PMIX_APP_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pa = (pmix_app_t*)p->data.darray->array; - sa = (pmix_app_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sa[n].cmd) { - pa[n].cmd = strdup(sa[n].cmd); - } - if (NULL != sa[n].argv) { - pa[n].argv = pmix_argv_copy(sa[n].argv); - } - if (NULL != sa[n].env) { - pa[n].env = pmix_argv_copy(sa[n].env); - } - if (NULL != sa[n].cwd) { - pa[n].cwd = strdup(sa[n].cwd); - } - pa[n].maxprocs = sa[n].maxprocs; - if (0 < sa[n].ninfo && NULL != sa[n].info) { - PMIX_INFO_CREATE(pa[n].info, sa[n].ninfo); - if (NULL == pa[n].info) { - PMIX_APP_FREE(pa, src->data.darray->size); - return PMIX_ERR_NOMEM; - } - pa[n].ninfo = sa[n].ninfo; - for (m=0; m < pa[n].ninfo; m++) { - PMIX_INFO_XFER(&pa[n].info[m], &sa[n].info[m]); - } - } - } - break; - case PMIX_INFO: - PMIX_INFO_CREATE(p->data.darray->array, src->data.darray->size); - p1 = (pmix_info_t*)p->data.darray->array; - s1 = (pmix_info_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_INFO_XFER(&p1[n], &s1[n]); - } - break; - case PMIX_PDATA: - PMIX_PDATA_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pd = (pmix_pdata_t*)p->data.darray->array; - sd = (pmix_pdata_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_PDATA_XFER(&pd[n], &sd[n]); - } - break; - case PMIX_BUFFER: - p->data.darray->array = (pmix_buffer_t*)malloc(src->data.darray->size * sizeof(pmix_buffer_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pb = (pmix_buffer_t*)p->data.darray->array; - sb = (pmix_buffer_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_CONSTRUCT(&pb[n], pmix_buffer_t); - pmix_bfrops_base_copy_payload(&pb[n], &sb[n]); - } - break; - case PMIX_BYTE_OBJECT: - case PMIX_COMPRESSED_STRING: - p->data.darray->array = (pmix_byte_object_t*)malloc(src->data.darray->size * sizeof(pmix_byte_object_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pbo = (pmix_byte_object_t*)p->data.darray->array; - sbo = (pmix_byte_object_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sbo[n].bytes && 0 < sbo[n].size) { - pbo[n].size = sbo[n].size; - pbo[n].bytes = (char*)malloc(pbo[n].size); - memcpy(pbo[n].bytes, sbo[n].bytes, pbo[n].size); - } else { - pbo[n].bytes = NULL; - pbo[n].size = 0; - } - } - break; - case PMIX_KVAL: - p->data.darray->array = (pmix_kval_t*)calloc(src->data.darray->size , sizeof(pmix_kval_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pk = (pmix_kval_t*)p->data.darray->array; - sk = (pmix_kval_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sk[n].key) { - pk[n].key = strdup(sk[n].key); - } - if (NULL != sk[n].value) { - PMIX_VALUE_CREATE(pk[n].value, 1); - if (NULL == pk[n].value) { - free(p->data.darray->array); - return PMIX_ERR_NOMEM; - } - if (PMIX_SUCCESS != (rc = pmix_value_xfer(pk[n].value, sk[n].value))) { - return rc; - } - } - } - break; - case PMIX_MODEX: - PMIX_MODEX_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pm = (pmix_modex_data_t*)p->data.darray->array; - sm = (pmix_modex_data_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - memcpy(&pm[n], &sm[n], sizeof(pmix_modex_data_t)); - if (NULL != sm[n].blob && 0 < sm[n].size) { - pm[n].blob = (uint8_t*)malloc(sm[n].size); - if (NULL == pm[n].blob) { - return PMIX_ERR_NOMEM; - } - memcpy(pm[n].blob, sm[n].blob, sm[n].size); - pm[n].size = sm[n].size; - } else { - pm[n].blob = NULL; - pm[n].size = 0; - } - } - break; - case PMIX_PERSIST: - p->data.darray->array = (pmix_persistence_t*)malloc(src->data.darray->size * sizeof(pmix_persistence_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_persistence_t)); - break; - case PMIX_POINTER: - p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - prarray = (char**)p->data.darray->array; - strarray = (char**)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - prarray[n] = strarray[n]; - } - break; - case PMIX_SCOPE: - p->data.darray->array = (pmix_scope_t*)malloc(src->data.darray->size * sizeof(pmix_scope_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_scope_t)); - break; - case PMIX_DATA_RANGE: - p->data.darray->array = (pmix_data_range_t*)malloc(src->data.darray->size * sizeof(pmix_data_range_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_data_range_t)); - break; - case PMIX_COMMAND: - p->data.darray->array = (pmix_cmd_t*)malloc(src->data.darray->size * sizeof(pmix_cmd_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_cmd_t)); - break; - case PMIX_INFO_DIRECTIVES: - p->data.darray->array = (pmix_info_directives_t*)malloc(src->data.darray->size * sizeof(pmix_info_directives_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_info_directives_t)); - break; - case PMIX_PROC_INFO: - PMIX_PROC_INFO_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pi = (pmix_proc_info_t*)p->data.darray->array; - si = (pmix_proc_info_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - memcpy(&pi[n].proc, &si[n].proc, sizeof(pmix_proc_t)); - if (NULL != si[n].hostname) { - pi[n].hostname = strdup(si[n].hostname); - } else { - pi[n].hostname = NULL; - } - if (NULL != si[n].executable_name) { - pi[n].executable_name = strdup(si[n].executable_name); - } else { - pi[n].executable_name = NULL; - } - pi[n].pid = si[n].pid; - pi[n].exit_code = si[n].exit_code; - pi[n].state = si[n].state; - } - break; - case PMIX_DATA_ARRAY: - PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); - return PMIX_ERR_NOT_SUPPORTED; // don't support iterative arrays - case PMIX_QUERY: - PMIX_QUERY_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pq = (pmix_query_t*)p->data.darray->array; - sq = (pmix_query_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sq[n].keys) { - pq[n].keys = pmix_argv_copy(sq[n].keys); - } - if (NULL != sq[n].qualifiers && 0 < sq[n].nqual) { - PMIX_INFO_CREATE(pq[n].qualifiers, sq[n].nqual); - if (NULL == pq[n].qualifiers) { - PMIX_QUERY_FREE(pq, src->data.darray->size); - return PMIX_ERR_NOMEM; - } - for (m=0; m < sq[n].nqual; m++) { - PMIX_INFO_XFER(&pq[n].qualifiers[m], &sq[n].qualifiers[m]); - } - pq[n].nqual = sq[n].nqual; - } else { - pq[n].qualifiers = NULL; - pq[n].nqual = 0; - } - } - break; - default: - return PMIX_ERR_UNKNOWN_DATA_TYPE; - } - break; + return pmix_bfrops_base_copy_darray(&p->data.darray, src->data.darray, PMIX_DATA_ARRAY); case PMIX_POINTER: - memcpy(&p->data.ptr, &src->data.ptr, sizeof(void*)); - break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - p->data.array->size = src->data.array->size; - if (0 < src->data.array->size) { - p->data.array->array = (pmix_info_t*)malloc(src->data.array->size * sizeof(pmix_info_t)); - if (NULL == p->data.array->array) { - return PMIX_ERR_NOMEM; - } - p1 = (pmix_info_t*)p->data.array->array; - s1 = (pmix_info_t*)src->data.array->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_INFO_XFER(&p1[n], &s1[n]); - } - } + p->data.ptr = src->data.ptr; break; - /********************/ + default: - pmix_output(0, "XFER-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)src->type); - assert(0); + pmix_output(0, "PMIX-XFER-VALUE: UNSUPPORTED TYPE %d", (int)src->type); return PMIX_ERROR; } return PMIX_SUCCESS; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_frame.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_frame.c index b1c1d5439de..f9db897ca68 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_frame.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_frame.c @@ -11,8 +11,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -90,7 +90,7 @@ static pmix_status_t pmix_bfrop_close(void) pmix_bfrops_globals.initialized = false; /* the components will cleanup when closed */ - PMIX_DESTRUCT(&pmix_bfrops_globals.actives); + PMIX_LIST_DESTRUCT(&pmix_bfrops_globals.actives); return pmix_mca_base_framework_components_close(&pmix_bfrops_base_framework, NULL); } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_pack.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_pack.c index a493ed989e8..c3cc9f21104 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_pack.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_pack.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -454,6 +454,7 @@ pmix_status_t pmix_bfrops_base_pack_status(pmix_buffer_t *buffer, const void *sr for (i = 0; i < num_vals; ++i) { status = (int32_t)ssrc[i]; if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_int32(buffer, &status, 1, PMIX_INT32))) { + PMIX_ERROR_LOG(ret); return ret; } } @@ -702,28 +703,6 @@ pmix_status_t pmix_bfrops_base_pack_kval(pmix_buffer_t *buffer, const void *src, return PMIX_SUCCESS; } -pmix_status_t pmix_bfrops_base_pack_modex(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ - pmix_modex_data_t *ptr; - int32_t i; - int ret; - - ptr = (pmix_modex_data_t *) src; - - for (i = 0; i < num_vals; ++i) { - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { - return ret; - } - if( 0 < ptr[i].size){ - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_byte(buffer, ptr[i].blob, ptr[i].size, PMIX_UINT8))) { - return ret; - } - } - } - return PMIX_SUCCESS; -} - pmix_status_t pmix_bfrops_base_pack_persist(pmix_buffer_t *buffer, const void *src, int32_t num_vals, pmix_data_type_t type) { @@ -1000,13 +979,7 @@ pmix_status_t pmix_bfrops_base_pack_darray(pmix_buffer_t *buffer, const void *sr return ret; } break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_array(buffer, p[i].array, p[i].size, PMIX_INFO_ARRAY))) { - return ret; - } - break; - /********************/ + default: pmix_output(0, "PACK-PMIX-VALUE[%s:%d]: UNSUPPORTED TYPE %d", __FILE__, __LINE__, (int)p[i].type); @@ -1223,17 +1196,11 @@ pmix_status_t pmix_bfrops_base_pack_val(pmix_buffer_t *buffer, return ret; } break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_array(buffer, p->data.array, 1, PMIX_INFO_ARRAY))) { - return ret; - } - break; - /********************/ + default: - pmix_output(0, "PACK-PMIX-VALUE[%s:%d]: UNSUPPORTED TYPE %d", - __FILE__, __LINE__, (int)p->type); - return PMIX_ERROR; + pmix_output(0, "PACK-PMIX-VALUE[%s:%d]: UNSUPPORTED TYPE %d", + __FILE__, __LINE__, (int)p->type); + return PMIX_ERROR; } return PMIX_SUCCESS; } @@ -1243,30 +1210,3 @@ pmix_status_t pmix_bfrops_base_pack_alloc_directive(pmix_buffer_t *buffer, const { return pmix_bfrops_base_pack_byte(buffer, src, num_vals, PMIX_UINT8); } - - -/**** DEPRECATED ****/ -pmix_status_t pmix_bfrops_base_pack_array(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ - pmix_info_array_t *ptr; - int32_t i; - pmix_status_t ret; - - ptr = (pmix_info_array_t *) src; - - for (i = 0; i < num_vals; ++i) { - /* pack the size */ - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { - return ret; - } - if (0 < ptr[i].size) { - /* pack the values */ - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_info(buffer, ptr[i].array, ptr[i].size, PMIX_INFO))) { - return ret; - } - } - } - - return PMIX_SUCCESS; -} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_print.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_print.c index 051efd08b96..247f53318df 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_print.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_print.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * @@ -1014,12 +1014,7 @@ int pmix_bfrops_base_print_status(char **output, char *prefix, rc = asprintf(output, "%sPMIX_VALUE: Data type: DATA_ARRAY\tARRAY SIZE: %ld", prefx, (long)src->data.darray->size); break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - rc = asprintf(output, "%sPMIX_VALUE: Data type: INFO_ARRAY\tARRAY SIZE: %ld", - prefx, (long)src->data.array->size); - break; - /********************/ + default: rc = asprintf(output, "%sPMIX_VALUE: Data type: UNKNOWN\tValue: UNPRINTABLE", prefx); break; @@ -1136,12 +1131,6 @@ int pmix_bfrops_base_print_kval(char **output, char *prefix, return PMIX_SUCCESS; } -pmix_status_t pmix_bfrops_base_print_modex(char **output, char *prefix, - pmix_modex_data_t *src, pmix_data_type_t type) -{ - return PMIX_SUCCESS; -} - int pmix_bfrops_base_print_persist(char **output, char *prefix, pmix_persistence_t *src, pmix_data_type_t type) { @@ -1635,37 +1624,3 @@ pmix_status_t pmix_bfrops_base_print_alloc_directive(char **output, char *prefix return PMIX_SUCCESS; } } - - -/**** DEPRECATED ****/ -pmix_status_t pmix_bfrops_base_print_array(char **output, char *prefix, - pmix_info_array_t *src, pmix_data_type_t type) -{ - size_t j; - char *tmp, *tmp2, *tmp3, *pfx; - pmix_info_t *s1; - - if (0 > asprintf(&tmp, "%sARRAY SIZE: %ld", prefix, (long)src->size)) { - return PMIX_ERR_NOMEM; - } - if (0 > asprintf(&pfx, "\n%s\t", (NULL == prefix) ? "" : prefix)) { - free(tmp); - return PMIX_ERR_NOMEM; - } - s1 = (pmix_info_t*)src->array; - - for (j=0; j < src->size; j++) { - pmix_bfrops_base_print_info(&tmp2, pfx, &s1[j], PMIX_INFO); - if (0 > asprintf(&tmp3, "%s%s", tmp, tmp2)) { - free(tmp); - free(tmp2); - return PMIX_ERR_NOMEM; - } - free(tmp); - free(tmp2); - tmp = tmp3; - } - *output = tmp; - return PMIX_SUCCESS; -} -/********************/ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c index 1b484772597..18c219ae722 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -111,6 +111,7 @@ pmix_status_t pmix_bfrops_base_unpack(pmix_pointer_array_t *regtypes, } if (PMIX_INT32 != local_type) { /* if the length wasn't first, then error */ *num_vals = 0; + PMIX_ERROR_LOG(PMIX_ERR_UNPACK_FAILURE); return PMIX_ERR_UNPACK_FAILURE; } } @@ -753,26 +754,14 @@ pmix_status_t pmix_bfrops_base_unpack_val(pmix_buffer_t *buffer, return ret; } break; - case PMIX_QUERY: - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_query(buffer, val->data.darray, &m, PMIX_QUERY))) { + case PMIX_ALLOC_DIRECTIVE: + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_alloc_directive(buffer, &val->data.adir, &m, PMIX_ALLOC_DIRECTIVE))) { return ret; } break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - /* this field is now a pointer, so we must allocate storage for it */ - val->data.array = (pmix_info_array_t*)malloc(sizeof(pmix_info_array_t)); - if (NULL == val->data.array) { - return PMIX_ERR_NOMEM; - } - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_array(buffer, val->data.array, &m, PMIX_INFO_ARRAY))) { - return ret; - } - break; - /********************/ default: - pmix_output(0, "UNPACK-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)val->type); - return PMIX_ERROR; + pmix_output(0, "UNPACK-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)val->type); + return PMIX_ERROR; } return PMIX_SUCCESS; @@ -830,7 +819,7 @@ pmix_status_t pmix_bfrops_base_unpack_info(pmix_buffer_t *buffer, void *dest, if (NULL == tmp) { return PMIX_ERROR; } - (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); + pmix_strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); /* unpack the directives */ m=1; @@ -884,7 +873,7 @@ pmix_status_t pmix_bfrops_base_unpack_pdata(pmix_buffer_t *buffer, void *dest, PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; } - (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); + pmix_strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); /* unpack value - since the value structure is statically-defined * instead of a pointer in this struct, we directly unpack it to @@ -976,7 +965,7 @@ pmix_status_t pmix_bfrops_base_unpack_proc(pmix_buffer_t *buffer, void *dest, PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; } - (void)strncpy(ptr[i].nspace, tmp, PMIX_MAX_NSLEN); + pmix_strncpy(ptr[i].nspace, tmp, PMIX_MAX_NSLEN); free(tmp); /* unpack the rank */ m=1; @@ -1105,38 +1094,6 @@ pmix_status_t pmix_bfrops_base_unpack_kval(pmix_buffer_t *buffer, void *dest, return PMIX_SUCCESS; } -pmix_status_t pmix_bfrops_base_unpack_modex(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) -{ - pmix_modex_data_t *ptr; - int32_t i, n, m; - pmix_status_t ret; - - pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, - "pmix_bfrop_unpack: %d modex", *num_vals); - - ptr = (pmix_modex_data_t *) dest; - n = *num_vals; - - for (i = 0; i < n; ++i) { - memset(&ptr[i], 0, sizeof(pmix_modex_data_t)); - /* unpack the number of bytes */ - m=1; - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_sizet(buffer, &ptr[i].size, &m, PMIX_SIZE))) { - return ret; - } - if (0 < ptr[i].size) { - ptr[i].blob = (uint8_t*)malloc(ptr[i].size * sizeof(uint8_t)); - m=ptr[i].size; - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_byte(buffer, ptr[i].blob, &m, PMIX_UINT8))) { - return ret; - } - } - } - return PMIX_SUCCESS; -} - - pmix_status_t pmix_bfrops_base_unpack_persist(pmix_buffer_t *buffer, void *dest, int32_t *num_vals, pmix_data_type_t type) { @@ -1524,17 +1481,6 @@ pmix_status_t pmix_bfrops_base_unpack_darray(pmix_buffer_t *buffer, void *dest, return ret; } break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - ptr[i].array = (pmix_info_array_t*)malloc(m * sizeof(pmix_info_array_t)); - if (NULL == ptr[i].array) { - return PMIX_ERR_NOMEM; - } - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_array(buffer, ptr[i].array, &m, ptr[i].type))) { - return ret; - } - break; - /********************/ default: return PMIX_ERR_NOT_SUPPORTED; } @@ -1602,38 +1548,3 @@ pmix_status_t pmix_bfrops_base_unpack_alloc_directive(pmix_buffer_t *buffer, voi { return pmix_bfrops_base_unpack_byte(buffer, dest, num_vals, PMIX_UINT8); } - - -/**** DEPRECATED ****/ -pmix_status_t pmix_bfrops_base_unpack_array(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) -{ - pmix_info_array_t *ptr; - int32_t i, n, m; - pmix_status_t ret; - - pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, - "pmix_bfrop_unpack: %d info arrays", *num_vals); - - ptr = (pmix_info_array_t*) dest; - n = *num_vals; - - for (i = 0; i < n; ++i) { - pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, - "pmix_bfrop_unpack: init array[%d]", i); - memset(&ptr[i], 0, sizeof(pmix_info_array_t)); - /* unpack the size of this array */ - m=1; - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_sizet(buffer, &ptr[i].size, &m, PMIX_SIZE))) { - return ret; - } - if (0 < ptr[i].size) { - ptr[i].array = (pmix_info_t*)malloc(ptr[i].size * sizeof(pmix_info_t)); - m=ptr[i].size; - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_value(buffer, ptr[i].array, &m, PMIX_INFO))) { - return ret; - } - } - } - return PMIX_SUCCESS; -} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/bfrops.h b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/bfrops.h index 3aeeebfe12e..edf4cb96e05 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/bfrops.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/bfrops.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -32,7 +32,7 @@ #define PMIX_BFROP_H_ #include - +#include #include #include "src/mca/mca.h" @@ -100,11 +100,6 @@ BEGIN_C_DECLS * NOTE: THESE FUNCTIONS ARE NOT TO BE USED INTERNALLY - * USE THE MACROS INSTEAD */ -pmix_status_t pmix_value_xfer(pmix_value_t *kv, pmix_value_t *src); -void pmix_value_load(pmix_value_t *v, const void *data, - pmix_data_type_t type); -pmix_status_t pmix_value_unload(pmix_value_t *kv, void **data, - size_t *sz, pmix_data_type_t type); bool pmix_value_cmp(pmix_value_t *p, pmix_value_t *p1); @@ -329,7 +324,7 @@ typedef pmix_status_t (*pmix_bfrop_print_fn_t)(char **output, char *prefix, * @retval PMIX_ERROR(s) An appropriate error code */ typedef pmix_status_t (*pmix_bfrop_value_xfer_fn_t)(pmix_value_t *dest, - pmix_value_t *src); + const pmix_value_t *src); /** diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/Makefile.am index db788f3a7c8..5855a6aeae3 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -51,6 +51,9 @@ mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component) mca_bfrops_v12_la_SOURCES = $(component_sources) mca_bfrops_v12_la_LDFLAGS = -module -avoid-version +if NEED_LIBPMIX +mca_bfrops_v12_la_LIBADD = $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_bfrops_v12_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/copy.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/copy.c index d833e4bdc86..47f4f674824 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/copy.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/copy.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -225,10 +225,8 @@ pmix_value_cmp_t pmix12_bfrop_value_cmp(pmix_value_t *p, pmix_value_t *p1) return PMIX_VALUE1_GREATER; } /* COPY FUNCTIONS FOR GENERIC PMIX TYPES */ -pmix_status_t pmix12_bfrop_value_xfer(pmix_value_t *p, pmix_value_t *src) +pmix_status_t pmix12_bfrop_value_xfer(pmix_value_t *p, const pmix_value_t *src) { - pmix_info_t *p1, *s1; - /* copy the right field */ p->type = src->type; switch (src->type) { @@ -300,22 +298,7 @@ pmix_status_t pmix12_bfrop_value_xfer(pmix_value_t *p, pmix_value_t *src) p->data.tv.tv_usec = src->data.tv.tv_usec; break; case PMIX_INFO_ARRAY: - p->data.array = (pmix_info_array_t*)malloc(sizeof(pmix_info_array_t)); - if (NULL == p->data.array) { - return PMIX_ERR_NOMEM; - } - p->data.array->size = src->data.array->size; - if (0 < src->data.array->size) { - p->data.array->array = (pmix_info_t*)malloc(src->data.array->size * sizeof(pmix_info_t)); - if (NULL == p->data.array->array) { - free(p->data.array); - return PMIX_ERR_NOMEM; - } - p1 = (pmix_info_t*)p->data.array->array; - s1 = (pmix_info_t*)src->data.array->array; - memcpy(p1, s1, src->data.array->size * sizeof(pmix_info_t)); - } - break; + return PMIX_ERR_NOT_SUPPORTED; case PMIX_BYTE_OBJECT: if (NULL != src->data.bo.bytes && 0 < src->data.bo.size) { p->data.bo.bytes = malloc(src->data.bo.size); @@ -356,7 +339,7 @@ pmix_status_t pmix12_bfrop_copy_info(pmix_info_t **dest, pmix_info_t *src, pmix_data_type_t type) { *dest = (pmix_info_t*)malloc(sizeof(pmix_info_t)); - (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); return pmix_value_xfer(&(*dest)->value, &src->value); } @@ -381,7 +364,7 @@ pmix_status_t pmix12_bfrop_copy_app(pmix_app_t **dest, pmix_app_t *src, (*dest)->ninfo = src->ninfo; (*dest)->info = (pmix_info_t*)malloc(src->ninfo * sizeof(pmix_info_t)); for (j=0; j < src->ninfo; j++) { - (void)strncpy((*dest)->info[j].key, src->info[j].key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->info[j].key, src->info[j].key, PMIX_MAX_KEYLEN); pmix_value_xfer(&(*dest)->info[j].value, &src->info[j].value); } return PMIX_SUCCESS; @@ -427,7 +410,7 @@ pmix_status_t pmix12_bfrop_copy_proc(pmix_proc_t **dest, pmix_proc_t *src, if (NULL == *dest) { return PMIX_ERR_OUT_OF_RESOURCE; } - (void)strncpy((*dest)->nspace, src->nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->nspace, src->nspace, PMIX_MAX_NSLEN); (*dest)->rank = src->rank; return PMIX_SUCCESS; } @@ -484,9 +467,9 @@ pmix_status_t pmix12_bfrop_copy_pdata(pmix_pdata_t **dest, pmix_data_type_t type) { *dest = (pmix_pdata_t*)malloc(sizeof(pmix_pdata_t)); - (void)strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); (*dest)->proc.rank = src->proc.rank; - (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); return pmix_value_xfer(&(*dest)->value, &src->value); } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/internal.h b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/internal.h index 78dbf47666f..e5989ed6ddc 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/internal.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/internal.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -58,7 +58,7 @@ pmix_status_t pmix12_bfrop_print(char **output, char *prefix, void *src, pmix_da pmix_status_t pmix12_bfrop_copy_payload(pmix_buffer_t *dest, pmix_buffer_t *src); -pmix_status_t pmix12_bfrop_value_xfer(pmix_value_t *p, pmix_value_t *src); +pmix_status_t pmix12_bfrop_value_xfer(pmix_value_t *p, const pmix_value_t *src); void pmix12_bfrop_value_load(pmix_value_t *v, const void *data, pmix_data_type_t type); diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/pack.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/pack.c index 07f9a74a27d..520b8984012 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/pack.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/pack.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2013 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -537,11 +537,6 @@ static pmix_status_t pack_val(pmix_buffer_t *buffer, return ret; } break; - case PMIX_INFO_ARRAY: - if (PMIX_SUCCESS != (ret = pmix12_bfrop_pack_buffer(buffer, &p->data.array, 1, PMIX_INFO_ARRAY))) { - return ret; - } - break; case PMIX_BYTE_OBJECT: if (PMIX_SUCCESS != (ret = pmix12_bfrop_pack_buffer(buffer, &p->data.bo, 1, PMIX_BYTE_OBJECT))) { return ret; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/unpack.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/unpack.c index 5604637f1b7..667f9c64841 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/unpack.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v12/unpack.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -119,7 +119,6 @@ pmix_status_t pmix12_bfrop_unpack(pmix_buffer_t *buffer, *num_vals = 0; /* don't error log here as the user may be unpacking past * the end of the buffer, which isn't necessarily an error */ - PMIX_ERROR_LOG(rc); return rc; } @@ -670,7 +669,13 @@ static pmix_status_t unpack_val(pmix_buffer_t *buffer, pmix_value_t *val) } break; case PMIX_INFO_ARRAY: - if (PMIX_SUCCESS != (ret = pmix12_bfrop_unpack_buffer(buffer, &val->data.array, &m, PMIX_INFO_ARRAY))) { + /* we don't know anything about info array's so we + * have to convert this to a data array */ + val->data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); + val->data.darray->type = PMIX_INFO_ARRAY; + val->data.darray->size = m; + /* unpack into it */ + if (PMIX_SUCCESS != (ret = pmix12_bfrop_unpack_buffer(buffer, &val->data.darray->array, &m, PMIX_INFO_ARRAY))) { return ret; } break; @@ -743,7 +748,7 @@ pmix_status_t pmix12_bfrop_unpack_info(pmix_buffer_t *buffer, void *dest, if (NULL == tmp) { return PMIX_ERROR; } - (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); + pmix_strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); /* unpack value - since the value structure is statically-defined * instead of a pointer in this struct, we directly unpack it to @@ -797,7 +802,7 @@ pmix_status_t pmix12_bfrop_unpack_pdata(pmix_buffer_t *buffer, void *dest, if (NULL == tmp) { return PMIX_ERROR; } - (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); + pmix_strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); /* unpack value - since the value structure is statically-defined * instead of a pointer in this struct, we directly unpack it to @@ -881,7 +886,7 @@ pmix_status_t pmix12_bfrop_unpack_proc(pmix_buffer_t *buffer, void *dest, if (NULL == tmp) { return PMIX_ERROR; } - (void)strncpy(ptr[i].nspace, tmp, PMIX_MAX_NSLEN); + pmix_strncpy(ptr[i].nspace, tmp, PMIX_MAX_NSLEN); free(tmp); /* unpack the rank */ m=1; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/Makefile.am index ffe16123206..42eb14940aa 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -51,6 +51,9 @@ mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component) mca_bfrops_v20_la_SOURCES = $(component_sources) mca_bfrops_v20_la_LDFLAGS = -module -avoid-version +if NEED_LIBPMIX +mca_bfrops_v20_la_LIBADD = $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_bfrops_v20_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/copy.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/copy.c index 53fbf9b264c..a5c945d96bc 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/copy.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/copy.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -337,7 +337,7 @@ bool pmix_value_cmp(pmix_value_t *p, pmix_value_t *p1) /* COPY FUNCTIONS FOR GENERIC PMIX TYPES - we * are not allocating memory and so we cannot * use the regular copy functions */ -pmix_status_t pmix20_bfrop_value_xfer(pmix_value_t *p, pmix_value_t *src) +pmix_status_t pmix20_bfrop_value_xfer(pmix_value_t *p, const pmix_value_t *src) { size_t n, m; pmix_status_t rc; @@ -356,518 +356,506 @@ pmix_status_t pmix20_bfrop_value_xfer(pmix_value_t *p, pmix_value_t *src) /* copy the right field */ p->type = src->type; switch (src->type) { - case PMIX_UNDEF: - break; - case PMIX_BOOL: - p->data.flag = src->data.flag; - break; - case PMIX_BYTE: - p->data.byte = src->data.byte; - break; - case PMIX_STRING: - if (NULL != src->data.string) { - p->data.string = strdup(src->data.string); - } else { - p->data.string = NULL; - } - break; - case PMIX_SIZE: - p->data.size = src->data.size; - break; - case PMIX_PID: - p->data.pid = src->data.pid; - break; - case PMIX_INT: - /* to avoid alignment issues */ - memcpy(&p->data.integer, &src->data.integer, sizeof(int)); - break; - case PMIX_INT8: - p->data.int8 = src->data.int8; - break; - case PMIX_INT16: - /* to avoid alignment issues */ - memcpy(&p->data.int16, &src->data.int16, 2); - break; - case PMIX_INT32: - /* to avoid alignment issues */ - memcpy(&p->data.int32, &src->data.int32, 4); - break; - case PMIX_INT64: - /* to avoid alignment issues */ - memcpy(&p->data.int64, &src->data.int64, 8); - break; - case PMIX_UINT: - /* to avoid alignment issues */ - memcpy(&p->data.uint, &src->data.uint, sizeof(unsigned int)); - break; - case PMIX_UINT8: - p->data.uint8 = src->data.uint8; - break; - case PMIX_UINT16: - /* to avoid alignment issues */ - memcpy(&p->data.uint16, &src->data.uint16, 2); - break; - case PMIX_UINT32: - /* to avoid alignment issues */ - memcpy(&p->data.uint32, &src->data.uint32, 4); - break; - case PMIX_UINT64: - /* to avoid alignment issues */ - memcpy(&p->data.uint64, &src->data.uint64, 8); - break; - case PMIX_FLOAT: - p->data.fval = src->data.fval; - break; - case PMIX_DOUBLE: - p->data.dval = src->data.dval; - break; - case PMIX_TIMEVAL: - memcpy(&p->data.tv, &src->data.tv, sizeof(struct timeval)); - break; - case PMIX_TIME: - memcpy(&p->data.time, &src->data.time, sizeof(time_t)); - break; - case PMIX_STATUS: - memcpy(&p->data.status, &src->data.status, sizeof(pmix_status_t)); - break; - case PMIX_PROC: - memcpy(&p->data.proc, &src->data.proc, sizeof(pmix_proc_t)); - break; - case PMIX_PROC_RANK: - memcpy(&p->data.proc, &src->data.rank, sizeof(pmix_rank_t)); - break; - case PMIX_BYTE_OBJECT: - case PMIX_COMPRESSED_STRING: - memset(&p->data.bo, 0, sizeof(pmix_byte_object_t)); - if (NULL != src->data.bo.bytes && 0 < src->data.bo.size) { - p->data.bo.bytes = malloc(src->data.bo.size); - memcpy(p->data.bo.bytes, src->data.bo.bytes, src->data.bo.size); - p->data.bo.size = src->data.bo.size; - } else { - p->data.bo.bytes = NULL; - p->data.bo.size = 0; - } - break; - case PMIX_PERSIST: - memcpy(&p->data.persist, &src->data.persist, sizeof(pmix_persistence_t)); - break; - case PMIX_SCOPE: - memcpy(&p->data.scope, &src->data.scope, sizeof(pmix_scope_t)); - break; - case PMIX_DATA_RANGE: - memcpy(&p->data.range, &src->data.range, sizeof(pmix_data_range_t)); - break; - case PMIX_PROC_STATE: - memcpy(&p->data.state, &src->data.state, sizeof(pmix_proc_state_t)); - break; - case PMIX_PROC_INFO: - PMIX_PROC_INFO_CREATE(p->data.pinfo, 1); - if (NULL != src->data.pinfo->hostname) { - p->data.pinfo->hostname = strdup(src->data.pinfo->hostname); - } - if (NULL != src->data.pinfo->executable_name) { - p->data.pinfo->executable_name = strdup(src->data.pinfo->executable_name); - } - memcpy(&p->data.pinfo->pid, &src->data.pinfo->pid, sizeof(pid_t)); - memcpy(&p->data.pinfo->exit_code, &src->data.pinfo->exit_code, sizeof(int)); - memcpy(&p->data.pinfo->state, &src->data.pinfo->state, sizeof(pmix_proc_state_t)); - break; - case PMIX_DATA_ARRAY: - p->data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); - p->data.darray->type = src->data.darray->type; - p->data.darray->size = src->data.darray->size; - if (0 == p->data.darray->size || NULL == src->data.darray->array) { - p->data.darray->array = NULL; - p->data.darray->size = 0; + case PMIX_UNDEF: break; - } - /* allocate space and do the copy */ - switch (src->data.darray->type) { - case PMIX_UINT8: - case PMIX_INT8: - case PMIX_BYTE: - p->data.darray->array = (char*)malloc(src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size); - break; - case PMIX_UINT16: - case PMIX_INT16: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint16_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint16_t)); - break; - case PMIX_UINT32: - case PMIX_INT32: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint32_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint32_t)); - break; - case PMIX_UINT64: - case PMIX_INT64: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint64_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint64_t)); - break; - case PMIX_BOOL: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(bool)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(bool)); - break; - case PMIX_SIZE: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(size_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(size_t)); - break; - case PMIX_PID: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(pid_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pid_t)); + case PMIX_BOOL: + p->data.flag = src->data.flag; + break; + case PMIX_BYTE: + p->data.byte = src->data.byte; + break; + case PMIX_STRING: + if (NULL != src->data.string) { + p->data.string = strdup(src->data.string); + } else { + p->data.string = NULL; + } + break; + case PMIX_SIZE: + p->data.size = src->data.size; + break; + case PMIX_PID: + p->data.pid = src->data.pid; + break; + case PMIX_INT: + /* to avoid alignment issues */ + memcpy(&p->data.integer, &src->data.integer, sizeof(int)); + break; + case PMIX_INT8: + p->data.int8 = src->data.int8; + break; + case PMIX_INT16: + /* to avoid alignment issues */ + memcpy(&p->data.int16, &src->data.int16, 2); + break; + case PMIX_INT32: + /* to avoid alignment issues */ + memcpy(&p->data.int32, &src->data.int32, 4); + break; + case PMIX_INT64: + /* to avoid alignment issues */ + memcpy(&p->data.int64, &src->data.int64, 8); + break; + case PMIX_UINT: + /* to avoid alignment issues */ + memcpy(&p->data.uint, &src->data.uint, sizeof(unsigned int)); + break; + case PMIX_UINT8: + p->data.uint8 = src->data.uint8; + break; + case PMIX_UINT16: + /* to avoid alignment issues */ + memcpy(&p->data.uint16, &src->data.uint16, 2); + break; + case PMIX_UINT32: + /* to avoid alignment issues */ + memcpy(&p->data.uint32, &src->data.uint32, 4); + break; + case PMIX_UINT64: + /* to avoid alignment issues */ + memcpy(&p->data.uint64, &src->data.uint64, 8); + break; + case PMIX_FLOAT: + p->data.fval = src->data.fval; + break; + case PMIX_DOUBLE: + p->data.dval = src->data.dval; + break; + case PMIX_TIMEVAL: + memcpy(&p->data.tv, &src->data.tv, sizeof(struct timeval)); + break; + case PMIX_TIME: + memcpy(&p->data.time, &src->data.time, sizeof(time_t)); + break; + case PMIX_STATUS: + memcpy(&p->data.status, &src->data.status, sizeof(pmix_status_t)); + break; + case PMIX_PROC: + memcpy(&p->data.proc, &src->data.proc, sizeof(pmix_proc_t)); + break; + case PMIX_PROC_RANK: + memcpy(&p->data.proc, &src->data.rank, sizeof(pmix_rank_t)); + break; + case PMIX_BYTE_OBJECT: + case PMIX_COMPRESSED_STRING: + memset(&p->data.bo, 0, sizeof(pmix_byte_object_t)); + if (NULL != src->data.bo.bytes && 0 < src->data.bo.size) { + p->data.bo.bytes = malloc(src->data.bo.size); + memcpy(p->data.bo.bytes, src->data.bo.bytes, src->data.bo.size); + p->data.bo.size = src->data.bo.size; + } else { + p->data.bo.bytes = NULL; + p->data.bo.size = 0; + } + break; + case PMIX_PERSIST: + memcpy(&p->data.persist, &src->data.persist, sizeof(pmix_persistence_t)); + break; + case PMIX_SCOPE: + memcpy(&p->data.scope, &src->data.scope, sizeof(pmix_scope_t)); + break; + case PMIX_DATA_RANGE: + memcpy(&p->data.range, &src->data.range, sizeof(pmix_data_range_t)); + break; + case PMIX_PROC_STATE: + memcpy(&p->data.state, &src->data.state, sizeof(pmix_proc_state_t)); + break; + case PMIX_PROC_INFO: + PMIX_PROC_INFO_CREATE(p->data.pinfo, 1); + if (NULL != src->data.pinfo->hostname) { + p->data.pinfo->hostname = strdup(src->data.pinfo->hostname); + } + if (NULL != src->data.pinfo->executable_name) { + p->data.pinfo->executable_name = strdup(src->data.pinfo->executable_name); + } + memcpy(&p->data.pinfo->pid, &src->data.pinfo->pid, sizeof(pid_t)); + memcpy(&p->data.pinfo->exit_code, &src->data.pinfo->exit_code, sizeof(int)); + memcpy(&p->data.pinfo->state, &src->data.pinfo->state, sizeof(pmix_proc_state_t)); + break; + case PMIX_DATA_ARRAY: + p->data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); + p->data.darray->type = src->data.darray->type; + p->data.darray->size = src->data.darray->size; + if (0 == p->data.darray->size || NULL == src->data.darray->array) { + p->data.darray->array = NULL; + p->data.darray->size = 0; break; - case PMIX_STRING: - p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - prarray = (char**)p->data.darray->array; - strarray = (char**)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != strarray[n]) { - prarray[n] = strdup(strarray[n]); + } + /* allocate space and do the copy */ + switch (src->data.darray->type) { + case PMIX_UINT8: + case PMIX_INT8: + case PMIX_BYTE: + p->data.darray->array = (char*)malloc(src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - } - break; - case PMIX_INT: - case PMIX_UINT: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(int)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(int)); - break; - case PMIX_FLOAT: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(float)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(float)); - break; - case PMIX_DOUBLE: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(double)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(double)); - break; - case PMIX_TIMEVAL: - p->data.darray->array = (struct timeval*)malloc(src->data.darray->size * sizeof(struct timeval)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(struct timeval)); - break; - case PMIX_TIME: - p->data.darray->array = (time_t*)malloc(src->data.darray->size * sizeof(time_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(time_t)); - break; - case PMIX_STATUS: - p->data.darray->array = (pmix_status_t*)malloc(src->data.darray->size * sizeof(pmix_status_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_status_t)); - break; - case PMIX_VALUE: - PMIX_VALUE_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pv = (pmix_value_t*)p->data.darray->array; - sv = (pmix_value_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (PMIX_SUCCESS != (rc = pmix20_bfrop_value_xfer(&pv[n], &sv[n]))) { - PMIX_VALUE_FREE(pv, src->data.darray->size); - return rc; + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size); + break; + case PMIX_UINT16: + case PMIX_INT16: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint16_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - } - break; - case PMIX_PROC: - PMIX_PROC_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_proc_t)); - break; - case PMIX_APP: - PMIX_APP_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pa = (pmix_app_t*)p->data.darray->array; - sa = (pmix_app_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sa[n].cmd) { - pa[n].cmd = strdup(sa[n].cmd); + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint16_t)); + break; + case PMIX_UINT32: + case PMIX_INT32: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint32_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - if (NULL != sa[n].argv) { - pa[n].argv = pmix_argv_copy(sa[n].argv); + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint32_t)); + break; + case PMIX_UINT64: + case PMIX_INT64: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint64_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - if (NULL != sa[n].env) { - pa[n].env = pmix_argv_copy(sa[n].env); + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint64_t)); + break; + case PMIX_BOOL: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(bool)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - if (NULL != sa[n].cwd) { - pa[n].cwd = strdup(sa[n].cwd); + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(bool)); + break; + case PMIX_SIZE: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(size_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(size_t)); + break; + case PMIX_PID: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(pid_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - pa[n].maxprocs = sa[n].maxprocs; - if (0 < sa[n].ninfo && NULL != sa[n].info) { - PMIX_INFO_CREATE(pa[n].info, sa[n].ninfo); - if (NULL == pa[n].info) { - PMIX_APP_FREE(pa, src->data.darray->size); - return PMIX_ERR_NOMEM; + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pid_t)); + break; + case PMIX_STRING: + p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + prarray = (char**)p->data.darray->array; + strarray = (char**)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + if (NULL != strarray[n]) { + prarray[n] = strdup(strarray[n]); } - pa[n].ninfo = sa[n].ninfo; - for (m=0; m < pa[n].ninfo; m++) { - PMIX_INFO_XFER(&pa[n].info[m], &sa[n].info[m]); + } + break; + case PMIX_INT: + case PMIX_UINT: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(int)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(int)); + break; + case PMIX_FLOAT: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(float)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(float)); + break; + case PMIX_DOUBLE: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(double)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(double)); + break; + case PMIX_TIMEVAL: + p->data.darray->array = (struct timeval*)malloc(src->data.darray->size * sizeof(struct timeval)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(struct timeval)); + break; + case PMIX_TIME: + p->data.darray->array = (time_t*)malloc(src->data.darray->size * sizeof(time_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(time_t)); + break; + case PMIX_STATUS: + p->data.darray->array = (pmix_status_t*)malloc(src->data.darray->size * sizeof(pmix_status_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_status_t)); + break; + case PMIX_VALUE: + PMIX_VALUE_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pv = (pmix_value_t*)p->data.darray->array; + sv = (pmix_value_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + if (PMIX_SUCCESS != (rc = pmix20_bfrop_value_xfer(&pv[n], &sv[n]))) { + PMIX_VALUE_FREE(pv, src->data.darray->size); + return rc; } } - } - break; - case PMIX_INFO: - PMIX_INFO_CREATE(p->data.darray->array, src->data.darray->size); - p1 = (pmix_info_t*)p->data.darray->array; - s1 = (pmix_info_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_INFO_LOAD(&p1[n], s1[n].key, &s1[n].value.data.flag, s1[n].value.type); - } - break; - case PMIX_PDATA: - PMIX_PDATA_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pd = (pmix_pdata_t*)p->data.darray->array; - sd = (pmix_pdata_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_PDATA_LOAD(&pd[n], &sd[n].proc, sd[n].key, &sd[n].value.data.flag, sd[n].value.type); - } - break; - case PMIX_BUFFER: - p->data.darray->array = (pmix_buffer_t*)malloc(src->data.darray->size * sizeof(pmix_buffer_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pb = (pmix_buffer_t*)p->data.darray->array; - sb = (pmix_buffer_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_CONSTRUCT(&pb[n], pmix_buffer_t); - pmix20_bfrop_copy_payload(&pb[n], &sb[n]); - } - break; - case PMIX_BYTE_OBJECT: - case PMIX_COMPRESSED_STRING: - p->data.darray->array = (pmix_byte_object_t*)malloc(src->data.darray->size * sizeof(pmix_byte_object_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pbo = (pmix_byte_object_t*)p->data.darray->array; - sbo = (pmix_byte_object_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sbo[n].bytes && 0 < sbo[n].size) { - pbo[n].size = sbo[n].size; - pbo[n].bytes = (char*)malloc(pbo[n].size); - memcpy(pbo[n].bytes, sbo[n].bytes, pbo[n].size); - } else { - pbo[n].bytes = NULL; - pbo[n].size = 0; + break; + case PMIX_PROC: + PMIX_PROC_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - } - break; - case PMIX_KVAL: - p->data.darray->array = (pmix_kval_t*)calloc(src->data.darray->size , sizeof(pmix_kval_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pk = (pmix_kval_t*)p->data.darray->array; - sk = (pmix_kval_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sk[n].key) { - pk[n].key = strdup(sk[n].key); + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_proc_t)); + break; + case PMIX_APP: + PMIX_APP_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - if (NULL != sk[n].value) { - PMIX_VALUE_CREATE(pk[n].value, 1); - if (NULL == pk[n].value) { - free(p->data.darray->array); - return PMIX_ERR_NOMEM; + pa = (pmix_app_t*)p->data.darray->array; + sa = (pmix_app_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + if (NULL != sa[n].cmd) { + pa[n].cmd = strdup(sa[n].cmd); } - if (PMIX_SUCCESS != (rc = pmix20_bfrop_value_xfer(pk[n].value, sk[n].value))) { - return rc; + if (NULL != sa[n].argv) { + pa[n].argv = pmix_argv_copy(sa[n].argv); + } + if (NULL != sa[n].env) { + pa[n].env = pmix_argv_copy(sa[n].env); + } + if (NULL != sa[n].cwd) { + pa[n].cwd = strdup(sa[n].cwd); + } + pa[n].maxprocs = sa[n].maxprocs; + if (0 < sa[n].ninfo && NULL != sa[n].info) { + PMIX_INFO_CREATE(pa[n].info, sa[n].ninfo); + if (NULL == pa[n].info) { + PMIX_APP_FREE(pa, src->data.darray->size); + return PMIX_ERR_NOMEM; + } + pa[n].ninfo = sa[n].ninfo; + for (m=0; m < pa[n].ninfo; m++) { + PMIX_INFO_XFER(&pa[n].info[m], &sa[n].info[m]); + } } } - } - break; - case PMIX_MODEX: - PMIX_MODEX_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pm = (pmix_modex_data_t*)p->data.darray->array; - sm = (pmix_modex_data_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - memcpy(&pm[n], &sm[n], sizeof(pmix_modex_data_t)); - if (NULL != sm[n].blob && 0 < sm[n].size) { - pm[n].blob = (uint8_t*)malloc(sm[n].size); - if (NULL == pm[n].blob) { - return PMIX_ERR_NOMEM; + break; + case PMIX_INFO: + PMIX_INFO_CREATE(p->data.darray->array, src->data.darray->size); + p1 = (pmix_info_t*)p->data.darray->array; + s1 = (pmix_info_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + PMIX_INFO_LOAD(&p1[n], s1[n].key, &s1[n].value.data.flag, s1[n].value.type); + } + break; + case PMIX_PDATA: + PMIX_PDATA_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pd = (pmix_pdata_t*)p->data.darray->array; + sd = (pmix_pdata_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + PMIX_PDATA_LOAD(&pd[n], &sd[n].proc, sd[n].key, &sd[n].value.data.flag, sd[n].value.type); + } + break; + case PMIX_BUFFER: + p->data.darray->array = (pmix_buffer_t*)malloc(src->data.darray->size * sizeof(pmix_buffer_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pb = (pmix_buffer_t*)p->data.darray->array; + sb = (pmix_buffer_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + PMIX_CONSTRUCT(&pb[n], pmix_buffer_t); + pmix20_bfrop_copy_payload(&pb[n], &sb[n]); + } + break; + case PMIX_BYTE_OBJECT: + case PMIX_COMPRESSED_STRING: + p->data.darray->array = (pmix_byte_object_t*)malloc(src->data.darray->size * sizeof(pmix_byte_object_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pbo = (pmix_byte_object_t*)p->data.darray->array; + sbo = (pmix_byte_object_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + if (NULL != sbo[n].bytes && 0 < sbo[n].size) { + pbo[n].size = sbo[n].size; + pbo[n].bytes = (char*)malloc(pbo[n].size); + memcpy(pbo[n].bytes, sbo[n].bytes, pbo[n].size); + } else { + pbo[n].bytes = NULL; + pbo[n].size = 0; } - memcpy(pm[n].blob, sm[n].blob, sm[n].size); - pm[n].size = sm[n].size; - } else { - pm[n].blob = NULL; - pm[n].size = 0; } - } - break; - case PMIX_PERSIST: - p->data.darray->array = (pmix_persistence_t*)malloc(src->data.darray->size * sizeof(pmix_persistence_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_persistence_t)); - break; - case PMIX_POINTER: - p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - prarray = (char**)p->data.darray->array; - strarray = (char**)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - prarray[n] = strarray[n]; - } - break; - case PMIX_SCOPE: - p->data.darray->array = (pmix_scope_t*)malloc(src->data.darray->size * sizeof(pmix_scope_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_scope_t)); - break; - case PMIX_DATA_RANGE: - p->data.darray->array = (pmix_data_range_t*)malloc(src->data.darray->size * sizeof(pmix_data_range_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_data_range_t)); - break; - case PMIX_COMMAND: - p->data.darray->array = (pmix_cmd_t*)malloc(src->data.darray->size * sizeof(pmix_cmd_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_cmd_t)); - break; - case PMIX_INFO_DIRECTIVES: - p->data.darray->array = (pmix_info_directives_t*)malloc(src->data.darray->size * sizeof(pmix_info_directives_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_info_directives_t)); - break; - case PMIX_PROC_INFO: - PMIX_PROC_INFO_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pi = (pmix_proc_info_t*)p->data.darray->array; - si = (pmix_proc_info_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - memcpy(&pi[n].proc, &si[n].proc, sizeof(pmix_proc_t)); - if (NULL != si[n].hostname) { - pi[n].hostname = strdup(si[n].hostname); - } else { - pi[n].hostname = NULL; + break; + case PMIX_KVAL: + p->data.darray->array = (pmix_kval_t*)calloc(src->data.darray->size , sizeof(pmix_kval_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - if (NULL != si[n].executable_name) { - pi[n].executable_name = strdup(si[n].executable_name); - } else { - pi[n].executable_name = NULL; + pk = (pmix_kval_t*)p->data.darray->array; + sk = (pmix_kval_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + if (NULL != sk[n].key) { + pk[n].key = strdup(sk[n].key); + } + if (NULL != sk[n].value) { + PMIX_VALUE_CREATE(pk[n].value, 1); + if (NULL == pk[n].value) { + free(p->data.darray->array); + return PMIX_ERR_NOMEM; + } + if (PMIX_SUCCESS != (rc = pmix20_bfrop_value_xfer(pk[n].value, sk[n].value))) { + return rc; + } + } } - pi[n].pid = si[n].pid; - pi[n].exit_code = si[n].exit_code; - pi[n].state = si[n].state; - } - break; - case PMIX_DATA_ARRAY: - return PMIX_ERR_NOT_SUPPORTED; // don't support iterative arrays - case PMIX_QUERY: - PMIX_QUERY_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pq = (pmix_query_t*)p->data.darray->array; - sq = (pmix_query_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sq[n].keys) { - pq[n].keys = pmix_argv_copy(sq[n].keys); + break; + case PMIX_MODEX: + PMIX_MODEX_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pm = (pmix_modex_data_t*)p->data.darray->array; + sm = (pmix_modex_data_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + memcpy(&pm[n], &sm[n], sizeof(pmix_modex_data_t)); + if (NULL != sm[n].blob && 0 < sm[n].size) { + pm[n].blob = (uint8_t*)malloc(sm[n].size); + if (NULL == pm[n].blob) { + return PMIX_ERR_NOMEM; + } + memcpy(pm[n].blob, sm[n].blob, sm[n].size); + pm[n].size = sm[n].size; + } else { + pm[n].blob = NULL; + pm[n].size = 0; + } + } + break; + case PMIX_PERSIST: + p->data.darray->array = (pmix_persistence_t*)malloc(src->data.darray->size * sizeof(pmix_persistence_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - if (NULL != sq[n].qualifiers && 0 < sq[n].nqual) { - PMIX_INFO_CREATE(pq[n].qualifiers, sq[n].nqual); - if (NULL == pq[n].qualifiers) { - PMIX_QUERY_FREE(pq, src->data.darray->size); - return PMIX_ERR_NOMEM; + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_persistence_t)); + break; + case PMIX_POINTER: + p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + prarray = (char**)p->data.darray->array; + strarray = (char**)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + prarray[n] = strarray[n]; + } + break; + case PMIX_SCOPE: + p->data.darray->array = (pmix_scope_t*)malloc(src->data.darray->size * sizeof(pmix_scope_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_scope_t)); + break; + case PMIX_DATA_RANGE: + p->data.darray->array = (pmix_data_range_t*)malloc(src->data.darray->size * sizeof(pmix_data_range_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_data_range_t)); + break; + case PMIX_COMMAND: + p->data.darray->array = (pmix_cmd_t*)malloc(src->data.darray->size * sizeof(pmix_cmd_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_cmd_t)); + break; + case PMIX_INFO_DIRECTIVES: + p->data.darray->array = (pmix_info_directives_t*)malloc(src->data.darray->size * sizeof(pmix_info_directives_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_info_directives_t)); + break; + case PMIX_PROC_INFO: + PMIX_PROC_INFO_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pi = (pmix_proc_info_t*)p->data.darray->array; + si = (pmix_proc_info_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + memcpy(&pi[n].proc, &si[n].proc, sizeof(pmix_proc_t)); + if (NULL != si[n].hostname) { + pi[n].hostname = strdup(si[n].hostname); + } else { + pi[n].hostname = NULL; } - for (m=0; m < sq[n].nqual; m++) { - PMIX_INFO_XFER(&pq[n].qualifiers[m], &sq[n].qualifiers[m]); + if (NULL != si[n].executable_name) { + pi[n].executable_name = strdup(si[n].executable_name); + } else { + pi[n].executable_name = NULL; } - pq[n].nqual = sq[n].nqual; - } else { - pq[n].qualifiers = NULL; - pq[n].nqual = 0; + pi[n].pid = si[n].pid; + pi[n].exit_code = si[n].exit_code; + pi[n].state = si[n].state; } - } - break; - default: - return PMIX_ERR_UNKNOWN_DATA_TYPE; - } - break; - case PMIX_POINTER: - memcpy(&p->data.ptr, &src->data.ptr, sizeof(void*)); - break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - p->data.array->size = src->data.array->size; - if (0 < src->data.array->size) { - p->data.array->array = (pmix_info_t*)malloc(src->data.array->size * sizeof(pmix_info_t)); - if (NULL == p->data.array->array) { - return PMIX_ERR_NOMEM; - } - p1 = (pmix_info_t*)p->data.array->array; - s1 = (pmix_info_t*)src->data.array->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_INFO_LOAD(&p1[n], s1[n].key, &s1[n].value.data.flag, s1[n].value.type); + break; + case PMIX_DATA_ARRAY: + return PMIX_ERR_NOT_SUPPORTED; // don't support iterative arrays + case PMIX_QUERY: + PMIX_QUERY_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pq = (pmix_query_t*)p->data.darray->array; + sq = (pmix_query_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + if (NULL != sq[n].keys) { + pq[n].keys = pmix_argv_copy(sq[n].keys); + } + if (NULL != sq[n].qualifiers && 0 < sq[n].nqual) { + PMIX_INFO_CREATE(pq[n].qualifiers, sq[n].nqual); + if (NULL == pq[n].qualifiers) { + PMIX_QUERY_FREE(pq, src->data.darray->size); + return PMIX_ERR_NOMEM; + } + for (m=0; m < sq[n].nqual; m++) { + PMIX_INFO_XFER(&pq[n].qualifiers[m], &sq[n].qualifiers[m]); + } + pq[n].nqual = sq[n].nqual; + } else { + pq[n].qualifiers = NULL; + pq[n].nqual = 0; + } + } + break; + default: + return PMIX_ERR_UNKNOWN_DATA_TYPE; } - } - break; - /********************/ - default: - pmix_output(0, "COPY-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)src->type); - return PMIX_ERROR; + break; + case PMIX_POINTER: + memcpy(&p->data.ptr, &src->data.ptr, sizeof(void*)); + break; + /**** DEPRECATED ****/ + case PMIX_INFO_ARRAY: + return PMIX_ERR_NOT_SUPPORTED; + /********************/ + default: + pmix_output(0, "COPY-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)src->type); + return PMIX_ERROR; } return PMIX_SUCCESS; } @@ -895,7 +883,7 @@ pmix_status_t pmix20_bfrop_copy_info(pmix_info_t **dest, pmix_info_t *src, pmix_data_type_t type) { *dest = (pmix_info_t*)malloc(sizeof(pmix_info_t)); - (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); (*dest)->flags = src->flags; return pmix20_bfrop_value_xfer(&(*dest)->value, &src->value); } @@ -924,7 +912,7 @@ pmix_status_t pmix20_bfrop_copy_app(pmix_app_t **dest, pmix_app_t *src, (*dest)->ninfo = src->ninfo; (*dest)->info = (pmix_info_t*)malloc(src->ninfo * sizeof(pmix_info_t)); for (j=0; j < src->ninfo; j++) { - (void)strncpy((*dest)->info[j].key, src->info[j].key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->info[j].key, src->info[j].key, PMIX_MAX_KEYLEN); pmix20_bfrop_value_xfer(&(*dest)->info[j].value, &src->info[j].value); } return PMIX_SUCCESS; @@ -955,7 +943,7 @@ pmix_status_t pmix20_bfrop_copy_proc(pmix_proc_t **dest, pmix_proc_t *src, if (NULL == *dest) { return PMIX_ERR_OUT_OF_RESOURCE; } - (void)strncpy((*dest)->nspace, src->nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->nspace, src->nspace, PMIX_MAX_NSLEN); (*dest)->rank = src->rank; return PMIX_SUCCESS; } @@ -1008,9 +996,9 @@ pmix_status_t pmix20_bfrop_copy_pdata(pmix_pdata_t **dest, pmix_pdata_t *src, pmix_data_type_t type) { *dest = (pmix_pdata_t*)malloc(sizeof(pmix_pdata_t)); - (void)strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); (*dest)->proc.rank = src->proc.rank; - (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); return pmix20_bfrop_value_xfer(&(*dest)->value, &src->value); } @@ -1018,7 +1006,7 @@ pmix_status_t pmix20_bfrop_copy_pinfo(pmix_proc_info_t **dest, pmix_proc_info_t pmix_data_type_t type) { *dest = (pmix_proc_info_t*)malloc(sizeof(pmix_proc_info_t)); - (void)strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); (*dest)->proc.rank = src->proc.rank; if (NULL != src->hostname) { (*dest)->hostname = strdup(src->hostname); diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/internal.h b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/internal.h index d85ac2985c2..528bc30962c 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/internal.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/internal.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -59,7 +59,7 @@ pmix_status_t pmix20_bfrop_print(char **output, char *prefix, void *src, pmix_da pmix_status_t pmix20_bfrop_copy_payload(pmix_buffer_t *dest, pmix_buffer_t *src); -pmix_status_t pmix20_bfrop_value_xfer(pmix_value_t *p, pmix_value_t *src); +pmix_status_t pmix20_bfrop_value_xfer(pmix_value_t *p, const pmix_value_t *src); void pmix20_bfrop_value_load(pmix_value_t *v, const void *data, pmix_data_type_t type); diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/pack.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/pack.c index 91636204497..8b11452c0d5 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/pack.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/pack.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2013 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -613,13 +613,6 @@ static pmix_status_t pack_val(pmix_buffer_t *buffer, return ret; } break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - if (PMIX_SUCCESS != (ret = pmix20_bfrop_pack_buffer(buffer, p->data.array, 1, PMIX_INFO_ARRAY))) { - return ret; - } - break; - /********************/ default: pmix_output(0, "PACK-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)p->type); return PMIX_ERROR; @@ -1035,10 +1028,9 @@ pmix_status_t pmix20_bfrop_pack_alloc_directive(pmix_buffer_t *buffer, const voi return pmix20_bfrop_pack_byte(buffer, src, num_vals, PMIX_UINT8); } - /**** DEPRECATED ****/ pmix_status_t pmix20_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) + int32_t num_vals, pmix_data_type_t type) { pmix_info_array_t *ptr; int32_t i; @@ -1048,12 +1040,12 @@ pmix_status_t pmix20_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, for (i = 0; i < num_vals; ++i) { /* pack the size */ - if (PMIX_SUCCESS != (ret = pmix20_bfrop_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { return ret; } if (0 < ptr[i].size) { /* pack the values */ - if (PMIX_SUCCESS != (ret = pmix20_bfrop_pack_info(buffer, ptr[i].array, ptr[i].size, PMIX_INFO))) { + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_info(buffer, ptr[i].array, ptr[i].size, PMIX_INFO))) { return ret; } } @@ -1062,3 +1054,4 @@ pmix_status_t pmix20_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, return PMIX_SUCCESS; } /********************/ + diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/print.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/print.c index 500bd87f1c1..f6b9e253b0d 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/print.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/print.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -761,7 +761,7 @@ pmix_status_t pmix20_bfrop_print_status(char **output, char *prefix, * PMIX_VALUE */ pmix_status_t pmix20_bfrop_print_value(char **output, char *prefix, - pmix_value_t *src, pmix_data_type_t type) + pmix_value_t *src, pmix_data_type_t type) { char *prefx; int rc; @@ -904,12 +904,6 @@ pmix_status_t pmix20_bfrop_print_status(char **output, char *prefix, rc = asprintf(output, "%sPMIX_VALUE: Data type: DATA_ARRAY\tARRAY SIZE: %ld", prefx, (long)src->data.darray->size); break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - rc = asprintf(output, "%sPMIX_VALUE: Data type: INFO_ARRAY\tARRAY SIZE: %ld", - prefx, (long)src->data.array->size); - break; - /********************/ default: rc = asprintf(output, "%sPMIX_VALUE: Data type: UNKNOWN\tValue: UNPRINTABLE", prefx); break; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/unpack.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/unpack.c index f812a005bc1..91ce264fd30 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/unpack.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v20/unpack.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -756,12 +756,13 @@ pmix_status_t pmix20_bfrop_unpack_status(pmix_buffer_t *buffer, void *dest, break; /**** DEPRECATED ****/ case PMIX_INFO_ARRAY: - /* this field is now a pointer, so we must allocate storage for it */ - val->data.array = (pmix_info_array_t*)malloc(sizeof(pmix_info_array_t)); - if (NULL == val->data.array) { - return PMIX_ERR_NOMEM; - } - if (PMIX_SUCCESS != (ret = pmix20_bfrop_unpack_buffer(buffer, val->data.array, &m, PMIX_INFO_ARRAY))) { + /* we don't know anything about info array's so we + * have to convert this to a data array */ + val->data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); + val->data.darray->type = PMIX_INFO_ARRAY; + val->data.darray->size = m; + /* unpack into it */ + if (PMIX_SUCCESS != (ret = pmix20_bfrop_unpack_buffer(buffer, &val->data.darray->array, &m, PMIX_INFO_ARRAY))) { return ret; } break; @@ -825,7 +826,7 @@ pmix_status_t pmix20_bfrop_unpack_info(pmix_buffer_t *buffer, void *dest, PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; } - (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); + pmix_strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); /* unpack the flags */ m=1; @@ -882,7 +883,7 @@ pmix_status_t pmix20_bfrop_unpack_pdata(pmix_buffer_t *buffer, void *dest, if (NULL == tmp) { return PMIX_ERROR; } - (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); + pmix_strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); /* unpack value - since the value structure is statically-defined * instead of a pointer in this struct, we directly unpack it to @@ -962,7 +963,7 @@ pmix_status_t pmix20_bfrop_unpack_proc(pmix_buffer_t *buffer, void *dest, if (NULL == tmp) { return PMIX_ERROR; } - (void)strncpy(ptr[i].nspace, tmp, PMIX_MAX_NSLEN); + pmix_strncpy(ptr[i].nspace, tmp, PMIX_MAX_NSLEN); free(tmp); /* unpack the rank */ m=1; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v21/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v21/Makefile.am index d4da3258b44..1658de9d1db 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v21/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v21/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -44,6 +44,9 @@ mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component) mca_bfrops_v21_la_SOURCES = $(component_sources) mca_bfrops_v21_la_LDFLAGS = -module -avoid-version +if NEED_LIBPMIX +mca_bfrops_v21_la_LIBADD = $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_bfrops_v21_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v21/bfrop_pmix21.c b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v21/bfrop_pmix21.c index adcc3cba565..7e0167c1e88 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v21/bfrop_pmix21.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/bfrops/v21/bfrop_pmix21.c @@ -13,7 +13,7 @@ * Copyright (c) 2011-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -63,6 +63,25 @@ pmix_bfrops_module_t pmix_bfrops_pmix21_module = { .data_type_string = data_type_string }; +static pmix_status_t pmix21_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_pack_modex(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_unpack_array(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_unpack_modex(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_copy_array(pmix_info_array_t **dest, + pmix_info_array_t *src, + pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_copy_modex(pmix_modex_data_t **dest, pmix_modex_data_t *src, + pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_print_array(char **output, char *prefix, + pmix_info_array_t *src, pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_print_modex(char **output, char *prefix, + pmix_modex_data_t *src, pmix_data_type_t type); + + static pmix_status_t init(void) { /* some standard types don't require anything special */ @@ -265,10 +284,10 @@ static pmix_status_t init(void) &mca_bfrops_v21_component.types); PMIX_REGISTER_TYPE("PMIX_MODEX", PMIX_MODEX, - pmix_bfrops_base_pack_modex, - pmix_bfrops_base_unpack_modex, - pmix_bfrops_base_copy_modex, - pmix_bfrops_base_print_modex, + pmix21_bfrop_pack_modex, + pmix21_bfrop_unpack_modex, + pmix21_bfrop_copy_modex, + pmix21_bfrop_print_modex, &mca_bfrops_v21_component.types); /* these are fixed-sized values and can be done by base */ @@ -374,10 +393,10 @@ static pmix_status_t init(void) /**** DEPRECATED ****/ PMIX_REGISTER_TYPE("PMIX_INFO_ARRAY", PMIX_INFO_ARRAY, - pmix_bfrops_base_pack_array, - pmix_bfrops_base_unpack_array, - pmix_bfrops_base_copy_array, - pmix_bfrops_base_print_array, + pmix21_bfrop_pack_array, + pmix21_bfrop_unpack_array, + pmix21_bfrop_copy_array, + pmix21_bfrop_print_array, &mca_bfrops_v21_component.types); /********************/ @@ -446,3 +465,198 @@ static const char* data_type_string(pmix_data_type_t type) { return pmix_bfrops_base_data_type_string(&mca_bfrops_v21_component.types, type); } + +/**** DEPRECATED ****/ +static pmix_status_t pmix21_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type) +{ + pmix_info_array_t *ptr; + int32_t i; + pmix_status_t ret; + + ptr = (pmix_info_array_t *) src; + + for (i = 0; i < num_vals; ++i) { + /* pack the size */ + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { + return ret; + } + if (0 < ptr[i].size) { + /* pack the values */ + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_info(buffer, ptr[i].array, ptr[i].size, PMIX_INFO))) { + return ret; + } + } + } + + return PMIX_SUCCESS; +} + +static pmix_status_t pmix21_bfrop_pack_modex(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type) +{ + pmix_modex_data_t *ptr; + int32_t i; + pmix_status_t ret; + + ptr = (pmix_modex_data_t *) src; + + for (i = 0; i < num_vals; ++i) { + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { + return ret; + } + if( 0 < ptr[i].size){ + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_byte(buffer, ptr[i].blob, ptr[i].size, PMIX_UINT8))) { + return ret; + } + } + } + return PMIX_SUCCESS; +} + +/********************/ + +/**** DEPRECATED ****/ +static pmix_status_t pmix21_bfrop_unpack_array(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) +{ + pmix_info_array_t *ptr; + int32_t i, n, m; + pmix_status_t ret; + + pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, + "pmix21_bfrop_unpack: %d info arrays", *num_vals); + + ptr = (pmix_info_array_t*) dest; + n = *num_vals; + + for (i = 0; i < n; ++i) { + pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, + "pmix21_bfrop_unpack: init array[%d]", i); + memset(&ptr[i], 0, sizeof(pmix_info_array_t)); + /* unpack the size of this array */ + m=1; + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_sizet(buffer, &ptr[i].size, &m, PMIX_SIZE))) { + return ret; + } + if (0 < ptr[i].size) { + ptr[i].array = (pmix_info_t*)malloc(ptr[i].size * sizeof(pmix_info_t)); + m=ptr[i].size; + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_value(buffer, ptr[i].array, &m, PMIX_INFO))) { + return ret; + } + } + } + return PMIX_SUCCESS; +} + +static pmix_status_t pmix21_bfrop_unpack_modex(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) +{ + pmix_modex_data_t *ptr; + int32_t i, n, m; + pmix_status_t ret; + + pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, + "pmix20_bfrop_unpack: %d modex", *num_vals); + + ptr = (pmix_modex_data_t *) dest; + n = *num_vals; + + for (i = 0; i < n; ++i) { + memset(&ptr[i], 0, sizeof(pmix_modex_data_t)); + /* unpack the number of bytes */ + m=1; + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_sizet(buffer, &ptr[i].size, &m, PMIX_SIZE))) { + return ret; + } + if (0 < ptr[i].size) { + ptr[i].blob = (uint8_t*)malloc(ptr[i].size * sizeof(uint8_t)); + m=ptr[i].size; + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_byte(buffer, ptr[i].blob, &m, PMIX_UINT8))) { + return ret; + } + } + } + return PMIX_SUCCESS; +} + +/********************/ + +/**** DEPRECATED ****/ +static pmix_status_t pmix21_bfrop_copy_array(pmix_info_array_t **dest, + pmix_info_array_t *src, + pmix_data_type_t type) +{ + pmix_info_t *d1, *s1; + + *dest = (pmix_info_array_t*)malloc(sizeof(pmix_info_array_t)); + (*dest)->size = src->size; + (*dest)->array = (pmix_info_t*)malloc(src->size * sizeof(pmix_info_t)); + d1 = (pmix_info_t*)(*dest)->array; + s1 = (pmix_info_t*)src->array; + memcpy(d1, s1, src->size * sizeof(pmix_info_t)); + return PMIX_SUCCESS; +} + +static pmix_status_t pmix21_bfrop_copy_modex(pmix_modex_data_t **dest, pmix_modex_data_t *src, + pmix_data_type_t type) +{ + *dest = (pmix_modex_data_t*)malloc(sizeof(pmix_modex_data_t)); + if (NULL == *dest) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + (*dest)->blob = NULL; + (*dest)->size = 0; + if (NULL != src->blob) { + (*dest)->blob = (uint8_t*)malloc(src->size * sizeof(uint8_t)); + if (NULL == (*dest)->blob) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + memcpy((*dest)->blob, src->blob, src->size * sizeof(uint8_t)); + (*dest)->size = src->size; + } + return PMIX_SUCCESS; +} + +/*******************/ + +/**** DEPRECATED ****/ +static pmix_status_t pmix21_bfrop_print_array(char **output, char *prefix, + pmix_info_array_t *src, pmix_data_type_t type) +{ + size_t j; + char *tmp, *tmp2, *tmp3, *pfx; + pmix_info_t *s1; + + if (0 > asprintf(&tmp, "%sARRAY SIZE: %ld", prefix, (long)src->size)) { + return PMIX_ERR_NOMEM; + } + if (0 > asprintf(&pfx, "\n%s\t", (NULL == prefix) ? "" : prefix)) { + free(tmp); + return PMIX_ERR_NOMEM; + } + s1 = (pmix_info_t*)src->array; + + for (j=0; j < src->size; j++) { + pmix_bfrops_base_print_info(&tmp2, pfx, &s1[j], PMIX_INFO); + if (0 > asprintf(&tmp3, "%s%s", tmp, tmp2)) { + free(tmp); + free(tmp2); + return PMIX_ERR_NOMEM; + } + free(tmp); + free(tmp2); + tmp = tmp3; + } + *output = tmp; + return PMIX_SUCCESS; +} + +static pmix_status_t pmix21_bfrop_print_modex(char **output, char *prefix, + pmix_modex_data_t *src, pmix_data_type_t type) +{ + return PMIX_SUCCESS; +} + +/********************/ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/common/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/common/Makefile.am new file mode 100644 index 00000000000..2a47f435a3c --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/common/Makefile.am @@ -0,0 +1,26 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2019 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Note that this file must exist, even though it is empty (there is no +# "base" directory for the common framework). autogen.pl and +# opal_mca.m4 assume that every framework has a top-level Makefile.am. +# We *could* adjust the framework glue code to exclude "common" from +# this requirement, but it's just a lot easier to have an empty +# Makefile.am here. diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/Makefile.am new file mode 100644 index 00000000000..49dfd8a46cf --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/Makefile.am @@ -0,0 +1,59 @@ +# +# Copyright (c) 2018 Mellanox Technologies. All rights reserved. +# +# Copyright (c) 2018-2019 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Header files + +AM_CPPFLAGS = $(LTDLINCL) + +headers = \ + dstore_common.h \ + dstore_base.h \ + dstore_segment.h \ + dstore_file.h + +# Source files + +sources = \ + dstore_base.c \ + dstore_segment.c + +lib_LTLIBRARIES = +noinst_LTLIBRARIES = +comp_inst = libmca_common_dstore.la +comp_noinst = libmca_common_dstore_noinst.la + +if MCA_BUILD_pmix_common_dstore_DSO +lib_LTLIBRARIES += $(comp_inst) +else +noinst_LTLIBRARIES += $(comp_noinst) +endif + +libmca_common_dstore_la_SOURCES = $(headers) $(sources) +libmca_common_dstore_la_LDFLAGS = -version-info $(libmca_common_dstore_so_version) +libmca_common_dstore_noinst_la_SOURCES = $(headers) $(sources) + +# Conditionally install the header files + +if WANT_INSTALL_HEADERS +pmixdir = $(pmixincludedir)/$(subdir) +pmix_HEADERS = $(headers) +endif + +all-local: + if test -z "$(lib_LTLIBRARIES)"; then \ + rm -f "$(comp_inst)"; \ + $(LN_S) "$(comp_noinst)" "$(comp_inst)"; \ + fi + +clean-local: + if test -z "$(lib_LTLIBRARIES)"; then \ + rm -f "$(comp_inst)"; \ + fi diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/configure.m4 b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/configure.m4 new file mode 100644 index 00000000000..730c49ceba7 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/configure.m4 @@ -0,0 +1,18 @@ +# -*- shell-script -*- +# +# Copyright (c) 2018 Mellanox Technologies. All rights reserved. +# Copyright (c) 2019 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# MCA_pmix_common_dstore_CONFIG([action-if-can-compile], +# [action-if-cant-compile]) +# ------------------------------------------------ +AC_DEFUN([MCA_pmix_common_dstore_CONFIG], [ + AC_CONFIG_FILES([src/mca/common/dstore/Makefile]) + $1 +])dnl diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_dstore.c b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_base.c similarity index 55% rename from opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_dstore.c rename to opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_base.c index 065a9bb4bad..5568c943477 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_dstore.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_base.c @@ -1,8 +1,10 @@ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016-2018 IBM Corporation. All rights reserved. - * Copyright (c) 2016-2017 Mellanox Technologies, Inc. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. * All rights reserved. + * Copyright (c) 2018-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * * $COPYRIGHT$ * @@ -48,8 +50,10 @@ #include "src/mca/preg/preg.h" #include "src/mca/gds/base/base.h" -#include "gds_dstore.h" #include "src/mca/pshmem/base/base.h" +#include "dstore_common.h" +#include "dstore_base.h" +#include "dstore_segment.h" #define ESH_REGION_EXTENSION "EXTENSION_SLOT" #define ESH_REGION_INVALIDATED "INVALIDATED" @@ -58,418 +62,89 @@ #define ESH_ENV_NS_DATA_SEG_SIZE "NS_DATA_SEG_SIZE" #define ESH_ENV_LINEAR "SM_USE_LINEAR_SEARCH" -#define ESH_MIN_KEY_LEN (sizeof(ESH_REGION_INVALIDATED)) - -#define ESH_KV_SIZE(addr) \ -__extension__ ({ \ - size_t sz; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - sz = ESH_KV_SIZE_V12(addr); \ - } else { \ - sz = ESH_KV_SIZE_V20(addr); \ - } \ - sz; \ -}) - -#define ESH_KNAME_PTR(addr) \ -__extension__ ({ \ - char *name_ptr; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - name_ptr = ESH_KNAME_PTR_V12(addr); \ - } else { \ - name_ptr = ESH_KNAME_PTR_V20(addr); \ - } \ - name_ptr; \ -}) - -#define ESH_KNAME_LEN(key) \ -__extension__ ({ \ - size_t len; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - len = ESH_KNAME_LEN_V12(key); \ - } else { \ - len = ESH_KNAME_LEN_V20(key); \ - } \ - len; \ -}) - -#define ESH_DATA_PTR(addr) \ -__extension__ ({ \ - uint8_t *data_ptr; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - data_ptr = ESH_DATA_PTR_V12(addr); \ - } else { \ - data_ptr = ESH_DATA_PTR_V20(addr); \ - } \ - data_ptr; \ -}) - -#define ESH_DATA_SIZE(addr, data_ptr) \ -__extension__ ({ \ - size_t sz; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - sz = ESH_DATA_SIZE_V12(addr); \ - } else { \ - sz = ESH_DATA_SIZE_V20(addr, data_ptr); \ - } \ - sz; \ -}) - -#define ESH_KEY_SIZE(key, size) \ -__extension__ ({ \ - size_t len; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - len = ESH_KEY_SIZE_V12(key, size); \ - } else { \ - len = ESH_KEY_SIZE_V20(key, size); \ - } \ - len; \ -}) - -#define EXT_SLOT_SIZE() \ -__extension__ ({ \ - size_t sz; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - sz = EXT_SLOT_SIZE_V12(); \ - } else { \ - sz = EXT_SLOT_SIZE_V20(); \ - } \ - sz; \ -}) - -#define ESH_PUT_KEY(addr, key, buffer, size) \ -__extension__ ({ \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - ESH_PUT_KEY_V12(addr, key, buffer, size); \ - } else { \ - ESH_PUT_KEY_V20(addr, key, buffer, size); \ - } \ -}) - -/* PMIx v2.x dstore specific macro */ -#define ESH_KV_SIZE_V20(addr) \ -__extension__ ({ \ - size_t sz; \ - memcpy(&sz, addr, sizeof(size_t)); \ - sz; \ -}) - -#define ESH_KNAME_PTR_V20(addr) \ -__extension__ ({ \ - char *name_ptr = (char *)addr + sizeof(size_t); \ - name_ptr; \ -}) - -#define ESH_KNAME_LEN_V20(key) \ -__extension__ ({ \ - size_t kname_len = strlen(key) + 1; \ - size_t len = (kname_len < ESH_MIN_KEY_LEN) ? \ - ESH_MIN_KEY_LEN : kname_len; \ - len; \ -}) - -#define ESH_DATA_PTR_V20(addr) \ -__extension__ ({ \ - size_t kname_len = \ - ESH_KNAME_LEN_V20(ESH_KNAME_PTR_V20(addr)); \ - uint8_t *data_ptr = addr + sizeof(size_t) + kname_len; \ - data_ptr; \ -}) - -#define ESH_DATA_SIZE_V20(addr, data_ptr) \ -__extension__ ({ \ - size_t sz = ESH_KV_SIZE_V20(addr); \ - size_t data_size = sz - (data_ptr - addr); \ - data_size; \ -}) - -#define ESH_KEY_SIZE_V20(key, size) \ -__extension__ ({ \ - size_t len = \ - sizeof(size_t) + ESH_KNAME_LEN_V20(key) + size; \ - len; \ -}) - -/* in ext slot new offset will be stored in case if - * new data were added for the same process during - * next commit - */ -#define EXT_SLOT_SIZE_V20() \ - (ESH_KEY_SIZE_V20(ESH_REGION_EXTENSION, sizeof(size_t))) - - -#define ESH_PUT_KEY_V20(addr, key, buffer, size) \ -__extension__ ({ \ - size_t sz = ESH_KEY_SIZE_V20(key, size); \ - memcpy(addr, &sz, sizeof(size_t)); \ - memset(addr + sizeof(size_t), 0, \ - ESH_KNAME_LEN_V20(key)); \ - strncpy((char *)addr + sizeof(size_t), \ - key, ESH_KNAME_LEN_V20(key)); \ - memcpy(addr + sizeof(size_t) + ESH_KNAME_LEN_V20(key), \ - buffer, size); \ -}) - -/* PMIx v1.2 dstore specific macro */ -#define ESH_KEY_SIZE_V12(key, size) \ -__extension__ ({ \ - size_t len = strlen(key) + 1 + sizeof(size_t) + size; \ - len; \ -}) - -/* in ext slot new offset will be stored in case if - * new data were added for the same process during - * next commit - */ -#define EXT_SLOT_SIZE_V12() \ - (ESH_KEY_SIZE_V12(ESH_REGION_EXTENSION, sizeof(size_t))) - -#define ESH_KV_SIZE_V12(addr) \ -__extension__ ({ \ - size_t sz; \ - memcpy(&sz, addr + \ - ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)), \ - sizeof(size_t)); \ - sz += ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)) + \ - sizeof(size_t); \ - sz; \ -}) - -#define ESH_KNAME_PTR_V12(addr) \ -__extension__ ({ \ - char *name_ptr = (char *)addr; \ - name_ptr; \ -}) - -#define ESH_KNAME_LEN_V12(key) \ -__extension__ ({ \ - size_t len = strlen((char*)key) + 1; \ - len; \ -}) - -#define ESH_DATA_PTR_V12(addr) \ -__extension__ ({ \ - uint8_t *data_ptr = \ - addr + \ - sizeof(size_t) + \ - ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)); \ - data_ptr; \ -}) - -#define ESH_DATA_SIZE_V12(addr) \ -__extension__ ({ \ - size_t data_size; \ - memcpy(&data_size, \ - addr + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)), \ - sizeof(size_t)); \ - data_size; \ -}) - -#define ESH_PUT_KEY_V12(addr, key, buffer, size) \ -__extension__ ({ \ - size_t sz = size; \ - memset(addr, 0, ESH_KNAME_LEN_V12(key)); \ - strncpy((char *)addr, key, ESH_KNAME_LEN_V12(key)); \ - memcpy(addr + ESH_KNAME_LEN_V12(key), &sz, \ - sizeof(size_t)); \ - memcpy(addr + ESH_KNAME_LEN_V12(key) + sizeof(size_t), \ - buffer, size); \ -}) - -#ifdef ESH_PTHREAD_LOCK -#define _ESH_LOCK(rwlock, func) \ -__extension__ ({ \ - pmix_status_t ret = PMIX_SUCCESS; \ - int rc; \ - rc = pthread_rwlock_##func(rwlock); \ - if (0 != rc) { \ - switch (errno) { \ - case EINVAL: \ - ret = PMIX_ERR_INIT; \ - break; \ - case EPERM: \ - ret = PMIX_ERR_NO_PERMISSIONS; \ - break; \ - } \ - } \ - if (ret) { \ - pmix_output(0, "%s %d:%s lock failed: %s", \ - __FILE__, __LINE__, __func__, strerror(errno)); \ - } \ - ret; \ -}) - -#define _ESH_WRLOCK(rwlock) _ESH_LOCK(rwlock, wrlock) -#define _ESH_RDLOCK(rwlock) _ESH_LOCK(rwlock, rdlock) -#define _ESH_UNLOCK(rwlock) _ESH_LOCK(rwlock, unlock) -#endif - -#ifdef ESH_FCNTL_LOCK -#define _ESH_LOCK(lockfd, operation) \ -__extension__ ({ \ - pmix_status_t ret = PMIX_SUCCESS; \ - int i; \ - struct flock fl = {0}; \ - fl.l_type = operation; \ - fl.l_whence = SEEK_SET; \ - for(i = 0; i < 10; i++) { \ - if( 0 > fcntl(lockfd, F_SETLKW, &fl) ) { \ - switch( errno ){ \ - case EINTR: \ - continue; \ - case ENOENT: \ - case EINVAL: \ - ret = PMIX_ERR_NOT_FOUND; \ - break; \ - case EBADF: \ - ret = PMIX_ERR_BAD_PARAM; \ - break; \ - case EDEADLK: \ - case EFAULT: \ - case ENOLCK: \ - ret = PMIX_ERR_RESOURCE_BUSY; \ - break; \ - default: \ - ret = PMIX_ERROR; \ - break; \ - } \ - } \ - break; \ - } \ - if (ret) { \ - pmix_output(0, "%s %d:%s lock failed: %s", \ - __FILE__, __LINE__, __func__, strerror(errno)); \ - } \ - ret; \ -}) - -#define _ESH_WRLOCK(lock) _ESH_LOCK(lock, F_WRLCK) -#define _ESH_RDLOCK(lock) _ESH_LOCK(lock, F_RDLCK) -#define _ESH_UNLOCK(lock) _ESH_LOCK(lock, F_UNLCK) -#endif - #define ESH_INIT_SESSION_TBL_SIZE 2 #define ESH_INIT_NS_MAP_TBL_SIZE 2 -static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_buffer_t *buf); -static seg_desc_t *_create_new_segment(segment_type type, const ns_map_data_t *ns_map, uint32_t id); -static seg_desc_t *_attach_new_segment(segment_type type, const ns_map_data_t *ns_map, uint32_t id); -static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info); -static int _put_ns_info_to_initial_segment(const ns_map_data_t *ns_map, pmix_pshmem_seg_t *metaseg, pmix_pshmem_seg_t *dataseg); -static ns_seg_info_t *_get_ns_info_from_initial_segment(const ns_map_data_t *ns_map); -static ns_track_elem_t *_get_track_elem_for_namespace(ns_map_data_t *ns_map); -static rank_meta_info *_get_rank_meta_info(pmix_rank_t rank, seg_desc_t *segdesc); -static uint8_t *_get_data_region_by_offset(seg_desc_t *segdesc, size_t offset); -static void _update_initial_segment_info(const ns_map_data_t *ns_map); -static void _set_constants_from_env(void); -static void _delete_sm_desc(seg_desc_t *desc); -static int _pmix_getpagesize(void); -static inline ssize_t _get_univ_size(const char *nspace); - -static inline ns_map_data_t * _esh_session_map_search_server(const char *nspace); -static inline ns_map_data_t * _esh_session_map_search_client(const char *nspace); -static inline ns_map_data_t * _esh_session_map(const char *nspace, size_t tbl_idx); -static inline void _esh_session_map_clean(ns_map_t *m); -static inline int _esh_jobuid_tbl_search(uid_t jobuid, size_t *tbl_idx); -static inline int _esh_session_tbl_add(size_t *tbl_idx); -static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid, int setjobuid); -static inline void _esh_session_release(session_t *s); -static inline void _esh_ns_track_cleanup(void); -static inline void _esh_sessions_cleanup(void); -static inline void _esh_ns_map_cleanup(void); +static int _store_data_for_rank(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_info, + pmix_rank_t rank, pmix_buffer_t *buf); +static int _update_ns_elem(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_elem, ns_seg_info_t *info); +static int _put_ns_info_to_initial_segment(pmix_common_dstore_ctx_t *ds_ctx, + const ns_map_data_t *ns_map, pmix_pshmem_seg_t *metaseg, + pmix_pshmem_seg_t *dataseg); +static ns_seg_info_t *_get_ns_info_from_initial_segment(pmix_common_dstore_ctx_t *ds_ctx, + const ns_map_data_t *ns_map); +static ns_track_elem_t *_get_track_elem_for_namespace(pmix_common_dstore_ctx_t *ds_ctx, + ns_map_data_t *ns_map); +static rank_meta_info *_get_rank_meta_info(pmix_common_dstore_ctx_t *ds_ctx, pmix_rank_t rank, + pmix_dstore_seg_desc_t *segdesc); +static uint8_t *_get_data_region_by_offset(pmix_common_dstore_ctx_t *ds_ctx, + pmix_dstore_seg_desc_t *segdesc, size_t offset); +static void _update_initial_segment_info(pmix_common_dstore_ctx_t *ds_ctx, + const ns_map_data_t *ns_map); +static void _set_constants_from_env(pmix_common_dstore_ctx_t *ds_ctx); +static inline ssize_t _get_univ_size(pmix_common_dstore_ctx_t *ds_ctx, const char *nspace); + +static inline ns_map_data_t * _esh_session_map_search_server(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace); +static inline ns_map_data_t * _esh_session_map_search_client(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace); +static inline ns_map_data_t * _esh_session_map(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, uint32_t local_size, + size_t tbl_idx); +static inline void _esh_session_map_clean(pmix_common_dstore_ctx_t *ds_ctx, ns_map_t *m); +static inline int _esh_jobuid_tbl_search(pmix_common_dstore_ctx_t *ds_ctx, + uid_t jobuid, size_t *tbl_idx); +static inline int _esh_session_tbl_add(pmix_common_dstore_ctx_t *ds_ctx, size_t *tbl_idx); +static int _esh_session_init(pmix_common_dstore_ctx_t *ds_ctx, size_t idx, ns_map_data_t *m, + uint32_t local_size, size_t jobuid, int setjobuid); +static void _esh_session_release(pmix_common_dstore_ctx_t *ds_ctx, size_t idx); +static inline void _esh_ns_track_cleanup(pmix_common_dstore_ctx_t *ds_ctx); +static inline void _esh_sessions_cleanup(pmix_common_dstore_ctx_t *ds_ctx); +static inline void _esh_ns_map_cleanup(pmix_common_dstore_ctx_t *ds_ctx); static inline int _esh_dir_del(const char *dirname); -static inline void _client_compat_save(pmix_peer_t *peer); -static inline pmix_peer_t * _client_peer(void); +static inline void _client_compat_save(pmix_common_dstore_ctx_t *ds_ctx, pmix_peer_t *peer); +static inline pmix_peer_t * _client_peer(pmix_common_dstore_ctx_t *ds_ctx); static inline int _my_client(const char *nspace, pmix_rank_t rank); -static pmix_status_t dstore_init(pmix_info_t info[], size_t ninfo); +static pmix_status_t _dstor_store_modex_cb(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_byte_object_t *bo); -static void dstore_finalize(void); +static pmix_status_t _dstore_store_nolock(pmix_common_dstore_ctx_t *ds_ctx, + ns_map_data_t *ns_map, + pmix_rank_t rank, + pmix_kval_t *kv); -static pmix_status_t dstore_setup_fork(const pmix_proc_t *peer, char ***env); +static pmix_status_t _dstore_fetch(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, pmix_rank_t rank, + const char *key, pmix_value_t **kvs); -static pmix_status_t dstore_cache_job_info(struct pmix_nspace_t *ns, - pmix_info_t info[], size_t ninfo); +ns_map_data_t * (*_esh_session_map_search)(const char *nspace) = NULL; -static pmix_status_t dstore_register_job_info(struct pmix_peer_t *pr, - pmix_buffer_t *reply); +#define _ESH_SESSION_lock(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].lock) -static pmix_status_t dstore_store_job_info(const char *nspace, - pmix_buffer_t *job_data); +#define _ESH_SESSION_path(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].nspace_path) -static pmix_status_t _dstore_store(const char *nspace, - pmix_rank_t rank, - pmix_kval_t *kv); +#define _ESH_SESSION_lockfile(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].lockfile) -static pmix_status_t dstore_store(const pmix_proc_t *proc, - pmix_scope_t scope, - pmix_kval_t *kv); - -static pmix_status_t _dstore_fetch(const char *nspace, - pmix_rank_t rank, - const char *key, pmix_value_t **kvs); - -static pmix_status_t dstore_fetch(const pmix_proc_t *proc, - pmix_scope_t scope, bool copy, - const char *key, - pmix_info_t info[], size_t ninfo, - pmix_list_t *kvs); - -static pmix_status_t dstore_add_nspace(const char *nspace, - pmix_info_t info[], - size_t ninfo); - -static pmix_status_t dstore_del_nspace(const char* nspace); - -static pmix_status_t dstore_assign_module(pmix_info_t *info, size_t ninfo, - int *priority); - -static pmix_status_t dstore_store_modex(struct pmix_nspace_t *nspace, - pmix_list_t *cbs, - pmix_byte_object_t *bo); - -pmix_gds_base_module_t pmix_ds12_module = { - .name = "ds12", - .init = dstore_init, - .finalize = dstore_finalize, - .assign_module = dstore_assign_module, - .cache_job_info = dstore_cache_job_info, - .register_job_info = dstore_register_job_info, - .store_job_info = dstore_store_job_info, - .store = dstore_store, - .store_modex = dstore_store_modex, - .fetch = dstore_fetch, - .setup_fork = dstore_setup_fork, - .add_nspace = dstore_add_nspace, - .del_nspace = dstore_del_nspace, -}; - -static char *_base_path = NULL; -static size_t _initial_segment_size = 0; -static size_t _max_ns_num; -static size_t _meta_segment_size = 0; -static size_t _max_meta_elems; -static size_t _data_segment_size = 0; -static size_t _lock_segment_size = 0; -static uid_t _jobuid; -static char _setjobuid = 0; -static pmix_peer_t *_clients_peer = NULL; - -static pmix_value_array_t *_session_array = NULL; -static pmix_value_array_t *_ns_map_array = NULL; -static pmix_value_array_t *_ns_track_array = NULL; +#define _ESH_SESSION_setjobuid(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].setjobuid) -ns_map_data_t * (*_esh_session_map_search)(const char *nspace) = NULL; -int (*_esh_lock_init)(size_t idx) = NULL; +#define _ESH_SESSION_jobuid(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].jobuid) + +#define _ESH_SESSION_sm_seg_first(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].sm_seg_first) +#define _ESH_SESSION_sm_seg_last(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].sm_seg_last) -#define _ESH_SESSION_path(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].nspace_path) -#define _ESH_SESSION_lockfile(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].lockfile) -#define _ESH_SESSION_setjobuid(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].setjobuid) -#define _ESH_SESSION_jobuid(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].jobuid) -#define _ESH_SESSION_sm_seg_first(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].sm_seg_first) -#define _ESH_SESSION_sm_seg_last(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].sm_seg_last) -#define _ESH_SESSION_ns_info(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].ns_info) +#define _ESH_SESSION_ns_info(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].ns_info) #ifdef ESH_PTHREAD_LOCK #define _ESH_SESSION_pthread_rwlock(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].rwlock) @@ -482,14 +157,13 @@ int (*_esh_lock_init)(size_t idx) = NULL; #define _ESH_SESSION_lock(tbl_idx) _ESH_SESSION_lockfd(tbl_idx) #endif -/* If _direct_mode is set, it means that we use linear search - * along the array of rank meta info objects inside a meta segment - * to find the requested rank. Otherwise, we do a fast lookup - * based on rank and directly compute offset. - * This mode is called direct because it's effectively used in - * sparse communication patterns when direct modex is usually used. - */ -static int _direct_mode = 0; +#define _ESH_LOCK(ds_ctx, session_id, operation) \ +__pmix_attribute_extension__ ({ \ + pmix_status_t rc = PMIX_SUCCESS; \ + rc = ds_ctx->lock_cbs->operation(_ESH_SESSION_lock(ds_ctx->session_array, \ + session_id)); \ + rc; \ +}) static void ncon(ns_track_elem_t *p) { memset(&p->ns_map, 0, sizeof(p->ns_map)); @@ -501,8 +175,8 @@ static void ncon(ns_track_elem_t *p) { } static void ndes(ns_track_elem_t *p) { - _delete_sm_desc(p->meta_seg); - _delete_sm_desc(p->data_seg); + pmix_common_dstor_delete_sm_desc(p->meta_seg); + pmix_common_dstor_delete_sm_desc(p->data_seg); memset(&p->ns_map, 0, sizeof(p->ns_map)); p->in_use = false; } @@ -511,153 +185,11 @@ PMIX_CLASS_INSTANCE(ns_track_elem_t, pmix_value_array_t, ncon, ndes); -static inline void _esh_session_map_clean(ns_map_t *m) { +static inline void _esh_session_map_clean(pmix_common_dstore_ctx_t *ds_ctx, ns_map_t *m) { memset(m, 0, sizeof(*m)); m->data.track_idx = -1; } -#ifdef ESH_FCNTL_LOCK -static inline int _flock_init(size_t idx) { - pmix_status_t rc = PMIX_SUCCESS; - - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - _ESH_SESSION_lock(idx) = open(_ESH_SESSION_lockfile(idx), O_CREAT | O_RDWR | O_EXCL, 0600); - - /* if previous launch was crashed, the lockfile might not be deleted and unlocked, - * so we delete it and create a new one. */ - if (_ESH_SESSION_lock(idx) < 0) { - unlink(_ESH_SESSION_lockfile(idx)); - _ESH_SESSION_lock(idx) = open(_ESH_SESSION_lockfile(idx), O_CREAT | O_RDWR, 0600); - if (_ESH_SESSION_lock(idx) < 0) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - } - if (_ESH_SESSION_setjobuid(idx) > 0) { - if (0 > chown(_ESH_SESSION_lockfile(idx), (uid_t) _ESH_SESSION_jobuid(idx), (gid_t) -1)) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - if (0 > chmod(_ESH_SESSION_lockfile(idx), S_IRUSR | S_IWGRP | S_IRGRP)) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - } - } - else { - _ESH_SESSION_lock(idx) = open(_ESH_SESSION_lockfile(idx), O_RDONLY); - if (-1 == _ESH_SESSION_lock(idx)) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - } - return rc; -} -#endif - -#ifdef ESH_PTHREAD_LOCK -static inline int _rwlock_init(size_t idx) { - pmix_status_t rc = PMIX_SUCCESS; - size_t size = _lock_segment_size; - pthread_rwlockattr_t attr; - - if ((NULL != _ESH_SESSION_pthread_seg(idx)) || (NULL != _ESH_SESSION_pthread_rwlock(idx))) { - rc = PMIX_ERR_INIT; - return rc; - } - _ESH_SESSION_pthread_seg(idx) = (pmix_pshmem_seg_t *)malloc(sizeof(pmix_pshmem_seg_t)); - if (NULL == _ESH_SESSION_pthread_seg(idx)) { - rc = PMIX_ERR_OUT_OF_RESOURCE; - return rc; - } - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - if (PMIX_SUCCESS != (rc = pmix_pshmem.segment_create(_ESH_SESSION_pthread_seg(idx), _ESH_SESSION_lockfile(idx), size))) { - return rc; - } - memset(_ESH_SESSION_pthread_seg(idx)->seg_base_addr, 0, size); - if (_ESH_SESSION_setjobuid(idx) > 0) { - if (0 > chown(_ESH_SESSION_lockfile(idx), (uid_t) _ESH_SESSION_jobuid(idx), (gid_t) -1)){ - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - /* set the mode as required */ - if (0 > chmod(_ESH_SESSION_lockfile(idx), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP )) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - } - _ESH_SESSION_pthread_rwlock(idx) = (pthread_rwlock_t *)_ESH_SESSION_pthread_seg(idx)->seg_base_addr; - - if (0 != pthread_rwlockattr_init(&attr)) { - rc = PMIX_ERR_INIT; - pmix_pshmem.segment_detach(_ESH_SESSION_pthread_seg(idx)); - return rc; - } - if (0 != pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) { - rc = PMIX_ERR_INIT; - pmix_pshmem.segment_detach(_ESH_SESSION_pthread_seg(idx)); - pthread_rwlockattr_destroy(&attr); - return rc; - } -#ifdef HAVE_PTHREAD_SETKIND - if (0 != pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)) { - rc = PMIX_ERR_INIT; - pmix_pshmem.segment_detach(_ESH_SESSION_pthread_seg(idx)); - pthread_rwlockattr_destroy(&attr); - return rc; - } -#endif - if (0 != pthread_rwlock_init(_ESH_SESSION_pthread_rwlock(idx), &attr)) { - rc = PMIX_ERR_INIT; - pmix_pshmem.segment_detach(_ESH_SESSION_pthread_seg(idx)); - pthread_rwlockattr_destroy(&attr); - return rc; - } - if (0 != pthread_rwlockattr_destroy(&attr)) { - rc = PMIX_ERR_INIT; - return rc; - } - - } - else { - _ESH_SESSION_pthread_seg(idx)->seg_size = size; - snprintf(_ESH_SESSION_pthread_seg(idx)->seg_name, PMIX_PATH_MAX, "%s", _ESH_SESSION_lockfile(idx)); - if (PMIX_SUCCESS != (rc = pmix_pshmem.segment_attach(_ESH_SESSION_pthread_seg(idx), PMIX_PSHMEM_RW))) { - return rc; - } - _ESH_SESSION_pthread_rwlock(idx) = (pthread_rwlock_t *)_ESH_SESSION_pthread_seg(idx)->seg_base_addr; - } - - return rc; -} - -static inline void _rwlock_release(session_t *s) { - pmix_status_t rc; - - if (0 != pthread_rwlock_destroy(s->rwlock)) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return; - } - - /* detach & unlink from current desc */ - if (s->rwlock_seg->seg_cpid == getpid()) { - pmix_pshmem.segment_unlink(s->rwlock_seg); - } - pmix_pshmem.segment_detach(s->rwlock_seg); - - free(s->rwlock_seg); - s->rwlock_seg = NULL; - s->rwlock = NULL; -} -#endif - static inline int _esh_dir_del(const char *path) { DIR *dir; @@ -713,132 +245,133 @@ static inline int _esh_dir_del(const char *path) return rc; } -static inline int _esh_tbls_init(void) +static inline int _esh_tbls_init(pmix_common_dstore_ctx_t *ds_ctx) { pmix_status_t rc = PMIX_SUCCESS; size_t idx; /* initial settings */ - _ns_track_array = NULL; - _session_array = NULL; - _ns_map_array = NULL; + ds_ctx->ns_track_array = NULL; + ds_ctx->session_array = NULL; + ds_ctx->ns_map_array = NULL; /* Setup namespace tracking array */ - if (NULL == (_ns_track_array = PMIX_NEW(pmix_value_array_t))) { + if (NULL == (ds_ctx->ns_track_array = PMIX_NEW(pmix_value_array_t))) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto err_exit; } - if (PMIX_SUCCESS != (rc = pmix_value_array_init(_ns_track_array, sizeof(ns_track_elem_t)))){ + if (PMIX_SUCCESS != (rc = pmix_value_array_init(ds_ctx->ns_track_array, sizeof(ns_track_elem_t)))){ PMIX_ERROR_LOG(rc); goto err_exit; } /* Setup sessions table */ - if (NULL == (_session_array = PMIX_NEW(pmix_value_array_t))){ + if (NULL == (ds_ctx->session_array = PMIX_NEW(pmix_value_array_t))){ rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto err_exit; } - if (PMIX_SUCCESS != (rc = pmix_value_array_init(_session_array, sizeof(session_t)))) { + if (PMIX_SUCCESS != (rc = pmix_value_array_init(ds_ctx->session_array, sizeof(session_t)))) { PMIX_ERROR_LOG(rc); goto err_exit; } - if (PMIX_SUCCESS != (rc = pmix_value_array_set_size(_session_array, ESH_INIT_SESSION_TBL_SIZE))) { + if (PMIX_SUCCESS != (rc = pmix_value_array_set_size(ds_ctx->session_array, ESH_INIT_SESSION_TBL_SIZE))) { PMIX_ERROR_LOG(rc); goto err_exit; } for (idx = 0; idx < ESH_INIT_SESSION_TBL_SIZE; idx++) { - memset(pmix_value_array_get_item(_session_array, idx), 0, sizeof(session_t)); + memset(pmix_value_array_get_item(ds_ctx->session_array, idx), 0, sizeof(session_t)); } /* Setup namespace map array */ - if (NULL == (_ns_map_array = PMIX_NEW(pmix_value_array_t))) { + if (NULL == (ds_ctx->ns_map_array = PMIX_NEW(pmix_value_array_t))) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto err_exit; } - if (PMIX_SUCCESS != (rc = pmix_value_array_init(_ns_map_array, sizeof(ns_map_t)))) { + if (PMIX_SUCCESS != (rc = pmix_value_array_init(ds_ctx->ns_map_array, sizeof(ns_map_t)))) { PMIX_ERROR_LOG(rc); goto err_exit; } - if (PMIX_SUCCESS != (rc = pmix_value_array_set_size(_ns_map_array, ESH_INIT_NS_MAP_TBL_SIZE))) { + if (PMIX_SUCCESS != (rc = pmix_value_array_set_size(ds_ctx->ns_map_array, ESH_INIT_NS_MAP_TBL_SIZE))) { PMIX_ERROR_LOG(rc); goto err_exit; } for (idx = 0; idx < ESH_INIT_NS_MAP_TBL_SIZE; idx++) { - _esh_session_map_clean(pmix_value_array_get_item(_ns_map_array, idx)); + _esh_session_map_clean(ds_ctx, pmix_value_array_get_item(ds_ctx->ns_map_array, idx)); } return PMIX_SUCCESS; err_exit: - if (NULL != _ns_track_array) { - PMIX_RELEASE(_ns_track_array); + if (NULL != ds_ctx->ns_track_array) { + PMIX_RELEASE(ds_ctx->ns_track_array); } - if (NULL != _session_array) { - PMIX_RELEASE(_session_array); + if (NULL != ds_ctx->session_array) { + PMIX_RELEASE(ds_ctx->session_array); } - if (NULL != _ns_map_array) { - PMIX_RELEASE(_ns_map_array); + if (NULL != ds_ctx->ns_map_array) { + PMIX_RELEASE(ds_ctx->ns_map_array); } return rc; } -static inline void _esh_ns_map_cleanup(void) +static inline void _esh_ns_map_cleanup(pmix_common_dstore_ctx_t *ds_ctx) { size_t idx; size_t size; ns_map_t *ns_map; - if (NULL == _ns_map_array) { + if (NULL == ds_ctx->ns_map_array) { return; } - size = pmix_value_array_get_size(_ns_map_array); - ns_map = PMIX_VALUE_ARRAY_GET_BASE(_ns_map_array, ns_map_t); + size = pmix_value_array_get_size(ds_ctx->ns_map_array); + ns_map = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->ns_map_array, ns_map_t); for (idx = 0; idx < size; idx++) { - if(ns_map[idx].in_use) - _esh_session_map_clean(&ns_map[idx]); + if(ns_map[idx].in_use) { + _esh_session_map_clean(ds_ctx, &ns_map[idx]); + } } - PMIX_RELEASE(_ns_map_array); - _ns_map_array = NULL; + PMIX_RELEASE(ds_ctx->ns_map_array); + ds_ctx->ns_map_array = NULL; } -static inline void _esh_sessions_cleanup(void) +static inline void _esh_sessions_cleanup(pmix_common_dstore_ctx_t *ds_ctx) { size_t idx; size_t size; session_t *s_tbl; - if (NULL == _session_array) { + if (NULL == ds_ctx->session_array) { return; } - size = pmix_value_array_get_size(_session_array); - s_tbl = PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t); + size = pmix_value_array_get_size(ds_ctx->session_array); + s_tbl = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->session_array, session_t); for (idx = 0; idx < size; idx++) { if(s_tbl[idx].in_use) - _esh_session_release(&s_tbl[idx]); + _esh_session_release(ds_ctx, idx); } - PMIX_RELEASE(_session_array); - _session_array = NULL; + PMIX_RELEASE(ds_ctx->session_array); + ds_ctx->session_array = NULL; } -static inline void _esh_ns_track_cleanup(void) +static inline void _esh_ns_track_cleanup(pmix_common_dstore_ctx_t *ds_ctx) { int size; ns_track_elem_t *ns_trk; - if (NULL == _ns_track_array) { + if (NULL == ds_ctx->ns_track_array) { return; } - size = pmix_value_array_get_size(_ns_track_array); - ns_trk = PMIX_VALUE_ARRAY_GET_BASE(_ns_track_array, ns_track_elem_t); + size = pmix_value_array_get_size(ds_ctx->ns_track_array); + ns_trk = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->ns_track_array, ns_track_elem_t); for (int i = 0; i < size; i++) { ns_track_elem_t *trk = ns_trk + i; @@ -847,15 +380,17 @@ static inline void _esh_ns_track_cleanup(void) } } - PMIX_RELEASE(_ns_track_array); - _ns_track_array = NULL; + PMIX_RELEASE(ds_ctx->ns_track_array); + ds_ctx->ns_track_array = NULL; } -static inline ns_map_data_t * _esh_session_map(const char *nspace, size_t tbl_idx) +static inline ns_map_data_t * _esh_session_map(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, uint32_t local_size, + size_t tbl_idx) { size_t map_idx; - size_t size = pmix_value_array_get_size(_ns_map_array);; - ns_map_t *ns_map = PMIX_VALUE_ARRAY_GET_BASE(_ns_map_array, ns_map_t);; + size_t size = pmix_value_array_get_size(ds_ctx->ns_map_array); + ns_map_t *ns_map = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->ns_map_array, ns_map_t); ns_map_t *new_map = NULL; if (NULL == nspace) { @@ -866,32 +401,33 @@ static inline ns_map_data_t * _esh_session_map(const char *nspace, size_t tbl_id for(map_idx = 0; map_idx < size; map_idx++) { if (!ns_map[map_idx].in_use) { ns_map[map_idx].in_use = true; - strncpy(ns_map[map_idx].data.name, nspace, sizeof(ns_map[map_idx].data.name)-1); + pmix_strncpy(ns_map[map_idx].data.name, nspace, sizeof(ns_map[map_idx].data.name)-1); ns_map[map_idx].data.tbl_idx = tbl_idx; return &ns_map[map_idx].data; } } - if (NULL == (new_map = pmix_value_array_get_item(_ns_map_array, map_idx))) { + if (NULL == (new_map = pmix_value_array_get_item(ds_ctx->ns_map_array, map_idx))) { PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); return NULL; } - _esh_session_map_clean(new_map); + _esh_session_map_clean(ds_ctx, new_map); new_map->in_use = true; new_map->data.tbl_idx = tbl_idx; - strncpy(new_map->data.name, nspace, sizeof(new_map->data.name)-1); + pmix_strncpy(new_map->data.name, nspace, sizeof(new_map->data.name)-1); return &new_map->data; } -static inline int _esh_jobuid_tbl_search(uid_t jobuid, size_t *tbl_idx) +static inline int _esh_jobuid_tbl_search(pmix_common_dstore_ctx_t *ds_ctx, + uid_t jobuid, size_t *tbl_idx) { size_t idx, size; session_t *session_tbl = NULL; - size = pmix_value_array_get_size(_session_array); - session_tbl = PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t); + size = pmix_value_array_get_size(ds_ctx->session_array); + session_tbl = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->session_array, session_t); for(idx = 0; idx < size; idx++) { if (session_tbl[idx].in_use && session_tbl[idx].jobuid == jobuid) { @@ -903,37 +439,38 @@ static inline int _esh_jobuid_tbl_search(uid_t jobuid, size_t *tbl_idx) return PMIX_ERR_NOT_FOUND; } -static inline int _esh_session_tbl_add(size_t *tbl_idx) +static inline int _esh_session_tbl_add(pmix_common_dstore_ctx_t *ds_ctx, size_t *tbl_idx) { size_t idx; - size_t size = pmix_value_array_get_size(_session_array); - session_t *s_tbl = PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t); + size_t size = pmix_value_array_get_size(ds_ctx->session_array); + session_t *s_tbl = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->session_array, session_t); session_t *new_sesion; pmix_status_t rc = PMIX_SUCCESS; for(idx = 0; idx < size; idx ++) { if (0 == s_tbl[idx].in_use) { - s_tbl[idx].in_use = 1; - *tbl_idx = idx; - return PMIX_SUCCESS; + goto done; } } - if (NULL == (new_sesion = pmix_value_array_get_item(_session_array, idx))) { + if (NULL == (new_sesion = pmix_value_array_get_item(ds_ctx->session_array, idx))) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); return rc; } + +done: s_tbl[idx].in_use = 1; *tbl_idx = idx; return PMIX_SUCCESS; } -static inline ns_map_data_t * _esh_session_map_search_server(const char *nspace) +static inline ns_map_data_t * _esh_session_map_search_server(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace) { - size_t idx, size = pmix_value_array_get_size(_ns_map_array); - ns_map_t *ns_map = PMIX_VALUE_ARRAY_GET_BASE(_ns_map_array, ns_map_t); + size_t idx, size = pmix_value_array_get_size(ds_ctx->ns_map_array); + ns_map_t *ns_map = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->ns_map_array, ns_map_t); if (NULL == nspace) { return NULL; } @@ -947,10 +484,11 @@ static inline ns_map_data_t * _esh_session_map_search_server(const char *nspace) return NULL; } -static inline ns_map_data_t * _esh_session_map_search_client(const char *nspace) +static inline ns_map_data_t * _esh_session_map_search_client(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace) { - size_t idx, size = pmix_value_array_get_size(_ns_map_array); - ns_map_t *ns_map = PMIX_VALUE_ARRAY_GET_BASE(_ns_map_array, ns_map_t); + size_t idx, size = pmix_value_array_get_size(ds_ctx->ns_map_array); + ns_map_t *ns_map = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->ns_map_array, ns_map_t); if (NULL == nspace) { return NULL; @@ -962,29 +500,19 @@ static inline ns_map_data_t * _esh_session_map_search_client(const char *nspace) return &ns_map[idx].data; } } - return _esh_session_map(nspace, 0); + return _esh_session_map(ds_ctx, nspace, 0, 0); } -static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid, int setjobuid) +static int _esh_session_init(pmix_common_dstore_ctx_t *ds_ctx, size_t idx, ns_map_data_t *m, + uint32_t local_size, size_t jobuid, int setjobuid) { - seg_desc_t *seg = NULL; - session_t *s = &(PMIX_VALUE_ARRAY_GET_ITEM(_session_array, session_t, idx)); + pmix_dstore_seg_desc_t *seg = NULL; + session_t *s = &(PMIX_VALUE_ARRAY_GET_ITEM(ds_ctx->session_array, session_t, idx)); pmix_status_t rc = PMIX_SUCCESS; s->setjobuid = setjobuid; s->jobuid = jobuid; - s->nspace_path = strdup(_base_path); - - /* create a lock file to prevent clients from reading while server is writing to the shared memory. - * This situation is quite often, especially in case of direct modex when clients might ask for data - * simultaneously.*/ - if(0 > asprintf(&s->lockfile, "%s/dstore_sm.lock", s->nspace_path)) { - rc = PMIX_ERR_OUT_OF_RESOURCE; - PMIX_ERROR_LOG(rc); - return rc; - } - PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, - "%s:%d:%s _lockfile_name: %s", __FILE__, __LINE__, __func__, s->lockfile)); + s->nspace_path = strdup(ds_ctx->base_path); if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { if (0 != mkdir(s->nspace_path, 0770)) { @@ -1003,7 +531,8 @@ static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid, return rc; } } - seg = _create_new_segment(INITIAL_SEGMENT, m, 0); + seg = pmix_common_dstor_create_new_segment(PMIX_DSTORE_INITIAL_SEGMENT, ds_ctx->base_path, + m->name, 0, ds_ctx->jobuid, ds_ctx->setjobuid); if( NULL == seg ){ rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); @@ -1011,245 +540,92 @@ static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid, } } else { - seg = _attach_new_segment(INITIAL_SEGMENT, m, 0); + seg = pmix_common_dstor_attach_new_segment(PMIX_DSTORE_INITIAL_SEGMENT, ds_ctx->base_path, m->name, 0); if( NULL == seg ){ rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); return rc; } } - - if (NULL == _esh_lock_init) { - rc = PMIX_ERR_INIT; - PMIX_ERROR_LOG(rc); - return rc; - } - if ( PMIX_SUCCESS != (rc = _esh_lock_init(m->tbl_idx))) { - PMIX_ERROR_LOG(rc); - return rc; - } - s->sm_seg_first = seg; s->sm_seg_last = s->sm_seg_first; + return PMIX_SUCCESS; } -static inline void _esh_session_release(session_t *s) +static void _esh_session_release(pmix_common_dstore_ctx_t *ds_ctx, size_t idx) { + session_t *s = &(PMIX_VALUE_ARRAY_GET_ITEM(ds_ctx->session_array, session_t, idx)); + if (!s->in_use) { return; } - _delete_sm_desc(s->sm_seg_first); - /* if the lock fd was somehow set, then we - * need to close it */ - if (0 != s->lockfd) { - close(s->lockfd); - } + pmix_common_dstor_delete_sm_desc(s->sm_seg_first); + + ds_ctx->lock_cbs->finalize(&_ESH_SESSION_lock(ds_ctx->session_array, idx)); - if (NULL != s->lockfile) { - if(PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - unlink(s->lockfile); - } - free(s->lockfile); - } if (NULL != s->nspace_path) { if(PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { _esh_dir_del(s->nspace_path); } free(s->nspace_path); } -#ifdef ESH_PTHREAD_LOCK - _rwlock_release(s); -#endif memset ((char *) s, 0, sizeof(*s)); } -static void _set_constants_from_env() +static void _set_constants_from_env(pmix_common_dstore_ctx_t *ds_ctx) { char *str; - int page_size = _pmix_getpagesize(); + int page_size = pmix_common_dstor_getpagesize(); if( NULL != (str = getenv(ESH_ENV_INITIAL_SEG_SIZE)) ) { - _initial_segment_size = strtoul(str, NULL, 10); - if ((size_t)page_size > _initial_segment_size) { - _initial_segment_size = (size_t)page_size; + ds_ctx->initial_segment_size = strtoul(str, NULL, 10); + if ((size_t)page_size > ds_ctx->initial_segment_size) { + ds_ctx->initial_segment_size = (size_t)page_size; } } - if (0 == _initial_segment_size) { - _initial_segment_size = INITIAL_SEG_SIZE; + if (0 == ds_ctx->initial_segment_size) { + ds_ctx->initial_segment_size = INITIAL_SEG_SIZE; } if( NULL != (str = getenv(ESH_ENV_NS_META_SEG_SIZE)) ) { - _meta_segment_size = strtoul(str, NULL, 10); - if ((size_t)page_size > _meta_segment_size) { - _meta_segment_size = (size_t)page_size; + ds_ctx->meta_segment_size = strtoul(str, NULL, 10); + if ((size_t)page_size > ds_ctx->meta_segment_size) { + ds_ctx->meta_segment_size = (size_t)page_size; } } - if (0 == _meta_segment_size) { - _meta_segment_size = NS_META_SEG_SIZE; + if (0 == ds_ctx->meta_segment_size) { + ds_ctx->meta_segment_size = NS_META_SEG_SIZE; } if( NULL != (str = getenv(ESH_ENV_NS_DATA_SEG_SIZE)) ) { - _data_segment_size = strtoul(str, NULL, 10); - if ((size_t)page_size > _data_segment_size) { - _data_segment_size = (size_t)page_size; + ds_ctx->data_segment_size = strtoul(str, NULL, 10); + if ((size_t)page_size > ds_ctx->data_segment_size) { + ds_ctx->data_segment_size = (size_t)page_size; } } - if (0 == _data_segment_size) { - _data_segment_size = NS_DATA_SEG_SIZE; + if (0 == ds_ctx->data_segment_size) { + ds_ctx->data_segment_size = NS_DATA_SEG_SIZE; } if (NULL != (str = getenv(ESH_ENV_LINEAR))) { if (1 == strtoul(str, NULL, 10)) { - _direct_mode = 1; - } - } - - _lock_segment_size = page_size; - _max_ns_num = (_initial_segment_size - sizeof(size_t) * 2) / sizeof(ns_seg_info_t); - _max_meta_elems = (_meta_segment_size - sizeof(size_t)) / sizeof(rank_meta_info); - -} - -static void _delete_sm_desc(seg_desc_t *desc) -{ - seg_desc_t *tmp; - - /* free all global segments */ - while (NULL != desc) { - tmp = desc->next; - /* detach & unlink from current desc */ - if (desc->seg_info.seg_cpid == getpid()) { - pmix_pshmem.segment_unlink(&desc->seg_info); - } - pmix_pshmem.segment_detach(&desc->seg_info); - free(desc); - desc = tmp; - } -} - -static int _pmix_getpagesize(void) -{ -#if defined(_SC_PAGESIZE ) - return sysconf(_SC_PAGESIZE); -#elif defined(_SC_PAGE_SIZE) - return sysconf(_SC_PAGE_SIZE); -#else - return 65536; /* safer to overestimate than under */ -#endif -} - -static seg_desc_t *_create_new_segment(segment_type type, const ns_map_data_t *ns_map, uint32_t id) -{ - pmix_status_t rc; - char file_name[PMIX_PATH_MAX]; - size_t size; - seg_desc_t *new_seg = NULL; - - PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, - "%s:%d:%s: segment type %d, nspace %s, id %u", - __FILE__, __LINE__, __func__, type, ns_map->name, id)); - - switch (type) { - case INITIAL_SEGMENT: - size = _initial_segment_size; - snprintf(file_name, PMIX_PATH_MAX, "%s/initial-pmix_shared-segment-%u", - _ESH_SESSION_path(ns_map->tbl_idx), id); - break; - case NS_META_SEGMENT: - size = _meta_segment_size; - snprintf(file_name, PMIX_PATH_MAX, "%s/smseg-%s-%u", - _ESH_SESSION_path(ns_map->tbl_idx), ns_map->name, id); - break; - case NS_DATA_SEGMENT: - size = _data_segment_size; - snprintf(file_name, PMIX_PATH_MAX, "%s/smdataseg-%s-%d", - _ESH_SESSION_path(ns_map->tbl_idx), ns_map->name, id); - break; - default: - PMIX_ERROR_LOG(PMIX_ERROR); - return NULL; - } - new_seg = (seg_desc_t*)malloc(sizeof(seg_desc_t)); - if (new_seg) { - new_seg->id = id; - new_seg->next = NULL; - new_seg->type = type; - rc = pmix_pshmem.segment_create(&new_seg->seg_info, file_name, size); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - goto err_exit; - } - memset(new_seg->seg_info.seg_base_addr, 0, size); - - - if (_ESH_SESSION_setjobuid(ns_map->tbl_idx) > 0){ - rc = PMIX_ERR_PERM; - if (0 > chown(file_name, (uid_t) _ESH_SESSION_jobuid(ns_map->tbl_idx), (gid_t) -1)){ - PMIX_ERROR_LOG(rc); - goto err_exit; - } - /* set the mode as required */ - if (0 > chmod(file_name, S_IRUSR | S_IRGRP | S_IWGRP )) { - PMIX_ERROR_LOG(rc); - goto err_exit; - } + ds_ctx->direct_mode = 1; } } - return new_seg; -err_exit: - if( NULL != new_seg ){ - free(new_seg); - } - return NULL; -} + ds_ctx->lock_segment_size = page_size; + ds_ctx->max_ns_num = (ds_ctx->initial_segment_size - sizeof(size_t) * 2) / sizeof(ns_seg_info_t); + ds_ctx->max_meta_elems = (ds_ctx->meta_segment_size - sizeof(size_t)) / sizeof(rank_meta_info); -static seg_desc_t *_attach_new_segment(segment_type type, const ns_map_data_t *ns_map, uint32_t id) -{ - pmix_status_t rc; - seg_desc_t *new_seg = NULL; - new_seg = (seg_desc_t*)malloc(sizeof(seg_desc_t)); - new_seg->id = id; - new_seg->next = NULL; - new_seg->type = type; + pmix_common_dstor_init_segment_info(ds_ctx->initial_segment_size, ds_ctx->meta_segment_size, + ds_ctx->data_segment_size); - PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, - "%s:%d:%s: segment type %d, nspace %s, id %u", - __FILE__, __LINE__, __func__, type, ns_map->name, id)); - - switch (type) { - case INITIAL_SEGMENT: - new_seg->seg_info.seg_size = _initial_segment_size; - snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/initial-pmix_shared-segment-%u", - _ESH_SESSION_path(ns_map->tbl_idx), id); - break; - case NS_META_SEGMENT: - new_seg->seg_info.seg_size = _meta_segment_size; - snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/smseg-%s-%u", - _ESH_SESSION_path(ns_map->tbl_idx), ns_map->name, id); - break; - case NS_DATA_SEGMENT: - new_seg->seg_info.seg_size = _data_segment_size; - snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/smdataseg-%s-%d", - _ESH_SESSION_path(ns_map->tbl_idx), ns_map->name, id); - break; - default: - free(new_seg); - PMIX_ERROR_LOG(PMIX_ERROR); - return NULL; - } - rc = pmix_pshmem.segment_attach(&new_seg->seg_info, PMIX_PSHMEM_RONLY); - if (PMIX_SUCCESS != rc) { - free(new_seg); - new_seg = NULL; - PMIX_ERROR_LOG(rc); - } - return new_seg; } /* This function synchronizes the content of initial shared segment and the local track list. */ -static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) +static int _update_ns_elem(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_elem, + ns_seg_info_t *info) { - seg_desc_t *seg, *tmp = NULL; + pmix_dstore_seg_desc_t *seg, *tmp = NULL; size_t i, offs; ns_map_data_t *ns_map = NULL; pmix_status_t rc; @@ -1258,7 +634,7 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) "%s:%d:%s", __FILE__, __LINE__, __func__)); - if (NULL == (ns_map = _esh_session_map_search(info->ns_map.name))) { + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, info->ns_map.name))) { rc = PMIX_ERR_NOT_AVAILABLE; PMIX_ERROR_LOG(rc); return rc; @@ -1274,14 +650,16 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) /* synchronize number of meta segments for the target namespace. */ for (i = ns_elem->num_meta_seg; i < info->num_meta_seg; i++) { if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - seg = _create_new_segment(NS_META_SEGMENT, &info->ns_map, i); + seg = pmix_common_dstor_create_new_segment(PMIX_DSTORE_NS_META_SEGMENT, ds_ctx->base_path, + info->ns_map.name, i, ds_ctx->jobuid, + ds_ctx->setjobuid); if (NULL == seg) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); return rc; } } else { - seg = _attach_new_segment(NS_META_SEGMENT, &info->ns_map, i); + seg = pmix_common_dstor_attach_new_segment(PMIX_DSTORE_NS_META_SEGMENT, ds_ctx->base_path, info->ns_map.name, i); if (NULL == seg) { rc = PMIX_ERR_NOT_AVAILABLE; PMIX_ERROR_LOG(rc); @@ -1307,7 +685,9 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) /* synchronize number of data segments for the target namespace. */ for (i = ns_elem->num_data_seg; i < info->num_data_seg; i++) { if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - seg = _create_new_segment(NS_DATA_SEGMENT, &info->ns_map, i); + seg = pmix_common_dstor_create_new_segment(PMIX_DSTORE_NS_DATA_SEGMENT, ds_ctx->base_path, + info->ns_map.name, i, ds_ctx->jobuid, + ds_ctx->setjobuid); if (NULL == seg) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); @@ -1316,7 +696,7 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) offs = sizeof(size_t);//shift on offset field itself memcpy(seg->seg_info.seg_base_addr, &offs, sizeof(size_t)); } else { - seg = _attach_new_segment(NS_DATA_SEGMENT, &info->ns_map, i); + seg = pmix_common_dstor_attach_new_segment(PMIX_DSTORE_NS_DATA_SEGMENT, ds_ctx->base_path, info->ns_map.name, i); if (NULL == seg) { rc = PMIX_ERR_NOT_AVAILABLE; PMIX_ERROR_LOG(rc); @@ -1336,66 +716,54 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) return PMIX_SUCCESS; } -static seg_desc_t *extend_segment(seg_desc_t *segdesc, const ns_map_data_t *ns_map) -{ - seg_desc_t *tmp, *seg; - - PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, - "%s:%d:%s", - __FILE__, __LINE__, __func__)); - /* find last segment */ - tmp = segdesc; - while (NULL != tmp->next) { - tmp = tmp->next; - } - /* create another segment, the old one is full. */ - seg = _create_new_segment(segdesc->type, ns_map, tmp->id + 1); - tmp->next = seg; - - return seg; -} - -static int _put_ns_info_to_initial_segment(const ns_map_data_t *ns_map, pmix_pshmem_seg_t *metaseg, pmix_pshmem_seg_t *dataseg) +static int _put_ns_info_to_initial_segment(pmix_common_dstore_ctx_t *ds_ctx, + const ns_map_data_t *ns_map, pmix_pshmem_seg_t *metaseg, + pmix_pshmem_seg_t *dataseg) { ns_seg_info_t elem; size_t num_elems; - num_elems = *((size_t*)(_ESH_SESSION_sm_seg_last(ns_map->tbl_idx)->seg_info.seg_base_addr)); - seg_desc_t *last_seg = _ESH_SESSION_sm_seg_last(ns_map->tbl_idx); + num_elems = *((size_t*)(_ESH_SESSION_sm_seg_last(ds_ctx->session_array, + ns_map->tbl_idx)->seg_info.seg_base_addr)); + pmix_dstore_seg_desc_t *last_seg = _ESH_SESSION_sm_seg_last(ds_ctx->session_array, ns_map->tbl_idx); pmix_status_t rc; PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s", __FILE__, __LINE__, __func__)); - if (_max_ns_num == num_elems) { + if (ds_ctx->max_ns_num == num_elems) { num_elems = 0; - if (NULL == (last_seg = extend_segment(last_seg, ns_map))) { + if (NULL == (last_seg = pmix_common_dstor_extend_segment(last_seg, ds_ctx->base_path, ns_map->name, + ds_ctx->jobuid, ds_ctx->setjobuid))) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); return rc; } /* mark previous segment as full */ size_t full = 1; - memcpy((uint8_t*)(_ESH_SESSION_sm_seg_last(ns_map->tbl_idx)->seg_info.seg_base_addr + sizeof(size_t)), &full, sizeof(size_t)); - _ESH_SESSION_sm_seg_last(ns_map->tbl_idx) = last_seg; - memset(_ESH_SESSION_sm_seg_last(ns_map->tbl_idx)->seg_info.seg_base_addr, 0, _initial_segment_size); + memcpy((uint8_t*)(_ESH_SESSION_sm_seg_last(ds_ctx->session_array, ns_map->tbl_idx)->seg_info.seg_base_addr + + sizeof(size_t)), &full, sizeof(size_t)); + _ESH_SESSION_sm_seg_last(ds_ctx->session_array, ns_map->tbl_idx) = last_seg; + memset(_ESH_SESSION_sm_seg_last(ds_ctx->session_array, ns_map->tbl_idx)->seg_info.seg_base_addr, + 0, ds_ctx->initial_segment_size); } memset(&elem.ns_map, 0, sizeof(elem.ns_map)); - strncpy(elem.ns_map.name, ns_map->name, sizeof(elem.ns_map.name)-1); + pmix_strncpy(elem.ns_map.name, ns_map->name, sizeof(elem.ns_map.name)-1); elem.ns_map.tbl_idx = ns_map->tbl_idx; elem.num_meta_seg = 1; elem.num_data_seg = 1; - memcpy((uint8_t*)(_ESH_SESSION_sm_seg_last(ns_map->tbl_idx)->seg_info.seg_base_addr) + sizeof(size_t) * 2 + num_elems * sizeof(ns_seg_info_t), - &elem, sizeof(ns_seg_info_t)); + memcpy((uint8_t*)(_ESH_SESSION_sm_seg_last(ds_ctx->session_array, ns_map->tbl_idx)->seg_info.seg_base_addr) + + sizeof(size_t) * 2 + num_elems * sizeof(ns_seg_info_t), &elem, sizeof(ns_seg_info_t)); num_elems++; - memcpy((uint8_t*)(_ESH_SESSION_sm_seg_last(ns_map->tbl_idx)->seg_info.seg_base_addr), &num_elems, sizeof(size_t)); + memcpy((uint8_t*)(_ESH_SESSION_sm_seg_last(ds_ctx->session_array, ns_map->tbl_idx)->seg_info.seg_base_addr), + &num_elems, sizeof(size_t)); return PMIX_SUCCESS; } /* clients should sync local info with information from initial segment regularly */ -static void _update_initial_segment_info(const ns_map_data_t *ns_map) +static void _update_initial_segment_info(pmix_common_dstore_ctx_t *ds_ctx, const ns_map_data_t *ns_map) { - seg_desc_t *tmp; - tmp = _ESH_SESSION_sm_seg_first(ns_map->tbl_idx); + pmix_dstore_seg_desc_t *tmp; + tmp = _ESH_SESSION_sm_seg_first(ds_ctx->session_array, ns_map->tbl_idx); PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, "%s:%d:%s", __FILE__, __LINE__, __func__)); @@ -1404,7 +772,8 @@ static void _update_initial_segment_info(const ns_map_data_t *ns_map) do { /* check if current segment was marked as full but no more next segment is in the chain */ if (NULL == tmp->next && 1 == *((size_t*)((uint8_t*)(tmp->seg_info.seg_base_addr) + sizeof(size_t)))) { - tmp->next = _attach_new_segment(INITIAL_SEGMENT, ns_map, tmp->id+1); + tmp->next = pmix_common_dstor_attach_new_segment(PMIX_DSTORE_INITIAL_SEGMENT, ds_ctx->base_path, + ns_map->name, tmp->id+1); } tmp = tmp->next; } @@ -1412,11 +781,12 @@ static void _update_initial_segment_info(const ns_map_data_t *ns_map) } /* this function will be used by clients to get ns data from the initial segment and add them to the tracker list */ -static ns_seg_info_t *_get_ns_info_from_initial_segment(const ns_map_data_t *ns_map) +static ns_seg_info_t *_get_ns_info_from_initial_segment(pmix_common_dstore_ctx_t *ds_ctx, + const ns_map_data_t *ns_map) { pmix_status_t rc; size_t i; - seg_desc_t *tmp; + pmix_dstore_seg_desc_t *tmp; ns_seg_info_t *elem, *cur_elem; elem = NULL; size_t num_elems; @@ -1424,7 +794,7 @@ static ns_seg_info_t *_get_ns_info_from_initial_segment(const ns_map_data_t *ns_ PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, "%s:%d:%s", __FILE__, __LINE__, __func__)); - tmp = _ESH_SESSION_sm_seg_first(ns_map->tbl_idx); + tmp = _ESH_SESSION_sm_seg_first(ds_ctx->session_array, ns_map->tbl_idx); rc = 1; /* go through all global segments */ @@ -1446,10 +816,11 @@ static ns_seg_info_t *_get_ns_info_from_initial_segment(const ns_map_data_t *ns_ return elem; } -static ns_track_elem_t *_get_track_elem_for_namespace(ns_map_data_t *ns_map) +static ns_track_elem_t *_get_track_elem_for_namespace(pmix_common_dstore_ctx_t *ds_ctx, + ns_map_data_t *ns_map) { ns_track_elem_t *new_elem = NULL; - size_t size = pmix_value_array_get_size(_ns_track_array); + size_t size = pmix_value_array_get_size(ds_ctx->ns_track_array); PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: nspace %s", @@ -1462,27 +833,27 @@ static ns_track_elem_t *_get_track_elem_for_namespace(ns_map_data_t *ns_map) } /* data for this namespace should be already stored in shared memory region. */ /* so go and just put new data. */ - return pmix_value_array_get_item(_ns_track_array, ns_map->track_idx); + return pmix_value_array_get_item(ds_ctx->ns_track_array, ns_map->track_idx); } /* create shared memory regions for this namespace and store its info locally * to operate with address and detach/unlink afterwards. */ - if (NULL == (new_elem = pmix_value_array_get_item(_ns_track_array, size))) { + if (NULL == (new_elem = pmix_value_array_get_item(ds_ctx->ns_track_array, size))) { return NULL; } PMIX_CONSTRUCT(new_elem, ns_track_elem_t); - strncpy(new_elem->ns_map.name, ns_map->name, sizeof(new_elem->ns_map.name)-1); + pmix_strncpy(new_elem->ns_map.name, ns_map->name, sizeof(new_elem->ns_map.name)-1); /* save latest track idx to info of nspace */ ns_map->track_idx = size; return new_elem; } -static rank_meta_info *_get_rank_meta_info(pmix_rank_t rank, seg_desc_t *segdesc) +static rank_meta_info *_get_rank_meta_info(pmix_common_dstore_ctx_t *ds_ctx, pmix_rank_t rank, pmix_dstore_seg_desc_t *segdesc) { size_t i; rank_meta_info *elem = NULL; - seg_desc_t *tmp = segdesc; + pmix_dstore_seg_desc_t *tmp = segdesc; size_t num_elems, rel_offset; int id; rank_meta_info *cur_elem; @@ -1493,7 +864,7 @@ static rank_meta_info *_get_rank_meta_info(pmix_rank_t rank, seg_desc_t *segdesc "%s:%d:%s", __FILE__, __LINE__, __func__)); - if (1 == _direct_mode) { + if (1 == ds_ctx->direct_mode) { /* do linear search to find the requested rank inside all meta segments * for this namespace. */ /* go through all existing meta segments for this namespace */ @@ -1512,8 +883,8 @@ static rank_meta_info *_get_rank_meta_info(pmix_rank_t rank, seg_desc_t *segdesc } else { /* directly compute index of meta segment (id) and relative offset (rel_offset) * inside this segment for fast lookup a rank_meta_info object for the requested rank. */ - id = rcount/_max_meta_elems; - rel_offset = (rcount%_max_meta_elems) * sizeof(rank_meta_info) + sizeof(size_t); + id = rcount/ds_ctx->max_meta_elems; + rel_offset = (rcount % ds_ctx->max_meta_elems) * sizeof(rank_meta_info) + sizeof(size_t); /* go through all existing meta segments for this namespace. * Stop at id number if it exists. */ while (NULL != tmp->next && 0 != id) { @@ -1532,10 +903,10 @@ static rank_meta_info *_get_rank_meta_info(pmix_rank_t rank, seg_desc_t *segdesc return elem; } -static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) +static int set_rank_meta_info(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_info, rank_meta_info *rinfo) { /* it's claimed that there is still no meta info for this rank stored */ - seg_desc_t *tmp; + pmix_dstore_seg_desc_t *tmp; size_t num_elems, rel_offset; int id, count; rank_meta_info *cur_elem; @@ -1552,18 +923,19 @@ static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) (unsigned long)rinfo->offset, (unsigned long)rinfo->count)); tmp = ns_info->meta_seg; - if (1 == _direct_mode) { + if (1 == ds_ctx->direct_mode) { /* get the last meta segment to put new rank_meta_info at the end. */ while (NULL != tmp->next) { tmp = tmp->next; } num_elems = *((size_t*)(tmp->seg_info.seg_base_addr)); - if (_max_meta_elems <= num_elems) { + if (ds_ctx->max_meta_elems <= num_elems) { PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, "%s:%d:%s: extend meta segment for nspace %s", __FILE__, __LINE__, __func__, ns_info->ns_map.name)); /* extend meta segment, so create a new one */ - tmp = extend_segment(tmp, &ns_info->ns_map); + tmp = pmix_common_dstor_extend_segment(tmp, ds_ctx->base_path, ns_info->ns_map.name, + ds_ctx->jobuid, ds_ctx->setjobuid); if (NULL == tmp) { PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; @@ -1571,7 +943,7 @@ static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) ns_info->num_meta_seg++; memset(tmp->seg_info.seg_base_addr, 0, sizeof(rank_meta_info)); /* update number of meta segments for namespace in initial_segment */ - ns_seg_info_t *elem = _get_ns_info_from_initial_segment(&ns_info->ns_map); + ns_seg_info_t *elem = _get_ns_info_from_initial_segment(ds_ctx, &ns_info->ns_map); if (NULL == elem) { PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; @@ -1589,8 +961,8 @@ static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) /* directly compute index of meta segment (id) and relative offset (rel_offset) * inside this segment for fast lookup a rank_meta_info object for the requested rank. */ size_t rcount = rinfo->rank == PMIX_RANK_WILDCARD ? 0 : rinfo->rank + 1; - id = rcount/_max_meta_elems; - rel_offset = (rcount % _max_meta_elems) * sizeof(rank_meta_info) + sizeof(size_t); + id = rcount/ds_ctx->max_meta_elems; + rel_offset = (rcount % ds_ctx->max_meta_elems) * sizeof(rank_meta_info) + sizeof(size_t); count = id; /* go through all existing meta segments for this namespace. * Stop at id number if it exists. */ @@ -1602,7 +974,8 @@ static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) if ((int)ns_info->num_meta_seg < (id+1)) { while ((int)ns_info->num_meta_seg != (id+1)) { /* extend meta segment, so create a new one */ - tmp = extend_segment(tmp, &ns_info->ns_map); + tmp = pmix_common_dstor_extend_segment(tmp, ds_ctx->base_path, ns_info->ns_map.name, + ds_ctx->jobuid, ds_ctx->setjobuid); if (NULL == tmp) { PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; @@ -1611,7 +984,7 @@ static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) ns_info->num_meta_seg++; } /* update number of meta segments for namespace in initial_segment */ - ns_seg_info_t *elem = _get_ns_info_from_initial_segment(&ns_info->ns_map); + ns_seg_info_t *elem = _get_ns_info_from_initial_segment(ds_ctx, &ns_info->ns_map); if (NULL == elem) { PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; @@ -1627,9 +1000,9 @@ static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) return PMIX_SUCCESS; } -static uint8_t *_get_data_region_by_offset(seg_desc_t *segdesc, size_t offset) +static uint8_t *_get_data_region_by_offset(pmix_common_dstore_ctx_t *ds_ctx, pmix_dstore_seg_desc_t *segdesc, size_t offset) { - seg_desc_t *tmp = segdesc; + pmix_dstore_seg_desc_t *tmp = segdesc; size_t rel_offset = offset; uint8_t *dataaddr = NULL; @@ -1639,8 +1012,8 @@ static uint8_t *_get_data_region_by_offset(seg_desc_t *segdesc, size_t offset) /* go through all existing data segments for this namespace */ do { - if (rel_offset >= _data_segment_size) { - rel_offset -= _data_segment_size; + if (rel_offset >= ds_ctx->data_segment_size) { + rel_offset -= ds_ctx->data_segment_size; } else { dataaddr = tmp->seg_info.seg_base_addr + rel_offset; } @@ -1650,10 +1023,10 @@ static uint8_t *_get_data_region_by_offset(seg_desc_t *segdesc, size_t offset) return dataaddr; } -static size_t get_free_offset(seg_desc_t *data_seg) +static size_t get_free_offset(pmix_common_dstore_ctx_t *ds_ctx, pmix_dstore_seg_desc_t *data_seg) { size_t offset; - seg_desc_t *tmp; + pmix_dstore_seg_desc_t *tmp; int id = 0; tmp = data_seg; /* first find the last data segment */ @@ -1666,35 +1039,42 @@ static size_t get_free_offset(seg_desc_t *data_seg) /* this is the first created data segment, the first 8 bytes are used to place the free offset value itself */ offset = sizeof(size_t); } - return (id * _data_segment_size + offset); + return (id * ds_ctx->data_segment_size + offset); } -static int put_empty_ext_slot(seg_desc_t *dataseg) +static int put_empty_ext_slot(pmix_common_dstore_ctx_t *ds_ctx, pmix_dstore_seg_desc_t *dataseg) { size_t global_offset, rel_offset, data_ended, val = 0; uint8_t *addr; - global_offset = get_free_offset(dataseg); - rel_offset = global_offset % _data_segment_size; - if (rel_offset + EXT_SLOT_SIZE() > _data_segment_size) { + pmix_status_t rc; + + global_offset = get_free_offset(ds_ctx, dataseg); + rel_offset = global_offset % ds_ctx->data_segment_size; + if (rel_offset + PMIX_DS_SLOT_SIZE(ds_ctx) > ds_ctx->data_segment_size) { PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; } - addr = _get_data_region_by_offset(dataseg, global_offset); - ESH_PUT_KEY(addr, ESH_REGION_EXTENSION, (void*)&val, sizeof(size_t)); - + addr = _get_data_region_by_offset(ds_ctx, dataseg, global_offset); + PMIX_DS_PUT_KEY(rc, ds_ctx, addr, ESH_REGION_EXTENSION, (void*)&val, sizeof(size_t)); + if (rc != PMIX_SUCCESS) { + PMIX_ERROR_LOG(rc); + return rc; + } /* update offset at the beginning of current segment */ - data_ended = rel_offset + EXT_SLOT_SIZE(); + data_ended = rel_offset + PMIX_DS_SLOT_SIZE(ds_ctx); addr = (uint8_t*)(addr - rel_offset); memcpy(addr, &data_ended, sizeof(size_t)); return PMIX_SUCCESS; } -static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, char *key, void *buffer, size_t size) +static size_t put_data_to_the_end(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_info, + pmix_dstore_seg_desc_t *dataseg, char *key, void *buffer, size_t size) { size_t offset, id = 0; - seg_desc_t *tmp; + pmix_dstore_seg_desc_t *tmp; size_t global_offset, data_ended; uint8_t *addr; + pmix_status_t rc; PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, "%s:%d:%s: key %s", @@ -1705,17 +1085,19 @@ static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, tmp = tmp->next; id++; } - global_offset = get_free_offset(dataseg); - offset = global_offset % _data_segment_size; + global_offset = get_free_offset(ds_ctx, dataseg); + offset = global_offset % ds_ctx->data_segment_size; /* We should provide additional space at the end of segment to * place EXTENSION_SLOT to have an ability to enlarge data for this rank.*/ - if ((sizeof(size_t) + ESH_KEY_SIZE(key, size) + EXT_SLOT_SIZE()) > _data_segment_size) { + if ((sizeof(size_t) + PMIX_DS_KEY_SIZE(ds_ctx, key, size) + PMIX_DS_SLOT_SIZE(ds_ctx)) > + ds_ctx->data_segment_size) { /* this is an error case: segment is so small that cannot place evem a single key-value pair. * warn a user about it and fail. */ offset = 0; /* offset cannot be 0 in normal case, so we use this value to indicate a problem. */ pmix_output(0, "PLEASE set NS_DATA_SEG_SIZE to value which is larger when %lu.", - (unsigned long)(sizeof(size_t) + strlen(key) + 1 + sizeof(size_t) + size + EXT_SLOT_SIZE())); + (unsigned long)(sizeof(size_t) + strlen(key) + 1 + sizeof(size_t) + + size + PMIX_DS_SLOT_SIZE(ds_ctx))); return offset; } @@ -1726,10 +1108,12 @@ static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, * new segment wasn't allocated to us but (global_offset % _data_segment_size) == 0 * so if offset is 0 here - we need to allocate the segment as well */ - if ( (0 == offset) || ( (offset + ESH_KEY_SIZE(key, size) + EXT_SLOT_SIZE()) > _data_segment_size) ) { + if ( (0 == offset) || ( (offset + PMIX_DS_KEY_SIZE(ds_ctx, key, size) + + PMIX_DS_SLOT_SIZE(ds_ctx)) > ds_ctx->data_segment_size) ) { id++; /* create a new data segment. */ - tmp = extend_segment(tmp, &ns_info->ns_map); + tmp = pmix_common_dstor_extend_segment(tmp, ds_ctx->base_path, ns_info->ns_map.name, + ds_ctx->jobuid, ds_ctx->setjobuid); if (NULL == tmp) { PMIX_ERROR_LOG(PMIX_ERR_NOMEM); offset = 0; /* offset cannot be 0 in normal case, so we use this value to indicate a problem. */ @@ -1737,7 +1121,7 @@ static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, } ns_info->num_data_seg++; /* update_ns_info_in_initial_segment */ - ns_seg_info_t *elem = _get_ns_info_from_initial_segment(&ns_info->ns_map); + ns_seg_info_t *elem = _get_ns_info_from_initial_segment(ds_ctx, &ns_info->ns_map); if (NULL == elem) { PMIX_ERROR_LOG(PMIX_ERR_NOMEM); offset = 0; /* offset cannot be 0 in normal case, so we use this value to indicate a problem. */ @@ -1746,12 +1130,16 @@ static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, elem->num_data_seg++; offset = sizeof(size_t); } - global_offset = offset + id * _data_segment_size; + global_offset = offset + id * ds_ctx->data_segment_size; addr = (uint8_t*)(tmp->seg_info.seg_base_addr)+offset; - ESH_PUT_KEY(addr, key, buffer, size); + PMIX_DS_PUT_KEY(rc, ds_ctx, addr, key, buffer, size); + if (rc != PMIX_SUCCESS) { + PMIX_ERROR_LOG(rc); + return 0; + } /* update offset at the beginning of current segment */ - data_ended = offset + ESH_KEY_SIZE(key, size); + data_ended = offset + PMIX_DS_KEY_SIZE(ds_ctx, key, size); addr = (uint8_t*)(tmp->seg_info.seg_base_addr); memcpy(addr, &data_ended, sizeof(size_t)); PMIX_OUTPUT_VERBOSE((1, pmix_gds_base_framework.framework_output, @@ -1759,17 +1147,18 @@ static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, __FILE__, __LINE__, __func__, key, (unsigned long)offset, (unsigned long)data_ended, - (unsigned long)(id * _data_segment_size), + (unsigned long)(id * ds_ctx->data_segment_size), (unsigned long)size)); return global_offset; } -static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t *kval, rank_meta_info **rinfo, int data_exist) +static int pmix_sm_store(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_info, + pmix_rank_t rank, pmix_kval_t *kval, rank_meta_info **rinfo, int data_exist) { size_t offset, size, kval_cnt; pmix_buffer_t buffer; pmix_status_t rc; - seg_desc_t *datadesc; + pmix_dstore_seg_desc_t *datadesc; uint8_t *addr; PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, @@ -1779,7 +1168,7 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t datadesc = ns_info->data_seg; /* pack value to the buffer */ PMIX_CONSTRUCT(&buffer, pmix_buffer_t); - PMIX_BFROPS_PACK(rc, _client_peer(), &buffer, kval->value, 1, PMIX_VALUE); + PMIX_BFROPS_PACK(rc, _client_peer(ds_ctx), &buffer, kval->value, 1, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto exit; @@ -1789,8 +1178,8 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t if (0 == data_exist) { /* there is no data blob for this rank yet, so add it. */ size_t free_offset; - free_offset = get_free_offset(datadesc); - offset = put_data_to_the_end(ns_info, datadesc, kval->key, buffer.base_ptr, size); + free_offset = get_free_offset(ds_ctx, datadesc); + offset = put_data_to_the_end(ds_ctx, ns_info, datadesc, kval->key, buffer.base_ptr, size); if (0 == offset) { /* this is an error */ rc = PMIX_ERROR; @@ -1804,11 +1193,15 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t * because previous segment is already full. */ if (free_offset != offset && NULL != *rinfo) { /* here we compare previous free offset with the offset where we just put data. - * It should be equal in the normal case. It it's not true, then it means that + * It should be equal in the normal case. If it's not true, then it means that * segment was extended, and we put data to the next segment, so we now need to * put extension slot at the end of previous segment with a "reference" to a new_offset */ - addr = _get_data_region_by_offset(datadesc, free_offset); - ESH_PUT_KEY(addr, ESH_REGION_EXTENSION, (void*)&offset, sizeof(size_t)); + addr = _get_data_region_by_offset(ds_ctx, datadesc, free_offset); + PMIX_DS_PUT_KEY(rc, ds_ctx, addr, ESH_REGION_EXTENSION, (void*)&offset, sizeof(size_t)); + if (rc != PMIX_SUCCESS) { + PMIX_ERROR_LOG(rc); + return 0; + } } if (NULL == *rinfo) { *rinfo = (rank_meta_info*)malloc(sizeof(rank_meta_info)); @@ -1819,7 +1212,7 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t (*rinfo)->count++; } else if (NULL != *rinfo) { /* there is data blob for this rank */ - addr = _get_data_region_by_offset(datadesc, (*rinfo)->offset); + addr = _get_data_region_by_offset(ds_ctx, datadesc, (*rinfo)->offset); if (NULL == addr) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); @@ -1841,8 +1234,8 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t * ..... * extension slot which has key = EXTENSION_SLOT and a size_t value for offset to next data address for this process. */ - if (0 == strncmp(ESH_KNAME_PTR(addr), ESH_REGION_EXTENSION, ESH_KNAME_LEN(ESH_REGION_EXTENSION))) { - memcpy(&offset, ESH_DATA_PTR(addr), sizeof(size_t)); + if(PMIX_DS_KEY_IS_EXTSLOT(ds_ctx, addr)) { + memcpy(&offset, PMIX_DS_DATA_PTR(ds_ctx, addr), sizeof(size_t)); if (0 < offset) { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %lu, replace flag %d %s is filled with %lu value", @@ -1850,7 +1243,7 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t (unsigned long)rank, data_exist, ESH_REGION_EXTENSION, (unsigned long)offset)); /* go to next item, updating address */ - addr = _get_data_region_by_offset(datadesc, offset); + addr = _get_data_region_by_offset(ds_ctx, datadesc, offset); if (NULL == addr) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); @@ -1859,20 +1252,21 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t } else { /* should not be, we should be out of cycle when this happens */ } - } else if (0 == strncmp(ESH_KNAME_PTR(addr), kval->key, ESH_KNAME_LEN(kval->key))) { + } else if (0 == strncmp(PMIX_DS_KNAME_PTR(ds_ctx, addr), kval->key, + PMIX_DS_KNAME_LEN(ds_ctx, kval->key))) { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %u, replace flag %d found target key %s", __FILE__, __LINE__, __func__, rank, data_exist, kval->key)); /* target key is found, compare value sizes */ - if (ESH_DATA_SIZE(addr, ESH_DATA_PTR(addr)) != size) { + if (PMIX_DS_DATA_SIZE(ds_ctx, addr, PMIX_DS_DATA_PTR(ds_ctx, addr)) != size) { //if (1) { /* if we want to test replacing values for existing keys. */ /* invalidate current value and store another one at the end of data region. */ - strncpy(ESH_KNAME_PTR(addr), ESH_REGION_INVALIDATED, ESH_KNAME_LEN(ESH_REGION_INVALIDATED)); + PMIX_DS_KEY_SET_INVALID(ds_ctx, addr); /* decrementing count, it will be incremented back when we add a new value for this key at the end of region. */ (*rinfo)->count--; kval_cnt--; /* go to next item, updating address */ - addr += ESH_KV_SIZE(addr); + addr += PMIX_DS_KV_SIZE(ds_ctx, addr); PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %u, replace flag %d mark key %s regions as invalidated. put new data at the end.", __FILE__, __LINE__, __func__, rank, data_exist, kval->key)); @@ -1881,23 +1275,25 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t "%s:%d:%s: for rank %u, replace flag %d replace data for key %s type %d in place", __FILE__, __LINE__, __func__, rank, data_exist, kval->key, kval->value->type)); /* replace old data with new one. */ - memset(ESH_DATA_PTR(addr), 0, ESH_DATA_SIZE(addr, ESH_DATA_PTR(addr))); - memcpy(ESH_DATA_PTR(addr), buffer.base_ptr, size); - addr += ESH_KV_SIZE(addr); + memset(PMIX_DS_DATA_PTR(ds_ctx, addr), 0, + PMIX_DS_DATA_SIZE(ds_ctx, addr, PMIX_DS_DATA_PTR(ds_ctx, addr))); + memcpy(PMIX_DS_DATA_PTR(ds_ctx, addr), buffer.base_ptr, size); + addr += PMIX_DS_KV_SIZE(ds_ctx, addr); add_to_the_end = 0; break; } } else { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %u, replace flag %d skip %s key, look for %s key", - __FILE__, __LINE__, __func__, rank, data_exist, ESH_KNAME_PTR(addr), kval->key)); + __FILE__, __LINE__, __func__, rank, data_exist, + PMIX_DS_KNAME_PTR(ds_ctx, addr), kval->key)); /* Skip it: key is "INVALIDATED" or key is valid but different from target one. */ - if (0 != strncmp(ESH_REGION_INVALIDATED, ESH_KNAME_PTR(addr), ESH_KNAME_LEN(ESH_KNAME_PTR(addr)))) { + if (!PMIX_DS_KEY_IS_INVALID(ds_ctx, addr)) { /* count only valid items */ kval_cnt--; } /* go to next item, updating address */ - addr += ESH_KV_SIZE(addr); + addr += PMIX_DS_KV_SIZE(ds_ctx, addr); } } if (1 == add_to_the_end) { @@ -1906,9 +1302,68 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t * for the same key. */ size_t free_offset; (*rinfo)->count++; - free_offset = get_free_offset(datadesc); + free_offset = get_free_offset(ds_ctx, datadesc); + + /* + * Remove trailing extention slot if we are continuing + * same ranks data. + * + * When keys are stored individually through _store_data_for_rank + * an empty extention slot is placed every time. + * + * This is required because there is no information about whether or not the next key + * will belong to the same rank. + * + * As the result EACH keys stored with _store_data_for_rank is + * followed by extension slot. This slows down search and increases + * the memory footprint. + * + * The following code tries to deal with such one-key-at-a-time + * situation by: + * - checking if the last key-value for this rank is an extention + * slot + * - If this is the case - checks if this key-value pair is the + * last one at the moment and can be safely deleted. + * - if it is - current segment's offset pointer is decreased by + * the size of the extention slot key-value effectively removing + * it from the dstor + */ + if (PMIX_DS_KEY_IS_EXTSLOT(ds_ctx, addr)){ + /* Find the last data segment */ + pmix_dstore_seg_desc_t *ldesc = datadesc; + uint8_t *segstart; + size_t offs_past_extslot = 0; + size_t offs_cur_segment = 0; + while (NULL != ldesc->next) { + ldesc = ldesc->next; + } + + /* Calculate the offset of the end of the extension slot */ + offs_cur_segment = free_offset % ds_ctx->data_segment_size; + segstart = ldesc->seg_info.seg_base_addr; + offs_past_extslot = (addr + PMIX_DS_KV_SIZE(ds_ctx, addr)) - segstart; + + /* We can erase extension slot if: + * - address of the ext slot belongs to the occupied part of the + * last segment + * - local offset within the segment is equal to the local + * offset of the end of extension slot + */ + if( ( (addr > segstart) && (addr < (segstart + offs_cur_segment)) ) + && (offs_cur_segment == offs_past_extslot) ) { + /* Calculate a new free offset that doesn't account this + * extension slot */ + size_t new_offset = addr - segstart; + /* Rewrite segment's offset information to exclude + * extension slot */ + memcpy(segstart, &new_offset, sizeof(size_t)); + /* Recalculate free_offset */ + free_offset = get_free_offset(ds_ctx, datadesc); + } + } + /* add to the end */ - offset = put_data_to_the_end(ns_info, datadesc, kval->key, buffer.base_ptr, size); + offset = put_data_to_the_end(ds_ctx, ns_info, datadesc, kval->key, buffer.base_ptr, size); if (0 == offset) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); @@ -1919,11 +1374,11 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t * data for different ranks, and that's why next element is EXTENSION_SLOT. * We put new data to the end of data region and just update EXTENSION_SLOT value by new offset. */ - if (0 == strncmp(ESH_KNAME_PTR(addr), ESH_REGION_EXTENSION, ESH_KNAME_LEN(ESH_REGION_EXTENSION))) { + if (PMIX_DS_KEY_IS_EXTSLOT(ds_ctx, addr)) { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %u, replace flag %d %s should be filled with offset %lu value", __FILE__, __LINE__, __func__, rank, data_exist, ESH_REGION_EXTENSION, offset)); - memcpy(ESH_DATA_PTR(addr), &offset, sizeof(size_t)); + memcpy(PMIX_DS_DATA_PTR(ds_ctx, addr), &offset, sizeof(size_t)); } else { /* (2) - we point to the first free offset, no more data is stored further in this segment. * There is no EXTENSION_SLOT by this addr since we continue pushing data for the same rank, @@ -1933,7 +1388,11 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t * forcibly and store new offset in its value. */ if (free_offset != offset) { /* segment was extended, need to put extension slot by free_offset indicating new_offset */ - ESH_PUT_KEY(addr, ESH_REGION_EXTENSION, (void*)&offset, sizeof(size_t)); + PMIX_DS_PUT_KEY(rc, ds_ctx, addr, ESH_REGION_EXTENSION, (void*)&offset, sizeof(size_t)); + if (rc != PMIX_SUCCESS) { + PMIX_ERROR_LOG(rc); + return 0; + } } } PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, @@ -1946,12 +1405,13 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t return rc; } -static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_buffer_t *buf) +static int _store_data_for_rank(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_info, + pmix_rank_t rank, pmix_buffer_t *buf) { pmix_status_t rc; pmix_kval_t *kp; - seg_desc_t *metadesc, *datadesc; + pmix_dstore_seg_desc_t *metadesc, *datadesc; int32_t cnt; rank_meta_info *rinfo = NULL; @@ -1972,11 +1432,11 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix num_elems = *((size_t*)(metadesc->seg_info.seg_base_addr)); data_exist = 0; - /* when we don't use linear search (_direct_mode ==0 ) we don't use num_elems field, + /* when we don't use linear search (direct_mode == 0) we don't use num_elems field, * so anyway try to get rank_meta_info first. */ - if (0 < num_elems || 0 == _direct_mode) { + if (0 < num_elems || 0 == ds_ctx->direct_mode) { /* go through all elements in meta segment and look for target rank. */ - rinfo = _get_rank_meta_info(rank, metadesc); + rinfo = _get_rank_meta_info(ds_ctx, rank, metadesc); if (NULL != rinfo) { data_exist = 1; } @@ -1985,14 +1445,14 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix * so unpack these buffers, and then unpack kvals from each modex buffer, * storing them in the shared memory dstore. */ - free_offset = get_free_offset(datadesc); + free_offset = get_free_offset(ds_ctx, datadesc); cnt = 1; kp = PMIX_NEW(pmix_kval_t); PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, buf, kp, &cnt, PMIX_KVAL); while(PMIX_SUCCESS == rc) { pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "pmix: unpacked key %s", kp->key); - if (PMIX_SUCCESS != (rc = pmix_sm_store(ns_info, rank, kp, &rinfo, data_exist))) { + if (PMIX_SUCCESS != (rc = pmix_sm_store(ds_ctx, ns_info, rank, kp, &rinfo, data_exist))) { PMIX_ERROR_LOG(rc); if (NULL != rinfo) { free(rinfo); @@ -2019,7 +1479,7 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix * in that case we don't reserve space for EXTENSION_SLOT, it's * already reserved. * */ - new_free_offset = get_free_offset(datadesc); + new_free_offset = get_free_offset(ds_ctx, datadesc); if (new_free_offset != free_offset) { /* Reserve space for EXTENSION_SLOT at the end of data blob. * We need it to split data for one rank from data for different @@ -2027,7 +1487,7 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix * We also put EXTENSION_SLOT at the end of each data segment, and * its value points to the beginning of next data segment. * */ - rc = put_empty_ext_slot(ns_info->data_seg); + rc = put_empty_ext_slot(ds_ctx, ns_info->data_seg); if (PMIX_SUCCESS != rc) { if ((0 == data_exist) && NULL != rinfo) { free(rinfo); @@ -2040,7 +1500,7 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix /* if this is the first data posted for this rank, then * update meta info for it */ if (0 == data_exist) { - set_rank_meta_info(ns_info, rinfo); + set_rank_meta_info(ds_ctx, ns_info, rinfo); if (NULL != rinfo) { free(rinfo); } @@ -2049,13 +1509,13 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix return rc; } -static inline ssize_t _get_univ_size(const char *nspace) +static inline ssize_t _get_univ_size(pmix_common_dstore_ctx_t *ds_ctx, const char *nspace) { ssize_t nprocs = 0; pmix_value_t *val; int rc; - rc = _dstore_fetch(nspace, PMIX_RANK_WILDCARD, PMIX_UNIV_SIZE, &val); + rc = _dstore_fetch(ds_ctx, nspace, PMIX_RANK_WILDCARD, PMIX_UNIV_SIZE, &val); if( PMIX_SUCCESS != rc ) { PMIX_ERROR_LOG(rc); return rc; @@ -2070,23 +1530,39 @@ static inline ssize_t _get_univ_size(const char *nspace) return nprocs; } -static pmix_status_t dstore_cache_job_info(struct pmix_nspace_t *ns, +PMIX_EXPORT pmix_status_t pmix_common_dstor_cache_job_info(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_namespace_t *ns, pmix_info_t info[], size_t ninfo) { return PMIX_SUCCESS; } -static pmix_status_t dstore_init(pmix_info_t info[], size_t ninfo) + +pmix_common_dstore_ctx_t *pmix_common_dstor_init(const char *ds_name, pmix_info_t info[], size_t ninfo, + pmix_common_lock_callbacks_t *lock_cb, + pmix_common_dstore_file_cbs_t *file_cb) { pmix_status_t rc; size_t n; char *dstor_tmpdir = NULL; - size_t tbl_idx=0; + size_t tbl_idx = 0; ns_map_data_t *ns_map = NULL; + pmix_common_dstore_ctx_t *ds_ctx = NULL; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "pmix:gds:dstore init"); + ds_ctx = (pmix_common_dstore_ctx_t*) malloc(sizeof(*ds_ctx)); + if (NULL == ds_ctx) { + PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); + return NULL; + } + memset(ds_ctx, 0, sizeof(*ds_ctx)); + + /* assign lock callbacks */ + ds_ctx->lock_cbs = lock_cb; + ds_ctx->file_cbs = file_cb; + /* open the pshmem and select the active plugins */ if( PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pshmem_base_framework, 0)) ) { PMIX_ERROR_LOG(rc); @@ -2097,17 +1573,10 @@ static pmix_status_t dstore_init(pmix_info_t info[], size_t ninfo) goto err_exit; } - _jobuid = getuid(); - _setjobuid = 0; - -#ifdef ESH_PTHREAD_LOCK - _esh_lock_init = _rwlock_init; -#endif -#ifdef ESH_FCNTL_LOCK - _esh_lock_init = _flock_init; -#endif + ds_ctx->jobuid = getuid(); + ds_ctx->setjobuid = 0; - if (PMIX_SUCCESS != (rc = _esh_tbls_init())) { + if (PMIX_SUCCESS != (rc = _esh_tbls_init(ds_ctx))) { PMIX_ERROR_LOG(rc); goto err_exit; } @@ -2118,23 +1587,19 @@ static pmix_status_t dstore_init(pmix_info_t info[], size_t ninfo) goto err_exit; } - _set_constants_from_env(); - - if (NULL != _base_path) { - free(_base_path); - _base_path = NULL; - } + _set_constants_from_env(ds_ctx); + ds_ctx->ds_name = strdup(ds_name); /* find the temp dir */ if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - _esh_session_map_search = _esh_session_map_search_server; + ds_ctx->session_map_search = (session_map_search_fn_t)_esh_session_map_search_server; /* scan incoming info for directives */ if (NULL != info) { for (n=0; n < ninfo; n++) { if (0 == strcmp(PMIX_USERID, info[n].key)) { - _jobuid = info[n].value.data.uint32; - _setjobuid = 1; + ds_ctx->jobuid = info[n].value.data.uint32; + ds_ctx->setjobuid = 1; continue; } if (0 == strcmp(PMIX_DSTPATH, info[n].key)) { @@ -2180,67 +1645,109 @@ static pmix_status_t dstore_init(pmix_info_t info[], size_t ninfo) } } - rc = asprintf(&_base_path, "%s/pmix_dstor_%d", dstor_tmpdir, getpid()); - if ((0 > rc) || (NULL == _base_path)) { + rc = asprintf(&ds_ctx->base_path, "%s/pmix_dstor_%s_%d", dstor_tmpdir, + ds_ctx->ds_name, getpid()); + if ((0 > rc) || (NULL == ds_ctx->base_path)) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto err_exit; } - if (0 != mkdir(_base_path, 0770)) { + if (0 != mkdir(ds_ctx->base_path, 0770)) { if (EEXIST != errno) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); goto err_exit; } } - if (_setjobuid > 0) { - if (chown(_base_path, (uid_t) _jobuid, (gid_t) -1) < 0){ + if (ds_ctx->setjobuid > 0) { + if (chown(ds_ctx->base_path, (uid_t) ds_ctx->jobuid, (gid_t) -1) < 0){ rc = PMIX_ERR_NO_PERMISSIONS; PMIX_ERROR_LOG(rc); goto err_exit; } } - _esh_session_map_search = _esh_session_map_search_server; - return PMIX_SUCCESS; + ds_ctx->session_map_search = _esh_session_map_search_server; + return ds_ctx; } /* for clients */ else { - if (NULL == (dstor_tmpdir = getenv(PMIX_DSTORE_ESH_BASE_PATH))){ - return PMIX_ERR_NOT_AVAILABLE; // simply disqualify ourselves + char *env_name = NULL; + int ds_ver = 0; + + sscanf(ds_ctx->ds_name, "ds%d", &ds_ver); + if (0 == ds_ver) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto err_exit; + } + if (0 > asprintf(&env_name, PMIX_DSTORE_VER_BASE_PATH_FMT, ds_ver)) { + rc = PMIX_ERR_NOMEM; + PMIX_ERROR_LOG(rc); + goto err_exit; + } + dstor_tmpdir = getenv(env_name); + free(env_name); + + if (NULL == dstor_tmpdir) { + dstor_tmpdir = getenv(PMIX_DSTORE_ESH_BASE_PATH); + } + if (NULL == dstor_tmpdir){ + rc = PMIX_ERR_NOT_AVAILABLE; // simply disqualify ourselves + goto err_exit; } - if (NULL == (_base_path = strdup(dstor_tmpdir))) { + if (NULL == (ds_ctx->base_path = strdup(dstor_tmpdir))) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto err_exit; } - _esh_session_map_search = _esh_session_map_search_client; + ds_ctx->session_map_search = _esh_session_map_search_client; + /* init ds_ctx protect lock */ + if (0 != pthread_mutex_init(&ds_ctx->lock, NULL)) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto err_exit; + } } - rc = _esh_session_tbl_add(&tbl_idx); + rc = _esh_session_tbl_add(ds_ctx, &tbl_idx); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto err_exit; } - ns_map = _esh_session_map(pmix_globals.myid.nspace, tbl_idx); + char *nspace = NULL; + /* if we don't see the required info, then we cannot init */ + if (NULL == (nspace = getenv("PMIX_NAMESPACE"))) { + rc = PMIX_ERR_INVALID_NAMESPACE; + PMIX_ERROR_LOG(rc); + goto err_exit; + } + /* lock init */ + rc = ds_ctx->lock_cbs->init(&_ESH_SESSION_lock(ds_ctx->session_array, tbl_idx), ds_ctx->base_path, nspace, 1, ds_ctx->jobuid, ds_ctx->setjobuid); + if (rc != PMIX_SUCCESS) { + goto err_exit; + } + ns_map = _esh_session_map(ds_ctx, nspace, 0, tbl_idx); if (NULL == ns_map) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto err_exit; } - if (PMIX_SUCCESS != (rc =_esh_session_init(tbl_idx, ns_map, _jobuid, _setjobuid))) { + if (PMIX_SUCCESS != (rc =_esh_session_init(ds_ctx, tbl_idx, ns_map, 1, + ds_ctx->jobuid, ds_ctx->setjobuid))) { PMIX_ERROR_LOG(rc); goto err_exit; } - return PMIX_SUCCESS; + return ds_ctx; err_exit: - return rc; + pmix_common_dstor_finalize(ds_ctx); + return NULL; } -static void dstore_finalize(void) +PMIX_EXPORT void pmix_common_dstor_finalize(pmix_common_dstore_ctx_t *ds_ctx) { struct stat st = {0}; pmix_status_t rc = PMIX_SUCCESS; @@ -2248,38 +1755,45 @@ static void dstore_finalize(void) PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s", __FILE__, __LINE__, __func__)); - _esh_sessions_cleanup(); - _esh_ns_map_cleanup(); - _esh_ns_track_cleanup(); + _esh_sessions_cleanup(ds_ctx); + _esh_ns_map_cleanup(ds_ctx); + _esh_ns_track_cleanup(ds_ctx); pmix_pshmem.finalize(); - if (NULL != _base_path){ + if (NULL != ds_ctx->base_path){ if(PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - if (lstat(_base_path, &st) >= 0){ - if (PMIX_SUCCESS != (rc = _esh_dir_del(_base_path))) { + if (lstat(ds_ctx->base_path, &st) >= 0){ + if (PMIX_SUCCESS != (rc = _esh_dir_del(ds_ctx->base_path))) { PMIX_ERROR_LOG(rc); } } } - free(_base_path); - _base_path = NULL; + free(ds_ctx->base_path); + ds_ctx->base_path = NULL; + } + if (NULL != ds_ctx->clients_peer) { + PMIX_RELEASE(ds_ctx->clients_peer->nptr); + PMIX_RELEASE(ds_ctx->clients_peer); } - if (NULL != _clients_peer) { - PMIX_RELEASE(_clients_peer->nptr); - PMIX_RELEASE(_clients_peer); + /* close the pshmem framework */ + if( PMIX_SUCCESS != (rc = pmix_mca_base_framework_close(&pmix_pshmem_base_framework)) ) { + PMIX_ERROR_LOG(rc); } + free(ds_ctx->ds_name); + free(ds_ctx->base_path); + free(ds_ctx); } -static pmix_status_t _dstore_store(const char *nspace, - pmix_rank_t rank, - pmix_kval_t *kv) +static pmix_status_t _dstore_store_nolock(pmix_common_dstore_ctx_t *ds_ctx, + ns_map_data_t *ns_map, + pmix_rank_t rank, + pmix_kval_t *kv) { - pmix_status_t rc = PMIX_SUCCESS, tmp_rc; + pmix_status_t rc = PMIX_SUCCESS; ns_track_elem_t *elem; pmix_buffer_t xfer; ns_seg_info_t ns_info; - ns_map_data_t *ns_map = NULL; if (NULL == kv) { return PMIX_ERROR; @@ -2287,19 +1801,7 @@ static pmix_status_t _dstore_store(const char *nspace, PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for %s:%u", - __FILE__, __LINE__, __func__, nspace, rank)); - - if (NULL == (ns_map = _esh_session_map_search(nspace))) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - - /* set exclusive lock */ - if (PMIX_SUCCESS != (rc = _ESH_WRLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { - PMIX_ERROR_LOG(rc); - return rc; - } + __FILE__, __LINE__, __func__, ns_map->name, rank)); /* First of all, we go through local track list (list of ns_track_elem_t structures) * and look for an element for the target namespace. @@ -2311,36 +1813,36 @@ static pmix_status_t _dstore_store(const char *nspace, * All this stuff is done inside _get_track_elem_for_namespace function. */ - elem = _get_track_elem_for_namespace(ns_map); + elem = _get_track_elem_for_namespace(ds_ctx, ns_map); if (NULL == elem) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); - goto err_exit; + goto exit; } /* If a new element was just created, we need to create corresponding meta and * data segments and update corresponding element's fields. */ if (NULL == elem->meta_seg || NULL == elem->data_seg) { memset(&ns_info.ns_map, 0, sizeof(ns_info.ns_map)); - strncpy(ns_info.ns_map.name, ns_map->name, sizeof(ns_info.ns_map.name)-1); + pmix_strncpy(ns_info.ns_map.name, ns_map->name, sizeof(ns_info.ns_map.name)-1); ns_info.ns_map.tbl_idx = ns_map->tbl_idx; ns_info.num_meta_seg = 1; ns_info.num_data_seg = 1; - rc = _update_ns_elem(elem, &ns_info); + rc = _update_ns_elem(ds_ctx, elem, &ns_info); if (PMIX_SUCCESS != rc || NULL == elem->meta_seg || NULL == elem->data_seg) { PMIX_ERROR_LOG(rc); - goto err_exit; + goto exit; } /* zero created shared memory segments for this namespace */ - memset(elem->meta_seg->seg_info.seg_base_addr, 0, _meta_segment_size); - memset(elem->data_seg->seg_info.seg_base_addr, 0, _data_segment_size); + memset(elem->meta_seg->seg_info.seg_base_addr, 0, ds_ctx->meta_segment_size); + memset(elem->data_seg->seg_info.seg_base_addr, 0, ds_ctx->data_segment_size); /* put ns's shared segments info to the global meta segment. */ - rc = _put_ns_info_to_initial_segment(ns_map, &elem->meta_seg->seg_info, &elem->data_seg->seg_info); + rc = _put_ns_info_to_initial_segment(ds_ctx, ns_map, &elem->meta_seg->seg_info, &elem->data_seg->seg_info); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - goto err_exit; + goto exit; } } @@ -2349,34 +1851,28 @@ static pmix_status_t _dstore_store(const char *nspace, PMIX_CONSTRUCT(&xfer, pmix_buffer_t); PMIX_LOAD_BUFFER(pmix_globals.mypeer, &xfer, kv->value->data.bo.bytes, kv->value->data.bo.size); - rc = _store_data_for_rank(elem, rank, &xfer); + rc = _store_data_for_rank(ds_ctx, elem, rank, &xfer); PMIX_DESTRUCT(&xfer); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - goto err_exit; - } - - /* unset lock */ - if (PMIX_SUCCESS != (rc = _ESH_UNLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { - PMIX_ERROR_LOG(rc); + goto exit; } - return rc; -err_exit: - /* unset lock */ - if (PMIX_SUCCESS != (tmp_rc = _ESH_UNLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { - PMIX_ERROR_LOG(tmp_rc); - } +exit: return rc; } -static pmix_status_t dstore_store(const pmix_proc_t *proc, - pmix_scope_t scope, - pmix_kval_t *kv) +PMIX_EXPORT pmix_status_t pmix_common_dstor_store(pmix_common_dstore_ctx_t *ds_ctx, + const pmix_proc_t *proc, + pmix_scope_t scope, + pmix_kval_t *kv) { pmix_status_t rc = PMIX_SUCCESS; + ns_map_data_t *ns_map; + pmix_kval_t *kv2; + pmix_buffer_t tmp; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "[%s:%d] gds: dstore store for key '%s' scope %d", @@ -2387,26 +1883,51 @@ static pmix_status_t dstore_store(const pmix_proc_t *proc, PMIX_ERROR_LOG(rc); return rc; } - else { - pmix_kval_t *kv2; - kv2 = PMIX_NEW(pmix_kval_t); - PMIX_VALUE_CREATE(kv2->value, 1); - kv2->value->type = PMIX_BYTE_OBJECT; - pmix_buffer_t tmp; - PMIX_CONSTRUCT(&tmp, pmix_buffer_t); + kv2 = PMIX_NEW(pmix_kval_t); + PMIX_VALUE_CREATE(kv2->value, 1); + kv2->value->type = PMIX_BYTE_OBJECT; - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &tmp, kv, 1, PMIX_KVAL); - PMIX_UNLOAD_BUFFER(&tmp, kv2->value->data.bo.bytes, kv2->value->data.bo.size); + PMIX_CONSTRUCT(&tmp, pmix_buffer_t); + + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &tmp, kv, 1, PMIX_KVAL); + PMIX_UNLOAD_BUFFER(&tmp, kv2->value->data.bo.bytes, kv2->value->data.bo.size); + + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, proc->nspace))) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto exit; + } + + /* set exclusive lock */ + rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, wr_lock); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto exit; + } - rc = _dstore_store(proc->nspace, proc->rank, kv2); - PMIX_RELEASE(kv2); - PMIX_DESTRUCT(&tmp); + rc = _dstore_store_nolock(ds_ctx, ns_map, proc->rank, kv2); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto exit; } + + /* unset lock */ + rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, wr_unlock); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto exit; + } + +exit: + PMIX_RELEASE(kv2); + PMIX_DESTRUCT(&tmp); + return rc; } -static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, +static pmix_status_t _dstore_fetch(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, pmix_rank_t rank, const char *key, pmix_value_t **kvs) { ns_seg_info_t *ns_info = NULL; @@ -2414,7 +1935,7 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, ns_track_elem_t *elem; rank_meta_info *rinfo = NULL; size_t kval_cnt = 0; - seg_desc_t *meta_seg, *data_seg; + pmix_dstore_seg_desc_t *meta_seg, *data_seg; uint8_t *addr; pmix_buffer_t buffer; pmix_value_t val, *kval = NULL; @@ -2425,6 +1946,8 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, bool key_found = false; pmix_info_t *info = NULL; size_t ninfo; + size_t keyhash = 0; + bool lock_is_set = false; PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for %s:%u look for key %s", @@ -2434,34 +1957,38 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, PMIX_OUTPUT_VERBOSE((7, pmix_gds_base_framework.framework_output, "dstore: Does not support passed parameters")); rc = PMIX_ERR_BAD_PARAM; - PMIX_ERROR_LOG(rc); - return rc; + goto error; } PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for %s:%u look for key %s", __FILE__, __LINE__, __func__, nspace, rank, key)); - if (NULL == (ns_map = _esh_session_map_search(nspace))) { + /* protect info of dstore segments before it will be updated */ + if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (0 != (rc = pthread_mutex_lock(&ds_ctx->lock))) { + goto error; + } + lock_is_set = true; + } + + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, nspace))) { /* This call is issued from the the client. * client must have the session, otherwise the error is fatal. */ rc = PMIX_ERR_FATAL; - PMIX_ERROR_LOG(rc); - return rc; + goto error; } if (NULL == kvs) { rc = PMIX_ERR_FATAL; - PMIX_ERROR_LOG(rc); - return rc; + goto error; } if (PMIX_RANK_UNDEF == rank) { - ssize_t _nprocs = _get_univ_size(ns_map->name); + ssize_t _nprocs = _get_univ_size(ds_ctx, ns_map->name); if( 0 > _nprocs ){ - PMIX_ERROR_LOG(rc); - return rc; + goto error; } nprocs = (size_t) _nprocs; cur_rank = 0; @@ -2471,11 +1998,11 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, } /* grab shared lock */ - if (PMIX_SUCCESS != (lock_rc = _ESH_RDLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { + lock_rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, rd_lock); + if (PMIX_SUCCESS != lock_rc) { /* Something wrong with the lock. The error is fatal */ - rc = PMIX_ERR_FATAL; - PMIX_ERROR_LOG(lock_rc); - return lock_rc; + rc = lock_rc; + goto error; } /* First of all, we go through all initial segments and look at their field. @@ -2491,9 +2018,9 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, */ /* first update local information about initial segments. they can be extended, so then we need to attach to new segments. */ - _update_initial_segment_info(ns_map); + _update_initial_segment_info(ds_ctx, ns_map); - ns_info = _get_ns_info_from_initial_segment(ns_map); + ns_info = _get_ns_info_from_initial_segment(ds_ctx, ns_map); if (NULL == ns_info) { /* no data for this namespace is found in the shared memory. */ PMIX_OUTPUT_VERBOSE((7, pmix_gds_base_framework.framework_output, @@ -2504,7 +2031,7 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, } /* get ns_track_elem_t object for the target namespace from the local track list. */ - elem = _get_track_elem_for_namespace(ns_map); + elem = _get_track_elem_for_namespace(ds_ctx, ns_map); if (NULL == elem) { /* Shouldn't happen! */ rc = PMIX_ERR_FATAL; @@ -2515,7 +2042,7 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, /* need to update tracker: * attach to shared memory regions for this namespace and store its info locally * to operate with address and detach/unlink afterwards. */ - rc = _update_ns_elem(elem, ns_info); + rc = _update_ns_elem(ds_ctx, elem, ns_info); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto done; @@ -2525,9 +2052,21 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, meta_seg = elem->meta_seg; data_seg = elem->data_seg; + if( NULL != key ) { + keyhash = PMIX_DS_KEY_HASH(ds_ctx, key); + } + + /* all segment data updated, ctx lock may released */ + if (lock_is_set) { + lock_is_set = false; + if (0 != (rc = pthread_mutex_unlock(&ds_ctx->lock))) { + goto error; + } + } + while (nprocs--) { /* Get the rank meta info in the shared meta segment. */ - rinfo = _get_rank_meta_info(cur_rank, meta_seg); + rinfo = _get_rank_meta_info(ds_ctx, cur_rank, meta_seg); if (NULL == rinfo) { PMIX_OUTPUT_VERBOSE((7, pmix_gds_base_framework.framework_output, "%s:%d:%s: no data for this rank is found in the shared memory. rank %u", @@ -2535,7 +2074,7 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, all_ranks_found = false; continue; } - addr = _get_data_region_by_offset(data_seg, rinfo->offset); + addr = _get_data_region_by_offset(ds_ctx, data_seg, rinfo->offset); if (NULL == addr) { /* This means that meta-info is broken - error is fatal */ rc = PMIX_ERR_FATAL; @@ -2587,22 +2126,22 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, * EXTENSION slot which has key = EXTENSION_SLOT and a size_t value for offset * to next data address for this process. */ - if (0 == strncmp(ESH_KNAME_PTR(addr), ESH_REGION_INVALIDATED, ESH_KNAME_LEN(ESH_REGION_INVALIDATED))) { + if (PMIX_DS_KEY_IS_INVALID(ds_ctx, addr)) { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %s:%u, skip %s region", __FILE__, __LINE__, __func__, nspace, cur_rank, ESH_REGION_INVALIDATED)); /* skip it * go to next item, updating address */ - addr += ESH_KV_SIZE(addr); - } else if (0 == strncmp(ESH_KNAME_PTR(addr), ESH_REGION_EXTENSION, ESH_KNAME_LEN(ESH_REGION_EXTENSION))) { + addr += PMIX_DS_KV_SIZE(ds_ctx, addr); + } else if (PMIX_DS_KEY_IS_EXTSLOT(ds_ctx, addr)) { size_t offset; - memcpy(&offset, ESH_DATA_PTR(addr), sizeof(size_t)); + memcpy(&offset, PMIX_DS_DATA_PTR(ds_ctx, addr), sizeof(size_t)); PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %s:%u, reached %s with %lu value", __FILE__, __LINE__, __func__, nspace, cur_rank, ESH_REGION_EXTENSION, offset)); if (0 < offset) { /* go to next item, updating address */ - addr = _get_data_region_by_offset(data_seg, offset); + addr = _get_data_region_by_offset(ds_ctx, data_seg, offset); if (NULL == addr) { /* This shouldn't happen - error is fatal */ rc = PMIX_ERR_FATAL; @@ -2619,21 +2158,22 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, } else if (NULL == key) { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %s:%u, found target key %s", - __FILE__, __LINE__, __func__, nspace, cur_rank, ESH_KNAME_PTR(addr))); + __FILE__, __LINE__, __func__, nspace, cur_rank, PMIX_DS_KNAME_PTR(ds_ctx, addr))); - uint8_t *data_ptr = ESH_DATA_PTR(addr); - size_t data_size = ESH_DATA_SIZE(addr, data_ptr); + uint8_t *data_ptr = PMIX_DS_DATA_PTR(ds_ctx, addr); + size_t data_size = PMIX_DS_DATA_SIZE(ds_ctx, addr, data_ptr); PMIX_CONSTRUCT(&buffer, pmix_buffer_t); - PMIX_LOAD_BUFFER(_client_peer(), &buffer, data_ptr, data_size); + PMIX_LOAD_BUFFER(_client_peer(ds_ctx), &buffer, data_ptr, data_size); int cnt = 1; /* unpack value for this key from the buffer. */ PMIX_VALUE_CONSTRUCT(&val); - PMIX_BFROPS_UNPACK(rc, _client_peer(), &buffer, &val, &cnt, PMIX_VALUE); + PMIX_BFROPS_UNPACK(rc, _client_peer(ds_ctx), &buffer, &val, &cnt, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto done; } - strncpy(info[kval_cnt - 1].key, ESH_KNAME_PTR(addr), ESH_KNAME_LEN((char *)addr)); + pmix_strncpy(info[kval_cnt - 1].key, PMIX_DS_KNAME_PTR(ds_ctx, addr), + PMIX_DS_KNAME_LEN(ds_ctx, addr)); pmix_value_xfer(&info[kval_cnt - 1].value, &val); PMIX_VALUE_DESTRUCT(&val); buffer.base_ptr = NULL; @@ -2642,30 +2182,24 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, key_found = true; kval_cnt--; - addr += ESH_KV_SIZE(addr); - } else if (0 == strncmp(ESH_KNAME_PTR(addr), key, ESH_KNAME_LEN(key))) { + addr += PMIX_DS_KV_SIZE(ds_ctx, addr); + } else if (PMIX_DS_KEY_MATCH(ds_ctx, addr, key, keyhash)) { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %s:%u, found target key %s", __FILE__, __LINE__, __func__, nspace, cur_rank, key)); /* target key is found, get value */ - uint8_t *data_ptr = ESH_DATA_PTR(addr); - size_t data_size = ESH_DATA_SIZE(addr, data_ptr); + uint8_t *data_ptr = PMIX_DS_DATA_PTR(ds_ctx, addr); + size_t data_size = PMIX_DS_DATA_SIZE(ds_ctx, addr, data_ptr); PMIX_CONSTRUCT(&buffer, pmix_buffer_t); - PMIX_LOAD_BUFFER(_client_peer(), &buffer, data_ptr, data_size); + PMIX_LOAD_BUFFER(_client_peer(ds_ctx), &buffer, data_ptr, data_size); int cnt = 1; /* unpack value for this key from the buffer. */ - PMIX_VALUE_CONSTRUCT(&val); - PMIX_BFROPS_UNPACK(rc, _client_peer(), &buffer, &val, &cnt, PMIX_VALUE); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - goto done; - } - PMIX_BFROPS_COPY(rc, _client_peer(), (void**)kvs, &val, PMIX_VALUE); + *kvs = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + PMIX_BFROPS_UNPACK(rc, _client_peer(ds_ctx), &buffer, (void*)*kvs, &cnt, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto done; } - PMIX_VALUE_DESTRUCT(&val); buffer.base_ptr = NULL; buffer.bytes_used = 0; PMIX_DESTRUCT(&buffer); @@ -2674,9 +2208,10 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, } else { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %s:%u, skip key %s look for key %s", - __FILE__, __LINE__, __func__, nspace, cur_rank, ESH_KNAME_PTR(addr), key)); + __FILE__, __LINE__, __func__, nspace, cur_rank, + PMIX_DS_KNAME_PTR(ds_ctx, addr), key)); /* go to next item, updating address */ - addr += ESH_KV_SIZE(addr); + addr += PMIX_DS_KV_SIZE(ds_ctx, addr); kval_cnt--; } } @@ -2688,10 +2223,16 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, done: /* unset lock */ - if (PMIX_SUCCESS != (lock_rc = _ESH_UNLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { + lock_rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, rd_unlock); + if (PMIX_SUCCESS != lock_rc) { PMIX_ERROR_LOG(lock_rc); } + /* unset ds_ctx lock */ + if (lock_is_set) { + pthread_mutex_unlock(&ds_ctx->lock); + } + if( rc != PMIX_SUCCESS ){ if ((NULL == key) && (kval_cnt > 0)) { if( NULL != info ) { @@ -2718,13 +2259,21 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, } rc = PMIX_ERR_NOT_FOUND; return rc; + +error: + if (lock_is_set) { + pthread_mutex_unlock(&ds_ctx->lock); + } + PMIX_ERROR_LOG(rc); + return rc; } -static pmix_status_t dstore_fetch(const pmix_proc_t *proc, - pmix_scope_t scope, bool copy, - const char *key, - pmix_info_t info[], size_t ninfo, - pmix_list_t *kvs) +PMIX_EXPORT pmix_status_t pmix_common_dstor_fetch(pmix_common_dstore_ctx_t *ds_ctx, + const pmix_proc_t *proc, + pmix_scope_t scope, bool copy, + const char *key, + pmix_info_t info[], size_t ninfo, + pmix_list_t *kvs) { pmix_kval_t *kv; pmix_value_t *val; @@ -2733,7 +2282,7 @@ static pmix_status_t dstore_fetch(const pmix_proc_t *proc, pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "gds: dstore fetch `%s`", key == NULL ? "NULL" : key); - rc = _dstore_fetch(proc->nspace, proc->rank, key, &val); + rc = _dstore_fetch(ds_ctx, proc->nspace, proc->rank, key, &val); if (PMIX_SUCCESS == rc) { if( NULL == key ) { pmix_info_t *info; @@ -2781,7 +2330,8 @@ static pmix_status_t dstore_fetch(const pmix_proc_t *proc, return rc; } -static pmix_status_t dstore_setup_fork(const pmix_proc_t *peer, char ***env) +PMIX_EXPORT pmix_status_t pmix_common_dstor_setup_fork(pmix_common_dstore_ctx_t *ds_ctx, const char *base_path_env, + const pmix_proc_t *peer, char ***env) { pmix_status_t rc = PMIX_SUCCESS; ns_map_data_t *ns_map = NULL; @@ -2789,41 +2339,43 @@ static pmix_status_t dstore_setup_fork(const pmix_proc_t *peer, char ***env) pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "gds: dstore setup fork"); - if (NULL == _esh_session_map_search) { + if (NULL == ds_ctx->session_map_search) { rc = PMIX_ERR_NOT_AVAILABLE; PMIX_ERROR_LOG(rc); return rc; } - if (NULL == (ns_map = _esh_session_map_search(peer->nspace))) { + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, peer->nspace))) { rc = PMIX_ERR_NOT_AVAILABLE; PMIX_ERROR_LOG(rc); return rc; } - if ((NULL == _base_path) || (strlen(_base_path) == 0)){ + if ((NULL == ds_ctx->base_path) || (strlen(ds_ctx->base_path) == 0)){ rc = PMIX_ERR_NOT_AVAILABLE; PMIX_ERROR_LOG(rc); return rc; } - if(PMIX_SUCCESS != (rc = pmix_setenv(PMIX_DSTORE_ESH_BASE_PATH, - _ESH_SESSION_path(ns_map->tbl_idx), true, env))){ + if(PMIX_SUCCESS != (rc = pmix_setenv(base_path_env, + _ESH_SESSION_path(ds_ctx->session_array, ns_map->tbl_idx), + true, env))){ PMIX_ERROR_LOG(rc); } + return rc; } -static pmix_status_t dstore_add_nspace(const char *nspace, - pmix_info_t info[], - size_t ninfo) +PMIX_EXPORT pmix_status_t pmix_common_dstor_add_nspace(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, pmix_info_t info[], size_t ninfo) { - pmix_status_t rc; + pmix_status_t rc = PMIX_SUCCESS; size_t tbl_idx=0; - uid_t jobuid = _jobuid; - char setjobuid = _setjobuid; + uid_t jobuid = ds_ctx->jobuid; + char setjobuid = ds_ctx->setjobuid; size_t n; ns_map_data_t *ns_map = NULL; + uint32_t local_size = 0; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "gds: dstore add nspace"); @@ -2835,31 +2387,36 @@ static pmix_status_t dstore_add_nspace(const char *nspace, setjobuid = 1; continue; } + if (0 == strcmp(PMIX_LOCAL_SIZE, info[n].key)) { + local_size = info[n].value.data.uint32; + continue; + } } } - if (PMIX_SUCCESS != _esh_jobuid_tbl_search(jobuid, &tbl_idx)) { + if (PMIX_SUCCESS != _esh_jobuid_tbl_search(ds_ctx, jobuid, &tbl_idx)) { - rc = _esh_session_tbl_add(&tbl_idx); + rc = _esh_session_tbl_add(ds_ctx, &tbl_idx); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } - ns_map = _esh_session_map(nspace, tbl_idx); + ns_map = _esh_session_map(ds_ctx, nspace, local_size, tbl_idx); if (NULL == ns_map) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); return rc; } - if (PMIX_SUCCESS != (rc =_esh_session_init(tbl_idx, ns_map, jobuid, setjobuid))) { + if (PMIX_SUCCESS != (rc =_esh_session_init(ds_ctx, tbl_idx, ns_map, + local_size, jobuid, setjobuid))) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); return rc; } } else { - ns_map = _esh_session_map(nspace, tbl_idx); + ns_map = _esh_session_map(ds_ctx, nspace, local_size, tbl_idx); if (NULL == ns_map) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); @@ -2867,10 +2424,19 @@ static pmix_status_t dstore_add_nspace(const char *nspace, } } + /* lock init */ + ds_ctx->lock_cbs->init(&_ESH_SESSION_lock(ds_ctx->session_array, tbl_idx), + ds_ctx->base_path, nspace, local_size, ds_ctx->jobuid, + ds_ctx->setjobuid); + if (NULL == _ESH_SESSION_lock(ds_ctx->session_array, tbl_idx)) { + PMIX_ERROR_LOG(rc); + return rc; + } + return PMIX_SUCCESS; } -static pmix_status_t dstore_del_nspace(const char* nspace) +PMIX_EXPORT pmix_status_t pmix_common_dstor_del_nspace(pmix_common_dstore_ctx_t *ds_ctx, const char* nspace) { pmix_status_t rc = PMIX_SUCCESS; size_t map_idx, size; @@ -2885,20 +2451,20 @@ static pmix_status_t dstore_del_nspace(const char* nspace) PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s delete nspace `%s`", __FILE__, __LINE__, __func__, nspace)); - if (NULL == (ns_map_data = _esh_session_map_search(nspace))) { + if (NULL == (ns_map_data = ds_ctx->session_map_search(ds_ctx, nspace))) { rc = PMIX_ERR_NOT_AVAILABLE; return rc; } dstor_track_idx = ns_map_data->track_idx; session_tbl_idx = ns_map_data->tbl_idx; - size = pmix_value_array_get_size(_ns_map_array); - ns_map = PMIX_VALUE_ARRAY_GET_BASE(_ns_map_array, ns_map_t); + size = pmix_value_array_get_size(ds_ctx->ns_map_array); + ns_map = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->ns_map_array, ns_map_t); for (map_idx = 0; map_idx < size; map_idx++){ if (ns_map[map_idx].in_use && (ns_map[map_idx].data.tbl_idx == ns_map_data->tbl_idx)) { if (0 == strcmp(ns_map[map_idx].data.name, nspace)) { - _esh_session_map_clean(&ns_map[map_idx]); + _esh_session_map_clean(ds_ctx, &ns_map[map_idx]); continue; } in_use++; @@ -2908,68 +2474,29 @@ static pmix_status_t dstore_del_nspace(const char* nspace) /* A lot of nspaces may be using same session info * session record can only be deleted once all references are gone */ if (!in_use) { - session_tbl = PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t); + session_tbl = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->session_array, session_t); PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s delete session for jobuid: %d", __FILE__, __LINE__, __func__, session_tbl[session_tbl_idx].jobuid)); - size = pmix_value_array_get_size(_ns_track_array); + size = pmix_value_array_get_size(ds_ctx->ns_track_array); if (size && (dstor_track_idx >= 0)) { - if((dstor_track_idx + 1) > size) { + if((dstor_track_idx + 1) > (int)size) { rc = PMIX_ERR_VALUE_OUT_OF_BOUNDS; PMIX_ERROR_LOG(rc); goto exit; } - trk = pmix_value_array_get_item(_ns_track_array, dstor_track_idx); + trk = pmix_value_array_get_item(ds_ctx->ns_track_array, dstor_track_idx); if (true == trk->in_use) { PMIX_DESTRUCT(trk); + pmix_value_array_remove_item(ds_ctx->ns_track_array, dstor_track_idx); } } - _esh_session_release(&session_tbl[session_tbl_idx]); + _esh_session_release(ds_ctx, session_tbl_idx); } exit: return rc; } -static pmix_status_t dstore_assign_module(pmix_info_t *info, size_t ninfo, - int *priority) -{ - size_t n, m; - char **options; - - *priority = 20; - if (NULL != info) { - for (n=0; n < ninfo; n++) { - if (0 == strncmp(info[n].key, PMIX_GDS_MODULE, PMIX_MAX_KEYLEN)) { - options = pmix_argv_split(info[n].value.data.string, ','); - for (m=0; NULL != options[m]; m++) { - if (0 == strcmp(options[m], "ds12")) { - /* they specifically asked for us */ - *priority = 100; - break; - } - if (0 == strcmp(options[m], "dstore")) { - /* they are asking for any dstore module - we - * take an intermediate priority in case another - * dstore is more modern than us */ - *priority = 50; - break; - } - } - pmix_argv_free(options); - break; - } - } - } - -#if 0 - if PMIX_GDS_MODULE != "ds12" - *proirity = 0; - else PMIX_GDS_MODULE == "ds12" || !PMIX_GDS_MODULE - *priority = -1; -#endif - return PMIX_SUCCESS; -} - static inline int _my_client(const char *nspace, pmix_rank_t rank) { pmix_peer_t *peer; @@ -2992,16 +2519,59 @@ static inline int _my_client(const char *nspace, pmix_rank_t rank) * host has received data from some other peer. It therefore * always contains data solely from remote procs, and we * shall store it accordingly */ -static pmix_status_t dstore_store_modex(struct pmix_nspace_t *nspace, - pmix_list_t *cbs, - pmix_byte_object_t *bo) +PMIX_EXPORT pmix_status_t pmix_common_dstor_store_modex(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_buffer_t *buf) +{ + pmix_status_t rc = PMIX_SUCCESS; + pmix_status_t rc1 = PMIX_SUCCESS; + pmix_namespace_t *ns = (pmix_namespace_t*)nspace; + ns_map_data_t *ns_map; + + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, ns->nspace))) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + return rc; + } + + /* set exclusive lock */ + rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, wr_lock); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } + + rc = pmix_gds_base_store_modex(nspace, cbs, buf, (pmix_gds_base_store_modex_cb_fn_t)_dstor_store_modex_cb, ds_ctx); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + + /* unset lock */ + rc1 = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, wr_unlock); + if (PMIX_SUCCESS != rc1) { + PMIX_ERROR_LOG(rc1); + if (PMIX_SUCCESS == rc) { + rc = rc1; + } + } + + return rc; +} + +static pmix_status_t _dstor_store_modex_cb(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_byte_object_t *bo) { - pmix_nspace_t *ns = (pmix_nspace_t*)nspace; + pmix_namespace_t *ns = (pmix_namespace_t*)nspace; pmix_status_t rc = PMIX_SUCCESS; int32_t cnt; pmix_buffer_t pbkt; pmix_proc_t proc; pmix_kval_t *kv; + ns_map_data_t *ns_map; + pmix_buffer_t tmp; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "[%s:%d] gds:dstore:store_modex for nspace %s", @@ -3042,6 +2612,10 @@ static pmix_status_t dstore_store_modex(struct pmix_nspace_t *nspace, PMIX_DESTRUCT(&pbkt); return PMIX_SUCCESS; } + + /* Prepare a buffer to be provided to the dstor store primitive */ + PMIX_CONSTRUCT(&tmp, pmix_buffer_t); + /* unpack the remaining values until we hit the end of the buffer */ cnt = 1; kv = PMIX_NEW(pmix_kval_t); @@ -3057,29 +2631,68 @@ static pmix_status_t dstore_store_modex(struct pmix_nspace_t *nspace, PMIX_DESTRUCT(&pbkt); return rc; } - if (PMIX_SUCCESS != (rc = dstore_store(&proc, PMIX_REMOTE, kv))) { - PMIX_ERROR_LOG(rc); - } - PMIX_RELEASE(kv); // maintain accounting as the hash increments the ref count - /* continue along */ + + /* place the key to the to be provided to _dstore_store_nolock */ + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &tmp, kv, 1, PMIX_KVAL); + + /* Release the kv to maintain accounting + * as the hash increments the ref count */ + PMIX_RELEASE(kv); + + /* proceed to the next element */ kv = PMIX_NEW(pmix_kval_t); cnt = 1; PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, &pbkt, kv, &cnt, PMIX_KVAL); } - PMIX_RELEASE(kv); // maintain accounting + + /* Release the kv that didn't received the value + * because input buffer was exhausted */ + PMIX_RELEASE(kv); if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) { PMIX_ERROR_LOG(rc); } else { rc = PMIX_SUCCESS; } + + /* Create a key-value pair with the buffer + * to be passed to _dstore_store_nolock */ + kv = PMIX_NEW(pmix_kval_t); + PMIX_VALUE_CREATE(kv->value, 1); + kv->value->type = PMIX_BYTE_OBJECT; + PMIX_UNLOAD_BUFFER(&tmp, kv->value->data.bo.bytes, kv->value->data.bo.size); + + /* Get the namespace map element for the process "proc" */ + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, proc.nspace))) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + bo->bytes = pbkt.base_ptr; + bo->size = pbkt.bytes_used; // restore the incoming data + pbkt.base_ptr = NULL; + PMIX_DESTRUCT(&pbkt); + return rc; + } + + /* Store all keys at once */ + rc = _dstore_store_nolock(ds_ctx, ns_map, proc.rank, kv); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + + /* Release all resources */ + PMIX_RELEASE(kv); + PMIX_DESTRUCT(&tmp); + + /* Reset the input buffer */ bo->bytes = pbkt.base_ptr; - bo->size = pbkt.bytes_used; // restore the incoming data + bo->size = pbkt.bytes_used; pbkt.base_ptr = NULL; PMIX_DESTRUCT(&pbkt); + return rc; } -static pmix_status_t _store_job_info(pmix_proc_t *proc) +static pmix_status_t _store_job_info(pmix_common_dstore_ctx_t *ds_ctx, ns_map_data_t *ns_map, + pmix_proc_t *proc) { pmix_cb_t cb; pmix_kval_t *kv; @@ -3107,7 +2720,7 @@ static pmix_status_t _store_job_info(pmix_proc_t *proc) } PMIX_LIST_FOREACH(kv, &cb.kvs, pmix_kval_t) { - if ((PMIX_PROC_IS_V1(_client_peer()) || PMIX_PROC_IS_V20(_client_peer())) && + if ((PMIX_PROC_IS_V1(_client_peer(ds_ctx)) || PMIX_PROC_IS_V20(_client_peer(ds_ctx))) && 0 != strncmp("pmix.", kv->key, 4) && kv->value->type == PMIX_DATA_ARRAY) { pmix_info_t *info; @@ -3144,7 +2757,7 @@ static pmix_status_t _store_job_info(pmix_proc_t *proc) } PMIX_UNLOAD_BUFFER(&buf, kvp->value->data.bo.bytes, kvp->value->data.bo.size); - if (PMIX_SUCCESS != (rc = _dstore_store(proc->nspace, proc->rank, kvp))) { + if (PMIX_SUCCESS != (rc = _dstore_store_nolock(ds_ctx, ns_map, proc->rank, kvp))) { PMIX_ERROR_LOG(rc); goto exit; } @@ -3156,11 +2769,12 @@ static pmix_status_t _store_job_info(pmix_proc_t *proc) return rc; } -static pmix_status_t dstore_register_job_info(struct pmix_peer_t *pr, - pmix_buffer_t *reply) +PMIX_EXPORT pmix_status_t pmix_common_dstor_register_job_info(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_peer_t *pr, + pmix_buffer_t *reply) { pmix_peer_t *peer = (pmix_peer_t*)pr; - pmix_nspace_t *ns = peer->nptr; + pmix_namespace_t *ns = peer->nptr; char *msg; pmix_status_t rc; pmix_proc_t proc; @@ -3172,10 +2786,25 @@ static pmix_status_t dstore_register_job_info(struct pmix_peer_t *pr, peer->info->pname.nspace, peer->info->pname.rank); if (0 == ns->ndelivered) { // don't store twice - _client_compat_save(peer); - (void)strncpy(proc.nspace, ns->nspace, PMIX_MAX_NSLEN); + ns_map_data_t *ns_map; + + _client_compat_save(ds_ctx, peer); + pmix_strncpy(proc.nspace, ns->nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; - rc = _store_job_info(&proc); + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, proc.nspace))) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + return rc; + } + + /* set exclusive lock */ + rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, wr_lock); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } + + rc = _store_job_info(ds_ctx, ns_map, &proc); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; @@ -3183,12 +2812,18 @@ static pmix_status_t dstore_register_job_info(struct pmix_peer_t *pr, for (rank=0; rank < ns->nprocs; rank++) { proc.rank = rank; - rc = _store_job_info(&proc); + rc = _store_job_info(ds_ctx, ns_map, &proc); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } } + /* unset lock */ + rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, wr_unlock); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } } /* answer to client */ @@ -3202,7 +2837,9 @@ static pmix_status_t dstore_register_job_info(struct pmix_peer_t *pr, return rc; } -static pmix_status_t dstore_store_job_info(const char *nspace, pmix_buffer_t *buf) +PMIX_EXPORT pmix_status_t pmix_common_dstor_store_job_info(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, + pmix_buffer_t *job_data) { pmix_status_t rc = PMIX_SUCCESS; @@ -3211,7 +2848,7 @@ static pmix_status_t dstore_store_job_info(const char *nspace, pmix_buffer_t *b pmix_globals.myid.nspace, pmix_globals.myid.rank, nspace); /* check buf data */ - if ((NULL == buf) || (0 == buf->bytes_used)) { + if ((NULL == job_data) || (0 == job_data->bytes_used)) { rc = PMIX_ERR_BAD_PARAM; PMIX_ERROR_LOG(rc); return rc; @@ -3219,23 +2856,23 @@ static pmix_status_t dstore_store_job_info(const char *nspace, pmix_buffer_t *b return rc; } -static void _client_compat_save(pmix_peer_t *peer) +static void _client_compat_save(pmix_common_dstore_ctx_t *ds_ctx, pmix_peer_t *peer) { - pmix_nspace_t *nptr = NULL; + pmix_namespace_t *nptr = NULL; - if (NULL == _clients_peer) { - _clients_peer = PMIX_NEW(pmix_peer_t); - nptr = PMIX_NEW(pmix_nspace_t); - _clients_peer->nptr = nptr; + if (NULL == ds_ctx->clients_peer) { + ds_ctx->clients_peer = PMIX_NEW(pmix_peer_t); + nptr = PMIX_NEW(pmix_namespace_t); + ds_ctx->clients_peer->nptr = nptr; } - _clients_peer->nptr->compat = peer->nptr->compat; - _clients_peer->proc_type = peer->proc_type; + ds_ctx->clients_peer->nptr->compat = peer->nptr->compat; + ds_ctx->clients_peer->proc_type = peer->proc_type; } -static inline pmix_peer_t * _client_peer(void) +static inline pmix_peer_t * _client_peer(pmix_common_dstore_ctx_t *ds_ctx) { - if (NULL == _clients_peer) { + if (NULL == ds_ctx->clients_peer) { return pmix_globals.mypeer; } - return _clients_peer; + return ds_ctx->clients_peer; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_base.h b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_base.h new file mode 100644 index 00000000000..719980bcc60 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_base.h @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2017 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_DSTORE_H +#define PMIX_DSTORE_H + +#include + + +#include "src/mca/gds/gds.h" +#include "src/mca/pshmem/pshmem.h" + +BEGIN_C_DECLS + +#include +#include "src/class/pmix_value_array.h" +#include "dstore_common.h" +#include "dstore_segment.h" +#include "dstore_file.h" + +#define INITIAL_SEG_SIZE 4096 +#define NS_META_SEG_SIZE (1<<22) +#define NS_DATA_SEG_SIZE (1<<22) + +#define PMIX_DSTORE_ESH_BASE_PATH "PMIX_DSTORE_ESH_BASE_PATH" +#define PMIX_DSTORE_VER_BASE_PATH_FMT "PMIX_DSTORE_%d_BASE_PATH" + +typedef struct ns_map_data_s ns_map_data_t; +typedef struct session_s session_t; +typedef struct ns_map_s ns_map_t; + +typedef ns_map_data_t * (*session_map_search_fn_t)(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace); + +struct pmix_common_dstore_ctx_s { + char *ds_name; + char *base_path; + uid_t jobuid; + char setjobuid; + + pmix_value_array_t *session_array; + pmix_value_array_t *ns_map_array; + pmix_value_array_t *ns_track_array; + + pmix_common_lock_callbacks_t *lock_cbs; + pmix_common_dstore_file_cbs_t *file_cbs; + + size_t initial_segment_size; + size_t meta_segment_size; + size_t data_segment_size; + size_t lock_segment_size; + + size_t max_ns_num; + size_t max_meta_elems; + + session_map_search_fn_t session_map_search; + pmix_peer_t *clients_peer; + /* If _direct_mode is set, it means that we use linear search + * along the array of rank meta info objects inside a meta segment + * to find the requested rank. Otherwise, we do a fast lookup + * based on rank and directly compute offset. + * This mode is called direct because it's effectively used in + * sparse communication patterns when direct modex is usually used. + */ + int direct_mode; + /* dstore ctx protect lock, uses for clients only */ + pthread_mutex_t lock; +}; + +struct session_s { + int in_use; + uid_t jobuid; + char setjobuid; + char *nspace_path; + pmix_dstore_seg_desc_t *sm_seg_first; + pmix_dstore_seg_desc_t *sm_seg_last; + pmix_common_dstor_lock_ctx_t lock; +}; + +struct ns_map_data_s { + char name[PMIX_MAX_NSLEN+1]; + size_t tbl_idx; + int track_idx; +}; + +struct ns_map_s { + int in_use; + ns_map_data_t data; +}; + +/* initial segment format: + * size_t num_elems; + * size_t full; //indicate to client that it needs to attach to the next segment + * ns_seg_info_t ns_seg_info[max_ns_num]; + */ + +typedef struct { + ns_map_data_t ns_map; + size_t num_meta_seg;/* read by clients to attach to this number of segments. */ + size_t num_data_seg; +} ns_seg_info_t; + +/* meta segment format: + * size_t num_elems; + * rank_meta_info meta_info[max_meta_elems]; + */ + +typedef struct { + size_t rank; + size_t offset; + size_t count; +} rank_meta_info; + +typedef struct { + pmix_value_array_t super; + ns_map_data_t ns_map; + size_t num_meta_seg; + size_t num_data_seg; + pmix_dstore_seg_desc_t *meta_seg; + pmix_dstore_seg_desc_t *data_seg; + bool in_use; +} ns_track_elem_t; + +typedef struct { + pmix_list_item_t super; + pmix_common_dstor_lock_ctx_t *lock; +} lock_track_item_t; + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_common.h b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_common.h new file mode 100644 index 00000000000..5edf2821583 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_common.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_GDS_DS_BASE_H_ +#define PMIX_GDS_DS_BASE_H_ + +#include +#include +#include + +#include "src/include/pmix_globals.h" +#include "src/class/pmix_list.h" +#include "src/mca/gds/gds.h" +#include "src/mca/pshmem/pshmem.h" +#include "src/mca/common/dstore/dstore_file.h" + +typedef void* pmix_common_dstor_lock_ctx_t; + +typedef pmix_status_t (*pmix_common_dstor_lock_init_fn_t)(pmix_common_dstor_lock_ctx_t *ctx, + const char *base_path, const char *name, + uint32_t local_size, uid_t uid, bool setuid); +typedef void (*pmix_common_dstor_lock_finalize_fn_t)(pmix_common_dstor_lock_ctx_t *ctx); +typedef pmix_status_t (*pmix_common_dstor_lock_rd_get_fn_t)(pmix_common_dstor_lock_ctx_t ctx); +typedef pmix_status_t (*pmix_common_dstor_lock_rd_rel_fn_t)(pmix_common_dstor_lock_ctx_t ctx); +typedef pmix_status_t (*pmix_common_dstor_lock_wr_get_fn_t)(pmix_common_dstor_lock_ctx_t ctx); +typedef pmix_status_t (*pmix_common_dstor_lock_wr_rel_fn_t)(pmix_common_dstor_lock_ctx_t ctx); + +typedef struct { + pmix_common_dstor_lock_init_fn_t init; + pmix_common_dstor_lock_finalize_fn_t finalize; + pmix_common_dstor_lock_rd_get_fn_t rd_lock; + pmix_common_dstor_lock_rd_rel_fn_t rd_unlock; + pmix_common_dstor_lock_wr_get_fn_t wr_lock; + pmix_common_dstor_lock_wr_rel_fn_t wr_unlock; +} pmix_common_lock_callbacks_t; + +typedef struct pmix_common_dstore_ctx_s pmix_common_dstore_ctx_t; + +PMIX_EXPORT pmix_common_dstore_ctx_t *pmix_common_dstor_init(const char *ds_name, pmix_info_t info[], size_t ninfo, + pmix_common_lock_callbacks_t *lock_cb, + pmix_common_dstore_file_cbs_t *file_cb); +PMIX_EXPORT void pmix_common_dstor_finalize(pmix_common_dstore_ctx_t *ds_ctx); +PMIX_EXPORT pmix_status_t pmix_common_dstor_add_nspace(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, pmix_info_t info[], size_t ninfo); +PMIX_EXPORT pmix_status_t pmix_common_dstor_del_nspace(pmix_common_dstore_ctx_t *ds_ctx, const char* nspace); +PMIX_EXPORT pmix_status_t pmix_common_dstor_setup_fork(pmix_common_dstore_ctx_t *ds_ctx, const char *base_path_env, + const pmix_proc_t *peer, char ***env); +PMIX_EXPORT pmix_status_t pmix_common_dstor_cache_job_info(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_namespace_t *ns, + pmix_info_t info[], size_t ninfo); +PMIX_EXPORT pmix_status_t pmix_common_dstor_register_job_info(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_peer_t *pr, + pmix_buffer_t *reply); +PMIX_EXPORT pmix_status_t pmix_common_dstor_store_job_info(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, + pmix_buffer_t *job_data); +PMIX_EXPORT pmix_status_t pmix_common_dstor_store(pmix_common_dstore_ctx_t *ds_ctx, + const pmix_proc_t *proc, + pmix_scope_t scope, + pmix_kval_t *kv); +PMIX_EXPORT pmix_status_t pmix_common_dstor_fetch(pmix_common_dstore_ctx_t *ds_ctx, + const pmix_proc_t *proc, + pmix_scope_t scope, bool copy, + const char *key, + pmix_info_t info[], size_t ninfo, + pmix_list_t *kvs); +PMIX_EXPORT pmix_status_t pmix_common_dstor_store_modex(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_buffer_t *buff); +#endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_file.h b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_file.h new file mode 100644 index 00000000000..576149b0569 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_file.h @@ -0,0 +1,161 @@ +#ifndef DSTORE_FORMAT_H +#define DSTORE_FORMAT_H + +typedef size_t (*pmix_common_dstore_kv_size_fn)(uint8_t *addr); +typedef char* (*pmix_common_dstore_key_name_ptr_fn)(uint8_t *addr); +typedef size_t (*pmix_common_dstore_key_name_len_fn)(char *key); +typedef uint8_t* (*pmix_common_dstore_data_ptr_fn)(uint8_t *addr); +typedef size_t (*pmix_common_dstore_data_size_fn)(uint8_t *addr, uint8_t* data_ptr); +typedef size_t (*pmix_common_dstore_key_size_fn)(char *key, size_t data_size); +typedef size_t (*pmix_common_dstore_ext_slot_size_fn)(void); +typedef int (*pmix_common_dstore_put_key_fn)(uint8_t *addr, char *key, void *buf, + size_t size); +typedef bool (*pmix_common_dstore_is_invalid_fn)(uint8_t *addr); +typedef bool (*pmix_common_dstore_is_extslot_fn)(uint8_t *addr); +typedef void (*pmix_common_dstore_set_invalid_fn)(uint8_t *addr); +typedef size_t (*pmix_common_dstore_key_hash_fn)(const char *key); +typedef bool (*pmix_common_dstore_key_match_fn)(uint8_t *addr, const char *key, + size_t key_hash); + +typedef struct { + const char *name; + pmix_common_dstore_kv_size_fn kval_size; + pmix_common_dstore_key_name_ptr_fn kname_ptr; + pmix_common_dstore_key_name_len_fn kname_len; + pmix_common_dstore_data_ptr_fn data_ptr; + pmix_common_dstore_data_size_fn data_size; + pmix_common_dstore_key_size_fn key_size; + pmix_common_dstore_ext_slot_size_fn ext_slot_size; + pmix_common_dstore_put_key_fn put_key; + pmix_common_dstore_is_invalid_fn is_invalid; + pmix_common_dstore_is_extslot_fn is_extslot; + pmix_common_dstore_set_invalid_fn set_invalid; + pmix_common_dstore_key_hash_fn key_hash; + pmix_common_dstore_key_match_fn key_match; +} pmix_common_dstore_file_cbs_t; + +#define ESH_REGION_EXTENSION "EXTENSION_SLOT" +#define ESH_REGION_INVALIDATED "INVALIDATED" +#define ESH_ENV_INITIAL_SEG_SIZE "INITIAL_SEG_SIZE" +#define ESH_ENV_NS_META_SEG_SIZE "NS_META_SEG_SIZE" +#define ESH_ENV_NS_DATA_SEG_SIZE "NS_DATA_SEG_SIZE" +#define ESH_ENV_LINEAR "SM_USE_LINEAR_SEARCH" + +#define ESH_MIN_KEY_LEN (sizeof(ESH_REGION_INVALIDATED)) + +#define PMIX_DS_PUT_KEY(rc, ctx, addr, key, buf, size) \ + do { \ + rc = PMIX_ERROR; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->put_key) { \ + rc = (ctx)->file_cbs->put_key(addr, key, buf, size); \ + } \ + } while(0) + +#define PMIX_DS_KV_SIZE(ctx, addr) \ +__pmix_attribute_extension__ ({ \ + size_t size = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->kval_size) { \ + size = (ctx)->file_cbs->kval_size(addr); \ + } \ + size; \ +}) + +#define PMIX_DS_KNAME_PTR(ctx, addr) \ +__pmix_attribute_extension__ ({ \ + char *name_ptr = NULL; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->kname_ptr) { \ + name_ptr = (ctx)->file_cbs->kname_ptr(addr); \ + } \ + name_ptr; \ +}) + +#define PMIX_DS_KNAME_LEN(ctx, addr) \ +__pmix_attribute_extension__ ({ \ + size_t len = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->kname_len) { \ + len = (ctx)->file_cbs->kname_len((char*)addr); \ + } \ + len; \ +}) + +#define PMIX_DS_DATA_PTR(ctx, addr) \ +__pmix_attribute_extension__ ({ \ + uint8_t *data_ptr = NULL; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->data_ptr) { \ + data_ptr = (ctx)->file_cbs->data_ptr(addr); \ + } \ + data_ptr; \ +}) + +#define PMIX_DS_DATA_SIZE(ctx, addr, data_ptr) \ +__pmix_attribute_extension__ ({ \ + size_t size = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->data_size) { \ + size = (ctx)->file_cbs->data_size(addr, data_ptr); \ + } \ + size; \ +}) + +#define PMIX_DS_KEY_SIZE(ctx, key, data_size) \ +__pmix_attribute_extension__ ({ \ + size_t __size = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->key_size) { \ + __size = (ctx)->file_cbs->key_size(key, data_size); \ + } \ + __size; \ +}) + +#define PMIX_DS_SLOT_SIZE(ctx) \ +__pmix_attribute_extension__ ({ \ + size_t __size = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->ext_slot_size) { \ + __size = (ctx)->file_cbs->ext_slot_size(); \ + } \ + __size; \ +}) + +#define PMIX_DS_KEY_HASH(ctx, key) \ +__pmix_attribute_extension__ ({ \ + size_t keyhash = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->key_hash) { \ + keyhash = (ctx)->file_cbs->key_hash(key); \ + } \ + keyhash; \ +}) + +#define PMIX_DS_KEY_MATCH(ctx, addr, key, hash) \ +__pmix_attribute_extension__ ({ \ + int ret = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->key_match) { \ + ret = (ctx)->file_cbs->key_match(addr, key, hash); \ + } \ + ret; \ +}) + +#define PMIX_DS_KEY_IS_INVALID(ctx, addr) \ +__pmix_attribute_extension__ ({ \ + int ret = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->is_invalid) { \ + ret = (ctx)->file_cbs->is_invalid(addr); \ + } \ + ret; \ +}) + +#define PMIX_DS_KEY_SET_INVALID(ctx, addr) \ + do { \ + if ((ctx)->file_cbs && (ctx)->file_cbs->set_invalid) { \ + (ctx)->file_cbs->set_invalid(addr); \ + } \ + } while(0) + +#define PMIX_DS_KEY_IS_EXTSLOT(ctx, addr) \ +__pmix_attribute_extension__ ({ \ + int ret = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->is_invalid) { \ + ret = (ctx)->file_cbs->is_extslot(addr); \ + } \ + ret; \ +}) + + +#endif // DSTORE_FORMAT_H diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_segment.c b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_segment.c new file mode 100644 index 00000000000..9dff9505a0d --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_segment.c @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2017 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#ifdef HAVE_FCNTL_H +#include +#endif + +#ifdef HAVE_SYS_AUXV_H +#include +#endif + +#include + +#include "src/include/pmix_globals.h" +#include "src/mca/gds/base/base.h" +#include "src/mca/pshmem/base/base.h" +#include "src/util/error.h" +#include "src/util/output.h" + +#include "dstore_common.h" +#include "dstore_segment.h" + +static size_t _initial_segment_size; +static size_t _meta_segment_size; +static size_t _data_segment_size; + +PMIX_EXPORT int pmix_common_dstor_getpagesize(void) +{ +#if defined(_SC_PAGESIZE ) + return sysconf(_SC_PAGESIZE); +#elif defined(_SC_PAGE_SIZE) + return sysconf(_SC_PAGE_SIZE); +#else + return 65536; /* safer to overestimate than under */ +#endif +} + +PMIX_EXPORT size_t pmix_common_dstor_getcacheblocksize(void) +{ + size_t cache_line = 0; + +#if defined(_SC_LEVEL1_DCACHE_LINESIZE) + cache_line = sysconf(_SC_LEVEL1_DCACHE_LINESIZE); +#endif +#if (defined(HAVE_SYS_AUXV_H)) && (defined(AT_DCACHEBSIZE)) + if (0 == cache_line) { + cache_line = getauxval(AT_DCACHEBSIZE); + } +#endif + return cache_line; +} + +PMIX_EXPORT void pmix_common_dstor_init_segment_info(size_t initial_segment_size, + size_t meta_segment_size, + size_t data_segment_size) +{ + _initial_segment_size = initial_segment_size; + _meta_segment_size = meta_segment_size; + _data_segment_size = data_segment_size; +} + +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_create_new_lock_seg(const char *base_path, size_t size, + const char *name, uint32_t id, uid_t uid, bool setuid) +{ + pmix_status_t rc; + char file_name[PMIX_PATH_MAX]; + pmix_dstore_seg_desc_t *new_seg = NULL; + + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s: segment type %d, nspace %s, id %u", + __FILE__, __LINE__, __func__, PMIX_DSTORE_NS_LOCK_SEGMENT, + name, id)); + + snprintf(file_name, PMIX_PATH_MAX, "%s/smlockseg-%s", base_path, name); + new_seg = (pmix_dstore_seg_desc_t*)malloc(sizeof(pmix_dstore_seg_desc_t)); + if (new_seg) { + new_seg->id = id; + new_seg->next = NULL; + new_seg->type = PMIX_DSTORE_NS_LOCK_SEGMENT; + rc = pmix_pshmem.segment_create(&new_seg->seg_info, file_name, size); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto err_exit; + } + memset(new_seg->seg_info.seg_base_addr, 0, size); + + if (setuid > 0){ + rc = PMIX_ERR_PERM; + if (0 > chown(file_name, (uid_t) uid, (gid_t) -1)){ + PMIX_ERROR_LOG(rc); + goto err_exit; + } + /* set the mode as required */ + if (0 > chmod(file_name, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP )) { + PMIX_ERROR_LOG(rc); + goto err_exit; + } + } + } + return new_seg; + + err_exit: + if( NULL != new_seg ){ + free(new_seg); + } + return NULL; + +} + +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_attach_new_lock_seg(const char *base_path, + size_t size, const char *name, uint32_t id) +{ + pmix_status_t rc; + pmix_dstore_seg_desc_t *new_seg = NULL; + new_seg = (pmix_dstore_seg_desc_t*)malloc(sizeof(pmix_dstore_seg_desc_t)); + new_seg->id = id; + new_seg->next = NULL; + new_seg->type = PMIX_DSTORE_NS_LOCK_SEGMENT; + new_seg->seg_info.seg_size = size; + + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s: segment type %d, name %s, id %u", + __FILE__, __LINE__, __func__, new_seg->type, name, id)); + + snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/smlockseg-%s", + base_path, name); + rc = pmix_pshmem.segment_attach(&new_seg->seg_info, PMIX_PSHMEM_RW); + if (PMIX_SUCCESS != rc) { + free(new_seg); + new_seg = NULL; + } + return new_seg; +} + +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_create_new_segment(pmix_dstore_segment_type type, + const char *base_path, const char *name, uint32_t id, + uid_t uid, bool setuid) +{ + pmix_status_t rc; + char file_name[PMIX_PATH_MAX]; + size_t size; + pmix_dstore_seg_desc_t *new_seg = NULL; + + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s: segment type %d, nspace %s, id %u", + __FILE__, __LINE__, __func__, type, name, id)); + + switch (type) { + case PMIX_DSTORE_INITIAL_SEGMENT: + size = _initial_segment_size; + snprintf(file_name, PMIX_PATH_MAX, "%s/initial-pmix_shared-segment-%u", + base_path, id); + break; + case PMIX_DSTORE_NS_META_SEGMENT: + size = _meta_segment_size; + snprintf(file_name, PMIX_PATH_MAX, "%s/smseg-%s-%u", base_path, name, id); + break; + case PMIX_DSTORE_NS_DATA_SEGMENT: + size = _data_segment_size; + snprintf(file_name, PMIX_PATH_MAX, "%s/smdataseg-%s-%d", base_path, name, id); + break; + default: + PMIX_ERROR_LOG(PMIX_ERROR); + return NULL; + } + new_seg = (pmix_dstore_seg_desc_t*)malloc(sizeof(pmix_dstore_seg_desc_t)); + if (new_seg) { + new_seg->id = id; + new_seg->next = NULL; + new_seg->type = type; + rc = pmix_pshmem.segment_create(&new_seg->seg_info, file_name, size); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto err_exit; + } + memset(new_seg->seg_info.seg_base_addr, 0, size); + + if (setuid > 0){ + rc = PMIX_ERR_PERM; + if (0 > chown(file_name, (uid_t) uid, (gid_t) -1)){ + PMIX_ERROR_LOG(rc); + goto err_exit; + } + /* set the mode as required */ + if (0 > chmod(file_name, S_IRUSR | S_IRGRP | S_IWGRP )) { + PMIX_ERROR_LOG(rc); + goto err_exit; + } + } + } + return new_seg; + +err_exit: + if( NULL != new_seg ){ + free(new_seg); + } + return NULL; +} + +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_attach_new_segment(pmix_dstore_segment_type type, const char *base_path, + const char *name, uint32_t id) +{ + pmix_status_t rc; + pmix_dstore_seg_desc_t *new_seg = NULL; + new_seg = (pmix_dstore_seg_desc_t*)malloc(sizeof(pmix_dstore_seg_desc_t)); + new_seg->id = id; + new_seg->next = NULL; + new_seg->type = type; + + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s: segment type %d, nspace %s, id %u", + __FILE__, __LINE__, __func__, type, name, id)); + + switch (type) { + case PMIX_DSTORE_INITIAL_SEGMENT: + new_seg->seg_info.seg_size = _initial_segment_size; + snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/initial-pmix_shared-segment-%u", + base_path, id); + break; + case PMIX_DSTORE_NS_META_SEGMENT: + new_seg->seg_info.seg_size = _meta_segment_size; + snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/smseg-%s-%u", + base_path, name, id); + break; + case PMIX_DSTORE_NS_DATA_SEGMENT: + new_seg->seg_info.seg_size = _data_segment_size; + snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/smdataseg-%s-%d", + base_path, name, id); + break; + default: + free(new_seg); + PMIX_ERROR_LOG(PMIX_ERROR); + return NULL; + } + rc = pmix_pshmem.segment_attach(&new_seg->seg_info, PMIX_PSHMEM_RONLY); + if (PMIX_SUCCESS != rc) { + free(new_seg); + new_seg = NULL; + PMIX_ERROR_LOG(rc); + } + return new_seg; +} + +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_extend_segment(pmix_dstore_seg_desc_t *segdesc, const char *base_path, + const char *name, uid_t uid, bool setuid) +{ + pmix_dstore_seg_desc_t *tmp, *seg; + + PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, + "%s:%d:%s", + __FILE__, __LINE__, __func__)); + /* find last segment */ + tmp = segdesc; + while (NULL != tmp->next) { + tmp = tmp->next; + } + /* create another segment, the old one is full. */ + seg = pmix_common_dstor_create_new_segment(segdesc->type, base_path, name, tmp->id + 1, uid, setuid); + tmp->next = seg; + + return seg; +} + +PMIX_EXPORT void pmix_common_dstor_delete_sm_desc(pmix_dstore_seg_desc_t *desc) +{ + pmix_dstore_seg_desc_t *tmp; + + /* free all global segments */ + while (NULL != desc) { + tmp = desc->next; + /* detach & unlink from current desc */ + if (desc->seg_info.seg_cpid == getpid()) { + pmix_pshmem.segment_unlink(&desc->seg_info); + } + pmix_pshmem.segment_detach(&desc->seg_info); + free(desc); + desc = tmp; + } +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_segment.h b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_segment.h new file mode 100644 index 00000000000..bd55f4e0821 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/common/dstore/dstore_segment.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2017 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include + +#include "src/include/pmix_globals.h" +#include "src/mca/gds/base/base.h" +#include "src/mca/pshmem/base/base.h" + +#include "dstore_common.h" + +#ifndef DSTORE_SEGMENT_H +#define DSTORE_SEGMENT_H + +/* this structs are used to store information about + * shared segments addresses locally at each process, + * so they are common for different types of segments + * and don't have a specific content (namespace's info, + * rank's meta info, ranks's data). */ + +typedef struct pmix_dstore_seg_desc_t pmix_dstore_seg_desc_t; + +typedef enum { + PMIX_DSTORE_INITIAL_SEGMENT, + PMIX_DSTORE_NS_META_SEGMENT, + PMIX_DSTORE_NS_DATA_SEGMENT, + PMIX_DSTORE_NS_LOCK_SEGMENT, +} pmix_dstore_segment_type; + +struct pmix_dstore_seg_desc_t { + pmix_dstore_segment_type type; + pmix_pshmem_seg_t seg_info; + uint32_t id; + pmix_dstore_seg_desc_t *next; +}; + +PMIX_EXPORT int pmix_common_dstor_getpagesize(void); +PMIX_EXPORT size_t pmix_common_dstor_getcacheblocksize(void); +PMIX_EXPORT void pmix_common_dstor_init_segment_info(size_t initial_segment_size, + size_t meta_segment_size, + size_t data_segment_size); +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_create_new_segment(pmix_dstore_segment_type type, + const char *base_path, const char *name, uint32_t id, + uid_t uid, bool setuid); +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_attach_new_segment(pmix_dstore_segment_type type, + const char *base_path, + const char *name, uint32_t id); +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_extend_segment(pmix_dstore_seg_desc_t *segdesc, + const char *base_path, + const char *name, uid_t uid, bool setuid); +PMIX_EXPORT void pmix_common_dstor_delete_sm_desc(pmix_dstore_seg_desc_t *desc); +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_create_new_lock_seg(const char *base_path, size_t size, + const char *name, uint32_t id, uid_t uid, bool setuid); +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_attach_new_lock_seg(const char *base_path, + size_t size, const char *name, uint32_t id); + +#endif // DSTORE_SEGMENT_H diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/base/base.h b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/base/base.h index 3ada366984f..9f0136a5481 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/base/base.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/base/base.h @@ -11,9 +11,10 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -76,6 +77,12 @@ struct pmix_gds_globals_t { }; typedef struct pmix_gds_globals_t pmix_gds_globals_t; +typedef void * pmix_gds_base_store_modex_cbdata_t; +typedef pmix_status_t (*pmix_gds_base_store_modex_cb_fn_t)(pmix_gds_base_store_modex_cbdata_t cbdata, + struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_byte_object_t *bo); + PMIX_EXPORT extern pmix_gds_globals_t pmix_gds_globals; /* get a list of available support - caller must free results @@ -98,6 +105,12 @@ PMIX_EXPORT pmix_gds_base_module_t* pmix_gds_base_assign_module(pmix_info_t *inf PMIX_EXPORT pmix_status_t pmix_gds_base_setup_fork(const pmix_proc_t *proc, char ***env); +PMIX_EXPORT pmix_status_t pmix_gds_base_store_modex(struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_buffer_t *xfer, + pmix_gds_base_store_modex_cb_fn_t cb_fn, + pmix_gds_base_store_modex_cbdata_t cbdata); + END_C_DECLS #endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/base/gds_base_fns.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/base/gds_base_fns.c index 16e88485c9f..abec7a744da 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/base/gds_base_fns.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/base/gds_base_fns.c @@ -1,8 +1,11 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * * $COPYRIGHT$ * @@ -76,10 +79,107 @@ pmix_status_t pmix_gds_base_setup_fork(const pmix_proc_t *proc, if (NULL == active->module->setup_fork) { continue; } - if (PMIX_SUCCESS != (rc = active->module->setup_fork(proc, env))) { + rc = active->module->setup_fork(proc, env); + if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_AVAILABLE != rc) { return rc; } } return PMIX_SUCCESS; } + +pmix_status_t pmix_gds_base_store_modex(struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_buffer_t * buff, + pmix_gds_base_store_modex_cb_fn_t cb_fn, + pmix_gds_base_store_modex_cbdata_t cbdata) +{ + pmix_status_t rc = PMIX_SUCCESS; + pmix_namespace_t * ns = (pmix_namespace_t *)nspace; + pmix_buffer_t bkt; + pmix_byte_object_t bo, bo2; + int32_t cnt = 1; + char byte; + pmix_collect_t ctype; + bool have_ctype = false; + + /* Loop over the enclosed byte object envelopes and + * store them in our GDS module */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + buff, &bo, &cnt, PMIX_BYTE_OBJECT); + while (PMIX_SUCCESS == rc) { + PMIX_CONSTRUCT(&bkt, pmix_buffer_t); + PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt, bo.bytes, bo.size); + /* unpack the data collection flag */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, &byte, &cnt, PMIX_BYTE); + if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) { + /* no data was returned, so we are done with this blob */ + PMIX_DESTRUCT(&bkt); + break; + } + if (PMIX_SUCCESS != rc) { + /* we have an error */ + PMIX_DESTRUCT(&bkt); + goto error; + } + + // Check that this blob was accumulated with the same data collection setting + if (have_ctype) { + if (ctype != (pmix_collect_t)byte) { + rc = PMIX_ERR_INVALID_ARG; + PMIX_DESTRUCT(&bkt); + goto error; + } + } + else { + ctype = (pmix_collect_t)byte; + have_ctype = true; + } + + /* unpack the enclosed blobs from the various peers */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, &bo2, &cnt, PMIX_BYTE_OBJECT); + while (PMIX_SUCCESS == rc) { + /* unpack all the kval's from this peer and store them in + * our GDS. Note that PMIx by design holds all data at + * the server level until requested. If our GDS is a + * shared memory region, then the data may be available + * right away - but the client still has to be notified + * of its presence. */ + rc = cb_fn(cbdata, (struct pmix_namespace_t *)ns, cbs, &bo2); + if (PMIX_SUCCESS != rc) { + PMIX_DESTRUCT(&bkt); + goto error; + } + PMIX_BYTE_OBJECT_DESTRUCT(&bo2); + /* get the next blob */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, &bo2, &cnt, PMIX_BYTE_OBJECT); + } + PMIX_DESTRUCT(&bkt); + if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) { + rc = PMIX_SUCCESS; + } else if (PMIX_SUCCESS != rc) { + goto error; + } + /* unpack and process the next blob */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + buff, &bo, &cnt, PMIX_BYTE_OBJECT); + } + if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) { + rc = PMIX_SUCCESS; + } + +error: + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + + return rc; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/Makefile.am index ac62d8a9aad..eae7ef34abc 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2017 Mellanox Technologies, Inc. @@ -24,11 +24,24 @@ # headers = \ - gds_dstore.h + gds_ds12_base.h \ + gds_ds12_lock.h \ + gds_ds12_file.h sources = \ - gds_dstore.c \ - gds_dstore_component.c + gds_ds12_base.c \ + gds_ds12_lock.c \ + gds_ds12_component.c \ + gds_ds12_file.c \ + gds_ds20_file.c + +if HAVE_DSTORE_PTHREAD_LOCK +sources += gds_ds12_lock_pthread.c +else +if HAVE_DSTORE_FCNTL_LOCK +sources += gds_ds12_lock_fcntl.c +endif +endif # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la @@ -49,7 +62,11 @@ endif mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component) mca_gds_ds12_la_SOURCES = $(component_sources) -mca_gds_ds12_la_LDFLAGS = -module -avoid-version +mca_gds_ds12_la_LDFLAGS = -module -avoid-version \ + $(PMIX_TOP_BUILDDIR)/src/mca/common/dstore/libmca_common_dstore.la +if NEED_LIBPMIX +mca_gds_ds12_la_LIBADD = $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_gds_ds12_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_base.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_base.c new file mode 100644 index 00000000000..bb11c9223f7 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_base.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include +#include "src/include/pmix_globals.h" +#include "src/util/error.h" +#include "src/mca/gds/base/base.h" +#include "src/util/argv.h" + +#include "src/mca/common/dstore/dstore_common.h" +#include "gds_ds12_base.h" +#include "gds_ds12_lock.h" +#include "gds_ds12_file.h" +#include "src/mca/common/dstore/dstore_base.h" + +static pmix_common_dstore_ctx_t *ds12_ctx; + +static pmix_status_t ds12_init(pmix_info_t info[], size_t ninfo) +{ + pmix_status_t rc = PMIX_SUCCESS; + pmix_common_dstore_file_cbs_t *dstore_file_cbs = NULL; + + if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + dstore_file_cbs = &pmix_ds20_file_module; + } + ds12_ctx = pmix_common_dstor_init("ds12", info, ninfo, + &pmix_ds12_lock_module, + dstore_file_cbs); + if (NULL == ds12_ctx) { + rc = PMIX_ERR_INIT; + } + + return rc; +} + +static void ds12_finalize(void) +{ + pmix_common_dstor_finalize(ds12_ctx); +} + +static pmix_status_t ds12_assign_module(pmix_info_t *info, size_t ninfo, + int *priority) +{ + size_t n, m; + char **options; + + *priority = 20; + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_GDS_MODULE, PMIX_MAX_KEYLEN)) { + options = pmix_argv_split(info[n].value.data.string, ','); + for (m=0; NULL != options[m]; m++) { + if (0 == strcmp(options[m], "ds12")) { + /* they specifically asked for us */ + *priority = 100; + break; + } + if (0 == strcmp(options[m], "dstore")) { + /* they are asking for any dstore module - we + * take an intermediate priority in case another + * dstore is more modern than us */ + *priority = 50; + break; + } + } + pmix_argv_free(options); + break; + } + } + } + + return PMIX_SUCCESS; +} + +static pmix_status_t ds12_cache_job_info(struct pmix_namespace_t *ns, + pmix_info_t info[], size_t ninfo) +{ + return PMIX_SUCCESS; +} + +static pmix_status_t ds12_register_job_info(struct pmix_peer_t *pr, + pmix_buffer_t *reply) +{ + if (PMIX_PROC_IS_V1(pr)) { + ds12_ctx->file_cbs = &pmix_ds12_file_module; + } else { + ds12_ctx->file_cbs = &pmix_ds20_file_module; + } + return pmix_common_dstor_register_job_info(ds12_ctx, pr, reply); +} + +static pmix_status_t ds12_store_job_info(const char *nspace, pmix_buffer_t *buf) +{ + return pmix_common_dstor_store_job_info(ds12_ctx, nspace, buf); +} + +static pmix_status_t ds12_store(const pmix_proc_t *proc, + pmix_scope_t scope, + pmix_kval_t *kv) +{ + return pmix_common_dstor_store(ds12_ctx, proc, scope, kv); +} + +/* this function is only called by the PMIx server when its + * host has received data from some other peer. It therefore + * always contains data solely from remote procs, and we + * shall store it accordingly */ +static pmix_status_t ds12_store_modex(struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_buffer_t *buf) +{ + return pmix_common_dstor_store_modex(ds12_ctx, nspace, cbs, buf); +} + +static pmix_status_t ds12_fetch(const pmix_proc_t *proc, + pmix_scope_t scope, bool copy, + const char *key, + pmix_info_t info[], size_t ninfo, + pmix_list_t *kvs) +{ + return pmix_common_dstor_fetch(ds12_ctx, proc, scope, copy, key, info, ninfo, kvs); +} + +static pmix_status_t ds12_setup_fork(const pmix_proc_t *peer, char ***env) +{ + return pmix_common_dstor_setup_fork(ds12_ctx, PMIX_DSTORE_ESH_BASE_PATH, peer, env); +} + +static pmix_status_t ds12_add_nspace(const char *nspace, + pmix_info_t info[], + size_t ninfo) +{ + return pmix_common_dstor_add_nspace(ds12_ctx, nspace, info, ninfo); +} + +static pmix_status_t ds12_del_nspace(const char* nspace) +{ + return pmix_common_dstor_del_nspace(ds12_ctx, nspace); +} + +pmix_gds_base_module_t pmix_ds12_module = { + .name = "ds12", + .is_tsafe = false, + .init = ds12_init, + .finalize = ds12_finalize, + .assign_module = ds12_assign_module, + .cache_job_info = ds12_cache_job_info, + .register_job_info = ds12_register_job_info, + .store_job_info = ds12_store_job_info, + .store = ds12_store, + .store_modex = ds12_store_modex, + .fetch = ds12_fetch, + .setup_fork = ds12_setup_fork, + .add_nspace = ds12_add_nspace, + .del_nspace = ds12_del_nspace, +}; + diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_base.h b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_base.h new file mode 100644 index 00000000000..5f8a1623b98 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_base.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef GDS_DSTORE_12_H +#define GDS_DSTORE_12_H + +#include "src/mca/gds/gds.h" + +/* the component must be visible data for the linker to find it */ +PMIX_EXPORT extern pmix_gds_base_component_t mca_gds_ds12_component; +extern pmix_gds_base_module_t pmix_ds12_module; + +#endif // GDS_DSTORE_12_H diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_dstore_component.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_component.c similarity index 90% rename from opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_dstore_component.c rename to opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_component.c index da955113b50..abf5a62bfe0 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_dstore_component.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_component.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2017 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -31,9 +31,9 @@ #include #include "pmix_common.h" - +#include "src/include/pmix_globals.h" #include "src/mca/gds/gds.h" -#include "gds_dstore.h" +#include "gds_ds12_base.h" static pmix_status_t component_open(void); static pmix_status_t component_close(void); @@ -74,6 +74,13 @@ static int component_open(void) static int component_query(pmix_mca_base_module_t **module, int *priority) { + /* launchers cannot use the dstore */ + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + *priority = 0; + *module = NULL; + return PMIX_ERROR; + } + *priority = 20; *module = (pmix_mca_base_module_t *)&pmix_ds12_module; return PMIX_SUCCESS; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_file.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_file.c new file mode 100644 index 00000000000..2664cd3108a --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_file.c @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include "src/include/pmix_globals.h" +#include "src/mca/gds/base/base.h" + +#include "src/mca/common/dstore/dstore_file.h" +#include "gds_ds12_file.h" + +#define ESH_KEY_SIZE_V12(key, size) \ +__pmix_attribute_extension__ ({ \ + size_t len = strlen((char*)key) + 1 + sizeof(size_t) + size; \ + len; \ +}) + +/* in ext slot new offset will be stored in case if + * new data were added for the same process during + * next commit + */ +#define EXT_SLOT_SIZE_V12() \ + (ESH_KEY_SIZE_V12(ESH_REGION_EXTENSION, sizeof(size_t))) + +#define ESH_KV_SIZE_V12(addr) \ +__pmix_attribute_extension__ ({ \ + size_t sz; \ + memcpy(&sz, addr + \ + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)), \ + sizeof(size_t)); \ + sz += ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)) + \ + sizeof(size_t); \ + sz; \ +}) + +#define ESH_KNAME_PTR_V12(addr) \ +__pmix_attribute_extension__ ({ \ + char *name_ptr = (char*)addr; \ + name_ptr; \ +}) + +#define ESH_KNAME_LEN_V12(key) \ +__pmix_attribute_extension__ ({ \ + size_t len = strlen((char*)key) + 1; \ + len; \ +}) + +#define ESH_DATA_PTR_V12(addr) \ +__pmix_attribute_extension__ ({ \ + uint8_t *data_ptr = \ + addr + \ + sizeof(size_t) + \ + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)); \ + data_ptr; \ +}) + +#define ESH_DATA_SIZE_V12(addr) \ +__pmix_attribute_extension__ ({ \ + size_t data_size; \ + memcpy(&data_size, \ + addr + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)), \ + sizeof(size_t)); \ + data_size; \ +}) + +#define ESH_PUT_KEY_V12(addr, key, buffer, size) \ +__pmix_attribute_extension__ ({ \ + size_t sz = size; \ + memset(addr, 0, ESH_KNAME_LEN_V12(key)); \ + strncpy((char *)addr, key, ESH_KNAME_LEN_V12(key)); \ + memcpy(addr + ESH_KNAME_LEN_V12(key), &sz, \ + sizeof(size_t)); \ + memcpy(addr + ESH_KNAME_LEN_V12(key) + sizeof(size_t), \ + buffer, size); \ +}) + +static size_t pmix_ds12_kv_size(uint8_t *addr) +{ + size_t size; + + memcpy(&size, addr + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)), + sizeof(size_t)); + size += ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)) + sizeof(size_t); + return size; +} + +static char* pmix_ds12_key_name_ptr(uint8_t *addr) +{ + return ESH_KNAME_PTR_V12(addr); +} + +static size_t pmix_ds12_key_name_len(char *key) +{ + return ESH_KNAME_LEN_V12(key); +} + +static uint8_t* pmix_ds12_data_ptr(uint8_t *addr) +{ + return ESH_DATA_PTR_V12(addr); +} + +static size_t pmix_ds12_data_size(uint8_t *addr, uint8_t* data_ptr) +{ + return ESH_DATA_SIZE_V12(addr); +} + +static size_t pmix_ds12_key_size(char *addr, size_t data_size) +{ + return ESH_KEY_SIZE_V12(addr, data_size); +} + +static size_t pmix_ds12_ext_slot_size(void) +{ + return EXT_SLOT_SIZE_V12(); +} + +static int pmix_ds12_put_key(uint8_t *addr, char *key, void *buf, size_t size) +{ + ESH_PUT_KEY_V12(addr, key, buf, size); + return PMIX_SUCCESS; +} + +static bool pmix_ds12_is_invalid(uint8_t *addr) +{ + bool ret = (0 == strncmp(ESH_REGION_INVALIDATED, ESH_KNAME_PTR_V12(addr), + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)))); + return ret; +} + +static void pmix_ds12_set_invalid(uint8_t *addr) +{ + strncpy(ESH_KNAME_PTR_V12(addr), ESH_REGION_INVALIDATED, + ESH_KNAME_LEN_V12(ESH_REGION_INVALIDATED)); +} + +static bool pmix_ds12_is_ext_slot(uint8_t *addr) +{ + bool ret; + ret = (0 == strncmp(ESH_REGION_EXTENSION, ESH_KNAME_PTR_V12(addr), + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)))); + return ret; +} + +static bool pmix_ds12_kname_match(uint8_t *addr, const char *key, size_t key_hash) +{ + bool ret = 0; + + ret = (0 == strncmp(ESH_KNAME_PTR_V12(addr), + key, ESH_KNAME_LEN_V12(key))); + return ret; +} + +pmix_common_dstore_file_cbs_t pmix_ds12_file_module = { + .name = "ds12", + .kval_size = pmix_ds12_kv_size, + .kname_ptr = pmix_ds12_key_name_ptr, + .kname_len = pmix_ds12_key_name_len, + .data_ptr = pmix_ds12_data_ptr, + .data_size = pmix_ds12_data_size, + .key_size = pmix_ds12_key_size, + .ext_slot_size = pmix_ds12_ext_slot_size, + .put_key = pmix_ds12_put_key, + .is_invalid = pmix_ds12_is_invalid, + .is_extslot = pmix_ds12_is_ext_slot, + .set_invalid = pmix_ds12_set_invalid, + .key_hash = NULL, + .key_match = pmix_ds12_kname_match +}; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_file.h b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_file.h new file mode 100644 index 00000000000..af383d222bb --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_file.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef GDS_DS12_FILE_H +#define GDS_DS12_FILE_H + +#include +#include + +extern pmix_common_dstore_file_cbs_t pmix_ds12_file_module; +extern pmix_common_dstore_file_cbs_t pmix_ds20_file_module; + +#endif // GDS_DS12_FILE_H diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock.c new file mode 100644 index 00000000000..e3082681684 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include + +#include "src/mca/common/dstore/dstore_common.h" + +#include "gds_ds12_lock.h" + +pmix_common_lock_callbacks_t pmix_ds12_lock_module = { + .init = pmix_gds_ds12_lock_init, + .finalize = pmix_ds12_lock_finalize, + .rd_lock = pmix_ds12_lock_rd_get, + .rd_unlock = pmix_ds12_lock_rw_rel, + .wr_lock = pmix_ds12_lock_wr_get, + .wr_unlock = pmix_ds12_lock_rw_rel +}; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock.h b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock.h new file mode 100644 index 00000000000..4c07ba4b989 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef DS12_LOCK_H +#define DS12_LOCK_H + +#include +#include + +#include "src/mca/common/dstore/dstore_common.h" + +pmix_status_t pmix_gds_ds12_lock_init(pmix_common_dstor_lock_ctx_t *lock_ctx, + const char *base_path, const char *name, + uint32_t local_size, uid_t uid, bool setuid); +void pmix_ds12_lock_finalize(pmix_common_dstor_lock_ctx_t *lock_ctx); +pmix_status_t pmix_ds12_lock_rd_get(pmix_common_dstor_lock_ctx_t lock_ctx); +pmix_status_t pmix_ds12_lock_wr_get(pmix_common_dstor_lock_ctx_t lock_ctx); +pmix_status_t pmix_ds12_lock_rw_rel(pmix_common_dstor_lock_ctx_t lock_ctx); + +extern pmix_common_lock_callbacks_t pmix_ds12_lock_module; + +#endif // DS12_LOCK_H diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock_fcntl.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock_fcntl.c new file mode 100644 index 00000000000..759bf409374 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock_fcntl.c @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#ifdef HAVE_FCNTL_H +#include +#endif + +#include + +#include "src/mca/common/dstore/dstore_common.h" +#include "src/mca/gds/base/base.h" + +#include "src/util/error.h" +#include "src/util/output.h" + +#include "gds_ds12_lock.h" + +#define _ESH_12_FCNTL_LOCK(lockfd, operation) \ +__pmix_attribute_extension__ ({ \ + pmix_status_t ret = PMIX_SUCCESS; \ + int i; \ + struct flock fl = {0}; \ + fl.l_type = operation; \ + fl.l_whence = SEEK_SET; \ + for(i = 0; i < 10; i++) { \ + if( 0 > fcntl(lockfd, F_SETLKW, &fl) ) { \ + switch( errno ){ \ + case EINTR: \ + continue; \ + case ENOENT: \ + case EINVAL: \ + ret = PMIX_ERR_NOT_FOUND; \ + break; \ + case EBADF: \ + ret = PMIX_ERR_BAD_PARAM; \ + break; \ + case EDEADLK: \ + case EFAULT: \ + case ENOLCK: \ + ret = PMIX_ERR_RESOURCE_BUSY; \ + break; \ + default: \ + ret = PMIX_ERROR; \ + break; \ + } \ + } \ + break; \ + } \ + if (ret) { \ + pmix_output(0, "%s %d:%s lock failed: %s", \ + __FILE__, __LINE__, __func__, strerror(errno)); \ + } \ + ret; \ +}) + +typedef struct { + char *lockfile; + int lockfd; +} ds12_lock_fcntl_ctx_t; + +pmix_status_t pmix_gds_ds12_lock_init(pmix_common_dstor_lock_ctx_t *ctx, const char *base_path, + const char *name, uint32_t local_size, uid_t uid, bool setuid) +{ + pmix_status_t rc = PMIX_SUCCESS; + ds12_lock_fcntl_ctx_t *lock_ctx; + + if (*ctx != NULL) { + return PMIX_SUCCESS; + } + + lock_ctx = (ds12_lock_fcntl_ctx_t*)malloc(sizeof(ds12_lock_fcntl_ctx_t)); + if (NULL == lock_ctx) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + *ctx = lock_ctx; + memset(lock_ctx, 0, sizeof(ds12_lock_fcntl_ctx_t)); + lock_ctx->lockfd = -1; + + /* create a lock file to prevent clients from reading while server is writing + * to the shared memory. This situation is quite often, especially in case of + * direct modex when clients might ask for data simultaneously. */ + if(0 > asprintf(&lock_ctx->lockfile, "%s/dstore_sm.lock", base_path)) { + rc = PMIX_ERR_OUT_OF_RESOURCE; + PMIX_ERROR_LOG(rc); + goto error; + } + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s _lockfile_name: %s", __FILE__, __LINE__, __func__, lock_ctx->lockfile)); + + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + lock_ctx->lockfd = open(lock_ctx->lockfile, O_CREAT | O_RDWR | O_EXCL, 0600); + + /* if previous launch was crashed, the lockfile might not be deleted and unlocked, + * so we delete it and create a new one. */ + if (lock_ctx->lockfd < 0) { + unlink(lock_ctx->lockfile); + lock_ctx->lockfd = open(lock_ctx->lockfile, O_CREAT | O_RDWR, 0600); + if (lock_ctx->lockfd < 0) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + } + if (0 != setuid) { + if (0 > chown(lock_ctx->lockfile, uid, (gid_t) -1)) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + if (0 > chmod(lock_ctx->lockfile, S_IRUSR | S_IWGRP | S_IRGRP)) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + } + } + else { + lock_ctx->lockfd = open(lock_ctx->lockfile, O_RDONLY); + if (0 > lock_ctx->lockfd) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + } + + return rc; + +error: + if (NULL != lock_ctx) { + if (NULL != lock_ctx->lockfile) { + free(lock_ctx->lockfile); + } + if (0 > lock_ctx->lockfd) { + close(lock_ctx->lockfd); + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + unlink(lock_ctx->lockfile); + } + } + free(lock_ctx); + lock_ctx = NULL; + } + *ctx = NULL; + + return rc; +} + +void pmix_ds12_lock_finalize(pmix_common_dstor_lock_ctx_t *lock_ctx) +{ + ds12_lock_fcntl_ctx_t *fcntl_lock = (ds12_lock_fcntl_ctx_t*)*lock_ctx; + + if (NULL == fcntl_lock) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND); + return; + } + + close(fcntl_lock->lockfd); + + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + unlink(fcntl_lock->lockfile); + } + free(fcntl_lock); + *lock_ctx = NULL; +} + +pmix_status_t pmix_ds12_lock_rd_get(pmix_common_dstor_lock_ctx_t lock_ctx) +{ ds12_lock_fcntl_ctx_t *fcntl_lock = (ds12_lock_fcntl_ctx_t*)lock_ctx; + pmix_status_t rc; + + if (NULL == fcntl_lock) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = _ESH_12_FCNTL_LOCK(fcntl_lock->lockfd, F_RDLCK); + + return rc; + +} + +pmix_status_t pmix_ds12_lock_wr_get(pmix_common_dstor_lock_ctx_t lock_ctx) +{ ds12_lock_fcntl_ctx_t *fcntl_lock = (ds12_lock_fcntl_ctx_t*)lock_ctx; + pmix_status_t rc; + + if (NULL == fcntl_lock) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = _ESH_12_FCNTL_LOCK(fcntl_lock->lockfd, F_WRLCK); + + return rc; + +} + +pmix_status_t pmix_ds12_lock_rw_rel(pmix_common_dstor_lock_ctx_t lock_ctx) +{ ds12_lock_fcntl_ctx_t *fcntl_lock = (ds12_lock_fcntl_ctx_t*)lock_ctx; + pmix_status_t rc; + + if (NULL == fcntl_lock) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = _ESH_12_FCNTL_LOCK(fcntl_lock->lockfd, F_UNLCK); + + return rc; + +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock_pthread.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock_pthread.c new file mode 100644 index 00000000000..1c9e9b79bb3 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds12_lock_pthread.c @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif + +#include + +#include "src/mca/common/dstore/dstore_common.h" +#include "src/mca/gds/base/base.h" +#include "src/mca/pshmem/pshmem.h" + +#include "src/util/error.h" +#include "src/util/output.h" + +#include "gds_ds12_lock.h" +#include "src/mca/common/dstore/dstore_segment.h" + +#define _ESH_12_PTHREAD_LOCK(rwlock, func) \ +__pmix_attribute_extension__ ({ \ + pmix_status_t ret = PMIX_SUCCESS; \ + int rc; \ + rc = pthread_rwlock_##func(rwlock); \ + if (0 != rc) { \ + switch (errno) { \ + case EINVAL: \ + ret = PMIX_ERR_INIT; \ + break; \ + case EPERM: \ + ret = PMIX_ERR_NO_PERMISSIONS; \ + break; \ + } \ + } \ + if (ret) { \ + pmix_output(0, "%s %d:%s lock failed: %s", \ + __FILE__, __LINE__, __func__, strerror(errno)); \ + } \ + ret; \ +}) + +typedef struct { + char *lockfile; + pmix_pshmem_seg_t *segment; + pthread_rwlock_t *rwlock; +} ds12_lock_pthread_ctx_t; + +pmix_status_t pmix_gds_ds12_lock_init(pmix_common_dstor_lock_ctx_t *ctx, const char *base_path, + const char * name, uint32_t local_size, uid_t uid, bool setuid) +{ + size_t size = pmix_common_dstor_getpagesize(); + pmix_status_t rc = PMIX_SUCCESS; + pthread_rwlockattr_t attr; + ds12_lock_pthread_ctx_t *lock_ctx = (ds12_lock_pthread_ctx_t*)ctx; + + if (*ctx != NULL) { + return PMIX_SUCCESS; + } + + lock_ctx = (ds12_lock_pthread_ctx_t*)malloc(sizeof(ds12_lock_pthread_ctx_t)); + if (NULL == lock_ctx) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + memset(lock_ctx, 0, sizeof(ds12_lock_pthread_ctx_t)); + *ctx = (pmix_common_dstor_lock_ctx_t*)lock_ctx; + + lock_ctx->segment = (pmix_pshmem_seg_t *)malloc(sizeof(pmix_pshmem_seg_t)); + if (NULL == lock_ctx->segment) { + rc = PMIX_ERR_OUT_OF_RESOURCE; + PMIX_ERROR_LOG(rc); + goto error; + } + + /* create a lock file to prevent clients from reading while server is writing + * to the shared memory. This situation is quite often, especially in case of + * direct modex when clients might ask for data simultaneously. */ + if(0 > asprintf(&lock_ctx->lockfile, "%s/dstore_sm.lock", base_path)) { + rc = PMIX_ERR_OUT_OF_RESOURCE; + PMIX_ERROR_LOG(rc); + goto error; + } + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s _lockfile_name: %s", __FILE__, __LINE__, __func__, lock_ctx->lockfile)); + + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_SUCCESS != (rc = pmix_pshmem.segment_create(lock_ctx->segment, + lock_ctx->lockfile, size))) { + PMIX_ERROR_LOG(rc); + goto error; + } + memset(lock_ctx->segment->seg_base_addr, 0, size); + if (0 != setuid) { + if (0 > chown(lock_ctx->lockfile, (uid_t) uid, (gid_t) -1)){ + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + /* set the mode as required */ + if (0 > chmod(lock_ctx->lockfile, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP )) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + } + lock_ctx->rwlock = (pthread_rwlock_t *)lock_ctx->segment->seg_base_addr; + + if (0 != pthread_rwlockattr_init(&attr)) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + if (0 != pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) { + pthread_rwlockattr_destroy(&attr); + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } +#ifdef HAVE_PTHREAD_SETKIND + if (0 != pthread_rwlockattr_setkind_np(&attr, + PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)) { + pthread_rwlockattr_destroy(&attr); + PMIX_ERROR_LOG(PMIX_ERR_INIT); + goto error; + } +#endif + if (0 != pthread_rwlock_init(lock_ctx->rwlock, &attr)) { + pthread_rwlockattr_destroy(&attr); + PMIX_ERROR_LOG(PMIX_ERR_INIT); + goto error; + } + if (0 != pthread_rwlockattr_destroy(&attr)) { + PMIX_ERROR_LOG(PMIX_ERR_INIT); + goto error; + } + + } + else { + lock_ctx->segment->seg_size = size; + snprintf(lock_ctx->segment->seg_name, PMIX_PATH_MAX, "%s", lock_ctx->lockfile); + if (PMIX_SUCCESS != (rc = pmix_pshmem.segment_attach(lock_ctx->segment, + PMIX_PSHMEM_RW))) { + PMIX_ERROR_LOG(rc); + goto error; + } + lock_ctx->rwlock = (pthread_rwlock_t *)lock_ctx->segment->seg_base_addr; + } + + return PMIX_SUCCESS; + +error: + if (NULL != lock_ctx) { + if (lock_ctx->segment) { + /* detach & unlink from current desc */ + if (lock_ctx->segment->seg_cpid == getpid()) { + pmix_pshmem.segment_unlink(lock_ctx->segment); + } + pmix_pshmem.segment_detach(lock_ctx->segment); + lock_ctx->rwlock = NULL; + } + if (NULL != lock_ctx->lockfile) { + free(lock_ctx->lockfile); + } + free(lock_ctx); + *ctx = (pmix_common_dstor_lock_ctx_t*)NULL; + } + + return rc; +} + +void pmix_ds12_lock_finalize(pmix_common_dstor_lock_ctx_t *lock_ctx) +{ + ds12_lock_pthread_ctx_t *pthread_lock = + (ds12_lock_pthread_ctx_t*)*lock_ctx; + + if (NULL == pthread_lock) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND); + return; + } + if (0 != pthread_rwlock_destroy(pthread_lock->rwlock)) { + PMIX_ERROR_LOG(PMIX_ERROR); + return; + } + + if (NULL == pthread_lock->segment) { + PMIX_ERROR_LOG(PMIX_ERROR); + return; + } + if (NULL == pthread_lock->lockfile) { + PMIX_ERROR_LOG(PMIX_ERROR); + return; + } + + /* detach & unlink from current desc */ + if (pthread_lock->segment->seg_cpid == getpid()) { + pmix_pshmem.segment_unlink(pthread_lock->segment); + } + pmix_pshmem.segment_detach(pthread_lock->segment); + + free(pthread_lock->segment); + pthread_lock->segment = NULL; + free(pthread_lock->lockfile); + pthread_lock->lockfile = NULL; + pthread_lock->rwlock = NULL; + free(pthread_lock); + *lock_ctx = NULL; +} + +pmix_status_t pmix_ds12_lock_rd_get(pmix_common_dstor_lock_ctx_t lock_ctx) +{ + ds12_lock_pthread_ctx_t *pthread_lock = (ds12_lock_pthread_ctx_t*)lock_ctx; + pmix_status_t rc; + + if (NULL == pthread_lock) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = _ESH_12_PTHREAD_LOCK(pthread_lock->rwlock, rdlock); + + return rc; +} + +pmix_status_t pmix_ds12_lock_wr_get(pmix_common_dstor_lock_ctx_t lock_ctx) +{ + ds12_lock_pthread_ctx_t *pthread_lock = (ds12_lock_pthread_ctx_t*)lock_ctx; + pmix_status_t rc; + + if (NULL == pthread_lock) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = _ESH_12_PTHREAD_LOCK(pthread_lock->rwlock, wrlock); + + return rc; +} + +pmix_status_t pmix_ds12_lock_rw_rel(pmix_common_dstor_lock_ctx_t lock_ctx) +{ + ds12_lock_pthread_ctx_t *pthread_lock = (ds12_lock_pthread_ctx_t*)lock_ctx; + pmix_status_t rc; + + if (NULL == pthread_lock) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = _ESH_12_PTHREAD_LOCK(pthread_lock->rwlock, unlock); + + return rc; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds20_file.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds20_file.c new file mode 100644 index 00000000000..75c816680b6 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_ds20_file.c @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include "src/include/pmix_globals.h" +#include "src/mca/gds/base/base.h" + +#include "src/mca/common/dstore/dstore_file.h" +#include "gds_ds12_file.h" + +#define ESH_KV_SIZE_V20(addr) \ +__pmix_attribute_extension__ ({ \ + size_t sz; \ + memcpy(&sz, addr, sizeof(size_t)); \ + sz; \ +}) + +#define ESH_KNAME_PTR_V20(addr) \ + ((char *)addr + sizeof(size_t)) + +#define ESH_KNAME_LEN_V20(key) \ +__pmix_attribute_extension__ ({ \ + size_t kname_len = strlen(key) + 1; \ + size_t len = (kname_len < ESH_MIN_KEY_LEN) ? \ + ESH_MIN_KEY_LEN : kname_len; \ + len; \ +}) + +#define ESH_DATA_PTR_V20(addr) \ +__pmix_attribute_extension__ ({ \ + size_t kname_len = \ + ESH_KNAME_LEN_V20(ESH_KNAME_PTR_V20(addr)); \ + uint8_t *data_ptr = addr + sizeof(size_t) + kname_len; \ + data_ptr; \ +}) + +#define ESH_DATA_SIZE_V20(addr, data_ptr) \ +__pmix_attribute_extension__ ({ \ + size_t __sz = ESH_KV_SIZE_V20(addr); \ + size_t data_size = __sz - (data_ptr - addr); \ + data_size; \ +}) + +#define ESH_KEY_SIZE_V20(key, size) \ + (sizeof(size_t) + ESH_KNAME_LEN_V20((char*)key) + size) + +/* in ext slot new offset will be stored in case if + * new data were added for the same process during + * next commit + */ +#define EXT_SLOT_SIZE_V20() \ + (ESH_KEY_SIZE_V20(ESH_REGION_EXTENSION, sizeof(size_t))) + + +#define ESH_PUT_KEY_V20(addr, key, buffer, size) \ +__pmix_attribute_extension__ ({ \ + size_t sz = ESH_KEY_SIZE_V20(key, size); \ + memcpy(addr, &sz, sizeof(size_t)); \ + memset(addr + sizeof(size_t), 0, \ + ESH_KNAME_LEN_V20(key)); \ + strncpy((char *)addr + sizeof(size_t), \ + key, ESH_KNAME_LEN_V20(key)); \ + memcpy(addr + sizeof(size_t) + ESH_KNAME_LEN_V20(key), \ + buffer, size); \ +}) + +static size_t pmix_ds20_kv_size(uint8_t *key) +{ + size_t size; + + memcpy(&size, key, sizeof(size_t)); + return size; +} + +static char* pmix_ds20_key_name_ptr(uint8_t *addr) +{ + return ESH_KNAME_PTR_V20(addr); +} + +static size_t pmix_ds20_key_name_len(char *key) +{ + return ESH_KNAME_LEN_V20(key); +} + +static uint8_t* pmix_ds20_data_ptr(uint8_t *addr) +{ + return ESH_DATA_PTR_V20(addr); +} + +static size_t pmix_ds20_data_size(uint8_t *addr, uint8_t* data_ptr) +{ + return ESH_DATA_SIZE_V20(addr, data_ptr); +} + +static size_t pmix_ds20_key_size(char *addr, size_t data_size) +{ + return ESH_KEY_SIZE_V20(addr, data_size); +} + +static size_t pmix_ds20_ext_slot_size(void) +{ + return EXT_SLOT_SIZE_V20(); +} + +static int pmix_ds20_put_key(uint8_t *addr, char *key, void *buf, size_t size) +{ + ESH_PUT_KEY_V20(addr, key, buf, size); + return PMIX_SUCCESS; +} + +static bool pmix_ds20_is_invalid(uint8_t *addr) +{ + bool ret = (0 == strncmp(ESH_REGION_INVALIDATED, ESH_KNAME_PTR_V20(addr), + ESH_KNAME_LEN_V20(ESH_KNAME_PTR_V20(addr)))); + return ret; +} + +static void pmix_ds20_set_invalid(uint8_t *addr) +{ + strncpy(ESH_KNAME_PTR_V20(addr), ESH_REGION_INVALIDATED, + ESH_KNAME_LEN_V20(ESH_REGION_INVALIDATED)); +} + +static bool pmix_ds20_is_ext_slot(uint8_t *addr) +{ + bool ret; + ret = (0 == strncmp(ESH_REGION_EXTENSION, ESH_KNAME_PTR_V20(addr), + ESH_KNAME_LEN_V20(ESH_KNAME_PTR_V20(addr)))); + return ret; +} + +static bool pmix_ds20_kname_match(uint8_t *addr, const char *key, size_t key_hash) +{ + bool ret = 0; + + ret = (0 == strncmp(ESH_KNAME_PTR_V20(addr), + key, ESH_KNAME_LEN_V20(key))); + return ret; +} + + +pmix_common_dstore_file_cbs_t pmix_ds20_file_module = { + .name = "ds20", + .kval_size = pmix_ds20_kv_size, + .kname_ptr = pmix_ds20_key_name_ptr, + .kname_len = pmix_ds20_key_name_len, + .data_ptr = pmix_ds20_data_ptr, + .data_size = pmix_ds20_data_size, + .key_size = pmix_ds20_key_size, + .ext_slot_size = pmix_ds20_ext_slot_size, + .put_key = pmix_ds20_put_key, + .is_invalid = pmix_ds20_is_invalid, + .is_extslot = pmix_ds20_is_ext_slot, + .set_invalid = pmix_ds20_set_invalid, + .key_hash = NULL, + .key_match = pmix_ds20_kname_match +}; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_dstore.h b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_dstore.h deleted file mode 100644 index abd4723ad25..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds12/gds_dstore.h +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2017 Mellanox Technologies, Inc. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef PMIX_DS12_H -#define PMIX_DS12_H - -#include - - -#include "src/mca/gds/gds.h" -#include "src/mca/pshmem/pshmem.h" - -BEGIN_C_DECLS - -#include -#include "src/class/pmix_value_array.h" - -#define INITIAL_SEG_SIZE 4096 -#define NS_META_SEG_SIZE (1<<22) -#define NS_DATA_SEG_SIZE (1<<22) - -#define PMIX_DSTORE_ESH_BASE_PATH "PMIX_DSTORE_ESH_BASE_PATH" - -#ifdef HAVE_PTHREAD_SHARED -#define ESH_PTHREAD_LOCK -#elif defined HAVE_FCNTL_FLOCK -#define ESH_FCNTL_LOCK -#else -#error No locking mechanism was found -#endif - -/* this structs are used to store information about - * shared segments addresses locally at each process, - * so they are common for different types of segments - * and don't have a specific content (namespace's info, - * rank's meta info, ranks's data). */ - -typedef enum { - INITIAL_SEGMENT, - NS_META_SEGMENT, - NS_DATA_SEGMENT -} segment_type; - -typedef struct seg_desc_t seg_desc_t; -struct seg_desc_t { - segment_type type; - pmix_pshmem_seg_t seg_info; - uint32_t id; - seg_desc_t *next; -}; - -typedef struct ns_map_data_s ns_map_data_t; -typedef struct session_s session_t; -typedef struct ns_map_s ns_map_t; - -struct session_s { - int in_use; - uid_t jobuid; - char setjobuid; - char *nspace_path; - char *lockfile; -#ifdef ESH_PTHREAD_LOCK - pmix_pshmem_seg_t *rwlock_seg; - pthread_rwlock_t *rwlock; -#endif - int lockfd; - seg_desc_t *sm_seg_first; - seg_desc_t *sm_seg_last; -}; - -struct ns_map_data_s { - char name[PMIX_MAX_NSLEN+1]; - size_t tbl_idx; - int track_idx; -}; - -struct ns_map_s { - int in_use; - ns_map_data_t data; -}; - -/* initial segment format: - * size_t num_elems; - * size_t full; //indicate to client that it needs to attach to the next segment - * ns_seg_info_t ns_seg_info[max_ns_num]; - */ - -typedef struct { - ns_map_data_t ns_map; - size_t num_meta_seg;/* read by clients to attach to this number of segments. */ - size_t num_data_seg; -} ns_seg_info_t; - -/* meta segment format: - * size_t num_elems; - * rank_meta_info meta_info[max_meta_elems]; - */ - -typedef struct { - size_t rank; - size_t offset; - size_t count; -} rank_meta_info; - -typedef struct { - pmix_value_array_t super; - ns_map_data_t ns_map; - size_t num_meta_seg; - size_t num_data_seg; - seg_desc_t *meta_seg; - seg_desc_t *data_seg; - bool in_use; -} ns_track_elem_t; - -/* the component must be visible data for the linker to find it */ -PMIX_EXPORT extern pmix_gds_base_component_t mca_gds_ds12_component; -extern pmix_gds_base_module_t pmix_ds12_module; - -END_C_DECLS - -#endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/Makefile.am new file mode 100644 index 00000000000..06e1dd13a90 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/Makefile.am @@ -0,0 +1,65 @@ +# -*- makefile -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. +# Copyright (c) 2017 Research Organization for Information Science +# and Technology (RIST). All rights reserved. +# Copyright (c) 2017-2018 Mellanox Technologies, Inc. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +headers = \ + gds_ds21_base.h \ + gds_ds21_lock.h \ + gds_ds21_file.h + +sources = \ + gds_ds21_base.c \ + gds_ds21_lock.c \ + gds_ds21_lock_pthread.c \ + gds_ds21_component.c \ + gds_ds21_file.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_pmix_gds_ds21_DSO +lib = +lib_sources = +component = mca_gds_ds21.la +component_sources = $(headers) $(sources) +else +lib = libmca_gds_ds21.la +lib_sources = $(headers) $(sources) +component = +component_sources = +endif + +mcacomponentdir = $(pmixlibdir) +mcacomponent_LTLIBRARIES = $(component) +mca_gds_ds21_la_SOURCES = $(component_sources) +mca_gds_ds21_la_LDFLAGS = -module -avoid-version \ + $(PMIX_TOP_BUILDDIR)/src/mca/common/dstore/libmca_common_dstore.la +if NEED_LIBPMIX +mca_gds_ds21_la_LIBADD = $(top_builddir)/src/libpmix.la +endif + +noinst_LTLIBRARIES = $(lib) +libmca_gds_ds21_la_SOURCES = $(lib_sources) +libmca_gds_ds21_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_base.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_base.c new file mode 100644 index 00000000000..9bb2b57c84b --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_base.c @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include +#include "src/include/pmix_globals.h" +#include "src/util/error.h" +#include "src/mca/gds/base/base.h" +#include "src/util/argv.h" + +#include "src/mca/common/dstore/dstore_common.h" +#include "gds_ds21_base.h" +#include "gds_ds21_lock.h" +#include "gds_ds21_file.h" +#include "src/mca/common/dstore/dstore_base.h" + +static pmix_common_dstore_ctx_t *ds21_ctx; + +static pmix_status_t ds21_init(pmix_info_t info[], size_t ninfo) +{ + pmix_status_t rc = PMIX_SUCCESS; + + ds21_ctx = pmix_common_dstor_init("ds21", info, ninfo, + &pmix_ds21_lock_module, + &pmix_ds21_file_module); + if (NULL == ds21_ctx) { + rc = PMIX_ERR_INIT; + } + + return rc; +} + +static void ds21_finalize(void) +{ + pmix_common_dstor_finalize(ds21_ctx); +} + +static pmix_status_t ds21_assign_module(pmix_info_t *info, size_t ninfo, + int *priority) +{ + size_t n, m; + char **options; + + *priority = 20; + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_GDS_MODULE, PMIX_MAX_KEYLEN)) { + options = pmix_argv_split(info[n].value.data.string, ','); + for (m=0; NULL != options[m]; m++) { + if (0 == strcmp(options[m], "ds21")) { + /* they specifically asked for us */ + *priority = 120; + break; + } + if (0 == strcmp(options[m], "dstore")) { + *priority = 60; + break; + } + } + pmix_argv_free(options); + break; + } + } + } + + return PMIX_SUCCESS; +} + +static pmix_status_t ds21_cache_job_info(struct pmix_namespace_t *ns, + pmix_info_t info[], size_t ninfo) +{ + return PMIX_SUCCESS; +} + +static pmix_status_t ds21_register_job_info(struct pmix_peer_t *pr, + pmix_buffer_t *reply) +{ + return pmix_common_dstor_register_job_info(ds21_ctx, pr, reply); +} + +static pmix_status_t ds21_store_job_info(const char *nspace, pmix_buffer_t *buf) +{ + return pmix_common_dstor_store_job_info(ds21_ctx, nspace, buf); +} + +static pmix_status_t ds21_store(const pmix_proc_t *proc, + pmix_scope_t scope, + pmix_kval_t *kv) +{ + return pmix_common_dstor_store(ds21_ctx, proc, scope, kv); +} + +/* this function is only called by the PMIx server when its + * host has received data from some other peer. It therefore + * always contains data solely from remote procs, and we + * shall store it accordingly */ +static pmix_status_t ds21_store_modex(struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_buffer_t *buf) +{ + return pmix_common_dstor_store_modex(ds21_ctx, nspace, cbs, buf); +} + +static pmix_status_t ds21_fetch(const pmix_proc_t *proc, + pmix_scope_t scope, bool copy, + const char *key, + pmix_info_t info[], size_t ninfo, + pmix_list_t *kvs) +{ + return pmix_common_dstor_fetch(ds21_ctx, proc, scope, copy, key, info, ninfo, kvs); +} + +static pmix_status_t ds21_setup_fork(const pmix_proc_t *peer, char ***env) +{ + pmix_status_t rc; + char *env_name = NULL; + int ds_ver = 0; + + sscanf(ds21_ctx->ds_name, "ds%d", &ds_ver); + if (0 == ds_ver) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + return rc; + } + if (0 > asprintf(&env_name, PMIX_DSTORE_VER_BASE_PATH_FMT, ds_ver)) { + rc = PMIX_ERR_NOMEM; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = pmix_common_dstor_setup_fork(ds21_ctx, env_name, peer, env); + free(env_name); + + return rc; +} + +static pmix_status_t ds21_add_nspace(const char *nspace, + pmix_info_t info[], + size_t ninfo) +{ + return pmix_common_dstor_add_nspace(ds21_ctx, nspace, info, ninfo); +} + +static pmix_status_t ds21_del_nspace(const char* nspace) +{ + return pmix_common_dstor_del_nspace(ds21_ctx, nspace); +} + +pmix_gds_base_module_t pmix_ds21_module = { + .name = "ds21", + .is_tsafe = true, + .init = ds21_init, + .finalize = ds21_finalize, + .assign_module = ds21_assign_module, + .cache_job_info = ds21_cache_job_info, + .register_job_info = ds21_register_job_info, + .store_job_info = ds21_store_job_info, + .store = ds21_store, + .store_modex = ds21_store_modex, + .fetch = ds21_fetch, + .setup_fork = ds21_setup_fork, + .add_nspace = ds21_add_nspace, + .del_nspace = ds21_del_nspace, +}; + diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_base.h b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_base.h new file mode 100644 index 00000000000..0779763bc7f --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_base.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef GDS_DSTORE_21_H +#define GDS_DSTORE_21_H + +#include "src/mca/gds/gds.h" + +/* the component must be visible data for the linker to find it */ +PMIX_EXPORT extern pmix_gds_base_component_t mca_gds_ds21_component; +extern pmix_gds_base_module_t pmix_ds21_module; + +#endif // GDS_DSTORE_21_H diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_component.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_component.c new file mode 100644 index 00000000000..88a21d0790d --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_component.c @@ -0,0 +1,93 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include +#include "pmix_common.h" + +#include "src/include/pmix_globals.h" +#include "src/mca/gds/gds.h" +#include "gds_ds21_base.h" + +static pmix_status_t component_open(void); +static pmix_status_t component_close(void); +static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +pmix_gds_base_component_t mca_gds_ds21_component = { + .base = { + PMIX_GDS_BASE_VERSION_1_0_0, + + /* Component name and version */ + .pmix_mca_component_name = "ds21", + PMIX_MCA_BASE_MAKE_VERSION(component, + PMIX_MAJOR_VERSION, + PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + + /* Component open and close functions */ + .pmix_mca_open_component = component_open, + .pmix_mca_close_component = component_close, + .pmix_mca_query_component = component_query, + }, + .data = { + /* The component is checkpoint ready */ + PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT + } +}; + + +static int component_open(void) +{ + return PMIX_SUCCESS; +} + + +static int component_query(pmix_mca_base_module_t **module, int *priority) +{ + /* launchers cannot use the dstore */ + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + *priority = 0; + *module = NULL; + return PMIX_ERROR; + } + + *priority = 30; + *module = (pmix_mca_base_module_t *)&pmix_ds21_module; + return PMIX_SUCCESS; +} + + +static int component_close(void) +{ + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_file.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_file.c new file mode 100644 index 00000000000..51276846d76 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_file.c @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include "src/include/pmix_globals.h" +#include "src/mca/gds/base/base.h" + +#include "src/mca/common/dstore/dstore_file.h" +#include "gds_ds21_file.h" + +#if 8 > SIZEOF_SIZE_T +#define ESH_REGION_EXTENSION_FLG 0x80000000 +#define ESH_REGION_INVALIDATED_FLG 0x40000000 +#define ESH_REGION_SIZE_MASK 0x3FFFFFFF +#else +#define ESH_REGION_EXTENSION_FLG 0x8000000000000000 +#define ESH_REGION_INVALIDATED_FLG 0x4000000000000000 +#define ESH_REGION_SIZE_MASK 0x3FFFFFFFFFFFFFFF +#endif + +#define ESH_KV_SIZE_V21(addr) \ +__pmix_attribute_extension__ ({ \ + size_t sz; \ + memcpy(&sz, addr, sizeof(size_t)); \ + /* drop flags in lsb's */ \ + (sz & ESH_REGION_SIZE_MASK); \ +}) + +#define ESH_KNAME_PTR_V21(addr) \ + ((char *)addr + 2 * sizeof(size_t)) + +#define ESH_KNAME_LEN_V21(key) \ + (strlen(key) + 1) + +#define ESH_DATA_PTR_V21(addr) \ +__pmix_attribute_extension__ ({ \ + char *key_ptr = ESH_KNAME_PTR_V21(addr); \ + size_t kname_len = ESH_KNAME_LEN_V21(key_ptr); \ + uint8_t *data_ptr = \ + addr + (key_ptr - (char*)addr) + kname_len; \ + data_ptr; \ +}) + +#define ESH_DATA_SIZE_V21(addr, data_ptr) \ +__pmix_attribute_extension__ ({ \ + size_t sz = ESH_KV_SIZE_V21(addr); \ + size_t data_size = sz - (data_ptr - addr); \ + data_size; \ +}) + +#define ESH_KEY_SIZE_V21(key, size) \ + (2 * sizeof(size_t) + ESH_KNAME_LEN_V21((char*)key) + size) + +/* in ext slot new offset will be stored in case if + * new data were added for the same process during + * next commit + */ +#define EXT_SLOT_SIZE_V21() \ + (ESH_KEY_SIZE_V21("", sizeof(size_t))) + +static bool pmix_ds21_is_invalid(uint8_t *addr) +{ + size_t sz; + memcpy(&sz, addr, sizeof(size_t)); + return !!(sz & ESH_REGION_INVALIDATED_FLG); +} + +static void pmix_ds21_set_invalid(uint8_t *addr) +{ + size_t sz; + memcpy(&sz, addr, sizeof(size_t)); + sz |= ESH_REGION_INVALIDATED_FLG; + memcpy(addr, &sz, sizeof(size_t)); +} + +static bool pmix_ds21_is_ext_slot(uint8_t *addr) +{ + size_t sz; + memcpy(&sz, addr, sizeof(size_t)); + return !!(sz & ESH_REGION_EXTENSION_FLG); +} + +static size_t pmix_ds21_key_hash(const char *key) +{ + size_t hash = 0; + int i; + for(i=0; key[i]; i++) { + hash += key[i]; + } + return hash; +} + +static bool pmix_ds21_kname_match(uint8_t *addr, const char *key, size_t key_hash) +{ + bool ret = 0; + size_t hash; + memcpy(&hash, (char*)addr + sizeof(size_t), sizeof(size_t)); + if( key_hash != hash ) { + return ret; + } + return (0 == strncmp(ESH_KNAME_PTR_V21(addr), key, ESH_KNAME_LEN_V21(key))); +} + +static size_t pmix_ds21_kval_size(uint8_t *key) +{ + return ESH_KV_SIZE_V21(key); ; +} + +static char* pmix_ds21_key_name_ptr(uint8_t *addr) +{ + return ESH_KNAME_PTR_V21(addr); +} + +static size_t pmix_ds21_key_name_len(char *key) +{ + return ESH_KNAME_LEN_V21(key); +} + +static uint8_t* pmix_ds21_data_ptr(uint8_t *addr) +{ + return ESH_DATA_PTR_V21(addr); +} + +static size_t pmix_ds21_data_size(uint8_t *addr, uint8_t* data_ptr) +{ + return ESH_DATA_SIZE_V21(addr, data_ptr); +} + +static size_t pmix_ds21_key_size(char *addr, size_t data_size) +{ + return ESH_KEY_SIZE_V21(addr, data_size); +} + +static size_t pmix_ds21_ext_slot_size(void) +{ + return EXT_SLOT_SIZE_V21(); +} + +static int pmix_ds21_put_key(uint8_t *addr, char *key, + void* buffer, size_t size) +{ + size_t flag = 0; + size_t hash = 0; + char *addr_ch = (char*)addr; + if( !strcmp(key, ESH_REGION_EXTENSION) ) { + /* we have a flag for this special key */ + key = ""; + flag |= ESH_REGION_EXTENSION_FLG; + } + size_t sz = ESH_KEY_SIZE_V21(key, size); + if( ESH_REGION_SIZE_MASK < sz ) { + return PMIX_ERROR; + } + sz |= flag; + memcpy(addr_ch, &sz, sizeof(size_t)); + hash = pmix_ds21_key_hash(key); + memcpy(addr_ch + sizeof(size_t), &hash, sizeof(size_t)); + strncpy(addr_ch + 2 * sizeof(size_t), key, ESH_KNAME_LEN_V21(key)); + memcpy(ESH_DATA_PTR_V21(addr), buffer, size); + return PMIX_SUCCESS; +} + +pmix_common_dstore_file_cbs_t pmix_ds21_file_module = { + .name = "ds21", + .kval_size = pmix_ds21_kval_size, + .kname_ptr = pmix_ds21_key_name_ptr, + .kname_len = pmix_ds21_key_name_len, + .data_ptr = pmix_ds21_data_ptr, + .data_size = pmix_ds21_data_size, + .key_size = pmix_ds21_key_size, + .ext_slot_size = pmix_ds21_ext_slot_size, + .put_key = pmix_ds21_put_key, + .is_invalid = pmix_ds21_is_invalid, + .is_extslot = pmix_ds21_is_ext_slot, + .set_invalid = pmix_ds21_set_invalid, + .key_hash = pmix_ds21_key_hash, + .key_match = pmix_ds21_kname_match +}; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_file.h b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_file.h new file mode 100644 index 00000000000..4fa64c7978e --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_file.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef GDS_DS21_FILE_H +#define GDS_DS21_FILE_H + +#include +#include + +extern pmix_common_dstore_file_cbs_t pmix_ds21_file_module; + +#endif // GDS_DS21_FILE_H diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock.c new file mode 100644 index 00000000000..09a3b367e6b --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include + +#include "src/mca/common/dstore/dstore_common.h" + +#include "gds_ds21_lock.h" + +pmix_common_lock_callbacks_t pmix_ds21_lock_module = { + .init = pmix_gds_ds21_lock_init, + .finalize = pmix_ds21_lock_finalize, + .rd_lock = pmix_ds21_lock_rd_get, + .rd_unlock = pmix_ds21_lock_rd_rel, + .wr_lock = pmix_ds21_lock_wr_get, + .wr_unlock = pmix_ds21_lock_wr_rel +}; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock.h b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock.h new file mode 100644 index 00000000000..e3054e97801 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef DS21_LOCK_H +#define DS21_LOCK_H + +#include +#include + +#include "src/mca/common/dstore/dstore_common.h" + +pmix_status_t pmix_gds_ds21_lock_init(pmix_common_dstor_lock_ctx_t *lock_ctx, + const char *base_path, const char *name, + uint32_t local_size, uid_t uid, bool setuid); +void pmix_ds21_lock_finalize(pmix_common_dstor_lock_ctx_t *lock_ctx); +pmix_status_t pmix_ds21_lock_rd_get(pmix_common_dstor_lock_ctx_t lock_ctx); +pmix_status_t pmix_ds21_lock_wr_get(pmix_common_dstor_lock_ctx_t lock_ctx); +pmix_status_t pmix_ds21_lock_rd_rel(pmix_common_dstor_lock_ctx_t lock_ctx); +pmix_status_t pmix_ds21_lock_wr_rel(pmix_common_dstor_lock_ctx_t lock_ctx); + +extern pmix_common_lock_callbacks_t pmix_ds21_lock_module; + +#endif // DS21_LOCK_H diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock_pthread.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock_pthread.c new file mode 100644 index 00000000000..99713f5651e --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/ds21/gds_ds21_lock_pthread.c @@ -0,0 +1,439 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include + +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif + +#include "src/mca/common/dstore/dstore_common.h" +#include "src/mca/gds/base/base.h" +#include "src/mca/pshmem/pshmem.h" +#include "src/class/pmix_list.h" + +#include "src/util/error.h" +#include "src/util/output.h" + +#include "gds_ds21_lock.h" +#include "src/mca/common/dstore/dstore_segment.h" + +typedef struct { + pmix_list_item_t super; + + char *lockfile; + pmix_dstore_seg_desc_t *seg_desc; + pthread_mutex_t *mutex; + uint32_t num_locks; + uint32_t lock_idx; +} lock_item_t; + +typedef struct { + pmix_list_t lock_traker; +} lock_ctx_t; + +typedef pmix_list_t ds21_lock_pthread_ctx_t; + +/* + * Lock segment format: + * 1. Segment size sizeof(size_t) + * 2. local_size: sizeof(uint32_t) + * 3. Align size sizeof(size_t) + * 4. Offset of mutexes sizeof(size_t) + * 5. Array of in use indexes: sizeof(int32_t)*local_size + * 6. Double array of locks: sizeof(pthread_mutex_t)*local_size*2 + */ +typedef struct { + size_t seg_size; + uint32_t num_locks; + size_t align_size; + size_t mutex_offs; +} segment_hdr_t; + +#define _GET_IDX_ARR_PTR(seg_ptr) \ + ((pmix_atomic_int32_t*)((char*)seg_ptr + sizeof(segment_hdr_t))) + +#define _GET_MUTEX_ARR_PTR(seg_hdr) \ + ((pthread_mutex_t*)((char*)seg_hdr + seg_hdr->mutex_offs)) + +#define _GET_MUTEX_PTR(seg_hdr, idx) \ + ((pthread_mutex_t*)((char*)seg_hdr + seg_hdr->mutex_offs + seg_hdr->align_size * (idx))) + + +static void ncon(lock_item_t *p) { + p->lockfile = NULL; + p->lock_idx = 0; + p->mutex = NULL; + p->num_locks = 0; + p->seg_desc = NULL; +} + +static void ldes(lock_item_t *p) { + uint32_t i; + + if(PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + segment_hdr_t *seg_hdr = (segment_hdr_t *)p->seg_desc->seg_info.seg_base_addr; + if (p->lockfile) { + unlink(p->lockfile); + } + for(i = 0; i < p->num_locks * 2; i++) { + pthread_mutex_t *mutex = _GET_MUTEX_PTR(seg_hdr, i); + if (0 != pthread_mutex_destroy(mutex)) { + PMIX_ERROR_LOG(PMIX_ERROR); + } + } + } + if (p->lockfile) { + free(p->lockfile); + } + if (p->seg_desc) { + pmix_common_dstor_delete_sm_desc(p->seg_desc); + } +} + +PMIX_CLASS_INSTANCE(lock_item_t, + pmix_list_item_t, + ncon, ldes); + +pmix_status_t pmix_gds_ds21_lock_init(pmix_common_dstor_lock_ctx_t *ctx, const char *base_path, const char * name, + uint32_t local_size, uid_t uid, bool setuid) +{ + pthread_mutexattr_t attr; + size_t size; + uint32_t i; + int page_size = pmix_common_dstor_getpagesize(); + segment_hdr_t *seg_hdr; + lock_item_t *lock_item = NULL; + lock_ctx_t *lock_ctx = (lock_ctx_t*)*ctx; + pmix_list_t *lock_tracker; + pmix_status_t rc = PMIX_SUCCESS; + + if (NULL == *ctx) { + lock_ctx = (lock_ctx_t*)malloc(sizeof(lock_ctx_t)); + if (NULL == lock_ctx) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + memset(lock_ctx, 0, sizeof(lock_ctx_t)); + PMIX_CONSTRUCT(&lock_ctx->lock_traker, pmix_list_t); + *ctx = lock_ctx; + } + + lock_tracker = &lock_ctx->lock_traker; + lock_item = PMIX_NEW(lock_item_t); + + if (NULL == lock_item) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + pmix_list_append(lock_tracker, &lock_item->super); + + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s local_size %d", __FILE__, __LINE__, __func__, local_size)); + + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + size_t seg_align_size; + size_t seg_hdr_size; + + if (0 != (seg_align_size = pmix_common_dstor_getcacheblocksize())) { + seg_align_size = (sizeof(pthread_mutex_t) / seg_align_size + 1) + * seg_align_size; + } else { + seg_align_size = sizeof(pthread_mutex_t); + } + + seg_hdr_size = ((sizeof(segment_hdr_t) + + sizeof(int32_t) * local_size) + / seg_align_size + 1) * seg_align_size; + + size = ((seg_hdr_size + + 2 * local_size * seg_align_size) /* array of mutexes */ + / page_size + 1) * page_size; + + lock_item->seg_desc = pmix_common_dstor_create_new_lock_seg(base_path, + size, name, 0, uid, setuid); + if (NULL == lock_item->seg_desc) { + rc = PMIX_ERR_OUT_OF_RESOURCE; + PMIX_ERROR_LOG(rc); + goto error; + } + + if (0 != pthread_mutexattr_init(&attr)) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + if (0 != pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) { + pthread_mutexattr_destroy(&attr); + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + + segment_hdr_t *seg_hdr = (segment_hdr_t*)lock_item->seg_desc->seg_info.seg_base_addr; + seg_hdr->num_locks = local_size; + seg_hdr->seg_size = size; + seg_hdr->align_size = seg_align_size; + seg_hdr->mutex_offs = seg_hdr_size; + + lock_item->lockfile = strdup(lock_item->seg_desc->seg_info.seg_name); + lock_item->num_locks = local_size; + lock_item->mutex = _GET_MUTEX_ARR_PTR(seg_hdr); + + for(i = 0; i < local_size * 2; i++) { + pthread_mutex_t *mutex = _GET_MUTEX_PTR(seg_hdr, i); + if (0 != pthread_mutex_init(mutex, &attr)) { + pthread_mutexattr_destroy(&attr); + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + } + if (0 != pthread_mutexattr_destroy(&attr)) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(PMIX_ERR_INIT); + goto error; + } + } + else { + pmix_atomic_int32_t *lock_idx_ptr; + bool idx_found = false; + + size = pmix_common_dstor_getpagesize(); + lock_item->seg_desc = pmix_common_dstor_attach_new_lock_seg(base_path, size, name, 0); + if (NULL == lock_item->seg_desc) { + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + seg_hdr = (segment_hdr_t*)lock_item->seg_desc->seg_info.seg_base_addr; + + if (seg_hdr->seg_size > size) { + size = seg_hdr->seg_size; + pmix_common_dstor_delete_sm_desc(lock_item->seg_desc); + lock_item->seg_desc = pmix_common_dstor_attach_new_lock_seg(base_path, size, name, 0); + if (NULL == lock_item->seg_desc) { + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + seg_hdr = (segment_hdr_t*)lock_item->seg_desc->seg_info.seg_base_addr; + } + + lock_item->num_locks = seg_hdr->num_locks; + lock_idx_ptr = _GET_IDX_ARR_PTR(seg_hdr); + lock_item->mutex = _GET_MUTEX_ARR_PTR(seg_hdr); + + for (i = 0; i < lock_item->num_locks; i++) { + int32_t expected = 0; + if (pmix_atomic_compare_exchange_strong_32(&lock_idx_ptr[i], &expected, 1)) { + lock_item->lock_idx = i; + lock_item->lockfile = strdup(lock_item->seg_desc->seg_info.seg_name); + idx_found = true; + break; + } + } + + if (false == idx_found) { + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + } + + return rc; + +error: + if (NULL != lock_item) { + pmix_list_remove_item(lock_tracker, &lock_item->super); + PMIX_RELEASE(lock_item); + lock_item = NULL; + } + *ctx = NULL; + + return rc; +} + +void pmix_ds21_lock_finalize(pmix_common_dstor_lock_ctx_t *lock_ctx) +{ + lock_item_t *lock_item, *item_next; + pmix_list_t *lock_tracker = &((lock_ctx_t*)*lock_ctx)->lock_traker; + + if (NULL == lock_tracker) { + return; + } + + PMIX_LIST_FOREACH_SAFE(lock_item, item_next, lock_tracker, lock_item_t) { + pmix_list_remove_item(lock_tracker, &lock_item->super); + PMIX_RELEASE(lock_item); + } + if (pmix_list_is_empty(lock_tracker)) { + PMIX_LIST_DESTRUCT(lock_tracker); + free(lock_tracker); + lock_tracker = NULL; + } + *lock_ctx = NULL; +} + +pmix_status_t pmix_ds21_lock_wr_get(pmix_common_dstor_lock_ctx_t lock_ctx) +{ + lock_item_t *lock_item; + pmix_list_t *lock_tracker = &((lock_ctx_t*)lock_ctx)->lock_traker; + uint32_t num_locks; + uint32_t i; + pmix_status_t rc; + segment_hdr_t *seg_hdr; + + if (NULL == lock_tracker) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND); + return rc; + } + + PMIX_LIST_FOREACH(lock_item, lock_tracker, lock_item_t) { + num_locks = lock_item->num_locks; + seg_hdr = (segment_hdr_t *)lock_item->seg_desc->seg_info.seg_base_addr; + + /* Lock the "signalling" lock first to let clients know that + * server is going to get a write lock. + * Clients do not hold this lock for a long time, + * so this loop should be relatively dast. + */ + for (i = 0; i < num_locks; i++) { + pthread_mutex_t *mutex = _GET_MUTEX_PTR(seg_hdr, 2*i); + if (0 != pthread_mutex_lock(mutex)) { + return PMIX_ERROR; + } + } + + /* Now we can go and grab the main locks + * New clients will be stopped at the previous + * "barrier" locks. + * We will wait here while all clients currently holding + * locks will be done + */ + for(i = 0; i < num_locks; i++) { + pthread_mutex_t *mutex = _GET_MUTEX_PTR(seg_hdr, 2*i + 1); + if (0 != pthread_mutex_lock(mutex)) { + return PMIX_ERROR; + } + } + } + return PMIX_SUCCESS; +} + +pmix_status_t pmix_ds21_lock_wr_rel(pmix_common_dstor_lock_ctx_t lock_ctx) +{ + lock_item_t *lock_item; + pmix_list_t *lock_tracker = &((lock_ctx_t*)lock_ctx)->lock_traker; + uint32_t num_locks; + uint32_t i; + pmix_status_t rc; + segment_hdr_t *seg_hdr; + + if (NULL == lock_tracker) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + + PMIX_LIST_FOREACH(lock_item, lock_tracker, lock_item_t) { + seg_hdr = (segment_hdr_t *)lock_item->seg_desc->seg_info.seg_base_addr; + num_locks = lock_item->num_locks; + + /* Lock the second lock first to ensure that all procs will see + * that we are trying to grab the main one */ + for(i=0; ilock_traker; + uint32_t idx; + pmix_status_t rc; + segment_hdr_t *seg_hdr; + + if (NULL == lock_tracker) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + + lock_item = (lock_item_t*)pmix_list_get_first(lock_tracker); + idx = lock_item->lock_idx; + seg_hdr = (segment_hdr_t *)lock_item->seg_desc->seg_info.seg_base_addr; + + /* This mutex is only used to acquire the next one, + * this is a barrier that server is using to let clients + * know that it is going to grab the write lock + */ + + if (0 != pthread_mutex_lock(_GET_MUTEX_PTR(seg_hdr, 2*idx))) { + return PMIX_ERROR; + } + + /* Now grab the main lock */ + if (0 != pthread_mutex_lock(_GET_MUTEX_PTR(seg_hdr, 2*idx + 1))) { + return PMIX_ERROR; + } + + /* Once done - release signalling lock */ + if (0 != pthread_mutex_unlock(_GET_MUTEX_PTR(seg_hdr, 2*idx))) { + return PMIX_ERROR; + } + + return PMIX_SUCCESS; +} + +pmix_status_t pmix_ds21_lock_rd_rel(pmix_common_dstor_lock_ctx_t lock_ctx) +{ + lock_item_t *lock_item; + pmix_list_t *lock_tracker = &((lock_ctx_t*)lock_ctx)->lock_traker; + pmix_status_t rc; + uint32_t idx; + segment_hdr_t *seg_hdr; + + if (NULL == lock_tracker) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + + lock_item = (lock_item_t*)pmix_list_get_first(lock_tracker); + seg_hdr = (segment_hdr_t *)lock_item->seg_desc->seg_info.seg_base_addr; + idx = lock_item->lock_idx; + + /* Release the main lock */ + if (0 != pthread_mutex_unlock(_GET_MUTEX_PTR(seg_hdr, 2*idx + 1))) { + return PMIX_SUCCESS; + } + + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/gds.h b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/gds.h index 9d90dbc565d..b68e55f139d 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/gds.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/gds.h @@ -1,8 +1,9 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2016-2017 Mellanox Technologies, Inc. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. * All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -44,7 +45,7 @@ BEGIN_C_DECLS /* forward declaration */ struct pmix_peer_t; -struct pmix_nspace_t; +struct pmix_namespace_t; /* backdoor to base verbosity */ PMIX_EXPORT extern int pmix_gds_base_output; @@ -117,7 +118,7 @@ typedef pmix_status_t (*pmix_gds_base_module_accept_kvs_resp_fn_t)(pmix_buffer_t * only we don't have packed data on the server side, and don't want * to incur the overhead of packing it just to unpack it in the function. */ -typedef pmix_status_t (*pmix_gds_base_module_cache_job_info_fn_t)(struct pmix_nspace_t *ns, +typedef pmix_status_t (*pmix_gds_base_module_cache_job_info_fn_t)(struct pmix_namespace_t *ns, pmix_info_t info[], size_t ninfo); /* define a convenience macro for caching job info */ @@ -127,7 +128,7 @@ typedef pmix_status_t (*pmix_gds_base_module_cache_job_info_fn_t)(struct pmix_ns pmix_output_verbose(1, pmix_gds_base_output, \ "[%s:%d] GDS CACHE JOB INFO WITH %s", \ __FILE__, __LINE__, _g->name); \ - (s) = _g->cache_job_info((struct pmix_nspace_t*)(n), (i), (ni)); \ + (s) = _g->cache_job_info((struct pmix_namespace_t*)(n), (i), (ni)); \ } while(0) /* register job-level info - this is provided as a special function @@ -135,7 +136,7 @@ typedef pmix_status_t (*pmix_gds_base_module_cache_job_info_fn_t)(struct pmix_ns * prepare the job-level info provided at PMIx_Register_nspace, because * we don't know the GDS component to use for that application until * a local client contacts us. Thus, the module is required to process - * the job-level info cached in the pmix_nspace_t for this job and + * the job-level info cached in the pmix_namespace_t for this job and * do whatever is necessary to support the client, packing any required * return message into the provided buffer. * @@ -155,7 +156,7 @@ typedef pmix_status_t (*pmix_gds_base_module_cache_job_info_fn_t)(struct pmix_ns * * The pmix_peer_t of the requesting client is provided here so that * the module can access the job-level info cached on the corresponding - * pmix_nspace_t pointed to by the pmix_peer_t + * pmix_namespace_t pointed to by the pmix_peer_t */ typedef pmix_status_t (*pmix_gds_base_module_register_job_info_fn_t)(struct pmix_peer_t *pr, pmix_buffer_t *reply); @@ -241,16 +242,16 @@ typedef pmix_status_t (*pmix_gds_base_module_store_fn_t)(const pmix_proc_t *proc * bo - pointer to the byte object containing the data * */ -typedef pmix_status_t (*pmix_gds_base_module_store_modex_fn_t)(struct pmix_nspace_t *ns, +typedef pmix_status_t (*pmix_gds_base_module_store_modex_fn_t)(struct pmix_namespace_t *ns, pmix_list_t *cbs, - pmix_byte_object_t *bo); + pmix_buffer_t *buff); /** * define a convenience macro for storing modex byte objects * * r - return status code * - * n - pointer to the pmix_nspace_t this blob is to be stored for + * n - pointer to the pmix_namespace_t this blob is to be stored for * * l - pointer to pmix_list_t containing pmix_server_caddy_t objects * of the local_cbs of the collective tracker @@ -262,7 +263,7 @@ typedef pmix_status_t (*pmix_gds_base_module_store_modex_fn_t)(struct pmix_nspac pmix_output_verbose(1, pmix_gds_base_output, \ "[%s:%d] GDS STORE MODEX WITH %s", \ __FILE__, __LINE__, (n)->compat.gds->name); \ - (r) = (n)->compat.gds->store_modex((struct pmix_nspace_t*)n, l, b); \ + (r) = (n)->compat.gds->store_modex((struct pmix_namespace_t*)n, l, b); \ } while (0) /** @@ -398,12 +399,26 @@ typedef pmix_status_t (*pmix_gds_base_module_del_nspace_fn_t)(const char* nspace } \ } while(0) +/* define a convenience macro for is_tsafe for fetch operation */ +#define PMIX_GDS_FETCH_IS_TSAFE(s, p) \ + do { \ + pmix_gds_base_module_t *_g = (p)->nptr->compat.gds; \ + pmix_output_verbose(1, pmix_gds_base_output, \ + "[%s:%d] GDS FETCH IS THREAD SAFE WITH %s", \ + __FILE__, __LINE__, _g->name); \ + if (true == _g->is_tsafe) { \ + (s) = PMIX_SUCCESS; \ + } else { \ + (s) = PMIX_ERR_NOT_SUPPORTED; \ + } \ +} while(0) /** * structure for gds modules */ typedef struct { const char *name; + const bool is_tsafe; pmix_gds_base_module_init_fn_t init; pmix_gds_base_module_fini_fn_t finalize; pmix_gds_base_assign_module_fn_t assign_module; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/hash/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/hash/Makefile.am index 7d9da0189e2..4067145ff28 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/hash/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/hash/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. # $COPYRIGHT$ @@ -49,6 +49,9 @@ mcacomponent_LTLIBRARIES = $(component) mca_gds_hash_la_SOURCES = $(component_sources) mca_gds_hash_la_LIBADD = $(gds_hash_LIBS) mca_gds_hash_la_LDFLAGS = -module -avoid-version $(gds_hash_LDFLAGS) +if NEED_LIBPMIX +mca_gds_hash_la_LIBADD += $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_gds_hash_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/hash/gds_hash.c b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/hash/gds_hash.c index d8611d8fff0..4e092fc5a68 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/gds/hash/gds_hash.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/gds/hash/gds_hash.c @@ -1,6 +1,10 @@ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 IBM Corporation. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. * * $COPYRIGHT$ * @@ -49,7 +53,7 @@ static void hash_finalize(void); static pmix_status_t hash_assign_module(pmix_info_t *info, size_t ninfo, int *priority); -static pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, +static pmix_status_t hash_cache_job_info(struct pmix_namespace_t *ns, pmix_info_t info[], size_t ninfo); static pmix_status_t hash_register_job_info(struct pmix_peer_t *pr, @@ -62,9 +66,14 @@ static pmix_status_t hash_store(const pmix_proc_t *proc, pmix_scope_t scope, pmix_kval_t *kv); -static pmix_status_t hash_store_modex(struct pmix_nspace_t *ns, +static pmix_status_t hash_store_modex(struct pmix_namespace_t *ns, pmix_list_t *cbs, - pmix_byte_object_t *bo); + pmix_buffer_t *buff); + +static pmix_status_t _hash_store_modex(void * cbdata, + struct pmix_namespace_t *ns, + pmix_list_t *cbs, + pmix_byte_object_t *bo); static pmix_status_t hash_fetch(const pmix_proc_t *proc, pmix_scope_t scope, bool copy, @@ -89,6 +98,7 @@ static pmix_status_t accept_kvs_resp(pmix_buffer_t *buf); pmix_gds_base_module_t pmix_hash_module = { .name = "hash", + .is_tsafe = false, .init = hash_init, .finalize = hash_finalize, .assign_module = hash_assign_module, @@ -108,7 +118,7 @@ pmix_gds_base_module_t pmix_hash_module = { typedef struct { pmix_list_item_t super; char *ns; - pmix_nspace_t *nptr; + pmix_namespace_t *nptr; pmix_hash_table_t internal; pmix_hash_table_t remote; pmix_hash_table_t local; @@ -135,8 +145,11 @@ static void htdes(pmix_hash_trkr_t *p) if (NULL != p->nptr) { PMIX_RELEASE(p->nptr); } + pmix_hash_remove_data(&p->internal, PMIX_RANK_WILDCARD, NULL); PMIX_DESTRUCT(&p->internal); + pmix_hash_remove_data(&p->remote, PMIX_RANK_WILDCARD, NULL); PMIX_DESTRUCT(&p->remote); + pmix_hash_remove_data(&p->local, PMIX_RANK_WILDCARD, NULL); PMIX_DESTRUCT(&p->local); } static PMIX_CLASS_INSTANCE(pmix_hash_trkr_t, @@ -188,8 +201,16 @@ static pmix_status_t hash_assign_module(pmix_info_t *info, size_t ninfo, return PMIX_SUCCESS; } +/* Define a bitmask to track what information may not have + * been provided but is computable from other info */ +#define PMIX_HASH_PROC_DATA 0x00000001 +#define PMIX_HASH_JOB_SIZE 0x00000002 +#define PMIX_HASH_MAX_PROCS 0x00000004 +#define PMIX_HASH_NUM_NODES 0x00000008 + static pmix_status_t store_map(pmix_hash_table_t *ht, - char **nodes, char **ppn) + char **nodes, char **ppn, + uint32_t flags) { pmix_status_t rc; pmix_value_t *val; @@ -199,6 +220,8 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, bool updated; pmix_kval_t *kp2; char **procs; + uint32_t totalprocs=0; + bool localldr; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "[%s:%d] gds:hash:store_map", @@ -210,6 +233,22 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, return PMIX_ERR_BAD_PARAM; } + /* if they didn't provide the number of nodes, then + * compute it from the list of nodes */ + if (!(PMIX_HASH_NUM_NODES & flags)) { + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_NUM_NODES); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_UINT32; + kp2->value->data.uint32 = pmix_argv_count(nodes); + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + } + for (n=0; NULL != nodes[n]; n++) { /* check and see if we already have data for this node */ val = NULL; @@ -227,18 +266,22 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, } iptr = (pmix_info_t*)val->data.darray->array; updated = false; + localldr = false; for (m=0; m < val->data.darray->size; m++) { - if (0 == strncmp(iptr[m].key, PMIX_LOCAL_PEERS, PMIX_MAX_KEYLEN)) { + if (PMIX_CHECK_KEY(&iptr[m], PMIX_LOCAL_PEERS)) { /* we will update this entry */ if (NULL != iptr[m].value.data.string) { free(iptr[m].value.data.string); } iptr[m].value.data.string = strdup(ppn[n]); - updated = true; - break; + updated = true; // no need to add the local_peers to the array + } else if (PMIX_CHECK_KEY(&iptr[m], PMIX_LOCALLDR)) { + rank = strtoul(ppn[n], NULL, 10); + iptr[m].value.data.rank = rank; + localldr = true; // no need to add localldr to the array } } - if (!updated) { + if (!updated || !localldr) { /* append this entry to the current data */ kp2 = PMIX_NEW(pmix_kval_t); if (NULL == kp2) { @@ -257,7 +300,18 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, return PMIX_ERR_NOMEM; } kp2->value->data.darray->type = PMIX_INFO; - kp2->value->data.darray->size = val->data.darray->size + 1; + /* if we didn't update the local leader, then we will + * add it here */ + m = 0; + if (!localldr) { + kp2->value->data.darray->size = val->data.darray->size + 1; + ++m; + } + /* if they didn't update the local peers, then we add it here */ + if (!updated) { + kp2->value->data.darray->size = val->data.darray->size + 1; + ++m; + } PMIX_INFO_CREATE(info, kp2->value->data.darray->size); if (NULL == info) { PMIX_RELEASE(kp2); @@ -267,7 +321,15 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, for (m=0; m < val->data.darray->size; m++) { PMIX_INFO_XFER(&info[m], &iptr[m]); } - PMIX_INFO_LOAD(&info[kp2->value->data.darray->size-1], PMIX_LOCAL_PEERS, ppn[n], PMIX_STRING); + if (!updated) { + PMIX_INFO_LOAD(&info[kp2->value->data.darray->size-m], PMIX_LOCAL_PEERS, ppn[n], PMIX_STRING); + --m; + } + if (!localldr) { + rank = strtoul(ppn[n], NULL, 10); + PMIX_INFO_LOAD(&info[kp2->value->data.darray->size-m], PMIX_LOCALLDR, &rank, PMIX_PROC_RANK); + --m; + } kp2->value->data.darray->array = info; if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) { PMIX_ERROR_LOG(rc); @@ -295,14 +357,16 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, return PMIX_ERR_NOMEM; } kp2->value->data.darray->type = PMIX_INFO; - PMIX_INFO_CREATE(info, 1); + PMIX_INFO_CREATE(info, 2); if (NULL == info) { PMIX_RELEASE(kp2); return PMIX_ERR_NOMEM; } PMIX_INFO_LOAD(&info[0], PMIX_LOCAL_PEERS, ppn[n], PMIX_STRING); + rank = strtoul(ppn[n], NULL, 10); + PMIX_INFO_LOAD(&info[1], PMIX_LOCALLDR, &rank, PMIX_PROC_RANK); kp2->value->data.darray->array = info; - kp2->value->data.darray->size = 1; + kp2->value->data.darray->size = 2; if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(kp2); @@ -313,6 +377,7 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, /* split the list of procs so we can store their * individual location data */ procs = pmix_argv_split(ppn[n], ','); + totalprocs += pmix_argv_count(procs); for (m=0; NULL != procs[m]; m++) { /* store the hostname for each proc */ kp2 = PMIX_NEW(pmix_kval_t); @@ -327,13 +392,56 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, pmix_argv_free(procs); return rc; } - PMIX_RELEASE(kp2); + PMIX_RELEASE(kp2); // maintain acctg + if (!(PMIX_HASH_PROC_DATA & flags)) { + /* add an entry for the nodeid */ + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_NODEID); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_UINT32; + kp2->value->data.uint32 = n; + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, rank, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + pmix_argv_free(procs); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + /* add an entry for the local rank */ + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_LOCAL_RANK); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_UINT16; + kp2->value->data.uint16 = m; + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, rank, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + pmix_argv_free(procs); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + /* add an entry for the node rank - for now, we assume + * only the one job is running */ + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_NODE_RANK); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_UINT16; + kp2->value->data.uint16 = m; + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, rank, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + pmix_argv_free(procs); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + } } pmix_argv_free(procs); } /* store the comma-delimited list of nodes hosting - * procs in this nspace */ + * procs in this nspace in case someone using PMIx v2 + * requests it */ kp2 = PMIX_NEW(pmix_kval_t); kp2->key = strdup(PMIX_NODE_LIST); kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); @@ -344,14 +452,50 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, PMIX_RELEASE(kp2); return rc; } + PMIX_RELEASE(kp2); // maintain acctg + + /* if they didn't provide the job size, compute it as + * being the number of provided procs (i.e., size of + * ppn list) */ + if (!(PMIX_HASH_JOB_SIZE & flags)) { + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_JOB_SIZE); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_UINT32; + kp2->value->data.uint32 = totalprocs; + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + } + + /* if they didn't provide a value for max procs, just + * assume it is the same as the number of procs in the + * job and store it */ + if (!(PMIX_HASH_MAX_PROCS & flags)) { + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_MAX_PROCS); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_UINT32; + kp2->value->data.uint32 = totalprocs; + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + } + return PMIX_SUCCESS; } -pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, +pmix_status_t hash_cache_job_info(struct pmix_namespace_t *ns, pmix_info_t info[], size_t ninfo) { - pmix_nspace_t *nptr = (pmix_nspace_t*)ns; + pmix_namespace_t *nptr = (pmix_namespace_t*)ns; pmix_hash_trkr_t *trk, *t; pmix_hash_table_t *ht; pmix_kval_t *kp2, *kvptr; @@ -361,6 +505,7 @@ pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, pmix_rank_t rank; pmix_status_t rc=PMIX_SUCCESS; size_t n, j, size, len; + uint32_t flags = 0; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "[%s:%d] gds:hash:cache_job_info for nspace %s", @@ -397,34 +542,33 @@ pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, ht = &trk->internal; for (n=0; n < ninfo; n++) { if (0 == strcmp(info[n].key, PMIX_NODE_MAP)) { + /* store the node map itself since that is + * what v3 uses */ + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_NODE_MAP); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_STRING; + kp2->value->data.string = strdup(info[n].value.data.string); + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + /* parse the regex to get the argv array of node names */ if (PMIX_SUCCESS != (rc = pmix_preg.parse_nodes(info[n].value.data.string, &nodes))) { PMIX_ERROR_LOG(rc); goto release; } - /* if we have already found the proc map, then parse - * and store the detailed map */ - if (NULL != procs) { - if (PMIX_SUCCESS != (rc = store_map(ht, nodes, procs))) { - PMIX_ERROR_LOG(rc); - goto release; - } - } } else if (0 == strcmp(info[n].key, PMIX_PROC_MAP)) { /* parse the regex to get the argv array containing proc ranks on each node */ if (PMIX_SUCCESS != (rc = pmix_preg.parse_procs(info[n].value.data.string, &procs))) { PMIX_ERROR_LOG(rc); goto release; } - /* if we have already recv'd the node map, then parse - * and store the detailed map */ - if (NULL != nodes) { - if (PMIX_SUCCESS != (rc = store_map(ht, nodes, procs))) { - PMIX_ERROR_LOG(rc); - goto release; - } - } } else if (0 == strcmp(info[n].key, PMIX_PROC_DATA)) { + flags |= PMIX_HASH_PROC_DATA; /* an array of data pertaining to a specific proc */ if (PMIX_DATA_ARRAY != info[n].value.type) { PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); @@ -513,9 +657,16 @@ pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, PMIX_RELEASE(kp2); goto release; } - /* if this is the job size, then store it */ - if (0 == strncmp(info[n].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN)) { + PMIX_RELEASE(kp2); // maintain acctg + /* if this is the job size, then store it in + * the nptr tracker and flag that we were given it */ + if (PMIX_CHECK_KEY(&info[n], PMIX_JOB_SIZE)) { nptr->nprocs = info[n].value.data.uint32; + flags |= PMIX_HASH_JOB_SIZE; + } else if (PMIX_CHECK_KEY(&info[n], PMIX_NUM_NODES)) { + flags |= PMIX_HASH_NUM_NODES; + } else if (PMIX_CHECK_KEY(&info[n], PMIX_MAX_PROCS)) { + flags |= PMIX_HASH_MAX_PROCS; } } } @@ -542,10 +693,22 @@ pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, PMIX_RELEASE(kp2); break; } + PMIX_RELEASE(kp2); // maintain acctg } trk->gdata_added = true; } + /* we must have the proc AND node maps */ + if (NULL == procs || NULL == nodes) { + rc = PMIX_ERR_NOT_FOUND; + goto release; + } + + if (PMIX_SUCCESS != (rc = store_map(ht, nodes, procs, flags))) { + PMIX_ERROR_LOG(rc); + goto release; + } + release: if (NULL != nodes) { pmix_argv_free(nodes); @@ -557,7 +720,7 @@ pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, } static pmix_status_t register_info(pmix_peer_t *peer, - pmix_nspace_t *ns, + pmix_namespace_t *ns, pmix_buffer_t *reply) { pmix_hash_trkr_t *trk, *t; @@ -613,25 +776,24 @@ static pmix_status_t register_info(pmix_peer_t *peer, for (rank=0; rank < ns->nprocs; rank++) { val = NULL; rc = pmix_hash_fetch(ht, rank, NULL, &val); - if (PMIX_SUCCESS != rc) { + if (PMIX_SUCCESS != rc && PMIX_ERR_PROC_ENTRY_NOT_FOUND != rc) { PMIX_ERROR_LOG(rc); if (NULL != val) { PMIX_VALUE_RELEASE(val); } return rc; } - if (NULL == val) { - return PMIX_ERR_NOT_FOUND; - } PMIX_CONSTRUCT(&buf, pmix_buffer_t); PMIX_BFROPS_PACK(rc, peer, &buf, &rank, 1, PMIX_PROC_RANK); - info = (pmix_info_t*)val->data.darray->array; - ninfo = val->data.darray->size; - for (n=0; n < ninfo; n++) { - kv.key = info[n].key; - kv.value = &info[n].value; - PMIX_BFROPS_PACK(rc, peer, &buf, &kv, 1, PMIX_KVAL); + if (NULL != val) { + info = (pmix_info_t*)val->data.darray->array; + ninfo = val->data.darray->size; + for (n=0; n < ninfo; n++) { + kv.key = info[n].key; + kv.value = &info[n].value; + PMIX_BFROPS_PACK(rc, peer, &buf, &kv, 1, PMIX_KVAL); + } } kv.key = PMIX_PROC_BLOB; kv.value = &blob; @@ -649,18 +811,19 @@ static pmix_status_t register_info(pmix_peer_t *peer, } /* the purpose of this function is to pack the job-level - * info stored in the pmix_nspace_t into a buffer and send + * info stored in the pmix_namespace_t into a buffer and send * it to the given client */ static pmix_status_t hash_register_job_info(struct pmix_peer_t *pr, pmix_buffer_t *reply) { pmix_peer_t *peer = (pmix_peer_t*)pr; - pmix_nspace_t *ns = peer->nptr; + pmix_namespace_t *ns = peer->nptr; char *msg; pmix_status_t rc; pmix_hash_trkr_t *trk, *t2; - if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { /* this function is only available on servers */ PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); return PMIX_ERR_NOT_SUPPORTED; @@ -760,7 +923,8 @@ static pmix_status_t hash_store_job_info(const char *nspace, "[%s:%u] pmix:gds:hash store job info for nspace %s", pmix_globals.myid.nspace, pmix_globals.myid.rank, nspace); - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { /* this function is NOT available on servers */ PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); return PMIX_ERR_NOT_SUPPORTED; @@ -915,7 +1079,7 @@ static pmix_status_t hash_store_job_info(const char *nspace, PMIX_DESTRUCT(&buf2); return rc; } - PMIX_RELEASE(kp2); + PMIX_RELEASE(kp2); // maintain acctg } else { /* nope - so add this by itself */ kp2 = PMIX_NEW(pmix_kval_t); @@ -947,7 +1111,7 @@ static pmix_status_t hash_store_job_info(const char *nspace, PMIX_DESTRUCT(&buf2); return rc; } - PMIX_RELEASE(kp2); + PMIX_RELEASE(kp2); // maintain acctg } /* split the list of procs so we can store their * individual location data */ @@ -970,7 +1134,7 @@ static pmix_status_t hash_store_job_info(const char *nspace, pmix_argv_free(procs); return rc; } - PMIX_RELEASE(kp2); + PMIX_RELEASE(kp2); // maintain acctg } pmix_argv_free(procs); PMIX_DESTRUCT(&kv); @@ -991,7 +1155,7 @@ static pmix_status_t hash_store_job_info(const char *nspace, PMIX_DESTRUCT(&buf2); return rc; } - PMIX_RELEASE(kp2); + PMIX_RELEASE(kp2); // maintain acctg } /* cleanup */ PMIX_DESTRUCT(&buf2); @@ -1046,10 +1210,10 @@ static pmix_status_t hash_store(const pmix_proc_t *proc, pmix_kval_t *kp; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, - "[%s:%d] gds:hash:hash_store for proc [%s:%d] key %s scope %s", + "[%s:%d] gds:hash:hash_store for proc [%s:%d] key %s type %s scope %s", pmix_globals.myid.nspace, pmix_globals.myid.rank, proc->nspace, proc->rank, kv->key, - PMIx_Scope_string(scope)); + PMIx_Data_type_string(kv->value->type), PMIx_Scope_string(scope)); if (NULL == kv->key) { return PMIX_ERR_BAD_PARAM; @@ -1093,8 +1257,10 @@ static pmix_status_t hash_store(const pmix_proc_t *proc, } if (PMIX_SUCCESS != (rc = pmix_hash_store(&trk->internal, proc->rank, kp))) { PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp); return rc; } + PMIX_RELEASE(kp); // maintain accounting } } @@ -1154,11 +1320,18 @@ static pmix_status_t hash_store(const pmix_proc_t *proc, * host has received data from some other peer. It therefore * always contains data solely from remote procs, and we * shall store it accordingly */ -static pmix_status_t hash_store_modex(struct pmix_nspace_t *nspace, +static pmix_status_t hash_store_modex(struct pmix_namespace_t *nspace, pmix_list_t *cbs, - pmix_byte_object_t *bo) + pmix_buffer_t *buf) { + return pmix_gds_base_store_modex(nspace, cbs, buf, _hash_store_modex, NULL); +} + +static pmix_status_t _hash_store_modex(void * cbdata, + struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_byte_object_t *bo) { - pmix_nspace_t *ns = (pmix_nspace_t*)nspace; + pmix_namespace_t *ns = (pmix_namespace_t*)nspace; pmix_hash_trkr_t *trk, *t; pmix_status_t rc = PMIX_SUCCESS; int32_t cnt; @@ -1285,7 +1458,6 @@ static pmix_status_t hash_fetch(const pmix_proc_t *proc, val = NULL; rc = pmix_hash_fetch(ht, PMIX_RANK_WILDCARD, NULL, &val); if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); if (NULL != val) { PMIX_VALUE_RELEASE(val); } @@ -1299,7 +1471,6 @@ static pmix_status_t hash_fetch(const pmix_proc_t *proc, if (PMIX_DATA_ARRAY != val->type || NULL == val->data.darray || PMIX_INFO != val->data.darray->type) { - PMIX_ERROR_LOG(PMIX_ERR_INVALID_VAL); PMIX_VALUE_RELEASE(val); return PMIX_ERR_INVALID_VAL; } @@ -1444,7 +1615,17 @@ static pmix_status_t nspace_add(const char *nspace, static pmix_status_t nspace_del(const char *nspace) { - /* we don't need to do anything here */ + pmix_hash_trkr_t *t; + + /* find the hash table for this nspace */ + PMIX_LIST_FOREACH(t, &myhashes, pmix_hash_trkr_t) { + if (0 == strcmp(nspace, t->ns)) { + /* release it */ + pmix_list_remove_item(&myhashes, &t->super); + PMIX_RELEASE(t); + break; + } + } return PMIX_SUCCESS; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pdl/base/base.h b/opal/mca/pmix/pmix2x/pmix/src/mca/pdl/base/base.h index 2cb37256885..b061f7ea6a1 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pdl/base/base.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pdl/base/base.h @@ -3,7 +3,7 @@ * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -26,7 +26,7 @@ BEGIN_C_DECLS /** * Globals */ -extern pmix_mca_base_framework_t pmix_pdl_base_framework; +PMIX_EXPORT extern pmix_mca_base_framework_t pmix_pdl_base_framework; extern pmix_pdl_base_component_t *pmix_pdl_base_selected_component; extern pmix_pdl_base_module_t *pmix_pdl; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pdl/configure.m4 b/opal/mca/pmix/pmix2x/pmix/src/mca/pdl/configure.m4 index c5082065b23..1e749df5b2d 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pdl/configure.m4 +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pdl/configure.m4 @@ -1,8 +1,8 @@ dnl -*- shell-script -*- dnl dnl Copyright (c) 2010-2015 Cisco Systems, Inc. All rights reserved. -dnl Copyright (c) 2016-2017 Intel, Inc. All rights reserved. -dnl Copyright (c) 2016 Research Organization for Information Science +dnl Copyright (c) 2016-2019 Intel, Inc. All rights reserved. +dnl Copyright (c) 2016-2019 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. dnl $COPYRIGHT$ dnl @@ -27,7 +27,7 @@ AC_DEFUN([MCA_pmix_pdl_CONFIG],[ # (we still need to configure them all so that things like "make # dist" work", but we just want the MCA system to (artificially) # conclude that it can't build any of the components. - AS_IF([test "$enable_dlopen" = "no"], + AS_IF([test $PMIX_ENABLE_DLOPEN_SUPPORT -eq 0], [want_pdl=0], [want_pdl=1]) MCA_CONFIGURE_FRAMEWORK([pdl], [$want_pdl]) @@ -35,7 +35,7 @@ AC_DEFUN([MCA_pmix_pdl_CONFIG],[ # If we found no suitable static pdl component and dlopen support # was not specifically disabled, this is an error. AS_IF([test "$MCA_pmix_pdl_STATIC_COMPONENTS" = "" && \ - test "$enable_dlopen" != "no"], + test $PMIX_ENABLE_DLOPEN_SUPPORT -eq 1], [AC_MSG_WARN([Did not find a suitable static pmix pdl component]) AC_MSG_WARN([You might need to install libltld (and its headers) or]) AC_MSG_WARN([specify --disable-dlopen to configure.]) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/bsdx_ipv4/pif_bsdx.c b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/bsdx_ipv4/pif_bsdx.c index 9157d546616..b5b49e035b2 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/bsdx_ipv4/pif_bsdx.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/bsdx_ipv4/pif_bsdx.c @@ -1,6 +1,7 @@ /* * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -35,23 +36,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include @@ -173,7 +158,7 @@ static int if_bsdx_open(void) /* fill values into the pmix_pif_t */ memcpy(&a4, &(sin_addr->sin_addr), sizeof(struct in_addr)); - strncpy(intf->if_name, cur_ifaddrs->ifa_name, IF_NAMESIZE); + pmix_strncpy(intf->if_name, cur_ifaddrs->ifa_name, IF_NAMESIZE-1); intf->if_index = pmix_list_get_size(&pmix_if_list) + 1; ((struct sockaddr_in*) &intf->if_addr)->sin_addr = a4; ((struct sockaddr_in*) &intf->if_addr)->sin_family = AF_INET; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/bsdx_ipv6/pif_bsdx_ipv6.c b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/bsdx_ipv6/pif_bsdx_ipv6.c index 2dac2550d37..699984369d1 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/bsdx_ipv6/pif_bsdx_ipv6.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/bsdx_ipv6/pif_bsdx_ipv6.c @@ -1,6 +1,7 @@ /* * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -35,23 +36,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include @@ -198,7 +183,7 @@ static int if_bsdx_ipv6_open(void) return PMIX_ERR_OUT_OF_RESOURCE; } intf->af_family = AF_INET6; - strncpy(intf->if_name, cur_ifaddrs->ifa_name, IF_NAMESIZE); + pmix_strncpy(intf->if_name, cur_ifaddrs->ifa_name, IF_NAMESIZE-1); intf->if_index = pmix_list_get_size(&pmix_if_list) + 1; ((struct sockaddr_in6*) &intf->if_addr)->sin6_addr = a6; ((struct sockaddr_in6*) &intf->if_addr)->sin6_family = AF_INET6; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c index 2f240f9d8a1..ec5261fbda4 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c @@ -1,6 +1,7 @@ /* * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -35,23 +36,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include @@ -95,7 +80,9 @@ static int if_linux_ipv6_open(void) { FILE *f; if ((f = fopen("/proc/net/if_inet6", "r"))) { - char ifname[IF_NAMESIZE]; + /* IF_NAMESIZE is normally 16 on Linux, + but the next scanf allows up to 21 bytes */ + char ifname[21]; unsigned int idx, pfxlen, scope, dadstat; struct in6_addr a6; int iter; @@ -142,7 +129,7 @@ static int if_linux_ipv6_open(void) } /* now construct the pmix_pif_t */ - strncpy(intf->if_name, ifname, IF_NAMESIZE); + pmix_strncpy(intf->if_name, ifname, IF_NAMESIZE-1); intf->if_index = pmix_list_get_size(&pmix_if_list)+1; intf->if_kernel_index = (uint16_t) idx; ((struct sockaddr_in6*) &intf->if_addr)->sin6_addr = a6; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/pif.h b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/pif.h index 29c75b869c7..1582122122f 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/pif.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/pif.h @@ -3,7 +3,7 @@ * Copyright (c) 2010-2013 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -40,23 +40,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/posix_ipv4/pif_posix.c b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/posix_ipv4/pif_posix.c index 3c2e4603a66..7f5deade20a 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/posix_ipv4/pif_posix.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/posix_ipv4/pif_posix.c @@ -4,7 +4,7 @@ * Copyright (c) 2013 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -39,23 +39,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include @@ -265,7 +249,7 @@ static int if_posix_open(void) /* copy entry over into our data structure */ memset(intf->if_name, 0, sizeof(intf->if_name)); - strncpy(intf->if_name, ifr->ifr_name, sizeof(intf->if_name) - 1); + pmix_strncpy(intf->if_name, ifr->ifr_name, sizeof(intf->if_name) - 1); intf->if_flags = ifr->ifr_flags; /* every new address gets its own internal if_index */ @@ -333,7 +317,8 @@ static int if_posix_open(void) } intf->ifmtu = ifr->ifr_mtu; #endif - + pmix_output_verbose(1, pmix_pif_base_framework.framework_output, + "adding interface %s", intf->if_name); pmix_list_append(&pmix_if_list, &(intf->super)); } free(ifconf.ifc_req); diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/solaris_ipv6/pif_solaris_ipv6.c b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/solaris_ipv6/pif_solaris_ipv6.c index 7403cebf0e2..6872b5c0c2d 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/solaris_ipv6/pif_solaris_ipv6.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/solaris_ipv6/pif_solaris_ipv6.c @@ -3,7 +3,7 @@ * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -146,8 +146,8 @@ static int if_solaris_ipv6_open(void) i += sizeof (*lifreq)) { lifreq = (struct lifreq *)((caddr_t)lifconf.lifc_buf + i); - strncpy (lifquery.lifr_name, lifreq->lifr_name, - sizeof (lifquery.lifr_name)); + pmix_strncpy (lifquery.lifr_name, lifreq->lifr_name, + sizeof (lifquery.lifr_name)-1); /* lookup kernel index */ error = ioctl (sd, SIOCGLIFINDEX, &lifquery); @@ -190,7 +190,7 @@ static int if_solaris_ipv6_open(void) } intf->af_family = AF_INET6; - strncpy (intf->if_name, lifreq->lifr_name, IF_NAMESIZE); + pmix_strncpy (intf->if_name, lifreq->lifr_name, IF_NAMESIZE-1); intf->if_index = pmix_list_get_size(&pmix_if_list)+1; memcpy(&intf->if_addr, my_addr, sizeof (*my_addr)); intf->if_mask = 64; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pinstalldirs/base/base.h b/opal/mca/pmix/pmix2x/pmix/src/mca/pinstalldirs/base/base.h index a7592f0927d..8e467f1f036 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pinstalldirs/base/base.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pinstalldirs/base/base.h @@ -3,7 +3,7 @@ * reserved. * Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -27,7 +27,7 @@ BEGIN_C_DECLS /** * Framework structure declaration */ -extern pmix_mca_base_framework_t pmix_pinstalldirs_base_framework; +PMIX_EXPORT extern pmix_mca_base_framework_t pmix_pinstalldirs_base_framework; /* Just like pmix_pinstall_dirs_expand() (see pinstalldirs.h), but will also insert the value of the environment variable $PMIX_DESTDIR, if diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pinstalldirs/config/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/pinstalldirs/config/Makefile.am index d05743fb5f6..cc77e74f074 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pinstalldirs/config/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pinstalldirs/config/Makefile.am @@ -4,7 +4,7 @@ # Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2009 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. -# Copyright (c) 2016 Intel, Inc. All rights reserved. +# Copyright (c) 2016-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -19,4 +19,4 @@ libmca_pinstalldirs_config_la_SOURCES = \ # This file is generated; we do not want to include it in the tarball nodist_libmca_pinstalldirs_config_la_SOURCES = \ - install_dirs.h + pinstall_dirs.h diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pinstalldirs/pinstalldirs.h b/opal/mca/pmix/pmix2x/pmix/src/mca/pinstalldirs/pinstalldirs.h index 4af02a5e214..404895c6ff7 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pinstalldirs/pinstalldirs.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pinstalldirs/pinstalldirs.h @@ -2,7 +2,7 @@ /* * Copyright (c) 2006-2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -54,7 +54,7 @@ struct pmix_pinstall_dirs_t { typedef struct pmix_pinstall_dirs_t pmix_pinstall_dirs_t; /* Install directories. Only available after pmix_init() */ -extern pmix_pinstall_dirs_t pmix_pinstall_dirs; +PMIX_EXPORT extern pmix_pinstall_dirs_t pmix_pinstall_dirs; /** * Expand out path variables (such as ${prefix}) in the input string diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/base/base.h b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/base/base.h index 24b9349a349..db1705c093b 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/base/base.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/base/base.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/base/pnet_base_fns.c b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/base/pnet_base_fns.c index 3572fdf8e82..b307a78e361 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/base/pnet_base_fns.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/base/pnet_base_fns.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * @@ -90,7 +90,8 @@ pmix_status_t pmix_pnet_base_setup_fork(const pmix_proc_t *peer, char ***env) PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { if (NULL != active->module->setup_fork) { - if (PMIX_SUCCESS != (rc = active->module->setup_fork(peer, env))) { + rc = active->module->setup_fork(peer, env); + if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_AVAILABLE != rc) { return rc; } } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/base/pnet_base_frame.c b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/base/pnet_base_frame.c index 0dd5410daef..93706f276ce 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/base/pnet_base_frame.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/base/pnet_base_frame.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/Makefile.am index 1223b43eca4..fe01cde836e 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. # $COPYRIGHT$ @@ -49,6 +49,9 @@ mcacomponent_LTLIBRARIES = $(component) mca_pnet_opa_la_SOURCES = $(component_sources) mca_pnet_opa_la_LIBADD = $(pnet_opa_LIBS) mca_pnet_opa_la_LDFLAGS = -module -avoid-version $(pnet_opa_LDFLAGS) +if NEED_LIBPMIX +mca_pnet_opa_la_LIBADD += $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_pnet_opa_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/configure.m4 b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/configure.m4 index 4d6d109a9dd..2273fabe47e 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/configure.m4 +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/configure.m4 @@ -12,7 +12,7 @@ # All rights reserved. # Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2013 Sandia National Laboratories. All rights reserved. -# Copyright (c) 2014-2016 Intel, Inc. All rights reserved. +# Copyright (c) 2014-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -30,6 +30,74 @@ AC_DEFUN([MCA_pmix_pnet_opa_CONFIG],[ [pnet_opa_happy="yes"], [pnet_opa_happy="no"]) + AC_ARG_WITH([opamgt], + [AC_HELP_STRING([--with-opamgt(=DIR)], + [Build OmniPath Fabric Management support (optionally adding DIR/include, DIR/include/opamgt, DIR/lib, and DIR/lib64 to the search path for headers and libraries])], [], [with_opamgt=no]) + + AC_ARG_WITH([opamgt-libdir], + [AC_HELP_STRING([--with-opamgt-libdir=DIR], + [Search for OmniPath Fabric Management libraries in DIR])]) + + pmix_check_opamgt_save_CPPFLAGS="$CPPFLAGS" + pmix_check_opamgt_save_LDFLAGS="$LDFLAGS" + pmix_check_opamgt_save_LIBS="$LIBS" + + pmix_check_opamgt_libdir= + pmix_check_opamgt_dir= + + AC_MSG_CHECKING([if opamgt requested]) + AS_IF([test "$with_opamgt" == "no"], + [AC_MSG_RESULT([no]) + pmix_check_opamgt_happy=no], + [AC_MSG_RESULT([yes]) + PMIX_CHECK_WITHDIR([opamgt-libdir], [$with_opamgt_libdir], [libopamgt.*]) + AS_IF([test ! -z "$with_opamgt" && test "$with_opamgt" != "yes"], + [pmix_check_opamgt_dir="$with_opamgt" + AS_IF([test ! -d "$pmix_check_opamgt_dir" || test ! -f "$pmix_check_opamgt_dir/opamgt.h"], + [$pmix_check_opamgt_dir=$pmix_check_opamgt_dir/include + AS_IF([test ! -d "$pmix_check_opamgt_dir" || test ! -f "$pmix_check_opamgt_dir/opamgt.h"], + [$pmix_check_opamgt_dir=$pmix_check_opamgt_dir/opamgt + AS_IF([test ! -d "$pmix_check_opamgt_dir" || test ! -f "$pmix_check_opamgt_dir/opamgt.h"], + [AC_MSG_WARN([OmniPath Fabric Management support requested, but]) + AC_MSG_WARN([required header file opamgt.h not found. Locations tested:]) + AC_MSG_WARN([ $with_opamgt]) + AC_MSG_WARN([ $with_opamgt/include]) + AC_MSG_WARN([ $with_opamgt/include/opamgt]) + AC_MSG_ERROR([Cannot continue])])])])], + [pmix_check_opamgt_dir="/usr/include/opamgt"]) + + AS_IF([test ! -z "$with_opamgt_libdir" && test "$with_opamgt_libdir" != "yes"], + [pmix_check_opamgt_libdir="$with_opamgt_libdir"]) + + # no easy way to check this, so let's ensure that the + # full opamgt install was done, including the iba support + AS_IF([test ! -d "$pmix_check_opamgt_dir/iba" || test ! -f "$pmix_check_opamgt_dir/iba/vpi.h"], + [pmix_check_opamgt_happy="no"], + [PMIX_CHECK_PACKAGE([pnet_opamgt], + [opamgt.h], + [opamgt], + [omgt_query_sa], + [], + [$pmix_check_opamgt_dir], + [$pmix_check_opamgt_libdir], + [pmix_check_opamgt_happy="yes" + pnet_opa_CFLAGS="$pnet_opa_CFLAGS $pnet_opamgt_CFLAGS" + pnet_opa_CPPFLAGS="$pnet_opa_CPPFLAGS $pnet_opamgt_CPPFLAGS" + pnet_opa_LDFLAGS="$pnet_opa_LDFLAGS $pnet_opamgt_LDFLAGS" + pnet_opa_LIBS="$pnet_opa_LIBS $pnet_opamgt_LIBS"], + [pmix_check_opamgt_happy="no"])]) + ]) + + AS_IF([test "$pmix_check_opamgt_happy" = "yes"], + [pmix_want_opamgt=1], + [pmix_want_opamgt=0]) + AC_DEFINE_UNQUOTED([PMIX_WANT_OPAMGT], [$pmix_want_opamgt], + [Whether or not to include OmniPath Fabric Manager support]) + + CPPFLAGS="$pmix_check_opamgt_save_CPPFLAGS" + LDFLAGS="$pmix_check_opamgt_save_LDFLAGS" + LIBS="$pmix_check_opamgt_save_LIBS" + AS_IF([test "$pnet_opa_happy" = "yes"], [$1], [$2]) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/pnet_opa.c b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/pnet_opa.c index e3740b5a640..0684d3b86d1 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/pnet_opa.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/pnet_opa.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * * $COPYRIGHT$ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/pnet_opa.h b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/pnet_opa.h index 278c894863a..0ca3ba5e39b 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/pnet_opa.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/pnet_opa.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * * $COPYRIGHT$ * diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/pnet_opa_component.c b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/pnet_opa_component.c index 7d07c400f6e..92dec15ca6b 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/pnet_opa_component.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/opa/pnet_opa_component.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/pnet.h b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/pnet.h index dedb1eb63f5..2afdf76c9d9 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/pnet.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pnet/pnet.h @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/preg/base/preg_base_frame.c b/opal/mca/pmix/pmix2x/pmix/src/mca/preg/base/preg_base_frame.c index dbf551ea640..5f6c5afce7b 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/preg/base/preg_base_frame.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/preg/base/preg_base_frame.c @@ -11,9 +11,10 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2019 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -98,6 +99,7 @@ static void rvcon(pmix_regex_value_t *p) p->prefix = NULL; p->suffix = NULL; p->num_digits = 0; + p->skip = false; PMIX_CONSTRUCT(&p->ranges, pmix_list_t); } static void rvdes(pmix_regex_value_t *p) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/preg/native/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/preg/native/Makefile.am index fa51393622f..607dcdb0c96 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/preg/native/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/preg/native/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -44,6 +44,9 @@ mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component) mca_preg_native_la_SOURCES = $(component_sources) mca_preg_native_la_LDFLAGS = -module -avoid-version +if NEED_LIBPMIX +mca_preg_native_la_LIBADD = $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_preg_native_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/preg/native/preg_native.c b/opal/mca/pmix/pmix2x/pmix/src/mca/preg/native/preg_native.c index 5535296b3d0..0c9d6188a0d 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/preg/native/preg_native.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/preg/native/preg_native.c @@ -1,6 +1,6 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 IBM Corporation. All rights reserved. * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * @@ -152,9 +152,22 @@ static pmix_status_t generate_node_regex(const char *input, suffix = NULL; numdigits = (int)strlen(&vptr[startnum]); } + /* is this value already on our list? */ found = false; PMIX_LIST_FOREACH(vreg, &vids, pmix_regex_value_t) { + // The regex must preserve ordering of the values. + // If we disqualified this entry in a previous check then exclude it + // from future checks as well. This will prevent a later entry from + // being 'pulled forward' accidentally. For example, given: + // "a28n01,a99n02,a28n02" + // Without this 'skip' the loop would have 'a28n02' combine with + // 'a28n01' jumping over the 'a99n02' entry, and thus not preserving + // the order of the list when the regex is unpacked. + if( vreg->skip ) { + continue; + } + if (0 < strlen(prefix) && NULL == vreg->prefix) { continue; } @@ -163,6 +176,7 @@ static pmix_status_t generate_node_regex(const char *input, } if (0 < strlen(prefix) && NULL != vreg->prefix && 0 != strcmp(prefix, vreg->prefix)) { + vreg->skip = true; continue; } if (NULL == suffix && NULL != vreg->suffix) { @@ -173,9 +187,11 @@ static pmix_status_t generate_node_regex(const char *input, } if (NULL != suffix && NULL != vreg->suffix && 0 != strcmp(suffix, vreg->suffix)) { + vreg->skip = true; continue; } if (numdigits != vreg->num_digits) { + vreg->skip = true; continue; } /* found a match - flag it */ @@ -501,27 +517,32 @@ static pmix_status_t parse_procs(const char *regexp, return rc; } -static void _resolve_peers(int sd, short args, void *cbdata) +static pmix_status_t resolve_peers(const char *nodename, + const char *nspace, + pmix_proc_t **procs, size_t *nprocs) { - pmix_cb_t *cb = (pmix_cb_t*)cbdata; + pmix_cb_t cb; pmix_status_t rc; pmix_kval_t *kv; pmix_proc_t proc; char **ptr; pmix_info_t *info; - pmix_proc_t *procs; - size_t ninfo, nprocs, n, j; + pmix_proc_t *p=NULL; + size_t ninfo, np=0, n, j; + PMIX_CONSTRUCT(&cb, pmix_cb_t); + + cb.key = strdup(nodename); /* this data isn't going anywhere, so we don't require a copy */ - cb->copy = false; + cb.copy = false; /* scope is irrelevant as the info we seek must be local */ - cb->scope = PMIX_SCOPE_UNDEF; + cb.scope = PMIX_SCOPE_UNDEF; /* let the proc point to the nspace */ - (void)strncpy(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; - cb->proc = &proc; + cb.proc = &proc; - PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, cb); + PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, &cb); if (PMIX_SUCCESS != rc) { if (PMIX_ERR_INVALID_NAMESPACE != rc) { PMIX_ERROR_LOG(rc); @@ -529,12 +550,12 @@ static void _resolve_peers(int sd, short args, void *cbdata) goto complete; } /* should just be the one value on the list */ - if (1 != pmix_list_get_size(&cb->kvs)) { + if (1 != pmix_list_get_size(&cb.kvs)) { PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); rc = PMIX_ERR_BAD_PARAM; goto complete; } - kv = (pmix_kval_t*)pmix_list_get_first(&cb->kvs); + kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs); /* the hostname used as a key with wildcard rank will return * a pmix_data_array_t of pmix_info_t structs */ if (NULL == kv->value || @@ -552,184 +573,103 @@ static void _resolve_peers(int sd, short args, void *cbdata) if (0 == strncmp(info[n].key, PMIX_LOCAL_PEERS, PMIX_MAX_KEYLEN)) { /* split the string */ ptr = pmix_argv_split(info[n].value.data.string, ','); - nprocs = pmix_argv_count(ptr); - PMIX_PROC_CREATE(procs, nprocs); - if (NULL == procs) { + np = pmix_argv_count(ptr); + PMIX_PROC_CREATE(p, np); + if (NULL == p) { rc = PMIX_ERR_NOMEM; pmix_argv_free(ptr); goto complete; } - for (j=0; j < nprocs; j++) { - (void)strncpy(procs[j].nspace, cb->pname.nspace, PMIX_MAX_NSLEN); - procs[j].rank = strtoul(ptr[j], NULL, 10); + for (j=0; j < np; j++) { + pmix_strncpy(p[j].nspace, nspace, PMIX_MAX_NSLEN); + p[j].rank = strtoul(ptr[j], NULL, 10); } - cb->procs = procs; - cb->nprocs = nprocs; rc = PMIX_SUCCESS; pmix_argv_free(ptr); - goto complete; + break; } } complete: - cb->status = rc; - if (NULL != cb->info) { - PMIX_INFO_FREE(cb->info, cb->ninfo); + if (NULL != cb.info) { + PMIX_INFO_FREE(cb.info, cb.ninfo); } - cb->pstatus = rc; - /* post the data so the receiving thread can acquire it */ - PMIX_POST_OBJECT(cb); - PMIX_WAKEUP_THREAD(&cb->lock); - return; -} - -static pmix_status_t resolve_peers(const char *nodename, - const char *nspace, - pmix_proc_t **procs, size_t *nprocs) -{ - pmix_cb_t *cb; - pmix_status_t rc; - pmix_proc_t proc; - - cb = PMIX_NEW(pmix_cb_t); - cb->key = (char*)nodename; - cb->pname.nspace = strdup(nspace); - - PMIX_THREADSHIFT(cb, _resolve_peers); - - /* wait for the result */ - PMIX_WAIT_THREAD(&cb->lock); - - /* if the nspace wasn't found, then we need to - * ask the server for that info */ - if (PMIX_ERR_INVALID_NAMESPACE == cb->status) { - (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); - proc.rank = PMIX_RANK_WILDCARD; - /* any key will suffice as it will bring down - * the entire data blob */ - rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, NULL); - if (PMIX_SUCCESS != rc) { - PMIX_RELEASE(cb); - return rc; - } - /* retry the fetch */ - cb->lock.active = true; - PMIX_THREADSHIFT(cb, _resolve_peers); - PMIX_WAIT_THREAD(&cb->lock); + if (NULL != cb.key) { + free(cb.key); + cb.key = NULL; } - *procs = cb->procs; - *nprocs = cb->nprocs; + PMIX_DESTRUCT(&cb); + *procs = p; + *nprocs = np; - rc = cb->status; - PMIX_RELEASE(cb); return rc; } -static void _resolve_nodes(int sd, short args, void *cbdata) +static pmix_status_t resolve_nodes(const char *nspace, + char **nodelist) { - pmix_cb_t *cb = (pmix_cb_t*)cbdata; + pmix_cb_t cb; pmix_status_t rc; pmix_kval_t *kv; pmix_proc_t proc; - /* create a pmix_info_t so we can pass the nspace - * into the fetch as a qualifier */ - PMIX_INFO_CREATE(cb->info, 1); - if (NULL == cb->info) { - cb->status = PMIX_ERR_NOMEM; - PMIX_POST_OBJECT(cb); - PMIX_WAKEUP_THREAD(&cb->lock); - return; - } - cb->ninfo = 1; - PMIX_INFO_LOAD(&cb->info[0], PMIX_NSPACE, cb->pname.nspace, PMIX_STRING); - /* tell the GDS what we want */ - cb->key = PMIX_NODE_LIST; - /* this data isn't going anywhere, so we don't require a copy */ - cb->copy = false; - /* scope is irrelevant as the info we seek must be local */ - cb->scope = PMIX_SCOPE_UNDEF; - /* put the nspace in the proc field */ - (void)strncpy(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN); - /* the info will be associated with PMIX_RANK_WILDCARD */ - proc.rank = PMIX_RANK_WILDCARD; - cb->proc = &proc; - - PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, cb); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - goto complete; - } - /* should just be the one value on the list */ - if (1 != pmix_list_get_size(&cb->kvs)) { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); - rc = PMIX_ERR_BAD_PARAM; - goto complete; - } - kv = (pmix_kval_t*)pmix_list_get_first(&cb->kvs); - /* the PMIX_NODE_LIST key is supposed to return a comma-delimited - * string of nodes in this - check that it did */ - if (NULL == kv->value || - PMIX_STRING != kv->value->type) { - PMIX_ERROR_LOG(PMIX_ERR_DATA_VALUE_NOT_FOUND); - rc = PMIX_ERR_DATA_VALUE_NOT_FOUND; - goto complete; - } - /* return the string */ - if (NULL != kv->value->data.string) { - cb->key = strdup(kv->value->data.string); - } - - complete: - cb->status = rc; - if (NULL != cb->info) { - PMIX_INFO_FREE(cb->info, cb->ninfo); - } - /* post the data so the receiving thread can acquire it */ - PMIX_POST_OBJECT(cb); - PMIX_WAKEUP_THREAD(&cb->lock); - return; -} + PMIX_CONSTRUCT(&cb, pmix_cb_t); -static pmix_status_t resolve_nodes(const char *nspace, - char **nodelist) -{ - pmix_cb_t *cb; - pmix_status_t rc; - pmix_proc_t proc; + /* setup default answer */ + *nodelist = NULL; - cb = PMIX_NEW(pmix_cb_t); - cb->pname.nspace = strdup(nspace); - - PMIX_THREADSHIFT(cb, _resolve_nodes); - - /* wait for the result */ - PMIX_WAIT_THREAD(&cb->lock); - - /* if the nspace wasn't found, then we need to - * ask the server for that info */ - if (PMIX_ERR_INVALID_NAMESPACE == cb->status) { - (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); - proc.rank = PMIX_RANK_WILDCARD; - /* any key will suffice as it will bring down - * the entire data blob */ - rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, NULL); - if (PMIX_SUCCESS != rc) { - PMIX_RELEASE(cb); - return rc; - } - /* retry the fetch */ - cb->lock.active = true; - PMIX_THREADSHIFT(cb, _resolve_nodes); - PMIX_WAIT_THREAD(&cb->lock); + /* create a pmix_info_t so we can pass the nspace + * into the fetch as a qualifier */ + PMIX_INFO_CREATE(cb.info, 1); + if (NULL == cb.info) { + PMIX_DESTRUCT(&cb); + return PMIX_ERR_NOMEM; } - /* the string we want is in the key field */ - *nodelist = cb->key; + cb.ninfo = 1; + PMIX_INFO_LOAD(&cb.info[0], PMIX_NSPACE, nspace, PMIX_STRING); - rc = cb->status; - PMIX_RELEASE(cb); - return rc; + /* tell the GDS what we want */ + cb.key = PMIX_NODE_MAP; + /* this data isn't going anywhere, so we don't require a copy */ + cb.copy = false; + /* scope is irrelevant as the info we seek must be local */ + cb.scope = PMIX_SCOPE_UNDEF; + /* put the nspace in the proc field */ + pmix_strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + /* the info will be associated with PMIX_RANK_WILDCARD */ + proc.rank = PMIX_RANK_WILDCARD; + cb.proc = &proc; + PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, &cb); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + /* should just be the one value on the list */ + if (1 != pmix_list_get_size(&cb.kvs)) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + rc = PMIX_ERR_BAD_PARAM; + goto complete; + } + kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs); + /* the PMIX_NODE_MAP key is supposed to return + * a regex string - check that it did */ + if (NULL == kv->value || + PMIX_STRING != kv->value->type) { + PMIX_ERROR_LOG(PMIX_ERR_DATA_VALUE_NOT_FOUND); + rc = PMIX_ERR_DATA_VALUE_NOT_FOUND; + goto complete; + } + /* return the string */ + if (NULL != kv->value->data.string) { + *nodelist = strdup(kv->value->data.string); + } + + complete: + if (NULL != cb.info) { + PMIX_INFO_FREE(cb.info, cb.ninfo); + } + return rc; } static pmix_status_t pmix_regex_extract_nodes(char *regexp, char ***names) @@ -953,7 +893,7 @@ static pmix_status_t regex_parse_value_range(char *base, char *range, for (found = false, i = 0; i < len; ++i) { if (isdigit((int) range[i])) { if (!found) { - start = atoi(range + i); + start = strtol(range + i, NULL, 10); found = true; break; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/preg/preg_types.h b/opal/mca/pmix/pmix2x/pmix/src/mca/preg/preg_types.h index 9f1b8a8ae51..572b321b04f 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/preg/preg_types.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/preg/preg_types.h @@ -12,7 +12,8 @@ * All rights reserved. * Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2019 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -51,6 +52,7 @@ typedef struct { char *suffix; int num_digits; pmix_list_t ranges; + bool skip; } pmix_regex_value_t; PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_regex_value_t); diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/Makefile.am new file mode 100644 index 00000000000..200176dd9d7 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/Makefile.am @@ -0,0 +1,56 @@ +# -*- makefile -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. +# Copyright (c) 2019 Mellanox Technologies, Inc. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +if MCA_BUILD_PSEC_DUMMY_HANDSHAKE + +headers = psec_dummy_handshake.h +sources = \ + psec_dummy_handshake_component.c \ + psec_dummy_handshake.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_pmix_psec_dummy_handshake_DSO +lib = +lib_sources = +component = mca_psec_dummy_handshake.la +component_sources = $(headers) $(sources) +else +lib = libmca_psec_dummy_handshake.la +lib_sources = $(headers) $(sources) +component = +component_sources = +endif + +mcacomponentdir = $(pmixlibdir) +mcacomponent_LTLIBRARIES = $(component) +mca_psec_dummy_handshake_la_SOURCES = $(component_sources) +mca_psec_dummy_handshake_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(lib) +libmca_psec_dummy_handshake_la_SOURCES = $(lib_sources) +libmca_psec_dummy_handshake_la_LDFLAGS = -module -avoid-version + +endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.c new file mode 100644 index 00000000000..ae1f9b62e59 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.c @@ -0,0 +1,170 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2019 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#ifdef HAVE_SYS_TYPES_H +#include +#endif + +#include + +#include "src/include/pmix_globals.h" +#include "src/util/error.h" +#include "src/util/output.h" + +#include "src/mca/psec/base/base.h" +#include "psec_dummy_handshake.h" + +#include "src/mca/ptl/base/base.h" + +#define PMIX_PSEC_DUMMY_HNDSHK_STR "PMIX_PSEC_DUMMY_HANDSHAKE_STRING" + +static pmix_status_t simple_init(void); +static void simple_finalize(void); +static pmix_status_t create_cred(struct pmix_peer_t *peer, + const pmix_info_t directives[], size_t ndirs, + pmix_info_t **info, size_t *ninfo, + pmix_byte_object_t *cred); +static pmix_status_t client_hndshk(int sd); +static pmix_status_t server_hndshk(int sd); + +pmix_psec_module_t pmix_dummy_handshake_module = { + .name = "dummy_handshake", + /** init/finalize */ + .init = simple_init, + .finalize = simple_finalize, + /** Client-side */ + .create_cred = create_cred, + .client_handshake = client_hndshk, + /** Server-side */ + .validate_cred = NULL, + .server_handshake = server_hndshk +}; + +static pmix_status_t simple_init(void) +{ + pmix_output_verbose(2, pmix_psec_base_framework.framework_output, + "psec: simple init"); + return PMIX_SUCCESS; +} + +static void simple_finalize(void) +{ + pmix_output_verbose(2, pmix_psec_base_framework.framework_output, + "psec: simple finalize"); +} + +static pmix_status_t create_cred(struct pmix_peer_t *peer, + const pmix_info_t directives[], size_t ndirs, + pmix_info_t **info, size_t *ninfo, + pmix_byte_object_t *cred) +{ + char mycred[] = "dymmy_cred"; + + pmix_output_verbose(2, pmix_psec_base_framework.framework_output, + "psec: simple create_cred"); + + /* ensure initialization */ + PMIX_BYTE_OBJECT_CONSTRUCT(cred); + + cred->bytes = strdup(mycred); + cred->size = strlen(mycred) + 1; + + return PMIX_SUCCESS; +} + +static pmix_status_t server_hndshk(int sd) +{ + pmix_status_t rc, status = PMIX_SUCCESS; + char *hndshk_msg = NULL; + size_t size; + + pmix_output_verbose(2, pmix_psec_base_framework.framework_output, + "psec: simple server_hndshk"); + + asprintf(&hndshk_msg, "%s", PMIX_PSEC_DUMMY_HNDSHK_STR); + size = strlen(hndshk_msg); + + /* send size of handshake message */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(sd, (char*)&size, + sizeof(size)))) { + goto exit; + } + /* send handshake message */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(sd, hndshk_msg, + size))) { + goto exit; + } + /* recv hadshake status from client */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_recv_blocking(sd, (char*)&status, + sizeof(status)))) { + goto exit; + } + rc = status; + pmix_output(0, "[%s:%d] psec handshake status %d recv from client", + __FILE__, __LINE__, status); + +exit: + if (NULL != hndshk_msg) { + free(hndshk_msg); + } + + return rc; +} + +static pmix_status_t client_hndshk(int sd) +{ + char *hndshk_msg = NULL; + size_t size; + pmix_status_t rc, status = PMIX_SUCCESS; + + pmix_output_verbose(2, pmix_psec_base_framework.framework_output, + "psec: simple client_hndshk"); + + /* recv size of handshake message */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_recv_blocking(sd, (char*)&size, + sizeof(size_t)))) { + return rc; + } + hndshk_msg = (char*)malloc(size); + /* recv handshake message */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_recv_blocking(sd, (char*)hndshk_msg, + size))) { + free(hndshk_msg); + return rc; + } + /* verifying handshake data */ + if (size != strlen(PMIX_PSEC_DUMMY_HNDSHK_STR)) { + rc = PMIX_ERR_HANDSHAKE_FAILED; + goto exit; + } + if (0 != strncmp(hndshk_msg, PMIX_PSEC_DUMMY_HNDSHK_STR, size)) { + rc = PMIX_ERR_HANDSHAKE_FAILED; + goto exit; + } + + /* send hadshake status to the server */ + status = PMIX_SUCCESS; + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(sd, (char*)&status, + sizeof(status)))) { + goto exit; + } + pmix_output(0, "[%s:%d] psec handshake status %d sent to server", + __FILE__, __LINE__, status); +exit: + if (NULL != hndshk_msg) { + free(hndshk_msg); + } + return rc; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.h b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.h new file mode 100644 index 00000000000..74cc3632213 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.h @@ -0,0 +1,29 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2019 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_SIMPLE_H +#define PMIX_SIMPLE_H + +#include + + +#include "src/mca/psec/psec.h" + +BEGIN_C_DECLS + +/* the component must be visible data for the linker to find it */ +PMIX_EXPORT extern pmix_psec_base_component_t mca_psec_dummy_handshake_component; +extern pmix_psec_module_t pmix_dummy_handshake_module; + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake_component.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake_component.c new file mode 100644 index 00000000000..53fb13b6fed --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake_component.c @@ -0,0 +1,73 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2019 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include "pmix_common.h" + +#include "src/mca/base/pmix_mca_base_var.h" +#include "src/mca/psec/psec.h" +#include "psec_dummy_handshake.h" + +static pmix_status_t component_open(void); +static pmix_status_t component_close(void); +static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority); +static pmix_psec_module_t* assign_module(void); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +pmix_psec_base_component_t mca_psec_dummy_handshake_component = { + .base = { + PMIX_PSEC_BASE_VERSION_1_0_0, + + /* Component name and version */ + .pmix_mca_component_name = "dummy_handshake", + PMIX_MCA_BASE_MAKE_VERSION(component, + PMIX_MAJOR_VERSION, + PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + + /* Component open and close functions */ + .pmix_mca_open_component = component_open, + .pmix_mca_close_component = component_close, + .pmix_mca_query_component = component_query, + }, + .data = { + /* The component is checkpoint ready */ + PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + .assign_module = assign_module +}; + +static int component_open(void) +{ + return PMIX_SUCCESS; +} + +static int component_query(pmix_mca_base_module_t **module, int *priority) +{ + *priority = 100; + *module = (pmix_mca_base_module_t *)&pmix_dummy_handshake_module; + return PMIX_SUCCESS; +} + + +static int component_close(void) +{ + return PMIX_SUCCESS; +} + +static pmix_psec_module_t* assign_module(void) +{ + return &pmix_dummy_handshake_module; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/Makefile.am index 5f01461190c..79756320d6a 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -47,6 +47,9 @@ mcacomponent_LTLIBRARIES = $(component) mca_psec_munge_la_SOURCES = $(component_sources) mca_psec_munge_la_LDFLAGS = -module -avoid-version $(psec_munge_LDFLAGS) mca_psec_munge_la_LIBADD = $(psec_munge_LIBS) +if NEED_LIBPMIX +mca_psec_munge_la_LIBADD += $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_psec_munge_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/psec_munge.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/psec_munge.c index 0742ad2ad6b..5ed607fc4fb 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/psec_munge.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/psec_munge.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * * NOTE: THE MUNGE CLIENT LIBRARY (libmunge) IS LICENSED AS LGPL * diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/Makefile.am index 9381d8ad60f..b1086a2aac2 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -44,6 +44,9 @@ mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component) mca_psec_native_la_SOURCES = $(component_sources) mca_psec_native_la_LDFLAGS = -module -avoid-version +if NEED_LIBPMIX +mca_psec_native_la_LIBADD = $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_psec_native_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/psec_native.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/psec_native.c index 77b3d2eaf2b..3c86ee1944c 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/psec_native.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/psec_native.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * * $COPYRIGHT$ @@ -63,8 +63,7 @@ static pmix_status_t create_cred(pmix_listener_protocol_t protocol, gid_t egid; char *tmp, *ptr; - if (PMIX_PROTOCOL_V1 == protocol || - PMIX_PROTOCOL_V3 == protocol) { + if (PMIX_PROTOCOL_V1 == protocol) { /* these are usock protocols - nothing to do */ *cred = NULL; *len = 0; @@ -112,8 +111,7 @@ static pmix_status_t validate_cred(int sd, uid_t uid, gid_t gid, pmix_output_verbose(2, pmix_globals.debug_output, "psec: native validate_cred %s", cred ? cred : "NULL"); - if (PMIX_PROTOCOL_V1 == protocol || - PMIX_PROTOCOL_V3 == protocol) { + if (PMIX_PROTOCOL_V1 == protocol) { /* these are usock protocols - get the remote side's uid/gid */ #if defined(SO_PEERCRED) && (defined(HAVE_STRUCT_UCRED_UID) || defined(HAVE_STRUCT_UCRED_CR_UID)) /* Ignore received 'cred' and validate ucred for socket instead. */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/Makefile.am index 74236996375..cde03ba502f 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -44,6 +44,9 @@ mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component) mca_psec_none_la_SOURCES = $(component_sources) mca_psec_none_la_LDFLAGS = -module -avoid-version +if NEED_LIBPMIX +mca_psec_none_la_LIBADD = $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_psec_none_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.c index d5769aec146..28c1ed53684 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none_component.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none_component.c index 17e9035b3ad..659f02f10de 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none_component.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none_component.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/psec.h b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/psec.h index f295fbb3fbf..a0ba22baea2 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/psec.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/psec.h @@ -1,10 +1,11 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -135,23 +136,12 @@ PMIX_EXPORT pmix_psec_module_t* pmix_psec_base_assign_module(const char *options pmix_output_verbose(2, pmix_globals.debug_output, \ "credential validated"); \ } \ - /* send them the result */ \ - if (PMIX_SUCCESS != (_r = pmix_ptl_base_send_blocking((p)->sd, (char*)&(_r), sizeof(int)))) { \ - PMIX_ERROR_LOG(_r); \ - } \ (r) = _r; \ } else if (NULL != (p)->nptr->compat.psec->server_handshake) { \ - /* execute the handshake if the security mode calls for it */ \ + /* request the handshake if the security mode calls for it */ \ pmix_output_verbose(2, pmix_globals.debug_output, \ - "executing handshake"); \ + "requesting handshake"); \ _r = PMIX_ERR_READY_FOR_HANDSHAKE; \ - if (PMIX_SUCCESS != (_r = pmix_ptl_base_send_blocking((p)->sd, (char*)&(_r), sizeof(int)))) { \ - PMIX_ERROR_LOG(_r); \ - } else { \ - if (PMIX_SUCCESS != (_r = p->nptr->compat.psec->server_handshake((p)->sd))) { \ - PMIX_ERROR_LOG(_r); \ - } \ - } \ (r) = _r; \ } else { \ /* this is not allowed */ \ @@ -159,6 +149,21 @@ PMIX_EXPORT pmix_psec_module_t* pmix_psec_base_assign_module(const char *options } \ } while(0) + +#define PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(r, p, d, nd, in, nin, c) \ + if(PMIX_ERR_READY_FOR_HANDSHAKE == r) { \ + int _r; \ + /* execute the handshake if the security mode calls for it */ \ + pmix_output_verbose(2, pmix_globals.debug_output, \ + "executing handshake"); \ + if (PMIX_SUCCESS != (_r = p->nptr->compat.psec->server_handshake((p)->sd))) { \ + PMIX_ERROR_LOG(_r); \ + } \ + /* Update the reply status */ \ + (r) = _r; \ + } + + /**** COMPONENT STRUCTURE DEFINITION ****/ /* define a component-level API for initializing the component */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/Makefile.am index 30dce46e38e..638fcd6a32a 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/Makefile.am @@ -1,6 +1,6 @@ # # Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2017 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -31,6 +31,9 @@ mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component_install) mca_psensor_file_la_SOURCES = $(sources) mca_psensor_file_la_LDFLAGS = -module -avoid-version +if NEED_LIBPMIX +mca_psensor_file_la_LIBADD = $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(component_noinst) libmca_psensor_file_la_SOURCES =$(sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/psensor_file.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/psensor_file.c index 914e895a40a..9a6d2c7b4a7 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/psensor_file.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/psensor_file.c @@ -6,7 +6,7 @@ * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * All rights reserved. * - * Copyright (c) 2017-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -345,7 +345,7 @@ static void file_sample(int sd, short args, void *cbdata) /* stop monitoring this client */ pmix_list_remove_item(&mca_psensor_file_component.trackers, &ft->super); /* generate an event */ - (void)strncpy(source.nspace, ft->requestor->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(source.nspace, ft->requestor->info->pname.nspace, PMIX_MAX_NSLEN); source.rank = ft->requestor->info->pname.rank; rc = PMIx_Notify_event(PMIX_MONITOR_FILE_ALERT, &source, ft->range, ft->info, ft->ninfo, opcbfunc, ft); diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/psensor_file.h b/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/psensor_file.h index 51f2a0e9f8e..bc9287bc721 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/psensor_file.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/file/psensor_file.h @@ -1,7 +1,7 @@ /* * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * - * Copyright (c) 2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/Makefile.am index df4fe0466a7..95b978415d3 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/Makefile.am @@ -1,7 +1,7 @@ # # Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. # -# Copyright (c) 2017 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -32,6 +32,9 @@ mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component_install) mca_psensor_heartbeat_la_SOURCES = $(sources) mca_psensor_heartbeat_la_LDFLAGS = -module -avoid-version +if NEED_LIBPMIX +mca_psensor_heartbeat_la_LIBADD = $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(component_noinst) libmca_psensor_heartbeat_la_SOURCES =$(sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/psensor_heartbeat.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/psensor_heartbeat.c index f88ef0cdb69..497c89a41d8 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/psensor_heartbeat.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psensor/heartbeat/psensor_heartbeat.c @@ -3,7 +3,7 @@ * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights * reserved. * - * Copyright (c) 2017-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -297,7 +297,7 @@ static void check_heartbeat(int fd, short dummy, void *cbdata) pmix_globals.myid.nspace, pmix_globals.myid.rank, ft->requestor->info->pname.nspace, ft->requestor->info->pname.rank)); /* generate an event */ - (void)strncpy(source.nspace, ft->requestor->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(source.nspace, ft->requestor->info->pname.nspace, PMIX_MAX_NSLEN); source.rank = ft->requestor->info->pname.rank; /* ensure the tracker remains throughout the process */ PMIX_RETAIN(ft); diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pshmem/base/pshmem_base_frame.c b/opal/mca/pmix/pmix2x/pmix/src/mca/pshmem/base/pshmem_base_frame.c index 4c38005da67..6d38a7f2e81 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pshmem/base/pshmem_base_frame.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pshmem/base/pshmem_base_frame.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -60,6 +60,9 @@ static pmix_status_t pmix_pshmem_close(void) static pmix_status_t pmix_pshmem_open(pmix_mca_base_open_flag_t flags) { + if (initialized) { + return PMIX_SUCCESS; + } /* initialize globals */ initialized = true; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pshmem/mmap/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/pshmem/mmap/Makefile.am index 68ba424b719..1483ae5de01 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pshmem/mmap/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pshmem/mmap/Makefile.am @@ -2,6 +2,7 @@ # # Copyright (c) 2017 Mellanox Technologies, Inc. # All rights reserved. +# Copyright (c) 2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -36,6 +37,9 @@ mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component) mca_pshmem_mmap_la_SOURCES = $(component_sources) mca_pshmem_mmap_la_LDFLAGS = -module -avoid-version +if NEED_LIBPMIX +mca_pshmem_mmap_la_LIBADD = $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_pshmem_mmap_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pshmem/mmap/pshmem_mmap.c b/opal/mca/pmix/pmix2x/pmix/src/mca/pshmem/mmap/pshmem_mmap.c index a004ac27316..587ce5ab2cc 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pshmem/mmap/pshmem_mmap.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pshmem/mmap/pshmem_mmap.c @@ -3,7 +3,7 @@ * All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -121,7 +121,7 @@ static int _mmap_segment_create(pmix_pshmem_seg_t *sm_seg, const char *file_name sm_seg->seg_cpid = my_pid; sm_seg->seg_size = size; sm_seg->seg_base_addr = (unsigned char *)seg_addr; - (void)strncpy(sm_seg->seg_name, file_name, PMIX_PATH_MAX - 1); + pmix_strncpy(sm_seg->seg_name, file_name, PMIX_PATH_MAX); out: if (-1 != sm_seg->seg_id) { diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/base.h b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/base.h index 97754b4cb1e..5705056e957 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/base.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/base.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -100,7 +100,7 @@ PMIX_EXPORT pmix_status_t pmix_ptl_base_cancel_recv(struct pmix_peer_t *peer, PMIX_EXPORT pmix_status_t pmix_ptl_base_start_listening(pmix_info_t *info, size_t ninfo); PMIX_EXPORT void pmix_ptl_base_stop_listening(void); - +PMIX_EXPORT pmix_status_t pmix_ptl_base_setup_fork(const pmix_proc_t *proc, char ***env); /* base support functions */ PMIX_EXPORT void pmix_ptl_base_send(int sd, short args, void *cbdata); diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_frame.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_frame.c index 2f1fd4f6a07..af863bcb08c 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_frame.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_frame.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -198,9 +198,11 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_ptl_sr_t, static void pccon(pmix_pending_connection_t *p) { + p->need_id = false; memset(p->nspace, 0, PMIX_MAX_NSLEN+1); p->info = NULL; p->ninfo = 0; + p->peer = NULL; p->bfrops = NULL; p->psec = NULL; p->gds = NULL; @@ -258,6 +260,8 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_listener_t, static void qcon(pmix_ptl_queue_t *p) { p->peer = NULL; + p->buf = NULL; + p->tag = UINT32_MAX; } static void qdes(pmix_ptl_queue_t *p) { diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c index d1f7971e5bb..043a68e1388 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2015-2017 Research Organization for Information Science @@ -55,17 +55,22 @@ static void _notify_complete(pmix_status_t status, void *cbdata) PMIX_RELEASE(chain); } +static void lcfn(pmix_status_t status, void *cbdata) +{ + pmix_peer_t *peer = (pmix_peer_t*)cbdata; + PMIX_RELEASE(peer); +} + void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err) { - pmix_server_trkr_t *trk; + pmix_server_trkr_t *trk, *tnxt; pmix_server_caddy_t *rinfo, *rnext; - pmix_trkr_caddy_t *tcd; - pmix_regevents_info_t *reginfoptr, *regnext; - pmix_peer_events_info_t *pr, *pnext; pmix_rank_info_t *info, *pinfo; pmix_ptl_posted_recv_t *rcv; pmix_buffer_t buf; pmix_ptl_hdr_t hdr; + pmix_proc_t proc; + pmix_status_t rc; /* stop all events */ if (peer->recv_ev_active) { @@ -82,20 +87,18 @@ void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err) } CLOSE_THE_SOCKET(peer->sd); - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && + !PMIX_PROC_IS_TOOL(pmix_globals.mypeer)) { /* if I am a server, then we need to ensure that * we properly account for the loss of this client * from any local collectives in which it was * participating - note that the proc would not * have been added to any collective tracker until * after it successfully connected */ - PMIX_LIST_FOREACH(trk, &pmix_server_globals.collectives, pmix_server_trkr_t) { + PMIX_LIST_FOREACH_SAFE(trk, tnxt, &pmix_server_globals.collectives, pmix_server_trkr_t) { /* see if this proc is participating in this tracker */ PMIX_LIST_FOREACH_SAFE(rinfo, rnext, &trk->local_cbs, pmix_server_caddy_t) { - if (0 != strncmp(rinfo->peer->info->pname.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN)) { - continue; - } - if (rinfo->peer->info->pname.rank != peer->info->pname.rank) { + if (!PMIX_CHECK_PROCID(&rinfo->peer->info->pname, &peer->info->pname)) { continue; } /* it is - adjust the count */ @@ -103,16 +106,65 @@ void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err) /* remove it from the list */ pmix_list_remove_item(&trk->local_cbs, &rinfo->super); PMIX_RELEASE(rinfo); - /* check for completion */ - if (pmix_list_get_size(&trk->local_cbs) == trk->nlocal) { - /* complete, so now we need to process it - * we don't want to block someone - * here, so kick any completed trackers into a - * new event for processing */ - PMIX_EXECUTE_COLLECTIVE(tcd, trk, pmix_server_execute_collective); + /* if the host has already been called for this tracker, + * then do nothing here - just wait for the host to return + * from the operation */ + if (trk->host_called) { + continue; + } + if (trk->def_complete && trk->nlocal == pmix_list_get_size(&trk->local_cbs)) { + /* if this is a local-only collective, then resolve it now */ + if (trk->local) { + /* everyone else has called in - we need to let them know + * that this proc has disappeared + * as otherwise the collective will never complete */ + if (PMIX_FENCENB_CMD == trk->type) { + if (NULL != trk->modexcbfunc) { + trk->modexcbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, NULL, 0, trk, NULL, NULL); + } + } else if (PMIX_CONNECTNB_CMD == trk->type) { + if (NULL != trk->op_cbfunc) { + trk->op_cbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, trk); + } + } else if (PMIX_DISCONNECTNB_CMD == trk->type) { + if (NULL != trk->op_cbfunc) { + trk->op_cbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, trk); + } + } + } else { + /* if the host has not been called, then we need to see if + * the collective is locally complete without this lost + * participant. If so, then we need to pass the call + * up to the host as otherwise the global collective will hang */ + if (PMIX_FENCENB_CMD == trk->type) { + trk->host_called = true; + rc = pmix_host_server.fence_nb(trk->pcs, trk->npcs, + trk->info, trk->ninfo, + NULL, 0, trk->modexcbfunc, trk); + if (PMIX_SUCCESS != rc) { + pmix_list_remove_item(&pmix_server_globals.collectives, &trk->super); + PMIX_RELEASE(trk); + } + } else if (PMIX_CONNECTNB_CMD == trk->type) { + trk->host_called = true; + rc = pmix_host_server.connect(trk->pcs, trk->npcs, trk->info, trk->ninfo, trk->op_cbfunc, trk); + if (PMIX_SUCCESS != rc) { + pmix_list_remove_item(&pmix_server_globals.collectives, &trk->super); + PMIX_RELEASE(trk); + } + } else if (PMIX_DISCONNECTNB_CMD == trk->type) { + trk->host_called = true; + rc = pmix_host_server.disconnect(trk->pcs, trk->npcs, trk->info, trk->ninfo, trk->op_cbfunc, trk); + if (PMIX_SUCCESS != rc) { + pmix_list_remove_item(&pmix_server_globals.collectives, &trk->super); + PMIX_RELEASE(trk); + } + } + } } } } + /* remove this proc from the list of ranks for this nspace if it is * still there - we must check for multiple copies as there will be * one for each "clone" of this peer */ @@ -122,38 +174,53 @@ void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err) } } /* reduce the number of local procs */ - --peer->nptr->nlocalprocs; + if (0 < peer->nptr->nlocalprocs) { + --peer->nptr->nlocalprocs; + } /* remove this client from our array */ pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); - /* cleanup any remaining events they have registered for */ - PMIX_LIST_FOREACH_SAFE(reginfoptr, regnext, &pmix_server_globals.events, pmix_regevents_info_t) { - PMIX_LIST_FOREACH_SAFE(pr, pnext, ®infoptr->peers, pmix_peer_events_info_t) { - if (peer == pr->peer) { - pmix_list_remove_item(®infoptr->peers, &pr->super); - PMIX_RELEASE(pr); - if (0 == pmix_list_get_size(®infoptr->peers)) { - pmix_list_remove_item(&pmix_server_globals.events, ®infoptr->super); - PMIX_RELEASE(reginfoptr); - break; - } - } - } + + /* purge any notifications cached for this client */ + pmix_server_purge_events(peer, NULL); + + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + /* only connection I can lose is to my server, so mark it */ + pmix_globals.connected = false; + } else { + /* cleanup any sensors that are monitoring them */ + pmix_psensor.stop(peer, NULL); } - /* cleanup any sensors that are monitoring them */ - pmix_psensor.stop(peer, NULL); - if (!peer->finalized && !PMIX_PROC_IS_TOOL(peer)) { + if (!peer->finalized && !PMIX_PROC_IS_TOOL(peer) && !pmix_globals.mypeer->finalized) { /* if this peer already called finalize, then * we are just seeing their connection go away * when they terminate - so do not generate * an event. If not, then we do */ - PMIX_REPORT_EVENT(err, peer, PMIX_RANGE_NAMESPACE, _notify_complete); + PMIX_REPORT_EVENT(err, peer, PMIX_RANGE_PROC_LOCAL, _notify_complete); } /* now decrease the refcount - might actually free the object */ PMIX_RELEASE(peer->info); + /* be sure to let the host know that the tool or client + * is gone - otherwise, it won't know to cleanup the + * resources it allocated to it */ + if (NULL != pmix_host_server.client_finalized && !peer->finalized) { + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + proc.rank = peer->info->pname.rank; + /* now tell the host server */ + rc = pmix_host_server.client_finalized(&proc, peer->info->server_object, + lcfn, peer); + if (PMIX_SUCCESS == rc) { + /* we will release the peer when the server calls us back */ + peer->finalized = true; + return; + } + } + /* mark the peer as "gone" since a release doesn't guarantee + * that the peer object doesn't persist */ + peer->finalized = true; /* Release peer info */ PMIX_RELEASE(peer); } else { @@ -183,7 +250,7 @@ void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err) PMIX_DESTRUCT(&buf); /* if I called finalize, then don't generate an event */ if (!pmix_globals.mypeer->finalized) { - PMIX_REPORT_EVENT(err, pmix_client_globals.myserver, PMIX_RANGE_LOCAL, _notify_complete); + PMIX_REPORT_EVENT(err, pmix_client_globals.myserver, PMIX_RANGE_PROC_LOCAL, _notify_complete); } } } @@ -335,7 +402,9 @@ void pmix_ptl_base_send_handler(int sd, short flags, void *cbdata) if (NULL != msg) { pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:base:send_handler SENDING MSG"); + "ptl:base:send_handler SENDING MSG TO %s:%d TAG %u", + peer->info->pname.nspace, peer->info->pname.rank, + ntohl(msg->hdr.tag)); if (PMIX_SUCCESS == (rc = send_msg(peer->sd, msg))) { // message is complete pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, @@ -466,13 +535,13 @@ void pmix_ptl_base_recv_handler(int sd, short flags, void *cbdata) pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "ptl:base:recv:handler allocate data region of size %lu", (unsigned long)peer->recv_msg->hdr.nbytes); + /* allocate the data region */ if (pmix_ptl_globals.max_msg_size < peer->recv_msg->hdr.nbytes) { pmix_show_help("help-pmix-runtime.txt", "ptl:msg_size", true, (unsigned long)peer->recv_msg->hdr.nbytes, (unsigned long)pmix_ptl_globals.max_msg_size); goto err_close; } - /* allocate the data region */ peer->recv_msg->data = (char*)malloc(peer->recv_msg->hdr.nbytes); memset(peer->recv_msg->data, 0, peer->recv_msg->hdr.nbytes); /* point to it */ @@ -566,10 +635,10 @@ void pmix_ptl_base_send(int sd, short args, void *cbdata) if (NULL == queue->peer || queue->peer->sd < 0 || NULL == queue->peer->info || NULL == queue->peer->nptr) { /* this peer has lost connection */ + if (NULL != queue->buf) { + PMIX_RELEASE(queue->buf); + } PMIX_RELEASE(queue); - /* ensure we post the object before another thread - * picks it back up */ - PMIX_POST_OBJECT(queue); return; } @@ -579,6 +648,12 @@ void pmix_ptl_base_send(int sd, short args, void *cbdata) (queue->peer)->info->pname.nspace, (queue->peer)->info->pname.rank, (queue->tag)); + if (NULL == queue->buf) { + /* nothing to send? */ + PMIX_RELEASE(queue); + return; + } + snd = PMIX_NEW(pmix_ptl_send_t); snd->hdr.pindex = htonl(pmix_globals.pindex); snd->hdr.tag = htonl(queue->tag); @@ -615,12 +690,19 @@ void pmix_ptl_base_send_recv(int fd, short args, void *cbdata) /* acquire the object */ PMIX_ACQUIRE_OBJECT(ms); - if (ms->peer->sd < 0) { - /* this peer's socket has been closed */ + if (NULL == ms->peer || ms->peer->sd < 0 || + NULL == ms->peer->info || NULL == ms->peer->nptr) { + /* this peer has lost connection */ + if (NULL != ms->bfr) { + PMIX_RELEASE(ms->bfr); + } + PMIX_RELEASE(ms); + return; + } + + if (NULL == ms->bfr) { + /* nothing to send? */ PMIX_RELEASE(ms); - /* ensure we post the object before another thread - * picks it back up */ - PMIX_POST_OBJECT(NULL); return; } @@ -731,7 +813,9 @@ void pmix_ptl_base_process_msg(int fd, short flags, void *cbdata) /* if the tag in this message is above the dynamic marker, then * that is an error */ if (PMIX_PTL_TAG_DYNAMIC <= msg->hdr.tag) { - pmix_output(0, "UNEXPECTED MESSAGE tag = %d", msg->hdr.tag); + pmix_output(0, "UNEXPECTED MESSAGE tag = %d from source %s:%d", + msg->hdr.tag, msg->peer->info->pname.nspace, + msg->peer->info->pname.rank); PMIX_REPORT_EVENT(PMIX_ERROR, msg->peer, PMIX_RANGE_NAMESPACE, _notify_complete); PMIX_RELEASE(msg); return; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_stubs.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_stubs.c index 67e0259b5e2..03d395da2af 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_stubs.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_stubs.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -30,6 +30,26 @@ #include "src/mca/ptl/base/base.h" +pmix_status_t pmix_ptl_base_setup_fork(const pmix_proc_t *proc, char ***env) +{ + pmix_ptl_base_active_t *active; + pmix_status_t rc; + + if (!pmix_ptl_globals.initialized) { + return PMIX_ERR_INIT; + } + + PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) { + if (NULL != active->component->setup_fork) { + rc = active->component->setup_fork(proc, env); + if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_AVAILABLE != rc) { + return rc; + } + } + } + return PMIX_SUCCESS; +} + pmix_status_t pmix_ptl_base_set_notification_cbfunc(pmix_ptl_cbfunc_t cbfunc) { pmix_ptl_posted_recv_t *req; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl.h b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl.h index 157f45f580c..0d1f784879f 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -143,11 +143,23 @@ typedef struct pmix_ptl_module_t pmix_ptl_module_t; /***** MACROS FOR EXECUTING PTL FUNCTIONS *****/ -#define PMIX_PTL_SEND_RECV(r, p, b, c, d) \ - (r) = (p)->nptr->compat.ptl->send_recv((struct pmix_peer_t*)(p), b, c, d) - -#define PMIX_PTL_SEND_ONEWAY(r, p, b, t) \ - (r) = (p)->nptr->compat.ptl->send((struct pmix_peer_t*)(p), b, t) +#define PMIX_PTL_SEND_RECV(r, p, b, c, d) \ + do { \ + if ((p)->finalized) { \ + (r) = PMIX_ERR_UNREACH; \ + } else { \ + (r) = (p)->nptr->compat.ptl->send_recv((struct pmix_peer_t*)(p), b, c, d); \ + } \ + } while(0) + +#define PMIX_PTL_SEND_ONEWAY(r, p, b, t) \ + do { \ + if ((p)->finalized) { \ + (r) = PMIX_ERR_UNREACH; \ + } else { \ + (r) = (p)->nptr->compat.ptl->send((struct pmix_peer_t*)(p), b, t); \ + } \ + } while(0) #define PMIX_PTL_RECV(r, p, c, t) \ (r) = (p)->nptr->compat.ptl->recv((struct pmix_peer_t*)(p), c, t) @@ -171,6 +183,10 @@ extern pmix_status_t pmix_ptl_base_connect_to_peer(struct pmix_peer_t* peer, typedef pmix_status_t (*pmix_ptl_base_setup_listener_fn_t)(pmix_info_t info[], size_t ninfo, bool *need_listener); +/* define a component-level API for obtaining any envars that are to + * be passed to client procs upon fork */ +typedef pmix_status_t (*pmix_ptl_base_setup_fork_fn_t)(const pmix_proc_t *proc, char ***env); + /* * the standard component data structure */ @@ -180,6 +196,8 @@ struct pmix_ptl_base_component_t { int priority; char* uri; pmix_ptl_base_setup_listener_fn_t setup_listener; + pmix_ptl_base_setup_fork_fn_t setup_fork; + }; typedef struct pmix_ptl_base_component_t pmix_ptl_base_component_t; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl_types.h b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl_types.h index 716b6571e20..8a2140959bc 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl_types.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl_types.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -63,13 +63,20 @@ struct pmix_ptl_module_t; /* define a process type */ typedef uint16_t pmix_proc_type_t; -#define PMIX_PROC_UNDEF 0x0000 -#define PMIX_PROC_CLIENT 0x0001 -#define PMIX_PROC_SERVER 0x0002 -#define PMIX_PROC_TOOL 0x0004 -#define PMIX_PROC_V1 0x0008 -#define PMIX_PROC_V20 0x0010 -#define PMIX_PROC_V21 0x0020 + +#define PMIX_PROC_UNDEF 0x0000 +#define PMIX_PROC_CLIENT 0x0001 // simple client process +#define PMIX_PROC_SERVER 0x0002 // simple server process +#define PMIX_PROC_TOOL 0x0004 // simple tool +#define PMIX_PROC_V1 0x0008 // process is using PMIx v1 protocols +#define PMIX_PROC_V20 0x0010 // process is using PMIx v2.0 protocols +#define PMIX_PROC_V21 0x0020 // process is using PMIx v2.1 protocols +#define PMIX_PROC_LAUNCHER_ACT 0x1000 // process acting as launcher +#define PMIX_PROC_LAUNCHER (PMIX_PROC_TOOL | PMIX_PROC_SERVER | PMIX_PROC_LAUNCHER_ACT) +#define PMIX_PROC_CLIENT_TOOL_ACT 0x2000 +#define PMIX_PROC_CLIENT_TOOL (PMIX_PROC_TOOL | PMIX_PROC_CLIENT | PMIX_PROC_CLIENT_TOOL_ACT) +#define PMIX_PROC_GATEWAY_ACT 0x4000 +#define PMIX_PROC_GATEWAY (PMIX_PROC_SERVER | PMIX_PROC_GATEWAY_ACT) /* defins some convenience macros for testing proc type */ #define PMIX_PROC_IS_CLIENT(p) (PMIX_PROC_CLIENT & (p)->proc_type) @@ -78,6 +85,9 @@ typedef uint16_t pmix_proc_type_t; #define PMIX_PROC_IS_V1(p) (PMIX_PROC_V1 & (p)->proc_type) #define PMIX_PROC_IS_V20(p) (PMIX_PROC_V20 & (p)->proc_type) #define PMIX_PROC_IS_V21(p) (PMIX_PROC_V21 & (p)->proc_type) +#define PMIX_PROC_IS_LAUNCHER(p) (PMIX_PROC_LAUNCHER_ACT & (p)->proc_type) +#define PMIX_PROC_IS_CLIENT_TOOL(p) (PMIX_PROC_CLIENT_TOOL_ACT & (p)->proc_type) +#define PMIX_PROC_IS_GATEWAY(p) (PMIX_PROC_GATEWAY_ACT & (p)->proc_type) /**** MESSAGING STRUCTURES ****/ @@ -87,6 +97,7 @@ typedef uint32_t pmix_ptl_tag_t; * within the system */ #define PMIX_PTL_TAG_NOTIFY 0 #define PMIX_PTL_TAG_HEARTBEAT 1 +#define PMIX_PTL_TAG_IOF 2 /* define the start of dynamic tags that are * assigned for send/recv operations */ @@ -174,9 +185,9 @@ PMIX_CLASS_DECLARATION(pmix_ptl_queue_t); /* define listener protocol types */ typedef uint16_t pmix_listener_protocol_t; -#define PMIX_PROTOCOL_V1 0 // legacy usock -#define PMIX_PROTOCOL_V2 1 // tcp -#define PMIX_PROTOCOL_V3 2 // updated usock +#define PMIX_PROTOCOL_UNDEF 0 +#define PMIX_PROTOCOL_V1 1 // legacy usock +#define PMIX_PROTOCOL_V2 2 // tcp /* connection support */ typedef struct { @@ -184,11 +195,14 @@ typedef struct { pmix_event_t ev; pmix_listener_protocol_t protocol; int sd; + bool need_id; + uint8_t flag; char nspace[PMIX_MAX_NSLEN+1]; pmix_info_t *info; size_t ninfo; pmix_status_t status; struct sockaddr_storage addr; + struct pmix_peer_t *peer; char *bfrops; char *psec; char *gds; @@ -224,9 +238,6 @@ PMIX_EXPORT extern int pmix_ptl_base_output; #define PMIX_ACTIVATE_POST_MSG(ms) \ do { \ - pmix_output_verbose(5, pmix_ptl_base_output, \ - "[%s:%d] post msg", \ - __FILE__, __LINE__); \ pmix_event_assign(&((ms)->ev), pmix_globals.evbase, -1, \ EV_WRITE, pmix_ptl_base_process_msg, (ms)); \ PMIX_POST_OBJECT(ms); \ @@ -247,37 +258,42 @@ PMIX_EXPORT extern int pmix_ptl_base_output; * t - tag to be sent to * b - buffer to be sent */ -#define PMIX_SERVER_QUEUE_REPLY(p, t, b) \ - do { \ - pmix_ptl_send_t *snd; \ - uint32_t nbytes; \ - pmix_output_verbose(5, pmix_ptl_base_output, \ +#define PMIX_SERVER_QUEUE_REPLY(r, p, t, b) \ + do { \ + pmix_ptl_send_t *snd; \ + uint32_t nbytes; \ + pmix_output_verbose(5, pmix_ptl_base_output, \ "[%s:%d] queue callback called: reply to %s:%d on tag %d size %d", \ - __FILE__, __LINE__, \ - (p)->info->pname.nspace, \ - (p)->info->pname.rank, (t), (int)(b)->bytes_used); \ - snd = PMIX_NEW(pmix_ptl_send_t); \ - snd->hdr.pindex = htonl(pmix_globals.pindex); \ - snd->hdr.tag = htonl(t); \ - nbytes = (b)->bytes_used; \ - snd->hdr.nbytes = htonl(nbytes); \ - snd->data = (b); \ - /* always start with the header */ \ - snd->sdptr = (char*)&snd->hdr; \ - snd->sdbytes = sizeof(pmix_ptl_hdr_t); \ - /* if there is no message on-deck, put this one there */ \ - if (NULL == (p)->send_msg) { \ - (p)->send_msg = snd; \ - } else { \ - /* add it to the queue */ \ - pmix_list_append(&(p)->send_queue, &snd->super); \ - } \ - /* ensure the send event is active */ \ - if (!(p)->send_ev_active && 0 <= (p)->sd) { \ - (p)->send_ev_active = true; \ - PMIX_POST_OBJECT(snd); \ - pmix_event_add(&(p)->send_event, 0); \ - } \ + __FILE__, __LINE__, \ + (p)->info->pname.nspace, \ + (p)->info->pname.rank, (t), (int)(b)->bytes_used); \ + if ((p)->finalized) { \ + (r) = PMIX_ERR_UNREACH; \ + } else { \ + snd = PMIX_NEW(pmix_ptl_send_t); \ + snd->hdr.pindex = htonl(pmix_globals.pindex); \ + snd->hdr.tag = htonl(t); \ + nbytes = (b)->bytes_used; \ + snd->hdr.nbytes = htonl(nbytes); \ + snd->data = (b); \ + /* always start with the header */ \ + snd->sdptr = (char*)&snd->hdr; \ + snd->sdbytes = sizeof(pmix_ptl_hdr_t); \ + /* if there is no message on-deck, put this one there */ \ + if (NULL == (p)->send_msg) { \ + (p)->send_msg = snd; \ + } else { \ + /* add it to the queue */ \ + pmix_list_append(&(p)->send_queue, &snd->super); \ + } \ + /* ensure the send event is active */ \ + if (!(p)->send_ev_active && 0 <= (p)->sd) { \ + (p)->send_ev_active = true; \ + PMIX_POST_OBJECT(snd); \ + pmix_event_add(&(p)->send_event, 0); \ + } \ + (r) = PMIX_SUCCESS; \ + } \ } while (0) #define CLOSE_THE_SOCKET(s) \ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/Makefile.am index 6788aba19c4..0a5b86bfdac 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -44,6 +44,9 @@ mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component) mca_ptl_tcp_la_SOURCES = $(component_sources) mca_ptl_tcp_la_LDFLAGS = -module -avoid-version +if NEED_LIBPMIX +mca_ptl_tcp_la_LIBADD = $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_ptl_tcp_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c index 5d317861825..04e9eaf87a0 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c @@ -13,7 +13,8 @@ * Copyright (c) 2011-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -42,7 +43,12 @@ #ifdef HAVE_SYS_STAT_H #include #endif +#ifdef HAVE_DIRENT_H #include +#endif +#ifdef HAVE_SYS_SYSCTL_H +#include +#endif #include "src/include/pmix_globals.h" #include "src/include/pmix_socket_errno.h" @@ -51,7 +57,9 @@ #include "src/util/argv.h" #include "src/util/error.h" #include "src/util/os_path.h" +#include "src/util/show_help.h" #include "src/mca/bfrops/base/base.h" +#include "src/mca/gds/gds.h" #include "src/mca/ptl/base/base.h" #include "ptl_tcp.h" @@ -76,8 +84,8 @@ pmix_ptl_module_t pmix_ptl_tcp_module = { .connect_to_peer = connect_to_peer }; -static pmix_status_t recv_connect_ack(int sd); -static pmix_status_t send_connect_ack(int sd); +static pmix_status_t recv_connect_ack(int sd, uint8_t myflag); +static pmix_status_t send_connect_ack(int sd, uint8_t *myflag, pmix_info_t info[], size_t ninfo); static pmix_status_t init(void) @@ -108,24 +116,30 @@ static pmix_status_t parse_uri_file(char *filename, char **uri, char **nspace, pmix_rank_t *rank); -static pmix_status_t try_connect(int *sd); +static pmix_status_t try_connect(char *uri, int *sd, pmix_info_t info[], size_t ninfo); static pmix_status_t df_search(char *dirname, char *prefix, + pmix_info_t info[], size_t ninfo, int *sd, char **nspace, - pmix_rank_t *rank); + pmix_rank_t *rank, char **uri); static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, pmix_info_t *info, size_t ninfo) { - char *evar, **uri, *suri; + char *evar, **uri, *suri = NULL, *suri2 = NULL; char *filename, *nspace=NULL; pmix_rank_t rank = PMIX_RANK_WILDCARD; - char *p, *p2; + char *p, *p2, *server_nspace = NULL, *rendfile = NULL; int sd, rc; size_t n; char myhost[PMIX_MAXHOSTNAMELEN]; bool system_level = false; bool system_level_only = false; - pid_t pid = 0; + pid_t pid = 0, mypid; + pmix_list_t ilist; + pmix_info_caddy_t *kv; + pmix_info_t *iptr = NULL, mypidinfo; + size_t niptr = 0; + pmix_kval_t *urikv = NULL; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "ptl:tcp: connecting to server"); @@ -163,6 +177,8 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, } /* the server will be using the same bfrops as us */ pmix_client_globals.myserver->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops; + /* mark that we are using the V2 (i.e., tcp) protocol */ + pmix_globals.mypeer->protocol = PMIX_PROTOCOL_V2; /* the URI consists of the following elements: * - server nspace.rank @@ -186,130 +202,244 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, ++p2; nspace = strdup(p); rank = strtoull(p2, NULL, 10); - - /* save the URI, but do not overwrite what we may have received from - * the info-key directives */ - if (NULL == mca_ptl_tcp_component.super.uri) { - mca_ptl_tcp_component.super.uri = strdup(uri[1]); - } - pmix_argv_free(uri); + suri = strdup(uri[1]); pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:tcp:client attempt connect to %s", - mca_ptl_tcp_component.super.uri); + "ptl:tcp:client attempt connect to %s", uri[1]); /* go ahead and try to connect */ - if (PMIX_SUCCESS != (rc = try_connect(&sd))) { + if (PMIX_SUCCESS != (rc = try_connect(uri[1], &sd, info, ninfo))) { free(nspace); + pmix_argv_free(uri); + free(suri); return rc; } + pmix_argv_free(uri); goto complete; } /* get here if we are a tool - check any provided directives * to see where they want us to connect to */ + suri = NULL; + PMIX_CONSTRUCT(&ilist, pmix_list_t); if (NULL != info) { for (n=0; n < ninfo; n++) { - if (0 == strcmp(info[n].key, PMIX_CONNECT_TO_SYSTEM)) { + if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_TO_SYSTEM)) { system_level_only = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strcmp(info[n].key, PMIX_CONNECT_SYSTEM_FIRST)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_SYSTEM_FIRST)) { /* try the system-level */ system_level = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strcmp(info[n].key, PMIX_SERVER_PIDINFO)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_PIDINFO)) { pid = info[n].value.data.pid; - } else if (0 == strcmp(info[n].key, PMIX_SERVER_URI)) { - if (NULL == mca_ptl_tcp_component.super.uri) { - free(mca_ptl_tcp_component.super.uri); + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_NSPACE)) { + if (NULL != server_nspace) { + /* they included it more than once */ + if (0 == strcmp(server_nspace, info[n].value.data.string)) { + /* same value, so ignore it */ + continue; + } + /* otherwise, we don't know which one to use */ + free(server_nspace); + if (NULL != suri) { + free(suri); + } + if (NULL != rendfile) { + free(rendfile); + } + return PMIX_ERR_BAD_PARAM; + } + server_nspace = strdup(info[n].value.data.string); + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_URI)) { + if (NULL != suri) { + /* they included it more than once */ + if (0 == strcmp(suri, info[n].value.data.string)) { + /* same value, so ignore it */ + continue; + } + /* otherwise, we don't know which one to use */ + free(suri); + if (NULL != server_nspace) { + free(server_nspace); + } + if (NULL != rendfile) { + free(rendfile); + } + return PMIX_ERR_BAD_PARAM; } - mca_ptl_tcp_component.super.uri = strdup(info[n].value.data.string); - } else if (0 == strcmp(info[n].key, PMIX_CONNECT_RETRY_DELAY)) { + suri = strdup(info[n].value.data.string); + } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_RETRY_DELAY)) { mca_ptl_tcp_component.wait_to_connect = info[n].value.data.uint32; - } else if (0 == strcmp(info[n].key, PMIX_CONNECT_MAX_RETRIES)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_MAX_RETRIES)) { mca_ptl_tcp_component.max_retries = info[n].value.data.uint32; + } else { + /* need to pass this to server */ + kv = PMIX_NEW(pmix_info_caddy_t); + kv->info = &info[n]; + pmix_list_append(&ilist, &kv->super); } } } + /* add our pid to the array */ + kv = PMIX_NEW(pmix_info_caddy_t); + mypid = getpid(); + PMIX_INFO_LOAD(&mypidinfo, PMIX_PROC_PID, &mypid, PMIX_PID); + kv->info = &mypidinfo; + pmix_list_append(&ilist, &kv->super); + + /* if we need to pass anything, setup an array */ + if (0 < (niptr = pmix_list_get_size(&ilist))) { + PMIX_INFO_CREATE(iptr, niptr); + n = 0; + while (NULL != (kv = (pmix_info_caddy_t*)pmix_list_remove_first(&ilist))) { + PMIX_INFO_XFER(&iptr[n], kv->info); + PMIX_RELEASE(kv); + ++n; + } + } + PMIX_LIST_DESTRUCT(&ilist); + + if (NULL == suri && NULL != mca_ptl_tcp_component.super.uri) { + suri = strdup(mca_ptl_tcp_component.super.uri); + } + + /* mark that we are using the V2 protocol */ + pmix_globals.mypeer->protocol = PMIX_PROTOCOL_V2; gethostname(myhost, sizeof(myhost)); /* if we were given a URI via MCA param, then look no further */ - if (NULL != mca_ptl_tcp_component.super.uri) { + if (NULL != suri) { + if (NULL != server_nspace) { + free(server_nspace); + server_nspace = NULL; + } /* if the string starts with "file:", then they are pointing * us to a file we need to read to get the URI itself */ - if (0 == strncmp(mca_ptl_tcp_component.super.uri, "file:", 5)) { + if (0 == strncmp(suri, "file:", 5)) { pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:tcp:tool getting connection info from %s", - mca_ptl_tcp_component.super.uri); + "ptl:tcp:tool getting connection info from %s", suri); nspace = NULL; - rc = parse_uri_file(&mca_ptl_tcp_component.super.uri[5], &suri, &nspace, &rank); + rc = parse_uri_file(&suri[5], &suri2, &nspace, &rank); if (PMIX_SUCCESS != rc) { + free(suri); + if (NULL != rendfile) { + free(rendfile); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return PMIX_ERR_UNREACH; } - free(mca_ptl_tcp_component.super.uri); - mca_ptl_tcp_component.super.uri = suri; + free(suri); + suri = suri2; } else { /* we need to extract the nspace/rank of the server from the string */ - p = strchr(mca_ptl_tcp_component.super.uri, ';'); + p = strchr(suri, ';'); if (NULL == p) { + free(suri); + if (NULL != rendfile) { + free(rendfile); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return PMIX_ERR_BAD_PARAM; } *p = '\0'; p++; - suri = strdup(p); // save the uri portion + suri2 = strdup(p); // save the uri portion /* the '.' in the first part of the original string separates * nspace from rank */ - p = strchr(mca_ptl_tcp_component.super.uri, '.'); + p = strchr(suri, '.'); if (NULL == p) { + free(suri2); free(suri); + if (NULL != rendfile) { + free(rendfile); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return PMIX_ERR_BAD_PARAM; } *p = '\0'; p++; - nspace = strdup(mca_ptl_tcp_component.super.uri); + nspace = strdup(suri); rank = strtoull(p, NULL, 10); + free(suri); + suri = suri2; /* now update the URI */ - free(mca_ptl_tcp_component.super.uri); - mca_ptl_tcp_component.super.uri = suri; } pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:tcp:tool attempt connect using given URI %s", - mca_ptl_tcp_component.super.uri); + "ptl:tcp:tool attempt connect using given URI %s", suri); /* go ahead and try to connect */ - if (PMIX_SUCCESS != (rc = try_connect(&sd))) { + if (PMIX_SUCCESS != (rc = try_connect(suri, &sd, iptr, niptr))) { if (NULL != nspace) { free(nspace); } + free(suri); + if (NULL != rendfile) { + free(rendfile); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return rc; } + /* cleanup */ + free(suri); + suri = NULL; + if (NULL != rendfile) { + free(rendfile); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } goto complete; } - /* if they gave us a pid, then look for it */ - if (0 != pid) { - if (0 > asprintf(&filename, "pmix.%s.tool.%d", myhost, pid)) { - return PMIX_ERR_NOMEM; - } - pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:tcp:tool searching for given session server %s", - filename); - nspace = NULL; - rc = df_search(mca_ptl_tcp_component.system_tmpdir, - filename, &sd, &nspace, &rank); - free(filename); + /* if they gave us a rendezvous file, use it */ + if (NULL != rendfile) { + /* try to read the file */ + rc = parse_uri_file(rendfile, &suri, &nspace, &rank); + free(rendfile); + rendfile = NULL; if (PMIX_SUCCESS == rc) { - goto complete; + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:tcp:tool attempt connect to system server at %s", suri); + /* go ahead and try to connect */ + if (PMIX_SUCCESS == try_connect(suri, &sd, iptr, niptr)) { + /* don't free nspace - we will use it below */ + if (NULL != rendfile) { + free(rendfile); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + goto complete; + } } + /* cleanup */ if (NULL != nspace) { free(nspace); } - /* since they gave us a specific pid and we couldn't + if (NULL != suri) { + free(suri); + } + free(rendfile); + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + /* since they gave us a specific rendfile and we couldn't * connect to it, return an error */ return PMIX_ERR_UNREACH; } - - /* if they asked for system-level, we start there */ + /* if they asked for system-level first or only, we start there */ if (system_level || system_level_only) { if (0 > asprintf(&filename, "%s/pmix.sys.%s", mca_ptl_tcp_component.system_tmpdir, myhost)) { + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return PMIX_ERR_NOMEM; } pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, @@ -319,12 +449,14 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, rc = parse_uri_file(filename, &suri, &nspace, &rank); free(filename); if (PMIX_SUCCESS == rc) { - mca_ptl_tcp_component.super.uri = suri; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:tcp:tool attempt connect to system server at %s", - mca_ptl_tcp_component.super.uri); + "ptl:tcp:tool attempt connect to system server at %s", suri); /* go ahead and try to connect */ - if (PMIX_SUCCESS == try_connect(&sd)) { + if (PMIX_SUCCESS == try_connect(suri, &sd, iptr, niptr)) { + /* don't free nspace - we will use it below */ + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } goto complete; } free(nspace); @@ -337,6 +469,83 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, if (system_level_only) { pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "ptl:tcp: connecting to system failed"); + if (NULL != suri) { + free(suri); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + return PMIX_ERR_UNREACH; + } + + /* if they gave us a pid, then look for it */ + if (0 != pid) { + if (NULL != server_nspace) { + free(server_nspace); + server_nspace = NULL; + } + if (0 > asprintf(&filename, "pmix.%s.tool.%d", myhost, pid)) { + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + return PMIX_ERR_NOMEM; + } + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:tcp:tool searching for given session server %s", + filename); + nspace = NULL; + rc = df_search(mca_ptl_tcp_component.system_tmpdir, + filename, iptr, niptr, &sd, &nspace, &rank, &suri); + free(filename); + if (PMIX_SUCCESS == rc) { + goto complete; + } + if (NULL != suri) { + free(suri); + } + if (NULL != nspace) { + free(nspace); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + /* since they gave us a specific pid and we couldn't + * connect to it, return an error */ + return PMIX_ERR_UNREACH; + } + + /* if they gave us an nspace, then look for it */ + if (NULL != server_nspace) { + if (0 > asprintf(&filename, "pmix.%s.tool.%s", myhost, server_nspace)) { + free(server_nspace); + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + return PMIX_ERR_NOMEM; + } + free(server_nspace); + server_nspace = NULL; + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:tcp:tool searching for given session server %s", + filename); + nspace = NULL; + rc = df_search(mca_ptl_tcp_component.system_tmpdir, + filename, iptr, niptr, &sd, &nspace, &rank, &suri); + free(filename); + if (PMIX_SUCCESS == rc) { + goto complete; + } + if (NULL != suri) { + free(suri); + } + if (NULL != nspace) { + free(nspace); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + /* since they gave us a specific nspace and we couldn't + * connect to it, return an error */ return PMIX_ERR_UNREACH; } @@ -346,6 +555,12 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, * one session per user on a node */ if (0 > asprintf(&filename, "pmix.%s.tool", myhost)) { + if (NULL != suri) { + free(suri); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return PMIX_ERR_NOMEM; } pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, @@ -353,24 +568,36 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, filename); nspace = NULL; rc = df_search(mca_ptl_tcp_component.system_tmpdir, - filename, &sd, &nspace, &rank); + filename, iptr, niptr, &sd, &nspace, &rank, &suri); free(filename); if (PMIX_SUCCESS != rc) { if (NULL != nspace){ free(nspace); } + if (NULL != suri) { + free(suri); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return PMIX_ERR_UNREACH; } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } complete: pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "sock_peer_try_connect: Connection across to server succeeded"); + "tcp_peer_try_connect: Connection across to server succeeded"); /* do a final bozo check */ if (NULL == nspace || PMIX_RANK_WILDCARD == rank) { if (NULL != nspace) { free(nspace); } + if (NULL != suri) { + free(suri); + } CLOSE_THE_SOCKET(sd); return PMIX_ERR_UNREACH; } @@ -378,22 +605,37 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, pmix_globals.connected = true; pmix_client_globals.myserver->sd = sd; - /* setup the server info */ - if (NULL == pmix_client_globals.myserver->info) { - pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); - } - if (NULL == pmix_client_globals.myserver->nptr) { - pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); - } - if (NULL == pmix_client_globals.myserver->nptr->nspace) { - pmix_client_globals.myserver->nptr->nspace = nspace; - } else { - free(nspace); - } - if (NULL == pmix_client_globals.myserver->info->pname.nspace) { + /* tools setup their server info in try_connect because they + * utilize a broader handshake */ + if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + /* setup the server info */ + if (NULL == pmix_client_globals.myserver->info) { + pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); + } + if (NULL == pmix_client_globals.myserver->nptr) { + pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t); + } + if (NULL != pmix_client_globals.myserver->nptr->nspace) { + free(pmix_client_globals.myserver->nptr->nspace); + } + pmix_client_globals.myserver->nptr->nspace = strdup(nspace); + + if (NULL != pmix_client_globals.myserver->info->pname.nspace) { + free(pmix_client_globals.myserver->info->pname.nspace); + } pmix_client_globals.myserver->info->pname.nspace = strdup(pmix_client_globals.myserver->nptr->nspace); + pmix_client_globals.myserver->info->pname.rank = rank; } - pmix_client_globals.myserver->info->pname.rank = rank; + /* store the URI for subsequent lookups */ + urikv = PMIX_NEW(pmix_kval_t); + urikv->key = strdup(PMIX_SERVER_URI); + PMIX_VALUE_CREATE(urikv->value, 1); + urikv->value->type = PMIX_STRING; + asprintf(&urikv->value->data.string, "%s.%u;%s", nspace, rank, suri); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, PMIX_INTERNAL, + urikv); + PMIX_RELEASE(urikv); // maintain accounting pmix_ptl_base_set_nonblocking(sd); @@ -415,6 +657,10 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, pmix_ptl_base_send_handler, pmix_client_globals.myserver); pmix_client_globals.myserver->send_ev_active = false; + free(nspace); + if (NULL != suri) { + free(suri); + } return PMIX_SUCCESS; } @@ -527,6 +773,7 @@ static pmix_status_t parse_uri_file(char *filename, p2 = pmix_getline(fp); if (NULL == p2) { pmix_client_globals.myserver->proc_type = PMIX_PROC_SERVER | PMIX_PROC_V20; + pmix_client_globals.myserver->protocol = PMIX_PROTOCOL_V2; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "V20 SERVER DETECTED"); } else { @@ -536,13 +783,11 @@ static pmix_status_t parse_uri_file(char *filename, } else { major = strtoul(p2, NULL, 10); } - if (2 <= major) { + if (2 == major) { pmix_client_globals.myserver->proc_type = PMIX_PROC_SERVER | PMIX_PROC_V21; + pmix_client_globals.myserver->protocol = PMIX_PROTOCOL_V2; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "V21 SERVER DETECTED"); - } else { - pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "UNKNOWN SERVER VERSION DETECTED: %s", p2); } } if (NULL != p2) { @@ -577,27 +822,27 @@ static pmix_status_t parse_uri_file(char *filename, return PMIX_SUCCESS; } -static pmix_status_t try_connect(int *sd) +static pmix_status_t try_connect(char *uri, int *sd, pmix_info_t iptr[], size_t niptr) { char *p, *p2, *host; struct sockaddr_in *in; struct sockaddr_in6 *in6; size_t len; pmix_status_t rc; - bool retried = false; + int retries = 0; + uint8_t myflag; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "pmix:tcp try connect to %s", - mca_ptl_tcp_component.super.uri); + "pmix:tcp try connect to %s", uri); /* mark that we are the active module for this server */ pmix_client_globals.myserver->nptr->compat.ptl = &pmix_ptl_tcp_module; /* setup the path to the daemon rendezvous point */ memset(&mca_ptl_tcp_component.connection, 0, sizeof(struct sockaddr_storage)); - if (0 == strncmp(mca_ptl_tcp_component.super.uri, "tcp4", 4)) { + if (0 == strncmp(uri, "tcp4", 4)) { /* need to skip the tcp4: part */ - p = strdup(&mca_ptl_tcp_component.super.uri[7]); + p = strdup(&uri[7]); if (NULL == p) { PMIX_ERROR_LOG(PMIX_ERR_NOMEM); return PMIX_ERR_NOMEM; @@ -626,7 +871,7 @@ static pmix_status_t try_connect(int *sd) len = sizeof(struct sockaddr_in); } else { /* need to skip the tcp6: part */ - p = strdup(&mca_ptl_tcp_component.super.uri[7]); + p = strdup(&uri[7]); if (NULL == p) { PMIX_ERROR_LOG(PMIX_ERR_NOMEM); return PMIX_ERR_NOMEM; @@ -664,39 +909,38 @@ static pmix_status_t try_connect(int *sd) retry: /* establish the connection */ if (PMIX_SUCCESS != (rc = pmix_ptl_base_connect(&mca_ptl_tcp_component.connection, len, sd))) { - PMIX_ERROR_LOG(rc); + /* do not error log - might just be a stale connection point */ return rc; } /* send our identity and any authentication credentials to the server */ - if (PMIX_SUCCESS != (rc = send_connect_ack(*sd))) { + if (PMIX_SUCCESS != (rc = send_connect_ack(*sd, &myflag, iptr, niptr))) { PMIX_ERROR_LOG(rc); CLOSE_THE_SOCKET(*sd); return rc; } /* do whatever handshake is required */ - if (PMIX_SUCCESS != (rc = recv_connect_ack(*sd))) { + if (PMIX_SUCCESS != (rc = recv_connect_ack(*sd, myflag))) { CLOSE_THE_SOCKET(*sd); if (PMIX_ERR_TEMP_UNAVAILABLE == rc) { - /* give it two tries */ - if (!retried) { - retried = true; + ++retries; + if( retries < mca_ptl_tcp_component.handshake_max_retries ) { goto retry; } } - PMIX_ERROR_LOG(rc); return rc; } return PMIX_SUCCESS; } -static pmix_status_t send_connect_ack(int sd) +static pmix_status_t send_connect_ack(int sd, uint8_t *myflag, + pmix_info_t iptr[], size_t niptr) { char *msg; pmix_ptl_hdr_t hdr; size_t sdsize=0, csize=0, len; - char *cred = NULL; + char *cred; char *sec, *bfrops, *gds; pmix_bfrop_buffer_type_t bftype; pmix_status_t rc; @@ -704,12 +948,14 @@ static pmix_status_t send_connect_ack(int sd) uid_t euid; gid_t egid; uint32_t u32; + pmix_buffer_t buf; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "pmix:tcp SEND CONNECT ACK"); /* if we are a server, then we shouldn't be here */ if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); return PMIX_ERR_NOT_SUPPORTED; } @@ -723,7 +969,7 @@ static pmix_status_t send_connect_ack(int sd) * local PMIx server, if known. Now use that module to * get a credential, if the security system provides one. Not * every psec module will do so, thus we must first check */ - PMIX_PSEC_CREATE_CRED(rc, pmix_client_globals.myserver, + PMIX_PSEC_CREATE_CRED(rc, pmix_globals.mypeer, PMIX_PROTOCOL_V2, &cred, &len); if (PMIX_SUCCESS != rc) { return rc; @@ -732,15 +978,42 @@ static pmix_status_t send_connect_ack(int sd) /* allow space for a marker indicating client vs tool */ sdsize = 1; - if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + /* Defined marker values: + * + * 0 => simple client process + * 1 => legacy tool - may or may not have an identifier + * ------------------------------------------ + * 3 => self-started tool process that needs an identifier + * 4 => self-started tool process that was given an identifier by caller + * 5 => tool that was started by a PMIx server - identifier specified by server + */ + if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer) && + !PMIX_PROC_IS_TOOL(pmix_globals.mypeer)) { + /* we are a simple client */ flag = 0; /* reserve space for our nspace and rank info */ - sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(int); - } else { - flag = 1; + sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t); + + } else { // must be a tool of some sort /* add space for our uid/gid for ACL purposes */ sdsize += 2*sizeof(uint32_t); + if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + /* if we are both tool and client, then we need + * to tell the server we are both */ + flag = 5; + /* add space for our identifier */ + sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t); + } else if (0 < strlen(pmix_globals.myid.nspace) && + PMIX_RANK_INVALID != pmix_globals.myid.rank) { + /* we were given an identifier by the caller, pass it */ + sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t); + flag = 4; + } else { + /* we are a self-started tool that needs an identifier */ + flag = 3; + } } + *myflag = flag; /* add the name of our active sec module - we selected it * in pmix_client.c prior to entering here */ @@ -754,18 +1027,26 @@ static pmix_status_t send_connect_ack(int sd) /* add our active gds module for working with the server */ gds = (char*)pmix_client_globals.myserver->nptr->compat.gds->name; - /* set the number of bytes to be read beyond the header */ + /* if we were given info structs to pass to the server, pack them */ + PMIX_CONSTRUCT(&buf, pmix_buffer_t); + if (NULL != iptr) { + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, &niptr, 1, PMIX_SIZE); + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, iptr, niptr, PMIX_INFO); + } + + /* set the number of bytes to be read beyond the header - must + * NULL terminate the strings! */ hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 \ + strlen(bfrops) + 1 + sizeof(bftype) \ - + strlen(gds) + 1 + sizeof(uint32_t) + len; // must NULL terminate the strings! + + strlen(gds) + 1 + sizeof(uint32_t) + len \ + + buf.bytes_used; /* create a space for our message */ sdsize = (sizeof(hdr) + hdr.nbytes); if (NULL == (msg = (char*)malloc(sdsize))) { - if (NULL != cred) { - free(cred); - } + free(cred); free(sec); + PMIX_DESTRUCT(&buf); return PMIX_ERR_OUT_OF_RESOURCE; } memset(msg, 0, sdsize); @@ -790,13 +1071,14 @@ static pmix_status_t send_connect_ack(int sd) memcpy(msg+csize, cred, len); csize += len; } + free(cred); /* load our process type - this is a single byte, * so no worry about heterogeneity here */ memcpy(msg+csize, &flag, 1); csize += 1; - if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + if (0 == flag) { /* if we are a client, provide our nspace/rank */ memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace)); csize += strlen(pmix_globals.myid.nspace)+1; @@ -804,9 +1086,18 @@ static pmix_status_t send_connect_ack(int sd) u32 = htonl((uint32_t)pmix_globals.myid.rank); memcpy(msg+csize, &u32, sizeof(uint32_t)); csize += sizeof(uint32_t); - } else { - /* if we are a tool, provide our uid/gid for ACL support - note - * that we have to convert so we can handle heterogeneity */ + } else if (3 == flag) { + /* we are a tool that needs an identifier - add our ACLs */ + euid = geteuid(); + u32 = htonl(euid); + memcpy(msg+csize, &u32, sizeof(uint32_t)); + csize += sizeof(uint32_t); + egid = getegid(); + u32 = htonl(egid); + memcpy(msg+csize, &u32, sizeof(uint32_t)); + csize += sizeof(uint32_t); + } else if (4 == flag || 5 == flag) { + /* we are a tool that has an identifier - start with our ACLs */ euid = geteuid(); u32 = htonl(euid); memcpy(msg+csize, &u32, sizeof(uint32_t)); @@ -815,6 +1106,17 @@ static pmix_status_t send_connect_ack(int sd) u32 = htonl(egid); memcpy(msg+csize, &u32, sizeof(uint32_t)); csize += sizeof(uint32_t); + /* now add our identifier */ + memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace)); + csize += strlen(pmix_globals.myid.nspace)+1; + /* again, need to convert */ + u32 = htonl((uint32_t)pmix_globals.myid.rank); + memcpy(msg+csize, &u32, sizeof(uint32_t)); + csize += sizeof(uint32_t); + } else { + /* not a valid flag */ + PMIX_DESTRUCT(&buf); + return PMIX_ERR_NOT_SUPPORTED; } /* provide our version */ @@ -833,33 +1135,33 @@ static pmix_status_t send_connect_ack(int sd) memcpy(msg+csize, gds, strlen(gds)); csize += strlen(gds)+1; + /* provide the info struct bytes */ + memcpy(msg+csize, buf.base_ptr, buf.bytes_used); + csize += buf.bytes_used; + /* send the entire message across */ if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) { free(msg); - if (NULL != cred) { - free(cred); - } + PMIX_DESTRUCT(&buf); return PMIX_ERR_UNREACH; } free(msg); - if (NULL != cred) { - free(cred); - } + PMIX_DESTRUCT(&buf); return PMIX_SUCCESS; } /* we receive a connection acknowledgement from the server, * consisting of nothing more than a status report. If success, * then we initiate authentication method */ -static pmix_status_t recv_connect_ack(int sd) +static pmix_status_t recv_connect_ack(int sd, uint8_t myflag) { pmix_status_t reply; pmix_status_t rc; struct timeval tv, save; pmix_socklen_t sz; bool sockopt = true; + pmix_nspace_t nspace; uint32_t u32; - char nspace[PMIX_MAX_NSLEN+1]; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "pmix: RECV CONNECT ACK FROM SERVER"); @@ -874,12 +1176,16 @@ static pmix_status_t recv_connect_ack(int sd) } } else { /* set a timeout on the blocking recv so we don't hang */ - tv.tv_sec = 2; + tv.tv_sec = mca_ptl_tcp_component.handshake_wait_time; tv.tv_usec = 0; if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) { - pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO"); - return PMIX_ERR_UNREACH; + if (ENOPROTOOPT == errno || EOPNOTSUPP == errno) { + sockopt = false; + } else { + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO"); + return PMIX_ERR_UNREACH; + } } } @@ -896,7 +1202,7 @@ static pmix_status_t recv_connect_ack(int sd) } reply = ntohl(u32); - if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + if (0 == myflag) { /* see if they want us to do the handshake */ if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) { PMIX_PSEC_CLIENT_HANDSHAKE(rc, pmix_client_globals.myserver, sd); @@ -918,23 +1224,31 @@ static pmix_status_t recv_connect_ack(int sd) } else { // we are a tool /* if the status indicates an error, then we are done */ if (PMIX_SUCCESS != reply) { - PMIX_ERROR_LOG(reply); return reply; } - /* recv our nspace */ - rc = pmix_ptl_base_recv_blocking(sd, (char*)&pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1); - if (PMIX_SUCCESS != rc) { - return rc; + /* if we needed an identifier, recv it */ + if (3 == myflag) { + /* first the nspace */ + rc = pmix_ptl_base_recv_blocking(sd, (char*)&nspace, PMIX_MAX_NSLEN+1); + if (PMIX_SUCCESS != rc) { + return rc; + } + PMIX_LOAD_NSPACE(pmix_globals.myid.nspace, nspace); + /* now the rank */ + rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t)); + if (PMIX_SUCCESS != rc) { + return rc; + } + /* convert and store */ + pmix_globals.myid.rank = htonl(u32); } - /* our rank is always zero */ - pmix_globals.myid.rank = 0; /* get the server's nspace and rank so we can send to it */ if (NULL == pmix_client_globals.myserver->info) { pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); } if (NULL == pmix_client_globals.myserver->nptr) { - pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); + pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t); } pmix_ptl_base_recv_blocking(sd, (char*)nspace, PMIX_MAX_NSLEN+1); if (NULL != pmix_client_globals.myserver->nptr->nspace) { @@ -945,7 +1259,8 @@ static pmix_status_t recv_connect_ack(int sd) free(pmix_client_globals.myserver->info->pname.nspace); } pmix_client_globals.myserver->info->pname.nspace = strdup(nspace); - pmix_ptl_base_recv_blocking(sd, (char*)&(pmix_client_globals.myserver->info->pname.rank), sizeof(int)); + pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t)); + pmix_client_globals.myserver->info->pname.rank = htonl(u32); pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "pmix: RECV CONNECT CONFIRMATION FOR TOOL %s:%d FROM SERVER %s:%d", @@ -954,7 +1269,18 @@ static pmix_status_t recv_connect_ack(int sd) pmix_client_globals.myserver->info->pname.rank); /* get the returned status from the security handshake */ - pmix_ptl_base_recv_blocking(sd, (char*)&reply, sizeof(pmix_status_t)); + rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(pmix_status_t)); + if (PMIX_SUCCESS != rc) { + if (sockopt) { + /* return the socket to normal */ + if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) { + return PMIX_ERR_UNREACH; + } + } + return rc; + } + + reply = ntohl(u32); if (PMIX_SUCCESS != reply) { /* see if they want us to do the handshake */ if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) { @@ -979,8 +1305,9 @@ static pmix_status_t recv_connect_ack(int sd) } static pmix_status_t df_search(char *dirname, char *prefix, + pmix_info_t info[], size_t ninfo, int *sd, char **nspace, - pmix_rank_t *rank) + pmix_rank_t *rank, char **uri) { char *suri, *nsp, *newdir; pmix_rank_t rk; @@ -1010,7 +1337,7 @@ static pmix_status_t df_search(char *dirname, char *prefix, } /* if it is a directory, down search */ if (S_ISDIR(buf.st_mode)) { - rc = df_search(newdir, prefix, sd, nspace, rank); + rc = df_search(newdir, prefix, info, ninfo, sd, nspace, rank, uri); free(newdir); if (PMIX_SUCCESS == rc) { closedir(cur_dirp); @@ -1027,20 +1354,18 @@ static pmix_status_t df_search(char *dirname, char *prefix, "pmix:tcp: reading file %s", newdir); rc = parse_uri_file(newdir, &suri, &nsp, &rk); if (PMIX_SUCCESS == rc) { - if (NULL != mca_ptl_tcp_component.super.uri) { - free(mca_ptl_tcp_component.super.uri); - } - mca_ptl_tcp_component.super.uri = suri; /* go ahead and try to connect */ pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "pmix:tcp: attempting to connect to %s", suri); - if (PMIX_SUCCESS == try_connect(sd)) { + if (PMIX_SUCCESS == try_connect(suri, sd, info, ninfo)) { (*nspace) = nsp; *rank = rk; closedir(cur_dirp); + *uri = suri; free(newdir); return PMIX_SUCCESS; } + free(suri); free(nsp); } } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.h b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.h index 6f7db225200..5813bc7085c 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.h @@ -9,7 +9,8 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -46,11 +47,16 @@ typedef struct { bool disable_ipv6_family; struct sockaddr_storage connection; char *session_filename; + char *nspace_filename; + char *pid_filename; char *system_filename; + char *rendezvous_filename; int wait_to_connect; int max_retries; char *report_uri; bool remote_connections; + int handshake_wait_time; + int handshake_max_retries; } pmix_ptl_tcp_component_t; extern pmix_ptl_tcp_component_t mca_ptl_tcp_component; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c index 2fcc6301dff..9379a4fd652 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c @@ -12,9 +12,11 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2017 Research Organization for Information Science + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. + * Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -60,6 +62,7 @@ #include "src/util/os_path.h" #include "src/util/parse_options.h" #include "src/util/pif.h" +#include "src/util/pmix_environ.h" #include "src/util/show_help.h" #include "src/util/strnlen.h" #include "src/server/pmix_server_ops.h" @@ -76,7 +79,7 @@ static int component_register(void); static int component_query(pmix_mca_base_module_t **module, int *priority); static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, bool *need_listener); - +static pmix_status_t setup_fork(const pmix_proc_t *proc, char ***env); /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it @@ -101,7 +104,8 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, }, .priority = 30, .uri = NULL, - .setup_listener = setup_listener + .setup_listener = setup_listener, + .setup_fork = setup_fork }, .session_tmpdir = NULL, .system_tmpdir = NULL, @@ -112,11 +116,16 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, .disable_ipv4_family = false, .disable_ipv6_family = true, .session_filename = NULL, + .nspace_filename = NULL, + .pid_filename = NULL, .system_filename = NULL, + .rendezvous_filename = NULL, .wait_to_connect = 4, .max_retries = 2, .report_uri = NULL, - .remote_connections = false + .remote_connections = false, + .handshake_wait_time = 4, + .handshake_max_retries = 2 }; static char **split_and_resolve(char **orig_str, char *name); @@ -145,7 +154,7 @@ static int component_register(void) (void)pmix_mca_base_component_var_register(component, "remote_connections", "Enable connections from remote tools", - PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, PMIX_INFO_LVL_2, PMIX_MCA_BASE_VAR_SCOPE_LOCAL, &mca_ptl_tcp_component.remote_connections); @@ -217,6 +226,20 @@ static int component_register(void) PMIX_MCA_BASE_VAR_SCOPE_READONLY, &mca_ptl_tcp_component.max_retries); + (void)pmix_mca_base_component_var_register(component, "handshake_wait_time", + "Number of seconds to wait for the server reply to the handshake request", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_4, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_ptl_tcp_component.handshake_wait_time); + + (void)pmix_mca_base_component_var_register(component, "handshake_max_retries", + "Number of times to retry the handshake request before giving up", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_4, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_ptl_tcp_component.handshake_max_retries); + return PMIX_SUCCESS; } @@ -230,27 +253,26 @@ static pmix_status_t component_open(void) /* check for environ-based directives * on system tmpdir to use */ - if (NULL != (tdir = getenv("PMIX_SERVER_TMPDIR"))) { - mca_ptl_tcp_component.session_tmpdir = strdup(tdir); - } - - if (NULL != (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) { - mca_ptl_tcp_component.system_tmpdir = strdup(tdir); + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + mca_ptl_tcp_component.session_tmpdir = strdup(pmix_server_globals.tmpdir); + } else { + if (NULL != (tdir = getenv("PMIX_SERVER_TMPDIR"))) { + mca_ptl_tcp_component.session_tmpdir = strdup(tdir); + } else { + mca_ptl_tcp_component.session_tmpdir = strdup(pmix_tmp_directory()); + } } - if (NULL == (tdir = getenv("TMPDIR"))) { - if (NULL == (tdir = getenv("TEMP"))) { - if (NULL == (tdir = getenv("TMP"))) { - tdir = "/tmp"; - } + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + mca_ptl_tcp_component.system_tmpdir = strdup(pmix_server_globals.system_tmpdir); + } else { + if (NULL != (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) { + mca_ptl_tcp_component.system_tmpdir = strdup(tdir); + } else { + mca_ptl_tcp_component.system_tmpdir = strdup(pmix_tmp_directory()); } } - if (NULL == mca_ptl_tcp_component.session_tmpdir) { - mca_ptl_tcp_component.session_tmpdir = strdup(tdir); - } - if (NULL == mca_ptl_tcp_component.system_tmpdir) { - mca_ptl_tcp_component.system_tmpdir = strdup(tdir); - } + if (NULL != mca_ptl_tcp_component.report_uri && 0 != strcmp(mca_ptl_tcp_component.report_uri, "-") && 0 != strcmp(mca_ptl_tcp_component.report_uri, "+")) { @@ -264,9 +286,23 @@ pmix_status_t component_close(void) { if (NULL != mca_ptl_tcp_component.system_filename) { unlink(mca_ptl_tcp_component.system_filename); + free(mca_ptl_tcp_component.system_filename); } if (NULL != mca_ptl_tcp_component.session_filename) { unlink(mca_ptl_tcp_component.session_filename); + free(mca_ptl_tcp_component.session_filename); + } + if (NULL != mca_ptl_tcp_component.nspace_filename) { + unlink(mca_ptl_tcp_component.nspace_filename); + free(mca_ptl_tcp_component.nspace_filename); + } + if (NULL != mca_ptl_tcp_component.pid_filename) { + unlink(mca_ptl_tcp_component.pid_filename); + free(mca_ptl_tcp_component.pid_filename); + } + if (NULL != mca_ptl_tcp_component.rendezvous_filename) { + unlink(mca_ptl_tcp_component.rendezvous_filename); + free(mca_ptl_tcp_component.rendezvous_filename); } if (NULL != urifile) { /* remove the file */ @@ -274,6 +310,12 @@ pmix_status_t component_close(void) free(urifile); urifile = NULL; } + if (NULL != mca_ptl_tcp_component.session_tmpdir) { + free(mca_ptl_tcp_component.session_tmpdir); + } + if (NULL != mca_ptl_tcp_component.system_tmpdir) { + free(mca_ptl_tcp_component.system_tmpdir); + } return PMIX_SUCCESS; } @@ -283,6 +325,14 @@ static int component_query(pmix_mca_base_module_t **module, int *priority) return PMIX_SUCCESS; } +static pmix_status_t setup_fork(const pmix_proc_t *proc, char ***env) +{ + pmix_setenv("PMIX_SERVER_TMPDIR", mca_ptl_tcp_component.session_tmpdir, true, env); + pmix_setenv("PMIX_SYSTEM_TMPDIR", mca_ptl_tcp_component.system_tmpdir, true, env); + + return PMIX_SUCCESS; +} + /* if we are the server, then we need to discover the available * interfaces, filter them thru any given directives, and select * the one we will listen on for connection requests. This will @@ -310,6 +360,7 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, char *prefix, myhost[PMIX_MAXHOSTNAMELEN]; char myconnhost[PMIX_MAXHOSTNAMELEN]; int myport; + pmix_kval_t *urikv; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "ptl:tcp setup_listener"); @@ -322,51 +373,51 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, /* scan the info keys and process any override instructions */ if (NULL != info) { for (n=0; n < ninfo; n++) { - if (0 == strcmp(info[n].key, PMIX_TCP_IF_INCLUDE)) { + if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IF_INCLUDE)) { if (NULL != mca_ptl_tcp_component.if_include) { free(mca_ptl_tcp_component.if_include); } mca_ptl_tcp_component.if_include = strdup(info[n].value.data.string); - } else if (0 == strcmp(info[n].key, PMIX_TCP_IF_EXCLUDE)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IF_EXCLUDE)) { if (NULL != mca_ptl_tcp_component.if_exclude) { free(mca_ptl_tcp_component.if_exclude); } mca_ptl_tcp_component.if_exclude = strdup(info[n].value.data.string); - } else if (0 == strcmp(info[n].key, PMIX_TCP_IPV4_PORT)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IPV4_PORT)) { mca_ptl_tcp_component.ipv4_port = info[n].value.data.integer; - } else if (0 == strcmp(info[n].key, PMIX_TCP_IPV6_PORT)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IPV6_PORT)) { mca_ptl_tcp_component.ipv6_port = info[n].value.data.integer; - } else if (0 == strcmp(info[n].key, PMIX_TCP_DISABLE_IPV4)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_DISABLE_IPV4)) { mca_ptl_tcp_component.disable_ipv4_family = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strcmp(info[n].key, PMIX_TCP_DISABLE_IPV6)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_DISABLE_IPV6)) { mca_ptl_tcp_component.disable_ipv6_family = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strcmp(info[n].key, PMIX_SERVER_REMOTE_CONNECTIONS)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_REMOTE_CONNECTIONS)) { mca_ptl_tcp_component.remote_connections = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strcmp(info[n].key, PMIX_TCP_URI)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_URI)) { if (NULL != mca_ptl_tcp_component.super.uri) { free(mca_ptl_tcp_component.super.uri); } mca_ptl_tcp_component.super.uri = strdup(info[n].value.data.string); - } else if (0 == strcmp(info[n].key, PMIX_TCP_REPORT_URI)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_REPORT_URI)) { if (NULL != mca_ptl_tcp_component.report_uri) { free(mca_ptl_tcp_component.report_uri); } mca_ptl_tcp_component.report_uri = strdup(info[n].value.data.string); - } else if (0 == strcmp(info[n].key, PMIX_SERVER_TMPDIR)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_TMPDIR)) { if (NULL != mca_ptl_tcp_component.session_tmpdir) { free(mca_ptl_tcp_component.session_tmpdir); } mca_ptl_tcp_component.session_tmpdir = strdup(info[n].value.data.string); - } else if (0 == strcmp(info[n].key, PMIX_SYSTEM_TMPDIR)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SYSTEM_TMPDIR)) { if (NULL != mca_ptl_tcp_component.system_tmpdir) { free(mca_ptl_tcp_component.system_tmpdir); } mca_ptl_tcp_component.system_tmpdir = strdup(info[n].value.data.string); } else if (0 == strcmp(info[n].key, PMIX_SERVER_TOOL_SUPPORT)) { session_tool = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strcmp(info[n].key, PMIX_SERVER_SYSTEM_SUPPORT)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_SYSTEM_SUPPORT)) { system_tool = PMIX_INFO_TRUE(&info[n]); - } + } } } @@ -516,6 +567,7 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, printf("%s:%d socket() failed\n", __FILE__, __LINE__); goto sockerror; } + /* set reusing ports flag */ if (setsockopt (lt->socket, SOL_SOCKET, SO_REUSEADDR, (const char *)&flags, sizeof(flags)) < 0) { pmix_output(0, "ptl:tcp:create_listen: unable to set the " @@ -589,6 +641,16 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "ptl:tcp URI %s", lt->uri); + /* save the URI internally so we can report it */ + urikv = PMIX_NEW(pmix_kval_t); + urikv->key = strdup(PMIX_SERVER_URI); + PMIX_VALUE_CREATE(urikv->value, 1); + PMIX_VALUE_LOAD(urikv->value, lt->uri, PMIX_STRING); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, PMIX_INTERNAL, + urikv); + PMIX_RELEASE(urikv); // maintain accounting + if (NULL != mca_ptl_tcp_component.report_uri) { /* if the string is a "-", then output to stdout */ if (0 == strcmp(mca_ptl_tcp_component.report_uri, "-")) { @@ -616,6 +678,38 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, } } + /* if we were given a rendezvous file, then drop it */ + if (NULL != mca_ptl_tcp_component.rendezvous_filename) { + FILE *fp; + + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "WRITING RENDEZVOUS FILE %s", + mca_ptl_tcp_component.rendezvous_filename); + fp = fopen(mca_ptl_tcp_component.rendezvous_filename, "w"); + if (NULL == fp) { + pmix_output(0, "Impossible to open the file %s in write mode\n", mca_ptl_tcp_component.rendezvous_filename); + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.rendezvous_filename); + mca_ptl_tcp_component.rendezvous_filename = NULL; + goto sockerror; + } + + /* output my nspace and rank plus the URI */ + fprintf(fp, "%s\n", lt->uri); + /* add a flag that indicates we accept v2.0 protocols */ + fprintf(fp, "v%s\n", PMIX_VERSION); + fclose(fp); + /* set the file mode */ + if (0 != chmod(mca_ptl_tcp_component.rendezvous_filename, S_IRUSR | S_IWUSR | S_IRGRP)) { + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.rendezvous_filename); + mca_ptl_tcp_component.rendezvous_filename = NULL; + goto sockerror; + } + } + /* if we are going to support tools, then drop contact file(s) */ if (system_tool) { FILE *fp; @@ -656,9 +750,10 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, FILE *fp; pid_t mypid; + /* first output to a std file */ mypid = getpid(); - if (0 > asprintf(&mca_ptl_tcp_component.session_filename, "%s/pmix.%s.tool.%d", - mca_ptl_tcp_component.session_tmpdir, myhost, mypid)) { + if (0 > asprintf(&mca_ptl_tcp_component.session_filename, "%s/pmix.%s.tool", + mca_ptl_tcp_component.session_tmpdir, myhost)) { CLOSE_THE_SOCKET(lt->socket); goto sockerror; } @@ -684,8 +779,76 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, if (0 != chmod(mca_ptl_tcp_component.session_filename, S_IRUSR | S_IWUSR | S_IRGRP)) { PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); CLOSE_THE_SOCKET(lt->socket); - free(mca_ptl_tcp_component.system_filename); - mca_ptl_tcp_component.system_filename = NULL; + free(mca_ptl_tcp_component.session_filename); + mca_ptl_tcp_component.session_filename = NULL; + goto sockerror; + } + + /* now output to a file based on pid */ + mypid = getpid(); + if (0 > asprintf(&mca_ptl_tcp_component.pid_filename, "%s/pmix.%s.tool.%d", + mca_ptl_tcp_component.session_tmpdir, myhost, mypid)) { + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "WRITING TOOL FILE %s", + mca_ptl_tcp_component.pid_filename); + fp = fopen(mca_ptl_tcp_component.pid_filename, "w"); + if (NULL == fp) { + pmix_output(0, "Impossible to open the file %s in write mode\n", mca_ptl_tcp_component.pid_filename); + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.pid_filename); + mca_ptl_tcp_component.pid_filename = NULL; + goto sockerror; + } + + /* output my URI */ + fprintf(fp, "%s\n", lt->uri); + /* add a flag that indicates we accept v2.1 protocols */ + fprintf(fp, "%s\n", PMIX_VERSION); + fclose(fp); + /* set the file mode */ + if (0 != chmod(mca_ptl_tcp_component.pid_filename, S_IRUSR | S_IWUSR | S_IRGRP)) { + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.pid_filename); + mca_ptl_tcp_component.pid_filename = NULL; + goto sockerror; + } + + /* now output it into a file based on my nspace */ + + if (0 > asprintf(&mca_ptl_tcp_component.nspace_filename, "%s/pmix.%s.tool.%s", + mca_ptl_tcp_component.session_tmpdir, myhost, pmix_globals.myid.nspace)) { + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "WRITING TOOL FILE %s", + mca_ptl_tcp_component.nspace_filename); + fp = fopen(mca_ptl_tcp_component.nspace_filename, "w"); + if (NULL == fp) { + pmix_output(0, "Impossible to open the file %s in write mode\n", mca_ptl_tcp_component.nspace_filename); + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.nspace_filename); + mca_ptl_tcp_component.nspace_filename = NULL; + goto sockerror; + } + + /* output my URI */ + fprintf(fp, "%s\n", lt->uri); + /* add a flag that indicates we accept v2.1 protocols */ + fprintf(fp, "%s\n", PMIX_VERSION); + fclose(fp); + /* set the file mode */ + if (0 != chmod(mca_ptl_tcp_component.nspace_filename, S_IRUSR | S_IWUSR | S_IRGRP)) { + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.nspace_filename); + mca_ptl_tcp_component.nspace_filename = NULL; goto sockerror; } } @@ -809,20 +972,20 @@ static void connection_handler(int sd, short args, void *cbdata) pmix_ptl_hdr_t hdr; pmix_peer_t *peer; pmix_rank_t rank=0; - pmix_status_t rc; + pmix_status_t rc, reply; char *msg, *mg, *version; char *sec, *bfrops, *gds; pmix_bfrop_buffer_type_t bftype; char *nspace; uint32_t len, u32; size_t cnt, msglen, n; - uint8_t flag; - pmix_nspace_t *nptr, *tmp; + pmix_namespace_t *nptr, *tmp; bool found; pmix_rank_info_t *info; pmix_proc_t proc; pmix_info_t ginfo; pmix_proc_type_t proc_type; + pmix_buffer_t buf; /* acquire the object */ PMIX_ACQUIRE_OBJECT(pnd); @@ -917,7 +1080,7 @@ static void connection_handler(int sd, short args, void *cbdata) /* get the process type of the connecting peer */ if (1 <= cnt) { - memcpy(&flag, mg, 1); + memcpy(&pnd->flag, mg, 1); ++mg; --cnt; } else { @@ -927,8 +1090,9 @@ static void connection_handler(int sd, short args, void *cbdata) goto error; } - if (0 == flag) { + if (0 == pnd->flag) { /* they must be a client, so get their nspace/rank */ + proc_type = PMIX_PROC_CLIENT; PMIX_STRNLEN(msglen, mg, cnt); if (msglen < cnt) { nspace = mg; @@ -953,8 +1117,9 @@ static void connection_handler(int sd, short args, void *cbdata) rc = PMIX_ERR_BAD_PARAM; goto error; } - } else if (1 == flag) { + } else if (1 == pnd->flag) { /* they are a tool */ + proc_type = PMIX_PROC_TOOL; /* extract the uid/gid */ if (sizeof(uint32_t) <= cnt) { memcpy(&u32, mg, sizeof(uint32_t)); @@ -978,8 +1143,87 @@ static void connection_handler(int sd, short args, void *cbdata) rc = PMIX_ERR_BAD_PARAM; goto error; } + } else if (3 == pnd->flag) { + /* they are a tool that needs an identifier */ + proc_type = PMIX_PROC_TOOL; + /* extract the uid/gid */ + if (sizeof(uint32_t) <= cnt) { + memcpy(&u32, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + pnd->uid = ntohl(u32); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + if (sizeof(uint32_t) <= cnt) { + memcpy(&u32, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + pnd->gid = ntohl(u32); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + /* they need an id */ + pnd->need_id = true; + } else if (4 == pnd->flag || 5 == pnd->flag) { + /* they are a tool that has an identifier - start with our ACLs */ + proc_type = PMIX_PROC_TOOL; + /* extract the uid/gid */ + if (sizeof(uint32_t) <= cnt) { + memcpy(&u32, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + pnd->uid = ntohl(u32); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + if (sizeof(uint32_t) <= cnt) { + memcpy(&u32, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + pnd->gid = ntohl(u32); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + PMIX_STRNLEN(msglen, mg, cnt); + if (msglen < cnt) { + nspace = mg; + mg += strlen(nspace) + 1; + cnt -= strlen(nspace) + 1; + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + + if (sizeof(pmix_rank_t) <= cnt) { + /* have to convert this to host order */ + memcpy(&u32, mg, sizeof(uint32_t)); + rank = ntohl(u32); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } } else { /* we don't know what they are! */ + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); rc = PMIX_ERR_NOT_SUPPORTED; free(msg); goto error; @@ -1001,22 +1245,18 @@ static void connection_handler(int sd, short args, void *cbdata) if (0 == strncmp(version, "2.0", 3)) { /* the 2.0 release handshake ends with the version string */ - proc_type = PMIX_PROC_V20; + proc_type = proc_type | PMIX_PROC_V20; bfrops = "v20"; bftype = pmix_bfrops_globals.default_type; // we can't know any better - gds = NULL; + gds = "ds12,hash"; } else { int major; major = strtoul(version, NULL, 10); - /* we already checked for version 2.0, so just check - * now for any version >= 2 as they must support - * V21 for backward compatibility */ if (2 <= major) { - proc_type = PMIX_PROC_V21; + proc_type = proc_type | PMIX_PROC_V21; } else { - /* must be a version 1.x or earlier, which cannot - * support us */ free(msg); + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); rc = PMIX_ERR_NOT_SUPPORTED; goto error; } @@ -1063,61 +1303,164 @@ static void connection_handler(int sd, short args, void *cbdata) } /* see if this is a tool connection request */ - if (1 == flag) { - /* does the server support tool connections? */ - if (NULL == pmix_host_server.tool_connected) { - /* send an error reply to the client */ - rc = PMIX_ERR_NOT_SUPPORTED; - goto error; + if (0 != pnd->flag) { + peer = PMIX_NEW(pmix_peer_t); + if (NULL == peer) { + /* probably cannot send an error reply if we are out of memory */ + free(msg); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; } - /* setup the info array to pass the relevant info - * to the server - starting with the version, if present */ + pnd->peer = peer; + /* if this is a tool we launched, then the host may + * have already registered it as a client - so check + * to see if we already have a peer for it */ + if (5 == pnd->flag) { + /* registration only adds the nspace and a rank in that + * nspace - it doesn't add the peer object to our array + * of local clients. So let's start by searching for + * the nspace object */ + nptr = NULL; + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { + if (0 == strcmp(tmp->nspace, nspace)) { + nptr = tmp; + break; + } + } + if (NULL == nptr) { + /* we don't know this namespace, reject it */ + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + /* now look for the rank */ + info = NULL; + found = false; + PMIX_LIST_FOREACH(info, &nptr->ranks, pmix_rank_info_t) { + if (info->pname.rank == rank) { + found = true; + break; + } + } + if (!found) { + /* rank unknown, reject it */ + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + PMIX_RETAIN(info); + peer->info = info; + PMIX_RETAIN(nptr); + } else { + nptr = PMIX_NEW(pmix_namespace_t); + if (NULL == nptr) { + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + PMIX_RELEASE(peer); + return; + } + } + peer->nptr = nptr; + /* select their bfrops compat module */ + peer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(bfrops); + if (NULL == peer->nptr->compat.bfrops) { + PMIX_RELEASE(peer); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + /* set the buffer type */ + peer->nptr->compat.type = bftype; n = 0; - PMIX_STRNLEN(msglen, mg, cnt); - if (msglen < cnt) { - pnd->ninfo = 4; + /* if info structs need to be passed along, then unpack them */ + if (0 < cnt) { + int32_t foo; + PMIX_CONSTRUCT(&buf, pmix_buffer_t); + PMIX_LOAD_BUFFER(peer, &buf, mg, cnt); + foo = 1; + PMIX_BFROPS_UNPACK(rc, peer, &buf, &pnd->ninfo, &foo, PMIX_SIZE); + foo = (int32_t)pnd->ninfo; + /* if we have an identifier, then we leave room to pass it */ + if (!pnd->need_id) { + pnd->ninfo += 5; + } else { + pnd->ninfo += 3; + } PMIX_INFO_CREATE(pnd->info, pnd->ninfo); - (void)strncpy(pnd->info[n].key, PMIX_VERSION_INFO, PMIX_MAX_KEYLEN); - pnd->info[n].value.type = PMIX_STRING; - pnd->info[n].value.data.string = strdup(mg); - ++n; + PMIX_BFROPS_UNPACK(rc, peer, &buf, pnd->info, &foo, PMIX_INFO); + n = foo; } else { - pnd->ninfo = 3; + if (!pnd->need_id) { + pnd->ninfo = 5; + } else { + pnd->ninfo = 3; + } PMIX_INFO_CREATE(pnd->info, pnd->ninfo); } - /* provide the user id */ - (void)strncpy(pnd->info[n].key, PMIX_USERID, PMIX_MAX_KEYLEN); - pnd->info[n].value.type = PMIX_UINT32; - pnd->info[n].value.data.uint32 = pnd->uid; - ++n; - /* and the group id */ - (void)strncpy(pnd->info[n].key, PMIX_GRPID, PMIX_MAX_KEYLEN); - pnd->info[n].value.type = PMIX_UINT32; - pnd->info[n].value.data.uint32 = pnd->gid; - ++n; + /* pass along the proc_type */ pnd->proc_type = proc_type; /* pass along the bfrop, buffer_type, and sec fields so * we can assign them once we create a peer object */ pnd->psec = strdup(sec); - if (NULL != bfrops) { - pnd->bfrops = strdup(bfrops); - } - pnd->buffer_type = bftype; if (NULL != gds) { pnd->gds = strdup(gds); } + + /* does the server support tool connections? */ + if (NULL == pmix_host_server.tool_connected) { + if (pnd->need_id) { + /* we need someone to provide the tool with an + * identifier and they aren't available */ + /* send an error reply to the client */ + rc = PMIX_ERR_NOT_SUPPORTED; + PMIX_RELEASE(peer); + /* release the msg */ + free(msg); + goto error; + } else { + /* just process it locally */ + PMIX_LOAD_PROCID(&proc, nspace, rank); + cnct_cbfunc(PMIX_SUCCESS, &proc, (void*)pnd); + /* release the msg */ + free(msg); + return; + } + } + + /* setup the info array to pass the relevant info + * to the server */ + /* provide the version */ + PMIX_INFO_LOAD(&pnd->info[n], PMIX_VERSION_INFO, version, PMIX_STRING); + ++n; + /* provide the user id */ + PMIX_INFO_LOAD(&pnd->info[n], PMIX_USERID, &pnd->uid, PMIX_UINT32); + ++n; + /* and the group id */ + PMIX_INFO_LOAD(&pnd->info[n], PMIX_GRPID, &pnd->gid, PMIX_UINT32); + ++n; + /* if we have it, pass along their ID */ + if (!pnd->need_id) { + PMIX_INFO_LOAD(&pnd->info[n], PMIX_NSPACE, nspace, PMIX_STRING); + ++n; + PMIX_INFO_LOAD(&pnd->info[n], PMIX_RANK, &rank, PMIX_PROC_RANK); + ++n; + } /* release the msg */ free(msg); - /* request an nspace for this requestor - it will - * automatically be assigned rank=0 */ + + /* pass it up for processing */ pmix_host_server.tool_connected(pnd->info, pnd->ninfo, cnct_cbfunc, pnd); return; } /* see if we know this nspace */ nptr = NULL; - PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(tmp->nspace, nspace)) { nptr = tmp; break; @@ -1160,12 +1503,20 @@ static void connection_handler(int sd, short args, void *cbdata) return; } /* mark that this peer is a client of the given type */ - peer->proc_type = PMIX_PROC_CLIENT | proc_type; + peer->proc_type = proc_type; + /* save the protocol */ + peer->protocol = pnd->protocol; /* add in the nspace pointer */ PMIX_RETAIN(nptr); peer->nptr = nptr; PMIX_RETAIN(info); peer->info = info; + /* update the epilog fields */ + peer->epilog.uid = info->uid; + peer->epilog.gid = info->gid; + /* ensure the nspace epilog is updated too */ + nptr->epilog.uid = info->uid; + nptr->epilog.gid = info->gid; info->proc_cnt++; /* increase number of processes on this rank */ peer->sd = pnd->sd; if (0 > (peer->index = pmix_pointer_array_add(&pmix_server_globals.clients, peer))) { @@ -1235,24 +1586,12 @@ static void connection_handler(int sd, short args, void *cbdata) peer->nptr->compat.ptl = &pmix_ptl_tcp_module; /* validate the connection */ - PMIX_PSEC_VALIDATE_CONNECTION(rc, peer, - PMIX_PROTOCOL_V2, - pnd->cred, pnd->len); - if (PMIX_SUCCESS != rc) { - pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "validation of client connection failed"); - info->proc_cnt--; - pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); - PMIX_RELEASE(peer); - /* send an error reply to the client */ - goto error; - } - + PMIX_PSEC_VALIDATE_CONNECTION(reply, peer, pnd->protocol, pnd->cred, pnd->len); pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "client connection validated"); + "client connection validated with status=%d", reply); - /* tell the client all is good */ - u32 = htonl(PMIX_SUCCESS); + /* Send the status to the client first */ + u32 = htonl(reply); if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { PMIX_ERROR_LOG(rc); info->proc_cnt--; @@ -1262,6 +1601,22 @@ static void connection_handler(int sd, short args, void *cbdata) PMIX_RELEASE(pnd); return; } + /* If needed perform the handshake. The macro will update reply */ + PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(reply, peer, NULL, 0, NULL, NULL, &cred); + + /* It is possible that connection validation failed + * We need to reply to the client first and cleanup after */ + if (PMIX_SUCCESS != reply) { + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "validation of client connection failed"); + info->proc_cnt--; + pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); + PMIX_RELEASE(peer); + /* send an error reply to the client */ + goto error; + } + + /* send the client's array index */ u32 = htonl(peer->index); if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { @@ -1277,12 +1632,16 @@ static void connection_handler(int sd, short args, void *cbdata) /* let the host server know that this client has connected */ if (NULL != pmix_host_server.client_connected) { - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; rc = pmix_host_server.client_connected(&proc, peer->info->server_object, NULL, NULL); - if (PMIX_SUCCESS != rc) { + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { PMIX_ERROR_LOG(rc); + info->proc_cnt--; + pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); + PMIX_RELEASE(peer); + goto error; } } @@ -1317,10 +1676,10 @@ static void process_cbfunc(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cd->cbdata; - pmix_nspace_t *nptr; + pmix_namespace_t *nptr; pmix_rank_info_t *info; pmix_peer_t *peer; - int rc; + pmix_status_t rc, reply; uint32_t u32; pmix_info_t ginfo; @@ -1332,6 +1691,7 @@ static void process_cbfunc(int sd, short args, void *cbdata) if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { PMIX_ERROR_LOG(rc); CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd->peer); PMIX_RELEASE(pnd); PMIX_RELEASE(cd); return; @@ -1339,24 +1699,41 @@ static void process_cbfunc(int sd, short args, void *cbdata) /* if the request failed, then we are done */ if (PMIX_SUCCESS != cd->status) { + PMIX_RELEASE(pnd->peer); PMIX_RELEASE(pnd); PMIX_RELEASE(cd); return; } - /* send the nspace back to the tool */ - if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, cd->proc.nspace, PMIX_MAX_NSLEN+1))) { - PMIX_ERROR_LOG(rc); - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; + /* if we got an identifier, send it back to the tool */ + if (pnd->need_id) { + /* start with the nspace */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, cd->proc.nspace, PMIX_MAX_NSLEN+1))) { + PMIX_ERROR_LOG(rc); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd->peer); + PMIX_RELEASE(pnd); + PMIX_RELEASE(cd); + return; + } + + /* now the rank, suitably converted */ + u32 = ntohl(cd->proc.rank); + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { + PMIX_ERROR_LOG(rc); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd->peer); + PMIX_RELEASE(pnd); + PMIX_RELEASE(cd); + return; + } } /* send my nspace back to the tool */ if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1))) { PMIX_ERROR_LOG(rc); CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd->peer); PMIX_RELEASE(pnd); PMIX_RELEASE(cd); return; @@ -1367,54 +1744,41 @@ static void process_cbfunc(int sd, short args, void *cbdata) if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { PMIX_ERROR_LOG(rc); CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd->peer); PMIX_RELEASE(pnd); PMIX_RELEASE(cd); return; } - /* add this nspace to our pool */ - nptr = PMIX_NEW(pmix_nspace_t); - if (NULL == nptr) { - PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; - } - nptr->nspace = strdup(cd->proc.nspace); - pmix_list_append(&pmix_server_globals.nspaces, &nptr->super); - /* add this tool rank to the nspace */ - info = PMIX_NEW(pmix_rank_info_t); - if (NULL == info) { - PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; - } - info->pname.nspace = strdup(cd->proc.nspace); - info->pname.rank = 0; - /* need to include the uid/gid for validation */ - info->uid = pnd->uid; - info->gid = pnd->gid; - pmix_list_append(&nptr->ranks, &info->super); + /* shortcuts */ + peer = (pmix_peer_t*)pnd->peer; + nptr = peer->nptr; - /* setup a peer object for this tool */ - peer = PMIX_NEW(pmix_peer_t); - if (NULL == peer) { - PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; + /* if this tool wasn't initially registered as a client, + * then add some required structures */ + if (5 != pnd->flag) { + PMIX_RETAIN(nptr); + nptr->nspace = strdup(cd->proc.nspace); + pmix_list_append(&pmix_server_globals.nspaces, &nptr->super); + info = PMIX_NEW(pmix_rank_info_t); + info->pname.nspace = strdup(nptr->nspace); + info->pname.rank = cd->proc.rank; + info->uid = pnd->uid; + info->gid = pnd->gid; + pmix_list_append(&nptr->ranks, &info->super); + PMIX_RETAIN(info); + peer->info = info; } + /* mark the peer proc type */ - peer->proc_type = PMIX_PROC_TOOL | pnd->proc_type; - /* add in the nspace pointer */ - PMIX_RETAIN(nptr); - peer->nptr = nptr; - PMIX_RETAIN(info); - peer->info = info; + peer->proc_type = pnd->proc_type; + /* save the protocol */ + peer->protocol = pnd->protocol; + /* save the uid/gid */ + peer->epilog.uid = peer->info->uid; + peer->epilog.gid = peer->info->gid; + nptr->epilog.uid = peer->info->uid; + nptr->epilog.gid = peer->info->gid; peer->proc_cnt = 1; peer->sd = pnd->sd; @@ -1432,17 +1796,6 @@ static void process_cbfunc(int sd, short args, void *cbdata) * tool as we received this request via that channel, so simply * record it here for future use */ peer->nptr->compat.ptl = &pmix_ptl_tcp_module; - /* select their bfrops compat module */ - peer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(pnd->bfrops); - if (NULL == peer->nptr->compat.bfrops) { - PMIX_RELEASE(peer); - pmix_list_remove_item(&pmix_server_globals.nspaces, &nptr->super); - PMIX_RELEASE(nptr); // will release the info object - CLOSE_THE_SOCKET(pnd->sd); - goto done; - } - /* set the buffer type */ - peer->nptr->compat.type = pnd->buffer_type; /* set the gds */ PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, pnd->gds, PMIX_STRING); peer->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1); @@ -1465,10 +1818,23 @@ static void process_cbfunc(int sd, short args, void *cbdata) } /* validate the connection */ - PMIX_PSEC_VALIDATE_CONNECTION(rc, peer, - PMIX_PROTOCOL_V2, - pnd->cred, pnd->len); - if (PMIX_SUCCESS != rc) { + PMIX_PSEC_VALIDATE_CONNECTION(reply, peer, pnd->protocol, pnd->cred, pnd->len); + /* communicate the result to the other side */ + u32 = htonl(reply); + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(peer); + pmix_list_remove_item(&pmix_server_globals.nspaces, &nptr->super); + PMIX_RELEASE(nptr); // will release the info object + CLOSE_THE_SOCKET(pnd->sd); + goto done; + } + + /* If needed perform the handshake. The macro will update reply */ + PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(reply, peer, NULL, 0, NULL, NULL, &cred); + + /* If verification wasn't successful - stop here */ + if (PMIX_SUCCESS != reply) { pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "validation of tool credentials failed: %s", PMIx_Error_string(rc)); @@ -1491,6 +1857,7 @@ static void process_cbfunc(int sd, short args, void *cbdata) /* probably cannot send an error reply if we are out of memory */ return; } + info->peerid = peer->index; /* start the events for this tool */ pmix_event_assign(&peer->recv_event, pmix_globals.evbase, peer->sd, @@ -1526,7 +1893,8 @@ static void cnct_cbfunc(pmix_status_t status, return; } cd->status = status; - (void)strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + cd->proc.rank = proc->rank; cd->cbdata = cbdata; PMIX_THREADSHIFT(cd, process_cbfunc); } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/Makefile.am index e6606e2e844..2c91ac37c8d 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -44,6 +44,9 @@ mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component) mca_ptl_usock_la_SOURCES = $(component_sources) mca_ptl_usock_la_LDFLAGS = -module -avoid-version +if NEED_LIBPMIX +mca_ptl_usock_la_LIBADD = $(top_builddir)/src/libpmix.la +endif noinst_LTLIBRARIES = $(lib) libmca_ptl_usock_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.c index 515744ddde0..46b5fb38f13 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.c @@ -13,7 +13,7 @@ * Copyright (c) 2011-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -103,6 +103,9 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, int sd; pmix_socklen_t len; bool retried = false; + pmix_kval_t *urikv; + char *nspace, *suri; + pmix_rank_t rank; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "[%s:%d] connect to server", @@ -140,12 +143,13 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; } + /* set the server nspace */ if (NULL == pmix_client_globals.myserver->info) { pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); } if (NULL == pmix_client_globals.myserver->nptr) { - pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); + pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t); } if (NULL == pmix_client_globals.myserver->nptr->nspace) { pmix_client_globals.myserver->nptr->nspace = strdup(uri[0]); @@ -157,6 +161,10 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, /* set the server rank */ pmix_client_globals.myserver->info->pname.rank = strtoull(uri[1], NULL, 10); + nspace = strdup(pmix_client_globals.myserver->nptr->nspace); + rank = pmix_client_globals.myserver->info->pname.rank; + suri = strdup(evar); + /* setup the path to the daemon rendezvous point */ memset(&mca_ptl_usock_component.connection, 0, sizeof(struct sockaddr_storage)); address = (struct sockaddr_un*)&mca_ptl_usock_component.connection; @@ -204,6 +212,19 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, /* mark the connection as made */ pmix_globals.connected = true; + /* store the URI for subsequent lookups */ + urikv = PMIX_NEW(pmix_kval_t); + urikv->key = strdup(PMIX_SERVER_URI); + PMIX_VALUE_CREATE(urikv->value, 1); + urikv->value->type = PMIX_STRING; + asprintf(&urikv->value->data.string, "%s.%u;%s", nspace, rank, suri); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, PMIX_INTERNAL, + urikv); + PMIX_RELEASE(urikv); // maintain accounting + free(nspace); + free(suri); + pmix_ptl_base_set_nonblocking(sd); /* setup recv event */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock_component.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock_component.c index 369a9777400..6b9fb03b5d1 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock_component.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock_component.c @@ -12,9 +12,11 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. + * Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -149,6 +151,10 @@ pmix_status_t component_close(void) static int component_query(pmix_mca_base_module_t **module, int *priority) { + if (PMIX_PROC_IS_TOOL(pmix_globals.mypeer)) { + return PMIX_ERR_NOT_SUPPORTED; + } + *module = (pmix_mca_base_module_t*)&pmix_ptl_usock_module; return PMIX_SUCCESS; } @@ -174,6 +180,7 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, bool disabled = false; char *pmix_pid; pid_t mypid; + pmix_kval_t *urikv; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "ptl:usock setup_listener"); @@ -235,6 +242,16 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, } /* save the rendezvous filename for later removal */ mca_ptl_usock_component.filename = strdup(address->sun_path); + /* save the URI internally so we can report it */ + urikv = PMIX_NEW(pmix_kval_t); + urikv->key = strdup(PMIX_SERVER_URI); + PMIX_VALUE_CREATE(urikv->value, 1); + PMIX_VALUE_LOAD(urikv->value, lt->uri, PMIX_STRING); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, PMIX_INTERNAL, + urikv); + PMIX_RELEASE(urikv); // maintain accounting + lt->protocol = PMIX_PROTOCOL_V1; lt->ptl = (struct pmix_ptl_module_t*)&pmix_ptl_usock_module; @@ -338,10 +355,10 @@ static void connection_handler(int sd, short args, void *cbdata) { pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata; char *msg, *ptr, *nspace, *version, *cred, *sec, *bfrops, *gds; - pmix_status_t rc; + pmix_status_t rc, reply; unsigned int rank; pmix_usock_hdr_t hdr; - pmix_nspace_t *nptr, *tmp; + pmix_namespace_t *nptr, *tmp; pmix_rank_info_t *info; pmix_peer_t *psave = NULL; bool found; @@ -353,6 +370,7 @@ static void connection_handler(int sd, short args, void *cbdata) unsigned int msglen; pmix_info_t ginfo; size_t credlen; + uint32_t u32; /* acquire the object */ PMIX_ACQUIRE_OBJECT(pnd); @@ -478,6 +496,10 @@ static void connection_handler(int sd, short args, void *cbdata) cred = ptr; ptr += credlen; len -= credlen; + } else { + /* set cred pointer to NULL to guard against validation + * methods that assume a zero length credential is NULL */ + cred = NULL; } } @@ -538,7 +560,7 @@ static void connection_handler(int sd, short args, void *cbdata) /* see if we know this nspace */ nptr = NULL; - PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(tmp->nspace, nspace)) { nptr = tmp; break; @@ -670,19 +692,41 @@ static void connection_handler(int sd, short args, void *cbdata) * record it here for future use */ nptr->compat.ptl = &pmix_ptl_usock_module; + /* now done with the msg */ + free(msg); + /* validate the connection */ if (NULL == cred) { len = 0; } else { len = strlen(cred); } - /* the macro will send the status result to the client */ - PMIX_PSEC_VALIDATE_CONNECTION(rc, psave, - PMIX_PROTOCOL_V1, cred, len); - /* now done with the msg */ - free(msg); - if (PMIX_SUCCESS != rc) { + /* validate the connection - the macro will send the status result to the client */ + PMIX_PSEC_VALIDATE_CONNECTION(reply, psave, PMIX_PROTOCOL_V1, cred, len); + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "client connection validated with status=%d", reply); + + /* Communicate the result of validation to the client */ + u32 = htonl(reply); + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { + PMIX_ERROR_LOG(rc); + info->proc_cnt--; + PMIX_RELEASE(info); + pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); + PMIX_RELEASE(psave); + /* error reply was sent by the above macro */ + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + + /* If needed perform the handshake. The macro will update reply */ + PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(reply, psave, NULL, 0, NULL, 0, &cred); + + /* It is possible that connection validation failed + * We need to reply to the client first and cleanup after */ + if (PMIX_SUCCESS != reply) { pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "validation of client credentials failed: %s", PMIx_Error_string(rc)); @@ -696,6 +740,8 @@ static void connection_handler(int sd, short args, void *cbdata) return; } + + /* send the client's array index */ if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&psave->index, sizeof(int)))) { PMIX_ERROR_LOG(rc); @@ -716,7 +762,7 @@ static void connection_handler(int sd, short args, void *cbdata) (void)strncpy(proc.nspace, psave->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = psave->info->pname.rank; rc = pmix_host_server.client_connected(&proc, psave->info->server_object, NULL, NULL); - if (PMIX_SUCCESS != rc) { + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { PMIX_ERROR_LOG(rc); info->proc_cnt--; PMIX_RELEASE(info); diff --git a/opal/mca/pmix/pmix2x/pmix/src/runtime/help-pmix-runtime.txt b/opal/mca/pmix/pmix2x/pmix/src/runtime/help-pmix-runtime.txt index 8a8899d33f1..58362b16b86 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/runtime/help-pmix-runtime.txt +++ b/opal/mca/pmix/pmix2x/pmix/src/runtime/help-pmix-runtime.txt @@ -12,7 +12,7 @@ # All rights reserved. # Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. # Copyright (c) 2014 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2017-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -59,12 +59,31 @@ by any of the following: Please check your installation and environment. # -[ptl:message-too-large] -A message has been received that exceeds the size limit: +[ptl:msg_size] +A received msg header indicates a size that is too large: - Message size: %lu - Size limit: %lu + Requested size: %lu + Size limit: %lu -The limit is enforced to protect against bad actors. If you believe that -the message was likely legitimate, then adjust the message size by -setting the ptl_base_max_msg_size MCA parameter. +If you believe this msg is legitimate, please increase the +max msg size via the ptl_base_max_msg_size parameter. +# +[tool:no-server] +A call was made to PMIx_tool_connect_to_server, but no information +was given as to which server the tool should be connected. Accepted +attributes include: + + - PMIX_CONNECT_TO_SYSTEM: connect solely to the system server + + - PMIX_CONNECT_SYSTEM_FIRST: a request to use the system server first, + if existing, and then look for the server specified in a different + attribute + + - PMIX_SERVER_URI: connect to the server at the given URI + + - PMIX_SERVER_NSPACE: connect to the server of a given nspace + + - PMIX_SERVER_PIDINFO: connect to a server embedded in the process with + the given pid + +Please correct your program and try again. diff --git a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_finalize.c b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_finalize.c index 13d3328a072..ebd653e72d3 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_finalize.c +++ b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_finalize.c @@ -12,8 +12,8 @@ * Copyright (c) 2008-2015 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010-2015 Los Alamos National Security, LLC. * All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2016 Research Organization for Information Science + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -49,19 +49,11 @@ extern int pmix_initialized; extern bool pmix_init_called; -static void __pmix_attribute_destructor__ pmix_cleanup (void) -{ - if (!pmix_initialized) { - /* nothing to do */ - return; - } - - /* finalize the class/object system */ - pmix_class_finalize(); -} - void pmix_rte_finalize(void) { + int i; + pmix_notify_caddy_t *cd; + if( --pmix_initialized != 0 ) { if( pmix_initialized < 0 ) { fprintf(stderr, "PMIx Finalize called too many times\n"); @@ -80,9 +72,6 @@ void pmix_rte_finalize(void) /* close the security framework */ (void)pmix_mca_base_framework_close(&pmix_psec_base_framework); - /* close the pnet framework */ - (void)pmix_mca_base_framework_close(&pmix_pnet_base_framework); - /* close bfrops */ (void)pmix_mca_base_framework_close(&pmix_bfrops_base_framework); @@ -114,6 +103,13 @@ void pmix_rte_finalize(void) PMIX_RELEASE(pmix_globals.mypeer); PMIX_DESTRUCT(&pmix_globals.events); PMIX_LIST_DESTRUCT(&pmix_globals.cached_events); + /* clear any notifications */ + for (i=0; i < pmix_globals.max_events; i++) { + pmix_hotel_checkout_and_return_occupant(&pmix_globals.notifications, i, (void**)&cd); + if (NULL != cd) { + PMIX_RELEASE(cd); + } + } PMIX_DESTRUCT(&pmix_globals.notifications); /* now safe to release the event base */ @@ -121,9 +117,4 @@ void pmix_rte_finalize(void) (void)pmix_progress_thread_stop(NULL); } - -#if PMIX_NO_LIB_DESTRUCTOR - pmix_cleanup(); -#endif - } diff --git a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_init.c b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_init.c index 88364d083f4..35bdf5277e5 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_init.c +++ b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_init.c @@ -15,7 +15,7 @@ * Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2010-2015 Los Alamos National Security, LLC. * All rights reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -50,6 +50,7 @@ #include "src/mca/preg/base/base.h" #include "src/mca/ptl/base/base.h" +#include "src/client/pmix_client_ops.h" #include "src/event/pmix_event.h" #include "src/include/types.h" #include "src/util/error.h" @@ -68,6 +69,8 @@ PMIX_EXPORT bool pmix_init_called = false; PMIX_EXPORT pmix_globals_t pmix_globals = { .init_cntr = 0, .mypeer = NULL, + .hostname = NULL, + .nodeid = UINT32_MAX, .pindex = 0, .evbase = NULL, .external_evbase = false, @@ -78,6 +81,15 @@ PMIX_EXPORT pmix_globals_t pmix_globals = { }; +static void _notification_eviction_cbfunc(struct pmix_hotel_t *hotel, + int room_num, + void *occupant) +{ + pmix_notify_caddy_t *cache = (pmix_notify_caddy_t*)occupant; + PMIX_RELEASE(cache); +} + + int pmix_rte_init(pmix_proc_type_t type, pmix_info_t info[], size_t ninfo, pmix_ptl_cbfunc_t cbfunc) @@ -85,6 +97,7 @@ int pmix_rte_init(pmix_proc_type_t type, int ret, debug_level; char *error = NULL, *evar; size_t n; + char hostname[PMIX_MAXHOSTNAMELEN]; if( ++pmix_initialized != 1 ) { if( pmix_initialized < 1 ) { @@ -145,14 +158,62 @@ int pmix_rte_init(pmix_proc_type_t type, } /* setup the globals structure */ - memset(&pmix_globals.myid, 0, sizeof(pmix_proc_t)); + gethostname(hostname, PMIX_MAXHOSTNAMELEN); + pmix_globals.hostname = strdup(hostname); + memset(&pmix_globals.myid.nspace, 0, PMIX_MAX_NSLEN+1); + pmix_globals.myid.rank = PMIX_RANK_INVALID; PMIX_CONSTRUCT(&pmix_globals.events, pmix_events_t); pmix_globals.event_window.tv_sec = pmix_event_caching_window; pmix_globals.event_window.tv_usec = 0; PMIX_CONSTRUCT(&pmix_globals.cached_events, pmix_list_t); /* construct the global notification ring buffer */ - PMIX_CONSTRUCT(&pmix_globals.notifications, pmix_ring_buffer_t); - pmix_ring_buffer_init(&pmix_globals.notifications, 256); + PMIX_CONSTRUCT(&pmix_globals.notifications, pmix_hotel_t); + ret = pmix_hotel_init(&pmix_globals.notifications, pmix_globals.max_events, + pmix_globals.evbase, pmix_globals.event_eviction_time, + _notification_eviction_cbfunc); + if (PMIX_SUCCESS != ret) { + error = "notification hotel init"; + goto return_error; + } + + /* Setup client verbosities as all procs are allowed to + * access client APIs */ + if (0 < pmix_client_globals.get_verbose) { + /* set default output */ + pmix_client_globals.get_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_client_globals.get_output, + pmix_client_globals.get_verbose); + } + if (0 < pmix_client_globals.connect_verbose) { + /* set default output */ + pmix_client_globals.connect_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_client_globals.connect_output, + pmix_client_globals.connect_verbose); + } + if (0 < pmix_client_globals.fence_verbose) { + /* set default output */ + pmix_client_globals.fence_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_client_globals.fence_output, + pmix_client_globals.fence_verbose); + } + if (0 < pmix_client_globals.pub_verbose) { + /* set default output */ + pmix_client_globals.pub_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_client_globals.pub_output, + pmix_client_globals.pub_verbose); + } + if (0 < pmix_client_globals.spawn_verbose) { + /* set default output */ + pmix_client_globals.spawn_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_client_globals.spawn_output, + pmix_client_globals.spawn_verbose); + } + if (0 < pmix_client_globals.event_verbose) { + /* set default output */ + pmix_client_globals.event_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_client_globals.event_output, + pmix_client_globals.event_verbose); + } /* get our effective id's */ pmix_globals.uid = geteuid(); @@ -173,7 +234,7 @@ int pmix_rte_init(pmix_proc_type_t type, pmix_globals.mypeer->proc_type = type | PMIX_PROC_V21; /* create an nspace object for ourselves - we will * fill in the nspace name later */ - pmix_globals.mypeer->nptr = PMIX_NEW(pmix_nspace_t); + pmix_globals.mypeer->nptr = PMIX_NEW(pmix_namespace_t); if (NULL == pmix_globals.mypeer->nptr) { PMIX_RELEASE(pmix_globals.mypeer); ret = PMIX_ERR_NOMEM; @@ -183,9 +244,19 @@ int pmix_rte_init(pmix_proc_type_t type, /* scan incoming info for directives */ if (NULL != info) { for (n=0; n < ninfo; n++) { - if (0 == strcmp(PMIX_EVENT_BASE, info[n].key)) { + if (PMIX_CHECK_KEY(&info[n], PMIX_EVENT_BASE)) { pmix_globals.evbase = (pmix_event_base_t*)info[n].value.data.ptr; pmix_globals.external_evbase = true; + } else if (PMIX_CHECK_KEY(&info[n], PMIX_HOSTNAME)) { + if (NULL != pmix_globals.hostname) { + free(pmix_globals.hostname); + } + pmix_globals.hostname = strdup(info[n].value.data.string); + } else if (PMIX_CHECK_KEY(&info[n], PMIX_NODEID)) { + PMIX_VALUE_GET_NUMBER(ret, &info[n].value, pmix_globals.nodeid, uint32_t); + if (PMIX_SUCCESS != ret) { + goto return_error; + } } } } @@ -195,21 +266,21 @@ int pmix_rte_init(pmix_proc_type_t type, * time of connection to that peer */ /* open the bfrops and select the active plugins */ - if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_bfrops_base_framework, 0)) ) { + if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_bfrops_base_framework, 0)) ) { error = "pmix_bfrops_base_open"; goto return_error; } - if( PMIX_SUCCESS != (ret = pmix_bfrop_base_select()) ) { + if (PMIX_SUCCESS != (ret = pmix_bfrop_base_select()) ) { error = "pmix_bfrops_base_select"; goto return_error; } /* open the ptl and select the active plugins */ - if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_ptl_base_framework, 0)) ) { + if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_ptl_base_framework, 0)) ) { error = "pmix_ptl_base_open"; goto return_error; } - if( PMIX_SUCCESS != (ret = pmix_ptl_base_select()) ) { + if (PMIX_SUCCESS != (ret = pmix_ptl_base_select()) ) { error = "pmix_ptl_base_select"; goto return_error; } @@ -230,11 +301,11 @@ int pmix_rte_init(pmix_proc_type_t type, } /* open the gds and select the active plugins */ - if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_gds_base_framework, 0)) ) { + if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_gds_base_framework, 0)) ) { error = "pmix_gds_base_open"; goto return_error; } - if( PMIX_SUCCESS != (ret = pmix_gds_base_select(info, ninfo)) ) { + if (PMIX_SUCCESS != (ret = pmix_gds_base_select(info, ninfo)) ) { error = "pmix_gds_base_select"; goto return_error; } @@ -245,22 +316,12 @@ int pmix_rte_init(pmix_proc_type_t type, return ret; } - /* open the pnet and select the active modules for this environment */ - if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_pnet_base_framework, 0))) { - error = "pmix_pnet_base_open"; - goto return_error; - } - if (PMIX_SUCCESS != (ret = pmix_pnet_base_select())) { - error = "pmix_pnet_base_select"; - goto return_error; - } - /* open the preg and select the active plugins */ - if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_preg_base_framework, 0)) ) { + if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_preg_base_framework, 0)) ) { error = "pmix_preg_base_open"; goto return_error; } - if( PMIX_SUCCESS != (ret = pmix_preg_base_select()) ) { + if (PMIX_SUCCESS != (ret = pmix_preg_base_select()) ) { error = "pmix_preg_base_select"; goto return_error; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_params.c b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_params.c index c0a40f98d7f..23e2f23d33f 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_params.c +++ b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_params.c @@ -19,9 +19,9 @@ * Copyright (c) 2014 Hochschule Esslingen. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015 Mellanox Technologies, Inc. + * Copyright (c) 2015-2018 Mellanox Technologies, Inc. * All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -35,6 +35,8 @@ #include "src/mca/base/pmix_mca_base_var.h" #include "src/runtime/pmix_rte.h" #include "src/util/timings.h" +#include "src/client/pmix_client_ops.h" +#include "src/server/pmix_server_ops.h" #if PMIX_ENABLE_TIMING char *pmix_timing_output = NULL; @@ -93,18 +95,120 @@ pmix_status_t pmix_register_params(void) } (void) pmix_mca_base_var_register ("pmix", "pmix", NULL, "event_caching_window", - "Time (in seconds) to aggregate events before reporting them - this " - "suppresses event cascades when processes abnormally terminate", - PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, - &pmix_event_caching_window); + "Time (in seconds) to aggregate events before reporting them - this " + "suppresses event cascades when processes abnormally terminate", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_event_caching_window); (void) pmix_mca_base_var_register ("pmix", "pmix", NULL, "suppress_missing_data_warning", - "Suppress warning that PMIx is missing job-level data that " - "is supposed to be provided by the host RM.", - PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, - PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, - &pmix_suppress_missing_data_warning); + "Suppress warning that PMIx is missing job-level data that " + "is supposed to be provided by the host RM.", + PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_suppress_missing_data_warning); + + /**** CLIENT: VERBOSE OUTPUT PARAMS ****/ + (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "get_verbose", + "Verbosity for client get operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_client_globals.get_verbose); + + (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "connect_verbose", + "Verbosity for client connect operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_client_globals.connect_verbose); + + (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "fence_verbose", + "Verbosity for client fence operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_client_globals.fence_verbose); + + (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "pub_verbose", + "Verbosity for client publish, lookup, and unpublish operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_client_globals.pub_verbose); + + (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "spawn_verbose", + "Verbosity for client spawn operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_client_globals.spawn_verbose); + + (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "event_verbose", + "Verbosity for client event notifications", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_client_globals.event_verbose); + + (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "base_verbose", + "Verbosity for basic client operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_client_globals.base_verbose); + + /**** SERVER: VERBOSE OUTPUT PARAMS ****/ + (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "get_verbose", + "Verbosity for server get operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_server_globals.get_verbose); + + (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "connect_verbose", + "Verbosity for server connect operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_server_globals.connect_verbose); + + (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "fence_verbose", + "Verbosity for server fence operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_server_globals.fence_verbose); + + (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "pub_verbose", + "Verbosity for server publish, lookup, and unpublish operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_server_globals.pub_verbose); + + (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "spawn_verbose", + "Verbosity for server spawn operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_server_globals.spawn_verbose); + + (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "event_verbose", + "Verbosity for server event operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_server_globals.event_verbose); + + (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "base_verbose", + "Verbosity for basic server operations", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_server_globals.base_verbose); + + /* max size of the notification hotel */ + pmix_globals.max_events = 512; + (void) pmix_mca_base_var_register ("pmix", "pmix", "max", "events", + "Maximum number of event notifications to cache", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_globals.max_events); + + /* how long to cache an event */ + pmix_globals.event_eviction_time = 120; + (void) pmix_mca_base_var_register ("pmix", "pmix", "event", "eviction_time", + "Maximum number of seconds to cache an event", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_globals.event_eviction_time); return PMIX_SUCCESS; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_progress_threads.c b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_progress_threads.c index df0af87c280..a66e4d0a768 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_progress_threads.c +++ b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_progress_threads.c @@ -1,8 +1,10 @@ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2019 Mellanox Technologies, Inc. + * All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -119,7 +121,7 @@ static void stop_progress_engine(pmix_progress_tracker_t *trk) /* break the event loop - this will cause the loop to exit upon completion of any current event */ - pmix_event_base_loopbreak(trk->ev_base); + pmix_event_base_loopexit(trk->ev_base); pmix_thread_join(&trk->engine, NULL); } diff --git a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c index 153bc9d66a4..6fb30026291 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c @@ -1,13 +1,14 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2014-2017 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. - * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2018 IBM Corporation. All rights reserved. + * Copyright (c) 2018 Cisco Systems, Inc. All rights reserved * $COPYRIGHT$ * * Additional copyrights may follow @@ -53,13 +54,14 @@ #include "src/util/argv.h" #include "src/util/error.h" +#include "src/util/name_fns.h" #include "src/util/output.h" #include "src/util/pmix_environ.h" #include "src/util/show_help.h" #include "src/mca/base/base.h" #include "src/mca/base/pmix_mca_base_var.h" #include "src/mca/pinstalldirs/base/base.h" -#include "src/mca/pnet/pnet.h" +#include "src/mca/pnet/base/base.h" #include "src/runtime/pmix_progress_threads.h" #include "src/runtime/pmix_rte.h" #include "src/mca/bfrops/base/base.h" @@ -84,9 +86,70 @@ static char *gds_mode = NULL; static pid_t mypid; // local functions for connection support -static void server_message_handler(struct pmix_peer_t *pr, - pmix_ptl_hdr_t *hdr, - pmix_buffer_t *buf, void *cbdata); +pmix_status_t pmix_server_initialize(void) +{ + /* setup the server-specific globals */ + PMIX_CONSTRUCT(&pmix_server_globals.clients, pmix_pointer_array_t); + pmix_pointer_array_init(&pmix_server_globals.clients, 1, INT_MAX, 1); + PMIX_CONSTRUCT(&pmix_server_globals.collectives, pmix_list_t); + PMIX_CONSTRUCT(&pmix_server_globals.remote_pnd, pmix_list_t); + PMIX_CONSTRUCT(&pmix_server_globals.gdata, pmix_list_t); + PMIX_CONSTRUCT(&pmix_server_globals.events, pmix_list_t); + PMIX_CONSTRUCT(&pmix_server_globals.local_reqs, pmix_list_t); + PMIX_CONSTRUCT(&pmix_server_globals.nspaces, pmix_list_t); + + pmix_output_verbose(2, pmix_server_globals.base_output, + "pmix:server init called"); + + /* setup the server verbosities */ + if (0 < pmix_server_globals.get_verbose) { + /* set default output */ + pmix_server_globals.get_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_server_globals.get_output, + pmix_server_globals.get_verbose); + } + if (0 < pmix_server_globals.connect_verbose) { + /* set default output */ + pmix_server_globals.connect_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_server_globals.connect_output, + pmix_server_globals.connect_verbose); + } + if (0 < pmix_server_globals.fence_verbose) { + /* set default output */ + pmix_server_globals.fence_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_server_globals.fence_output, + pmix_server_globals.fence_verbose); + } + if (0 < pmix_server_globals.pub_verbose) { + /* set default output */ + pmix_server_globals.pub_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_server_globals.pub_output, + pmix_server_globals.pub_verbose); + } + if (0 < pmix_server_globals.spawn_verbose) { + /* set default output */ + pmix_server_globals.spawn_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_server_globals.spawn_output, + pmix_server_globals.spawn_verbose); + } + if (0 < pmix_server_globals.event_verbose) { + /* set default output */ + pmix_server_globals.event_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_server_globals.event_output, + pmix_server_globals.event_verbose); + } + /* setup the base verbosity */ + if (0 < pmix_server_globals.base_verbose) { + /* set default output */ + pmix_server_globals.base_output = pmix_output_open(NULL); + pmix_output_set_verbosity(pmix_server_globals.base_output, + pmix_server_globals.base_verbose); + } + + return PMIX_SUCCESS; +} + +static pmix_server_module_t myhostserver = {0}; PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, pmix_info_t info[], size_t ninfo) @@ -107,36 +170,58 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, }; char *evar; pmix_rank_info_t *rinfo; + pmix_proc_type_t ptype = PMIX_PROC_SERVER; PMIX_ACQUIRE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server init called"); + /* setup the function pointers */ + if (NULL == module) { + pmix_host_server = myhostserver; + } else { + pmix_host_server = *module; + } + + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_SERVER_TMPDIR, PMIX_MAX_KEYLEN)) { + pmix_server_globals.tmpdir = strdup(info[n].value.data.string); + } else if (0 == strncmp(info[n].key, PMIX_SYSTEM_TMPDIR, PMIX_MAX_KEYLEN)) { + pmix_server_globals.system_tmpdir = strdup(info[n].value.data.string); + } + } + } + if (NULL == pmix_server_globals.tmpdir) { + if (NULL == (evar = getenv("PMIX_SERVER_TMPDIR"))) { + pmix_server_globals.tmpdir = strdup(pmix_tmp_directory()); + } else { + pmix_server_globals.tmpdir = strdup(evar); + } + } + if (NULL == pmix_server_globals.system_tmpdir) { + if (NULL == (evar = getenv("PMIX_SYSTEM_TMPDIR"))) { + pmix_server_globals.system_tmpdir = strdup(pmix_tmp_directory()); + } else { + pmix_server_globals.system_tmpdir = strdup(evar); + } + } + /* setup the runtime - this init's the globals, * opens and initializes the required frameworks */ - if (PMIX_SUCCESS != (rc = pmix_rte_init(PMIX_PROC_SERVER, info, ninfo, NULL))) { + if (PMIX_SUCCESS != (rc = pmix_rte_init(ptype, info, ninfo, NULL))) { PMIX_ERROR_LOG(rc); PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } /* setup the server-specific globals */ - PMIX_CONSTRUCT(&pmix_server_globals.clients, pmix_pointer_array_t); - pmix_pointer_array_init(&pmix_server_globals.clients, 1, INT_MAX, 1); - PMIX_CONSTRUCT(&pmix_server_globals.collectives, pmix_list_t); - PMIX_CONSTRUCT(&pmix_server_globals.remote_pnd, pmix_list_t); - PMIX_CONSTRUCT(&pmix_server_globals.gdata, pmix_list_t); - PMIX_CONSTRUCT(&pmix_server_globals.events, pmix_list_t); - PMIX_CONSTRUCT(&pmix_server_globals.local_reqs, pmix_list_t); - PMIX_CONSTRUCT(&pmix_server_globals.nspaces, pmix_list_t); - - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:server init called"); - - /* setup the function pointers */ - memset(&pmix_host_server, 0, sizeof(pmix_server_module_t)); - pmix_host_server = *module; + if (PMIX_SUCCESS != (rc = pmix_server_initialize())) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } /* assign our internal bfrops module */ pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(NULL); @@ -172,15 +257,13 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } + PMIX_INFO_DESTRUCT(&ginfo); - /* copy need parts over to the client_globals.myserver field + /* copy needed parts over to the client_globals.myserver field * so that calls into client-side functions will use our peer */ pmix_client_globals.myserver = PMIX_NEW(pmix_peer_t); PMIX_RETAIN(pmix_globals.mypeer->nptr); pmix_client_globals.myserver->nptr = pmix_globals.mypeer->nptr; - /* construct the global notification ring buffer */ - PMIX_CONSTRUCT(&pmix_globals.notifications, pmix_ring_buffer_t); - pmix_ring_buffer_init(&pmix_globals.notifications, 256); /* get our available security modules */ security_mode = pmix_psec_base_get_available_modules(); @@ -200,39 +283,37 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, if (NULL != info) { for (n=0; n < ninfo; n++) { if (0 == strncmp(info[n].key, PMIX_SERVER_NSPACE, PMIX_MAX_KEYLEN)) { - (void)strncpy(pmix_globals.myid.nspace, info[n].value.data.string, PMIX_MAX_NSLEN); + pmix_strncpy(pmix_globals.myid.nspace, info[n].value.data.string, PMIX_MAX_NSLEN); nspace_given = true; - continue; - } - if (0 == strncmp(info[n].key, PMIX_SERVER_RANK, PMIX_MAX_KEYLEN)) { + } else if (0 == strncmp(info[n].key, PMIX_SERVER_RANK, PMIX_MAX_KEYLEN)) { pmix_globals.myid.rank = info[n].value.data.rank; rank_given = true; - continue; - } - /* check the list of protected keys */ - protect = false; - for (m=0; NULL != protected[m]; m++) { - if (0 == strcmp(info[n].key, protected[m])) { - protect = true; - break; + } else { + /* check the list of protected keys */ + protect = false; + for (m=0; NULL != protected[m]; m++) { + if (0 == strcmp(info[n].key, protected[m])) { + protect = true; + break; + } } + if (protect) { + continue; + } + /* store and pass along to every client */ + kv = PMIX_NEW(pmix_kval_t); + kv->key = strdup(info[n].key); + PMIX_VALUE_CREATE(kv->value, 1); + PMIX_BFROPS_VALUE_XFER(rc, pmix_globals.mypeer, + kv->value, &info[n].value); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(kv); + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + pmix_list_append(&pmix_server_globals.gdata, &kv->super); } - if (protect) { - continue; - } - /* store and pass along to every client */ - kv = PMIX_NEW(pmix_kval_t); - kv->key = strdup(info[n].key); - PMIX_VALUE_CREATE(kv->value, 1); - PMIX_BFROPS_VALUE_XFER(rc, pmix_globals.mypeer, - kv->value, &info[n].value); - if (PMIX_SUCCESS != rc) { - PMIX_RELEASE(kv); - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - pmix_list_append(&pmix_server_globals.gdata, &kv->super); } } @@ -240,9 +321,9 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, /* look for our namespace, if one was given */ if (NULL == (evar = getenv("PMIX_SERVER_NAMESPACE"))) { /* use a fake namespace */ - (void)strncpy(pmix_globals.myid.nspace, "pmix-server", PMIX_MAX_NSLEN); + pmix_strncpy(pmix_globals.myid.nspace, "pmix-server", PMIX_MAX_NSLEN); } else { - (void)strncpy(pmix_globals.myid.nspace, evar, PMIX_MAX_NSLEN); + pmix_strncpy(pmix_globals.myid.nspace, evar, PMIX_MAX_NSLEN); } } if (!rank_given) { @@ -264,7 +345,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, rinfo = pmix_globals.mypeer->info; } if (NULL == pmix_globals.mypeer->nptr) { - pmix_globals.mypeer->nptr = PMIX_NEW(pmix_nspace_t); + pmix_globals.mypeer->nptr = PMIX_NEW(pmix_namespace_t); /* ensure our own nspace is first on the list */ PMIX_RETAIN(pmix_globals.mypeer->nptr); pmix_list_prepend(&pmix_server_globals.nspaces, &pmix_globals.mypeer->nptr->super); @@ -277,6 +358,16 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, PMIX_RETAIN(pmix_globals.mypeer->info); pmix_client_globals.myserver->info = pmix_globals.mypeer->info; + /* open the pnet framework and select the active modules for this environment */ + if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pnet_base_framework, 0))) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + if (PMIX_SUCCESS != (rc = pmix_pnet_base_select())) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + /* open the psensor framework */ if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_psensor_base_framework, 0))) { PMIX_RELEASE_THREAD(&pmix_global_lock); @@ -290,7 +381,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, /* setup the wildcard recv for inbound messages from clients */ req = PMIX_NEW(pmix_ptl_posted_recv_t); req->tag = UINT32_MAX; - req->cbfunc = server_message_handler; + req->cbfunc = pmix_server_message_handler; /* add it to the end of the list of recvs */ pmix_list_append(&pmix_ptl_globals.posted_recvs, &req->super); @@ -312,6 +403,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) { int i; pmix_peer_t *peer; + pmix_namespace_t *ns; PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { @@ -325,9 +417,8 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) return PMIX_SUCCESS; } pmix_globals.init_cntr = 0; - PMIX_RELEASE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server finalize called"); if (!pmix_globals.external_evbase) { @@ -342,6 +433,10 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) for (i=0; i < pmix_server_globals.clients.size; i++) { if (NULL != (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, i))) { + /* ensure that we do the specified cleanup - if this is an + * abnormal termination, then the peer object may not be + * at zero refcount */ + pmix_execute_epilog(&peer->epilog); PMIX_RELEASE(peer); } } @@ -351,6 +446,12 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) PMIX_LIST_DESTRUCT(&pmix_server_globals.local_reqs); PMIX_LIST_DESTRUCT(&pmix_server_globals.gdata); PMIX_LIST_DESTRUCT(&pmix_server_globals.events); + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { + /* ensure that we do the specified cleanup - if this is an + * abnormal termination, then the nspace object may not be + * at zero refcount */ + pmix_execute_epilog(&ns->epilog); + } PMIX_LIST_DESTRUCT(&pmix_server_globals.nspaces); if (NULL != security_mode) { @@ -368,40 +469,61 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) if (NULL != gds_mode) { free(gds_mode); } - + if (NULL != pmix_server_globals.tmpdir) { + free(pmix_server_globals.tmpdir); + } /* close the psensor framework */ (void)pmix_mca_base_framework_close(&pmix_psensor_base_framework); + /* close the pnet framework */ + (void)pmix_mca_base_framework_close(&pmix_pnet_base_framework); + + + PMIX_RELEASE_THREAD(&pmix_global_lock); + PMIX_DESTRUCT_LOCK(&pmix_global_lock); pmix_rte_finalize(); + if (NULL != pmix_globals.mypeer) { + PMIX_RELEASE(pmix_globals.mypeer); + } - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server finalize complete"); + /* finalize the class/object system */ + pmix_class_finalize(); + return PMIX_SUCCESS; } +static void opcbfunc(pmix_status_t status, void *cbdata) +{ + pmix_lock_t *lock = (pmix_lock_t*)cbdata; + lock->status = status; + PMIX_WAKEUP_THREAD(lock); +} + static void _register_nspace(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; - pmix_nspace_t *nptr, *tmp; + pmix_namespace_t *nptr, *tmp; pmix_status_t rc; size_t i; PMIX_ACQUIRE_OBJECT(caddy); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server _register_nspace %s", cd->proc.nspace); /* see if we already have this nspace */ nptr = NULL; - PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(tmp->nspace, cd->proc.nspace)) { nptr = tmp; break; } } if (NULL == nptr) { - nptr = PMIX_NEW(pmix_nspace_t); + nptr = PMIX_NEW(pmix_namespace_t); if (NULL == nptr) { rc = PMIX_ERR_NOMEM; goto release; @@ -439,18 +561,18 @@ static void _register_nspace(int sd, short args, void *cbdata) cd->info, cd->ninfo); release: - if (NULL != cd->opcbfunc) { - cd->opcbfunc(rc, cd->cbdata); - } + cd->opcbfunc(rc, cd->cbdata); PMIX_RELEASE(cd); } /* setup the data for a job */ -PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const char nspace[], int nlocalprocs, +PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const pmix_nspace_t nspace, int nlocalprocs, pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_setup_caddy_t *cd; + pmix_status_t rc; + pmix_lock_t mylock; PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { @@ -460,7 +582,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const char nspace[], int n PMIX_RELEASE_THREAD(&pmix_global_lock); cd = PMIX_NEW(pmix_setup_caddy_t); - (void)strncpy(cd->proc.nspace, nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->proc.nspace, nspace, PMIX_MAX_NSLEN); cd->nlocalprocs = nlocalprocs; cd->opcbfunc = cbfunc; cd->cbdata = cbdata; @@ -470,53 +592,157 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const char nspace[], int n cd->info = info; } + /* if the provided callback is NULL, then substitute + * our own internal cbfunc and block here */ + if (NULL == cbfunc) { + PMIX_CONSTRUCT_LOCK(&mylock); + cd->opcbfunc = opcbfunc; + cd->cbdata = &mylock; + PMIX_THREADSHIFT(cd, _register_nspace); + PMIX_WAIT_THREAD(&mylock); + rc = mylock.status; + PMIX_DESTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS == rc) { + rc = PMIX_OPERATION_SUCCEEDED; + } + return rc; + } + /* we have to push this into our event library to avoid * potential threading issues */ PMIX_THREADSHIFT(cd, _register_nspace); return PMIX_SUCCESS; } +void pmix_server_purge_events(pmix_peer_t *peer, + pmix_proc_t *proc) +{ + pmix_regevents_info_t *reginfo, *regnext; + pmix_peer_events_info_t *prev, *pnext; + int i; + pmix_notify_caddy_t *ncd; + size_t n, m, p, ntgs; + pmix_proc_t *tgs, *tgt; + pmix_dmdx_local_t *dlcd, *dnxt; + + /* since the client is finalizing, remove them from any event + * registrations they may still have on our list */ + PMIX_LIST_FOREACH_SAFE(reginfo, regnext, &pmix_server_globals.events, pmix_regevents_info_t) { + PMIX_LIST_FOREACH_SAFE(prev, pnext, ®info->peers, pmix_peer_events_info_t) { + if ((NULL != peer && prev->peer == peer) || + (NULL != proc && PMIX_CHECK_PROCID(proc, &prev->peer->info->pname))) { + pmix_list_remove_item(®info->peers, &prev->super); + PMIX_RELEASE(prev); + if (0 == pmix_list_get_size(®info->peers)) { + pmix_list_remove_item(&pmix_server_globals.events, ®info->super); + PMIX_RELEASE(reginfo); + break; + } + } + } + } + + /* see if this proc is involved in any direct modex requests */ + PMIX_LIST_FOREACH_SAFE(dlcd, dnxt, &pmix_server_globals.local_reqs, pmix_dmdx_local_t) { + if ((NULL != peer && PMIX_CHECK_PROCID(&peer->info->pname, &dlcd->proc)) || + (NULL != proc && PMIX_CHECK_PROCID(proc, &dlcd->proc))) { + /* cleanup this request */ + pmix_list_remove_item(&pmix_server_globals.local_reqs, &dlcd->super); + /* we can release the dlcd item here because we are not + * releasing the tracker held by the host - we are only + * releasing one item on that tracker */ + PMIX_RELEASE(dlcd); + } + } + + /* purge this client from any cached notifications */ + for (i=0; i < pmix_globals.max_events; i++) { + pmix_hotel_knock(&pmix_globals.notifications, i, (void**)&ncd); + if (NULL != ncd && NULL != ncd->targets && 0 < ncd->ntargets) { + tgt = NULL; + for (n=0; n < ncd->ntargets; n++) { + if ((NULL != peer && PMIX_CHECK_PROCID(&peer->info->pname, &ncd->targets[n])) || + (NULL != proc && PMIX_CHECK_PROCID(proc, &ncd->targets[n]))) { + tgt = &ncd->targets[n]; + break; + } + } + if (NULL != tgt) { + /* if this client was the only target, then just + * evict the notification */ + if (1 == ncd->ntargets) { + pmix_hotel_checkout(&pmix_globals.notifications, i); + PMIX_RELEASE(ncd); + } else if (PMIX_RANK_WILDCARD == tgt->rank && + NULL != proc && PMIX_RANK_WILDCARD == proc->rank) { + /* we have to remove this target, but leave the rest */ + ntgs = ncd->ntargets - 1; + PMIX_PROC_CREATE(tgs, ntgs); + p=0; + for (m=0; m < ncd->ntargets; m++) { + if (tgt != &ncd->targets[m]) { + memcpy(&tgs[p], &ncd->targets[n], sizeof(pmix_proc_t)); + ++p; + } + } + PMIX_PROC_FREE(ncd->targets, ncd->ntargets); + ncd->targets = tgs; + ncd->ntargets = ntgs; + } + } + } + } + + if (NULL != peer) { + /* ensure we honor any peer-level epilog requests */ + pmix_execute_epilog(&peer->epilog); + } +} + static void _deregister_nspace(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; - pmix_nspace_t *tmp; + pmix_namespace_t *tmp; pmix_status_t rc; PMIX_ACQUIRE_OBJECT(cd); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server _deregister_nspace %s", cd->proc.nspace); - /* see if we already have this nspace */ - PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { - if (0 == strcmp(tmp->nspace, cd->proc.nspace)) { + /* let our local storage clean up */ + PMIX_GDS_DEL_NSPACE(rc, cd->proc.nspace); + + /* remove any event registrations and + * cached notifications targeting procs from this nspace */ + pmix_server_purge_events(NULL, &cd->proc); + + /* release this nspace */ + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { + if (PMIX_CHECK_NSPACE(tmp->nspace, cd->proc.nspace)) { + /* perform any nspace-level epilog */ + pmix_execute_epilog(&tmp->epilog); + /* remove and release it */ pmix_list_remove_item(&pmix_server_globals.nspaces, &tmp->super); PMIX_RELEASE(tmp); break; } } - /* let our local storage clean up */ - PMIX_GDS_DEL_NSPACE(rc, cd->proc.nspace); - - /* release any job-level messaging resources */ - pmix_pnet.local_app_finalized(cd->proc.nspace); - /* release the caller */ - if (NULL != cd->opcbfunc) { - cd->opcbfunc(rc, cd->cbdata); - } + cd->opcbfunc(rc, cd->cbdata); PMIX_RELEASE(cd); } -PMIX_EXPORT void PMIx_server_deregister_nspace(const char nspace[], +PMIX_EXPORT void PMIx_server_deregister_nspace(const pmix_nspace_t nspace, pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_setup_caddy_t *cd; + pmix_lock_t mylock; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server deregister nspace %s", nspace); @@ -530,11 +756,23 @@ PMIX_EXPORT void PMIx_server_deregister_nspace(const char nspace[], } PMIX_RELEASE_THREAD(&pmix_global_lock); - cd = PMIX_NEW(pmix_setup_caddy_t); - (void)strncpy(cd->proc.nspace, nspace, PMIX_MAX_NSLEN); + cd = PMIX_NEW(pmix_setup_caddy_t); + PMIX_LOAD_PROCID(&cd->proc, nspace, PMIX_RANK_WILDCARD); cd->opcbfunc = cbfunc; cd->cbdata = cbdata; + /* if the provided callback is NULL, then substitute + * our own internal cbfunc and block here */ + if (NULL == cbfunc) { + PMIX_CONSTRUCT_LOCK(&mylock); + cd->opcbfunc = opcbfunc; + cd->cbdata = &mylock; + PMIX_THREADSHIFT(cd, _deregister_nspace); + PMIX_WAIT_THREAD(&mylock); + PMIX_DESTRUCT_LOCK(&mylock); + return; + } + /* we have to push this into our event library to avoid * potential threading issues */ PMIX_THREADSHIFT(cd, _deregister_nspace); @@ -598,7 +836,7 @@ void pmix_server_execute_collective(int sd, short args, void *cbdata) PMIX_BFROPS_PACK(rc, peer, &bucket, &tmp, 1, PMIX_BYTE); if (PMIX_COLLECT_YES == trk->collect_type) { - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "fence - assembling data"); first = true; PMIX_CONSTRUCT(&pnames, pmix_list_t); @@ -621,7 +859,7 @@ void pmix_server_execute_collective(int sd, short args, void *cbdata) } if (trk->hybrid || first) { /* setup the nspace */ - (void)strncpy(proc.nspace, cd->peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, cd->peer->info->pname.nspace, PMIX_MAX_NSLEN); first = false; } proc.rank = cd->peer->info->pname.rank; @@ -698,7 +936,7 @@ static void _register_client(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; pmix_rank_info_t *info, *iptr; - pmix_nspace_t *nptr, *ns; + pmix_namespace_t *nptr, *ns; pmix_server_trkr_t *trk; pmix_trkr_caddy_t *tcd; bool all_def; @@ -707,20 +945,21 @@ static void _register_client(int sd, short args, void *cbdata) PMIX_ACQUIRE_OBJECT(cd); - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:server _register_client for nspace %s rank %d", - cd->proc.nspace, cd->proc.rank); + pmix_output_verbose(2, pmix_server_globals.base_output, + "pmix:server _register_client for nspace %s rank %d %s object", + cd->proc.nspace, cd->proc.rank, + (NULL == cd->server_object) ? "NULL" : "NON-NULL"); /* see if we already have this nspace */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(ns->nspace, cd->proc.nspace)) { nptr = ns; break; } } if (NULL == nptr) { - nptr = PMIX_NEW(pmix_nspace_t); + nptr = PMIX_NEW(pmix_namespace_t); if (NULL == nptr) { rc = PMIX_ERR_NOMEM; goto cleanup; @@ -767,7 +1006,7 @@ static void _register_client(int sd, short args, void *cbdata) * if the nspaces are all defined */ if (all_def) { /* so far, they have all been defined - check this one */ - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 < ns->nlocalprocs && 0 == strcmp(trk->pcs[i].nspace, ns->nspace)) { all_def = ns->all_registered; @@ -810,9 +1049,7 @@ static void _register_client(int sd, short args, void *cbdata) cleanup: /* let the caller know we are done */ - if (NULL != cd->opcbfunc) { - cd->opcbfunc(rc, cd->cbdata); - } + cd->opcbfunc(rc, cd->cbdata); PMIX_RELEASE(cd); } @@ -821,6 +1058,8 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_client(const pmix_proc_t *proc, pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_setup_caddy_t *cd; + pmix_status_t rc; + pmix_lock_t mylock; PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { @@ -829,7 +1068,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_client(const pmix_proc_t *proc, } PMIX_RELEASE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server register client %s:%d", proc->nspace, proc->rank); @@ -837,7 +1076,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_client(const pmix_proc_t *proc, if (NULL == cd) { return PMIX_ERR_NOMEM; } - (void)strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); cd->proc.rank = proc->rank; cd->uid = uid; cd->gid = gid; @@ -845,6 +1084,22 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_client(const pmix_proc_t *proc, cd->opcbfunc = cbfunc; cd->cbdata = cbdata; + /* if the provided callback is NULL, then substitute + * our own internal cbfunc and block here */ + if (NULL == cbfunc) { + PMIX_CONSTRUCT_LOCK(&mylock); + cd->opcbfunc = opcbfunc; + cd->cbdata = &mylock; + PMIX_THREADSHIFT(cd, _register_client); + PMIX_WAIT_THREAD(&mylock); + rc = mylock.status; + PMIX_DESTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS == rc) { + rc = PMIX_OPERATION_SUCCEEDED; + } + return rc; + } + /* we have to push this into our event library to avoid * potential threading issues */ PMIX_THREADSHIFT(cd, _register_client); @@ -855,18 +1110,18 @@ static void _deregister_client(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; pmix_rank_info_t *info; - pmix_nspace_t *nptr, *tmp; + pmix_namespace_t *nptr, *tmp; pmix_peer_t *peer; PMIX_ACQUIRE_OBJECT(cd); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server _deregister_client for nspace %s rank %d", cd->proc.nspace, cd->proc.rank); /* see if we already have this nspace */ nptr = NULL; - PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(tmp->nspace, cd->proc.nspace)) { nptr = tmp; break; @@ -879,8 +1134,44 @@ static void _deregister_client(int sd, short args, void *cbdata) /* find and remove this client */ PMIX_LIST_FOREACH(info, &nptr->ranks, pmix_rank_info_t) { if (info->pname.rank == cd->proc.rank) { - if (NULL != (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, info->peerid))) { - pmix_psensor.stop(peer, NULL); + /* if this client failed to call finalize, we still need + * to restore any allocations that were given to it */ + if (NULL == (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, info->peerid))) { + /* this peer never connected, and hence it won't finalize, + * so account for it here */ + nptr->nfinalized++; + /* even if they never connected, resources were allocated + * to them, so we need to ensure they are properly released */ + pmix_pnet.child_finalized(peer); + } else { + if (!peer->finalized) { + /* this peer connected to us, but is being deregistered + * without having finalized. This usually means an + * abnormal termination that was picked up by + * our host prior to our seeing the connection drop. + * It is also possible that we missed the dropped + * connection, so mark the peer as finalized so + * we don't duplicate account for it and take care + * of it here */ + peer->finalized = true; + nptr->nfinalized++; + } + /* resources may have been allocated to them, so + * ensure they get cleaned up - this isn't true + * for tools, so don't clean them up */ + if (!PMIX_PROC_IS_TOOL(peer)) { + pmix_pnet.child_finalized(peer); + pmix_psensor.stop(peer, NULL); + } + /* honor any registered epilogs */ + pmix_execute_epilog(&peer->epilog); + /* ensure we close the socket to this peer so we don't + * generate "connection lost" events should it be + * subsequently "killed" by the host */ + CLOSE_THE_SOCKET(peer->sd); + } + if (nptr->nlocalprocs == nptr->nfinalized) { + pmix_pnet.local_app_finalized(cd->proc.nspace); } pmix_list_remove_item(&nptr->ranks, &info->super); PMIX_RELEASE(info); @@ -889,9 +1180,7 @@ static void _deregister_client(int sd, short args, void *cbdata) } cleanup: - if (NULL != cd->opcbfunc) { - cd->opcbfunc(PMIX_SUCCESS, cd->cbdata); - } + cd->opcbfunc(PMIX_SUCCESS, cd->cbdata); PMIX_RELEASE(cd); } @@ -899,6 +1188,7 @@ PMIX_EXPORT void PMIx_server_deregister_client(const pmix_proc_t *proc, pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_setup_caddy_t *cd; + pmix_lock_t mylock; PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { @@ -910,7 +1200,7 @@ PMIX_EXPORT void PMIx_server_deregister_client(const pmix_proc_t *proc, } PMIX_RELEASE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server deregister client %s:%d", proc->nspace, proc->rank); @@ -921,11 +1211,23 @@ PMIX_EXPORT void PMIx_server_deregister_client(const pmix_proc_t *proc, } return; } - (void)strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); cd->proc.rank = proc->rank; cd->opcbfunc = cbfunc; cd->cbdata = cbdata; + /* if the provided callback is NULL, then substitute + * our own internal cbfunc and block here */ + if (NULL == cbfunc) { + PMIX_CONSTRUCT_LOCK(&mylock); + cd->opcbfunc = opcbfunc; + cd->cbdata = &mylock; + PMIX_THREADSHIFT(cd, _deregister_client); + PMIX_WAIT_THREAD(&mylock); + PMIX_DESTRUCT_LOCK(&mylock); + return; + } + /* we have to push this into our event library to avoid * potential threading issues */ PMIX_THREADSHIFT(cd, _deregister_client); @@ -947,7 +1249,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_fork(const pmix_proc_t *proc, char * } PMIX_RELEASE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server setup_fork for nspace %s rank %d", proc->nspace, proc->rank); @@ -979,6 +1281,12 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_fork(const pmix_proc_t *proc, char * /* pass our available gds modules */ pmix_setenv("PMIX_GDS_MODULE", gds_mode, true, env); + /* get any PTL contribution such as tmpdir settings for session files */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_setup_fork(proc, env))) { + PMIX_ERROR_LOG(rc); + return rc; + } + /* get any network contribution */ if (PMIX_SUCCESS != (rc = pmix_pnet.setup_fork(proc, env))) { PMIX_ERROR_LOG(rc); @@ -1003,7 +1311,7 @@ static void _dmodex_req(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; pmix_rank_info_t *info, *iptr; - pmix_nspace_t *nptr, *ns; + pmix_namespace_t *nptr, *ns; char *data = NULL; size_t sz = 0; pmix_dmdx_remote_t *dcd; @@ -1014,7 +1322,7 @@ static void _dmodex_req(int sd, short args, void *cbdata) PMIX_ACQUIRE_OBJECT(cd); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "DMODX LOOKING FOR %s:%d", cd->proc.nspace, cd->proc.rank); @@ -1023,7 +1331,7 @@ static void _dmodex_req(int sd, short args, void *cbdata) * been informed of it - so first check to see if we know * about this nspace yet */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(ns->nspace, cd->proc.nspace)) { nptr = ns; break; @@ -1038,10 +1346,8 @@ static void _dmodex_req(int sd, short args, void *cbdata) rc = PMIX_ERR_NOMEM; goto cleanup; } - PMIX_RETAIN(cd); dcd->cd = cd; pmix_list_append(&pmix_server_globals.remote_pnd, &dcd->super); - PMIX_WAKEUP_THREAD(&cd->lock); // ensure the request doesn't hang return; } @@ -1070,13 +1376,7 @@ static void _dmodex_req(int sd, short args, void *cbdata) PMIX_DESTRUCT(&cb); PMIX_UNLOAD_BUFFER(&pbkt, data, sz); PMIX_DESTRUCT(&pbkt); - /* execute the callback */ - cd->cbfunc(rc, data, sz, cd->cbdata); - PMIX_WAKEUP_THREAD(&cd->lock); // ensure the request doesn't hang - if (NULL != data) { - free(data); - } - return; + goto cleanup; } /* see if we have this peer in our list */ @@ -1091,10 +1391,8 @@ static void _dmodex_req(int sd, short args, void *cbdata) /* rank isn't known yet - defer * the request until we do */ dcd = PMIX_NEW(pmix_dmdx_remote_t); - PMIX_RETAIN(cd); dcd->cd = cd; pmix_list_append(&pmix_server_globals.remote_pnd, &dcd->super); - PMIX_WAKEUP_THREAD(&cd->lock); // ensure the request doesn't hang return; } @@ -1104,10 +1402,8 @@ static void _dmodex_req(int sd, short args, void *cbdata) /* track the request so we can fulfill it once * data is recvd */ dcd = PMIX_NEW(pmix_dmdx_remote_t); - PMIX_RETAIN(cd); dcd->cd = cd; pmix_list_append(&pmix_server_globals.remote_pnd, &dcd->super); - PMIX_WAKEUP_THREAD(&cd->lock); // ensure the request doesn't hang return; } @@ -1139,7 +1435,7 @@ static void _dmodex_req(int sd, short args, void *cbdata) if (NULL != data) { free(data); } - PMIX_WAKEUP_THREAD(&cd->lock); + PMIX_RELEASE(cd); } PMIX_EXPORT pmix_status_t PMIx_server_dmodex_request(const pmix_proc_t *proc, @@ -1160,12 +1456,12 @@ PMIX_EXPORT pmix_status_t PMIx_server_dmodex_request(const pmix_proc_t *proc, return PMIX_ERR_BAD_PARAM; } - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server dmodex request%s:%d", proc->nspace, proc->rank); cd = PMIX_NEW(pmix_setup_caddy_t); - (void)strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); cd->proc.rank = proc->rank; cd->cbfunc = cbfunc; cd->cbdata = cbdata; @@ -1173,9 +1469,6 @@ PMIX_EXPORT pmix_status_t PMIx_server_dmodex_request(const pmix_proc_t *proc, /* we have to push this into our event library to avoid * potential threading issues */ PMIX_THREADSHIFT(cd, _dmodex_req); - - PMIX_WAIT_THREAD(&cd->lock); - PMIX_RELEASE(cd); return PMIX_SUCCESS; } @@ -1186,7 +1479,7 @@ static void _store_internal(int sd, short args, void *cbdata) PMIX_ACQUIRE_OBJECT(cd); - (void)strncpy(proc.nspace, cd->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, cd->pname.nspace, PMIX_MAX_NSLEN); proc.rank = cd->pname.rank; PMIX_GDS_STORE_KV(cd->status, pmix_globals.mypeer, &proc, PMIX_INTERNAL, cd->kv); @@ -1196,7 +1489,7 @@ static void _store_internal(int sd, short args, void *cbdata) } PMIX_EXPORT pmix_status_t PMIx_Store_internal(const pmix_proc_t *proc, - const char *key, pmix_value_t *val) + const pmix_key_t key, pmix_value_t *val) { pmix_shift_caddy_t *cd; pmix_status_t rc; @@ -1275,19 +1568,18 @@ static void _setup_op(pmix_status_t rc, void *cbdata) static void _setup_app(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; - pmix_buffer_t buffer; - pmix_byte_object_t blob; pmix_setup_caddy_t *fcd = NULL; pmix_status_t rc; pmix_list_t ilist; pmix_kval_t *kv; + size_t n; PMIX_ACQUIRE_OBJECT(cd); PMIX_CONSTRUCT(&ilist, pmix_list_t); - /* pass to the network libraries */ - if (PMIX_SUCCESS != (rc = pmix_pnet.setup_app(cd->nspace, &ilist))) { + rc = pmix_pnet.setup_app(cd->nspace, &ilist); + if (PMIX_SUCCESS != rc) { goto depart; } @@ -1299,31 +1591,20 @@ static void _setup_app(int sd, short args, void *cbdata) goto depart; } - /* if anything came back, construct the blob */ - if (0 < pmix_list_get_size(&ilist)) { - PMIX_CONSTRUCT(&buffer, pmix_buffer_t); - PMIX_LIST_FOREACH(kv, &ilist, pmix_kval_t) { - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buffer, kv, 1, PMIX_KVAL); - if (PMIX_SUCCESS != rc) { - PMIX_DESTRUCT(&blob); - PMIX_RELEASE(fcd); - fcd = NULL; - goto depart; - } - } - PMIX_INFO_CREATE(fcd->info, 1); + /* if anything came back, construct an info array */ + if (0 < (fcd->ninfo = pmix_list_get_size(&ilist))) { + PMIX_INFO_CREATE(fcd->info, fcd->ninfo); if (NULL == fcd->info) { - PMIX_DESTRUCT(&blob); + rc = PMIX_ERR_NOMEM; PMIX_RELEASE(fcd); - fcd = NULL; goto depart; } - fcd->ninfo = 1; - PMIX_BYTE_OBJECT_CONSTRUCT(&blob); - PMIX_BYTE_OBJECT_LOAD(&blob, buffer.base_ptr, buffer.bytes_used); - PMIX_DESTRUCT(&buffer); - PMIX_INFO_LOAD(&fcd->info[0], PMIX_PNET_SETUP_APP, &blob, PMIX_BYTE_OBJECT); - PMIX_BYTE_OBJECT_DESTRUCT(&blob); + n = 0; + PMIX_LIST_FOREACH(kv, &ilist, pmix_kval_t) { + pmix_strncpy(fcd->info[n].key, kv->key, PMIX_MAX_KEYLEN); + pmix_value_xfer(&fcd->info[n].value, kv->value); + ++n; + } } depart: @@ -1344,7 +1625,7 @@ static void _setup_app(int sd, short args, void *cbdata) PMIX_RELEASE(cd); } -pmix_status_t PMIx_server_setup_application(const char nspace[], +pmix_status_t PMIx_server_setup_application(const pmix_nspace_t nspace, pmix_info_t info[], size_t ninfo, pmix_setup_application_cbfunc_t cbfunc, void *cbdata) { @@ -1381,7 +1662,6 @@ static void _setup_local_support(int sd, short args, void *cbdata) PMIX_ACQUIRE_OBJECT(cd); - /* pass to the network libraries */ rc = pmix_pnet.setup_local_network(cd->nspace, cd->info, cd->ninfo); /* pass the info back */ @@ -1395,7 +1675,7 @@ static void _setup_local_support(int sd, short args, void *cbdata) PMIX_RELEASE(cd); } -pmix_status_t PMIx_server_setup_local_support(const char nspace[], +pmix_status_t PMIx_server_setup_local_support(const pmix_nspace_t nspace, pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { @@ -1425,7 +1705,6 @@ pmix_status_t PMIx_server_setup_local_support(const char nspace[], return PMIX_SUCCESS; } - /**** THE FOLLOWING CALLBACK FUNCTIONS ARE USED BY THE HOST SERVER **** **** THEY THEREFORE CAN OCCUR IN EITHER THE HOST SERVER'S THREAD **** **** CONTEXT, OR IN OUR OWN THREAD CONTEXT IF THE CALLBACK OCCURS **** @@ -1468,6 +1747,61 @@ static void op_cbfunc(pmix_status_t status, void *cbdata) PMIX_RELEASE(cd); } +static void connection_cleanup(int sd, short args, void *cbdata) +{ + pmix_server_caddy_t *cd = (pmix_server_caddy_t*)cbdata; + + /* ensure that we know the peer has finalized else we + * will generate an event - yes, it should have been + * done, but it is REALLY important that it be set */ + cd->peer->finalized = true; + pmix_ptl_base_lost_connection(cd->peer, PMIX_SUCCESS); + /* cleanup the caddy */ + PMIX_RELEASE(cd); +} + +static void op_cbfunc2(pmix_status_t status, void *cbdata) +{ + pmix_server_caddy_t *cd = (pmix_server_caddy_t*)cbdata; + pmix_buffer_t *reply; + pmix_status_t rc; + + /* no need to thread-shift here as no global data is + * being accessed */ + + /* setup the reply with the returned status */ + if (NULL == (reply = PMIX_NEW(pmix_buffer_t))) { + PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); + PMIX_RELEASE(cd); + return; + } + PMIX_BFROPS_PACK(rc, cd->peer, reply, &status, 1, PMIX_STATUS); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(reply); + PMIX_RELEASE(cd); + return; + } + + /* the function that created the server_caddy did a + * retain on the peer, so we don't have to worry about + * it still being present - send a copy to the originator */ + PMIX_PTL_SEND_ONEWAY(rc, cd->peer, reply, cd->hdr.tag); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(reply); + } + + /* cleanup any lingering references to this peer - note + * that we cannot call the lost_connection function + * directly as we need the connection to still + * exist for the message (queued above) to be + * sent. So we push this into an event, thus + * ensuring that it will "fire" after the message + * event has completed */ + PMIX_THREADSHIFT(cd, connection_cleanup); +} + static void _spcb(int sd, short args, void *cbdata) { pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata; @@ -1482,51 +1816,54 @@ static void _spcb(int sd, short args, void *cbdata) /* setup the reply with the returned status */ if (NULL == (reply = PMIX_NEW(pmix_buffer_t))) { PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); - PMIX_RELEASE(cd->cd); - PMIX_WAKEUP_THREAD(&cd->lock); - return; + goto cleanup; } PMIX_BFROPS_PACK(rc, cd->cd->peer, reply, &cd->status, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - PMIX_RELEASE(cd->cd); - PMIX_WAKEUP_THREAD(&cd->lock); - return; + PMIX_RELEASE(reply); + goto cleanup; } - if (PMIX_SUCCESS == cd->status) { - /* pass back the name of the nspace */ - PMIX_BFROPS_PACK(rc, cd->cd->peer, reply, &cd->pname.nspace, 1, PMIX_STRING); - /* add the job-level info, if we have it */ - (void)strncpy(proc.nspace, cd->pname.nspace, PMIX_MAX_NSLEN); - proc.rank = PMIX_RANK_WILDCARD; - /* this is going to a local client, so let the gds - * have the option of returning a copy of the data, - * or a pointer to local storage */ - PMIX_CONSTRUCT(&cb, pmix_cb_t); - cb.proc = &proc; - cb.scope = PMIX_SCOPE_UNDEF; - cb.copy = false; - PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb); - if (PMIX_SUCCESS == rc) { - PMIX_LIST_FOREACH(kv, &cb.kvs, pmix_kval_t) { - PMIX_BFROPS_PACK(rc, cd->cd->peer, reply, kv, 1, PMIX_KVAL); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(cd->cd); - PMIX_RELEASE(reply); - PMIX_DESTRUCT(&cb); - PMIX_WAKEUP_THREAD(&cd->lock); - return; - } + /* pass back the name of the nspace */ + PMIX_BFROPS_PACK(rc, cd->cd->peer, reply, &cd->pname.nspace, 1, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(reply); + goto cleanup; + } + /* add the job-level info, if we have it */ + pmix_strncpy(proc.nspace, cd->pname.nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + /* this is going to a local client, so let the gds + * have the option of returning a copy of the data, + * or a pointer to local storage */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + cb.proc = &proc; + cb.scope = PMIX_SCOPE_UNDEF; + cb.copy = false; + PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb); + if (PMIX_SUCCESS == rc) { + PMIX_LIST_FOREACH(kv, &cb.kvs, pmix_kval_t) { + PMIX_BFROPS_PACK(rc, cd->cd->peer, reply, kv, 1, PMIX_KVAL); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(reply); + PMIX_DESTRUCT(&cb); + goto cleanup; } - PMIX_DESTRUCT(&cb); } + PMIX_DESTRUCT(&cb); } /* the function that created the server_caddy did a * retain on the peer, so we don't have to worry about * it still being present - tell the originator the result */ - PMIX_SERVER_QUEUE_REPLY(cd->cd->peer, cd->cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->cd->peer, cd->cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + + cleanup: /* cleanup */ PMIX_RELEASE(cd->cd); PMIX_RELEASE(cd); @@ -1584,7 +1921,10 @@ static void lookup_cbfunc(pmix_status_t status, pmix_pdata_t pdata[], size_t nda /* the function that created the server_caddy did a * retain on the peer, so we don't have to worry about * it still being present - tell the originator the result */ - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } /* cleanup */ PMIX_RELEASE(cd); } @@ -1598,20 +1938,34 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata) { pmix_shift_caddy_t *scd = (pmix_shift_caddy_t*)cbdata; pmix_server_trkr_t *tracker = scd->tracker; - pmix_buffer_t xfer, *reply, bkt; - pmix_byte_object_t bo, bo2; - pmix_server_caddy_t *cd; + pmix_buffer_t xfer, *reply; + pmix_server_caddy_t *cd, *nxt; pmix_status_t rc = PMIX_SUCCESS, ret; pmix_nspace_caddy_t *nptr; pmix_list_t nslist; - int32_t cnt = 1; - char byte; bool found; - pmix_collect_t ctype; PMIX_ACQUIRE_OBJECT(scd); - /* pass the blobs being returned */ + if (NULL == tracker) { + /* give them a release if they want it - this should + * never happen, but protect against the possibility */ + if (NULL != scd->cbfunc.relfn) { + scd->cbfunc.relfn(scd->cbdata); + } + PMIX_RELEASE(scd); + return; + } + + /* if we get here, then there are processes waiting + * for a response */ + + /* if the timer is active, clear it */ + if (tracker->event_active) { + pmix_event_del(&tracker->ev); + } + + /* pass the blobs being returned */ PMIX_CONSTRUCT(&xfer, pmix_buffer_t); PMIX_LOAD_BUFFER(pmix_globals.mypeer, &xfer, scd->data, scd->ndata); PMIX_CONSTRUCT(&nslist, pmix_list_t); @@ -1632,7 +1986,7 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata) goto finish_collective; } - // collect the pmix_nspace_t's of all local participants + // collect the pmix_namespace_t's of all local participants PMIX_LIST_FOREACH(cd, &tracker->local_cbs, pmix_server_caddy_t) { // see if we already have this nspace found = false; @@ -1651,74 +2005,17 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata) } } - /* Loop over the enclosed byte object envelopes and - * store them in our GDS module */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &xfer, &bo, &cnt, PMIX_BYTE_OBJECT); - while (PMIX_SUCCESS == rc) { - PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt, bo.bytes, bo.size); - /* unpack the data collection flag */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &byte, &cnt, PMIX_BYTE); - if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) { - /* no data was returned, so we are done with this blob */ - break; - } + PMIX_LIST_FOREACH(nptr, &nslist, pmix_nspace_caddy_t) { + PMIX_GDS_STORE_MODEX(rc, nptr->ns, &tracker->local_cbs, &xfer); if (PMIX_SUCCESS != rc) { - /* we have an error */ - break; - } - - // Check that this blob was accumulated with the same data collection setting - ctype = (pmix_collect_t)byte; - if (ctype != tracker->collect_type) { - rc = PMIX_ERR_INVALID_ARG; - break; - } - /* unpack the enclosed blobs from the various peers */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &bo2, &cnt, PMIX_BYTE_OBJECT); - while (PMIX_SUCCESS == rc) { - /* unpack all the kval's from this peer and store them in - * our GDS. Note that PMIx by design holds all data at - * the server level until requested. If our GDS is a - * shared memory region, then the data may be available - * right away - but the client still has to be notified - * of its presence. */ - PMIX_LIST_FOREACH(nptr, &nslist, pmix_nspace_caddy_t) { - PMIX_GDS_STORE_MODEX(rc, nptr->ns, &tracker->local_cbs, &bo2); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - break; - } - } - PMIX_BYTE_OBJECT_DESTRUCT(&bo2); - /* get the next blob */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &bo2, &cnt, PMIX_BYTE_OBJECT); - } - if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) { - rc = PMIX_SUCCESS; - } else if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - goto finish_collective; + break; } - /* unpack and process the next blob */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &xfer, &bo, &cnt, PMIX_BYTE_OBJECT); - } - if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) { - rc = PMIX_SUCCESS; } finish_collective: /* loop across all procs in the tracker, sending them the reply */ - PMIX_LIST_FOREACH(cd, &tracker->local_cbs, pmix_server_caddy_t) { + PMIX_LIST_FOREACH_SAFE(cd, nxt, &tracker->local_cbs, pmix_server_caddy_t) { reply = PMIX_NEW(pmix_buffer_t); if (NULL == reply) { rc = PMIX_ERR_NOMEM; @@ -1730,10 +2027,16 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata) PMIX_ERROR_LOG(ret); goto cleanup; } - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "server:modex_cbfunc reply being sent to %s:%u", cd->peer->info->pname.nspace, cd->peer->info->pname.rank); - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + /* remove this entry */ + pmix_list_remove_item(&tracker->local_cbs, &cd->super); + PMIX_RELEASE(cd); } cleanup: @@ -1763,18 +2066,9 @@ static void modex_cbfunc(pmix_status_t status, const char *data, size_t ndata, v pmix_server_trkr_t *tracker = (pmix_server_trkr_t*)cbdata; pmix_shift_caddy_t *scd; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "server:modex_cbfunc called with %d bytes", (int)ndata); - if (NULL == tracker) { - /* nothing to do - but be sure to give them - * a release if they want it */ - if (NULL != relfn) { - relfn(relcbd); - } - return; - } - /* need to thread-shift this callback as it accesses global data */ scd = PMIX_NEW(pmix_shift_caddy_t); if (NULL == scd) { @@ -1800,10 +2094,13 @@ static void get_cbfunc(pmix_status_t status, const char *data, size_t ndata, voi pmix_buffer_t *reply, buf; pmix_status_t rc; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "server:get_cbfunc called with %d bytes", (int)ndata); - /* no need to thread-shift here as no global data is accessed */ + /* no need to thread-shift here as no global data is accessed + * and we are called from another internal function + * (see pmix_server_get.c:pmix_pending_resolve) that + * has already been thread-shifted */ if (NULL == cd) { /* nothing to do - but be sure to give them @@ -1833,13 +2130,16 @@ static void get_cbfunc(pmix_status_t status, const char *data, size_t ndata, voi buf.bytes_used = 0; PMIX_DESTRUCT(&buf); /* send the data to the requestor */ - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "server:get_cbfunc reply being sent to %s:%u", cd->peer->info->pname.nspace, cd->peer->info->pname.rank); - pmix_output_hexdump(10, pmix_globals.debug_output, + pmix_output_hexdump(10, pmix_server_globals.base_output, reply->base_ptr, (reply->bytes_used < 256 ? reply->bytes_used : 256)); - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } cleanup: /* if someone wants a release, give it to them */ @@ -1866,24 +2166,35 @@ static void _cnct(int sd, short args, void *cbdata) PMIX_ACQUIRE_OBJECT(scd); - if (PMIX_CONNECTNB_CMD == tracker->type) { - /* find the unique nspaces that are participating */ - PMIX_LIST_FOREACH(cd, &tracker->local_cbs, pmix_server_caddy_t) { - if (NULL == nspaces) { - pmix_argv_append_nosize(&nspaces, cd->peer->info->pname.nspace); - } else { - found = false; - for (i=0; NULL != nspaces[i]; i++) { - if (0 == strcmp(nspaces[i], cd->peer->info->pname.nspace)) { - found = true; - break; - } - } - if (!found) { - pmix_argv_append_nosize(&nspaces, cd->peer->info->pname.nspace); - } - } - } + if (NULL == tracker) { + /* nothing to do */ + return; + } + + /* if we get here, then there are processes waiting + * for a response */ + + /* if the timer is active, clear it */ + if (tracker->event_active) { + pmix_event_del(&tracker->ev); + } + + /* find the unique nspaces that are participating */ + PMIX_LIST_FOREACH(cd, &tracker->local_cbs, pmix_server_caddy_t) { + if (NULL == nspaces) { + pmix_argv_append_nosize(&nspaces, cd->peer->info->pname.nspace); + } else { + found = false; + for (i=0; NULL != nspaces[i]; i++) { + if (0 == strcmp(nspaces[i], cd->peer->info->pname.nspace)) { + found = true; + break; + } + } + if (!found) { + pmix_argv_append_nosize(&nspaces, cd->peer->info->pname.nspace); + } + } } /* loop across all local procs in the tracker, sending them the reply */ @@ -1895,13 +2206,14 @@ static void _cnct(int sd, short args, void *cbdata) rc = PMIX_ERR_NOMEM; goto cleanup; } + /* start with the status */ PMIX_BFROPS_PACK(rc, cd->peer, reply, &scd->status, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(reply); goto cleanup; } - if (PMIX_CONNECTNB_CMD == tracker->type) { + if (PMIX_SUCCESS == scd->status) { /* loop across all participating nspaces and include their * job-related info */ for (i=0; NULL != nspaces[i]; i++) { @@ -1916,7 +2228,7 @@ static void _cnct(int sd, short args, void *cbdata) * local storage */ /* add the job-level info, if necessary */ proc.rank = PMIX_RANK_WILDCARD; - (void)strncpy(proc.nspace, nspaces[i], PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nspaces[i], PMIX_MAX_NSLEN); PMIX_CONSTRUCT(&cb, pmix_cb_t); /* this is for a local client, so give the gds the * option of returning a complete copy of the data, @@ -1977,10 +2289,13 @@ static void _cnct(int sd, short args, void *cbdata) PMIX_DESTRUCT(&pbkt); } } - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "server:cnct_cbfunc reply being sent to %s:%u", cd->peer->info->pname.nspace, cd->peer->info->pname.rank); - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } } cleanup: @@ -1999,14 +2314,87 @@ static void cnct_cbfunc(pmix_status_t status, void *cbdata) pmix_server_trkr_t *tracker = (pmix_server_trkr_t*)cbdata; pmix_shift_caddy_t *scd; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "server:cnct_cbfunc called"); + /* need to thread-shift this callback as it accesses global data */ + scd = PMIX_NEW(pmix_shift_caddy_t); + if (NULL == scd) { + /* nothing we can do */ + return; + } + scd->status = status; + scd->tracker = tracker; + PMIX_THREADSHIFT(scd, _cnct); +} + +static void _discnct(int sd, short args, void *cbdata) +{ + pmix_shift_caddy_t *scd = (pmix_shift_caddy_t*)cbdata; + pmix_server_trkr_t *tracker = scd->tracker; + pmix_buffer_t *reply; + pmix_status_t rc; + pmix_server_caddy_t *cd; + + PMIX_ACQUIRE_OBJECT(scd); + if (NULL == tracker) { /* nothing to do */ return; } + /* if we get here, then there are processes waiting + * for a response */ + + /* if the timer is active, clear it */ + if (tracker->event_active) { + pmix_event_del(&tracker->ev); + } + + /* loop across all local procs in the tracker, sending them the reply */ + PMIX_LIST_FOREACH(cd, &tracker->local_cbs, pmix_server_caddy_t) { + /* setup the reply */ + reply = PMIX_NEW(pmix_buffer_t); + if (NULL == reply) { + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + /* return the status */ + PMIX_BFROPS_PACK(rc, cd->peer, reply, &scd->status, 1, PMIX_STATUS); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(reply); + goto cleanup; + } + pmix_output_verbose(2, pmix_server_globals.base_output, + "server:cnct_cbfunc reply being sent to %s:%u", + cd->peer->info->pname.nspace, cd->peer->info->pname.rank); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + } + + cleanup: + /* cleanup the tracker -- the host RM is responsible for + * telling us when to remove the nspace from our data */ + pmix_list_remove_item(&pmix_server_globals.collectives, &tracker->super); + PMIX_RELEASE(tracker); + + /* we are done */ + PMIX_RELEASE(scd); +} + +static void discnct_cbfunc(pmix_status_t status, void *cbdata) +{ + pmix_server_trkr_t *tracker = (pmix_server_trkr_t*)cbdata; + pmix_shift_caddy_t *scd; + + pmix_output_verbose(2, pmix_server_globals.base_output, + "server:discnct_cbfunc called on nspace %s", + (NULL == tracker) ? "NULL" : tracker->pname.nspace); + /* need to thread-shift this callback as it accesses global data */ scd = PMIX_NEW(pmix_shift_caddy_t); if (NULL == scd) { @@ -2015,16 +2403,17 @@ static void cnct_cbfunc(pmix_status_t status, void *cbdata) } scd->status = status; scd->tracker = tracker; - PMIX_THREADSHIFT(scd, _cnct); + PMIX_THREADSHIFT(scd, _discnct); } + static void regevents_cbfunc(pmix_status_t status, void *cbdata) { pmix_status_t rc; pmix_server_caddy_t *cd = (pmix_server_caddy_t*) cbdata; pmix_buffer_t *reply; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "server:regevents_cbfunc called status = %d", status); reply = PMIX_NEW(pmix_buffer_t); @@ -2038,7 +2427,10 @@ static void regevents_cbfunc(pmix_status_t status, void *cbdata) PMIX_ERROR_LOG(rc); } // send reply - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } PMIX_RELEASE(cd); } @@ -2048,7 +2440,7 @@ static void notifyerror_cbfunc (pmix_status_t status, void *cbdata) pmix_server_caddy_t *cd = (pmix_server_caddy_t*) cbdata; pmix_buffer_t *reply; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "server:notifyerror_cbfunc called status = %d", status); reply = PMIX_NEW(pmix_buffer_t); @@ -2062,10 +2454,71 @@ static void notifyerror_cbfunc (pmix_status_t status, void *cbdata) PMIX_ERROR_LOG(rc); } // send reply - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } PMIX_RELEASE(cd); } +static void alloc_cbfunc(pmix_status_t status, + pmix_info_t *info, size_t ninfo, + void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + pmix_query_caddy_t *qcd = (pmix_query_caddy_t*)cbdata; + pmix_server_caddy_t *cd = (pmix_server_caddy_t*)qcd->cbdata; + pmix_buffer_t *reply; + pmix_status_t rc; + + pmix_output_verbose(2, pmix_server_globals.base_output, + "pmix:alloc callback with status %d", status); + + reply = PMIX_NEW(pmix_buffer_t); + if (NULL == reply) { + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + PMIX_RELEASE(cd); + return; + } + PMIX_BFROPS_PACK(rc, cd->peer, reply, &status, 1, PMIX_STATUS); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + /* pack the returned data */ + PMIX_BFROPS_PACK(rc, cd->peer, reply, &ninfo, 1, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + if (0 < ninfo) { + PMIX_BFROPS_PACK(rc, cd->peer, reply, info, ninfo, PMIX_INFO); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + } + + complete: + // send reply + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + + // cleanup + if (NULL != qcd->queries) { + PMIX_QUERY_FREE(qcd->queries, qcd->nqueries); + } + if (NULL != qcd->info) { + PMIX_INFO_FREE(qcd->info, qcd->ninfo); + } + PMIX_RELEASE(qcd); + PMIX_RELEASE(cd); + if (NULL != release_fn) { + release_fn(release_cbdata); + } +} static void query_cbfunc(pmix_status_t status, pmix_info_t *info, size_t ninfo, @@ -2078,7 +2531,7 @@ static void query_cbfunc(pmix_status_t status, pmix_buffer_t *reply; pmix_status_t rc; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:query callback with status %d", status); reply = PMIX_NEW(pmix_buffer_t); @@ -2105,9 +2558,133 @@ static void query_cbfunc(pmix_status_t status, } } + /* cache the data for any future requests */ + + complete: + // send reply + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + + // cleanup + if (NULL != qcd->queries) { + PMIX_QUERY_FREE(qcd->queries, qcd->nqueries); + } + if (NULL != qcd->info) { + PMIX_INFO_FREE(qcd->info, qcd->ninfo); + } + PMIX_RELEASE(qcd); + PMIX_RELEASE(cd); + if (NULL != release_fn) { + release_fn(release_cbdata); + } +} + +static void jctrl_cbfunc(pmix_status_t status, + pmix_info_t *info, size_t ninfo, + void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + pmix_query_caddy_t *qcd = (pmix_query_caddy_t*)cbdata; + pmix_server_caddy_t *cd = (pmix_server_caddy_t*)qcd->cbdata; + pmix_buffer_t *reply; + pmix_status_t rc; + + pmix_output_verbose(2, pmix_server_globals.base_output, + "pmix:jctrl callback with status %d", status); + + reply = PMIX_NEW(pmix_buffer_t); + if (NULL == reply) { + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + PMIX_RELEASE(cd); + return; + } + PMIX_BFROPS_PACK(rc, cd->peer, reply, &status, 1, PMIX_STATUS); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + /* pack the returned data */ + PMIX_BFROPS_PACK(rc, cd->peer, reply, &ninfo, 1, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + if (0 < ninfo) { + PMIX_BFROPS_PACK(rc, cd->peer, reply, info, ninfo, PMIX_INFO); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + } + + complete: + // send reply + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + + // cleanup + if (NULL != qcd->queries) { + PMIX_QUERY_FREE(qcd->queries, qcd->nqueries); + } + if (NULL != qcd->info) { + PMIX_INFO_FREE(qcd->info, qcd->ninfo); + } + PMIX_RELEASE(qcd); + PMIX_RELEASE(cd); + if (NULL != release_fn) { + release_fn(release_cbdata); + } +} + +static void monitor_cbfunc(pmix_status_t status, + pmix_info_t *info, size_t ninfo, + void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + pmix_query_caddy_t *qcd = (pmix_query_caddy_t*)cbdata; + pmix_server_caddy_t *cd = (pmix_server_caddy_t*)qcd->cbdata; + pmix_buffer_t *reply; + pmix_status_t rc; + + pmix_output_verbose(2, pmix_server_globals.base_output, + "pmix:monitor callback with status %d", status); + + reply = PMIX_NEW(pmix_buffer_t); + if (NULL == reply) { + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + PMIX_RELEASE(cd); + return; + } + PMIX_BFROPS_PACK(rc, cd->peer, reply, &status, 1, PMIX_STATUS); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + /* pack the returned data */ + PMIX_BFROPS_PACK(rc, cd->peer, reply, &ninfo, 1, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + if (0 < ninfo) { + PMIX_BFROPS_PACK(rc, cd->peer, reply, info, ninfo, PMIX_INFO); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + } + complete: // send reply - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + // cleanup if (NULL != qcd->queries) { PMIX_QUERY_FREE(qcd->queries, qcd->nqueries); @@ -2117,8 +2694,12 @@ static void query_cbfunc(pmix_status_t status, } PMIX_RELEASE(qcd); PMIX_RELEASE(cd); + if (NULL != release_fn) { + release_fn(release_cbdata); + } } + /* the switchyard is the primary message handling function. It's purpose * is to take incoming commands (packed into a buffer), unpack them, * and then call the corresponding host server's function to execute @@ -2145,8 +2726,6 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, pmix_server_caddy_t *cd; pmix_proc_t proc; pmix_buffer_t *reply; - pmix_regevents_info_t *reginfo; - pmix_peer_events_info_t *prev; /* retrieve the cmd */ cnt = 1; @@ -2155,9 +2734,9 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, PMIX_ERROR_LOG(rc); return rc; } - pmix_output_verbose(2, pmix_globals.debug_output, - "recvd pmix cmd %d from %s:%u", - cmd, peer->info->pname.nspace, peer->info->pname.rank); + pmix_output_verbose(2, pmix_server_globals.base_output, + "recvd pmix cmd %s from %s:%u", + pmix_command_string(cmd), peer->info->pname.nspace, peer->info->pname.rank); if (PMIX_REQ_CMD == cmd) { reply = PMIX_NEW(pmix_buffer_t); @@ -2170,7 +2749,10 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, PMIX_ERROR_LOG(rc); return rc; } - PMIX_SERVER_QUEUE_REPLY(peer, tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, peer, tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } peer->nptr->ndelivered++; return PMIX_SUCCESS; } @@ -2195,7 +2777,10 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } - PMIX_SERVER_QUEUE_REPLY(peer, tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, peer, tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } } return PMIX_SUCCESS; // don't reply twice } @@ -2217,45 +2802,54 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, } if (PMIX_FINALIZE_CMD == cmd) { - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "recvd FINALIZE"); - /* mark that this peer called finalize */ - peer->finalized = true; + peer->nptr->nfinalized++; + /* purge events */ + pmix_server_purge_events(peer, NULL); + /* turn off the recv event - we shouldn't hear anything + * more from this proc */ + if (peer->recv_ev_active) { + pmix_event_del(&peer->recv_event); + peer->recv_ev_active = false; + } + PMIX_GDS_CADDY(cd, peer, tag); /* call the local server, if supported */ if (NULL != pmix_host_server.client_finalized) { - PMIX_GDS_CADDY(cd, peer, tag); - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; - /* since the client is finalizing, remove them from any event - * registrations they may still have on our list */ - PMIX_LIST_FOREACH(reginfo, &pmix_server_globals.events, pmix_regevents_info_t) { - PMIX_LIST_FOREACH(prev, ®info->peers, pmix_peer_events_info_t) { - if (prev->peer == peer) { - pmix_list_remove_item(®info->peers, &prev->super); - PMIX_RELEASE(prev); - break; - } - } - } /* now tell the host server */ - if (PMIX_SUCCESS != (rc = pmix_host_server.client_finalized(&proc, peer->info->server_object, - op_cbfunc, cd))) { - PMIX_RELEASE(cd); - } else { + rc = pmix_host_server.client_finalized(&proc, peer->info->server_object, + op_cbfunc2, cd); + if (PMIX_SUCCESS == rc) { /* don't reply to them ourselves - we will do so when the host * server calls us back */ return rc; + } else if (PMIX_OPERATION_SUCCEEDED == rc) { + /* they did it atomically */ + rc = PMIX_SUCCESS; } + /* if the call doesn't succeed (e.g., they provided the stub + * but return NOT_SUPPORTED), then the callback function + * won't be called, but we still need to cleanup + * any lingering references to this peer and answer + * the client. Thus, we call the callback function ourselves + * in this case */ + op_cbfunc2(rc, cd); + /* return SUCCESS as the cbfunc generated the return msg + * and released the cd object */ + return PMIX_SUCCESS; } - /* turn off the recv event - we shouldn't hear anything - * more from this proc */ - if (peer->recv_ev_active) { - pmix_event_del(&peer->recv_event); - peer->recv_ev_active = false; - } - /* let the network libraries cleanup */ - pmix_pnet.child_finalized(peer); - return rc; + /* if the host doesn't provide a client_finalized function, + * we still need to ensure that we cleanup any lingering + * references to this peer. We use the callback function + * here as well to ensure the client gets its required + * response and that we delay before cleaning up the + * connection*/ + op_cbfunc2(PMIX_SUCCESS, cd); + /* return SUCCESS as the cbfunc generated the return msg + * and released the cd object */ + return PMIX_SUCCESS; } @@ -2297,15 +2891,19 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, if (PMIX_CONNECTNB_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_connect(cd, buf, false, cnct_cbfunc); - PMIX_RELEASE(cd); + rc = pmix_server_connect(cd, buf, cnct_cbfunc); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_DISCONNECTNB_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_connect(cd, buf, true, cnct_cbfunc); - PMIX_RELEASE(cd); + rc = pmix_server_disconnect(cd, buf, discnct_cbfunc); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(cd); + } return rc; } @@ -2324,52 +2922,64 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, if (PMIX_NOTIFY_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_event_recvd_from_client(peer, buf, notifyerror_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_event_recvd_from_client(peer, buf, notifyerror_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_QUERY_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_query(peer, buf, query_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_query(peer, buf, query_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_LOG_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_log(peer, buf, op_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_log(peer, buf, op_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_ALLOC_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_alloc(peer, buf, query_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_alloc(peer, buf, alloc_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_JOB_CONTROL_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_job_ctrl(peer, buf, query_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_job_ctrl(peer, buf, jctrl_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_MONITOR_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_monitor(peer, buf, query_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_monitor(peer, buf, monitor_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } return PMIX_ERR_NOT_SUPPORTED; } -static void server_message_handler(struct pmix_peer_t *pr, - pmix_ptl_hdr_t *hdr, - pmix_buffer_t *buf, void *cbdata) +void pmix_server_message_handler(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, + pmix_buffer_t *buf, void *cbdata) { pmix_peer_t *peer = (pmix_peer_t*)pr; pmix_buffer_t *reply; pmix_status_t rc, ret; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "SWITCHYARD for %s:%u:%d", peer->info->pname.nspace, peer->info->pname.rank, peer->sd); @@ -2389,6 +2999,9 @@ static void server_message_handler(struct pmix_peer_t *pr, if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } - PMIX_SERVER_QUEUE_REPLY(peer, hdr->tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, peer, hdr->tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } } } diff --git a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_get.c b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_get.c index c1fd35e4402..d798b27bf95 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_get.c +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_get.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Artem Y. Polyakov . @@ -84,14 +84,17 @@ PMIX_CLASS_INSTANCE(pmix_dmdx_reply_caddy_t, static void dmdx_cbfunc(pmix_status_t status, const char *data, size_t ndata, void *cbdata, pmix_release_cbfunc_t relfn, void *relcbdata); -static pmix_status_t _satisfy_request(pmix_nspace_t *ns, pmix_rank_t rank, +static pmix_status_t _satisfy_request(pmix_namespace_t *ns, pmix_rank_t rank, pmix_server_caddy_t *cd, pmix_modex_cbfunc_t cbfunc, void *cbdata, bool *scope); static pmix_status_t create_local_tracker(char nspace[], pmix_rank_t rank, pmix_info_t info[], size_t ninfo, pmix_modex_cbfunc_t cbfunc, void *cbdata, - pmix_dmdx_local_t **lcd); + pmix_dmdx_local_t **lcd, + pmix_dmdx_request_t **rq); + +static void get_timeout(int sd, short args, void *cbdata); /* declare a function whose sole purpose is to @@ -116,12 +119,14 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, pmix_rank_t rank; char *cptr; char nspace[PMIX_MAX_NSLEN+1]; - pmix_nspace_t *ns, *nptr; + pmix_namespace_t *ns, *nptr; pmix_info_t *info=NULL; size_t ninfo=0; pmix_dmdx_local_t *lcd; + pmix_dmdx_request_t *req; bool local; bool localonly = false; + struct timeval tv = {0, 0}; pmix_buffer_t pbkt, pkt; pmix_byte_object_t bo; pmix_cb_t cb; @@ -130,7 +135,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, size_t sz, n; pmix_peer_t *peer; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.get_output, "recvd GET"); /* setup */ @@ -143,7 +148,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, PMIX_ERROR_LOG(rc); return rc; } - (void)strncpy(nspace, cptr, PMIX_MAX_NSLEN); + pmix_strncpy(nspace, cptr, PMIX_MAX_NSLEN); free(cptr); cnt = 1; PMIX_BFROPS_UNPACK(rc, cd->peer, buf, &rank, &cnt, PMIX_PROC_RANK); @@ -175,23 +180,25 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, /* search for directives we can deal with here */ for (n=0; n < ninfo; n++) { - if (0 == strcmp(info[n].key, PMIX_IMMEDIATE)) { + if (0 == strncmp(info[n].key, PMIX_IMMEDIATE, PMIX_MAX_KEYLEN)) { /* just check our own data - don't wait * or request it from someone else */ localonly = PMIX_INFO_TRUE(&info[n]); + } else if (0 == strncmp(info[n].key, PMIX_TIMEOUT, PMIX_MAX_KEYLEN)) { + tv.tv_sec = info[n].value.data.uint32; } } /* find the nspace object for this client */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(nspace, ns->nspace)) { nptr = ns; break; } } - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.get_output, "%s:%d EXECUTE GET FOR %s:%d ON BEHALF OF %s:%d", pmix_globals.myid.nspace, pmix_globals.myid.rank, nspace, rank, @@ -234,18 +241,26 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, * back when we receive it */ rc = create_local_tracker(nspace, rank, info, ninfo, - cbfunc, cbdata, &lcd); + cbfunc, cbdata, &lcd, &req); if (PMIX_ERR_NOMEM == rc) { PMIX_INFO_FREE(info, ninfo); return rc; - } else if (PMIX_ERR_NOT_FOUND != rc) { - return rc; } - - /* do NOT create the nspace tracker here so any request - * by another local client that hits before the RM responds - * to our request will get added to the local tracker so - * they receive their data upon completion */ + if (PMIX_SUCCESS == rc) { + /* if they specified a timeout for this specific + * request, set it up now */ + if (0 < tv.tv_sec) { + pmix_event_evtimer_set(pmix_globals.evbase, &req->ev, + get_timeout, req); + pmix_event_evtimer_add(&req->ev, &tv); + req->event_active = true; + } + /* we already asked for this info - no need to + * do it again */ + return PMIX_SUCCESS; + } + /* only other return code is NOT_FOUND, indicating that + * we created a new tracker */ /* Its possible there will be no local processes on this * host, so lets ask for this explicitly. There can @@ -253,7 +268,27 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, * up on its own, but at worst the direct modex * will simply overwrite the info later */ if (NULL != pmix_host_server.direct_modex) { - pmix_host_server.direct_modex(&lcd->proc, info, ninfo, dmdx_cbfunc, lcd); + rc = pmix_host_server.direct_modex(&lcd->proc, info, ninfo, dmdx_cbfunc, lcd); + if (PMIX_SUCCESS != rc) { + PMIX_INFO_FREE(info, ninfo); + pmix_list_remove_item(&pmix_server_globals.local_reqs, &lcd->super); + PMIX_RELEASE(lcd); + return rc; + } + /* if they specified a timeout for this specific + * request, set it up now */ + if (0 < tv.tv_sec) { + pmix_event_evtimer_set(pmix_globals.evbase, &req->ev, + get_timeout, req); + pmix_event_evtimer_add(&req->ev, &tv); + req->event_active = true; + } + } else { + /* if we don't have direct modex feature, just respond with "not found" */ + PMIX_INFO_FREE(info, ninfo); + pmix_list_remove_item(&pmix_server_globals.local_reqs, &lcd->super); + PMIX_RELEASE(lcd); + return PMIX_ERR_NOT_FOUND; } return PMIX_SUCCESS; @@ -268,7 +303,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, * for it, so there is no guarantee we have it */ data = NULL; sz = 0; - (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; /* if we have local procs for this nspace, then we * can retrieve the info from that GDS. Otherwise, @@ -309,7 +344,11 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, /* unload the resulting payload */ PMIX_UNLOAD_BUFFER(&pbkt, data, sz); PMIX_DESTRUCT(&pbkt); + /* call the internal callback function - it will + * release the cbdata */ cbfunc(PMIX_SUCCESS, data, sz, cbdata, relfn, data); + /* return success so the server doesn't duplicate + * the release of cbdata */ return PMIX_SUCCESS; } @@ -320,49 +359,91 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, * we do know how many clients to expect, so first check to see if * all clients have been registered with us */ if (!nptr->all_registered) { + pmix_output_verbose(2, pmix_server_globals.get_output, + "%s:%d NSPACE %s not all registered", + pmix_globals.myid.nspace, + pmix_globals.myid.rank, nspace); + if (localonly) { /* the client asked that we not wait, so return now */ + pmix_output_verbose(2, pmix_server_globals.get_output, + "%s:%d CLIENT REQUESTED IMMEDIATE", + pmix_globals.myid.nspace, + pmix_globals.myid.rank); return PMIX_ERR_NOT_FOUND; } /* we cannot do anything further, so just track this request * for now */ rc = create_local_tracker(nspace, rank, info, ninfo, - cbfunc, cbdata, &lcd); + cbfunc, cbdata, &lcd, &req); if (PMIX_ERR_NOMEM == rc) { PMIX_INFO_FREE(info, ninfo); + return rc; } - return rc; + pmix_output_verbose(2, pmix_server_globals.get_output, + "%s:%d TRACKER CREATED - WAITING", + pmix_globals.myid.nspace, + pmix_globals.myid.rank); + /* if they specified a timeout, set it up now */ + if (0 < tv.tv_sec) { + pmix_event_evtimer_set(pmix_globals.evbase, &req->ev, + get_timeout, req); + pmix_event_evtimer_add(&req->ev, &tv); + req->event_active = true; + } + /* the peer object has been added to the new lcd tracker, + * so return success here */ + return PMIX_SUCCESS; } - /* see if we already have this data */ + /* if everyone has registered, see if we already have this data */ rc = _satisfy_request(nptr, rank, cd, cbfunc, cbdata, &local); if( PMIX_SUCCESS == rc ){ /* request was successfully satisfied */ PMIX_INFO_FREE(info, ninfo); - return rc; + /* return success as the satisfy_request function + * calls the cbfunc for us, and it will have + * released the cbdata object */ + return PMIX_SUCCESS; } + pmix_output_verbose(2, pmix_server_globals.get_output, + "%s:%d DATA NOT FOUND", + pmix_globals.myid.nspace, + pmix_globals.myid.rank); + /* If we get here, then we don't have the data at this time. If * the user doesn't want to look for it, then we are done */ if (localonly) { + pmix_output_verbose(2, pmix_server_globals.get_output, + "%s:%d CLIENT REQUESTED IMMEDIATE", + pmix_globals.myid.nspace, + pmix_globals.myid.rank); return PMIX_ERR_NOT_FOUND; } /* Check to see if we already have a pending request for the data - if * we do, then we can just wait for it to arrive */ rc = create_local_tracker(nspace, rank, info, ninfo, - cbfunc, cbdata, &lcd); + cbfunc, cbdata, &lcd, &req); + if (PMIX_ERR_NOMEM == rc || NULL == lcd) { + /* we have a problem */ + PMIX_INFO_FREE(info, ninfo); + return PMIX_ERR_NOMEM; + } + /* if they specified a timeout, set it up now */ + if (0 < tv.tv_sec) { + pmix_event_evtimer_set(pmix_globals.evbase, &req->ev, + get_timeout, req); + pmix_event_evtimer_add(&req->ev, &tv); + req->event_active = true; + } if (PMIX_SUCCESS == rc) { /* we are already waiting for the data - nothing more * for us to do as the function added the new request * to the tracker for us */ return PMIX_SUCCESS; } - if (PMIX_ERR_NOT_FOUND != rc || NULL == lcd) { - /* we have a problem - e.g., out of memory */ - PMIX_INFO_FREE(info, ninfo); - return rc; - } /* Getting here means that we didn't already have a request for * for data pending, and so we created a new tracker for this @@ -378,9 +459,18 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, * whomever is hosting the target process */ if (NULL != pmix_host_server.direct_modex) { rc = pmix_host_server.direct_modex(&lcd->proc, info, ninfo, dmdx_cbfunc, lcd); + if (PMIX_SUCCESS != rc) { + /* may have a function entry but not support the request */ + PMIX_INFO_FREE(info, ninfo); + pmix_list_remove_item(&pmix_server_globals.local_reqs, &lcd->super); + PMIX_RELEASE(lcd); + } } else { + pmix_output_verbose(2, pmix_server_globals.get_output, + "%s:%d NO SERVER SUPPORT", + pmix_globals.myid.nspace, + pmix_globals.myid.rank); /* if we don't have direct modex feature, just respond with "not found" */ - cbfunc(PMIX_ERR_NOT_FOUND, NULL, 0, cbdata, NULL, NULL); PMIX_INFO_FREE(info, ninfo); pmix_list_remove_item(&pmix_server_globals.local_reqs, &lcd->super); PMIX_RELEASE(lcd); @@ -394,7 +484,8 @@ static pmix_status_t create_local_tracker(char nspace[], pmix_rank_t rank, pmix_info_t info[], size_t ninfo, pmix_modex_cbfunc_t cbfunc, void *cbdata, - pmix_dmdx_local_t **ld) + pmix_dmdx_local_t **ld, + pmix_dmdx_request_t **rq) { pmix_dmdx_local_t *lcd, *cd; pmix_dmdx_request_t *req; @@ -402,6 +493,7 @@ static pmix_status_t create_local_tracker(char nspace[], pmix_rank_t rank, /* define default */ *ld = NULL; + *rq = NULL; /* see if we already have an existing request for data * from this namespace/rank */ @@ -426,7 +518,7 @@ static pmix_status_t create_local_tracker(char nspace[], pmix_rank_t rank, if (NULL == lcd){ return PMIX_ERR_NOMEM; } - strncpy(lcd->proc.nspace, nspace, PMIX_MAX_NSLEN); + pmix_strncpy(lcd->proc.nspace, nspace, PMIX_MAX_NSLEN); lcd->proc.rank = rank; lcd->info = info; lcd->ninfo = ninfo; @@ -437,16 +529,24 @@ static pmix_status_t create_local_tracker(char nspace[], pmix_rank_t rank, /* track this specific requestor so we return the * data to them */ req = PMIX_NEW(pmix_dmdx_request_t); + if (NULL == req) { + *ld = lcd; + return PMIX_ERR_NOMEM; + } + PMIX_RETAIN(lcd); + req->lcd = lcd; req->cbfunc = cbfunc; req->cbdata = cbdata; pmix_list_append(&lcd->loc_reqs, &req->super); *ld = lcd; + *rq = req; return rc; } -void pmix_pending_nspace_requests(pmix_nspace_t *nptr) +void pmix_pending_nspace_requests(pmix_namespace_t *nptr) { pmix_dmdx_local_t *cd, *cd_next; + pmix_status_t rc; /* Now that we know all local ranks, go along request list and ask for remote data * for the non-local ranks, and resolve all pending requests for local procs @@ -469,10 +569,12 @@ void pmix_pending_nspace_requests(pmix_nspace_t *nptr) /* if not found - this is remote process and we need to send * corresponding direct modex request */ - if( !found ){ - if( NULL != pmix_host_server.direct_modex ){ - pmix_host_server.direct_modex(&cd->proc, cd->info, cd->ninfo, dmdx_cbfunc, cd); - } else { + if (!found){ + rc = PMIX_ERR_NOT_SUPPORTED; + if (NULL != pmix_host_server.direct_modex){ + rc = pmix_host_server.direct_modex(&cd->proc, cd->info, cd->ninfo, dmdx_cbfunc, cd); + } + if (PMIX_SUCCESS != rc) { pmix_dmdx_request_t *req, *req_next; PMIX_LIST_FOREACH_SAFE(req, req_next, &cd->loc_reqs, pmix_dmdx_request_t) { req->cbfunc(PMIX_ERR_NOT_FOUND, NULL, 0, req->cbdata, NULL, NULL); @@ -486,7 +588,7 @@ void pmix_pending_nspace_requests(pmix_nspace_t *nptr) } } -static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, +static pmix_status_t _satisfy_request(pmix_namespace_t *nptr, pmix_rank_t rank, pmix_server_caddy_t *cd, pmix_modex_cbfunc_t cbfunc, void *cbdata, bool *local) @@ -497,18 +599,23 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, pmix_rank_info_t *iptr; pmix_proc_t proc; pmix_cb_t cb; - pmix_peer_t *peer; + pmix_peer_t *peer = NULL; pmix_byte_object_t bo; char *data = NULL; size_t sz = 0; pmix_scope_t scope = PMIX_SCOPE_UNDEF; + pmix_output_verbose(2, pmix_server_globals.get_output, + "%s:%d SATISFY REQUEST CALLED", + pmix_globals.myid.nspace, + pmix_globals.myid.rank); + /* check to see if this data already has been * obtained as a result of a prior direct modex request from * a remote peer, or due to data from a local client * having been committed */ PMIX_CONSTRUCT(&pbkt, pmix_buffer_t); - (void)strncpy(proc.nspace, nptr->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nptr->nspace, PMIX_MAX_NSLEN); /* if we have local clients of this nspace, then we use * the corresponding GDS to retrieve the data. Otherwise, @@ -569,6 +676,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, /* assemble the provided data into a byte object */ PMIX_GDS_ASSEMB_KVS_REQ(rc, cd->peer, &proc, &cb.kvs, &pkt, cd); if (rc != PMIX_SUCCESS) { + PMIX_ERROR_LOG(rc); PMIX_DESTRUCT(&pkt); PMIX_DESTRUCT(&pbkt); PMIX_DESTRUCT(&cb); @@ -582,6 +690,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, PMIX_CONSTRUCT(&xfer, pmix_buffer_t); PMIX_BFROPS_PACK(rc, cd->peer, &xfer, &pkt, 1, PMIX_BUFFER); if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); PMIX_DESTRUCT(&pkt); PMIX_DESTRUCT(&pbkt); PMIX_DESTRUCT(&xfer); @@ -597,6 +706,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, /* pack it for transmission */ PMIX_BFROPS_PACK(rc, cd->peer, &pbkt, &bo, 1, PMIX_BYTE_OBJECT); if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); PMIX_DESTRUCT(&pbkt); PMIX_DESTRUCT(&cb); return rc; @@ -611,12 +721,12 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, /* retrieve the data for the specific rank they are asking about */ if (PMIX_RANK_WILDCARD != rank) { if (!PMIX_PROC_IS_SERVER(peer) && !peer->commit_cnt) { - /* this condition works only for local requests, server does - * count commits for local ranks, and check this count when + /* this condition works only for local requests, server does + * count commits for local ranks, and check this count when * local request. - * if that request performs for remote rank on the remote + * if that request performs for remote rank on the remote * node (by direct modex) so `peer->commit_cnt` should be ignored, - * it is can not be counted for the remote side and this condition + * it is can not be counted for the remote side and this condition * does not matter for remote case */ return PMIX_ERR_NOT_FOUND; } @@ -635,6 +745,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, /* assemble the provided data into a byte object */ PMIX_GDS_ASSEMB_KVS_REQ(rc, cd->peer, &proc, &cb.kvs, &pkt, cd); if (rc != PMIX_SUCCESS) { + PMIX_ERROR_LOG(rc); PMIX_DESTRUCT(&pkt); PMIX_DESTRUCT(&pbkt); PMIX_DESTRUCT(&cb); @@ -647,6 +758,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, /* pack the rank */ PMIX_BFROPS_PACK(rc, cd->peer, &pbkt, &rank, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); PMIX_DESTRUCT(&pkt); PMIX_DESTRUCT(&pbkt); PMIX_DESTRUCT(&cb); @@ -655,6 +767,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, /* now pack the data itself as a buffer */ PMIX_BFROPS_PACK(rc, cd->peer, &pbkt, &pkt, 1, PMIX_BUFFER); if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); PMIX_DESTRUCT(&pkt); PMIX_DESTRUCT(&pbkt); PMIX_DESTRUCT(&cb); @@ -667,6 +780,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, /* pack it for transmission */ PMIX_BFROPS_PACK(rc, cd->peer, &pbkt, &bo, 1, PMIX_BYTE_OBJECT); if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); PMIX_DESTRUCT(&pbkt); PMIX_DESTRUCT(&cb); return rc; @@ -684,17 +798,11 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, return rc; } - if ((PMIX_LOCAL == scope) && !found) { - /* pass PMIX_ERR_NOT_FOUND for local request if it's not found*/ - cbfunc(PMIX_ERR_NOT_FOUND, NULL, 0, cbdata, NULL, NULL); - return PMIX_SUCCESS; - } - return PMIX_ERR_NOT_FOUND; } /* Resolve pending requests to this namespace/rank */ -pmix_status_t pmix_pending_resolve(pmix_nspace_t *nptr, pmix_rank_t rank, +pmix_status_t pmix_pending_resolve(pmix_namespace_t *nptr, pmix_rank_t rank, pmix_status_t status, pmix_dmdx_local_t *lcd) { pmix_dmdx_local_t *cd, *ptr; @@ -706,7 +814,7 @@ pmix_status_t pmix_pending_resolve(pmix_nspace_t *nptr, pmix_rank_t rank, ptr = NULL; if (NULL != nptr) { PMIX_LIST_FOREACH(cd, &pmix_server_globals.local_reqs, pmix_dmdx_local_t) { - if (0 != strncmp(nptr->nspace, cd->proc.nspace, PMIX_MAX_NSLEN) || + if (!PMIX_CHECK_NSPACE(nptr->nspace, cd->proc.nspace) || rank != cd->proc.rank) { continue; } @@ -721,6 +829,13 @@ pmix_status_t pmix_pending_resolve(pmix_nspace_t *nptr, pmix_rank_t rank, ptr = lcd; } + /* if there are no local reqs on this request (e.g., only + * one proc requested it and that proc has died), then + * just remove the request */ + if (0 == pmix_list_get_size(&ptr->loc_reqs)) { + goto cleanup; + } + /* somebody was interested in this rank */ if (PMIX_SUCCESS != status){ /* if we've got an error for this request - just forward it*/ @@ -745,8 +860,10 @@ pmix_status_t pmix_pending_resolve(pmix_nspace_t *nptr, pmix_rank_t rank, } PMIX_RELEASE(scd); } + + cleanup: /* remove all requests to this rank and cleanup the corresponding structure */ - pmix_list_remove_item(&pmix_server_globals.local_reqs, (pmix_list_item_t*)ptr); + pmix_list_remove_item(&pmix_server_globals.local_reqs, &ptr->super); PMIX_RELEASE(ptr); return PMIX_SUCCESS; @@ -761,7 +878,7 @@ static void _process_dmdx_reply(int fd, short args, void *cbdata) pmix_rank_info_t *rinfo; int32_t cnt; pmix_kval_t *kv; - pmix_nspace_t *ns, *nptr; + pmix_namespace_t *ns, *nptr; pmix_status_t rc; pmix_list_t nspaces; pmix_nspace_caddy_t *nm; @@ -772,14 +889,14 @@ static void _process_dmdx_reply(int fd, short args, void *cbdata) PMIX_ACQUIRE_OBJECT(caddy); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.get_output, "[%s:%d] process dmdx reply from %s:%u", __FILE__, __LINE__, caddy->lcd->proc.nspace, caddy->lcd->proc.rank); /* find the nspace object for the proc whose data is being received */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(caddy->lcd->proc.nspace, ns->nspace)) { nptr = ns; break; @@ -790,7 +907,7 @@ static void _process_dmdx_reply(int fd, short args, void *cbdata) /* We may not have this namespace because there are no local * processes from it running on this host - so just record it * so we know we have the data for any future requests */ - nptr = PMIX_NEW(pmix_nspace_t); + nptr = PMIX_NEW(pmix_namespace_t); nptr->nspace = strdup(caddy->lcd->proc.nspace); /* add to the list */ pmix_list_append(&pmix_server_globals.nspaces, &nptr->super); @@ -854,7 +971,7 @@ static void _process_dmdx_reply(int fd, short args, void *cbdata) PMIX_DESTRUCT(&cb); goto complete; } - (void)strncpy(cb.proc->nspace, nm->ns->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cb.proc->nspace, nm->ns->nspace, PMIX_MAX_NSLEN); cb.proc->rank = PMIX_RANK_WILDCARD; cb.scope = PMIX_INTERNAL; cb.copy = false; @@ -941,9 +1058,24 @@ static void dmdx_cbfunc(pmix_status_t status, caddy->data = data; caddy->ndata = ndata; caddy->lcd = (pmix_dmdx_local_t *)cbdata; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.get_output, "[%s:%d] queue dmdx reply for %s:%u", __FILE__, __LINE__, caddy->lcd->proc.nspace, caddy->lcd->proc.rank); PMIX_THREADSHIFT(caddy, _process_dmdx_reply); } + +static void get_timeout(int sd, short args, void *cbdata) +{ + pmix_dmdx_request_t *req = (pmix_dmdx_request_t*)cbdata; + + pmix_output_verbose(2, pmix_server_globals.get_output, + "ALERT: get timeout fired"); + /* execute the provided callback function with the error */ + if (NULL != req->cbfunc) { + req->cbfunc(PMIX_ERR_TIMEOUT, NULL, 0, req->cbdata, NULL, NULL); + } + req->event_active = false; + pmix_list_remove_item(&req->lcd->loc_reqs, &req->super); + PMIX_RELEASE(req); +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c index 3205b761a4e..2c82c501b9f 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c @@ -1,8 +1,8 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2014-2017 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2016-2017 Mellanox Technologies, Inc. @@ -46,8 +46,12 @@ #ifdef HAVE_SYS_TYPES_H #include #endif +#ifdef HAVE_TIME_H +#include +#endif #include PMIX_EVENT_HEADER +#include "src/class/pmix_hotel.h" #include "src/class/pmix_list.h" #include "src/mca/bfrops/bfrops.h" #include "src/mca/psensor/psensor.h" @@ -71,7 +75,7 @@ pmix_status_t pmix_server_abort(pmix_peer_t *peer, pmix_buffer_t *buf, pmix_proc_t *procs = NULL; pmix_proc_t proc; - pmix_output_verbose(2, pmix_globals.debug_output, "recvd ABORT"); + pmix_output_verbose(2, pmix_server_globals.base_output, "recvd ABORT"); /* unpack the status */ cnt = 1; @@ -114,16 +118,12 @@ pmix_status_t pmix_server_abort(pmix_peer_t *peer, pmix_buffer_t *buf, /* let the local host's server execute it */ if (NULL != pmix_host_server.abort) { - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; rc = pmix_host_server.abort(&proc, peer->info->server_object, status, msg, procs, nprocs, cbfunc, cbdata); } else { rc = PMIX_ERR_NOT_SUPPORTED; - /* release the caller */ - if (NULL != cbfunc) { - cbfunc(rc, cbdata); - } } PMIX_PROC_FREE(procs, nprocs); @@ -143,7 +143,7 @@ pmix_status_t pmix_server_commit(pmix_peer_t *peer, pmix_buffer_t *buf) pmix_buffer_t b2, pbkt; pmix_kval_t *kp; pmix_scope_t scope; - pmix_nspace_t *nptr; + pmix_namespace_t *nptr; pmix_rank_info_t *info; pmix_proc_t proc; pmix_dmdx_remote_t *dcd, *dcdnext; @@ -154,10 +154,10 @@ pmix_status_t pmix_server_commit(pmix_peer_t *peer, pmix_buffer_t *buf) /* shorthand */ info = peer->info; nptr = peer->nptr; - (void)strncpy(proc.nspace, nptr->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nptr->nspace, PMIX_MAX_NSLEN); proc.rank = info->pname.rank; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "%s:%d EXECUTE COMMIT FOR %s:%d", pmix_globals.myid.nspace, pmix_globals.myid.rank, @@ -291,18 +291,18 @@ pmix_status_t pmix_server_commit(pmix_peer_t *peer, pmix_buffer_t *buf) * regardless of location * nprocs - the number of procs in the array */ -static pmix_server_trkr_t* get_tracker(pmix_proc_t *procs, +static pmix_server_trkr_t* get_tracker(char *id, pmix_proc_t *procs, size_t nprocs, pmix_cmd_t type) { pmix_server_trkr_t *trk; size_t i, j; size_t matches; - pmix_output_verbose(5, pmix_globals.debug_output, + pmix_output_verbose(5, pmix_server_globals.base_output, "get_tracker called with %d procs", (int)nprocs); /* bozo check - should never happen outside of programmer error */ - if (NULL == procs) { + if (NULL == procs && NULL == id) { PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); return NULL; } @@ -315,28 +315,35 @@ static pmix_server_trkr_t* get_tracker(pmix_proc_t *procs, * shouldn't take long */ PMIX_LIST_FOREACH(trk, &pmix_server_globals.collectives, pmix_server_trkr_t) { /* Collective operation if unique identified by - * the set of participating processes and the type of collective + * the set of participating processes and the type of collective, + * or by the operation ID */ - if (nprocs != trk->npcs) { - continue; - } - if (type != trk->type) { - continue; - } - matches = 0; - for (i=0; i < nprocs; i++) { - /* the procs may be in different order, so we have - * to do an exhaustive search */ - for (j=0; j < trk->npcs; j++) { - if (0 == strcmp(procs[i].nspace, trk->pcs[j].nspace) && - procs[i].rank == trk->pcs[j].rank) { - ++matches; - break; + if (NULL != id) { + if (NULL != trk->id && 0 == strcmp(id, trk->id)) { + return trk; + } + } else { + if (nprocs != trk->npcs) { + continue; + } + if (type != trk->type) { + continue; + } + matches = 0; + for (i=0; i < nprocs; i++) { + /* the procs may be in different order, so we have + * to do an exhaustive search */ + for (j=0; j < trk->npcs; j++) { + if (0 == strcmp(procs[i].nspace, trk->pcs[j].nspace) && + procs[i].rank == trk->pcs[j].rank) { + ++matches; + break; + } } } - } - if (trk->npcs == matches) { - return trk; + if (trk->npcs == matches) { + return trk; + } } } /* No tracker was found */ @@ -359,16 +366,16 @@ static pmix_server_trkr_t* get_tracker(pmix_proc_t *procs, * regardless of location * nprocs - the number of procs in the array */ -static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs, +static pmix_server_trkr_t* new_tracker(char *id, pmix_proc_t *procs, size_t nprocs, pmix_cmd_t type) { pmix_server_trkr_t *trk; size_t i; bool all_def; - pmix_nspace_t *nptr, *ns; + pmix_namespace_t *nptr, *ns; pmix_rank_info_t *info; - pmix_output_verbose(5, pmix_globals.debug_output, + pmix_output_verbose(5, pmix_server_globals.base_output, "new_tracker called with %d procs", (int)nprocs); /* bozo check - should never happen outside of programmer error */ @@ -377,8 +384,9 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs, return NULL; } - pmix_output_verbose(5, pmix_globals.debug_output, - "adding new tracker with %d procs", (int)nprocs); + pmix_output_verbose(5, pmix_server_globals.base_output, + "adding new tracker %s with %d procs", + (NULL == id) ? "NO-ID" : id, (int)nprocs); /* this tracker is new - create it */ trk = PMIX_NEW(pmix_server_trkr_t); @@ -387,26 +395,35 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs, return NULL; } - /* copy the procs */ - PMIX_PROC_CREATE(trk->pcs, nprocs); - if (NULL == trk->pcs) { - PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - PMIX_RELEASE(trk); - return NULL; + if (NULL != id) { + trk->id = strdup(id); + } + + if (NULL != procs) { + /* copy the procs */ + PMIX_PROC_CREATE(trk->pcs, nprocs); + if (NULL == trk->pcs) { + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + PMIX_RELEASE(trk); + return NULL; + } + memcpy(trk->pcs, procs, nprocs * sizeof(pmix_proc_t)); + trk->npcs = nprocs; } - trk->npcs = nprocs; trk->type = type; all_def = true; for (i=0; i < nprocs; i++) { - (void)strncpy(trk->pcs[i].nspace, procs[i].nspace, PMIX_MAX_NSLEN); - trk->pcs[i].rank = procs[i].rank; + if (NULL == id) { + pmix_strncpy(trk->pcs[i].nspace, procs[i].nspace, PMIX_MAX_NSLEN); + trk->pcs[i].rank = procs[i].rank; + } if (!all_def) { continue; } /* is this nspace known to us? */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(procs[i].nspace, ns->nspace)) { nptr = ns; break; @@ -414,7 +431,7 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs, } if (NULL == nptr) { /* cannot be a local proc */ - pmix_output_verbose(5, pmix_globals.debug_output, + pmix_output_verbose(5, pmix_server_globals.base_output, "new_tracker: unknown nspace %s", procs[i].nspace); continue; @@ -424,7 +441,7 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs, /* nope, so no point in going further on this one - we'll * process it once all the procs are known */ all_def = false; - pmix_output_verbose(5, pmix_globals.debug_output, + pmix_output_verbose(5, pmix_server_globals.base_output, "new_tracker: all clients not registered nspace %s", procs[i].nspace); /* we have to continue processing the list of procs @@ -435,7 +452,7 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs, PMIX_LIST_FOREACH(info, &nptr->ranks, pmix_rank_info_t) { if (procs[i].rank == info->pname.rank || PMIX_RANK_WILDCARD == procs[i].rank) { - pmix_output_verbose(5, pmix_globals.debug_output, + pmix_output_verbose(5, pmix_server_globals.base_output, "adding local proc %s.%d to tracker", info->pname.nspace, info->pname.rank); /* track the count */ @@ -453,6 +470,24 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs, return trk; } +static void fence_timeout(int sd, short args, void *cbdata) +{ + pmix_server_caddy_t *cd = (pmix_server_caddy_t*)cbdata; + + pmix_output_verbose(2, pmix_server_globals.fence_output, + "ALERT: fence timeout fired"); + + /* execute the provided callback function with the error */ + if (NULL != cd->trk->modexcbfunc) { + cd->trk->modexcbfunc(PMIX_ERR_TIMEOUT, NULL, 0, cd->trk, NULL, NULL); + return; // the cbfunc will have cleaned up the tracker + } + cd->event_active = false; + /* remove it from the list */ + pmix_list_remove_item(&cd->trk->local_cbs, &cd->super); + PMIX_RELEASE(cd); +} + pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, pmix_buffer_t *buf, pmix_modex_cbfunc_t modexcbfunc, @@ -473,8 +508,9 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, pmix_byte_object_t bo; pmix_info_t *info = NULL; size_t ninfo=0, n; + struct timeval tv = {0, 0}; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.fence_output, "recvd FENCE"); if (NULL == pmix_host_server.fence_nb) { @@ -488,7 +524,7 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, if (PMIX_SUCCESS != rc) { return rc; } - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.fence_output, "recvd fence from %s:%u with %d procs", cd->peer->info->pname.nspace, cd->peer->info->pname.rank, (int)nprocs); /* there must be at least one as the client has to at least provide @@ -527,20 +563,21 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, if (PMIX_SUCCESS != rc) { goto cleanup; } - /* see if we are to collect data - we don't internally care + /* see if we are to collect data or enforce a timeout - we don't internally care * about any other directives */ for (n=0; n < ninfo; n++) { if (0 == strcmp(info[n].key, PMIX_COLLECT_DATA)) { collect_data = true; - break; + } else if (0 == strncmp(info[n].key, PMIX_TIMEOUT, PMIX_MAX_KEYLEN)) { + tv.tv_sec = info[n].value.data.uint32; } } } /* find/create the local tracker for this operation */ - if (NULL == (trk = get_tracker(procs, nprocs, PMIX_FENCENB_CMD))) { + if (NULL == (trk = get_tracker(NULL, procs, nprocs, PMIX_FENCENB_CMD))) { /* If no tracker was found - create and initialize it once */ - if (NULL == (trk = new_tracker(procs, nprocs, PMIX_FENCENB_CMD))) { + if (NULL == (trk = new_tracker(NULL, procs, nprocs, PMIX_FENCENB_CMD))) { /* only if a bozo error occurs */ PMIX_ERROR_LOG(PMIX_ERROR); /* DO NOT HANG */ @@ -589,6 +626,15 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, /* add this contributor to the tracker so they get * notified when we are done */ pmix_list_append(&trk->local_cbs, &cd->super); + /* if a timeout was specified, set it */ + if (0 < tv.tv_sec) { + PMIX_RETAIN(trk); + cd->trk = trk; + pmix_event_evtimer_set(pmix_globals.evbase, &cd->ev, + fence_timeout, cd); + pmix_event_evtimer_add(&cd->ev, &tv); + cd->event_active = true; + } /* if all local contributions have been received, * let the local host's server know that we are at the @@ -596,7 +642,7 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, * across all participants has been completed */ if (trk->def_complete && pmix_list_get_size(&trk->local_cbs) == trk->nlocal) { - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "fence complete"); /* if the user asked us to collect data, then we have * to provide any locally collected data to the host @@ -614,12 +660,12 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, &tmp, 1, PMIX_BYTE); if (PMIX_COLLECT_YES == trk->collect_type) { - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.fence_output, "fence - assembling data"); PMIX_LIST_FOREACH(scd, &trk->local_cbs, pmix_server_caddy_t) { /* get any remote contribution - note that there * may not be a contribution */ - (void)strncpy(pcs.nspace, scd->peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(pcs.nspace, scd->peer->info->pname.nspace, PMIX_MAX_NSLEN); pcs.rank = scd->peer->info->pname.rank; PMIX_CONSTRUCT(&cb, pmix_cb_t); cb.proc = &pcs; @@ -678,9 +724,14 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, /* now unload the blob and pass it upstairs */ PMIX_UNLOAD_BUFFER(&bucket, data, sz); PMIX_DESTRUCT(&bucket); - pmix_host_server.fence_nb(trk->pcs, trk->npcs, - trk->info, trk->ninfo, - data, sz, trk->modexcbfunc, trk); + trk->host_called = true; + rc = pmix_host_server.fence_nb(trk->pcs, trk->npcs, + trk->info, trk->ninfo, + data, sz, trk->modexcbfunc, trk); + if (PMIX_SUCCESS != rc) { + pmix_list_remove_item(&pmix_server_globals.collectives, &trk->super); + PMIX_RELEASE(trk); + } } cleanup: @@ -718,7 +769,7 @@ pmix_status_t pmix_server_publish(pmix_peer_t *peer, pmix_proc_t proc; uint32_t uid; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.pub_output, "recvd PUBLISH"); if (NULL == pmix_host_server.publish) { @@ -761,12 +812,12 @@ pmix_status_t pmix_server_publish(pmix_peer_t *peer, goto cleanup; } } - (void)strncpy(cd->info[cd->ninfo-1].key, PMIX_USERID, PMIX_MAX_KEYLEN); + pmix_strncpy(cd->info[cd->ninfo-1].key, PMIX_USERID, PMIX_MAX_KEYLEN); cd->info[cd->ninfo-1].value.type = PMIX_UINT32; cd->info[cd->ninfo-1].value.data.uint32 = uid; /* call the local server */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; rc = pmix_host_server.publish(&proc, cd->info, cd->ninfo, opcbfunc, cd); @@ -813,7 +864,7 @@ pmix_status_t pmix_server_lookup(pmix_peer_t *peer, pmix_proc_t proc; uint32_t uid; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.pub_output, "recvd LOOKUP"); if (NULL == pmix_host_server.lookup) { @@ -875,12 +926,12 @@ pmix_status_t pmix_server_lookup(pmix_peer_t *peer, goto cleanup; } } - (void)strncpy(cd->info[cd->ninfo-1].key, PMIX_USERID, PMIX_MAX_KEYLEN); + pmix_strncpy(cd->info[cd->ninfo-1].key, PMIX_USERID, PMIX_MAX_KEYLEN); cd->info[cd->ninfo-1].value.type = PMIX_UINT32; cd->info[cd->ninfo-1].value.data.uint32 = uid; /* call the local server */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; rc = pmix_host_server.lookup(&proc, cd->keys, cd->info, cd->ninfo, lkcbfunc, cd); @@ -909,7 +960,7 @@ pmix_status_t pmix_server_unpublish(pmix_peer_t *peer, pmix_proc_t proc; uint32_t uid; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.pub_output, "recvd UNPUBLISH"); if (NULL == pmix_host_server.unpublish) { @@ -971,12 +1022,12 @@ pmix_status_t pmix_server_unpublish(pmix_peer_t *peer, goto cleanup; } } - (void)strncpy(cd->info[cd->ninfo-1].key, PMIX_USERID, PMIX_MAX_KEYLEN); + pmix_strncpy(cd->info[cd->ninfo-1].key, PMIX_USERID, PMIX_MAX_KEYLEN); cd->info[cd->ninfo-1].value.type = PMIX_UINT32; cd->info[cd->ninfo-1].value.data.uint32 = uid; /* call the local server */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; rc = pmix_host_server.unpublish(&proc, cd->keys, cd->info, cd->ninfo, opcbfunc, cd); @@ -1003,7 +1054,7 @@ static void spcbfunc(pmix_status_t status, PMIX_INFO_FREE(cd->info, cd->ninfo); } if (NULL != cd->apps) { - PMIX_APP_CREATE(cd->apps, cd->napps); + PMIX_APP_FREE(cd->apps, cd->napps); } if (NULL != cd->spcbfunc) { cd->spcbfunc(status, nspace, cd->cbdata); @@ -1022,11 +1073,10 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, pmix_proc_t proc; size_t ninfo; - pmix_output_verbose(2, pmix_globals.debug_output, - "recvd SPAWN"); + pmix_output_verbose(2, pmix_server_globals.spawn_output, + "recvd SPAWN from %s:%d", peer->info->pname.nspace, peer->info->pname.rank); if (NULL == pmix_host_server.spawn) { - PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); return PMIX_ERR_NOT_SUPPORTED; } @@ -1035,11 +1085,10 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, if (NULL == cd) { return PMIX_ERR_NOMEM; } + PMIX_RETAIN(peer); + cd->peer = peer; cd->spcbfunc = cbfunc; cd->cbdata = cbdata; - /* setup the proc name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); - proc.rank = peer->info->pname.rank; /* unpack the number of job-level directives */ cnt=1; @@ -1052,11 +1101,6 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, /* always add one directive that indicates whether the requestor * is a tool or client */ cd->ninfo = ninfo + 1; - /* if it is a client, then we set the parent and - * "spawned" keys as well */ - if (!PMIX_PROC_IS_TOOL(peer)) { - cd->ninfo += 2; - } PMIX_INFO_CREATE(cd->info, cd->ninfo); if (NULL == cd->info) { rc = PMIX_ERR_NOMEM; @@ -1072,13 +1116,12 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, goto cleanup; } } + /* add the directive to the end */ if (PMIX_PROC_IS_TOOL(peer)) { PMIX_INFO_LOAD(&cd->info[ninfo], PMIX_REQUESTOR_IS_TOOL, NULL, PMIX_BOOL); } else { - PMIX_INFO_LOAD(&cd->info[cd->ninfo-3], PMIX_SPAWNED, NULL, PMIX_BOOL); - PMIX_INFO_LOAD(&cd->info[cd->ninfo-2], PMIX_PARENT_ID, &proc, PMIX_PROC); - PMIX_INFO_LOAD(&cd->info[cd->ninfo-1], PMIX_REQUESTOR_IS_CLIENT, NULL, PMIX_BOOL); + PMIX_INFO_LOAD(&cd->info[ninfo], PMIX_REQUESTOR_IS_CLIENT, NULL, PMIX_BOOL); } /* unpack the number of apps */ @@ -1103,6 +1146,8 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, } } /* call the local server */ + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + proc.rank = peer->info->pname.rank; rc = pmix_host_server.spawn(&proc, cd->info, cd->ninfo, cd->apps, cd->napps, spcbfunc, cd); cleanup: @@ -1118,25 +1163,156 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, return rc; } +pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, + pmix_buffer_t *buf, + pmix_op_cbfunc_t cbfunc) +{ + int32_t cnt; + pmix_status_t rc; + pmix_info_t *info = NULL; + size_t nprocs, ninfo; + pmix_server_trkr_t *trk; + pmix_proc_t *procs = NULL; + + if (NULL == pmix_host_server.disconnect) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* unpack the number of procs */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, cd->peer, buf, &nprocs, &cnt, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto cleanup; + } + /* there must be at least one proc - we do not allow the client + * to send us NULL proc as the server has no idea what to do + * with that situation. Instead, the client should at least send + * us their own namespace for the use-case where the connection + * spans all procs in that namespace */ + if (nprocs < 1) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + rc = PMIX_ERR_BAD_PARAM; + goto cleanup; + } + + /* unpack the procs */ + PMIX_PROC_CREATE(procs, nprocs); + if (NULL == procs) { + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + cnt = nprocs; + PMIX_BFROPS_UNPACK(rc, cd->peer, buf, procs, &cnt, PMIX_PROC); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto cleanup; + } + + /* unpack the number of provided info structs */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, cd->peer, buf, &ninfo, &cnt, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + return rc; + } + if (0 < ninfo) { + PMIX_INFO_CREATE(info, ninfo); + if (NULL == info) { + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + /* unpack the info */ + cnt = ninfo; + PMIX_BFROPS_UNPACK(rc, cd->peer, buf, info, &cnt, PMIX_INFO); + if (PMIX_SUCCESS != rc) { + goto cleanup; + } + } + + /* find/create the local tracker for this operation */ + if (NULL == (trk = get_tracker(NULL, procs, nprocs, PMIX_DISCONNECTNB_CMD))) { + /* we don't have this tracker yet, so get a new one */ + if (NULL == (trk = new_tracker(NULL, procs, nprocs, PMIX_DISCONNECTNB_CMD))) { + /* only if a bozo error occurs */ + PMIX_ERROR_LOG(PMIX_ERROR); + rc = PMIX_ERROR; + goto cleanup; + } + trk->op_cbfunc = cbfunc; + } + + /* if the info keys have not been provided yet, pass + * them along here */ + if (NULL == trk->info && NULL != info) { + trk->info = info; + trk->ninfo = ninfo; + info = NULL; + ninfo = 0; + } + + /* add this contributor to the tracker so they get + * notified when we are done */ + pmix_list_append(&trk->local_cbs, &cd->super); + /* if all local contributions have been received, + * let the local host's server know that we are at the + * "fence" point - they will callback once the [dis]connect + * across all participants has been completed */ + if (trk->def_complete && + pmix_list_get_size(&trk->local_cbs) == trk->nlocal) { + trk->host_called = true; + rc = pmix_host_server.disconnect(trk->pcs, trk->npcs, trk->info, trk->ninfo, cbfunc, trk); + if (PMIX_SUCCESS != rc) { + /* remove this contributor from the list - they will be notified + * by the switchyard */ + pmix_list_remove_item(&trk->local_cbs, &cd->super); + } + } else { + rc = PMIX_SUCCESS; + } + + cleanup: + if (NULL != info) { + PMIX_INFO_FREE(info, ninfo); + } + return rc; +} + +static void connect_timeout(int sd, short args, void *cbdata) +{ + pmix_server_caddy_t *cd = (pmix_server_caddy_t*)cbdata; + + pmix_output_verbose(2, pmix_server_globals.connect_output, + "ALERT: connect timeout fired"); + + /* execute the provided callback function with the error */ + if (NULL != cd->trk->op_cbfunc) { + cd->trk->op_cbfunc(PMIX_ERR_TIMEOUT, cd->trk); + return; // the cbfunc will have cleaned up the tracker + } + cd->event_active = false; + /* remove it from the list */ + pmix_list_remove_item(&cd->trk->local_cbs, &cd->super); + PMIX_RELEASE(cd); +} + pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, - pmix_buffer_t *buf, bool disconnect, + pmix_buffer_t *buf, pmix_op_cbfunc_t cbfunc) { int32_t cnt; pmix_status_t rc; pmix_proc_t *procs = NULL; pmix_info_t *info = NULL; - size_t nprocs, ninfo; + size_t nprocs, ninfo, n; pmix_server_trkr_t *trk; - pmix_cmd_t type = PMIX_CONNECTNB_CMD; + struct timeval tv = {0, 0}; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.connect_output, "recvd CONNECT from peer %s:%d", cd->peer->info->pname.nspace, cd->peer->info->pname.rank); - if ((disconnect && NULL == pmix_host_server.disconnect) || - (!disconnect && NULL == pmix_host_server.connect)) { + if (NULL == pmix_host_server.connect) { return PMIX_ERR_NOT_SUPPORTED; } @@ -1189,15 +1365,19 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, if (PMIX_SUCCESS != rc) { goto cleanup; } + /* check for a timeout */ + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_TIMEOUT, PMIX_MAX_KEYLEN)) { + tv.tv_sec = info[n].value.data.uint32; + break; + } + } } /* find/create the local tracker for this operation */ - if (disconnect) { - type = PMIX_DISCONNECTNB_CMD; - } - if (NULL == (trk = get_tracker(procs, nprocs, type))) { + if (NULL == (trk = get_tracker(NULL, procs, nprocs, PMIX_CONNECTNB_CMD))) { /* we don't have this tracker yet, so get a new one */ - if (NULL == (trk = new_tracker(procs, nprocs, type))) { + if (NULL == (trk = new_tracker(NULL, procs, nprocs, PMIX_CONNECTNB_CMD))) { /* only if a bozo error occurs */ PMIX_ERROR_LOG(PMIX_ERROR); /* DO NOT HANG */ @@ -1209,6 +1389,7 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, } trk->op_cbfunc = cbfunc; } + /* if the info keys have not been provided yet, pass * them along here */ if (NULL == trk->info && NULL != info) { @@ -1220,22 +1401,33 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, /* add this contributor to the tracker so they get * notified when we are done */ - PMIX_RETAIN(cd); // prevent the caddy from being released when we return pmix_list_append(&trk->local_cbs, &cd->super); + /* if all local contributions have been received, * let the local host's server know that we are at the * "fence" point - they will callback once the [dis]connect * across all participants has been completed */ if (trk->def_complete && pmix_list_get_size(&trk->local_cbs) == trk->nlocal) { - if (disconnect) { - rc = pmix_host_server.disconnect(trk->pcs, trk->npcs, trk->info, trk->ninfo, cbfunc, trk); - } else { - rc = pmix_host_server.connect(trk->pcs, trk->npcs, trk->info, trk->ninfo, cbfunc, trk); + trk->host_called = true; + rc = pmix_host_server.connect(trk->pcs, trk->npcs, trk->info, trk->ninfo, cbfunc, trk); + if (PMIX_SUCCESS != rc) { + /* remove this contributor from the list - they will be notified + * by the switchyard */ + pmix_list_remove_item(&trk->local_cbs, &cd->super); } } else { rc = PMIX_SUCCESS; } + /* if a timeout was specified, set it */ + if (PMIX_SUCCESS == rc && 0 < tv.tv_sec) { + PMIX_RETAIN(trk); + cd->trk = trk; + pmix_event_evtimer_set(pmix_globals.evbase, &cd->ev, + connect_timeout, cd); + pmix_event_evtimer_add(&cd->ev, &tv); + cd->event_active = true; + } cleanup: if (NULL != procs) { @@ -1247,6 +1439,177 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, return rc; } +static void _check_cached_events(int sd, short args, void *cbdata) +{ + pmix_setup_caddy_t *scd = (pmix_setup_caddy_t*)cbdata; + pmix_notify_caddy_t *cd; + pmix_range_trkr_t rngtrk; + pmix_proc_t proc; + int i; + size_t k, n; + bool found, matched; + pmix_buffer_t *relay; + pmix_status_t ret = PMIX_SUCCESS; + pmix_cmd_t cmd = PMIX_NOTIFY_CMD; + + /* check if any matching notifications have been cached */ + rngtrk.procs = NULL; + rngtrk.nprocs = 0; + for (i=0; i < pmix_globals.max_events; i++) { + pmix_hotel_knock(&pmix_globals.notifications, i, (void**)&cd); + if (NULL == cd) { + continue; + } + found = false; + if (NULL == scd->codes) { + if (!cd->nondefault) { + /* they registered a default event handler - always matches */ + found = true; + } + } else { + for (k=0; k < scd->ncodes; k++) { + if (scd->codes[k] == cd->status) { + found = true; + break; + } + } + } + if (!found) { + continue; + } + /* check if the affected procs (if given) match those they + * wanted to know about */ + if (!pmix_notify_check_affected(cd->affected, cd->naffected, + scd->procs, scd->nprocs)) { + continue; + } + /* check the range */ + if (NULL == cd->targets) { + rngtrk.procs = &cd->source; + rngtrk.nprocs = 1; + } else { + rngtrk.procs = cd->targets; + rngtrk.nprocs = cd->ntargets; + } + rngtrk.range = cd->range; + PMIX_LOAD_PROCID(&proc, scd->peer->info->pname.nspace, scd->peer->info->pname.rank); + if (!pmix_notify_check_range(&rngtrk, &proc)) { + continue; + } + /* if we were given specific targets, check if this is one */ + found = false; + if (NULL != cd->targets) { + matched = false; + for (n=0; n < cd->ntargets; n++) { + /* if the source of the event is the same peer just registered, then ignore it + * as the event notification system will have already locally + * processed it */ + if (PMIX_CHECK_PROCID(&cd->source, &scd->peer->info->pname)) { + continue; + } + if (PMIX_CHECK_PROCID(&scd->peer->info->pname, &cd->targets[n])) { + matched = true; + /* track the number of targets we have left to notify */ + --cd->nleft; + /* if this is the last one, then evict this event + * from the cache */ + if (0 == cd->nleft) { + pmix_hotel_checkout(&pmix_globals.notifications, cd->room); + found = true; // mark that we should release cd + } + break; + } + } + if (!matched) { + /* do not notify this one */ + continue; + } + } + + /* all matches - notify */ + relay = PMIX_NEW(pmix_buffer_t); + if (NULL == relay) { + /* nothing we can do */ + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + ret = PMIX_ERR_NOMEM; + break; + } + /* pack the info data stored in the event */ + PMIX_BFROPS_PACK(ret, scd->peer, relay, &cmd, 1, PMIX_COMMAND); + if (PMIX_SUCCESS != ret) { + PMIX_ERROR_LOG(ret); + break; + } + PMIX_BFROPS_PACK(ret, scd->peer, relay, &cd->status, 1, PMIX_STATUS); + if (PMIX_SUCCESS != ret) { + PMIX_ERROR_LOG(ret); + break; + } + PMIX_BFROPS_PACK(ret, scd->peer, relay, &cd->source, 1, PMIX_PROC); + if (PMIX_SUCCESS != ret) { + PMIX_ERROR_LOG(ret); + break; + } + PMIX_BFROPS_PACK(ret, scd->peer, relay, &cd->ninfo, 1, PMIX_SIZE); + if (PMIX_SUCCESS != ret) { + PMIX_ERROR_LOG(ret); + break; + } + if (0 < cd->ninfo) { + PMIX_BFROPS_PACK(ret, scd->peer, relay, cd->info, cd->ninfo, PMIX_INFO); + if (PMIX_SUCCESS != ret) { + PMIX_ERROR_LOG(ret); + break; + } + } + PMIX_SERVER_QUEUE_REPLY(ret, scd->peer, 0, relay); + if (PMIX_SUCCESS != ret) { + PMIX_RELEASE(relay); + } + if (found) { + PMIX_RELEASE(cd); + } + } + /* release the caddy */ + if (NULL != scd->codes) { + free(scd->codes); + } + if (NULL != scd->info) { + PMIX_INFO_FREE(scd->info, scd->ninfo); + } + if (NULL != scd->opcbfunc) { + scd->opcbfunc(ret, scd->cbdata); + } + PMIX_RELEASE(scd); +} + +/* provide a callback function for the host when it finishes + * processing the registration */ +static void regevopcbfunc(pmix_status_t status, void *cbdata) +{ + pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; + + /* if the registration succeeded, then check local cache */ + if (PMIX_SUCCESS == status) { + _check_cached_events(0, 0, cd); + return; + } + + /* it didn't succeed, so cleanup and execute the callback + * so we don't hang */ + if (NULL != cd->codes) { + free(cd->codes); + } + if (NULL != cd->info) { + PMIX_INFO_FREE(cd->info, cd->ninfo); + } + if (NULL != cd->opcbfunc) { + cd->opcbfunc(status, cd->cbdata); + } + PMIX_RELEASE(cd); +} + + pmix_status_t pmix_server_register_events(pmix_peer_t *peer, pmix_buffer_t *buf, pmix_op_cbfunc_t cbfunc, @@ -1256,21 +1619,18 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, pmix_status_t rc; pmix_status_t *codes = NULL; pmix_info_t *info = NULL; - size_t ninfo=0, ncodes, n, k; + size_t ninfo=0, ncodes, n; pmix_regevents_info_t *reginfo; - pmix_peer_events_info_t *prev; - pmix_notify_caddy_t *cd; + pmix_peer_events_info_t *prev = NULL; pmix_setup_caddy_t *scd; - int i; bool enviro_events = false; - bool found, matched; - pmix_buffer_t *relay; - pmix_cmd_t cmd = PMIX_NOTIFY_CMD; + bool found; pmix_proc_t *affected = NULL; size_t naffected = 0; - pmix_output_verbose(2, pmix_globals.debug_output, - "recvd register events"); + pmix_output_verbose(2, pmix_server_globals.event_output, + "recvd register events for peer %s:%d", + peer->info->pname.nspace, peer->info->pname.rank); /* unpack the number of codes */ cnt=1; @@ -1318,9 +1678,7 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, /* check the directives */ for (n=0; n < ninfo; n++) { - if (0 == strncmp(info[n].key, PMIX_EVENT_ENVIRO_LEVEL, PMIX_MAX_KEYLEN)) { - enviro_events = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strncmp(info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + if (PMIX_CHECK_KEY(&info[n], PMIX_EVENT_AFFECTED_PROC)) { if (NULL != affected) { PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); rc = PMIX_ERR_BAD_PARAM; @@ -1329,7 +1687,7 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, naffected = 1; PMIX_PROC_CREATE(affected, naffected); memcpy(affected, info[n].value.data.proc, sizeof(pmix_proc_t)); - } else if (0 == strncmp(info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_EVENT_AFFECTED_PROCS)) { if (NULL != affected) { PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); rc = PMIX_ERR_BAD_PARAM; @@ -1341,6 +1699,14 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, } } + /* check the codes for system events */ + for (n=0; n < ncodes; n++) { + if (PMIX_SYSTEM_EVENT(codes[n])) { + enviro_events = true; + break; + } + } + /* if they asked for enviro events, and our host doesn't support * register_events, then we cannot meet the request */ if (enviro_events && NULL == pmix_host_server.register_events) { @@ -1349,10 +1715,36 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, goto cleanup; } + /* if they didn't send us any codes, then they are registering a + * default event handler. In that case, check only for default + * handlers and add this request to it, if not already present */ + if (0 == ncodes) { + PMIX_LIST_FOREACH(reginfo, &pmix_server_globals.events, pmix_regevents_info_t) { + if (PMIX_MAX_ERR_CONSTANT == reginfo->code) { + /* both are default handlers */ + prev = PMIX_NEW(pmix_peer_events_info_t); + if (NULL == prev) { + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + PMIX_RETAIN(peer); + prev->peer = peer; + if (NULL != affected) { + PMIX_PROC_CREATE(prev->affected, naffected); + prev->naffected = naffected; + memcpy(prev->affected, affected, naffected * sizeof(pmix_proc_t)); + } + pmix_list_append(®info->peers, &prev->super); + break; + } + } + rc = PMIX_OPERATION_SUCCEEDED; + goto cleanup; + } + /* store the event registration info so we can call the registered * client when the server notifies the event */ - k=0; - do { + for (n=0; n < ncodes; n++) { found = false; PMIX_LIST_FOREACH(reginfo, &pmix_server_globals.events, pmix_regevents_info_t) { if (NULL == codes) { @@ -1366,35 +1758,28 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, } else { if (PMIX_MAX_ERR_CONSTANT == reginfo->code) { continue; - } else if (codes[k] == reginfo->code) { + } else if (codes[n] == reginfo->code) { found = true; break; } } } if (found) { - /* found it - add this peer if we don't already have it */ - found = false; - PMIX_LIST_FOREACH(prev, ®info->peers, pmix_peer_events_info_t) { - if (prev->peer == peer) { - /* already have it */ - rc = PMIX_SUCCESS; - found = true; - break; - } + /* found it - add this request */ + prev = PMIX_NEW(pmix_peer_events_info_t); + if (NULL == prev) { + rc = PMIX_ERR_NOMEM; + goto cleanup; } - if (!found) { - /* get here if we don't already have this peer */ - prev = PMIX_NEW(pmix_peer_events_info_t); - if (NULL == prev) { - rc = PMIX_ERR_NOMEM; - goto cleanup; - } - PMIX_RETAIN(peer); - prev->peer = peer; - prev->enviro_events = enviro_events; - pmix_list_append(®info->peers, &prev->super); + PMIX_RETAIN(peer); + prev->peer = peer; + if (NULL != affected) { + PMIX_PROC_CREATE(prev->affected, naffected); + prev->naffected = naffected; + memcpy(prev->affected, affected, naffected * sizeof(pmix_proc_t)); } + prev->enviro_events = enviro_events; + pmix_list_append(®info->peers, &prev->super); } else { /* if we get here, then we didn't find an existing registration for this code */ reginfo = PMIX_NEW(pmix_regevents_info_t); @@ -1405,7 +1790,7 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, if (NULL == codes) { reginfo->code = PMIX_MAX_ERR_CONSTANT; } else { - reginfo->code = codes[k]; + reginfo->code = codes[n]; } pmix_list_append(&pmix_server_globals.events, ®info->super); prev = PMIX_NEW(pmix_peer_events_info_t); @@ -1415,14 +1800,23 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, } PMIX_RETAIN(peer); prev->peer = peer; + if (NULL != affected) { + PMIX_PROC_CREATE(prev->affected, naffected); + prev->naffected = naffected; + memcpy(prev->affected, affected, naffected * sizeof(pmix_proc_t)); + } prev->enviro_events = enviro_events; pmix_list_append(®info->peers, &prev->super); } - ++k; - } while (k < ncodes); + } /* if they asked for enviro events, call the local server */ if (enviro_events) { + /* if they don't support this, then we cannot do it */ + if (NULL == pmix_host_server.register_events) { + rc = PMIX_ERR_NOT_SUPPORTED; + goto cleanup; + } /* need to ensure the arrays don't go away until after the * host RM is done with them */ scd = PMIX_NEW(pmix_setup_caddy_t); @@ -1430,166 +1824,83 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, rc = PMIX_ERR_NOMEM; goto cleanup; } - if (NULL != codes) { - scd->codes = (pmix_status_t*)malloc(ncodes * sizeof(pmix_status_t)); - if (NULL == scd->codes) { - rc = PMIX_ERR_NOMEM; - PMIX_RELEASE(scd); - goto cleanup; - } - memcpy(scd->codes, codes, ncodes * sizeof(pmix_status_t)); - scd->ncodes = ncodes; - } - if (NULL != info) { - PMIX_INFO_CREATE(scd->info, ninfo); - if (NULL == scd->info) { - rc = PMIX_ERR_NOMEM; - if (NULL != scd->codes) { - free(scd->codes); - } - PMIX_RELEASE(scd); - goto cleanup; - } - /* copy the info across */ - for (n=0; n < ninfo; n++) { - PMIX_INFO_XFER(&scd->info[n], &info[n]); - } - scd->ninfo = ninfo; - } + PMIX_RETAIN(peer); + scd->peer = peer; + scd->codes = codes; + scd->ncodes = ncodes; + scd->info = info; + scd->ninfo = ninfo; scd->opcbfunc = cbfunc; scd->cbdata = cbdata; - if (PMIX_SUCCESS != (rc = pmix_host_server.register_events(scd->codes, scd->ncodes, scd->info, scd->ninfo, opcbfunc, scd))) { - pmix_output_verbose(2, pmix_globals.debug_output, - "server register events: host server reg events returned rc =%d", rc); - if (NULL != scd->codes) { - free(scd->codes); - } - if (NULL != scd->info) { - PMIX_INFO_FREE(scd->info, scd->ninfo); + if (PMIX_SUCCESS == (rc = pmix_host_server.register_events(scd->codes, scd->ncodes, scd->info, scd->ninfo, regevopcbfunc, scd))) { + /* the host will call us back when completed */ + pmix_output_verbose(2, pmix_server_globals.event_output, + "server register events: host server processing event registration"); + if (NULL != affected) { + free(affected); } - PMIX_RELEASE(scd); + return rc; + } else if (PMIX_OPERATION_SUCCEEDED == rc) { + /* we need to check cached notifications, but we want to ensure + * that occurs _after_ the client returns from registering the + * event handler in case the event is flagged for do_not_cache. + * Setup an event to fire after we return as that means it will + * occur after we send the registration response back to the client, + * thus guaranteeing that the client will get their registration + * callback prior to delivery of an event notification */ + PMIX_RETAIN(peer); + scd->peer = peer; + scd->procs = affected; + scd->nprocs = naffected; + scd->opcbfunc = NULL; + scd->cbdata = NULL; + PMIX_THREADSHIFT(scd, _check_cached_events); + return rc; } else { - goto check; + /* host returned a genuine error and won't be calling the callback function */ + pmix_output_verbose(2, pmix_server_globals.event_output, + "server register events: host server reg events returned rc =%d", rc); + PMIX_RELEASE(scd); + goto cleanup; + } + } else { + rc = PMIX_OPERATION_SUCCEEDED; + /* we need to check cached notifications, but we want to ensure + * that occurs _after_ the client returns from registering the + * event handler in case the event is flagged for do_not_cache. + * Setup an event to fire after we return as that means it will + * occur after we send the registration response back to the client, + * thus guaranteeing that the client will get their registration + * callback prior to delivery of an event notification */ + scd = PMIX_NEW(pmix_setup_caddy_t); + PMIX_RETAIN(peer); + scd->peer = peer; + scd->codes = codes; + scd->ncodes = ncodes; + scd->procs = affected; + scd->nprocs = naffected; + scd->opcbfunc = NULL; + scd->cbdata = NULL; + PMIX_THREADSHIFT(scd, _check_cached_events); + if (NULL != info) { + PMIX_INFO_FREE(info, ninfo); } + return rc; } cleanup: - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.event_output, "server register events: ninfo =%lu rc =%d", ninfo, rc); - /* be sure to execute the callback */ - if (NULL != cbfunc) { - cbfunc(rc, cbdata); - } if (NULL != info) { PMIX_INFO_FREE(info, ninfo); } - if (PMIX_SUCCESS != rc) { - if (NULL != codes) { - free(codes); - } - if (NULL != affected) { - PMIX_PROC_FREE(affected, naffected); - } - return rc; - } - - check: - /* check if any matching notifications have been cached */ - for (i=0; i < pmix_globals.notifications.size; i++) { - if (NULL == (cd = (pmix_notify_caddy_t*)pmix_ring_buffer_poke(&pmix_globals.notifications, i))) { - break; - } - found = false; - if (NULL == codes) { - /* they registered a default event handler - always matches */ - found = true; - } else { - for (k=0; k < ncodes; k++) { - if (codes[k] == cd->status) { - found = true; - break; - } - } - } - if (found) { - /* if we were given specific targets, check if this is one */ - if (NULL != cd->targets) { - matched = false; - for (n=0; n < cd->ntargets; n++) { - if (0 != strncmp(peer->info->pname.nspace, cd->targets[n].nspace, PMIX_MAX_NSLEN)) { - continue; - } - /* if the source of the event is the same peer just registered, then ignore it - * as the event notification system will have already locally - * processed it */ - if (0 == strncmp(peer->info->pname.nspace, cd->source.nspace, PMIX_MAX_NSLEN) && - peer->info->pname.rank == cd->source.rank) { - continue; - } - if (PMIX_RANK_WILDCARD == cd->targets[n].rank || - peer->info->pname.rank == cd->targets[n].rank) { - matched = true; - break; - } - } - if (!matched) { - /* do not notify this one */ - continue; - } - } - /* if they specified affected proc(s) they wanted to know about, check */ - if (!pmix_notify_check_affected(cd->affected, cd->naffected, - affected, naffected)) { - continue; - } - /* all matches - notify */ - relay = PMIX_NEW(pmix_buffer_t); - if (NULL == relay) { - /* nothing we can do */ - PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - return PMIX_ERR_NOMEM; - } - /* pack the info data stored in the event */ - PMIX_BFROPS_PACK(rc, peer, relay, &cmd, 1, PMIX_COMMAND); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - break; - } - PMIX_BFROPS_PACK(rc, peer, relay, &cd->status, 1, PMIX_STATUS); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - break; - } - PMIX_BFROPS_PACK(rc, peer, relay, &cd->source, 1, PMIX_PROC); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - break; - } - PMIX_BFROPS_PACK(rc, peer, relay, &cd->ninfo, 1, PMIX_SIZE); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - break; - } - if (0 < cd->ninfo) { - PMIX_BFROPS_PACK(rc, peer, relay, cd->info, cd->ninfo, PMIX_INFO); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - break; - } - } - PMIX_SERVER_QUEUE_REPLY(peer, 0, relay); - } - } - if (!enviro_events) { - if (NULL != codes) { - free(codes); - } + if (NULL != codes) { + free(codes); } if (NULL != affected) { PMIX_PROC_FREE(affected, naffected); } - - return PMIX_SUCCESS; + return rc; } void pmix_server_deregister_events(pmix_peer_t *peer, @@ -1601,7 +1912,7 @@ void pmix_server_deregister_events(pmix_peer_t *peer, pmix_regevents_info_t *reginfo_next; pmix_peer_events_info_t *prev; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.event_output, "recvd deregister events"); /* unpack codes and process until done */ @@ -1635,7 +1946,6 @@ void pmix_server_deregister_events(pmix_peer_t *peer, } } - static void local_cbfunc(pmix_status_t status, void *cbdata) { pmix_notify_caddy_t *cd = (pmix_notify_caddy_t*)cbdata; @@ -1669,6 +1979,13 @@ static void intermed_step(pmix_status_t status, void *cbdata) goto complete; } + /* since our host is going to send this everywhere, it may well + * come back to us. We already processed it, so mark it here + * to ensure we don't do it again. We previously inserted the + * PMIX_SERVER_INTERNAL_NOTIFY key at the very end of the + * info array - just overwrite that position */ + PMIX_INFO_LOAD(&cd->info[cd->ninfo-1], PMIX_EVENT_PROXY, &pmix_globals.myid, PMIX_PROC); + /* pass it to our host RM for distribution */ rc = pmix_host_server.notify_event(cd->status, &cd->source, cd->range, cd->info, cd->ninfo, local_cbfunc, cd); @@ -1687,6 +2004,11 @@ static void intermed_step(pmix_status_t status, void *cbdata) PMIX_RELEASE(cd); } +/* Receive an event sent by the client library. Since it was sent + * to us by one client, we have to both process it locally to ensure + * we notify all relevant local clients AND (assuming a range other + * than LOCAL) deliver to our host, requesting that they send it + * to all peer servers in the current session */ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, pmix_buffer_t *buf, pmix_op_cbfunc_t cbfunc, @@ -1695,11 +2017,12 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, int32_t cnt; pmix_status_t rc; pmix_notify_caddy_t *cd; - size_t ninfo; + size_t ninfo, n; - pmix_output_verbose(2, pmix_globals.debug_output, - "%s:%d recvd event notification from client", - pmix_globals.myid.nspace, pmix_globals.myid.rank); + pmix_output_verbose(2, pmix_server_globals.event_output, + "%s:%d recvd event notification from client %s:%d", + pmix_globals.myid.nspace, pmix_globals.myid.rank, + peer->info->pname.nspace, peer->info->pname.rank); cd = PMIX_NEW(pmix_notify_caddy_t); if (NULL == cd) { @@ -1708,8 +2031,7 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, cd->cbfunc = cbfunc; cd->cbdata = cbdata; /* set the source */ - (void)strncpy(cd->source.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); - cd->source.rank = peer->info->pname.rank; + PMIX_LOAD_PROCID(&cd->source, peer->info->pname.nspace, peer->info->pname.rank); /* unpack status */ cnt = 1; @@ -1748,6 +2070,18 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, goto exit; } } + + /* check to see if we already processed this event - it is possible + * that a local client "echoed" it back to us and we want to avoid + * a potential infinite loop */ + for (n=0; n < ninfo; n++) { + if (PMIX_CHECK_KEY(&cd->info[n], PMIX_SERVER_INTERNAL_NOTIFY)) { + /* yep, we did - so don't do it again! */ + rc = PMIX_OPERATION_SUCCEEDED; + goto exit; + } + } + /* add an info object to mark that we recvd this internally */ PMIX_INFO_LOAD(&cd->info[ninfo], PMIX_SERVER_INTERNAL_NOTIFY, NULL, PMIX_BOOL); /* process it */ @@ -1758,12 +2092,13 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, intermed_step, cd))) { goto exit; } - /* tell the switchyard we will handle it from here */ - return PMIX_SUCCESS; + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(cd); + } + return rc; exit: PMIX_RELEASE(cd); - cbfunc(rc, cbdata); return rc; } @@ -1776,14 +2111,14 @@ pmix_status_t pmix_server_query(pmix_peer_t *peer, pmix_status_t rc; pmix_query_caddy_t *cd; pmix_proc_t proc; + pmix_cb_t cb; + size_t n, p; + pmix_list_t results; + pmix_kval_t *kv, *kvnxt; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "recvd query from client"); - if (NULL == pmix_host_server.query) { - return PMIX_ERR_NOT_SUPPORTED; - } - cd = PMIX_NEW(pmix_query_caddy_t); if (NULL == cd) { return PMIX_ERR_NOMEM; @@ -1794,36 +2129,136 @@ pmix_status_t pmix_server_query(pmix_peer_t *peer, PMIX_BFROPS_UNPACK(rc, peer, buf, &cd->nqueries, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - goto exit; + PMIX_RELEASE(cd); + return rc; } /* unpack the queries */ if (0 < cd->nqueries) { PMIX_QUERY_CREATE(cd->queries, cd->nqueries); if (NULL == cd->queries) { rc = PMIX_ERR_NOMEM; - goto exit; + PMIX_RELEASE(cd); + return rc; } cnt = cd->nqueries; PMIX_BFROPS_UNPACK(rc, peer, buf, cd->queries, &cnt, PMIX_QUERY); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - goto exit; + PMIX_RELEASE(cd); + return rc; } } + /* check the directives to see if they want us to refresh + * the local cached results - if we wanted to optimize this + * more, we would check each query and allow those that don't + * want to be refreshed to be executed locally, and those that + * did would be sent to the host. However, for now we simply + * determine that if we don't have it, then ask for everything */ + memset(proc.nspace, 0, PMIX_MAX_NSLEN+1); + proc.rank = PMIX_RANK_INVALID; + PMIX_CONSTRUCT(&results, pmix_list_t); + + for (n=0; n < cd->nqueries; n++) { + for (p=0; p < cd->queries[n].nqual; p++) { + if (PMIX_CHECK_KEY(&cd->queries[n].qualifiers[p], PMIX_QUERY_REFRESH_CACHE)) { + if (PMIX_INFO_TRUE(&cd->queries[n].qualifiers[p])) { + PMIX_LIST_DESTRUCT(&results); + goto query; + } + } else if (PMIX_CHECK_KEY(&cd->queries[n].qualifiers[p], PMIX_PROCID)) { + PMIX_LOAD_NSPACE(proc.nspace, cd->queries[n].qualifiers[p].value.data.proc->nspace); + proc.rank = cd->queries[n].qualifiers[p].value.data.proc->rank; + } else if (PMIX_CHECK_KEY(&cd->queries[n].qualifiers[p], PMIX_NSPACE)) { + PMIX_LOAD_NSPACE(proc.nspace, cd->queries[n].qualifiers[p].value.data.string); + } else if (PMIX_CHECK_KEY(&cd->queries[n].qualifiers[p], PMIX_RANK)) { + proc.rank = cd->queries[n].qualifiers[p].value.data.rank; + } else if (PMIX_CHECK_KEY(&cd->queries[n].qualifiers[p], PMIX_HOSTNAME)) { + if (0 != strcmp(cd->queries[n].qualifiers[p].value.data.string, pmix_globals.hostname)) { + /* asking about a different host, so ask for the info */ + PMIX_LIST_DESTRUCT(&results); + goto query; + } + } + } + /* we get here if a refresh isn't required - first try a local + * "get" on the data to see if we already have it */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + cb.copy = false; + /* set the proc */ + if (PMIX_RANK_INVALID == proc.rank && + 0 == strlen(proc.nspace)) { + /* use our id */ + cb.proc = &pmix_globals.myid; + } else { + if (0 == strlen(proc.nspace)) { + /* use our nspace */ + PMIX_LOAD_NSPACE(cb.proc->nspace, pmix_globals.myid.nspace); + } + if (PMIX_RANK_INVALID == proc.rank) { + /* user the wildcard rank */ + proc.rank = PMIX_RANK_WILDCARD; + } + cb.proc = &proc; + } + for (p=0; NULL != cd->queries[n].keys[p]; p++) { + cb.key = cd->queries[n].keys[p]; + PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb); + if (PMIX_SUCCESS != rc) { + /* needs to be passed to the host */ + PMIX_LIST_DESTRUCT(&results); + PMIX_DESTRUCT(&cb); + goto query; + } + /* need to retain this result */ + PMIX_LIST_FOREACH_SAFE(kv, kvnxt, &cb.kvs, pmix_kval_t) { + pmix_list_remove_item(&cb.kvs, &kv->super); + pmix_list_append(&results, &kv->super); + } + PMIX_DESTRUCT(&cb); + } + } + + /* if we get here, then all queries were completely locally + * resolved, so construct the results for return */ + rc = PMIX_ERR_NOT_FOUND; + if (0 < (cd->ninfo = pmix_list_get_size(&results))) { + PMIX_INFO_CREATE(cd->info, cd->ninfo); + n = 0; + PMIX_LIST_FOREACH_SAFE(kv, kvnxt, &results, pmix_kval_t) { + PMIX_LOAD_KEY(cd->info[n].key, kv->key); + rc = pmix_value_xfer(&cd->info[n].value, kv->value); + if (PMIX_SUCCESS != rc) { + PMIX_INFO_FREE(cd->info, cd->ninfo); + cd->info = NULL; + cd->ninfo = 0; + break; + } + ++n; + } + } + /* done with the list of results */ + PMIX_LIST_DESTRUCT(&results); + /* we can just call the cbfunc here as we are already + * in an event - let our internal cbfunc do a threadshift + * if necessary */ + cbfunc(PMIX_SUCCESS, cd->info, cd->ninfo, cd, NULL, NULL); + return PMIX_SUCCESS; + + query: + if (NULL == pmix_host_server.query) { + PMIX_RELEASE(cd); + return PMIX_ERR_NOT_SUPPORTED; + } + /* setup the requesting peer name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); - proc.rank = peer->info->pname.rank; + PMIX_LOAD_PROCID(&proc, peer->info->pname.nspace, peer->info->pname.rank); /* ask the host for the info */ if (PMIX_SUCCESS != (rc = pmix_host_server.query(&proc, cd->queries, cd->nqueries, cbfunc, cd))) { - goto exit; + PMIX_RELEASE(cd); } - return PMIX_SUCCESS; - - exit: - PMIX_RELEASE(cd); return rc; } @@ -1919,7 +2354,7 @@ pmix_status_t pmix_server_alloc(pmix_peer_t *peer, pmix_proc_t proc; pmix_alloc_directive_t directive; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "recvd query from client"); if (NULL == pmix_host_server.allocate) { @@ -1959,7 +2394,7 @@ pmix_status_t pmix_server_alloc(pmix_peer_t *peer, } /* setup the requesting peer name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; /* ask the host to execute the request */ @@ -1985,7 +2420,7 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, pmix_query_caddy_t *cd; pmix_proc_t proc; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "recvd job control request from client"); if (NULL == pmix_host_server.job_control) { @@ -2014,6 +2449,7 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, goto exit; } } + /* unpack the number of info objects */ cnt = 1; PMIX_BFROPS_UNPACK(rc, peer, buf, &cd->ninfo, &cnt, PMIX_SIZE); @@ -2033,8 +2469,7 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, } /* setup the requesting peer name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); - proc.rank = peer->info->pname.rank; + PMIX_LOAD_PROCID(&proc, peer->info->pname.nspace, peer->info->pname.rank); /* ask the host to execute the request */ if (PMIX_SUCCESS != (rc = pmix_host_server.job_control(&proc, @@ -2061,7 +2496,7 @@ pmix_status_t pmix_server_monitor(pmix_peer_t *peer, pmix_query_caddy_t *cd; pmix_proc_t proc; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_server_globals.base_output, "recvd monitor request from client"); @@ -2125,7 +2560,7 @@ pmix_status_t pmix_server_monitor(pmix_peer_t *peer, } /* setup the requesting peer name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; /* ask the host to execute the request */ @@ -2145,6 +2580,11 @@ pmix_status_t pmix_server_monitor(pmix_peer_t *peer, /***** INSTANCE SERVER LIBRARY CLASSES *****/ static void tcon(pmix_server_trkr_t *t) { + t->event_active = false; + t->host_called = false; + t->id = NULL; + memset(t->pname.nspace, 0, PMIX_MAX_NSLEN+1); + t->pname.rank = PMIX_RANK_UNDEF; t->pcs = NULL; t->npcs = 0; PMIX_CONSTRUCT_LOCK(&t->lock); @@ -2162,6 +2602,9 @@ static void tcon(pmix_server_trkr_t *t) } static void tdes(pmix_server_trkr_t *t) { + if (NULL != t->id) { + free(t->id); + } PMIX_DESTRUCT_LOCK(&t->lock); if (NULL != t->pcs) { free(t->pcs); @@ -2177,10 +2620,19 @@ PMIX_CLASS_INSTANCE(pmix_server_trkr_t, static void cdcon(pmix_server_caddy_t *cd) { + memset(&cd->ev, 0, sizeof(pmix_event_t)); + cd->event_active = false; + cd->trk = NULL; cd->peer = NULL; } static void cddes(pmix_server_caddy_t *cd) { + if (cd->event_active) { + pmix_event_del(&cd->ev); + } + if (NULL != cd->trk) { + PMIX_RELEASE(cd->trk); + } if (NULL != cd->peer) { PMIX_RELEASE(cd->peer); } @@ -2192,6 +2644,7 @@ PMIX_CLASS_INSTANCE(pmix_server_caddy_t, static void scadcon(pmix_setup_caddy_t *p) { + p->peer = NULL; memset(&p->proc, 0, sizeof(pmix_proc_t)); PMIX_CONSTRUCT_LOCK(&p->lock); p->nspace = NULL; @@ -2199,11 +2652,15 @@ static void scadcon(pmix_setup_caddy_t *p) p->ncodes = 0; p->procs = NULL; p->nprocs = 0; + p->apps = NULL; + p->napps = 0; p->server_object = NULL; p->nlocalprocs = 0; p->info = NULL; p->ninfo = 0; p->keys = NULL; + p->bo = NULL; + p->nbo = 0; p->cbfunc = NULL; p->opcbfunc = NULL; p->setupcbfunc = NULL; @@ -2213,6 +2670,16 @@ static void scadcon(pmix_setup_caddy_t *p) } static void scaddes(pmix_setup_caddy_t *p) { + if (NULL != p->peer) { + PMIX_RELEASE(p->peer); + } + PMIX_PROC_FREE(p->procs, p->nprocs); + if (NULL != p->apps) { + PMIX_APP_FREE(p->apps, p->napps); + } + if (NULL != p->bo) { + PMIX_BYTE_OBJECT_FREE(p->bo, p->nbo); + } PMIX_DESTRUCT_LOCK(&p->lock); } PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_setup_caddy_t, @@ -2222,11 +2689,25 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_setup_caddy_t, static void ncon(pmix_notify_caddy_t *p) { PMIX_CONSTRUCT_LOCK(&p->lock); +#if defined(__linux__) && OPAL_HAVE_CLOCK_GETTIME + struct timespec tp; + (void) clock_gettime(CLOCK_MONOTONIC, &tp); + p->ts = tp.tv_sec; +#else + /* Fall back to gettimeofday() if we have nothing else */ + struct timeval tv; + gettimeofday(&tv, NULL); + p->ts = tv.tv_sec; +#endif + p->room = -1; memset(p->source.nspace, 0, PMIX_MAX_NSLEN+1); p->source.rank = PMIX_RANK_UNDEF; p->range = PMIX_RANGE_UNDEF; p->targets = NULL; p->ntargets = 0; + p->nleft = SIZE_MAX; + p->affected = NULL; + p->naffected = 0; p->nondefault = false; p->info = NULL; p->ninfo = 0; @@ -2237,6 +2718,7 @@ static void ndes(pmix_notify_caddy_t *p) if (NULL != p->info) { PMIX_INFO_FREE(p->info, p->ninfo); } + PMIX_PROC_FREE(p->affected, p->naffected); if (NULL != p->targets) { free(p->targets); } @@ -2264,9 +2746,24 @@ PMIX_CLASS_INSTANCE(pmix_dmdx_remote_t, pmix_list_item_t, dmcon, dmdes); +static void dmrqcon(pmix_dmdx_request_t *p) +{ + memset(&p->ev, 0, sizeof(pmix_event_t)); + p->event_active = false; + p->lcd = NULL; +} +static void dmrqdes(pmix_dmdx_request_t *p) +{ + if (p->event_active) { + pmix_event_del(&p->ev); + } + if (NULL != p->lcd) { + PMIX_RELEASE(p->lcd); + } +} PMIX_CLASS_INSTANCE(pmix_dmdx_request_t, pmix_list_item_t, - NULL, NULL); + dmrqcon, dmrqdes); static void lmcon(pmix_dmdx_local_t *p) { @@ -2289,12 +2786,17 @@ PMIX_CLASS_INSTANCE(pmix_dmdx_local_t, static void prevcon(pmix_peer_events_info_t *p) { p->peer = NULL; + p->affected = NULL; + p->naffected = 0; } static void prevdes(pmix_peer_events_info_t *p) { if (NULL != p->peer) { PMIX_RELEASE(p->peer); } + if (NULL != p->affected) { + PMIX_PROC_FREE(p->affected, p->naffected); + } } PMIX_CLASS_INSTANCE(pmix_peer_events_info_t, pmix_list_item_t, diff --git a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h index 7a8c380beaa..4710ba57ff3 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h @@ -1,24 +1,29 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. - * Copyright (c) 2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2016-2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ */ #ifndef PMIX_SERVER_OPS_H #define PMIX_SERVER_OPS_H +#include +#ifdef HAVE_SYS_TYPES_H +#include +#endif + #include #include "src/include/types.h" #include -#include +#include #include #include "src/threads/threads.h" #include "src/include/pmix_globals.h" @@ -35,6 +40,7 @@ typedef struct { pmix_object_t super; pmix_event_t ev; pmix_lock_t lock; + pmix_peer_t *peer; char *nspace; pmix_status_t status; pmix_status_t *codes; @@ -51,6 +57,13 @@ typedef struct { char **keys; pmix_app_t *apps; size_t napps; + pmix_byte_object_t *bo; + size_t nbo; + /* timestamp receipt of the notification so we + * can evict the oldest one if we get overwhelmed */ + time_t ts; + /* what room of the hotel they are in */ + int room; pmix_op_cbfunc_t opcbfunc; pmix_dmodex_response_fn_t cbfunc; pmix_setup_application_cbfunc_t setupcbfunc; @@ -60,19 +73,35 @@ typedef struct { } pmix_setup_caddy_t; PMIX_CLASS_DECLARATION(pmix_setup_caddy_t); +/* define a callback function returning inventory */ +typedef void (*pmix_inventory_cbfunc_t)(pmix_status_t status, + pmix_list_t *inventory, + void *cbdata); + +/* define an object for rolling up the inventory*/ +typedef struct { + pmix_object_t super; + pmix_lock_t lock; + pmix_event_t ev; + pmix_status_t status; + int requests; + int replies; + pmix_list_t payload; // list of pmix_kval_t containing the replies + pmix_info_t *info; + size_t ninfo; + pmix_inventory_cbfunc_t cbfunc; + pmix_info_cbfunc_t infocbfunc; + pmix_op_cbfunc_t opcbfunc; + void *cbdata; +} pmix_inventory_rollup_t; +PMIX_CLASS_DECLARATION(pmix_inventory_rollup_t); + typedef struct { pmix_list_item_t super; pmix_setup_caddy_t *cd; } pmix_dmdx_remote_t; PMIX_CLASS_DECLARATION(pmix_dmdx_remote_t); -typedef struct { - pmix_list_item_t super; - pmix_modex_cbfunc_t cbfunc; // cbfunc to be executed when data is available - void *cbdata; -} pmix_dmdx_request_t; -PMIX_CLASS_DECLARATION(pmix_dmdx_request_t); - typedef struct { pmix_list_item_t super; pmix_proc_t proc; // id of proc whose data is being requested @@ -83,17 +112,29 @@ typedef struct { } pmix_dmdx_local_t; PMIX_CLASS_DECLARATION(pmix_dmdx_local_t); +typedef struct { + pmix_list_item_t super; + pmix_event_t ev; + bool event_active; + pmix_dmdx_local_t *lcd; + pmix_modex_cbfunc_t cbfunc; // cbfunc to be executed when data is available + void *cbdata; +} pmix_dmdx_request_t; +PMIX_CLASS_DECLARATION(pmix_dmdx_request_t); + /* event/error registration book keeping */ typedef struct { pmix_list_item_t super; pmix_peer_t *peer; bool enviro_events; + pmix_proc_t *affected; + size_t naffected; } pmix_peer_events_info_t; PMIX_CLASS_DECLARATION(pmix_peer_events_info_t); typedef struct { pmix_list_item_t super; - pmix_list_t peers; // list of pmix_prevents_info_t + pmix_list_t peers; // list of pmix_peer_events_info_t int code; } pmix_regevents_info_t; PMIX_CLASS_DECLARATION(pmix_regevents_info_t); @@ -107,6 +148,30 @@ typedef struct { pmix_list_t gdata; // cache of data given to me for passing to all clients pmix_list_t events; // list of pmix_regevents_info_t registered events bool tool_connections_allowed; + char *tmpdir; // temporary directory for this server + char *system_tmpdir; // system tmpdir + // verbosity for server get operations + int get_output; + int get_verbose; + // verbosity for server connect operations + int connect_output; + int connect_verbose; + // verbosity for server fence operations + int fence_output; + int fence_verbose; + // verbosity for server pub operations + int pub_output; + int pub_verbose; + // verbosity for server spawn operations + int spawn_output; + int spawn_verbose; + // verbosity for server event operations + int event_output; + int event_verbose; + // verbosity for basic server functions + int base_output; + int base_verbose; + } pmix_server_globals_t; #define PMIX_GDS_CADDY(c, p, t) \ @@ -135,8 +200,8 @@ typedef struct { bool pmix_server_trk_update(pmix_server_trkr_t *trk); -void pmix_pending_nspace_requests(pmix_nspace_t *nptr); -pmix_status_t pmix_pending_resolve(pmix_nspace_t *nptr, pmix_rank_t rank, +void pmix_pending_nspace_requests(pmix_namespace_t *nptr); +pmix_status_t pmix_pending_resolve(pmix_namespace_t *nptr, pmix_rank_t rank, pmix_status_t status, pmix_dmdx_local_t *lcd); @@ -175,9 +240,13 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, void *cbdata); pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, - pmix_buffer_t *buf, bool disconnect, + pmix_buffer_t *buf, pmix_op_cbfunc_t cbfunc); +pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, + pmix_buffer_t *buf, + pmix_op_cbfunc_t cbfunc); + pmix_status_t pmix_server_notify_error(pmix_status_t status, pmix_proc_t procs[], size_t nprocs, pmix_proc_t error_procs[], size_t error_nprocs, @@ -223,7 +292,17 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, void *cbdata); void pmix_server_execute_collective(int sd, short args, void *cbdata); +pmix_status_t pmix_server_initialize(void); + +void pmix_server_message_handler(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, + pmix_buffer_t *buf, void *cbdata); + +void pmix_server_purge_events(pmix_peer_t *peer, + pmix_proc_t *proc); + PMIX_EXPORT extern pmix_server_module_t pmix_host_server; PMIX_EXPORT extern pmix_server_globals_t pmix_server_globals; + #endif // PMIX_SERVER_OPS_H diff --git a/opal/mca/pmix/pmix2x/pmix/src/threads/mutex_unix.h b/opal/mca/pmix/pmix2x/pmix/src/threads/mutex_unix.h index f61d549923e..b7d4587a498 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/threads/mutex_unix.h +++ b/opal/mca/pmix/pmix2x/pmix/src/threads/mutex_unix.h @@ -14,7 +14,7 @@ * reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -77,14 +77,14 @@ PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_recursive_mutex_t); .m_lock_debug = 0, \ .m_lock_file = NULL, \ .m_lock_line = 0, \ - .m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_LOCK_UNLOCKED } },\ + .m_lock_atomic = {PMIX_ATOMIC_LOCK_UNLOCKED}, \ } #else #define PMIX_MUTEX_STATIC_INIT \ { \ .super = PMIX_OBJ_STATIC_INIT(pmix_mutex_t), \ .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER, \ - .m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_LOCK_UNLOCKED } },\ + .m_lock_atomic = {PMIX_ATOMIC_LOCK_UNLOCKED}, \ } #endif @@ -98,14 +98,14 @@ PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_recursive_mutex_t); .m_lock_debug = 0, \ .m_lock_file = NULL, \ .m_lock_line = 0, \ - .m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_LOCK_UNLOCKED } },\ + .m_lock_atomic = {PMIX_ATOMIC_LOCK_UNLOCKED}, \ } #else #define PMIX_RECURSIVE_MUTEX_STATIC_INIT \ { \ .super = PMIX_OBJ_STATIC_INIT(pmix_mutex_t), \ .m_lock_pthread = PMIX_PTHREAD_RECURSIVE_MUTEX_INITIALIZER, \ - .m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_LOCK_UNLOCKED } },\ + .m_lock_atomic = {PMIX_ATOMIC_LOCK_UNLOCKED}, \ } #endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/threads/thread_usage.h b/opal/mca/pmix/pmix2x/pmix/src/threads/thread_usage.h index 59825645c90..31bf8beee7b 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/threads/thread_usage.h +++ b/opal/mca/pmix/pmix2x/pmix/src/threads/thread_usage.h @@ -15,7 +15,7 @@ * and Technology (RIST). All rights reserved. * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -33,31 +33,30 @@ /** - * Use an atomic operation for increment/decrement if pmix_using_threads() - * indicates that threads are in use by the application or library. + * Use an atomic operation for increment/decrement */ #define PMIX_THREAD_DEFINE_ATOMIC_OP(type, name, operator, suffix) \ -static inline type pmix_thread_ ## name ## _fetch_ ## suffix (volatile type *addr, type delta) \ +static inline type pmix_thread_ ## name ## _fetch_ ## suffix (pmix_atomic_ ## type *addr, type delta) \ { \ return pmix_atomic_ ## name ## _fetch_ ## suffix (addr, delta); \ } \ \ -static inline type pmix_thread_fetch_ ## name ## _ ## suffix (volatile type *addr, type delta) \ +static inline type pmix_thread_fetch_ ## name ## _ ## suffix (pmix_atomic_ ## type *addr, type delta) \ { \ return pmix_atomic_fetch_ ## name ## _ ## suffix (addr, delta); \ } #define PMIX_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(type, addr_type, suffix) \ -static inline bool pmix_thread_compare_exchange_strong_ ## suffix (volatile addr_type *addr, type *compare, type value) \ +static inline bool pmix_thread_compare_exchange_strong_ ## suffix (pmix_atomic_ ## addr_type *addr, type *compare, type value) \ { \ - return pmix_atomic_compare_exchange_strong_ ## suffix ((volatile type *) addr, compare, value); \ + return pmix_atomic_compare_exchange_strong_ ## suffix (addr, (addr_type *) compare, (addr_type) value); \ } #define PMIX_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix) \ -static inline type pmix_thread_swap_ ## suffix (volatile addr_type *ptr, type newvalue) \ +static inline type pmix_thread_swap_ ## suffix (pmix_atomic_ ## addr_type *ptr, type newvalue) \ { \ - return pmix_atomic_swap_ ## suffix ((volatile type *) ptr, newvalue); \ + return (type) pmix_atomic_swap_ ## suffix (ptr, (addr_type) newvalue); \ } PMIX_THREAD_DEFINE_ATOMIC_OP(int32_t, add, +, 32) @@ -112,13 +111,13 @@ PMIX_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr) #define PMIX_THREAD_COMPARE_EXCHANGE_STRONG_32 pmix_thread_compare_exchange_strong_32 #define PMIX_ATOMIC_COMPARE_EXCHANGE_STRONG_32 pmix_thread_compare_exchange_strong_32 -#define PMIX_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z) pmix_thread_compare_exchange_strong_ptr ((volatile intptr_t *) x, (void *) y, (void *) z) +#define PMIX_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z) pmix_thread_compare_exchange_strong_ptr ((pmix_atomic_intptr_t *) x, (intptr_t *) y, (intptr_t) z) #define PMIX_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR PMIX_THREAD_COMPARE_EXCHANGE_STRONG_PTR #define PMIX_THREAD_SWAP_32 pmix_thread_swap_32 #define PMIX_ATOMIC_SWAP_32 pmix_thread_swap_32 -#define PMIX_THREAD_SWAP_PTR(x, y) pmix_thread_swap_ptr ((volatile intptr_t *) x, (void *) y) +#define PMIX_THREAD_SWAP_PTR(x, y) pmix_thread_swap_ptr ((pmix_atomic_intptr_t *) x, (intptr_t) y) #define PMIX_ATOMIC_SWAP_PTR PMIX_THREAD_SWAP_PTR /* define 64-bit macros is 64-bit atomic math is available */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/threads/threads.h b/opal/mca/pmix/pmix2x/pmix/src/threads/threads.h index d66e594ead6..df04c95d3e0 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/threads/threads.h +++ b/opal/mca/pmix/pmix2x/pmix/src/threads/threads.h @@ -13,7 +13,7 @@ * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2015-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -65,6 +65,7 @@ typedef pthread_cond_t pmix_condition_t; #define PMIX_CONDITION_STATIC_INIT PTHREAD_COND_INITIALIZER typedef struct { + pmix_status_t status; pmix_mutex_t mutex; pmix_condition_t cond; volatile bool active; diff --git a/opal/mca/pmix/pmix2x/pmix/src/threads/tsd.h b/opal/mca/pmix/pmix2x/pmix/src/threads/tsd.h index 589027217ed..89252f37de5 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/threads/tsd.h +++ b/opal/mca/pmix/pmix2x/pmix/src/threads/tsd.h @@ -4,7 +4,7 @@ * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -38,74 +38,6 @@ BEGIN_C_DECLS */ typedef void (*pmix_tsd_destructor_t)(void *value); -#if defined(DOXYGEN) - -/** - * Typedef for thread-specific data key - */ -typedef void* pmix_tsd_key_t; - - -/** - * Delete a thread-specific data key - * - * Delete a thread-specific data key previously returned by - * pmix_tsd_key_create(). The destructor associated with the key is - * not fired in any thread and memory cleanup is the responsibility of - * the caller. - * - * @note Unlike pthread_key_delete, this function should not be called - * from within a destructor. It can not be universally supported at - * this time. - * - * @param key[in] The key for accessing thread-specific data - * - * @retval PMIX_SUCCESS Success - * @retval EINVAL Invalid key - */ -PMIX_EXPORT int pmix_tsd_key_delete(pmix_tsd_key_t key); - - -/** - * Set a thread-specific data value - * - * Associates value with key in the current thread. The value for the - * key in other threads is not changed. Different threads may assign - * different values to the same key. - * - * @note This function should not be called within - * pmix_tsd_key_delete(). - * - * @param key[in] Thread specific data key to modify - * @param value[in] Value to associate with key - * - * @retval PMIX_SUCCESS Success - * @retval ENOMEM Insufficient memory exists to associate the - * value with the key - * @retval EINVAL Invalid key - */ -PMIX_EXPORT int pmix_tsd_setspecific(pmix_tsd_key_t key, void *value); - - -/** - * Get a thread-specific data value - * - * Get the data associated with the given key, as set by - * pmix_tsd_setspecific(). If pmix_tsd_setspecific() hasn't been - * called in the current thread with the given key, NULL is returned - * in valuep. - * - * @param key[in] Thread specific data key to modify - * @param value[out] Value to associate with key - * - * @retval PMIX_SUCCESS Success - * @retval ENOMEM Insufficient memory exists to associate the - * value with the key - * @retval EINVAL Invalid key - */ -PMIX_EXPORT int pmix_tsd_getspecific(pmix_tsd_key_t key, void **valuep); - -#else typedef pthread_key_t pmix_tsd_key_t; @@ -128,8 +60,6 @@ pmix_tsd_getspecific(pmix_tsd_key_t key, void **valuep) return PMIX_SUCCESS; } -#endif - /** * Create thread-specific data key * diff --git a/opal/mca/pmix/pmix2x/pmix/src/threads/wait_sync.h b/opal/mca/pmix/pmix2x/pmix/src/threads/wait_sync.h index 311ecbfe7f1..e6a11cb0c2c 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/threads/wait_sync.h +++ b/opal/mca/pmix/pmix2x/pmix/src/threads/wait_sync.h @@ -8,7 +8,7 @@ * Copyright (c) 2016 Mellanox Technologies. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -28,7 +28,7 @@ BEGIN_C_DECLS typedef struct pmix_wait_sync_t { - int32_t count; + pmix_atomic_int32_t count; int32_t status; pthread_cond_t condition; pthread_mutex_t lock; diff --git a/opal/mca/pmix/pmix2x/pmix/src/tool/pmix_tool.c b/opal/mca/pmix/pmix2x/pmix/src/tool/pmix_tool.c index 9afbebb09d0..7da60671c57 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/tool/pmix_tool.c +++ b/opal/mca/pmix/pmix2x/pmix/src/tool/pmix_tool.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . @@ -52,23 +52,26 @@ #include PMIX_EVENT_HEADER #include PMIX_EVENT2_THREAD_HEADER -extern pmix_client_globals_t pmix_client_globals; - #include "src/class/pmix_list.h" #include "src/util/argv.h" #include "src/util/error.h" #include "src/util/hash.h" #include "src/util/output.h" +#include "src/util/pmix_environ.h" +#include "src/util/show_help.h" #include "src/runtime/pmix_progress_threads.h" #include "src/runtime/pmix_rte.h" #include "src/mca/bfrops/base/base.h" #include "src/mca/gds/base/base.h" -#include "src/mca/ptl/ptl.h" +#include "src/mca/ptl/base/base.h" #include "src/mca/psec/psec.h" #include "src/include/pmix_globals.h" +#include "src/server/pmix_server_ops.h" #define PMIX_MAX_RETRIES 10 +extern pmix_client_globals_t pmix_client_globals; + static void _notify_complete(pmix_status_t status, void *cbdata) { pmix_event_chain_t *chain = (pmix_event_chain_t*)cbdata; @@ -85,7 +88,7 @@ static void pmix_tool_notify_recv(struct pmix_peer_t *peer, pmix_event_chain_t *chain; size_t ninfo; - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix:tool_notify_recv - processing event"); /* a zero-byte buffer indicates that this recv is being @@ -147,6 +150,7 @@ static void pmix_tool_notify_recv(struct pmix_peer_t *peer, } if (0 < ninfo) { + chain->ninfo = ninfo; cnt = ninfo; PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver, buf, chain->info, &cnt, PMIX_INFO); @@ -159,22 +163,52 @@ static void pmix_tool_notify_recv(struct pmix_peer_t *peer, /* prep the chain for processing */ pmix_prep_event_chain(chain, chain->info, ninfo, false); - pmix_output_verbose(2, pmix_globals.debug_output, - "[%s:%d] pmix:tool_notify_recv - processing event %d, calling errhandler", - pmix_globals.myid.nspace, pmix_globals.myid.rank, chain->status); + pmix_output_verbose(2, pmix_client_globals.event_output, + "[%s:%d] pmix:tool_notify_recv - processing event %s from source %s:%d, calling errhandler", + pmix_globals.myid.nspace, pmix_globals.myid.rank, PMIx_Error_string(chain->status), + chain->source.nspace, chain->source.rank); pmix_invoke_local_event_hdlr(chain); return; - error: + error: /* we always need to return */ - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.event_output, "pmix:tool_notify_recv - unpack error status =%d, calling def errhandler", rc); chain = PMIX_NEW(pmix_event_chain_t); chain->status = rc; pmix_invoke_local_event_hdlr(chain); } +/* callback to receive job info */ +static void job_data(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, + pmix_buffer_t *buf, void *cbdata) +{ + pmix_status_t rc; + char *nspace; + int32_t cnt = 1; + pmix_cb_t *cb = (pmix_cb_t*)cbdata; + + /* unpack the nspace - should be same as our own */ + PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver, + buf, &nspace, &cnt, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + cb->status = PMIX_ERROR; + PMIX_POST_OBJECT(cb); + PMIX_WAKEUP_THREAD(&cb->lock); + return; + } + + /* decode it */ + PMIX_GDS_STORE_JOB_INFO(cb->status, + pmix_client_globals.myserver, + nspace, buf); + cb->status = PMIX_SUCCESS; + PMIX_POST_OBJECT(cb); + PMIX_WAKEUP_THREAD(&cb->lock); +} PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, pmix_info_t info[], size_t ninfo) @@ -182,11 +216,20 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, pmix_kval_t *kptr; pmix_status_t rc; char hostname[PMIX_MAX_NSLEN]; - bool found, do_not_connect = false; + char *evar, *nspace = NULL; + pmix_rank_t rank = PMIX_RANK_UNDEF; + bool gdsfound, do_not_connect = false; bool nspace_given = false; + bool nspace_in_enviro = false; bool rank_given = false; pmix_info_t ginfo; size_t n; + pmix_ptl_posted_recv_t *rcv; + pmix_proc_t wildcard; + pmix_proc_type_t ptype; + pmix_cb_t cb; + pmix_buffer_t *req; + pmix_cmd_t cmd = PMIX_REQ_CMD; PMIX_ACQUIRE_THREAD(&pmix_global_lock); @@ -198,8 +241,8 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, /* since we have been called before, the nspace and * rank should be known. So return them here if * requested */ - if (NULL != proc) { - (void)strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + if (NULL != proc) { + pmix_strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); proc->rank = pmix_globals.myid.rank; } ++pmix_globals.init_cntr; @@ -207,85 +250,244 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, return PMIX_SUCCESS; } - /* if we were given an nspace in the environment, then we - * must have been spawned by a PMIx server - so even though - * we technically will operate as a tool, we are actually - * a "client" of the PMIx server and should connect that way */ - if (NULL != getenv("PMIX_NAMESPACE")) { + /* parse the input directives */ + gdsfound = false; + ptype = PMIX_PROC_TOOL; + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_GDS_MODULE, PMIX_MAX_KEYLEN)) { + PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, info[n].value.data.string, PMIX_STRING); + gdsfound = true; + } else if (0 == strncmp(info[n].key, PMIX_TOOL_DO_NOT_CONNECT, PMIX_MAX_KEYLEN)) { + do_not_connect = PMIX_INFO_TRUE(&info[n]); + } else if (0 == strncmp(info[n].key, PMIX_TOOL_NSPACE, PMIX_MAX_KEYLEN)) { + if (NULL != nspace) { + /* cannot define it twice */ + free(nspace); + if (gdsfound) { + PMIX_INFO_DESTRUCT(&ginfo); + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_BAD_PARAM; + } + nspace = strdup(info[n].value.data.string); + nspace_given = true; + } else if (0 == strncmp(info[n].key, PMIX_TOOL_RANK, PMIX_MAX_KEYLEN)) { + rank = info[n].value.data.rank; + rank_given = true; + } else if (0 == strncmp(info[n].key, PMIX_SERVER_TMPDIR, PMIX_MAX_KEYLEN)) { + pmix_server_globals.tmpdir = strdup(info[n].value.data.string); + } else if (0 == strncmp(info[n].key, PMIX_SYSTEM_TMPDIR, PMIX_MAX_KEYLEN)) { + pmix_server_globals.system_tmpdir = strdup(info[n].value.data.string); + } + } + } + if (NULL == pmix_server_globals.tmpdir) { + if (NULL == (evar = getenv("PMIX_SERVER_TMPDIR"))) { + pmix_server_globals.tmpdir = strdup(pmix_tmp_directory()); + } else { + pmix_server_globals.tmpdir = strdup(evar); + } + } + if (NULL == pmix_server_globals.system_tmpdir) { + if (NULL == (evar = getenv("PMIX_SYSTEM_TMPDIR"))) { + pmix_server_globals.system_tmpdir = strdup(pmix_tmp_directory()); + } else { + pmix_server_globals.system_tmpdir = strdup(evar); + } + } + + if ((nspace_given && !rank_given) || + (!nspace_given && rank_given)) { + /* can't have one and not the other */ + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + if (NULL != nspace) { + free(nspace); + } + if (gdsfound) { + PMIX_INFO_DESTRUCT(&ginfo); + } PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIx_Init(proc, info, ninfo); + return PMIX_ERR_BAD_PARAM; + } + + /* if we were not passed an nspace in the info keys, + * check to see if we were given one in the env - this + * will be the case when we are launched by a PMIx-enabled + * daemon */ + if (!nspace_given) { + if (NULL != (evar = getenv("PMIX_NAMESPACE"))) { + nspace = strdup(evar); + nspace_in_enviro = true; + } + } + /* also look for the rank - it normally is zero, but if we + * were launched, then it might have been as part of a + * multi-process tool */ + if (!rank_given) { + if (NULL != (evar = getenv("PMIX_RANK"))) { + rank = strtol(evar, NULL, 10); + if (!nspace_in_enviro) { + /* this is an error - we can't have one and not + * the other */ + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_BAD_PARAM; + } + /* flag that this tool is also a client */ + ptype |= PMIX_PROC_CLIENT_TOOL; + } else if (nspace_in_enviro) { + /* this is an error - we can't have one and not + * the other */ + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + if (NULL != nspace) { + free(nspace); + } + if (gdsfound) { + PMIX_INFO_DESTRUCT(&ginfo); + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_BAD_PARAM; + } + } + + /* if we are a launcher, then we also need to act as a server, + * so setup the server-related structures here */ + if (PMIX_PROC_LAUNCHER_ACT & ptype) { + if (PMIX_SUCCESS != (rc = pmix_server_initialize())) { + PMIX_ERROR_LOG(rc); + if (NULL != nspace) { + free(nspace); + } + if (gdsfound) { + PMIX_INFO_DESTRUCT(&ginfo); + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + /* setup the function pointers */ + memset(&pmix_host_server, 0, sizeof(pmix_server_module_t)); } /* setup the runtime - this init's the globals, * opens and initializes the required frameworks */ - if (PMIX_SUCCESS != (rc = pmix_rte_init(PMIX_PROC_TOOL, info, ninfo, + if (PMIX_SUCCESS != (rc = pmix_rte_init(ptype, info, ninfo, pmix_tool_notify_recv))) { PMIX_ERROR_LOG(rc); + if (NULL != nspace) { + free(nspace); + } + if (gdsfound) { + PMIX_INFO_DESTRUCT(&ginfo); + } PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } + /* if we were given a name, then set it now */ + if (nspace_given || nspace_in_enviro) { + pmix_strncpy(pmix_globals.myid.nspace, nspace, PMIX_MAX_NSLEN); + free(nspace); + pmix_globals.myid.rank = rank; + } + /* setup the globals */ PMIX_CONSTRUCT(&pmix_client_globals.pending_requests, pmix_list_t); PMIX_CONSTRUCT(&pmix_client_globals.peers, pmix_pointer_array_t); pmix_pointer_array_init(&pmix_client_globals.peers, 1, INT_MAX, 1); pmix_client_globals.myserver = PMIX_NEW(pmix_peer_t); - pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); + if (NULL == pmix_client_globals.myserver) { + if (gdsfound) { + PMIX_INFO_DESTRUCT(&ginfo); + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_NOMEM; + } + pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t); + if (NULL == pmix_client_globals.myserver->nptr) { + PMIX_RELEASE(pmix_client_globals.myserver); + if (gdsfound) { + PMIX_INFO_DESTRUCT(&ginfo); + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_NOMEM; + } + pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); + if (NULL == pmix_client_globals.myserver->info) { + PMIX_RELEASE(pmix_client_globals.myserver); + if (gdsfound) { + PMIX_INFO_DESTRUCT(&ginfo); + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_NOMEM; + } pmix_output_verbose(2, pmix_globals.debug_output, "pmix: init called"); - /* select our bfrops compat module */ - pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(NULL); - if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) { - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_INIT; + if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + /* if we are a client, then we need to pickup the + * rest of the envar-based server assignments */ + pmix_globals.pindex = -1; + /* setup a rank_info object for us */ + pmix_globals.mypeer->info = PMIX_NEW(pmix_rank_info_t); + if (NULL == pmix_globals.mypeer->info) { + if (gdsfound) { + PMIX_INFO_DESTRUCT(&ginfo); + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_NOMEM; + } + pmix_globals.mypeer->info->pname.nspace = strdup(pmix_globals.myid.nspace); + pmix_globals.mypeer->info->pname.rank = pmix_globals.myid.rank; + /* our bfrops module will be set when we connect to the server */ + } else { + /* select our bfrops compat module */ + pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(NULL); + if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) { + if (gdsfound) { + PMIX_INFO_DESTRUCT(&ginfo); + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_INIT; + } + /* the server will be using the same */ + pmix_client_globals.myserver->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops; } - /* the server will be using the same */ - pmix_client_globals.myserver->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops; - - /* set the buffer type */ - pmix_globals.mypeer->nptr->compat.type = pmix_bfrops_globals.default_type; - /* the server will be using the same */ - pmix_client_globals.myserver->nptr->compat.type = pmix_globals.mypeer->nptr->compat.type; - /* select our psec compat module */ - pmix_globals.mypeer->nptr->compat.psec = pmix_psec_base_assign_module(NULL); + /* select our psec compat module - the selection may be based + * on the corresponding envars that should have been passed + * to us at launch */ + evar = getenv("PMIX_SECURITY_MODE"); + pmix_globals.mypeer->nptr->compat.psec = pmix_psec_base_assign_module(evar); if (NULL == pmix_globals.mypeer->nptr->compat.psec) { + if (gdsfound) { + PMIX_INFO_DESTRUCT(&ginfo); + } PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; } /* the server will be using the same */ pmix_client_globals.myserver->nptr->compat.psec = pmix_globals.mypeer->nptr->compat.psec; - /* select the gds compat module */ - pmix_client_globals.myserver->nptr->compat.gds = pmix_gds_base_assign_module(NULL, 0); - if (NULL == pmix_client_globals.myserver->nptr->compat.gds) { - PMIX_INFO_DESTRUCT(&ginfo); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_INIT; + /* set the buffer type - the selection will be based + * on the corresponding envars that should have been passed + * to us at launch */ + evar = getenv("PMIX_BFROP_BUFFER_TYPE"); + if (NULL == evar) { + /* just set to our default */ + pmix_globals.mypeer->nptr->compat.type = pmix_bfrops_globals.default_type; + } else if (0 == strcmp(evar, "PMIX_BFROP_BUFFER_FULLY_DESC")) { + pmix_globals.mypeer->nptr->compat.type = PMIX_BFROP_BUFFER_FULLY_DESC; + } else { + pmix_globals.mypeer->nptr->compat.type = PMIX_BFROP_BUFFER_NON_DESC; } + /* the server will be using the same */ + pmix_client_globals.myserver->nptr->compat.type = pmix_globals.mypeer->nptr->compat.type; - /* now select a GDS module for our own internal use - the user may + /* select a GDS module for our own internal use - the user may * have passed down a directive for this purpose. If they did, then * use it. Otherwise, we want the "hash" module */ - found = false; - if (NULL != info) { - for (n=0; n < ninfo; n++) { - if (0 == strncmp(info[n].key, PMIX_GDS_MODULE, PMIX_MAX_KEYLEN)) { - PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, info[n].value.data.string, PMIX_STRING); - found = true; - } else if (0 == strncmp(info[n].key, PMIX_TOOL_DO_NOT_CONNECT, PMIX_MAX_KEYLEN)) { - do_not_connect = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strncmp(info[n].key, PMIX_TOOL_NSPACE, PMIX_MAX_KEYLEN)) { - (void)strncpy(pmix_globals.myid.nspace, info[n].value.data.string, PMIX_MAX_NSLEN); - nspace_given = true; - } else if (0 == strncmp(info[n].key, PMIX_TOOL_RANK, PMIX_MAX_KEYLEN)) { - pmix_globals.myid.rank = info[n].value.data.rank; - rank_given = true; - } - } - } - if (!found) { + if (!gdsfound) { PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, "hash", PMIX_STRING); } pmix_globals.mypeer->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1); @@ -295,6 +497,22 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, return PMIX_ERR_INIT; } PMIX_INFO_DESTRUCT(&ginfo); + /* select the gds compat module we will use to interact with + * our server- the selection will be based + * on the corresponding envars that should have been passed + * to us at launch */ + evar = getenv("PMIX_GDS_MODULE"); + if (NULL != evar) { + PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, evar, PMIX_STRING); + pmix_client_globals.myserver->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1); + PMIX_INFO_DESTRUCT(&ginfo); + } else { + pmix_client_globals.myserver->nptr->compat.gds = pmix_gds_base_assign_module(NULL, 0); + } + if (NULL == pmix_client_globals.myserver->nptr->compat.gds) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_INIT; + } if (do_not_connect) { /* ensure we mark that we are not connected */ @@ -305,19 +523,23 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, return PMIX_ERR_INIT; } } else { - /* connect to the server - returns job info if successful */ + /* connect to the server */ rc = pmix_ptl_base_connect_to_peer((struct pmix_peer_t*)pmix_client_globals.myserver, info, ninfo); if (PMIX_SUCCESS != rc){ PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } } - /* Success, so copy the nspace and rank */ - (void)strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); - proc->rank = pmix_globals.myid.rank; + if (!nspace_given) { + /* Success, so copy the nspace and rank to the proc struct they gave us */ + pmix_strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + } + if (!rank_given) { + proc->rank = pmix_globals.myid.rank; + } /* and into our own peer object */ if (NULL == pmix_globals.mypeer->nptr->nspace) { - pmix_globals.mypeer->nptr->nspace = strdup(proc->nspace); + pmix_globals.mypeer->nptr->nspace = strdup(pmix_globals.myid.nspace); } /* setup a rank_info object for us */ pmix_globals.mypeer->info = PMIX_NEW(pmix_rank_info_t); @@ -325,320 +547,372 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOMEM; } - pmix_globals.mypeer->info->pname.nspace = strdup(proc->nspace); - pmix_globals.mypeer->info->pname.rank = proc->rank; + pmix_globals.mypeer->info->pname.nspace = strdup(pmix_globals.myid.nspace); + pmix_globals.mypeer->info->pname.rank = pmix_globals.myid.rank; + + /* if we are acting as a server, then start listening */ + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + /* setup the wildcard recv for inbound messages from clients */ + rcv = PMIX_NEW(pmix_ptl_posted_recv_t); + rcv->tag = UINT32_MAX; + rcv->cbfunc = pmix_server_message_handler; + /* add it to the end of the list of recvs */ + pmix_list_append(&pmix_ptl_globals.posted_recvs, &rcv->super); + } /* increment our init reference counter */ pmix_globals.init_cntr++; - - /* now finish the initialization by filling our local - * datastore with typical job-related info. No point - * in having the server generate these as we are - * obviously a singleton, and so the values are well-known */ - - /* the jobid is just our nspace */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_JOBID); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_STRING; - kptr->value->data.string = strdup(pmix_globals.myid.nspace); - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* our rank */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_RANK); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_INT; - kptr->value->data.integer = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* nproc offset */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_NPROC_OFFSET); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* node size */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_NODE_SIZE); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 1; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* local peers */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_LOCAL_PEERS); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_STRING; - kptr->value->data.string = strdup("0"); - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* local leader */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_LOCALLDR); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* universe size */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_UNIV_SIZE); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 1; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* job size - we are our very own job, so we have no peers */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_JOB_SIZE); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 1; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* local size - only us in our job */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_LOCAL_SIZE); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 1; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* max procs - since we are a self-started tool, there is no - * allocation within which we can grow ourselves */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_MAX_PROCS); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 1; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* app number */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_APPNUM); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* app leader */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_APPLDR); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* app rank */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_APP_RANK); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* global rank */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_GLOBAL_RANK); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* local rank - we are alone in our job */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_LOCAL_RANK); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_UINT32; - kptr->value->data.uint32 = 0; - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* we cannot know the node rank as we don't know what - * other processes are executing on this node - so - * we'll add that info to the server-tool handshake - * and load it from there */ - - /* hostname */ - gethostname(hostname, PMIX_MAX_NSLEN); - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_HOSTNAME); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_STRING; - kptr->value->data.string = strdup(hostname); - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - PMIX_RELEASE(kptr); // maintain accounting - - /* we cannot know the RM's nodeid for this host, so - * we'll add that info to the server-tool handshake - * and load it from there */ - - /* the nodemap is simply our hostname as there is no - * regex to generate */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_NODE_MAP); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_STRING; - kptr->value->data.string = strdup(hostname); - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; + /* if we are acting as a client, then send a request for our + * job info - we do this as a non-blocking + * transaction because some systems cannot handle very large + * blocking operations and error out if we try them. */ + if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + req = PMIX_NEW(pmix_buffer_t); + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + req, &cmd, 1, PMIX_COMMAND); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(req); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + /* send to the server */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, + req, job_data, (void*)&cb); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + /* wait for the data to return */ + PMIX_WAIT_THREAD(&cb.lock); + rc = cb.status; + PMIX_DESTRUCT(&cb); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + } else { + /* now finish the initialization by filling our local + * datastore with typical job-related info. No point + * in having the server generate these as we are + * obviously a singleton, and so the values are well-known */ + pmix_strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + wildcard.rank = pmix_globals.myid.rank; + + /* the jobid is just our nspace */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_JOBID); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_STRING; + kptr->value->data.string = strdup(pmix_globals.myid.nspace); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* our rank */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_RANK); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_INT; + kptr->value->data.integer = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* nproc offset */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_NPROC_OFFSET); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* node size */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_NODE_SIZE); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 1; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* local peers */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_LOCAL_PEERS); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_STRING; + kptr->value->data.string = strdup("0"); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* local leader */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_LOCALLDR); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* universe size */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_UNIV_SIZE); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 1; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* job size - we are our very own job, so we have no peers */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_JOB_SIZE); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 1; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* local size - only us in our job */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_LOCAL_SIZE); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 1; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* max procs - since we are a self-started tool, there is no + * allocation within which we can grow ourselves */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_MAX_PROCS); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 1; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* app number */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_APPNUM); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* app leader */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_APPLDR); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* app rank */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_APP_RANK); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* global rank */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_GLOBAL_RANK); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT32; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* local rank - we are alone in our job */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_LOCAL_RANK); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_UINT16; + kptr->value->data.uint32 = 0; + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* we cannot know the node rank as we don't know what + * other processes are executing on this node - so + * we'll add that info to the server-tool handshake + * and load it from there */ + + /* hostname */ + gethostname(hostname, PMIX_MAX_NSLEN); + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_HOSTNAME); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_STRING; + kptr->value->data.string = strdup(hostname); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* we cannot know the RM's nodeid for this host, so + * we'll add that info to the server-tool handshake + * and load it from there */ + + /* the nodemap is simply our hostname as there is no + * regex to generate */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_NODE_MAP); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_STRING; + kptr->value->data.string = strdup(hostname); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting + + /* likewise, the proc map is just our rank as we are + * the only proc in this job */ + kptr = PMIX_NEW(pmix_kval_t); + kptr->key = strdup(PMIX_PROC_MAP); + PMIX_VALUE_CREATE(kptr->value, 1); + kptr->value->type = PMIX_STRING; + kptr->value->data.string = strdup("0"); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &wildcard, + PMIX_INTERNAL, kptr); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + PMIX_RELEASE(kptr); // maintain accounting } - PMIX_RELEASE(kptr); // maintain accounting - - /* likewise, the proc map is just our rank as we are - * the only proc in this job */ - kptr = PMIX_NEW(pmix_kval_t); - kptr->key = strdup(PMIX_PROC_MAP); - PMIX_VALUE_CREATE(kptr->value, 1); - kptr->value->type = PMIX_STRING; - kptr->value->data.string = strdup("0"); - PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, - &pmix_globals.myid, - PMIX_INTERNAL, kptr); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; + PMIX_RELEASE_THREAD(&pmix_global_lock); + + /* if we are acting as a server, then start listening */ + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + /* start listening for connections */ + if (PMIX_SUCCESS != pmix_ptl_base_start_listening(info, ninfo)) { + pmix_show_help("help-pmix-server.txt", "listener-thread-start", true); + return PMIX_ERR_INIT; + } } - PMIX_RELEASE(kptr); // maintain accounting - PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } @@ -674,8 +948,8 @@ static void finwait_cbfunc(struct pmix_peer_t *pr, if (tev->active) { tev->active = false; pmix_event_del(&tev->ev); // stop the timer - PMIX_WAKEUP_THREAD(&tev->lock); } + PMIX_WAKEUP_THREAD(&tev->lock); } PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) @@ -684,7 +958,7 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) pmix_cmd_t cmd = PMIX_FINALIZE_CMD; pmix_status_t rc; pmix_tool_timeout_t tev; - struct timeval tv = {2, 0}; + struct timeval tv = {5, 0}; int n; pmix_peer_t *peer; @@ -695,57 +969,56 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) return PMIX_SUCCESS; } pmix_globals.init_cntr = 0; - /* if we are not connected, then we are done */ - if (!pmix_globals.connected) { - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_SUCCESS; - } + pmix_globals.mypeer->finalized = true; PMIX_RELEASE_THREAD(&pmix_global_lock); pmix_output_verbose(2, pmix_globals.debug_output, "pmix:tool finalize called"); - /* setup a cmd message to notify the PMIx - * server that we are normally terminating */ - msg = PMIX_NEW(pmix_buffer_t); - /* pack the cmd */ - PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, - msg, &cmd, 1, PMIX_COMMAND); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - return rc; - } + /* if we are connected, then disconnect */ + if (pmix_globals.connected) { + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:tool sending finalize sync to server"); + + /* setup a cmd message to notify the PMIx + * server that we are normally terminating */ + msg = PMIX_NEW(pmix_buffer_t); + /* pack the cmd */ + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + msg, &cmd, 1, PMIX_COMMAND); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + return rc; + } + /* setup a timer to protect ourselves should the server be unable + * to answer for some reason */ + PMIX_CONSTRUCT_LOCK(&tev.lock); + pmix_event_assign(&tev.ev, pmix_globals.evbase, -1, 0, + fin_timeout, &tev); + tev.active = true; + PMIX_POST_OBJECT(&tev); + pmix_event_add(&tev.ev, &tv); + PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg, + finwait_cbfunc, (void*)&tev); + if (PMIX_SUCCESS != rc) { + if (tev.active) { + pmix_event_del(&tev.ev); + } + return rc; + } + /* wait for the ack to return */ + PMIX_WAIT_THREAD(&tev.lock); + PMIX_DESTRUCT_LOCK(&tev.lock); - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:tool sending finalize sync to server"); - - /* setup a timer to protect ourselves should the server be unable - * to answer for some reason */ - PMIX_CONSTRUCT_LOCK(&tev.lock); - pmix_event_assign(&tev.ev, pmix_globals.evbase, -1, 0, - fin_timeout, &tev); - tev.active = true; - PMIX_POST_OBJECT(&tev); - pmix_event_add(&tev.ev, &tv); - PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg, - finwait_cbfunc, (void*)&tev); - if (PMIX_SUCCESS != rc) { if (tev.active) { pmix_event_del(&tev.ev); } - return rc; - } + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:tool finalize sync received"); - /* wait for the ack to return */ - PMIX_WAIT_THREAD(&tev.lock); - PMIX_DESTRUCT_LOCK(&tev.lock); - if (tev.active) { - pmix_event_del(&tev.ev); } - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:tool finalize sync received"); if (!pmix_globals.external_evbase) { /* stop the progress thread, but leave the event base @@ -755,7 +1028,6 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) (void)pmix_progress_thread_pause(NULL); } - PMIX_RELEASE(pmix_client_globals.myserver); PMIX_LIST_DESTRUCT(&pmix_client_globals.pending_requests); for (n=0; n < pmix_client_globals.peers.size; n++) { if (NULL != (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_client_globals.peers, n))) { @@ -765,6 +1037,12 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) /* shutdown services */ pmix_rte_finalize(); + if (NULL != pmix_globals.mypeer) { + PMIX_RELEASE(pmix_globals.mypeer); + } + + /* finalize the class/object system */ + pmix_class_finalize(); return PMIX_SUCCESS; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/Makefile.include b/opal/mca/pmix/pmix2x/pmix/src/util/Makefile.include index 1a4065dd325..9cd0070e7ad 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/Makefile.include +++ b/opal/mca/pmix/pmix2x/pmix/src/util/Makefile.include @@ -12,7 +12,7 @@ # All rights reserved. # Copyright (c) 2007-2016 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2013 NVIDIA Corporation. All rights reserved. -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # Copyright (c) 2016 Research Organization for Information Science # and Technology (RIST). All rights reserved. # $COPYRIGHT$ @@ -30,6 +30,7 @@ LEX_OUTPUT_ROOT = lex.pmix_show_help_yy headers += \ util/alfg.h \ util/argv.h \ + util/cmd_line.h \ util/error.h \ util/printf.h \ util/output.h \ @@ -46,15 +47,16 @@ headers += \ util/getid.h \ util/strnlen.h \ util/hash.h \ + util/name_fns.h \ util/net.h \ util/pif.h \ - util/tsd.h \ util/parse_options.h \ util/compress.h sources += \ util/alfg.c \ util/argv.c \ + util/cmd_line.c \ util/error.c \ util/printf.c \ util/output.c \ @@ -70,6 +72,7 @@ sources += \ util/path.c \ util/getid.c \ util/hash.c \ + util/name_fns.c \ util/net.c \ util/pif.c \ util/parse_options.c \ diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/argv.c b/opal/mca/pmix/pmix2x/pmix/src/util/argv.c index f5c08f80a0c..d90941ae0a9 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/argv.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/argv.c @@ -11,7 +11,7 @@ * All rights reserved. * Copyright (c) 2007 Voltaire. All rights reserved. * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved. - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -221,7 +221,7 @@ static char **pmix_argv_split_inter(const char *src_string, int delimiter, if (NULL == argtemp) return NULL; - strncpy(argtemp, src_string, arglen); + pmix_strncpy(argtemp, src_string, arglen); argtemp[arglen] = '\0'; if (PMIX_SUCCESS != pmix_argv_append(&argc, &argv, argtemp)) { @@ -235,7 +235,7 @@ static char **pmix_argv_split_inter(const char *src_string, int delimiter, /* short argument, copy to buffer and add */ else { - strncpy(arg, src_string, arglen); + pmix_strncpy(arg, src_string, arglen); arg[arglen] = '\0'; if (PMIX_SUCCESS != pmix_argv_append(&argc, &argv, arg)) diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/basename.c b/opal/mca/pmix/pmix2x/pmix/src/util/basename.c index 64e5c27e7e9..0de3c18882e 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/basename.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/basename.c @@ -12,7 +12,7 @@ * Copyright (c) 2009-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2014 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -123,11 +123,7 @@ char* pmix_dirname(const char* filename) } if( p != filename ) { char* ret = (char*)malloc( p - filename + 1 ); -#ifdef HAVE_STRNCPY_S - strncpy_s( ret, (p - filename + 1), filename, p - filename ); -#else - strncpy(ret, filename, p - filename); -#endif + pmix_strncpy(ret, filename, p - filename); ret[p - filename] = '\0'; return pmix_make_filename_os_friendly(ret); } diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/cmd_line.c b/opal/mca/pmix/pmix2x/pmix/src/util/cmd_line.c new file mode 100644 index 00000000000..2d370ab8444 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/util/cmd_line.c @@ -0,0 +1,1406 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2013 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2012-2015 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015-2017 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2017 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "pmix_config.h" + +#include +#include +#include + +#include "src/class/pmix_object.h" +#include "src/class/pmix_list.h" +#include "src/threads/mutex.h" +#include "src/util/argv.h" +#include "src/util/cmd_line.h" +#include "src/util/output.h" +#include "src/util/pmix_environ.h" + +#include "src/mca/base/pmix_mca_base_var.h" +#include "pmix_common.h" + + +/* + * Some usage message constants + * + * Max width for param listings before the description will be listed + * on the next line + */ +#define PARAM_WIDTH 25 +/* + * Max length of any line in the usage message + */ +#define MAX_WIDTH 76 + +/* + * Description of a command line option + */ +struct pmix_cmd_line_option_t { + pmix_list_item_t super; + + char clo_short_name; + char *clo_single_dash_name; + char *clo_long_name; + + int clo_num_params; + char *clo_description; + + pmix_cmd_line_type_t clo_type; + char *clo_mca_param_env_var; + void *clo_variable_dest; + bool clo_variable_set; + pmix_cmd_line_otype_t clo_otype; +}; +typedef struct pmix_cmd_line_option_t pmix_cmd_line_option_t; +static void option_constructor(pmix_cmd_line_option_t *cmd); +static void option_destructor(pmix_cmd_line_option_t *cmd); + +PMIX_CLASS_INSTANCE(pmix_cmd_line_option_t, + pmix_list_item_t, + option_constructor, option_destructor); + +/* + * An option that was used in the argv that was parsed + */ +struct pmix_cmd_line_param_t { + pmix_list_item_t super; + + /* Note that clp_arg points to storage "owned" by someone else; it + has the original option string by referene, not by value. + Hence, it should not be free()'ed. */ + + char *clp_arg; + + /* Pointer to the existing option. This is also by reference; it + should not be free()ed. */ + + pmix_cmd_line_option_t *clp_option; + + /* This argv array is a list of all the parameters of this option. + It is owned by this parameter, and should be freed when this + param_t is freed. */ + + int clp_argc; + char **clp_argv; +}; +typedef struct pmix_cmd_line_param_t pmix_cmd_line_param_t; +static void param_constructor(pmix_cmd_line_param_t *cmd); +static void param_destructor(pmix_cmd_line_param_t *cmd); +PMIX_CLASS_INSTANCE(pmix_cmd_line_param_t, + pmix_list_item_t, + param_constructor, param_destructor); + +/* + * Instantiate the pmix_cmd_line_t class + */ +static void cmd_line_constructor(pmix_cmd_line_t *cmd); +static void cmd_line_destructor(pmix_cmd_line_t *cmd); +PMIX_CLASS_INSTANCE(pmix_cmd_line_t, + pmix_object_t, + cmd_line_constructor, + cmd_line_destructor); + +/* + * Private variables + */ +static char special_empty_token[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, '\0' +}; + +/* + * Private functions + */ +static int make_opt(pmix_cmd_line_t *cmd, pmix_cmd_line_init_t *e); +static void free_parse_results(pmix_cmd_line_t *cmd); +static int split_shorts(pmix_cmd_line_t *cmd, + char *token, char **args, + int *output_argc, char ***output_argv, + int *num_args_used, bool ignore_unknown); +static pmix_cmd_line_option_t *find_option(pmix_cmd_line_t *cmd, + const char *option_name) __pmix_attribute_nonnull__(1) __pmix_attribute_nonnull__(2); +static int set_dest(pmix_cmd_line_option_t *option, char *sval); +static void fill(const pmix_cmd_line_option_t *a, char result[3][BUFSIZ]); +static int qsort_callback(const void *a, const void *b); +static pmix_cmd_line_otype_t get_help_otype(pmix_cmd_line_t *cmd); +static char *build_parsable(pmix_cmd_line_option_t *option); + + +/* + * Create an entire command line handle from a table + */ +int pmix_cmd_line_create(pmix_cmd_line_t *cmd, + pmix_cmd_line_init_t *table) +{ + int ret = PMIX_SUCCESS; + + /* Check bozo case */ + + if (NULL == cmd) { + return PMIX_ERR_BAD_PARAM; + } + PMIX_CONSTRUCT(cmd, pmix_cmd_line_t); + + if (NULL != table) { + ret = pmix_cmd_line_add(cmd, table); + } + return ret; +} + +/* Add a table to an existing cmd line object */ +int pmix_cmd_line_add(pmix_cmd_line_t *cmd, + pmix_cmd_line_init_t *table) +{ + int i, ret; + + /* Ensure we got a table */ + if (NULL == table) { + return PMIX_SUCCESS; + } + + /* Loop through the table */ + + for (i = 0; ; ++i) { + /* Is this the end? */ + if ('\0' == table[i].ocl_cmd_short_name && + NULL == table[i].ocl_cmd_single_dash_name && + NULL == table[i].ocl_cmd_long_name) { + break; + } + + /* Nope -- it's an entry. Process it. */ + ret = make_opt(cmd, &table[i]); + if (PMIX_SUCCESS != ret) { + return ret; + } + } + + return PMIX_SUCCESS; +} +/* + * Append a command line entry to the previously constructed command line + */ +int pmix_cmd_line_make_opt_mca(pmix_cmd_line_t *cmd, + pmix_cmd_line_init_t entry) +{ + /* Ensure we got an entry */ + if ('\0' == entry.ocl_cmd_short_name && + NULL == entry.ocl_cmd_single_dash_name && + NULL == entry.ocl_cmd_long_name) { + return PMIX_SUCCESS; + } + + return make_opt(cmd, &entry); +} + + +/* + * Create a command line option, --long-name and/or -s (short name). + */ +int pmix_cmd_line_make_opt3(pmix_cmd_line_t *cmd, char short_name, + const char *sd_name, const char *long_name, + int num_params, const char *desc) +{ + pmix_cmd_line_init_t e; + + e.ocl_mca_param_name = NULL; + + e.ocl_cmd_short_name = short_name; + e.ocl_cmd_single_dash_name = sd_name; + e.ocl_cmd_long_name = long_name; + + e.ocl_num_params = num_params; + + e.ocl_variable_dest = NULL; + e.ocl_variable_type = PMIX_CMD_LINE_TYPE_NULL; + + e.ocl_description = desc; + + return make_opt(cmd, &e); +} + + +/* + * Parse a command line according to a pre-built PMIX command line + * handle. + */ +int pmix_cmd_line_parse(pmix_cmd_line_t *cmd, bool ignore_unknown, bool ignore_unknown_option, + int argc, char **argv) +{ + int i, j, orig, ret; + pmix_cmd_line_option_t *option; + pmix_cmd_line_param_t *param; + bool is_unknown_option; + bool is_unknown_token; + bool is_option; + char **shortsv; + int shortsc; + int num_args_used; + bool have_help_option = false; + bool printed_error = false; + bool help_without_arg = false; + + /* Bozo check */ + + if (0 == argc || NULL == argv) { + return PMIX_SUCCESS; + } + + /* Thread serialization */ + + pmix_mutex_lock(&cmd->lcl_mutex); + + /* Free any parsed results that are already on this handle */ + + free_parse_results(cmd); + + /* Analyze each token */ + + cmd->lcl_argc = argc; + cmd->lcl_argv = pmix_argv_copy(argv); + + /* Check up front: do we have a --help option? */ + + option = find_option(cmd, "help"); + if (NULL != option) { + have_help_option = true; + } + + /* Now traverse the easy-to-parse sequence of tokens. Note that + incrementing i must happen elsehwere; it can't be the third + clause in the "if" statement. */ + + param = NULL; + option = NULL; + for (i = 1; i < cmd->lcl_argc; ) { + is_unknown_option = false; + is_unknown_token = false; + is_option = false; + + /* Are we done? i.e., did we find the special "--" token? If + so, copy everying beyond it into the tail (i.e., don't + bother copying the "--" into the tail). */ + + if (0 == strcmp(cmd->lcl_argv[i], "--")) { + ++i; + while (i < cmd->lcl_argc) { + pmix_argv_append(&cmd->lcl_tail_argc, &cmd->lcl_tail_argv, + cmd->lcl_argv[i]); + ++i; + } + + break; + } + + /* If it's not an option, then this is an error. Note that + this is different than an unrecognized token; an + unrecognized option is *always* an error. */ + + else if ('-' != cmd->lcl_argv[i][0]) { + is_unknown_token = true; + } + + /* Nope, this is supposedly an option. Is it a long name? */ + + else if (0 == strncmp(cmd->lcl_argv[i], "--", 2)) { + is_option = true; + option = find_option(cmd, cmd->lcl_argv[i] + 2); + } + + /* It could be a short name. Is it? */ + + else { + option = find_option(cmd, cmd->lcl_argv[i] + 1); + + /* If we didn't find it, try to split it into shorts. If + we find the short option, replace lcl_argv[i] and + insert the rest into lcl_argv starting after position + i. If we don't find the short option, don't do + anything to lcl_argv so that it can fall through to the + error condition, below. */ + + if (NULL == option) { + shortsv = NULL; + shortsc = 0; + ret = split_shorts(cmd, cmd->lcl_argv[i] + 1, + &(cmd->lcl_argv[i + 1]), + &shortsc, &shortsv, + &num_args_used, ignore_unknown); + if (PMIX_SUCCESS == ret) { + option = find_option(cmd, shortsv[0] + 1); + + if (NULL != option) { + pmix_argv_delete(&cmd->lcl_argc, + &cmd->lcl_argv, i, + 1 + num_args_used); + pmix_argv_insert(&cmd->lcl_argv, i, shortsv); + cmd->lcl_argc = pmix_argv_count(cmd->lcl_argv); + } else { + is_unknown_option = true; + } + pmix_argv_free(shortsv); + } else { + is_unknown_option = true; + } + } + + if (NULL != option) { + is_option = true; + } + } + + /* If we figured out above that this is an option, handle it */ + + if (is_option) { + if (NULL == option) { + is_unknown_option = true; + } else { + is_unknown_option = false; + orig = i; + ++i; + + /* Suck down the following parameters that belong to + this option. If we run out of parameters, or find + that any of them are the special_empty_param + (insertted by split_shorts()), then print an error + and return. */ + + param = PMIX_NEW(pmix_cmd_line_param_t); + if (NULL == param) { + pmix_mutex_unlock(&cmd->lcl_mutex); + return PMIX_ERR_OUT_OF_RESOURCE; + } + param->clp_arg = cmd->lcl_argv[i]; + param->clp_option = option; + + /* If we have any parameters to this option, suck down + tokens starting one beyond the token that we just + recognized */ + + for (j = 0; j < option->clo_num_params; ++j, ++i) { + /* If we run out of parameters, error, unless its a help request + which can have 0 or 1 arguments */ + if (i >= cmd->lcl_argc) { + /* If this is a help request, can have no arguments */ + if((NULL != option->clo_single_dash_name && + 0 == strcmp(option->clo_single_dash_name, "h")) || + (NULL != option->clo_long_name && + 0 == strcmp(option->clo_long_name, "help"))) { + help_without_arg = true; + continue; + } + fprintf(stderr, "%s: Error: option \"%s\" did not " + "have enough parameters (%d)\n", + cmd->lcl_argv[0], + cmd->lcl_argv[orig], + option->clo_num_params); + if (have_help_option) { + fprintf(stderr, "Type '%s --help' for usage.\n", + cmd->lcl_argv[0]); + } + PMIX_RELEASE(param); + printed_error = true; + goto error; + } else { + if (0 == strcmp(cmd->lcl_argv[i], + special_empty_token)) { + fprintf(stderr, "%s: Error: option \"%s\" did not " + "have enough parameters (%d)\n", + cmd->lcl_argv[0], + cmd->lcl_argv[orig], + option->clo_num_params); + if (have_help_option) { + fprintf(stderr, "Type '%s --help' for usage.\n", + cmd->lcl_argv[0]); + } + if (NULL != param->clp_argv) { + pmix_argv_free(param->clp_argv); + } + PMIX_RELEASE(param); + printed_error = true; + goto error; + } + + /* Otherwise, save this parameter */ + + else { + /* Save in the argv on the param entry */ + + pmix_argv_append(¶m->clp_argc, + ¶m->clp_argv, + cmd->lcl_argv[i]); + + /* If it's the first, save it in the + variable dest and/or MCA parameter */ + + if (0 == j && + (NULL != option->clo_mca_param_env_var || + NULL != option->clo_variable_dest)) { + if (PMIX_SUCCESS != (ret = set_dest(option, cmd->lcl_argv[i]))) { + pmix_mutex_unlock(&cmd->lcl_mutex); + return ret; + } + } + } + } + } + + /* If there are no options to this command or it is + a help request with no argument, see if we need to + set a boolean value to "true". */ + + if (0 == option->clo_num_params || help_without_arg) { + if (PMIX_SUCCESS != (ret = set_dest(option, "1"))) { + pmix_mutex_unlock(&cmd->lcl_mutex); + return ret; + } + } + + /* If we succeeded in all that, save the param to the + list on the pmix_cmd_line_t handle */ + + if (NULL != param) { + pmix_list_append(&cmd->lcl_params, ¶m->super); + } + } + } + + /* If we figured out above that this was an unknown option, + handle it. Copy everything (including the current token) + into the tail. If we're not ignoring unknowns, then print + an error and return. */ + if (is_unknown_option || is_unknown_token) { + if (!ignore_unknown || (is_unknown_option && !ignore_unknown_option)) { + fprintf(stderr, "%s: Error: unknown option \"%s\"\n", + cmd->lcl_argv[0], cmd->lcl_argv[i]); + printed_error = true; + if (have_help_option) { + fprintf(stderr, "Type '%s --help' for usage.\n", + cmd->lcl_argv[0]); + } + } + error: + while (i < cmd->lcl_argc) { + pmix_argv_append(&cmd->lcl_tail_argc, &cmd->lcl_tail_argv, + cmd->lcl_argv[i]); + ++i; + } + + /* Because i has advanced, we'll fall out of the loop */ + } + } + + /* Thread serialization */ + + pmix_mutex_unlock(&cmd->lcl_mutex); + + /* All done */ + if (printed_error) { + return PMIX_ERR_SILENT; + } + + return PMIX_SUCCESS; +} + + +/* + * Return a consolidated "usage" message for a PMIX command line handle. + */ +char *pmix_cmd_line_get_usage_msg(pmix_cmd_line_t *cmd) +{ + size_t i, len; + int argc; + size_t j; + char **argv; + char *ret, temp[MAX_WIDTH * 2], line[MAX_WIDTH * 2]; + char *start, *desc, *ptr; + pmix_list_item_t *item; + pmix_cmd_line_option_t *option, **sorted; + pmix_cmd_line_otype_t otype; + + /* Thread serialization */ + + pmix_mutex_lock(&cmd->lcl_mutex); + + /* Make an argv of all the usage strings */ + + argc = 0; + argv = NULL; + ret = NULL; + + /* First, take the original list and sort it */ + + sorted = (pmix_cmd_line_option_t**)malloc(sizeof(pmix_cmd_line_option_t *) * + pmix_list_get_size(&cmd->lcl_options)); + if (NULL == sorted) { + pmix_mutex_unlock(&cmd->lcl_mutex); + return NULL; + } + i = 0; + PMIX_LIST_FOREACH(item, &cmd->lcl_options, pmix_list_item_t) { + sorted[i++] = (pmix_cmd_line_option_t *) item; + } + qsort(sorted, i, sizeof(pmix_cmd_line_option_t*), qsort_callback); + + /* Find if a help argument was passed, and return its type if it was. */ + + otype = get_help_otype(cmd); + + /* Now go through the sorted array and make the strings */ + + for (j = 0; j < pmix_list_get_size(&cmd->lcl_options); ++j) { + option = sorted[j]; + if(otype == PMIX_CMD_LINE_OTYPE_PARSABLE) { + ret = build_parsable(option); + pmix_argv_append(&argc, &argv, ret); + free(ret); + ret = NULL; + } else if(otype == PMIX_CMD_LINE_OTYPE_NULL || option->clo_otype == otype) { + if (NULL != option->clo_description) { + bool filled = false; + + /* Build up the output line */ + + memset(line, 0, sizeof(line)); + if ('\0' != option->clo_short_name) { + line[0] = '-'; + line[1] = option->clo_short_name; + filled = true; + } else { + line[0] = ' '; + line[1] = ' '; + } + if (NULL != option->clo_single_dash_name) { + line[2] = (filled) ? '|' : ' '; + strncat(line, "-", sizeof(line) - 1); + strncat(line, option->clo_single_dash_name, sizeof(line) - 1); + filled = true; + } + if (NULL != option->clo_long_name) { + if (filled) { + strncat(line, "|", sizeof(line) - 1); + } else { + strncat(line, " ", sizeof(line) - 1); + } + strncat(line, "--", sizeof(line) - 1); + strncat(line, option->clo_long_name, sizeof(line) - 1); + } + strncat(line, " ", sizeof(line) - 1); + for (i = 0; (int)i < option->clo_num_params; ++i) { + len = sizeof(temp); + snprintf(temp, len, " ", (int)i); + strncat(line, temp, sizeof(line) - 1); + } + if (option->clo_num_params > 0) { + strncat(line, " ", sizeof(line) - 1); + } + + /* If we're less than param width, then start adding the + description to this line. Otherwise, finish this line + and start adding the description on the next line. */ + + if (strlen(line) > PARAM_WIDTH) { + pmix_argv_append(&argc, &argv, line); + + /* Now reset the line to be all blanks up to + PARAM_WIDTH so that we can start adding the + description */ + + memset(line, ' ', PARAM_WIDTH); + line[PARAM_WIDTH] = '\0'; + } else { + + /* Add enough blanks to the end of the line so that we + can start adding the description */ + + for (i = strlen(line); i < PARAM_WIDTH; ++i) { + line[i] = ' '; + } + line[i] = '\0'; + } + + /* Loop over adding the description to the array, breaking + the string at most at MAX_WIDTH characters. We need a + modifyable description (for simplicity), so strdup the + clo_description (because it's likely a cpmixler + constant, and may barf if we write temporary \0's in + the middle). */ + + desc = strdup(option->clo_description); + if (NULL == desc) { + free(sorted); + pmix_mutex_unlock(&cmd->lcl_mutex); + return strdup(""); + } + start = desc; + len = strlen(desc); + do { + + /* Trim off leading whitespace */ + + while (isspace(*start) && start < desc + len) { + ++start; + } + if (start >= desc + len) { + break; + } + + /* Last line */ + + if (strlen(start) < (MAX_WIDTH - PARAM_WIDTH)) { + strncat(line, start, sizeof(line) - 1); + pmix_argv_append(&argc, &argv, line); + break; + } + + /* We have more than 1 line's worth left -- find this + line's worth and add it to the array. Then reset + and loop around to get the next line's worth. */ + + for (ptr = start + (MAX_WIDTH - PARAM_WIDTH); + ptr > start; --ptr) { + if (isspace(*ptr)) { + *ptr = '\0'; + strncat(line, start, sizeof(line) - 1); + pmix_argv_append(&argc, &argv, line); + + start = ptr + 1; + memset(line, ' ', PARAM_WIDTH); + line[PARAM_WIDTH] = '\0'; + break; + } + } + + /* If we got all the way back to the beginning of the + string, then go forward looking for a whitespace + and break there. */ + + if (ptr == start) { + for (ptr = start + (MAX_WIDTH - PARAM_WIDTH); + ptr < start + len; ++ptr) { + if (isspace(*ptr)) { + *ptr = '\0'; + + strncat(line, start, sizeof(line) - 1); + pmix_argv_append(&argc, &argv, line); + + start = ptr + 1; + memset(line, ' ', PARAM_WIDTH); + line[PARAM_WIDTH] = '\0'; + break; + } + } + + /* If we reached the end of the string with no + whitespace, then just add it on and be done */ + + if (ptr >= start + len) { + strncat(line, start, sizeof(line) - 1); + pmix_argv_append(&argc, &argv, line); + start = desc + len + 1; + } + } + } while (start < desc + len); + free(desc); + } + } + } + if(otype == PMIX_CMD_LINE_OTYPE_NULL || otype == PMIX_CMD_LINE_OTYPE_GENERAL) { + char *argument_line = "\nFor additional mpirun arguments, run 'mpirun --help '\n\nThe following categories exist: general (Defaults to this option), debug,\n output, input, mapping, ranking, binding, devel (arguments useful to PMIX\n Developers), compatibility (arguments supported for backwards compatibility),\n launch (arguments to modify launch options), and dvm (Distributed Virtual\n Machine arguments)."; + + pmix_argv_append(&argc, &argv, argument_line); + } + if (NULL != argv) { + ret = pmix_argv_join(argv, '\n'); + pmix_argv_free(argv); + } else { + ret = strdup(""); + } + free(sorted); + + /* Thread serialization */ + pmix_mutex_unlock(&cmd->lcl_mutex); + + /* All done */ + return ret; +} + + +/* + * Test if a given option was taken on the parsed command line. + */ +bool pmix_cmd_line_is_taken(pmix_cmd_line_t *cmd, const char *opt) +{ + return (pmix_cmd_line_get_ninsts(cmd, opt) > 0); +} + + +/* + * Return the number of instances of an option found during parsing. + */ +int pmix_cmd_line_get_ninsts(pmix_cmd_line_t *cmd, const char *opt) +{ + int ret; + pmix_cmd_line_param_t *param; + pmix_cmd_line_option_t *option; + + /* Thread serialization */ + + pmix_mutex_lock(&cmd->lcl_mutex); + + /* Find the corresponding option. If we find it, look through all + the parsed params and see if we have any matches. */ + + ret = 0; + option = find_option(cmd, opt); + if (NULL != option) { + PMIX_LIST_FOREACH(param, &cmd->lcl_params, pmix_cmd_line_param_t) { + if (param->clp_option == option) { + ++ret; + } + } + } + + /* Thread serialization */ + + pmix_mutex_unlock(&cmd->lcl_mutex); + + /* All done */ + + return ret; +} + + +/* + * Return a specific parameter for a specific instance of a option + * from the parsed command line. + */ +char *pmix_cmd_line_get_param(pmix_cmd_line_t *cmd, const char *opt, int inst, + int idx) +{ + int num_found; + pmix_cmd_line_param_t *param; + pmix_cmd_line_option_t *option; + + /* Thread serialization */ + + pmix_mutex_lock(&cmd->lcl_mutex); + + /* Find the corresponding option. If we find it, look through all + the parsed params and see if we have any matches. */ + + num_found = 0; + option = find_option(cmd, opt); + if (NULL != option) { + + /* Ensure to check for the case where the user has asked for a + parameter index greater than we will have */ + + if (idx < option->clo_num_params) { + PMIX_LIST_FOREACH(param, &cmd->lcl_params, pmix_cmd_line_param_t) { + if (param->clp_argc > 0 && param->clp_option == option) { + if (num_found == inst) { + pmix_mutex_unlock(&cmd->lcl_mutex); + return param->clp_argv[idx]; + } + ++num_found; + } + } + } + } + + /* Thread serialization */ + + pmix_mutex_unlock(&cmd->lcl_mutex); + + /* All done */ + + return NULL; +} + + +/* + * Return the number of arguments parsed on a PMIX command line handle. + */ +int pmix_cmd_line_get_argc(pmix_cmd_line_t *cmd) +{ + return (NULL != cmd) ? cmd->lcl_argc : PMIX_ERROR; +} + + +/* + * Return a string argument parsed on a PMIX command line handle. + */ +char *pmix_cmd_line_get_argv(pmix_cmd_line_t *cmd, int index) +{ + return (NULL == cmd) ? NULL : + (index >= cmd->lcl_argc || index < 0) ? NULL : cmd->lcl_argv[index]; +} + + +/* + * Return the entire "tail" of unprocessed argv from a PMIX command + * line handle. + */ +int pmix_cmd_line_get_tail(pmix_cmd_line_t *cmd, int *tailc, char ***tailv) +{ + if (NULL != cmd) { + pmix_mutex_lock(&cmd->lcl_mutex); + *tailc = cmd->lcl_tail_argc; + *tailv = pmix_argv_copy(cmd->lcl_tail_argv); + pmix_mutex_unlock(&cmd->lcl_mutex); + return PMIX_SUCCESS; + } else { + return PMIX_ERROR; + } +} + + +/************************************************************************** + * Static functions + **************************************************************************/ + +static void option_constructor(pmix_cmd_line_option_t *o) +{ + o->clo_short_name = '\0'; + o->clo_single_dash_name = NULL; + o->clo_long_name = NULL; + o->clo_num_params = 0; + o->clo_description = NULL; + + o->clo_type = PMIX_CMD_LINE_TYPE_NULL; + o->clo_mca_param_env_var = NULL; + o->clo_variable_dest = NULL; + o->clo_variable_set = false; + o->clo_otype = PMIX_CMD_LINE_OTYPE_NULL; +} + + +static void option_destructor(pmix_cmd_line_option_t *o) +{ + if (NULL != o->clo_single_dash_name) { + free(o->clo_single_dash_name); + } + if (NULL != o->clo_long_name) { + free(o->clo_long_name); + } + if (NULL != o->clo_description) { + free(o->clo_description); + } + if (NULL != o->clo_mca_param_env_var) { + free(o->clo_mca_param_env_var); + } +} + + +static void param_constructor(pmix_cmd_line_param_t *p) +{ + p->clp_arg = NULL; + p->clp_option = NULL; + p->clp_argc = 0; + p->clp_argv = NULL; +} + + +static void param_destructor(pmix_cmd_line_param_t *p) +{ + if (NULL != p->clp_argv) { + pmix_argv_free(p->clp_argv); + } +} + + +static void cmd_line_constructor(pmix_cmd_line_t *cmd) +{ + /* Initialize the mutex. Since we're creating (and therefore the + only thread that has this instance), there's no need to lock it + right now. */ + + PMIX_CONSTRUCT(&cmd->lcl_mutex, pmix_recursive_mutex_t); + + /* Initialize the lists */ + + PMIX_CONSTRUCT(&cmd->lcl_options, pmix_list_t); + PMIX_CONSTRUCT(&cmd->lcl_params, pmix_list_t); + + /* Initialize the argc/argv pairs */ + + cmd->lcl_argc = 0; + cmd->lcl_argv = NULL; + cmd->lcl_tail_argc = 0; + cmd->lcl_tail_argv = NULL; +} + + +static void cmd_line_destructor(pmix_cmd_line_t *cmd) +{ + pmix_list_item_t *item; + + /* Free the contents of the options list (do not free the list + itself; it was not allocated from the heap) */ + + for (item = pmix_list_remove_first(&cmd->lcl_options); + NULL != item; + item = pmix_list_remove_first(&cmd->lcl_options)) { + PMIX_RELEASE(item); + } + + /* Free any parsed results */ + + free_parse_results(cmd); + + /* Destroy the lists */ + + PMIX_DESTRUCT(&cmd->lcl_options); + PMIX_DESTRUCT(&cmd->lcl_params); + + /* Destroy the mutex */ + + PMIX_DESTRUCT(&cmd->lcl_mutex); +} + + +static int make_opt(pmix_cmd_line_t *cmd, pmix_cmd_line_init_t *e) +{ + pmix_cmd_line_option_t *option; + + /* Bozo checks */ + + if (NULL == cmd) { + return PMIX_ERR_BAD_PARAM; + } else if ('\0' == e->ocl_cmd_short_name && + NULL == e->ocl_cmd_single_dash_name && + NULL == e->ocl_cmd_long_name) { + return PMIX_ERR_BAD_PARAM; + } else if (e->ocl_num_params < 0) { + return PMIX_ERR_BAD_PARAM; + } + + /* see if the option already exists */ + if (NULL != e->ocl_cmd_single_dash_name && + NULL != find_option(cmd, e->ocl_cmd_single_dash_name)) { + pmix_output(0, "Duplicate cmd line entry %s", e->ocl_cmd_single_dash_name); + return PMIX_ERR_BAD_PARAM; + } + if (NULL != e->ocl_cmd_long_name && + NULL != find_option(cmd, e->ocl_cmd_long_name)) { + pmix_output(0, "Duplicate cmd line entry %s", e->ocl_cmd_long_name); + return PMIX_ERR_BAD_PARAM; + } + + /* Allocate and fill an option item */ + option = PMIX_NEW(pmix_cmd_line_option_t); + if (NULL == option) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + + option->clo_short_name = e->ocl_cmd_short_name; + if (NULL != e->ocl_cmd_single_dash_name) { + option->clo_single_dash_name = strdup(e->ocl_cmd_single_dash_name); + } + if (NULL != e->ocl_cmd_long_name) { + option->clo_long_name = strdup(e->ocl_cmd_long_name); + } + option->clo_num_params = e->ocl_num_params; + if (NULL != e->ocl_description) { + option->clo_description = strdup(e->ocl_description); + } + + option->clo_type = e->ocl_variable_type; + option->clo_variable_dest = e->ocl_variable_dest; + if (NULL != e->ocl_mca_param_name) { + (void) pmix_mca_base_var_env_name (e->ocl_mca_param_name, + &option->clo_mca_param_env_var); + } + + option->clo_otype = e->ocl_otype; + + /* Append the item, serializing thread access */ + + pmix_mutex_lock(&cmd->lcl_mutex); + pmix_list_append(&cmd->lcl_options, &option->super); + pmix_mutex_unlock(&cmd->lcl_mutex); + + /* All done */ + + return PMIX_SUCCESS; +} + + +static void free_parse_results(pmix_cmd_line_t *cmd) +{ + pmix_list_item_t *item; + + /* Free the contents of the params list (do not free the list + itself; it was not allocated from the heap) */ + + for (item = pmix_list_remove_first(&cmd->lcl_params); + NULL != item; + item = pmix_list_remove_first(&cmd->lcl_params)) { + PMIX_RELEASE(item); + } + + /* Free the argv's */ + + if (NULL != cmd->lcl_argv) { + pmix_argv_free(cmd->lcl_argv); + } + cmd->lcl_argv = NULL; + cmd->lcl_argc = 0; + + if (NULL != cmd->lcl_tail_argv) { + pmix_argv_free(cmd->lcl_tail_argv); + } + cmd->lcl_tail_argv = NULL; + cmd->lcl_tail_argc = 0; +} + + +/* + * Traverse a token and split it into individual letter options (the + * token has already been certified to not be a long name and not be a + * short name). Ensure to differentiate the resulting options from + * "single dash" names. + */ +static int split_shorts(pmix_cmd_line_t *cmd, char *token, char **args, + int *output_argc, char ***output_argv, + int *num_args_used, bool ignore_unknown) +{ + int i, j, len; + pmix_cmd_line_option_t *option; + char fake_token[3]; + int num_args; + + /* Setup that we didn't use any of the args */ + + num_args = pmix_argv_count(args); + *num_args_used = 0; + + /* Traverse the token. If it's empty (e.g., if someone passes a + "-" token, which, since the upper level calls this function as + (argv[i] + 1), will be empty by the time it gets down here), + just return that we didn't find a short option. */ + + len = (int)strlen(token); + if (0 == len) { + return PMIX_ERR_BAD_PARAM; + } + fake_token[0] = '-'; + fake_token[2] = '\0'; + for (i = 0; i < len; ++i) { + fake_token[1] = token[i]; + option = find_option(cmd, fake_token + 1); + + /* If we don't find the option, either return an error or pass + it through unmodified to the new argv */ + + if (NULL == option) { + if (!ignore_unknown) { + return PMIX_ERR_BAD_PARAM; + } else { + pmix_argv_append(output_argc, output_argv, fake_token); + } + } + + /* If we do find the option, copy it and all of its parameters + to the output args. If we run out of paramters (i.e., no + more tokens in the original argv), that error will be + handled at a higher level) */ + + else { + pmix_argv_append(output_argc, output_argv, fake_token); + for (j = 0; j < option->clo_num_params; ++j) { + if (*num_args_used < num_args) { + pmix_argv_append(output_argc, output_argv, + args[*num_args_used]); + ++(*num_args_used); + } else { + pmix_argv_append(output_argc, output_argv, + special_empty_token); + } + } + } + } + + /* All done */ + + return PMIX_SUCCESS; +} + + +static pmix_cmd_line_option_t *find_option(pmix_cmd_line_t *cmd, + const char *option_name) +{ + pmix_cmd_line_option_t *option; + + /* Iterate through the list of options hanging off the + pmix_cmd_line_t and see if we find a match in either the short + or long names */ + + PMIX_LIST_FOREACH(option, &cmd->lcl_options, pmix_cmd_line_option_t) { + if ((NULL != option->clo_long_name && + 0 == strcmp(option_name, option->clo_long_name)) || + (NULL != option->clo_single_dash_name && + 0 == strcmp(option_name, option->clo_single_dash_name)) || + (strlen(option_name) == 1 && + option_name[0] == option->clo_short_name)) { + return option; + } + } + + /* Not found */ + + return NULL; +} + + +static int set_dest(pmix_cmd_line_option_t *option, char *sval) +{ + int ival = atol(sval); + long lval = strtoul(sval, NULL, 10); + size_t i; + + /* Set MCA param. We do this in the environment because the MCA + parameter may not have been registered yet -- and if it isn't + registered, we don't really want to register a dummy one + because we don't know what it's type and default value should + be. These are solvable problems (e.g., make a re-registration + overwrite everything), but it's far simpler to just leave the + registered table alone and set an environment variable with the + desired value. The environment variable will get picked up + during a nromal parameter lookup, and all will be well. */ + + if (NULL != option->clo_mca_param_env_var) { + switch(option->clo_type) { + case PMIX_CMD_LINE_TYPE_STRING: + case PMIX_CMD_LINE_TYPE_INT: + case PMIX_CMD_LINE_TYPE_SIZE_T: + pmix_setenv(option->clo_mca_param_env_var, sval, true, &environ); + break; + case PMIX_CMD_LINE_TYPE_BOOL: + pmix_setenv(option->clo_mca_param_env_var, "1", true, &environ); + break; + default: + break; + } + } + + /* Set variable */ + + if (NULL != option->clo_variable_dest) { + switch(option->clo_type) { + case PMIX_CMD_LINE_TYPE_STRING: + *((char**) option->clo_variable_dest) = strdup(sval); + break; + case PMIX_CMD_LINE_TYPE_INT: + /* check to see that the value given to us truly is an int */ + for (i=0; i < strlen(sval); i++) { + if (!isdigit(sval[i]) && '-' != sval[i]) { + /* show help isn't going to be available yet, so just + * print the msg + */ + fprintf(stderr, "----------------------------------------------------------------------------\n"); + fprintf(stderr, "Open MPI has detected that a parameter given to a command line\n"); + fprintf(stderr, "option does not match the expected format:\n\n"); + if (NULL != option->clo_long_name) { + fprintf(stderr, " Option: %s\n", option->clo_long_name); + } else if ('\0' != option->clo_short_name) { + fprintf(stderr, " Option: %c\n", option->clo_short_name); + } else { + fprintf(stderr, " Option: \n"); + } + fprintf(stderr, " Param: %s\n\n", sval); + fprintf(stderr, "This is frequently caused by omitting to provide the parameter\n"); + fprintf(stderr, "to an option that requires one. Please check the command line and try again.\n"); + fprintf(stderr, "----------------------------------------------------------------------------\n"); + return PMIX_ERR_SILENT; + } + } + *((int*) option->clo_variable_dest) = ival; + break; + case PMIX_CMD_LINE_TYPE_SIZE_T: + /* check to see that the value given to us truly is a size_t */ + for (i=0; i < strlen(sval); i++) { + if (!isdigit(sval[i]) && '-' != sval[i]) { + /* show help isn't going to be available yet, so just + * print the msg + */ + fprintf(stderr, "----------------------------------------------------------------------------\n"); + fprintf(stderr, "Open MPI has detected that a parameter given to a command line\n"); + fprintf(stderr, "option does not match the expected format:\n\n"); + if (NULL != option->clo_long_name) { + fprintf(stderr, " Option: %s\n", option->clo_long_name); + } else if ('\0' != option->clo_short_name) { + fprintf(stderr, " Option: %c\n", option->clo_short_name); + } else { + fprintf(stderr, " Option: \n"); + } + fprintf(stderr, " Param: %s\n\n", sval); + fprintf(stderr, "This is frequently caused by omitting to provide the parameter\n"); + fprintf(stderr, "to an option that requires one. Please check the command line and try again.\n"); + fprintf(stderr, "----------------------------------------------------------------------------\n"); + return PMIX_ERR_SILENT; + } + } + *((size_t*) option->clo_variable_dest) = lval; + break; + case PMIX_CMD_LINE_TYPE_BOOL: + *((bool*) option->clo_variable_dest) = 1; + break; + default: + break; + } + } + return PMIX_SUCCESS; +} + + +/* + * Helper function to qsort_callback + */ +static void fill(const pmix_cmd_line_option_t *a, char result[3][BUFSIZ]) +{ + int i = 0; + + result[0][0] = '\0'; + result[1][0] = '\0'; + result[2][0] = '\0'; + + if ('\0' != a->clo_short_name) { + snprintf(&result[i][0], BUFSIZ, "%c", a->clo_short_name); + ++i; + } + if (NULL != a->clo_single_dash_name) { + snprintf(&result[i][0], BUFSIZ, "%s", a->clo_single_dash_name); + ++i; + } + if (NULL != a->clo_long_name) { + snprintf(&result[i][0], BUFSIZ, "%s", a->clo_long_name); + ++i; + } +} + + +static int qsort_callback(const void *aa, const void *bb) +{ + int ret, i; + char str1[3][BUFSIZ], str2[3][BUFSIZ]; + const pmix_cmd_line_option_t *a = *((const pmix_cmd_line_option_t**) aa); + const pmix_cmd_line_option_t *b = *((const pmix_cmd_line_option_t**) bb); + + /* Icky comparison of command line options. There are multiple + forms of each command line option, so we first have to check + which forms each option has. Compare, in order: short name, + single-dash name, long name. */ + + fill(a, str1); + fill(b, str2); + + for (i = 0; i < 3; ++i) { + if (0 != (ret = strcasecmp(str1[i], str2[i]))) { + return ret; + } + } + + /* Shrug -- they must be equal */ + + return 0; +} + + +/* + * Helper function to find the option type specified in the help + * command. + */ +static pmix_cmd_line_otype_t get_help_otype(pmix_cmd_line_t *cmd) +{ + /* Initialize to NULL, if it remains so, the user asked for + "full" help output */ + pmix_cmd_line_otype_t otype = PMIX_CMD_LINE_OTYPE_NULL; + char *arg; + + arg = pmix_cmd_line_get_param(cmd, "help", 0, 0); + + /* If not "help", check for "h" */ + if(NULL == arg) { + arg = pmix_cmd_line_get_param(cmd, "h", 0, 0); + } + + /* If arg is still NULL, give them the General info by default */ + if(NULL == arg) { + arg = "general"; + } + + if (0 == strcmp(arg, "debug")) { + otype = PMIX_CMD_LINE_OTYPE_DEBUG; + } else if (0 == strcmp(arg, "output")) { + otype = PMIX_CMD_LINE_OTYPE_OUTPUT; + } else if (0 == strcmp(arg, "input")) { + otype = PMIX_CMD_LINE_OTYPE_INPUT; + } else if (0 == strcmp(arg, "mapping")) { + otype = PMIX_CMD_LINE_OTYPE_MAPPING; + } else if (0 == strcmp(arg, "ranking")) { + otype = PMIX_CMD_LINE_OTYPE_RANKING; + } else if (0 == strcmp(arg, "binding")) { + otype = PMIX_CMD_LINE_OTYPE_BINDING; + } else if (0 == strcmp(arg, "devel")) { + otype = PMIX_CMD_LINE_OTYPE_DEVEL; + } else if (0 == strcmp(arg, "compatibility")) { + otype = PMIX_CMD_LINE_OTYPE_COMPAT; + } else if (0 == strcmp(arg, "launch")) { + otype = PMIX_CMD_LINE_OTYPE_LAUNCH; + } else if (0 == strcmp(arg, "dvm")) { + otype = PMIX_CMD_LINE_OTYPE_DVM; + } else if (0 == strcmp(arg, "general")) { + otype = PMIX_CMD_LINE_OTYPE_GENERAL; + } else if (0 == strcmp(arg, "parsable")) { + otype = PMIX_CMD_LINE_OTYPE_PARSABLE; + } + + return otype; +} + +/* + * Helper function to build a parsable string for the help + * output. + */ +static char *build_parsable(pmix_cmd_line_option_t *option) { + char *line; + int length; + + length = snprintf(NULL, 0, "%c:%s:%s:%d:%s\n", option->clo_short_name, option->clo_single_dash_name, + option->clo_long_name, option->clo_num_params, option->clo_description); + + line = (char *)malloc(length * sizeof(char)); + + if('\0' == option->clo_short_name) { + snprintf(line, length, "0:%s:%s:%d:%s\n", option->clo_single_dash_name, option->clo_long_name, + option->clo_num_params, option->clo_description); + } else { + snprintf(line, length, "%c:%s:%s:%d:%s\n", option->clo_short_name, option->clo_single_dash_name, + option->clo_long_name, option->clo_num_params, option->clo_description); + } + + return line; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/cmd_line.h b/opal/mca/pmix/pmix2x/pmix/src/util/cmd_line.h new file mode 100644 index 00000000000..7f0ca5a178a --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/util/cmd_line.h @@ -0,0 +1,622 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2017 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/** + * @file + * + * General command line parsing facility for use throughout Open MPI. + * + * This scheme is inspired by the GNU getopt package. Command line + * options are registered. Each option can have up to three different + * matching tokens: a "short" name, a "single dash" name, and a "long" + * name. Each option can also take 0 or more arguments. Finally, + * each option can be repeated on the command line an arbitrary number + * of times. + * + * The "short" name can only be a single letter, and will be found + * after a single dash (e.g., "-a"). Multiple "short" names can be + * combined into a single command line argument (e.g., "-abc" can be + * equivalent to "-a -b -c"). + * + * The "single dash" name is a multi-character name that only + * requires a single dash. This only exists to provide backwards + * compatibility for some well-known command line options in prior + * MPI implementations (e.g., "mpirun -np 3"). It should be used + * sparingly. + * + * The "long" name is a multi-character name that is found after a + * pair of dashes. For example, "--some-option-name". + * + * A command line option is a combination of 1 or more of a short + * name, single dash name, and a long name. Any of the names may be + * used on the command line; they are treated as synonyms. For + * example, say the following was used in for an executable named + * "foo": + * + * \code + * pmix_cmd_line_make_opt3(cmd, 'a', NULL, 'add', 1, "Add a user"); + * \endcode + * + * In this case, the following command lines are exactly equivalent: + * + * \verbatim + * shell$ foo -a jsmith + * shell$ foo --add jsmith + * \endverbatim + * + * Note that this interface can also track multiple invocations of the + * same option. For example, the following is both legal and able to + * be retrieved through this interface: + * + * \verbatim + * shell$ foo -a jsmith -add bjones + * \endverbatim + * + * The caller to this interface creates a command line handle + * (pmix_cmd_line_t) with OBJ_NEW() and then uses it to register the + * desired parameters via pmix_cmd_line_make_opt3(). Once all the + * parameters have been registered, the user invokes + * pmix_cmd_line_parse() with the command line handle and the argv/argc + * pair to be parsed (typically the arguments from main()). The parser + * will examine the argv and find registered options and parameters. + * It will stop parsing when it runs into an recognized string token or + * the special "--" token. + * + * After the parse has occurred, various accessor functions can be + * used to determine which options were selected, what parameters were + * passed to them, etc.: + * + * - pmix_cmd_line_get_usage_msg() returns a string suitable for "help" + * kinds of messages. + * - pmix_cmd_line_is_taken() returns a true or false indicating + * whether a given command line option was found on the command + * line. + * - pmix_cmd_line_get_argc() returns the number of tokens parsed on + * the handle. + * - pmix_cmd_line_get_argv() returns any particular string from the + * original argv. + * - pmix_cmd_line_get_ninsts() returns the number of times a + * particular option was found on a command line. + * - pmix_cmd_line_get_param() returns the Nth parameter in the Mth + * instance of a given parameter. + * - pmix_cmd_line_get_tail() returns an array of tokens not parsed + * (i.e., if the parser ran into "--" or an unrecognized token). + * + * Note that a shortcut to creating a large number of options exists + * -- one can make a table of pmix_cmd_line_init_t instances and the + * table to pmix_cmd_line_create(). This creates an pmix_cmd_line_t + * handle that is pre-seeded with all the options from the table + * without the need to repeatedly invoke pmix_cmd_line_make_opt3() (or + * equivalent). This pmix_cmd_line_t instance is just like any other; + * it is still possible to add more options via + * pmix_cmd_line_make_opt3(), etc. + */ + +#ifndef PMIX_CMD_LINE_H +#define PMIX_CMD_LINE_H + +#include "pmix_config.h" + +#include "src/class/pmix_object.h" +#include "src/class/pmix_list.h" +#include "src/threads/mutex.h" + +BEGIN_C_DECLS + /** + * \internal + * + * Main top-level handle. This interface should not be used by users! + */ + struct pmix_cmd_line_t { + /** Make this an OBJ handle */ + pmix_object_t super; + + /** Thread safety */ + pmix_recursive_mutex_t lcl_mutex; + + /** List of ompi_cmd_line_option_t's (defined internally) */ + pmix_list_t lcl_options; + + /** Duplicate of argc from pmix_cmd_line_parse() */ + int lcl_argc; + /** Duplicate of argv from pmix_cmd_line_parse() */ + char **lcl_argv; + + /** Parsed output; list of ompi_cmd_line_param_t's (defined internally) */ + pmix_list_t lcl_params; + + /** List of tail (unprocessed) arguments */ + int lcl_tail_argc; + /** List of tail (unprocessed) arguments */ + char **lcl_tail_argv; + }; + /** + * \internal + * + * Convenience typedef + */ + typedef struct pmix_cmd_line_t pmix_cmd_line_t; + + /** + * Data types supported by the parser + */ + enum pmix_cmd_line_type_t { + PMIX_CMD_LINE_TYPE_NULL, + PMIX_CMD_LINE_TYPE_STRING, + PMIX_CMD_LINE_TYPE_INT, + PMIX_CMD_LINE_TYPE_SIZE_T, + PMIX_CMD_LINE_TYPE_BOOL, + + PMIX_CMD_LINE_TYPE_MAX + }; + /** + * \internal + * + * Convenience typedef + */ + typedef enum pmix_cmd_line_type_t pmix_cmd_line_type_t; + + /** + * Command line option type, for use in + * mpirun --help output. + */ + enum pmix_cmd_line_otype_t { + PMIX_CMD_LINE_OTYPE_GENERAL, + PMIX_CMD_LINE_OTYPE_DEBUG, + PMIX_CMD_LINE_OTYPE_OUTPUT, + PMIX_CMD_LINE_OTYPE_INPUT, + PMIX_CMD_LINE_OTYPE_MAPPING, + PMIX_CMD_LINE_OTYPE_RANKING, + PMIX_CMD_LINE_OTYPE_BINDING, + PMIX_CMD_LINE_OTYPE_DEVEL, + PMIX_CMD_LINE_OTYPE_COMPAT, /* Backwards compatibility */ + PMIX_CMD_LINE_OTYPE_LAUNCH, + PMIX_CMD_LINE_OTYPE_DVM, + PMIX_CMD_LINE_OTYPE_UNSUPPORTED, + PMIX_CMD_LINE_OTYPE_PARSABLE, + PMIX_CMD_LINE_OTYPE_NULL + }; + /** + * \internal + * + * Convenience typedef + */ + typedef enum pmix_cmd_line_otype_t pmix_cmd_line_otype_t; + + /** + * Datatype used to construct a command line handle; see + * pmix_cmd_line_create(). + */ + struct pmix_cmd_line_init_t { + /** If want to set an MCA parameter, set its parameter name + here. */ + const char *ocl_mca_param_name; + + /** "Short" name (i.e., "-X", where "X" is a single letter) */ + char ocl_cmd_short_name; + /** "Single dash" name (i.e., "-foo"). The use of these are + discouraged. */ + const char *ocl_cmd_single_dash_name; + /** Long name (i.e., "--foo"). */ + const char *ocl_cmd_long_name; + + /** Number of parameters that this option takes */ + int ocl_num_params; + + /** If this parameter is encountered, its *first* parameter it + saved here. If the parameter is encountered again, the + value is overwritten. */ + void *ocl_variable_dest; + /** If an ocl_variable_dest is given, its datatype must be + supplied as well. */ + pmix_cmd_line_type_t ocl_variable_type; + + /** Description of the command line option, to be used with + pmix_cmd_line_get_usage_msg(). */ + const char *ocl_description; + + /** Category for mpirun --help output */ + pmix_cmd_line_otype_t ocl_otype; + }; + /** + * \internal + * + * Convenience typedef + */ + typedef struct pmix_cmd_line_init_t pmix_cmd_line_init_t; + + /** + * Top-level command line handle. + * + * This handle is used for accessing all command line functionality + * (i.e., all pmix_cmd_line*() functions). Multiple handles can be + * created and simultaneously processed; each handle is independant + * from others. + * + * The pmix_cmd_line_t handles are [simplisticly] thread safe; + * processing is guaranteed to be mutually exclusive if multiple + * threads invoke functions on the same handle at the same time -- + * access will be serialized in an unspecified order. + * + * Once finished, handles should be released with OBJ_RELEASE(). The + * destructor for pmix_cmd_line_t handles will free all memory + * associated with the handle. + */ + PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_cmd_line_t); + + /** + * Make a command line handle from a table of initializers. + * + * @param cmd PMIX command line handle. + * @param table Table of pmix_cmd_line_init_t instances for all + * the options to be included in the resulting command line + * handler. + * + * @retval PMIX_SUCCESS Upon success. + * + * This function takes a table of pmix_cmd_line_init_t instances + * to pre-seed an PMIX command line handle. The last instance in + * the table must have '\0' for the short name and NULL for the + * single-dash and long names. The handle is expected to have + * been OBJ_NEW'ed or OBJ_CONSTRUCT'ed already. + * + * Upon return, the command line handle is just like any other. A + * sample using this syntax: + * + * \code + * pmix_cmd_line_init_t cmd_line_init[] = { + * { NULL, NULL, NULL, 'h', NULL, "help", 0, + * &orterun_globals.help, PMIX_CMD_LINE_TYPE_BOOL, + * "This help message" }, + * + * { NULL, NULL, NULL, '\0', NULL, "wd", 1, + * &orterun_globals.wd, PMIX_CMD_LINE_TYPE_STRING, + * "Set the working directory of the started processes" }, + * + * { NULL, NULL, NULL, '\0', NULL, NULL, 0, + * NULL, PMIX_CMD_LINE_TYPE_NULL, NULL } + * }; + * \endcode + */ + PMIX_EXPORT int pmix_cmd_line_create(pmix_cmd_line_t *cmd, + pmix_cmd_line_init_t *table); + + /* Add a table of pmix_cmd_line_init_t instances + * to an existing PMIX command line handle. + * + * Multiple calls to pmix_cmd_line_add are permitted - each + * subsequent call will simply append new options to the existing + * handle. Note that any duplicates will return an error. + */ + PMIX_EXPORT int pmix_cmd_line_add(pmix_cmd_line_t *cmd, + pmix_cmd_line_init_t *table); + + /** + * Create a command line option. + * + * @param cmd PMIX command line handle. + * @param entry Command line entry to add to the command line. + * + * @retval PMIX_SUCCESS Upon success. + * + */ + PMIX_EXPORT int pmix_cmd_line_make_opt_mca(pmix_cmd_line_t *cmd, + pmix_cmd_line_init_t entry); + + /** + * Create a command line option. + * + * @param cmd PMIX command line handle. + * @param short_name "Short" name of the command line option. + * @param sd_name "Single dash" name of the command line option. + * @param long_name "Long" name of the command line option. + * @param num_params How many parameters this option takes. + * @param dest Short string description of this option. + * + * @retval PMIX_ERR_OUT_OF_RESOURCE If out of memory. + * @retval PMIX_ERR_BAD_PARAM If bad parameters passed. + * @retval PMIX_SUCCESS Upon success. + * + * Adds a command line option to the list of options that an PMIX + * command line handle will accept. The short_name may take the + * special value '\0' to not have a short name. Likewise, the + * sd_name and long_name may take the special value NULL to not have + * a single dash or long name, respectively. However, one of the + * three must have a name. + * + * num_params indicates how many parameters this option takes. It + * must be greater than or equal to 0. + * + * Finally, desc is a short string description of this option. It is + * used to generate the output from pmix_cmd_line_get_usage_msg(). + * + */ + PMIX_EXPORT int pmix_cmd_line_make_opt3(pmix_cmd_line_t *cmd, + char short_name, + const char *sd_name, + const char *long_name, + int num_params, + const char *desc); + + /** + * Parse a command line according to a pre-built PMIX command line + * handle. + * + * @param cmd PMIX command line handle. + * @param ignore_unknown Whether to print an error message upon + * finding an unknown token or not + * @param ignore_unknown_option Whether to print an error message upon + * finding an unknown option or not + * @param argc Length of the argv array. + * @param argv Array of strings from the command line. + * + * @retval PMIX_SUCCESS Upon success. + * @retval PMIX_ERR_SILENT If an error message was printed. This + * value will only be returned if the command line was not + * successfully parsed. + * + * Parse a series of command line tokens according to the option + * descriptions from a PMIX command line handle. The PMIX command line + * handle can then be queried to see what options were used, what + * their parameters were, etc. + * + * If an unknown token is found in the command line (i.e., a token + * that is not a parameter or a registered option), the parsing will + * stop (see below). If ignore_unknown is false, an error message + * is displayed. If ignore_unknown is true, the error message is + * not displayed. + * + * Error messages are always displayed regardless of the value + * of ignore_unknown (to stderr, and PMIX_ERR_SILENT is + * returned) if: + * + * 1. A token was encountered that required N parameters, but #include +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif #include "src/util/error.h" #include "src/util/fd.h" @@ -93,3 +99,31 @@ pmix_status_t pmix_fd_set_cloexec(int fd) return PMIX_SUCCESS; } + + +bool pmix_fd_is_regular(int fd) +{ + struct stat buf; + if (fstat(fd, &buf)) { + return false; + } + return S_ISREG(buf.st_mode); +} + +bool pmix_fd_is_chardev(int fd) +{ + struct stat buf; + if (fstat(fd, &buf)) { + return false; + } + return S_ISCHR(buf.st_mode); +} + +bool pmix_fd_is_blkdev(int fd) +{ + struct stat buf; + if (fstat(fd, &buf)) { + return false; + } + return S_ISBLK(buf.st_mode); +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/fd.h b/opal/mca/pmix/pmix2x/pmix/src/util/fd.h index d67fe248359..32fd4a31104 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/fd.h +++ b/opal/mca/pmix/pmix2x/pmix/src/util/fd.h @@ -1,7 +1,7 @@ /* * Copyright (c) 2008-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2009 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -67,6 +67,37 @@ PMIX_EXPORT pmix_status_t pmix_fd_write(int fd, int len, const void *buffer); */ PMIX_EXPORT pmix_status_t pmix_fd_set_cloexec(int fd); +/** + * Convenience function to check if fd point to an accessible regular file. + * + * @param fd File descriptor + * + * @returns true if "fd" points to a regular file. + * @returns false otherwise. + */ +PMIX_EXPORT bool pmix_fd_is_regular(int fd); + +/** + * Convenience function to check if fd point to an accessible character device. + * + * @param fd File descriptor + * + * @returns true if "fd" points to a regular file. + * @returns false otherwise. + */ +PMIX_EXPORT bool pmix_fd_is_chardev(int fd); + +/** + * Convenience function to check if fd point to an accessible block device. + * + * @param fd File descriptor + * + * @returns true if "fd" points to a regular file. + * @returns false otherwise. + */ +PMIX_EXPORT bool pmix_fd_is_blkdev(int fd); + + END_C_DECLS #endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/hash.c b/opal/mca/pmix/pmix2x/pmix/src/util/hash.c index 76d6c70723e..ec5fc1f62b9 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/hash.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/hash.c @@ -6,8 +6,8 @@ * reserved. * Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. @@ -172,7 +172,7 @@ pmix_status_t pmix_hash_fetch(pmix_hash_table_t *table, pmix_rank_t rank, /* copy the list elements */ n=0; PMIX_LIST_FOREACH(hv, &proc_data->data, pmix_kval_t) { - (void)strncpy(info[n].key, hv->key, PMIX_MAX_KEYLEN); + pmix_strncpy(info[n].key, hv->key, PMIX_MAX_KEYLEN); pmix_value_xfer(&info[n].value, hv->value); ++n; } @@ -282,7 +282,7 @@ pmix_status_t pmix_hash_remove_data(pmix_hash_table_t *table, /* if the rank is wildcard, we want to apply this to * all rank entries */ - if (PMIX_RANK_UNDEF == rank) { + if (PMIX_RANK_WILDCARD == rank) { rc = pmix_hash_table_get_first_key_uint64(table, &id, (void**)&proc_data, (void**)&node); while (PMIX_SUCCESS == rc) { @@ -302,6 +302,7 @@ pmix_status_t pmix_hash_remove_data(pmix_hash_table_t *table, rc = pmix_hash_table_get_next_key_uint64(table, &id, (void**)&proc_data, node, (void**)&node); } + return PMIX_SUCCESS; } /* lookup the specified proc */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/keyval_parse.c b/opal/mca/pmix/pmix2x/pmix/src/util/keyval_parse.c index c07e65e6681..70ac512c3ab 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/keyval_parse.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/keyval_parse.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -136,7 +136,7 @@ static int parse_line(void) key_buffer = tmp; } - strncpy(key_buffer, pmix_util_keyval_yytext, key_buffer_len); + pmix_strncpy(key_buffer, pmix_util_keyval_yytext, key_buffer_len-1); /* The first thing we have to see is an "=" */ @@ -259,7 +259,7 @@ static int save_param_name (void) key_buffer = tmp; } - strncpy (key_buffer, pmix_util_keyval_yytext, key_buffer_len); + pmix_strncpy (key_buffer, pmix_util_keyval_yytext, key_buffer_len-1); return PMIX_SUCCESS; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/keyval_parse.h b/opal/mca/pmix/pmix2x/pmix/src/util/keyval_parse.h index 2d6e9f4c4e1..276c9717bda 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/keyval_parse.h +++ b/opal/mca/pmix/pmix2x/pmix/src/util/keyval_parse.h @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -26,7 +26,7 @@ BEGIN_C_DECLS -extern int pmix_util_keyval_parse_lineno; +PMIX_EXPORT extern int pmix_util_keyval_parse_lineno; /** * Callback triggered for each key = value pair @@ -46,14 +46,14 @@ typedef void (*pmix_keyval_parse_fn_t)(const char *key, const char *value); * called exactly once. In a multithreaded context, calls to * pmix_util_keyval_parse() will serialize multiple calls. */ -int pmix_util_keyval_parse(const char *filename, - pmix_keyval_parse_fn_t callback); +PMIX_EXPORT int pmix_util_keyval_parse(const char *filename, + pmix_keyval_parse_fn_t callback); -int pmix_util_keyval_parse_init(void); +PMIX_EXPORT int pmix_util_keyval_parse_init(void); -int pmix_util_keyval_parse_finalize(void); +PMIX_EXPORT int pmix_util_keyval_parse_finalize(void); -int pmix_util_keyval_save_internal_envars(pmix_keyval_parse_fn_t callback); +PMIX_EXPORT int pmix_util_keyval_save_internal_envars(pmix_keyval_parse_fn_t callback); END_C_DECLS diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/name_fns.c b/opal/mca/pmix/pmix2x/pmix/src/util/name_fns.c new file mode 100644 index 00000000000..ca00f4f8c82 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/util/name_fns.c @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014-2016 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "pmix_config.h" + +#include +#include + +#include "pmix_common.h" + +#include "src/threads/tsd.h" +#include "src/util/error.h" +#include "src/util/name_fns.h" +#include "src/util/printf.h" + +#define PMIX_PRINT_NAME_ARGS_MAX_SIZE 300 +#define PMIX_PRINT_NAME_ARG_NUM_BUFS 16 + +#define PMIX_SCHEMA_DELIMITER_CHAR '.' +#define PMIX_SCHEMA_WILDCARD_CHAR '*' +#define PMIX_SCHEMA_WILDCARD_STRING "*" +#define PMIX_SCHEMA_INVALID_CHAR '$' +#define PMIX_SCHEMA_INVALID_STRING "$" + +static bool fns_init=false; + +static pmix_tsd_key_t print_args_tsd_key; +char* pmix_print_args_null = "NULL"; +typedef struct { + char *buffers[PMIX_PRINT_NAME_ARG_NUM_BUFS]; + int cntr; +} pmix_print_args_buffers_t; + +static void +buffer_cleanup(void *value) +{ + int i; + pmix_print_args_buffers_t *ptr; + + if (NULL != value) { + ptr = (pmix_print_args_buffers_t*)value; + for (i=0; i < PMIX_PRINT_NAME_ARG_NUM_BUFS; i++) { + free(ptr->buffers[i]); + } + free (ptr); + } +} + +static pmix_print_args_buffers_t* +get_print_name_buffer(void) +{ + pmix_print_args_buffers_t *ptr; + int ret, i; + + if (!fns_init) { + /* setup the print_args function */ + if (PMIX_SUCCESS != (ret = pmix_tsd_key_create(&print_args_tsd_key, buffer_cleanup))) { + PMIX_ERROR_LOG(ret); + return NULL; + } + fns_init = true; + } + + ret = pmix_tsd_getspecific(print_args_tsd_key, (void**)&ptr); + if (PMIX_SUCCESS != ret) return NULL; + + if (NULL == ptr) { + ptr = (pmix_print_args_buffers_t*)malloc(sizeof(pmix_print_args_buffers_t)); + for (i=0; i < PMIX_PRINT_NAME_ARG_NUM_BUFS; i++) { + ptr->buffers[i] = (char *) malloc((PMIX_PRINT_NAME_ARGS_MAX_SIZE+1) * sizeof(char)); + } + ptr->cntr = 0; + ret = pmix_tsd_setspecific(print_args_tsd_key, (void*)ptr); + } + + return (pmix_print_args_buffers_t*) ptr; +} + +char* pmix_util_print_name_args(const pmix_proc_t *name) +{ + pmix_print_args_buffers_t *ptr; + char *rank; + + /* get the next buffer */ + ptr = get_print_name_buffer(); + if (NULL == ptr) { + PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); + return pmix_print_args_null; + } + /* cycle around the ring */ + if (PMIX_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) { + ptr->cntr = 0; + } + + /* protect against NULL names */ + if (NULL == name) { + snprintf(ptr->buffers[ptr->cntr++], PMIX_PRINT_NAME_ARGS_MAX_SIZE, "[NO-NAME]"); + return ptr->buffers[ptr->cntr-1]; + } + + rank = pmix_util_print_rank(name->rank); + + snprintf(ptr->buffers[ptr->cntr++], + PMIX_PRINT_NAME_ARGS_MAX_SIZE, + "[%s,%s]", name->nspace, rank); + + return ptr->buffers[ptr->cntr-1]; +} + +char* pmix_util_print_rank(const pmix_rank_t vpid) +{ + pmix_print_args_buffers_t *ptr; + + ptr = get_print_name_buffer(); + + if (NULL == ptr) { + PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); + return pmix_print_args_null; + } + + /* cycle around the ring */ + if (PMIX_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) { + ptr->cntr = 0; + } + + if (PMIX_RANK_UNDEF == vpid) { + snprintf(ptr->buffers[ptr->cntr++], PMIX_PRINT_NAME_ARGS_MAX_SIZE, "UNDEF"); + } else if (PMIX_RANK_WILDCARD == vpid) { + snprintf(ptr->buffers[ptr->cntr++], PMIX_PRINT_NAME_ARGS_MAX_SIZE, "WILDCARD"); + } else { + snprintf(ptr->buffers[ptr->cntr++], + PMIX_PRINT_NAME_ARGS_MAX_SIZE, + "%ld", (long)vpid); + } + return ptr->buffers[ptr->cntr-1]; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/name_fns.h b/opal/mca/pmix/pmix2x/pmix/src/util/name_fns.h new file mode 100644 index 00000000000..16b0b5ecbde --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/util/name_fns.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014-2016 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef _PMIX_NAME_FNS_H_ +#define _PMIX_NAME_FNS_H_ + +#include "pmix_config.h" + +#ifdef HAVE_STDINT_h +#include +#endif + +#include "pmix_common.h" + +BEGIN_C_DECLS + +/* useful define to print name args in output messages */ +PMIX_EXPORT char* pmix_util_print_name_args(const pmix_proc_t *name); +#define PMIX_NAME_PRINT(n) \ + pmix_util_print_name_args(n) + +PMIX_EXPORT char* pmix_util_print_rank(const pmix_rank_t vpid); +#define PMIX_RANK_PRINT(n) \ + pmix_util_print_rank(n) + + +END_C_DECLS +#endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/net.c b/opal/mca/pmix/pmix2x/pmix/src/util/net.c index b074a14dfae..11647276e7b 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/net.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/net.c @@ -12,7 +12,7 @@ * Copyright (c) 2007 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2013 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -50,23 +50,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include @@ -79,7 +63,7 @@ #include "src/util/output.h" #include "src/util/argv.h" #include "src/util/show_help.h" -#include "src/util/tsd.h" +#include "src/threads/tsd.h" #include "src/runtime/pmix_rte.h" /* this function doesn't depend on sockaddr_h */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/os_path.c b/opal/mca/pmix/pmix2x/pmix/src/util/os_path.c index 6abedaf8a30..b7ac618702f 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/os_path.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/os_path.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -34,7 +34,7 @@ static const char *path_sep = PMIX_PATH_SEP; -char *pmix_os_path(bool relative, ...) +char *pmix_os_path(int relative, ...) { va_list ap; char *element, *path; diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/os_path.h b/opal/mca/pmix/pmix2x/pmix/src/util/os_path.h index 9f3c71bfb57..d71dbec7715 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/os_path.h +++ b/opal/mca/pmix/pmix2x/pmix/src/util/os_path.h @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -64,8 +64,13 @@ BEGIN_C_DECLS * provided path elements, separated by the path separator character * appropriate to the local operating system. The path_name string has been malloc'd * and therefore the user is responsible for free'ing the field. + * + * Note that the "relative" argument is int instead of bool, because + * passing a parameter that undergoes default argument promotion to + * va_start() has undefined behavior (according to clang warnings on + * MacOS High Sierra). */ -PMIX_EXPORT char *pmix_os_path(bool relative, ...) __pmix_attribute_malloc__ __pmix_attribute_sentinel__ __pmix_attribute_warn_unused_result__; +PMIX_EXPORT char *pmix_os_path(int relative, ...) __pmix_attribute_malloc__ __pmix_attribute_sentinel__ __pmix_attribute_warn_unused_result__; /** * Convert the path to be OS friendly. On UNIX this function will diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/output.c b/opal/mca/pmix/pmix2x/pmix/src/util/output.c index 1d3d4148dcd..5441cf9c12c 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/output.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/output.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -694,7 +694,7 @@ static int open_file(int i) if (NULL == filename) { return PMIX_ERR_OUT_OF_RESOURCE; } - strncpy(filename, output_dir, PMIX_PATH_MAX); + pmix_strncpy(filename, output_dir, PMIX_PATH_MAX-1); strcat(filename, "/"); if (NULL != output_prefix) { strcat(filename, output_prefix); diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/path.c b/opal/mca/pmix/pmix2x/pmix/src/util/path.c index 6903d10df61..6fe5759309b 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/path.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/path.c @@ -13,7 +13,7 @@ * Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. * All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 University of Houston. All rights reserved. * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -402,6 +402,7 @@ char* pmix_find_absolute_path( char* app_name ) char* resolved_path = (char*)malloc(PMIX_PATH_MAX); if (NULL == realpath( abs_app_name, resolved_path )) { free(resolved_path); + free(abs_app_name); return NULL; } if( abs_app_name != app_name ) { diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/pif.c b/opal/mca/pmix/pmix2x/pmix/src/util/pif.c index 2f58f0203ab..9949b5e3b8b 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/pif.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/pif.c @@ -16,7 +16,7 @@ * reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -51,23 +51,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include @@ -218,7 +202,7 @@ int pmix_ifaddrtoname(const char* if_addr, char* if_name, int length) memcpy (&ipv4, r->ai_addr, r->ai_addrlen); if (inaddr->sin_addr.s_addr == ipv4.sin_addr.s_addr) { - strncpy(if_name, intf->if_name, length); + pmix_strncpy(if_name, intf->if_name, length-1); freeaddrinfo (res); return PMIX_SUCCESS; } @@ -226,7 +210,7 @@ int pmix_ifaddrtoname(const char* if_addr, char* if_name, int length) else { if (IN6_ARE_ADDR_EQUAL(&((struct sockaddr_in6*) &intf->if_addr)->sin6_addr, &((struct sockaddr_in6*) r->ai_addr)->sin6_addr)) { - strncpy(if_name, intf->if_name, length); + pmix_strncpy(if_name, intf->if_name, length-1); freeaddrinfo (res); return PMIX_SUCCESS; } @@ -269,13 +253,13 @@ int16_t pmix_ifaddrtokindex(const char* if_addr) for (r = res; r != NULL; r = r->ai_next) { PMIX_LIST_FOREACH(intf, &pmix_if_list, pmix_pif_t) { if (AF_INET == r->ai_family && AF_INET == intf->af_family) { - struct sockaddr ipv4, intv4; - memset(&ipv4, 0, sizeof(struct sockaddr)); + struct sockaddr_in ipv4, intv4; + memset(&ipv4, 0, sizeof(struct sockaddr_in)); len = (r->ai_addrlen < sizeof(struct sockaddr_in)) ? r->ai_addrlen : sizeof(struct sockaddr_in); memcpy(&ipv4, r->ai_addr, len); - memset(&intv4, 0, sizeof(struct sockaddr)); - memcpy(&intv4, &intf->if_addr, sizeof(struct sockaddr)); - if (pmix_net_samenetwork(&ipv4, &intv4, intf->if_mask)) { + memset(&intv4, 0, sizeof(struct sockaddr_in)); + memcpy(&intv4, &intf->if_addr, sizeof(struct sockaddr_in)); + if (pmix_net_samenetwork((struct sockaddr*)&ipv4, (struct sockaddr*)&intv4, intf->if_mask)) { if_kernel_index = intf->if_kernel_index; freeaddrinfo (res); return if_kernel_index; @@ -493,7 +477,7 @@ int pmix_ifindextoname(int if_index, char* if_name, int length) intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list); intf = (pmix_pif_t*)pmix_list_get_next(intf)) { if (intf->if_index == if_index) { - strncpy(if_name, intf->if_name, length); + pmix_strncpy(if_name, intf->if_name, length-1); return PMIX_SUCCESS; } } @@ -514,7 +498,7 @@ int pmix_ifkindextoname(int if_kindex, char* if_name, int length) intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list); intf = (pmix_pif_t*)pmix_list_get_next(intf)) { if (intf->if_kernel_index == if_kindex) { - strncpy(if_name, intf->if_name, length); + pmix_strncpy(if_name, intf->if_name, length-1); return PMIX_SUCCESS; } } diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/show_help.c b/opal/mca/pmix/pmix2x/pmix/src/util/show_help.c index 1654d39ead3..3e9b8b44ddd 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/show_help.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/show_help.c @@ -12,7 +12,7 @@ * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -49,9 +49,9 @@ static char **search_dirs = NULL; * Local functions */ static int pmix_show_vhelp_internal(const char *filename, const char *topic, - bool want_error_header, va_list arglist); + int want_error_header, va_list arglist); static int pmix_show_help_internal(const char *filename, const char *topic, - bool want_error_header, ...); + int want_error_header, ...); pmix_show_help_fn_t pmix_show_help = pmix_show_help_internal; pmix_show_vhelp_fn_t pmix_show_vhelp = pmix_show_vhelp_internal; @@ -90,7 +90,7 @@ int pmix_show_help_finalize(void) * not optimization. :-) */ static int array2string(char **outstring, - bool want_error_header, char **lines) + int want_error_header, char **lines) { int i, count; size_t len; @@ -298,7 +298,7 @@ static int load_array(char ***array, const char *filename, const char *topic) } char *pmix_show_help_vstring(const char *filename, const char *topic, - bool want_error_header, va_list arglist) + int want_error_header, va_list arglist) { int rc; char *single_string, *output, **array = NULL; @@ -324,7 +324,7 @@ char *pmix_show_help_vstring(const char *filename, const char *topic, } char *pmix_show_help_string(const char *filename, const char *topic, - bool want_error_handler, ...) + int want_error_handler, ...) { char *output; va_list arglist; @@ -338,7 +338,7 @@ char *pmix_show_help_string(const char *filename, const char *topic, } static int pmix_show_vhelp_internal(const char *filename, const char *topic, - bool want_error_header, va_list arglist) + int want_error_header, va_list arglist) { char *output; @@ -356,7 +356,7 @@ static int pmix_show_vhelp_internal(const char *filename, const char *topic, } static int pmix_show_help_internal(const char *filename, const char *topic, - bool want_error_header, ...) + int want_error_header, ...) { va_list arglist; int rc; diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/show_help.h b/opal/mca/pmix/pmix2x/pmix/src/util/show_help.h index 8c23887dc38..5f753427937 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/show_help.h +++ b/opal/mca/pmix/pmix2x/pmix/src/util/show_help.h @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -129,9 +129,14 @@ PMIX_EXPORT int pmix_show_help_finalize(void); * (typically $prefix/share/pmix), and looks up the message * based on the topic, and displays it. If want_error_header is * true, a header and footer of asterisks are also displayed. + * + * Note that the "want_error_header" argument is int instead of bool, + * because passing a parameter that undergoes default argument + * promotion to va_start() has undefined behavior (according to clang + * warnings on MacOS High Sierra). */ typedef int (*pmix_show_help_fn_t)(const char *filename, const char *topic, - bool want_error_header, ...); + int want_error_header, ...); PMIX_EXPORT extern pmix_show_help_fn_t pmix_show_help; /** @@ -139,7 +144,7 @@ PMIX_EXPORT extern pmix_show_help_fn_t pmix_show_help; * a va_list form of varargs. */ typedef int (*pmix_show_vhelp_fn_t)(const char *filename, const char *topic, - bool want_error_header, va_list ap); + int want_error_header, va_list ap); PMIX_EXPORT extern pmix_show_vhelp_fn_t pmix_show_vhelp; /** @@ -148,7 +153,7 @@ PMIX_EXPORT extern pmix_show_vhelp_fn_t pmix_show_vhelp; */ PMIX_EXPORT char* pmix_show_help_string(const char *filename, const char *topic, - bool want_error_header, ...); + int want_error_header, ...); /** * This function does the same thing as pmix_show_help_string(), but @@ -156,7 +161,7 @@ PMIX_EXPORT char* pmix_show_help_string(const char *filename, */ PMIX_EXPORT char* pmix_show_help_vstring(const char *filename, const char *topic, - bool want_error_header, va_list ap); + int want_error_header, va_list ap); /** * This function adds another search location for the files that diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/tsd.h b/opal/mca/pmix/pmix2x/pmix/src/util/tsd.h deleted file mode 100644 index 1149fca10e0..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/src/util/tsd.h +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright (c) 2007-2013 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#ifndef PMIX_THREADS_TSD_H -#define PMIX_THREADS_TSD_H - -#include "pmix_config.h" -#include "pmix_common.h" - -#include - - -BEGIN_C_DECLS - -/** - * @file - * - * Thread Specific Datastore Interface - * - * Functions for providing thread-specific datastore capabilities. - */ - - -/** - * Prototype for callback when tsd data is being destroyed - */ -typedef void (*pmix_tsd_destructor_t)(void *value); - -#if defined(DOXYGEN) - -/** - * Typedef for thread-specific data key - */ -typedef void* pmix_tsd_key_t; - - -/** - * Create thread-specific data key - * - * Create a thread-specific data key visible to all threads in the - * current process. The returned key is valid in all threads, - * although the values bound to the key by pmix_tsd_setspecific() are - * allocated on a per-thread basis and persist for the life of the - * calling thread. - * - * Upon key creation, the value NULL is associated with the new key in - * all active threads. When a new thread is created, the value NULL - * is associated with all defined keys in the new thread. - * - * The destructor parameter may be NULL. At thread exit, if - * destructor is non-NULL AND the thread has a non-NULL value - * associated with the key, the function is called with the current - * value as its argument. - * - * @param key[out] The key for accessing thread-specific data - * @param destructor[in] Cleanup function to call when a thread exits - * - * @retval PMIX_SUCCESS Success - * @retval EAGAIN The system lacked the necessary resource to - * create another thread specific data key - * @retval ENOMEM Insufficient memory exists to create the key - */ -PMIX_EXPORT int pmix_tsd_key_create(pmix_tsd_key_t *key, - pmix_tsd_destructor_t destructor); - - -/** - * Delete a thread-specific data key - * - * Delete a thread-specific data key previously returned by - * pmix_tsd_key_create(). The destructor associated with the key is - * not fired in any thread and memory cleanup is the responsibility of - * the caller. - * - * @note Unlike pthread_key_delete, this function should not be called - * from within a destructor. It can not be universally supported at - * this time. - * - * @param key[in] The key for accessing thread-specific data - * - * @retval PMIX_SUCCESS Success - * @retval EINVAL Invalid key - */ -PMIX_EXPORT int pmix_tsd_key_delete(pmix_tsd_key_t key); - - -/** - * Set a thread-specific data value - * - * Associates value with key in the current thread. The value for the - * key in other threads is not changed. Different threads may assign - * different values to the same key. - * - * @note This function should not be called within - * pmix_tsd_key_delete(). - * - * @param key[in] Thread specific data key to modify - * @param value[in] Value to associate with key - * - * @retval PMIX_SUCCESS Success - * @retval ENOMEM Insufficient memory exists to associate the - * value with the key - * @retval EINVAL Invalid key - */ -PMIX_EXPORT int pmix_tsd_setspecific(pmix_tsd_key_t key, void *value); - - -/** - * Get a thread-specific data value - * - * Get the data associated with the given key, as set by - * pmix_tsd_setspecific(). If pmix_tsd_setspecific() hasn't been - * called in the current thread with the given key, NULL is returned - * in valuep. - * - * @param key[in] Thread specific data key to modify - * @param value[out] Value to associate with key - * - * @retval PMIX_SUCCESS Success - * @retval ENOMEM Insufficient memory exists to associate the - * value with the key - * @retval EINVAL Invalid key - */ -PMIX_EXPORT int pmix_tsd_getspecific(pmix_tsd_key_t key, void **valuep); - -#else - -typedef pthread_key_t pmix_tsd_key_t; - -static inline int -pmix_tsd_key_create(pmix_tsd_key_t *key, - pmix_tsd_destructor_t destructor) -{ - return pthread_key_create(key, destructor); -} - -static inline int -pmix_tsd_key_delete(pmix_tsd_key_t key) -{ - return pthread_key_delete(key); -} - -static inline int -pmix_tsd_setspecific(pmix_tsd_key_t key, void *value) -{ - return pthread_setspecific(key, value); -} - -static inline int -pmix_tsd_getspecific(pmix_tsd_key_t key, void **valuep) -{ - *valuep = pthread_getspecific(key); - return PMIX_SUCCESS; -} - -#endif - -END_C_DECLS - -#endif /* PMIX_MTHREADS_TSD_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/test/Makefile.am b/opal/mca/pmix/pmix2x/pmix/test/Makefile.am index 9140989a4e9..fce6c8599ec 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/test/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/test/cli_stages.c b/opal/mca/pmix/pmix2x/pmix/test/cli_stages.c index 04049ad9cc3..593bc319ae1 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/cli_stages.c +++ b/opal/mca/pmix/pmix2x/pmix/test/cli_stages.c @@ -1,8 +1,10 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Mellanox Technologies, Inc. + * All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -21,18 +23,27 @@ int cli_rank(cli_info_t *cli) int i; for(i=0; i < cli_info_cnt; i++){ if( cli == &cli_info[i] ){ - return i; + return cli->rank; } } return -1; } -void cli_init(int nprocs, cli_state_t order[]) +void cli_init(int nprocs) { int n, i; + cli_state_t order[CLI_TERM+1]; + cli_info = malloc( sizeof(cli_info_t) * nprocs); cli_info_cnt = nprocs; + order[CLI_UNINIT] = CLI_FORKED; + order[CLI_FORKED] = CLI_FIN; + order[CLI_CONNECTED] = CLI_UNDEF; + order[CLI_FIN] = CLI_TERM; + order[CLI_DISCONN] = CLI_UNDEF; + order[CLI_TERM] = CLI_UNDEF; + for (n=0; n < nprocs; n++) { cli_info[n].sd = -1; cli_info[n].ev = NULL; @@ -198,8 +209,9 @@ void cli_wait_all(double timeout) TEST_VERBOSE(("waitpid = %d", pid)); for(i=0; i < cli_info_cnt; i++){ if( cli_info[i].pid == pid ){ - TEST_VERBOSE(("the child with pid = %d has rank = %d\n" - "\t\texited = %d, signalled = %d", pid, i, + TEST_VERBOSE(("the child with pid = %d has rank = %d, ns = %s\n" + "\t\texited = %d, signalled = %d", pid, + cli_info[i].rank, cli_info[i].ns, WIFEXITED(status), WIFSIGNALED(status) )); if( WIFEXITED(status) || WIFSIGNALED(status) ){ cli_cleanup(&cli_info[i]); @@ -211,6 +223,9 @@ void cli_wait_all(double timeout) if( errno == ECHILD ){ TEST_VERBOSE(("No more children to wait. Happens on the last cli_wait_all call " "which is used to ensure that all children terminated.\n")); + if (pmix_test_verbose) { + sleep(1); + } break; } else { TEST_ERROR(("waitpid(): %d : %s", errno, strerror(errno))); diff --git a/opal/mca/pmix/pmix2x/pmix/test/cli_stages.h b/opal/mca/pmix/pmix2x/pmix/test/cli_stages.h index 343af2de043..5a36fd90303 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/cli_stages.h +++ b/opal/mca/pmix/pmix2x/pmix/test/cli_stages.h @@ -1,8 +1,10 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Mellanox Technologies, Inc. + * All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -10,6 +12,9 @@ * $HEADER$ */ +#ifndef CLI_STAGES_H +#define CLI_STAGES_H + #include #include #include @@ -41,7 +46,7 @@ typedef struct { pmix_event_t *ev; cli_state_t state; cli_state_t next_state[CLI_TERM+1]; - int rank; + pmix_rank_t rank; char *ns; } cli_info_t; @@ -50,7 +55,7 @@ extern int cli_info_cnt; extern bool test_abort; int cli_rank(cli_info_t *cli); -void cli_init(int nprocs, cli_state_t order[]); +void cli_init(int nprocs); void cli_connect(cli_info_t *cli, int sd, struct event_base * ebase, event_callback_fn callback); void cli_finalize(cli_info_t *cli); void cli_disconnect(cli_info_t *cli); @@ -75,3 +80,5 @@ void op_callbk(pmix_status_t status, void errhandler_reg_callbk (pmix_status_t status, size_t errhandler_ref, void *cbdata); + +#endif // CLI_STAGES_H diff --git a/opal/mca/pmix/pmix2x/pmix/test/pmix_client.c b/opal/mca/pmix/pmix2x/pmix/test/pmix_client.c index e00a44e22e2..08caa7ce418 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/pmix_client.c +++ b/opal/mca/pmix/pmix2x/pmix/test/pmix_client.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015-2017 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * diff --git a/opal/mca/pmix/pmix2x/pmix/test/pmix_regex.c b/opal/mca/pmix/pmix2x/pmix/test/pmix_regex.c index cb347f704a5..1c1e5e97705 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/pmix_regex.c +++ b/opal/mca/pmix/pmix2x/pmix/test/pmix_regex.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/test/pmix_test.c b/opal/mca/pmix/pmix2x/pmix/test/pmix_test.c index c1a8130b1a4..3285d8bfa12 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/pmix_test.c +++ b/opal/mca/pmix/pmix2x/pmix/test/pmix_test.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. @@ -47,7 +47,6 @@ int main(int argc, char **argv) struct stat stat_buf; struct timeval tv; double test_start; - cli_state_t order[CLI_TERM+1]; test_params params; INIT_TEST_PARAMS(params); int test_fail = 0; @@ -98,13 +97,7 @@ int main(int argc, char **argv) PMIx_Register_event_handler(NULL, 0, NULL, 0, errhandler, errhandler_reg_callbk, NULL); - order[CLI_UNINIT] = CLI_FORKED; - order[CLI_FORKED] = CLI_FIN; - order[CLI_CONNECTED] = CLI_UNDEF; - order[CLI_FIN] = CLI_TERM; - order[CLI_DISCONN] = CLI_UNDEF; - order[CLI_TERM] = CLI_UNDEF; - cli_init(params.nprocs, order); + cli_init(params.nprocs); /* set common argv and env */ client_env = pmix_argv_copy(environ); diff --git a/opal/mca/pmix/pmix2x/pmix/test/server_callbacks.c b/opal/mca/pmix/pmix2x/pmix/test/server_callbacks.c index be6b06754f6..01766d2b318 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/server_callbacks.c +++ b/opal/mca/pmix/pmix2x/pmix/test/server_callbacks.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. @@ -94,12 +94,25 @@ pmix_status_t connected(const pmix_proc_t *proc, void *server_object, pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, pmix_op_cbfunc_t cbfunc, void *cbdata) { - if( CLI_TERM <= cli_info[proc->rank].state ){ + cli_info_t *cli = NULL; + int i; + for (i = 0; i < cli_info_cnt; i++) { + if((proc->rank == cli_info[i].rank) && + (0 == strcmp(proc->nspace, cli_info[i].ns))){ + cli = &cli_info[i]; + break; + } + } + if (NULL == cli) { + TEST_ERROR(("cannot found rank %d", proc->rank)); + return PMIX_SUCCESS; + } + if( CLI_TERM <= cli->state ){ TEST_ERROR(("double termination of rank %d", proc->rank)); return PMIX_SUCCESS; } - TEST_VERBOSE(("Rank %d terminated", proc->rank)); - cli_finalize(&cli_info[proc->rank]); + TEST_VERBOSE(("Rank %s:%d terminated", proc->nspace, proc->rank)); + cli_finalize(cli); finalized_count++; if (finalized_count == cli_info_cnt) { if (NULL != pmix_test_published_list) { @@ -311,10 +324,9 @@ pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, pmix_op_cbfunc_t cbfunc, void *cbdata) { if (NULL != cbfunc) { - /* return PMIX_EXISTS here just to ensure we get the correct status on the client */ - cbfunc(PMIX_EXISTS, cbdata); + cbfunc(PMIX_SUCCESS, cbdata); } - return PMIX_SUCCESS; + return PMIX_SUCCESS; } pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, diff --git a/opal/mca/pmix/pmix2x/pmix/test/server_callbacks.h b/opal/mca/pmix/pmix2x/pmix/test/server_callbacks.h index df0d5ca5776..eef43f9666c 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/server_callbacks.h +++ b/opal/mca/pmix/pmix2x/pmix/test/server_callbacks.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/Makefile.am b/opal/mca/pmix/pmix2x/pmix/test/simple/Makefile.am index 198e8f4891d..731fb5d7d86 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/simple/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -24,77 +24,82 @@ AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_buildd headers = simptest.h noinst_PROGRAMS = simptest simpclient simppub simpdyn simpft simpdmodex \ - test_pmix simptool simpdie simplegacy stability quietclient \ - test_pmix simptool simpdie simplegacy \ + test_pmix simptool simpdie simplegacy simptimeout \ stability quietclient simpjctrl -simptest_SOURCES = $(headers) \ +simptest_SOURCES = \ simptest.c simptest_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) simptest_LDADD = \ $(top_builddir)/src/libpmix.la -simpclient_SOURCES = $(headers) \ +simpclient_SOURCES = \ simpclient.c simpclient_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) simpclient_LDADD = \ $(top_builddir)/src/libpmix.la -simppub_SOURCES = $(headers) \ +simppub_SOURCES = \ simppub.c simppub_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) simppub_LDADD = \ $(top_builddir)/src/libpmix.la -simpdmodex_SOURCES = $(headers) \ +simpdmodex_SOURCES = \ simpdmodex.c simpdmodex_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) simpdmodex_LDADD = \ $(top_builddir)/src/libpmix.la -simpft_SOURCES = $(headers) \ +simpft_SOURCES = \ simpft.c simpft_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) simpft_LDADD = \ $(top_builddir)/src/libpmix.la -simpdyn_SOURCES = $(headers) \ +simpdyn_SOURCES = \ simpdyn.c simpdyn_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) simpdyn_LDADD = \ $(top_builddir)/src/libpmix.la -test_pmix_SOURCES = $(headers) \ +test_pmix_SOURCES = \ test_pmix.c test_pmix_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) test_pmix_LDADD = \ $(top_builddir)/src/libpmix.la -simptool_SOURCES = $(headers) \ +simptool_SOURCES = \ simptool.c simptool_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) simptool_LDADD = \ $(top_builddir)/src/libpmix.la -simpdie_SOURCES = $(headers) \ +simpdie_SOURCES = \ simpdie.c simpdie_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) simpdie_LDADD = \ $(top_builddir)/src/libpmix.la -simplegacy_SOURCES = $(headers) \ +simplegacy_SOURCES = \ simplegacy.c simplegacy_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) simplegacy_LDADD = \ $(top_builddir)/src/libpmix.la -stability_SOURCES = $(headers) \ +simptimeout_SOURCES = \ + simptimeout.c +simptimeout_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) +simptimeout_LDADD = \ + $(top_builddir)/src/libpmix.la + +stability_SOURCES = \ stability.c stability_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) stability_LDADD = \ $(top_builddir)/src/libpmix.la -quietclient_SOURCES = $(headers) \ +quietclient_SOURCES = \ quietclient.c quietclient_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) quietclient_LDADD = \ diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/simpclient.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simpclient.c index 4bc66e5d9ea..710ae4b7b60 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/simple/simpclient.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simpclient.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -163,6 +163,15 @@ int main(int argc, char **argv) } PMIX_VALUE_RELEASE(val); + /* test something */ + if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, PMIX_SERVER_URI, NULL, 0, &val))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Get failed: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + exit(rc); + } + pmix_output(0, "CLIENT SERVER URI: %s", val->data.string); + PMIX_VALUE_RELEASE(val); + /* register a handler specifically for when models declare */ active = true; ninfo = 1; @@ -184,6 +193,7 @@ int main(int argc, char **argv) usleep(10); } + /* get our universe size */ (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; @@ -320,13 +330,16 @@ int main(int argc, char **argv) /* log something */ PMIX_INFO_CONSTRUCT(&info); - (void)strncpy(info.key, "foobar", PMIX_MAX_KEYLEN); - info.value.type = PMIX_BOOL; - info.value.data.flag = true; + PMIX_INFO_LOAD(&info, PMIX_LOG_STDERR, "test log msg", PMIX_STRING); active = true; - PMIx_Log_nb(&info, 1, NULL, 0, opcbfunc, (void*)&active); - while (active) { - usleep(10); + rc = PMIx_Log_nb(&info, 1, NULL, 0, opcbfunc, (void*)&active); + if (PMIX_SUCCESS != rc) { + pmix_output(0, "Client ns %s rank %d - log_nb returned %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + } else { + while (active) { + usleep(10); + } } PMIX_INFO_DESTRUCT(&info); diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/simpdie.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simpdie.c index db62d7832c4..7e0e98fca5c 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/simple/simpdie.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simpdie.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -87,7 +87,9 @@ int main(int argc, char **argv) pmix_value_t *val = &value; pmix_proc_t proc; uint32_t nprocs; - + pmix_status_t code[5] = {PMIX_ERR_PROC_ABORTING, PMIX_ERR_PROC_ABORTED, + PMIX_ERR_PROC_REQUESTED_ABORT, PMIX_ERR_JOB_TERMINATED, + PMIX_ERR_UNREACH}; /* init us */ if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %d", myproc.nspace, myproc.rank, rc); @@ -108,7 +110,7 @@ int main(int argc, char **argv) completed = false; /* register our errhandler */ - PMIx_Register_event_handler(NULL, 0, NULL, 0, + PMIx_Register_event_handler(code, 5, NULL, 0, notification_fn, errhandler_reg_callbk, NULL); /* call fence to sync */ diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/simpdmodex.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simpdmodex.c index 2151caf2b33..56f71815bce 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/simple/simpdmodex.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simpdmodex.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -84,7 +84,7 @@ static void valcbfunc(pmix_status_t status, } pmix_output(0, "%s:%d PMIx_Get_nb Key %s returned correctly", myproc.nspace, myproc.rank, key); } else { - pmix_output(0, "%s:%d PMIx_Get_nb Key %s failed", myproc.nspace, myproc.rank, key); + pmix_output(0, "%s:%d PMIx_Get_nb Key %s failed: %s", myproc.nspace, myproc.rank, key, PMIx_Error_string(status)); } done: free(key); @@ -100,6 +100,11 @@ int main(int argc, char **argv) pmix_proc_t proc; uint32_t n, num_gets; bool active; + bool dofence = true; + + if (NULL != getenv("PMIX_SIMPDMODEX_ASYNC")) { + dofence = false; + } /* init us */ if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { @@ -139,7 +144,7 @@ int main(int argc, char **argv) (void)asprintf(&tmp, "%s-%d-remote", myproc.nspace, myproc.rank); value.type = PMIX_STRING; value.data.string = "1234"; - if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_REMOTE, tmp, &value))) { + if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_GLOBAL, tmp, &value))) { pmix_output(0, "Client ns %s rank %d: PMIx_Put internal failed: %d", myproc.nspace, myproc.rank, rc); goto done; } @@ -148,6 +153,7 @@ int main(int argc, char **argv) * if a "get" is received prior to data being provided */ if (0 == myproc.rank) { sleep(2); + pmix_output(0, "\n\n\nWOKE UP"); } /* commit the data to the server */ @@ -156,14 +162,16 @@ int main(int argc, char **argv) goto done; } - /* call fence_nb, but don't return any data */ - PMIX_PROC_CONSTRUCT(&proc); - (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); - proc.rank = PMIX_RANK_WILDCARD; - active = true; - if (PMIX_SUCCESS != (rc = PMIx_Fence_nb(&proc, 1, NULL, 0, opcbfunc, &active))) { - pmix_output(0, "Client ns %s rank %d: PMIx_Fence failed: %d", myproc.nspace, myproc.rank, rc); - goto done; + if (dofence) { + /* call fence_nb, but don't return any data */ + PMIX_PROC_CONSTRUCT(&proc); + (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + active = true; + if (PMIX_SUCCESS != (rc = PMIx_Fence_nb(&proc, 1, NULL, 0, opcbfunc, &active))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Fence failed: %d", myproc.nspace, myproc.rank, rc); + goto done; + } } /* get the committed data - ask for someone who doesn't exist as well */ @@ -186,8 +194,10 @@ int main(int argc, char **argv) ++num_gets; } - /* wait for the first fence to finish */ - PMIX_WAIT_FOR_COMPLETION(active); + if (dofence) { + /* wait for the first fence to finish */ + PMIX_WAIT_FOR_COMPLETION(active); + } /* wait for all my "get" calls to complete */ while (getcount < num_gets) { diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/simpdyn.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simpdyn.c index 2dfbc572174..3c8ff689283 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/simple/simpdyn.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simpdyn.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -132,7 +132,8 @@ int main(int argc, char **argv) pmix_output(0, "Client ns %s rank %d: PMIx_Connect failed: %d", myproc.nspace, myproc.rank, rc); goto done; } - pmix_output(0, "Client ns %s rank %d: PMIx_Connect succeeded", myproc.nspace, myproc.rank); + pmix_output(0, "Client ns %s rank %d: PMIx_Connect succeeded", + myproc.nspace, myproc.rank); if (PMIX_SUCCESS != (rc = PMIx_Disconnect(&proc, 1, NULL, 0))) { pmix_output(0, "Client ns %s rank %d: PMIx_Disonnect failed: %d", myproc.nspace, myproc.rank, rc); goto done; diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.c index fe6472e61fd..b6a08d5dbfe 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.c @@ -13,9 +13,9 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -46,6 +46,8 @@ #include "src/util/printf.h" #include "src/util/argv.h" +#include "simptest.h" + static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, pmix_op_cbfunc_t cbfunc, void *cbdata); static pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, @@ -147,25 +149,28 @@ PMIX_CLASS_INSTANCE(pmix_locdat_t, typedef struct { pmix_object_t super; - volatile bool active; + mylock_t lock; + pmix_event_t ev; pmix_proc_t caller; pmix_info_t *info; size_t ninfo; pmix_op_cbfunc_t cbfunc; pmix_spawn_cbfunc_t spcbfunc; + pmix_release_cbfunc_t relcbfunc; void *cbdata; } myxfer_t; static void xfcon(myxfer_t *p) { + DEBUG_CONSTRUCT_LOCK(&p->lock); p->info = NULL; p->ninfo = 0; - p->active = true; p->cbfunc = NULL; p->spcbfunc = NULL; p->cbdata = NULL; } static void xfdes(myxfer_t *p) { + DEBUG_DESTRUCT_LOCK(&p->lock); if (NULL != p->info) { PMIX_INFO_FREE(p->info, p->ninfo); } @@ -184,9 +189,12 @@ PMIX_CLASS_INSTANCE(wait_tracker_t, NULL, NULL); static volatile int wakeup; +static int exit_code = 0; static pmix_list_t pubdata; static pmix_event_t handler; static pmix_list_t children; +static bool istimeouttest = false; +static mylock_t globallock; static void set_namespace(int nprocs, char *ranks, char *nspace, pmix_op_cbfunc_t cbfunc, myxfer_t *x); @@ -210,9 +218,10 @@ static void opcbfunc(pmix_status_t status, void *cbdata) if (NULL != x->cbfunc) { x->cbfunc(PMIX_SUCCESS, x->cbdata); } - x->active = false; + DEBUG_WAKEUP_THREAD(&x->lock); } + /* this is an event notification function that we explicitly request * be called when the PMIX_MODEL_DECLARED notification is issued. * We could catch it in the general event notification function and test @@ -245,7 +254,7 @@ static void model_callback(size_t evhdlr_registration_id, if (NULL != cbfunc) { cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); } - wakeup = 0; + DEBUG_WAKEUP_THREAD(&globallock); } /* event handler registration is done asynchronously */ @@ -253,13 +262,14 @@ static void model_registration_callback(pmix_status_t status, size_t evhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; if (PMIX_SUCCESS != status) { fprintf(stderr, "simptest EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n", status, (unsigned long)evhandler_ref); } - *active = status; + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); } static void set_handler_default(int sig) @@ -289,8 +299,9 @@ int main(int argc, char **argv) size_t ninfo; bool cross_version = false; bool usock = true; - volatile int active; + mylock_t mylock; pmix_status_t code; + sigset_t unblock; /* smoke test */ if (PMIX_SUCCESS != 0) { @@ -298,8 +309,6 @@ int main(int argc, char **argv) exit(1); } - fprintf(stderr, "Testing version %s\n", PMIx_Get_version()); - /* see if we were passed the number of procs to run or * the executable to use */ for (n=1; n < argc; n++) { @@ -310,6 +319,10 @@ int main(int argc, char **argv) } else if (0 == strcmp("-e", argv[n]) && NULL != argv[n+1]) { executable = strdup(argv[n+1]); + /* check for timeout test */ + if (NULL != strstr(executable, "simptimeout")) { + istimeouttest = true; + } for (k=n+2; NULL != argv[k]; k++) { pmix_argv_append_nosize(&client_argv, argv[k]); } @@ -340,47 +353,61 @@ int main(int argc, char **argv) exit(1); } + fprintf(stderr, "Testing version %s\n", PMIx_Get_version()); + + /* ensure that SIGCHLD is unblocked as we need to capture it */ + if (0 != sigemptyset(&unblock)) { + fprintf(stderr, "SIGEMPTYSET FAILED\n"); + exit(1); + } + if (0 != sigaddset(&unblock, SIGCHLD)) { + fprintf(stderr, "SIGADDSET FAILED\n"); + exit(1); + } + if (0 != sigprocmask(SIG_UNBLOCK, &unblock, NULL)) { + fprintf(stderr, "SIG_UNBLOCK FAILED\n"); + exit(1); + } + + /* setup the server library and tell it to support tool connections */ - ninfo = 2; + ninfo = 1; PMIX_INFO_CREATE(info, ninfo); PMIX_INFO_LOAD(&info[0], PMIX_SERVER_TOOL_SUPPORT, NULL, PMIX_BOOL); - PMIX_INFO_LOAD(&info[1], PMIX_USOCK_DISABLE, &usock, PMIX_BOOL); - if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, 2))) { + if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, ninfo))) { fprintf(stderr, "Init failed with error %d\n", rc); return rc; } PMIX_INFO_FREE(info, ninfo); /* register the default errhandler */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); ninfo = 1; PMIX_INFO_CREATE(info, ninfo); PMIX_INFO_LOAD(&info[0], PMIX_EVENT_HDLR_NAME, "SIMPTEST-DEFAULT", PMIX_STRING); PMIx_Register_event_handler(NULL, 0, info, ninfo, - errhandler, errhandler_reg_callbk, (void*)&active); - while (-1 == active) { - usleep(10); - } + errhandler, errhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); PMIX_INFO_FREE(info, ninfo); - if (0 != active) { - exit(active); + if (PMIX_SUCCESS != mylock.status) { + exit(mylock.status); } + DEBUG_DESTRUCT_LOCK(&mylock); /* register a handler specifically for when models declare */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); ninfo = 1; PMIX_INFO_CREATE(info, ninfo); PMIX_INFO_LOAD(&info[0], PMIX_EVENT_HDLR_NAME, "SIMPTEST-MODEL", PMIX_STRING); code = PMIX_MODEL_DECLARED; PMIx_Register_event_handler(&code, 1, info, ninfo, - model_callback, model_registration_callback, (void*)&active); - while (-1 == active) { - usleep(10); - } + model_callback, model_registration_callback, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); PMIX_INFO_FREE(info, ninfo); - if (0 != active) { - exit(active); + if (PMIX_SUCCESS != mylock.status) { + exit(mylock.status); } + DEBUG_DESTRUCT_LOCK(&mylock); /* setup the pub data, in case it is used */ PMIX_CONSTRUCT(&pubdata, pmix_list_t); @@ -413,7 +440,7 @@ int main(int argc, char **argv) /* if the nspace registration hasn't completed yet, * wait for it here */ - PMIX_WAIT_FOR_COMPLETION(x->active); + DEBUG_WAIT_THREAD(&x->lock); free(tmp); PMIX_RELEASE(x); @@ -434,6 +461,9 @@ int main(int argc, char **argv) } else { pmix_setenv("PMIX_MCA_ptl", "usock", true, &client_env); } + } else if (!usock) { + /* don't disable usock => enable it on client */ + pmix_setenv("PMIX_MCA_ptl", "usock", true, &client_env); } x = PMIX_NEW(myxfer_t); if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid, @@ -444,7 +474,7 @@ int main(int argc, char **argv) } /* don't fork/exec the client until we know it is registered * so we avoid a potential race condition in the server */ - PMIX_WAIT_FOR_COMPLETION(x->active); + DEBUG_WAIT_THREAD(&x->lock); PMIX_RELEASE(x); pid = fork(); if (pid < 0) { @@ -470,7 +500,6 @@ int main(int argc, char **argv) pmix_list_append(&children, &child->super); } } - free(executable); pmix_argv_free(client_argv); pmix_argv_free(client_env); @@ -482,14 +511,20 @@ int main(int argc, char **argv) nanosleep(&ts, NULL); } - /* see if anyone exited with non-zero status */ - n=0; - PMIX_LIST_FOREACH(child, &children, wait_tracker_t) { - if (0 != child->exit_code) { - fprintf(stderr, "Child %d [%d] exited with status %d - test FAILED\n", n, child->pid, child->exit_code); - } - ++n; + /* see if anyone exited with non-zero status unless the test + * was expected to do so */ + if (NULL == strstr(executable, "simpdie")) { + n=0; + PMIX_LIST_FOREACH(child, &children, wait_tracker_t) { + if (0 != child->exit_code) { + fprintf(stderr, "Child %d [%d] exited with status %d - test FAILED\n", n, child->pid, child->exit_code); + } + ++n; + } + } else if (1 == exit_code) { + exit_code = 0; } + free(executable); /* try notifying ourselves */ ninfo = 3; @@ -498,29 +533,54 @@ int main(int argc, char **argv) PMIX_INFO_LOAD(&info[1], PMIX_MODEL_LIBRARY_NAME, "test", PMIX_STRING); /* mark that it is not to go to any default handlers */ PMIX_INFO_LOAD(&info[2], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL); - wakeup = -1; + DEBUG_CONSTRUCT_LOCK(&globallock); PMIx_Notify_event(PMIX_MODEL_DECLARED, &pmix_globals.myid, PMIX_RANGE_PROC_LOCAL, info, ninfo, NULL, NULL); - while (-1 == wakeup) { - usleep(10); - } + DEBUG_WAIT_THREAD(&globallock); + DEBUG_DESTRUCT_LOCK(&globallock); PMIX_INFO_FREE(info, ninfo); +#if 0 + fprintf(stderr, "TEST NONDEFAULT NOTIFICATION\n"); + /* verify that notifications don't recirculate */ + ninfo = 1; + PMIX_INFO_CREATE(info, ninfo); + /* mark that it is not to go to any default handlers */ + PMIX_INFO_LOAD(&info[0], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL); + PMIx_Notify_event(PMIX_ERR_DEBUGGER_RELEASE, + &pmix_globals.myid, PMIX_RANGE_LOCAL, + info, ninfo, NULL, NULL); + PMIX_INFO_FREE(info, ninfo); + /* wait a little in case we get notified */ + for (ninfo=0; ninfo < 100000; ninfo++) { + struct timespec t = {0, 100}; + nanosleep(&t, NULL); + } +#endif + /* deregister the event handlers */ PMIx_Deregister_event_handler(0, NULL, NULL); /* release any pub data */ PMIX_LIST_DESTRUCT(&pubdata); + /* release the child tracker */ + PMIX_LIST_DESTRUCT(&children); + /* finalize the server library */ if (PMIX_SUCCESS != (rc = PMIx_server_finalize())) { fprintf(stderr, "Finalize failed with error %d\n", rc); + exit_code = rc; } - fprintf(stderr, "Test finished OK!\n"); + if (0 == exit_code) { + fprintf(stderr, "Test finished OK!\n"); + } else { + fprintf(stderr, "TEST FAILED WITH ERROR %d\n", exit_code); + } - return rc; + return exit_code; } static void set_namespace(int nprocs, char *ranks, char *nspace, @@ -576,37 +636,37 @@ static void errhandler(size_t evhdlr_registration_id, void *cbdata) { pmix_output(0, "SERVER: ERRHANDLER CALLED WITH STATUS %d", status); + /* we must NOT tell the event handler state machine that we + * are the last step as that will prevent it from notifying + * anyone else that might be listening for declarations */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } } static void errhandler_reg_callbk (pmix_status_t status, size_t errhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; pmix_output(0, "SERVER: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu", status, (unsigned long)errhandler_ref); - *active = status; + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); } static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, pmix_op_cbfunc_t cbfunc, void *cbdata) { - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_output(0, "SERVER: FINALIZED %s:%d WAKEUP %d", proc->nspace, proc->rank, wakeup); - /* ensure we call the cbfunc so the proc can exit! */ - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static void abcbfunc(pmix_status_t status, void *cbdata) @@ -664,17 +724,31 @@ static pmix_status_t abort_fn(const pmix_proc_t *proc, return PMIX_SUCCESS; } +static void fencbfn(int sd, short args, void *cbdata) +{ + pmix_shift_caddy_t *scd = (pmix_shift_caddy_t*)cbdata; + /* pass the provided data back to each participating proc */ + if (NULL != scd->cbfunc.modexcbfunc) { + scd->cbfunc.modexcbfunc(scd->status, scd->data, scd->ndata, scd->cbdata, NULL, NULL); + } + PMIX_RELEASE(scd); +} static pmix_status_t fencenb_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, char *data, size_t ndata, pmix_modex_cbfunc_t cbfunc, void *cbdata) { + pmix_shift_caddy_t *scd; + pmix_output(0, "SERVER: FENCENB"); - /* pass the provided data back to each participating proc */ - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, data, ndata, cbdata, NULL, NULL); - } + scd = PMIX_NEW(pmix_shift_caddy_t); + scd->status = PMIX_SUCCESS; + scd->data = data; + scd->ndata = ndata; + scd->cbfunc.modexcbfunc = cbfunc; + scd->cbdata = cbdata; + PMIX_THREADSHIFT(scd, fencbfn); return PMIX_SUCCESS; } @@ -683,13 +757,21 @@ static pmix_status_t dmodex_fn(const pmix_proc_t *proc, const pmix_info_t info[], size_t ninfo, pmix_modex_cbfunc_t cbfunc, void *cbdata) { + pmix_shift_caddy_t *scd; + pmix_output(0, "SERVER: DMODEX"); - /* we don't have any data for remote procs as this - * test only runs one server - so report accordingly */ - if (NULL != cbfunc) { - cbfunc(PMIX_ERR_NOT_FOUND, NULL, 0, cbdata, NULL, NULL); + /* if this is a timeout test, then do nothing */ + if (istimeouttest) { + return PMIX_SUCCESS; } + + scd = PMIX_NEW(pmix_shift_caddy_t); + scd->status = PMIX_ERR_NOT_FOUND; + scd->cbfunc.modexcbfunc = cbfunc; + scd->cbdata = cbdata; + PMIX_THREADSHIFT(scd, fencbfn); + return PMIX_SUCCESS; } @@ -711,12 +793,26 @@ static pmix_status_t publish_fn(const pmix_proc_t *proc, pmix_value_xfer(&p->pdata.value, (pmix_value_t*)&info[n].value); pmix_list_append(&pubdata, &p->super); } - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - return PMIX_SUCCESS; + + return PMIX_OPERATION_SUCCEEDED; } +typedef struct { + pmix_event_t ev; + pmix_pdata_t *pd; + size_t n; + pmix_lookup_cbfunc_t cbfunc; + void *cbdata; +} lkobj_t; + +static void lkcbfn(int sd, short args, void *cbdata) +{ + lkobj_t *lk = (lkobj_t*)cbdata; + + lk->cbfunc(PMIX_SUCCESS, lk->pd, lk->n, lk->cbdata); + PMIX_PDATA_FREE(lk->pd, lk->n); + free(lk); +} static pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, const pmix_info_t info[], size_t ninfo, @@ -727,6 +823,7 @@ static pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, size_t i, n; pmix_pdata_t *pd = NULL; pmix_status_t ret = PMIX_ERR_NOT_FOUND; + lkobj_t *lk; pmix_output(0, "SERVER: LOOKUP"); @@ -759,13 +856,16 @@ static pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, } } PMIX_LIST_DESTRUCT(&results); - if (NULL != cbfunc) { - cbfunc(ret, pd, n, cbdata); + if (PMIX_SUCCESS == ret) { + lk = (lkobj_t*)malloc(sizeof(lkobj_t)); + lk->pd = pd; + lk->n = n; + lk->cbfunc = cbfunc; + lk->cbdata = cbdata; + PMIX_THREADSHIFT(lk, lkcbfn); } - if (0 < n) { - PMIX_PDATA_FREE(pd, n); - } - return PMIX_SUCCESS; + + return ret; } @@ -787,10 +887,7 @@ static pmix_status_t unpublish_fn(const pmix_proc_t *proc, char **keys, } } } - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static void spcbfunc(pmix_status_t status, void *cbdata) @@ -841,6 +938,7 @@ static pmix_status_t spawn_fn(const pmix_proc_t *proc, return PMIX_SUCCESS; } +static int numconnects = 0; static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, @@ -851,11 +949,9 @@ static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, /* in practice, we would pass this request to the local * resource manager for handling */ - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } + numconnects++; - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } @@ -865,30 +961,20 @@ static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, { pmix_output(0, "SERVER: DISCONNECT"); - /* in practice, we would pass this request to the local - * resource manager for handling */ - - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static pmix_status_t register_event_fn(pmix_status_t *codes, size_t ncodes, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static pmix_status_t deregister_events(pmix_status_t *codes, size_t ncodes, pmix_op_cbfunc_t cbfunc, void *cbdata) { - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static pmix_status_t notify_event(pmix_status_t code, @@ -902,10 +988,21 @@ static pmix_status_t notify_event(pmix_status_t code, } typedef struct query_data_t { + pmix_event_t ev; pmix_info_t *data; size_t ndata; + pmix_info_cbfunc_t cbfunc; + void *cbdata; } query_data_t; +static void qfn(int sd, short args, void *cbdata) +{ + query_data_t *qd = (query_data_t*)cbdata; + + qd->cbfunc(PMIX_SUCCESS, qd->data, qd->ndata, qd->cbdata, NULL, NULL); + PMIX_INFO_FREE(qd->data, qd->ndata); +} + static pmix_status_t query_fn(pmix_proc_t *proct, pmix_query_t *queries, size_t nqueries, pmix_info_cbfunc_t cbfunc, @@ -913,6 +1010,7 @@ static pmix_status_t query_fn(pmix_proc_t *proct, { size_t n; pmix_info_t *info; + query_data_t qd; pmix_output(0, "SERVER: QUERY"); @@ -929,7 +1027,11 @@ static pmix_status_t query_fn(pmix_proc_t *proct, return PMIX_ERROR; } } - cbfunc(PMIX_SUCCESS, info, nqueries, cbdata, NULL, NULL); + qd.data = info; + qd.ndata = nqueries; + qd.cbfunc = cbfunc; + qd.cbdata = cbdata; + PMIX_THREADSHIFT(&qd, qfn); return PMIX_SUCCESS; } @@ -950,16 +1052,29 @@ static void tool_connect_fn(pmix_info_t *info, size_t ninfo, } } +typedef struct { + pmix_event_t ev; + pmix_op_cbfunc_t cbfunc; + void *cbdata; +} mylog_t; + +static void foobar(int sd, short args, void *cbdata) +{ + mylog_t *lg = (mylog_t*)cbdata; + lg->cbfunc(PMIX_SUCCESS, lg->cbdata); +} static void log_fn(const pmix_proc_t *client, const pmix_info_t data[], size_t ndata, const pmix_info_t directives[], size_t ndirs, pmix_op_cbfunc_t cbfunc, void *cbdata) { + mylog_t *lg = (mylog_t *)malloc(sizeof(mylog_t)); + pmix_output(0, "SERVER: LOG"); - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } + lg->cbfunc = cbfunc; + lg->cbdata = cbdata; + PMIX_THREADSHIFT(lg, foobar); } static pmix_status_t alloc_fn(const pmix_proc_t *client, @@ -967,7 +1082,7 @@ static pmix_status_t alloc_fn(const pmix_proc_t *client, const pmix_info_t data[], size_t ndata, pmix_info_cbfunc_t cbfunc, void *cbdata) { - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static pmix_status_t jctrl_fn(const pmix_proc_t *requestor, @@ -1016,8 +1131,18 @@ static void wait_signal_callback(int fd, short event, void *arg) PMIX_LIST_FOREACH(t2, &children, wait_tracker_t) { if (pid == t2->pid) { /* found it! */ + if (WIFEXITED(status)) { + t2->exit_code = WEXITSTATUS(status); + } else { + if (WIFSIGNALED(status)) { + t2->exit_code = WTERMSIG(status) + 128; + } + } + if (0 != t2->exit_code && 0 == exit_code) { + exit_code = t2->exit_code; + } --wakeup; - return; + break; } } } diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.h b/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.h deleted file mode 100644 index 89f1c00dd3d..00000000000 --- a/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2018 Intel, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - */ - -#include -#include -#include -#include -#include - -typedef struct { - pthread_mutex_t mutex; - pthread_cond_t cond; - volatile bool active; - pmix_status_t status; -} mylock_t; - -#define DEBUG_CONSTRUCT_LOCK(l) \ - do { \ - pthread_mutex_init(&(l)->mutex, NULL); \ - pthread_cond_init(&(l)->cond, NULL); \ - (l)->active = true; \ - (l)->status = PMIX_SUCCESS; \ - } while(0) - -#define DEBUG_DESTRUCT_LOCK(l) \ - do { \ - pthread_mutex_destroy(&(l)->mutex); \ - pthread_cond_destroy(&(l)->cond); \ - } while(0) - -#define DEBUG_WAIT_THREAD(lck) \ - do { \ - pthread_mutex_lock(&(lck)->mutex); \ - while ((lck)->active) { \ - pthread_cond_wait(&(lck)->cond, &(lck)->mutex); \ - } \ - pthread_mutex_unlock(&(lck)->mutex); \ - } while(0) - -#define DEBUG_WAKEUP_THREAD(lck) \ - do { \ - pthread_mutex_lock(&(lck)->mutex); \ - (lck)->active = false; \ - pthread_cond_broadcast(&(lck)->cond); \ - pthread_mutex_unlock(&(lck)->mutex); \ - } while(0) diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/simptimeout.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simptimeout.c new file mode 100644 index 00000000000..a62ab99f1ae --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simptimeout.c @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include +#include + +#include +#include +#include +#include + +#include "src/class/pmix_object.h" +#include "src/util/output.h" +#include "src/util/printf.h" + +#define MAXCNT 1 + +static volatile bool completed = false; +static pmix_proc_t myproc; + +static void notification_fn(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) +{ + pmix_output(0, "Client %s:%d NOTIFIED with status %s", myproc.nspace, myproc.rank, PMIx_Error_string(status)); + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } + completed = true; +} + +static void errhandler_reg_callbk(pmix_status_t status, + size_t errhandler_ref, + void *cbdata) +{ + volatile bool *active = (volatile bool*)cbdata; + + pmix_output(0, "Client: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu", + status, (unsigned long)errhandler_ref); + *active = false; +} + + +int main(int argc, char **argv) +{ + int rc; + pmix_value_t value; + pmix_value_t *val = &value; + pmix_proc_t proc; + uint32_t nprocs, n; + volatile bool active; + pmix_info_t info; + + /* init us */ + if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + exit(rc); + } + pmix_output(0, "Client ns %s rank %d: Running", myproc.nspace, myproc.rank); + + /* test something */ + (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_JOB_SIZE, NULL, 0, &val))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Get failed: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + exit(rc); + } + PMIX_VALUE_RELEASE(val); + + /* register our errhandler */ + active = true; + PMIx_Register_event_handler(NULL, 0, NULL, 0, + notification_fn, errhandler_reg_callbk, (void*)&active); + while (active) { + usleep(10); + } + + /* get our universe size */ + (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &val))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Get universe size failed: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + goto done; + } + nprocs = val->data.uint32; + PMIX_VALUE_RELEASE(val); + pmix_output(0, "Client %s:%d universe size %d", myproc.nspace, myproc.rank, nprocs); + + /* if we are rank=0, then do a fence with timeout */ + if (0 == myproc.rank) { + PMIX_INFO_CONSTRUCT(&info); + n = 1; + PMIX_INFO_LOAD(&info, PMIX_TIMEOUT, &n, PMIX_UINT32); + PMIX_PROC_CONSTRUCT(&proc); + (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + pmix_output(0, "TEST FENCE TIMEOUT"); + if (PMIX_ERR_TIMEOUT != (rc = PMIx_Fence(&proc, 1, &info, 1))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Fence did not timeout: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + goto done; + } + pmix_output(0, "FENCE TIMEOUT SUCCEEDED"); + + /* check timeout on connect */ + pmix_output(0, "TEST CONNECT TIMEOUT"); + if (PMIX_ERR_TIMEOUT != (rc = PMIx_Connect(&proc, 1, &info, 1))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Connect did not timeout: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + goto done; + } + pmix_output(0, "CONNECT TIMEOUT SUCCEEDED"); + + /* check timeout on Get */ + proc.rank = 1; + pmix_output(0, "TEST GET TIMEOUT"); + if (PMIX_ERR_TIMEOUT == (rc = PMIx_Get(&proc, "1234", &info, 1, &val))) { + pmix_output(0, "Client ns %s rank %d: PMIx_Get did not timeout: %s", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + goto done; + } + pmix_output(0, "GET TIMEOUT SUCCEEDED"); + + } else { + sleep(5); + } + + done: + /* finalize us */ + pmix_output(0, "Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank); + if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { + fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + } else { + fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank); + } + fflush(stderr); + return(rc); +} diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/simptool.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simptool.c index 2af6f395ede..44fe60a2a0f 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/simple/simptool.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simptool.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -95,6 +95,7 @@ int main(int argc, char **argv) PMIX_QUERY_CREATE(query, nq); pmix_argv_append_nosize(&query[0].keys, "foobar"); pmix_argv_append_nosize(&query[1].keys, "spastic"); + pmix_argv_append_nosize(&query[1].keys, PMIX_SERVER_URI); active = true; if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&active))) { pmix_output(0, "Client ns %s rank %d: PMIx_Query_info failed: %d", myproc.nspace, myproc.rank, rc); diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/stability.c b/opal/mca/pmix/pmix2x/pmix/test/simple/stability.c index 914dc49b37e..a075384b6da 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/simple/stability.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/stability.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/test_pmix.c b/opal/mca/pmix/pmix2x/pmix/test/simple/test_pmix.c index de1d1fc5cf8..8ecf6c1f6b6 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/simple/test_pmix.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/test_pmix.c @@ -5,52 +5,63 @@ int main(int argc, char **argv) { - pmix_proc_t myproc; - pmix_status_t rc; - - rc = PMIx_Init(&myproc, NULL, 0); - assert(PMIX_SUCCESS == rc); - - if (myproc.rank == 0 ) { - pmix_info_t *info; - PMIX_INFO_CREATE(info, 1); - snprintf(info[0].key, PMIX_MAX_KEYLEN, "magic-found"); - info[0].value.type = PMIX_STRING; - info[0].value.data.string = "yes"; - rc = PMIx_Publish(info, 1); - assert(PMIX_SUCCESS == rc); - } - - printf("I am rank %d\n", myproc.rank); - - { - bool flag; - pmix_info_t *info; - PMIX_INFO_CREATE(info, 1); - flag = true; - PMIX_INFO_LOAD(info, PMIX_COLLECT_DATA, &flag, PMIX_BOOL); - rc = PMIx_Fence(&myproc, 1, info, 1); - assert(PMIX_SUCCESS == rc); - PMIX_INFO_FREE(info, 1); - } - - if (myproc.rank == 1) { - int i; - pmix_pdata_t *pdata; - PMIX_PDATA_CREATE(pdata, 2); - snprintf(pdata[0].key, PMIX_MAX_KEYLEN, "magic-found"); - snprintf(pdata[1].key, PMIX_MAX_KEYLEN, "magic-not-found"); - rc = PMIx_Lookup(&pdata[0], 2, NULL, 0); - assert((PMIX_SUCCESS == rc) || (PMIX_ERR_NOT_FOUND == rc)); - for ( i = 0 ; i < 2 ; i++ ) - if (pdata[i].value.type == PMIX_STRING) - printf("Found[%d] %d %s\n", i, pdata[i].value.type, pdata[i].value.data.string); - else - printf("Found[%d] %d\n", i, pdata[i].value.type); - PMIX_PDATA_FREE(pdata, 1); - } - - rc = PMIx_Finalize(NULL, 0); - assert(PMIX_SUCCESS == rc); + pmix_proc_t myproc; + pmix_status_t rc; + + int rank; + rc = PMIx_Init(&myproc, NULL, 0); + assert(PMIX_SUCCESS == rc); + + { + pmix_value_t *value; + rc = PMIx_Get(&myproc, PMIX_RANK, NULL, 0, &value); + assert(PMIX_SUCCESS == rc); + printf("%d\n", value->type); + assert(value->type == PMIX_INT); + rank = value->data.uint32; + PMIX_VALUE_RELEASE(value); + } + + if (rank == 0 ) { + pmix_info_t *info; + PMIX_INFO_CREATE(info, 1); + snprintf(info[0].key, PMIX_MAX_KEYLEN, "magic-found"); + info[0].value.type = PMIX_STRING; + info[0].value.data.string = "yes"; + rc = PMIx_Publish(info, 1); + assert(PMIX_SUCCESS == rc); + } + + printf("I am rank %d\n", rank); + + { + bool flag; + pmix_info_t *info; + PMIX_INFO_CREATE(info, 1); + flag = true; + PMIX_INFO_LOAD(info, PMIX_COLLECT_DATA, &flag, PMIX_BOOL); + rc = PMIx_Fence(&myproc, 1, info, 1); + assert(PMIX_SUCCESS == rc); + PMIX_INFO_FREE(info, 1); + } + + if (rank == 1) { + int i; + pmix_pdata_t *pdata; + PMIX_PDATA_CREATE(pdata, 2); + snprintf(pdata[0].key, PMIX_MAX_KEYLEN, "magic-found"); + snprintf(pdata[1].key, PMIX_MAX_KEYLEN, "magic-not-found"); + rc = PMIx_Lookup(&pdata[0], 2, NULL, 0); + assert((PMIX_SUCCESS == rc) || (PMIX_ERR_NOT_FOUND == rc)); + for ( i = 0 ; i < 2 ; i++ ) + if (pdata[i].value.type == PMIX_STRING) + printf("Found[%d] %d %s\n", i, pdata[i].value.type, pdata[i].value.data.string); + else + printf("Found[%d] %d\n", i, pdata[i].value.type); + PMIX_PDATA_FREE(pdata, 1); + } + + rc = PMIx_Finalize(NULL, 0); + assert(PMIX_SUCCESS == rc); } diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_cd.c b/opal/mca/pmix/pmix2x/pmix/test/test_cd.c index 65471dff708..e8528e39af6 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/test_cd.c +++ b/opal/mca/pmix/pmix2x/pmix/test/test_cd.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_cd.h b/opal/mca/pmix/pmix2x/pmix/test/test_cd.h index d4e789102f2..e044a993fd5 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/test_cd.h +++ b/opal/mca/pmix/pmix2x/pmix/test/test_cd.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_common.c b/opal/mca/pmix/pmix2x/pmix/test/test_common.c index 9021e58ba21..a0a71dc59f7 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/test_common.c +++ b/opal/mca/pmix/pmix2x/pmix/test/test_common.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2015-2017 Mellanox Technologies, Inc. @@ -600,7 +600,6 @@ int get_total_ns_number(test_params params) int get_all_ranks_from_namespace(test_params params, char *nspace, pmix_proc_t **ranks, size_t *nranks) { - int base_rank = 0; size_t num_ranks = 0; int num = -1; size_t j; @@ -616,7 +615,6 @@ int get_all_ranks_from_namespace(test_params params, char *nspace, pmix_proc_t * char *pch = tmp; int ns_id = (int)strtol(nspace + strlen(TEST_NAMESPACE) + 1, NULL, 10); while (NULL != pch && num != ns_id) { - base_rank += num_ranks; pch = strtok((-1 == num ) ? tmp : NULL, ":"); if (NULL == pch) { break; @@ -629,7 +627,7 @@ int get_all_ranks_from_namespace(test_params params, char *nspace, pmix_proc_t * PMIX_PROC_CREATE(*ranks, num_ranks); for (j = 0; j < num_ranks; j++) { (void)strncpy((*ranks)[j].nspace, nspace, PMIX_MAX_NSLEN); - (*ranks)[j].rank = base_rank+j; + (*ranks)[j].rank = j; } } else { free(tmp); diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_common.h b/opal/mca/pmix/pmix2x/pmix/test/test_common.h index 0c2837553ae..8057e478d71 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/test_common.h +++ b/opal/mca/pmix/pmix2x/pmix/test/test_common.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2015 Research Organization for Information Science @@ -33,6 +33,13 @@ #define TEST_NAMESPACE "smoky_nspace" #define TEST_CREDENTIAL "dummy" +#define PMIX_WAIT_FOR_COMPLETION(m) \ + do { \ + while ((m)) { \ + usleep(10); \ + } \ + } while(0) + /* WARNING: pmix_test_output_prepare is currently not threadsafe! * fix it once needed! */ @@ -250,7 +257,7 @@ typedef struct { TEST_VERBOSE(("%s:%d want to get from %s:%d key %s", my_nspace, my_rank, ns, r, key)); \ if (blocking) { \ if (PMIX_SUCCESS != (rc = PMIx_Get(&foobar, key, NULL, 0, &val))) { \ - if( !( rc == PMIX_ERR_NOT_FOUND && ok_notfnd ) ){ \ + if( !( (rc == PMIX_ERR_NOT_FOUND || rc == PMIX_ERR_PROC_ENTRY_NOT_FOUND) && ok_notfnd ) ){ \ TEST_ERROR(("%s:%d: PMIx_Get failed: %d from %s:%d, key %s", my_nspace, my_rank, rc, ns, r, key)); \ } \ rc = PMIX_ERROR; \ @@ -277,7 +284,7 @@ typedef struct { } \ if (PMIX_SUCCESS == rc) { \ if( PMIX_SUCCESS != cbdata.status ){ \ - if( !( cbdata.status == PMIX_ERR_NOT_FOUND && ok_notfnd ) ){ \ + if( !( (cbdata.status == PMIX_ERR_NOT_FOUND || cbdata.status == PMIX_ERR_PROC_ENTRY_NOT_FOUND) && ok_notfnd ) ){ \ TEST_ERROR(("%s:%d: PMIx_Get_nb failed: %d from %s:%d, key=%s", \ my_nspace, my_rank, rc, my_nspace, r)); \ } \ diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_fence.c b/opal/mca/pmix/pmix2x/pmix/test/test_fence.c index 909edcffa3f..69408b5357a 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/test_fence.c +++ b/opal/mca/pmix/pmix2x/pmix/test/test_fence.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015-2017 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -63,7 +63,7 @@ int test_fence(test_params params, char *my_nspace, pmix_rank_t my_rank) pmix_proc_t *pcs; bool participate; int fence_num = 0; - char sval[50]; + char *sval; int put_ind; if (NULL != params.noise) { @@ -98,13 +98,14 @@ int test_fence(test_params params, char *my_nspace, pmix_rank_t my_rank) /*run fence test on this range */ /* first put value (my_ns, my_rank) with key based on fence_num to split results of different fences*/ put_ind = 0; - (void)snprintf(sval, 50, "%d:%s:%d", fence_num, my_nspace, my_rank); + (void)asprintf(&sval, "%d:%s:%d", fence_num, my_nspace, my_rank); PUT(string, sval, PMIX_GLOBAL, fence_num, put_ind++, params.use_same_keys); if (PMIX_SUCCESS != rc) { TEST_ERROR(("%s:%d: PMIx_Put failed: %d", my_nspace, my_rank, rc)); PMIX_LIST_DESTRUCT(&test_fences); return rc; } + free(sval); PUT(int, fence_num+my_rank, PMIX_GLOBAL, fence_num, put_ind++, params.use_same_keys); if (PMIX_SUCCESS != rc) { @@ -186,7 +187,7 @@ int test_fence(test_params params, char *my_nspace, pmix_rank_t my_rank) /* get data from all participating in this fence clients */ PMIX_LIST_FOREACH(p, desc->participants, participant_t) { put_ind = 0; - snprintf(sval, 50, "%d:%s:%d", fence_num, p->proc.nspace, p->proc.rank); + asprintf(&sval, "%d:%s:%d", fence_num, p->proc.nspace, p->proc.rank); GET(string, sval, p->proc.nspace, p->proc.rank, fence_num, put_ind++, params.use_same_keys, 1, 0); if (PMIX_SUCCESS != rc) { TEST_ERROR(("%s:%d: PMIx_Get failed (%d) from %s:%d", my_nspace, my_rank, rc, p->proc.nspace, p->proc.rank)); @@ -194,6 +195,7 @@ int test_fence(test_params params, char *my_nspace, pmix_rank_t my_rank) PMIX_LIST_DESTRUCT(&test_fences); return rc; } + free(sval); GET(int, (int)(fence_num+p->proc.rank), p->proc.nspace, p->proc.rank, fence_num, put_ind++, params.use_same_keys, 0, 0); if (PMIX_SUCCESS != rc) { TEST_ERROR(("%s:%d: PMIx_Get failed (%d) from %s:%d", my_nspace, my_rank, rc, p->proc.nspace, p->proc.rank)); @@ -384,7 +386,7 @@ int test_job_fence(test_params params, char *my_nspace, pmix_rank_t my_rank) if( local ){ GET(int, (12340+j), my_nspace, i+params.base_rank, 100, j, 0, 0, 0); if (PMIX_SUCCESS != rc) { - TEST_ERROR(("%s:%d: PMIx_Get failed: %d", my_nspace, my_rank, rc)); + TEST_ERROR(("%s:%d: PMIx_Get failed: %s", my_nspace, my_rank, PMIx_Error_string(rc))); return PMIX_ERROR; } @@ -423,9 +425,10 @@ int test_job_fence(test_params params, char *my_nspace, pmix_rank_t my_rank) my_nspace, my_rank)); return PMIX_ERROR; } - if (PMIX_ERR_NOT_FOUND != rc) { - TEST_ERROR(("%s:%d [ERROR]: PMIx_Get returned %d instead of not_found", - my_nspace, my_rank, rc)); + if (PMIX_ERR_NOT_FOUND != rc && PMIX_ERR_PROC_ENTRY_NOT_FOUND != rc) { + TEST_ERROR(("%s:%d [ERROR]: PMIx_Get returned %s instead of not_found", + my_nspace, my_rank, PMIx_Error_string(rc))); + return PMIX_ERROR; } if (NULL != val) { TEST_ERROR(("%s:%d [ERROR]: PMIx_Get did not return NULL value", my_nspace, my_rank)); diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_resolve_peers.c b/opal/mca/pmix/pmix2x/pmix/test/test_resolve_peers.c index aac26067474..4179260e98a 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/test_resolve_peers.c +++ b/opal/mca/pmix/pmix2x/pmix/test/test_resolve_peers.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -100,9 +100,9 @@ int test_resolve_peers(char *my_nspace, int my_rank, test_params params) /* make a connection between processes from own namespace and processes from this namespace. */ rc = test_cd_common(procs, 2, 1, 0); if (PMIX_SUCCESS == rc) { - TEST_VERBOSE(("%s:%d: Connect to %s succeeded %s.", my_nspace, my_rank, nspace)); + TEST_VERBOSE(("%s:%d: Connect to %s succeeded %s.", my_nspace, my_rank, nspace, PMIx_Error_string(rc))); } else { - TEST_ERROR(("%s:%d: Connect to %s failed %s.", my_nspace, my_rank, nspace)); + TEST_ERROR(("%s:%d: Connect to %s failed %s.", my_nspace, my_rank, nspace, PMIx_Error_string(rc))); return PMIX_ERROR; } diff --git a/opal/mca/pmix/pmix2x/pmix/test/utils.c b/opal/mca/pmix/pmix2x/pmix/test/utils.c index 5fb1a0e78b3..a2ccc55a711 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/utils.c +++ b/opal/mca/pmix/pmix2x/pmix/test/utils.c @@ -1,6 +1,6 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2015-2017 Mellanox Technologies, Inc. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -88,7 +88,9 @@ static void set_namespace(int nprocs, char *ranks, char *name) info[7].value.data.uint32 = getpid (); int in_progress = 1, rc; - if (PMIX_SUCCESS == (rc = PMIx_server_register_nspace(name, nprocs, info, ninfo, release_cb, &in_progress))) { + if (PMIX_SUCCESS == (rc = PMIx_server_register_nspace(name, nprocs, info, + ninfo, release_cb, + &in_progress))) { PMIX_WAIT_FOR_COMPLETION(in_progress); } PMIX_INFO_FREE(info, ninfo); @@ -171,7 +173,7 @@ void set_client_argv(test_params *params, char ***argv) } } -int launch_clients(int num_procs, char *binary, char *** client_env, char ***base_argv) +int launch_clients(int nprocs, char *binary, char *** client_env, char ***base_argv) { int n; uid_t myuid; @@ -182,16 +184,17 @@ int launch_clients(int num_procs, char *binary, char *** client_env, char ***bas static int counter = 0; static int num_ns = 0; pmix_proc_t proc; + int base_rank = 0; TEST_VERBOSE(("Setting job info")); - fill_seq_ranks_array(num_procs, counter, &ranks); + fill_seq_ranks_array(nprocs, base_rank, &ranks); if (NULL == ranks) { PMIx_server_finalize(); TEST_ERROR(("fill_seq_ranks_array failed")); return PMIX_ERROR; } (void)snprintf(proc.nspace, PMIX_MAX_NSLEN, "%s-%d", TEST_NAMESPACE, num_ns); - set_namespace(num_procs, ranks, proc.nspace); + set_namespace(nprocs, ranks, proc.nspace); if (NULL != ranks) { free(ranks); } @@ -200,15 +203,16 @@ int launch_clients(int num_procs, char *binary, char *** client_env, char ***bas mygid = getgid(); /* fork/exec the test */ - for (n = 0; n < num_procs; n++) { - proc.rank = counter; + for (n = 0; n < nprocs; n++) { + proc.rank = n; if (PMIX_SUCCESS != (rc = PMIx_server_setup_fork(&proc, client_env))) {//n TEST_ERROR(("Server fork setup failed with error %d", rc)); PMIx_server_finalize(); cli_kill_all(); return rc; } - if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid, NULL, NULL, NULL))) {//n + rc = PMIx_server_register_client(&proc, myuid, mygid, NULL, NULL, NULL); + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) {//n TEST_ERROR(("Server fork setup failed with error %d", rc)); PMIx_server_finalize(); cli_kill_all(); @@ -222,20 +226,20 @@ int launch_clients(int num_procs, char *binary, char *** client_env, char ***bas cli_kill_all(); return -1; } - cli_info[counter].rank = counter;//n + cli_info[counter].rank = proc.rank; cli_info[counter].ns = strdup(proc.nspace); char **client_argv = pmix_argv_copy(*base_argv); /* add two last arguments: -r */ - sprintf(digit, "%d", counter);//n + sprintf(digit, "%d", proc.rank); pmix_argv_append_nosize(&client_argv, "-r"); pmix_argv_append_nosize(&client_argv, digit); pmix_argv_append_nosize(&client_argv, "-s"); pmix_argv_append_nosize(&client_argv, proc.nspace); - sprintf(digit, "%d", num_procs); + sprintf(digit, "%d", nprocs); pmix_argv_append_nosize(&client_argv, "--ns-size"); pmix_argv_append_nosize(&client_argv, digit); @@ -243,7 +247,7 @@ int launch_clients(int num_procs, char *binary, char *** client_env, char ***bas pmix_argv_append_nosize(&client_argv, "--ns-id"); pmix_argv_append_nosize(&client_argv, digit); - sprintf(digit, "%d", (counter-n)); + sprintf(digit, "%d", base_rank); pmix_argv_append_nosize(&client_argv, "--base-rank"); pmix_argv_append_nosize(&client_argv, digit); diff --git a/opal/mca/pmix/pmix2x/pmix/test/utils.h b/opal/mca/pmix/pmix2x/pmix/test/utils.h index bbeebaa2f7a..4d96b394c85 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/utils.h +++ b/opal/mca/pmix/pmix2x/pmix/test/utils.h @@ -1,6 +1,6 @@ /* - * Copyright (c) 2015 Intel, Inc. All rights reserved. - * Copyright (c) 2015 Mellanox Technologies, Inc. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ * @@ -19,4 +19,4 @@ #include "test_common.h" void set_client_argv(test_params *params, char ***argv); -int launch_clients(int num_procs, char *binary, char *** client_env, char ***client_argv); +int launch_clients(int nprocs, char *binary, char *** client_env, char ***base_argv);