From 3e9bae389d9790bc5407e18c2f148f2e09c4530b Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Tue, 1 Nov 2016 15:02:54 -0600 Subject: [PATCH 1/2] mpi/neighbor_allgatherv: fix copy&paste error and add helpers This commit adds a helper function to get the inbound and outbound neighbor count and updates the neighbor_allgatherv bindings to use the correct count when checking the input parameters. Fixes #2324 Signed-off-by: Nathan Hjelm (cherry picked from commit 3c0e94afab03a27364114dcf3d429943d15cd4f4) Signed-off-by: Nathan Hjelm --- ompi/mca/topo/base/base.h | 3 +++ ompi/mca/topo/base/topo_base_frame.c | 27 +++++++++++++++++++++ ompi/mpi/c/neighbor_allgatherv.c | 35 ++++++---------------------- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/ompi/mca/topo/base/base.h b/ompi/mca/topo/base/base.h index 5e05a8009d4..51714eee6fa 100644 --- a/ompi/mca/topo/base/base.h +++ b/ompi/mca/topo/base/base.h @@ -194,6 +194,9 @@ OMPI_DECLSPEC int mca_topo_base_dist_graph_neighbors_count(ompi_communicator_t *comm, int *inneighbors, int *outneighbors, int *weighted); + +int mca_topo_base_neighbor_count (ompi_communicator_t *comm, int *indegree, int *outdegree); + END_C_DECLS #endif /* MCA_BASE_TOPO_H */ diff --git a/ompi/mca/topo/base/topo_base_frame.c b/ompi/mca/topo/base/topo_base_frame.c index 062786f9308..4ed9049fc26 100644 --- a/ompi/mca/topo/base/topo_base_frame.c +++ b/ompi/mca/topo/base/topo_base_frame.c @@ -71,6 +71,33 @@ static int mca_topo_base_open(mca_base_open_flag_t flags) return mca_base_framework_components_open(&ompi_topo_base_framework, flags); } +int mca_topo_base_neighbor_count (ompi_communicator_t *comm, int *indegree, int *outdegree) { + if (!OMPI_COMM_IS_TOPO(comm)) { + return OMPI_ERR_BAD_PARAM; + } + + if (OMPI_COMM_IS_CART(comm)) { + /* cartesian */ + /* outdegree is always 2*ndims because we need to iterate over + empty buffers for MPI_PROC_NULL */ + *outdegree = *indegree = 2 * comm->c_topo->mtc.cart->ndims; + } else if (OMPI_COMM_IS_GRAPH(comm)) { + /* graph */ + int rank, nneighbors; + + rank = ompi_comm_rank (comm); + mca_topo_base_graph_neighbors_count (comm, rank, &nneighbors); + + *outdegree = *indegree = nneighbors; + } else if (OMPI_COMM_IS_DIST_GRAPH(comm)) { + /* graph */ + *indegree = comm->c_topo->mtc.dist_graph->indegree; + *outdegree = comm->c_topo->mtc.dist_graph->outdegree; + } + + return OMPI_SUCCESS; +} + MCA_BASE_FRAMEWORK_DECLARE(ompi, topo, "OMPI Topo", NULL, mca_topo_base_open, mca_topo_base_close, mca_topo_base_static_components, 0); diff --git a/ompi/mpi/c/neighbor_allgatherv.c b/ompi/mpi/c/neighbor_allgatherv.c index 9a2f87a2467..93737518f36 100644 --- a/ompi/mpi/c/neighbor_allgatherv.c +++ b/ompi/mpi/c/neighbor_allgatherv.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2010 University of Houston. All rights reserved. * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2016 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -32,6 +32,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/topo/base/base.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -50,20 +51,20 @@ int MPI_Neighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sen void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype, MPI_Comm comm) { - int i, size, err; + int in_size, out_size, err; MEMCHECKER( int rank; ptrdiff_t ext; rank = ompi_comm_rank(comm); - size = ompi_comm_size(comm); + mca_topo_base_neighbor_count (comm, &in_size, &out_size); ompi_datatype_type_extent(recvtype, &ext); memchecker_datatype(recvtype); memchecker_comm (comm); /* check whether the receive buffer is addressable. */ - for (i = 0; i < size; i++) { + for (int i = 0; i < in_size; ++i) { memchecker_call(&opal_memchecker_base_isaddressable, (char *)(recvbuf)+displs[i]*ext, recvcounts[i], recvtype); @@ -107,8 +108,8 @@ int MPI_Neighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sen get the size of the remote group here for both intra- and intercommunicators */ - size = ompi_comm_remote_size(comm); - for (i = 0; i < size; ++i) { + mca_topo_base_neighbor_count (comm, &in_size, &out_size); + for (int i = 0; i < in_size; ++i) { if (recvcounts[i] < 0) { return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_COUNT, FUNC_NAME); } @@ -141,27 +142,6 @@ int MPI_Neighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sen } } - /* Do we need to do anything? Everyone had to give the same - signature, which means that everyone must have given a - sum(recvounts) > 0 if there's anything to do. */ - - if ( OMPI_COMM_IS_INTRA( comm) ) { - for (i = 0; i < ompi_comm_size(comm); ++i) { - if (0 != recvcounts[i]) { - break; - } - } - if (i >= ompi_comm_size(comm)) { - return MPI_SUCCESS; - } - } - /* There is no rule that can be applied for inter-communicators, since - recvcount(s)=0 only indicates that the processes in the other group - do not send anything, sendcount=0 only indicates that I do not send - anything. However, other processes in my group might very well send - something */ - - OPAL_CR_ENTER_LIBRARY(); /* Invoke the coll component to perform the back-end operation */ @@ -170,4 +150,3 @@ int MPI_Neighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sen recvtype, comm, comm->c_coll->coll_neighbor_allgatherv_module); OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } - From c19d561e3991bcbbed362a5bf2512caf5321cc92 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Wed, 12 Jul 2017 14:13:12 -0600 Subject: [PATCH 2/2] ompi: clean up topo helper functions This commit removes the communicator topo helper functions in favor of functions in mca/topo/base. Signed-off-by: Nathan Hjelm (cherry picked from commit 9b702fb9bd6ccf744af8b8d92aa7c4c5d2f58f2f) Signed-off-by: Nathan Hjelm --- ompi/communicator/Makefile.am | 8 ++- ompi/communicator/comm_helpers.c | 92 -------------------------------- ompi/communicator/comm_helpers.h | 41 -------------- ompi/mpi/c/ineighbor_alltoallv.c | 9 ++-- ompi/mpi/c/ineighbor_alltoallw.c | 9 ++-- ompi/mpi/c/neighbor_alltoallv.c | 9 ++-- ompi/mpi/c/neighbor_alltoallw.c | 9 ++-- 7 files changed, 19 insertions(+), 158 deletions(-) delete mode 100644 ompi/communicator/comm_helpers.c delete mode 100644 ompi/communicator/comm_helpers.h diff --git a/ompi/communicator/Makefile.am b/ompi/communicator/Makefile.am index e7f6dc731ee..6f57a3787f9 100644 --- a/ompi/communicator/Makefile.am +++ b/ompi/communicator/Makefile.am @@ -10,7 +10,7 @@ # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. -# Copyright (c) 2013 Los Alamos National Security, LLC. All rights +# Copyright (c) 2013-2017 Los Alamos National Security, LLC. All rights # reserved. # Copyright (c) 2014 Research Organization for Information Science # and Technology (RIST). All rights reserved. @@ -26,13 +26,11 @@ headers += \ communicator/communicator.h \ - communicator/comm_request.h \ - communicator/comm_helpers.h + communicator/comm_request.h lib@OMPI_LIBMPI_NAME@_la_SOURCES += \ communicator/comm_init.c \ communicator/comm.c \ communicator/comm_cid.c \ - communicator/comm_request.c \ - communicator/comm_helpers.c + communicator/comm_request.c diff --git a/ompi/communicator/comm_helpers.c b/ompi/communicator/comm_helpers.c deleted file mode 100644 index 584e80ee983..00000000000 --- a/ompi/communicator/comm_helpers.c +++ /dev/null @@ -1,92 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2006 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2006 The Technical University of Chemnitz. All - * rights reserved. - * Copyright (c) 2014 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2015 Los Alamos National Security, LLC. All rights - * reserved. - * - * Author(s): Torsten Hoefler - * - */ - -#include "comm_helpers.h" - -int ompi_comm_neighbors_count(MPI_Comm comm, int *indegree, int *outdegree, int *weighted) { - int res; - - if (OMPI_COMM_IS_CART(comm)) { - int ndims; - res = MPI_Cartdim_get(comm, &ndims) ; - if (MPI_SUCCESS != res) { - return res; - } - /* outdegree is always 2*ndims because we need to iterate over empty buffers for MPI_PROC_NULL */ - *outdegree = *indegree = 2*ndims; - *weighted = 0; - } else if (OMPI_COMM_IS_GRAPH(comm)) { - int rank, nneighbors; - rank = ompi_comm_rank ((ompi_communicator_t *) comm); - res = MPI_Graph_neighbors_count(comm, rank, &nneighbors); - if (MPI_SUCCESS != res) { - return res; - } - *outdegree = *indegree = nneighbors; - *weighted = 0; - } else if (OMPI_COMM_IS_DIST_GRAPH(comm)) { - res = MPI_Dist_graph_neighbors_count(comm, indegree, outdegree, weighted); - } else { - return MPI_ERR_ARG; - } - - return MPI_SUCCESS; -} - -int ompi_comm_neighbors(MPI_Comm comm, int maxindegree, int sources[], int sourceweights[], int maxoutdegree, int destinations[], int destweights[]) { - int res; - int index = 0; - - int indeg, outdeg, wgtd; - res = ompi_comm_neighbors_count(comm, &indeg, &outdeg, &wgtd); - if (MPI_SUCCESS != res) { - return res; - } - if(indeg > maxindegree && outdeg > maxoutdegree) return MPI_ERR_TRUNCATE; /* we want to return *all* neighbors */ - - if (OMPI_COMM_IS_CART(comm)) { - int ndims, i, rpeer, speer; - res = MPI_Cartdim_get(comm, &ndims); - if (MPI_SUCCESS != res) { - return res; - } - - for(i = 0; i - * - * $HEADER$ - */ -#ifndef __TOPO_HELPERS_H__ -#define __TOPO_HELPERS_H__ -#include "ompi_config.h" - -#include "mpi.h" - -#include "ompi/include/ompi/constants.h" -#include "ompi/communicator/communicator.h" - -#include -#include -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -int ompi_comm_neighbors_count(MPI_Comm comm, int *indegree, int *outdegree, int *weighted); -int ompi_comm_neighbors(MPI_Comm comm, int maxindegree, int sources[], int sourceweights[], int maxoutdegree, int destinations[], int destweights[]); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/ompi/mpi/c/ineighbor_alltoallv.c b/ompi/mpi/c/ineighbor_alltoallv.c index 728e9bfebce..3f30bd42a0a 100644 --- a/ompi/mpi/c/ineighbor_alltoallv.c +++ b/ompi/mpi/c/ineighbor_alltoallv.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -29,7 +29,6 @@ #include "ompi/mpi/c/bindings.h" #include "ompi/runtime/params.h" #include "ompi/communicator/communicator.h" -#include "ompi/communicator/comm_helpers.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/memchecker.h" @@ -52,7 +51,7 @@ int MPI_Ineighbor_alltoallv(const void *sendbuf, const int sendcounts[], const i MPI_Request *request) { int i, err; - int indegree, outdegree, weighted; + int indegree, outdegree; MEMCHECKER( ptrdiff_t recv_ext; @@ -68,7 +67,7 @@ int MPI_Ineighbor_alltoallv(const void *sendbuf, const int sendcounts[], const i memchecker_datatype(recvtype); ompi_datatype_type_extent(sendtype, &send_ext); - err = ompi_comm_neighbors_count(comm, &indegree, &outdegree, &weighted); + err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree); if (MPI_SUCCESS == err) { if (MPI_IN_PLACE != sendbuf) { for ( i = 0; i < outdegree; i++ ) { @@ -105,7 +104,7 @@ int MPI_Ineighbor_alltoallv(const void *sendbuf, const int sendcounts[], const i return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME); } - err = ompi_comm_neighbors_count(comm, &indegree, &outdegree, &weighted); + err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree); OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME); for (i = 0; i < outdegree; ++i) { OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]); diff --git a/ompi/mpi/c/ineighbor_alltoallw.c b/ompi/mpi/c/ineighbor_alltoallw.c index a13115d1627..4601d5bc598 100644 --- a/ompi/mpi/c/ineighbor_alltoallw.c +++ b/ompi/mpi/c/ineighbor_alltoallw.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -29,7 +29,6 @@ #include "ompi/mpi/c/bindings.h" #include "ompi/runtime/params.h" #include "ompi/communicator/communicator.h" -#include "ompi/communicator/comm_helpers.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/memchecker.h" @@ -52,7 +51,7 @@ int MPI_Ineighbor_alltoallw(const void *sendbuf, const int sendcounts[], const M MPI_Request *request) { int i, err; - int indegree, outdegree, weighted; + int indegree, outdegree; MEMCHECKER( ptrdiff_t recv_ext; @@ -60,7 +59,7 @@ int MPI_Ineighbor_alltoallw(const void *sendbuf, const int sendcounts[], const M memchecker_comm(comm); - err = ompi_comm_neighbors_count(comm, &indegree, &outdegree, &weighted); + err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree); if (MPI_SUCCESS == err) { if (MPI_IN_PLACE != sendbuf) { for ( i = 0; i < outdegree; i++ ) { @@ -105,7 +104,7 @@ int MPI_Ineighbor_alltoallw(const void *sendbuf, const int sendcounts[], const M return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME); } - err = ompi_comm_neighbors_count(comm, &indegree, &outdegree, &weighted); + err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree); OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME); for (i = 0; i < outdegree; ++i) { OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]); diff --git a/ompi/mpi/c/neighbor_alltoallv.c b/ompi/mpi/c/neighbor_alltoallv.c index acadf1ab799..5004e6b42d6 100644 --- a/ompi/mpi/c/neighbor_alltoallv.c +++ b/ompi/mpi/c/neighbor_alltoallv.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -32,7 +32,6 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/memchecker.h" -#include "ompi/communicator/comm_helpers.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -52,7 +51,7 @@ int MPI_Neighbor_alltoallv(const void *sendbuf, const int sendcounts[], const in MPI_Datatype recvtype, MPI_Comm comm) { int i, err; - int indegree, outdegree, weighted; + int indegree, outdegree; MEMCHECKER( ptrdiff_t recv_ext; @@ -68,7 +67,7 @@ int MPI_Neighbor_alltoallv(const void *sendbuf, const int sendcounts[], const in memchecker_datatype(recvtype); ompi_datatype_type_extent(sendtype, &send_ext); - err = ompi_comm_neighbors_count(comm, &indegree, &outdegree, &weighted); + err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree); if (MPI_SUCCESS == err) { if (MPI_IN_PLACE != sendbuf) { for ( i = 0; i < outdegree; i++ ) { @@ -105,7 +104,7 @@ int MPI_Neighbor_alltoallv(const void *sendbuf, const int sendcounts[], const in return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME); } - err = ompi_comm_neighbors_count(comm, &indegree, &outdegree, &weighted); + err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree); OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME); for (i = 0; i < outdegree; ++i) { OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtype, sendcounts[i]); diff --git a/ompi/mpi/c/neighbor_alltoallw.c b/ompi/mpi/c/neighbor_alltoallw.c index 347d0d81432..5d339bfa6d6 100644 --- a/ompi/mpi/c/neighbor_alltoallw.c +++ b/ompi/mpi/c/neighbor_alltoallw.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -32,7 +32,6 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/memchecker.h" -#include "ompi/communicator/comm_helpers.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -52,7 +51,7 @@ int MPI_Neighbor_alltoallw(const void *sendbuf, const int sendcounts[], const MP const MPI_Datatype recvtypes[], MPI_Comm comm) { int i, err; - int indegree, outdegree, weighted; + int indegree, outdegree; MEMCHECKER( ptrdiff_t recv_ext; @@ -60,7 +59,7 @@ int MPI_Neighbor_alltoallw(const void *sendbuf, const int sendcounts[], const MP memchecker_comm(comm); - err = ompi_comm_neighbors_count(comm, &indegree, &outdegree, &weighted); + err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree); if (MPI_SUCCESS == err) { if (MPI_IN_PLACE != sendbuf) { for ( i = 0; i < outdegree; i++ ) { @@ -101,7 +100,7 @@ int MPI_Neighbor_alltoallw(const void *sendbuf, const int sendcounts[], const MP return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG, FUNC_NAME); } - err = ompi_comm_neighbors_count(comm, &indegree, &outdegree, &weighted); + err = mca_topo_base_neighbor_count (comm, &indegree, &outdegree); OMPI_ERRHANDLER_CHECK(err, comm, err, FUNC_NAME); for (i = 0; i < outdegree; ++i) { OMPI_CHECK_DATATYPE_FOR_SEND(err, sendtypes[i], sendcounts[i]);