Skip to content

Commit 75b0e6d

Browse files
bosilcabosilca
authored
Merge pull request #11552 from bosilca/topic/null_as_unused_buffers_for_collectives
Pass NULL to the collective components for unused buffers.
2 parents 69f5596 + 1edc1b5 commit 75b0e6d

16 files changed

+188
-49
lines changed

‎ompi/mca/coll/coll.h

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@
3737
* handler invocation, but the collective components provide all other
3838
* functionality.
3939
*
40-
* Component selection is done per commuicator, at Communicator
40+
* Component selection is done per communicator, at Communicator
4141
* construction time. mca_coll_base_comm_select() is used to
42-
* create the list of components available to the componenent
42+
* create the list of components available to the component
4343
* collm_comm_query function, instantiating a module for each
4444
* component that is usable, and sets the module collective function pointers.
4545
* mca_coll_base_comm_select() then loops through the list of available
@@ -59,6 +59,15 @@
5959
* components should be able to handle either style of communicator
6060
* during initialization (although handling may include indicating the
6161
* component is not available).
62+
*
63+
* Unlike the MPI standard, all buffers that are not supposed to be used (and
64+
* therefore where the MPI standard does not require the tuple
65+
* (buffer, datatype, count) to be accessible, are replaced with NULL.
66+
* As an example, the recvbuf for all non-root ranks in an MPI_Reduce,
67+
* will be set to NULL at the MPI API level. The reason behind this is to
68+
* allow collective components to indicate when buffers are really valid,
69+
* such that collective modules delegating collectives to other modules
70+
* can share temporary buffer.
6271
*/
6372

6473
#ifndef OMPI_MCA_COLL_COLL_H
@@ -508,7 +517,7 @@ typedef struct mca_coll_base_component_2_4_0_t mca_coll_base_component_t;
508517
*
509518
* Module interface to the Collective framework. Modules are
510519
* reference counted based on the number of functions from the module
511-
* used on the commuicator. There is at most one module per component
520+
* used on the communicator. There is at most one module per component
512521
* on a given communicator, and there can be many component modules on
513522
* a given communicator.
514523
*

‎ompi/mpi/c/gather.c

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2020 The University of Tennessee and The University
6+
* Copyright (c) 2004-2023 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -189,10 +189,16 @@ int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
189189
(0 == recvcount && (MPI_ROOT == root || MPI_PROC_NULL == root))) {
190190
return MPI_SUCCESS;
191191
}
192+
void* updated_recvbuf;
193+
if (OMPI_COMM_IS_INTRA(comm)) {
194+
updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
195+
} else {
196+
updated_recvbuf = (root == MPI_ROOT) ? recvbuf : NULL;
197+
}
192198

193199
/* Invoke the coll component to perform the back-end operation */
194-
err = comm->c_coll->coll_gather(sendbuf, sendcount, sendtype, recvbuf,
195-
recvcount, recvtype, root, comm,
196-
comm->c_coll->coll_gather_module);
200+
err = comm->c_coll->coll_gather(sendbuf, sendcount, sendtype, updated_recvbuf,
201+
recvcount, recvtype, root, comm,
202+
comm->c_coll->coll_gather_module);
197203
OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
198204
}

‎ompi/mpi/c/gather_init.c

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2020 The University of Tennessee and The University
6+
* Copyright (c) 2004-2023 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -168,8 +168,15 @@ int MPI_Gather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
168168
}
169169
}
170170

171+
void* updated_recvbuf;
172+
if (OMPI_COMM_IS_INTRA(comm)) {
173+
updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
174+
} else {
175+
updated_recvbuf = (root == MPI_ROOT) ? recvbuf : NULL;
176+
}
177+
171178
/* Invoke the coll component to perform the back-end operation */
172-
err = comm->c_coll->coll_gather_init(sendbuf, sendcount, sendtype, recvbuf,
179+
err = comm->c_coll->coll_gather_init(sendbuf, sendcount, sendtype, updated_recvbuf,
173180
recvcount, recvtype, root, comm, info, request,
174181
comm->c_coll->coll_gather_init_module);
175182
if (OPAL_LIKELY(OMPI_SUCCESS == err)) {

‎ompi/mpi/c/gatherv.c

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2020 The University of Tennessee and The University
6+
* Copyright (c) 2004-2023 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -202,8 +202,15 @@ int MPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
202202
}
203203
#endif
204204

205+
void* updated_recvbuf;
206+
if (OMPI_COMM_IS_INTRA(comm)) {
207+
updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
208+
} else {
209+
updated_recvbuf = (root == MPI_ROOT) ? recvbuf : NULL;
210+
}
211+
205212
/* Invoke the coll component to perform the back-end operation */
206-
err = comm->c_coll->coll_gatherv(sendbuf, sendcount, sendtype, recvbuf,
213+
err = comm->c_coll->coll_gatherv(sendbuf, sendcount, sendtype, updated_recvbuf,
207214
recvcounts, displs,
208215
recvtype, root, comm,
209216
comm->c_coll->coll_gatherv_module);

‎ompi/mpi/c/gatherv_init.c

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2020 The University of Tennessee and The University
6+
* Copyright (c) 2004-2023 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -192,8 +192,15 @@ int MPI_Gatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
192192
}
193193
}
194194

195+
void* updated_recvbuf;
196+
if (OMPI_COMM_IS_INTRA(comm)) {
197+
updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
198+
} else {
199+
updated_recvbuf = (root == MPI_ROOT) ? recvbuf : NULL;
200+
}
201+
195202
/* Invoke the coll component to perform the back-end operation */
196-
err = comm->c_coll->coll_gatherv_init(sendbuf, sendcount, sendtype, recvbuf,
203+
err = comm->c_coll->coll_gatherv_init(sendbuf, sendcount, sendtype, updated_recvbuf,
197204
recvcounts, displs, recvtype,
198205
root, comm, info, request,
199206
comm->c_coll->coll_gatherv_init_module);

‎ompi/mpi/c/igather.c

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2020 The University of Tennessee and The University
6+
* Copyright (c) 2004-2023 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -168,8 +168,15 @@ int MPI_Igather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
168168
}
169169
}
170170

171+
void* updated_recvbuf;
172+
if (OMPI_COMM_IS_INTRA(comm)) {
173+
updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
174+
} else {
175+
updated_recvbuf = (root == MPI_ROOT) ? recvbuf : NULL;
176+
}
177+
171178
/* Invoke the coll component to perform the back-end operation */
172-
err = comm->c_coll->coll_igather(sendbuf, sendcount, sendtype, recvbuf,
179+
err = comm->c_coll->coll_igather(sendbuf, sendcount, sendtype, updated_recvbuf,
173180
recvcount, recvtype, root, comm, request,
174181
comm->c_coll->coll_igather_module);
175182
if (OPAL_LIKELY(OMPI_SUCCESS == err)) {

‎ompi/mpi/c/igatherv.c

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2020 The University of Tennessee and The University
6+
* Copyright (c) 2004-2023 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -191,8 +191,15 @@ int MPI_Igatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
191191
}
192192
}
193193

194+
void* updated_recvbuf;
195+
if (OMPI_COMM_IS_INTRA(comm)) {
196+
updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
197+
} else {
198+
updated_recvbuf = (root == MPI_ROOT) ? recvbuf : NULL;
199+
}
200+
194201
/* Invoke the coll component to perform the back-end operation */
195-
err = comm->c_coll->coll_igatherv(sendbuf, sendcount, sendtype, recvbuf,
202+
err = comm->c_coll->coll_igatherv(sendbuf, sendcount, sendtype, updated_recvbuf,
196203
recvcounts, displs, recvtype,
197204
root, comm, request, comm->c_coll->coll_igatherv_module);
198205
if (OPAL_LIKELY(OMPI_SUCCESS == err)) {

‎ompi/mpi/c/ireduce.c

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2020 The University of Tennessee and The University
6+
* Copyright (c) 2004-2023 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -135,10 +135,20 @@ int MPI_Ireduce(const void *sendbuf, void *recvbuf, int count,
135135
return MPI_SUCCESS;
136136
}
137137

138+
void *updated_recvbuf;
139+
const void *updated_sendbuf;
140+
if(OMPI_COMM_IS_INTRA(comm)) {
141+
updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
142+
updated_sendbuf = sendbuf;
143+
} else {
144+
updated_sendbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : sendbuf;
145+
updated_recvbuf = (MPI_ROOT == root) ? recvbuf : NULL;
146+
}
147+
138148
/* Invoke the coll component to perform the back-end operation */
139-
err = comm->c_coll->coll_ireduce(sendbuf, recvbuf, count,
140-
datatype, op, root, comm, request,
141-
comm->c_coll->coll_ireduce_module);
149+
err = comm->c_coll->coll_ireduce(updated_sendbuf, updated_recvbuf, count,
150+
datatype, op, root, comm, request,
151+
comm->c_coll->coll_ireduce_module);
142152
if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
143153
ompi_coll_base_retain_op(*request, op, datatype);
144154
}

‎ompi/mpi/c/iscatter.c

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2020 The University of Tennessee and The University
6+
* Copyright (c) 2004-2023 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -151,10 +151,20 @@ int MPI_Iscatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
151151
}
152152
}
153153

154+
const void *updated_sendbuf;
155+
void *updated_recvbuf;
156+
if (OMPI_COMM_IS_INTRA(comm)) {
157+
updated_sendbuf = (ompi_comm_rank(comm) != root) ? NULL : sendbuf;
158+
updated_recvbuf = recvbuf;
159+
} else {
160+
updated_sendbuf = (MPI_ROOT != root ) ? NULL : sendbuf;
161+
updated_recvbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : recvbuf;
162+
}
163+
154164
/* Invoke the coll component to perform the back-end operation */
155-
err = comm->c_coll->coll_iscatter(sendbuf, sendcount, sendtype, recvbuf,
156-
recvcount, recvtype, root, comm, request,
157-
comm->c_coll->coll_iscatter_module);
165+
err = comm->c_coll->coll_iscatter(updated_sendbuf, sendcount, sendtype, updated_recvbuf,
166+
recvcount, recvtype, root, comm, request,
167+
comm->c_coll->coll_iscatter_module);
158168
if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
159169
if (OMPI_COMM_IS_INTRA(comm)) {
160170
if (MPI_IN_PLACE == recvbuf) {

‎ompi/mpi/c/iscatterv.c

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2020 The University of Tennessee and The University
6+
* Copyright (c) 2004-2023 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -191,10 +191,20 @@ int MPI_Iscatterv(const void *sendbuf, const int sendcounts[], const int displs[
191191
}
192192
}
193193

194+
const void *updated_sendbuf;
195+
void *updated_recvbuf;
196+
if (OMPI_COMM_IS_INTRA(comm)) {
197+
updated_sendbuf = (ompi_comm_rank(comm) != root) ? NULL : sendbuf;
198+
updated_recvbuf = recvbuf;
199+
} else {
200+
updated_sendbuf = (MPI_ROOT != root ) ? NULL : sendbuf;
201+
updated_recvbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : recvbuf;
202+
}
203+
194204
/* Invoke the coll component to perform the back-end operation */
195-
err = comm->c_coll->coll_iscatterv(sendbuf, sendcounts, displs,
196-
sendtype, recvbuf, recvcount, recvtype, root, comm,
197-
request, comm->c_coll->coll_iscatterv_module);
205+
err = comm->c_coll->coll_iscatterv(updated_sendbuf, sendcounts, displs,
206+
sendtype, updated_recvbuf, recvcount, recvtype, root, comm,
207+
request, comm->c_coll->coll_iscatterv_module);
198208
if (OPAL_LIKELY(OMPI_SUCCESS == err)) {
199209
if (OMPI_COMM_IS_INTRA(comm)) {
200210
if (MPI_IN_PLACE == recvbuf) {

‎ompi/mpi/c/reduce.c

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2020 The University of Tennessee and The University
6+
* Copyright (c) 2004-2023 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -144,12 +144,21 @@ int MPI_Reduce(const void *sendbuf, void *recvbuf, int count,
144144
return MPI_SUCCESS;
145145
}
146146

147-
/* Invoke the coll component to perform the back-end operation */
147+
void *updated_recvbuf;
148+
const void *updated_sendbuf;
149+
if(OMPI_COMM_IS_INTRA(comm)) {
150+
updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
151+
updated_sendbuf = sendbuf;
152+
} else {
153+
updated_sendbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : sendbuf;
154+
updated_recvbuf = (MPI_ROOT == root) ? recvbuf : NULL;
155+
}
148156

157+
/* Invoke the coll component to perform the back-end operation */
149158
OBJ_RETAIN(op);
150-
err = comm->c_coll->coll_reduce(sendbuf, recvbuf, count,
151-
datatype, op, root, comm,
152-
comm->c_coll->coll_reduce_module);
159+
err = comm->c_coll->coll_reduce(updated_sendbuf, updated_recvbuf, count,
160+
datatype, op, root, comm,
161+
comm->c_coll->coll_reduce_module);
153162
OBJ_RELEASE(op);
154163
OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
155164
}

‎ompi/mpi/c/reduce_init.c

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2020 The University of Tennessee and The University
6+
* Copyright (c) 2004-2023 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -136,8 +136,18 @@ int MPI_Reduce_init(const void *sendbuf, void *recvbuf, int count,
136136
OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
137137
}
138138

139+
void *updated_recvbuf;
140+
const void *updated_sendbuf;
141+
if(OMPI_COMM_IS_INTRA(comm)) {
142+
updated_recvbuf = (ompi_comm_rank(comm) == root) ? recvbuf : NULL;
143+
updated_sendbuf = sendbuf;
144+
} else {
145+
updated_sendbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : sendbuf;
146+
updated_recvbuf = (MPI_ROOT == root) ? recvbuf : NULL;
147+
}
148+
139149
/* Invoke the coll component to perform the back-end operation */
140-
err = comm->c_coll->coll_reduce_init(sendbuf, recvbuf, count,
150+
err = comm->c_coll->coll_reduce_init(updated_sendbuf, updated_recvbuf, count,
141151
datatype, op, root, comm, info, request,
142152
comm->c_coll->coll_reduce_init_module);
143153
if (OPAL_LIKELY(OMPI_SUCCESS == err)) {

‎ompi/mpi/c/scatter.c

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2020 The University of Tennessee and The University
6+
* Copyright (c) 2004-2023 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -173,9 +173,19 @@ int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
173173
return MPI_SUCCESS;
174174
}
175175

176+
const void *updated_sendbuf;
177+
void *updated_recvbuf;
178+
if (OMPI_COMM_IS_INTRA(comm)) {
179+
updated_sendbuf = (ompi_comm_rank(comm) != root) ? NULL : sendbuf;
180+
updated_recvbuf = recvbuf;
181+
} else {
182+
updated_sendbuf = (MPI_ROOT != root ) ? NULL : sendbuf;
183+
updated_recvbuf = ((MPI_ROOT == root) || (MPI_PROC_NULL == root)) ? NULL : recvbuf;
184+
}
185+
176186
/* Invoke the coll component to perform the back-end operation */
177-
err = comm->c_coll->coll_scatter(sendbuf, sendcount, sendtype, recvbuf,
178-
recvcount, recvtype, root, comm,
179-
comm->c_coll->coll_scatter_module);
187+
err = comm->c_coll->coll_scatter(updated_sendbuf, sendcount, sendtype, updated_recvbuf,
188+
recvcount, recvtype, root, comm,
189+
comm->c_coll->coll_scatter_module);
180190
OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME);
181191
}

0 commit comments

Comments
 (0)