diff --git a/ompi/communicator/comm.c b/ompi/communicator/comm.c index 8f9c95ade16..3ae6513d5bb 100644 --- a/ompi/communicator/comm.c +++ b/ompi/communicator/comm.c @@ -1411,7 +1411,7 @@ int ompi_intercomm_create (ompi_communicator_t *local_comm, int local_leader, om } /* put group elements in the list */ - new_group_pointer = ompi_group_allocate_plist_w_procs (rprocs, rsize); + new_group_pointer = ompi_group_allocate_plist_w_procs (local_comm->c_local_group, rprocs, rsize); if (OPAL_UNLIKELY(NULL == new_group_pointer)) { free (rprocs); return MPI_ERR_GROUP; @@ -1525,7 +1525,7 @@ int ompi_intercomm_create_from_groups (ompi_group_t *local_group, int local_lead return OMPI_ERR_OUT_OF_RESOURCE; } - leader_group = ompi_group_allocate_plist_w_procs (leader_procs, 2); + leader_group = ompi_group_allocate_plist_w_procs (NULL, leader_procs, 2); ompi_set_group_rank (leader_group, my_proc); if (OPAL_UNLIKELY(NULL == leader_group)) { free (sub_tag); @@ -1577,7 +1577,7 @@ int ompi_intercomm_create_from_groups (ompi_group_t *local_group, int local_lead if (!i_am_leader) { /* create a new group containing the remote processes for non-leader ranks */ - remote_group = ompi_group_allocate_plist_w_procs (rprocs, rsize); + remote_group = ompi_group_allocate_plist_w_procs (local_group, rprocs, rsize); if (OPAL_UNLIKELY(NULL == remote_group)) { free (rprocs); ompi_comm_free (&local_comm); @@ -2341,6 +2341,10 @@ static int ompi_comm_fill_rest(ompi_communicator_t *comm, int my_rank, ompi_errhandler_t *errh) { + ompi_group_t *new_group; + + new_group = ompi_group_allocate_plist_w_procs(comm->c_local_group, proc_pointers, num_procs); + /* properly decrement the ref counts on the groups. We are doing this because this function is sort of a redo of what is done in comm.c. No need to decrement the ref @@ -2356,7 +2360,7 @@ static int ompi_comm_fill_rest(ompi_communicator_t *comm, } /* allocate a group structure for the new communicator */ - comm->c_local_group = ompi_group_allocate_plist_w_procs (proc_pointers, num_procs); + comm->c_local_group = new_group; /* set the remote group to be the same as local group */ comm->c_remote_group = comm->c_local_group; diff --git a/ompi/communicator/comm_init.c b/ompi/communicator/comm_init.c index da26158ee86..626b9ceb824 100644 --- a/ompi/communicator/comm_init.c +++ b/ompi/communicator/comm_init.c @@ -23,7 +23,7 @@ * and Technology (RIST). All rights reserved. * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016-2017 IBM Corporation. All rights reserved. - * Copyright (c) 2018-2019 Triad National Security, LLC. All rights + * Copyright (c) 2018-2022 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -303,33 +303,6 @@ int ompi_comm_init_mpi3 (void) return OMPI_SUCCESS; } - -ompi_communicator_t *ompi_comm_allocate ( int local_size, int remote_size ) -{ - ompi_communicator_t *new_comm; - - /* create new communicator element */ - new_comm = OBJ_NEW(ompi_communicator_t); - new_comm->super.s_info = NULL; - new_comm->c_local_group = ompi_group_allocate ( local_size ); - if ( 0 < remote_size ) { - new_comm->c_remote_group = ompi_group_allocate (remote_size); - new_comm->c_flags |= OMPI_COMM_INTER; - } else { - /* - * simplifies some operations (e.g. p2p), if - * we can always use the remote group - */ - new_comm->c_remote_group = new_comm->c_local_group; - OBJ_RETAIN(new_comm->c_remote_group); - } - - /* fill in the inscribing hyper-cube dimensions */ - new_comm->c_cube_dim = opal_cube_dim(local_size); - - return new_comm; -} - static int ompi_comm_finalize (void) { int max, i; diff --git a/ompi/communicator/communicator.h b/ompi/communicator/communicator.h index a90f1f076e0..67c481c309d 100644 --- a/ompi/communicator/communicator.h +++ b/ompi/communicator/communicator.h @@ -22,7 +22,7 @@ * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016-2017 IBM Corporation. All rights reserved. - * Copyright (c) 2018-2021 Triad National Security, LLC. All rights + * Copyright (c) 2018-2022 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -935,19 +935,6 @@ int ompi_comm_compare(ompi_communicator_t *comm1, ompi_communicator_t *comm2, in */ OMPI_DECLSPEC int ompi_comm_free (ompi_communicator_t **comm); -/** - * allocate a new communicator structure - * @param local_group_size - * @param remote_group_size - * - * This routine allocates the structure, the according local and - * remote groups, the proc-arrays in the local and remote group. - * It furthermore sets the fortran index correctly, - * and sets all other elements to zero. - */ -ompi_communicator_t* ompi_comm_allocate (int local_group_size, - int remote_group_size); - /** * allocate new communicator ID * @param newcomm: pointer to the new communicator diff --git a/ompi/dpm/dpm.c b/ompi/dpm/dpm.c index d45f37b8596..b1206c2868c 100644 --- a/ompi/dpm/dpm.c +++ b/ompi/dpm/dpm.c @@ -21,7 +21,7 @@ * and Technology (RIST). All rights reserved. * Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All Rights reserved. * Copyright (c) 2021 Nanook Consulting. All rights reserved. - * Copyright (c) 2018-2021 Triad National Security, LLC. All rights + * Copyright (c) 2018-2022 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -468,7 +468,7 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root, /* now deal with the remote group */ rsize = opal_list_get_size(&rlist); - new_group_pointer=ompi_group_allocate(rsize); + new_group_pointer=ompi_group_allocate(NULL, rsize); if (NULL == new_group_pointer) { rc = OMPI_ERR_OUT_OF_RESOURCE; OPAL_LIST_DESTRUCT(&rlist); diff --git a/ompi/group/group.h b/ompi/group/group.h index 1e87ecd8556..6b0cd37501d 100644 --- a/ompi/group/group.h +++ b/ompi/group/group.h @@ -18,7 +18,7 @@ * reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2018 Triad National Security, LLC. All rights + * Copyright (c) 2018-2022 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -163,15 +163,16 @@ OMPI_DECLSPEC extern struct opal_mutex_t ompi_group_afp_mutex; /** * Allocate a new group structure. * + * @param orig_group Original group * @param group_size Number of MPI processes in the group * * @return Pointer to new group structure */ -OMPI_DECLSPEC ompi_group_t *ompi_group_allocate(int group_size); -ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_proc_t **procs, int group_size); -ompi_group_t *ompi_group_allocate_sporadic(int group_size); -ompi_group_t *ompi_group_allocate_strided(void); -ompi_group_t *ompi_group_allocate_bmap(int orig_group_size, int group_size); +OMPI_DECLSPEC ompi_group_t *ompi_group_allocate(ompi_group_t *orig_group, int group_size); +ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_group_t *orig_group, ompi_proc_t **procs, int group_size); +ompi_group_t *ompi_group_allocate_sporadic(ompi_group_t *orig_group, int group_size); +ompi_group_t *ompi_group_allocate_strided(ompi_group_t *orig_group); +ompi_group_t *ompi_group_allocate_bmap(ompi_group_t *orig_group, int group_size); /** * @brief Allocate a dense group from a group diff --git a/ompi/group/group_bitmap.c b/ompi/group/group_bitmap.c index c93184f162c..21bf95acd6b 100644 --- a/ompi/group/group_bitmap.c +++ b/ompi/group/group_bitmap.c @@ -14,6 +14,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. + * Copyright (c) 2022 Triad National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -159,7 +161,7 @@ int ompi_group_incl_bmap(ompi_group_t* group, int n, const int *ranks, return OMPI_SUCCESS; } - new_group_pointer = ompi_group_allocate_bmap(group->grp_proc_count, n); + new_group_pointer = ompi_group_allocate_bmap(group, n); if( NULL == new_group_pointer ) { return MPI_ERR_GROUP; } diff --git a/ompi/group/group_init.c b/ompi/group/group_init.c index 3a92d888f2e..158cb32224c 100644 --- a/ompi/group/group_init.c +++ b/ompi/group/group_init.c @@ -16,7 +16,7 @@ * Copyright (c) 2012 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2018 Triad National Security, LLC. All rights + * Copyright (c) 2018-2022 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -71,7 +71,7 @@ opal_mutex_t ompi_group_afp_mutex = OPAL_MUTEX_STATIC_INIT; /* * Allocate a new group structure */ -ompi_group_t *ompi_group_allocate(int group_size) +ompi_group_t *ompi_group_allocate(ompi_group_t *orig_group, int group_size) { /* local variables */ ompi_proc_t **procs = calloc (group_size, sizeof (ompi_proc_t *)); @@ -81,7 +81,7 @@ ompi_group_t *ompi_group_allocate(int group_size) return NULL; } - new_group = ompi_group_allocate_plist_w_procs (procs, group_size); + new_group = ompi_group_allocate_plist_w_procs (orig_group, procs, group_size); if (NULL == new_group) { free (procs); } @@ -89,7 +89,7 @@ ompi_group_t *ompi_group_allocate(int group_size) return new_group; } -ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_proc_t **procs, int group_size) +ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_group_t *orig_group, ompi_proc_t **procs, int group_size) { /* local variables */ ompi_group_t * new_group = NULL; @@ -121,12 +121,18 @@ ompi_group_t *ompi_group_allocate_plist_w_procs (ompi_proc_t **procs, int group_ new_group->grp_my_rank = MPI_UNDEFINED; OMPI_GROUP_SET_DENSE(new_group); + if (NULL != orig_group) { + new_group->grp_instance = orig_group->grp_instance; + } else { + new_group->grp_instance = NULL; + } + ompi_group_increment_proc_count (new_group); return new_group; } -ompi_group_t *ompi_group_allocate_sporadic(int group_size) +ompi_group_t *ompi_group_allocate_sporadic(ompi_group_t *orig_group, int group_size) { /* local variables */ ompi_group_t *new_group = NULL; @@ -165,13 +171,14 @@ ompi_group_t *ompi_group_allocate_sporadic(int group_size) /* initialize our rank to MPI_UNDEFINED */ new_group->grp_my_rank = MPI_UNDEFINED; new_group->grp_proc_pointers = NULL; + new_group->grp_instance = orig_group->grp_instance; OMPI_GROUP_SET_SPORADIC(new_group); error_exit: return new_group; } -ompi_group_t *ompi_group_allocate_strided(void) +ompi_group_t *ompi_group_allocate_strided(ompi_group_t *orig_group) { ompi_group_t *new_group = NULL; @@ -188,6 +195,7 @@ ompi_group_t *ompi_group_allocate_strided(void) /* initialize our rank to MPI_UNDEFINED */ new_group->grp_my_rank = MPI_UNDEFINED; new_group->grp_proc_pointers = NULL; + new_group->grp_instance = orig_group->grp_instance; OMPI_GROUP_SET_STRIDED(new_group); new_group->sparse_data.grp_strided.grp_strided_stride = -1; new_group->sparse_data.grp_strided.grp_strided_offset = -1; @@ -196,9 +204,13 @@ ompi_group_t *ompi_group_allocate_strided(void) /* return */ return new_group; } -ompi_group_t *ompi_group_allocate_bmap(int orig_group_size , int group_size) + +ompi_group_t *ompi_group_allocate_bmap(ompi_group_t *orig_group , int group_size) { ompi_group_t *new_group = NULL; + int orig_group_size; + + orig_group_size = orig_group->grp_proc_count; assert (group_size >= 0); @@ -224,6 +236,7 @@ ompi_group_t *ompi_group_allocate_bmap(int orig_group_size , int group_size) /* initialize our rank to MPI_UNDEFINED */ new_group->grp_my_rank = MPI_UNDEFINED; new_group->grp_proc_pointers = NULL; + new_group->grp_instance = orig_group->grp_instance; OMPI_GROUP_SET_BITMAP(new_group); error_exit: diff --git a/ompi/group/group_plist.c b/ompi/group/group_plist.c index 771bd921efa..39fce07069f 100644 --- a/ompi/group/group_plist.c +++ b/ompi/group/group_plist.c @@ -17,6 +17,8 @@ * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 Intel, Inc. All rights reserved. + * Copyright (c) 2022 Triad National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -81,7 +83,7 @@ int ompi_group_incl_plist(ompi_group_t* group, int n, const int *ranks, } /* get new group struct */ - new_group_pointer=ompi_group_allocate(n); + new_group_pointer=ompi_group_allocate(group,n); if( NULL == new_group_pointer ) { return MPI_ERR_GROUP; } @@ -103,6 +105,8 @@ int ompi_group_incl_plist(ompi_group_t* group, int n, const int *ranks, new_group_pointer->grp_my_rank = MPI_UNDEFINED; } + new_group_pointer->grp_instance = group->grp_instance; + *new_group = (MPI_Group)new_group_pointer; return OMPI_SUCCESS; @@ -148,7 +152,7 @@ int ompi_group_union (ompi_group_t* group1, ompi_group_t* group2, } /* get new group struct */ - new_group_pointer = ompi_group_allocate(new_group_size); + new_group_pointer = ompi_group_allocate(group1, new_group_size); if (NULL == new_group_pointer) { OBJ_DESTRUCT(&bitmap); return MPI_ERR_GROUP; @@ -230,7 +234,7 @@ int ompi_group_difference(ompi_group_t* group1, ompi_group_t* group2, } /* allocate a new ompi_group_t structure */ - new_group_pointer = ompi_group_allocate(new_group_size); + new_group_pointer = ompi_group_allocate(group1, new_group_size); if( NULL == new_group_pointer ) { OBJ_DESTRUCT(&bitmap); return MPI_ERR_GROUP; diff --git a/ompi/group/group_sporadic.c b/ompi/group/group_sporadic.c index 46e0d6bc9f8..ac630a76086 100644 --- a/ompi/group/group_sporadic.c +++ b/ompi/group/group_sporadic.c @@ -14,6 +14,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. + * Copyright (c) 2022 Triad National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -144,7 +146,7 @@ int ompi_group_incl_spor(ompi_group_t* group, int n, const int *ranks, } } - new_group_pointer = ompi_group_allocate_sporadic(l); + new_group_pointer = ompi_group_allocate_sporadic(group,l); if( NULL == new_group_pointer ) { return MPI_ERR_GROUP; } diff --git a/ompi/group/group_strided.c b/ompi/group/group_strided.c index d5f7f2848a8..fa6185204f0 100644 --- a/ompi/group/group_strided.c +++ b/ompi/group/group_strided.c @@ -14,6 +14,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. + * Copyright (c) 2022 Triad National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -117,7 +119,7 @@ int ompi_group_incl_strided(ompi_group_t* group, int n, const int *ranks, } stride = check_stride(ranks,n); - new_group_pointer = ompi_group_allocate_strided(); + new_group_pointer = ompi_group_allocate_strided(group); if( NULL == new_group_pointer ) { return MPI_ERR_GROUP; } diff --git a/ompi/instance/instance.c b/ompi/instance/instance.c index 98d3bf82a8b..9e4f0916734 100644 --- a/ompi/instance/instance.c +++ b/ompi/instance/instance.c @@ -1093,7 +1093,7 @@ static int ompi_instance_group_world (ompi_instance_t *instance, ompi_group_t ** size = ompi_process_info.num_procs; - group = ompi_group_allocate (size); + group = ompi_group_allocate (NULL,size); if (OPAL_UNLIKELY(NULL == group)) { return OMPI_ERR_OUT_OF_RESOURCE; } @@ -1144,7 +1144,7 @@ static int ompi_instance_group_shared (ompi_instance_t *instance, ompi_group_t * size = opal_argv_count (peers); - group = ompi_group_allocate (size); + group = ompi_group_allocate (NULL,size); if (OPAL_UNLIKELY(NULL == group)) { opal_argv_free (peers); return OMPI_ERR_OUT_OF_RESOURCE; @@ -1206,7 +1206,7 @@ static int ompi_instance_group_pmix_pset (ompi_instance_t *instance, const char size_t size = 0; /* make the group large enough to hold world */ - group = ompi_group_allocate (ompi_process_info.num_procs); + group = ompi_group_allocate (NULL, ompi_process_info.num_procs); if (OPAL_UNLIKELY(NULL == group)) { return OMPI_ERR_OUT_OF_RESOURCE; }