Skip to content

Update to hwloc 2.0.0a #3951

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 1, 2017
Merged

Update to hwloc 2.0.0a #3951

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7a83fdb
Update to hwloc 2.0.0a with shmem support.
Jun 22, 2017
6ebaed8
Restore support for user-provided cpulist
Jul 26, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,9 +302,7 @@ opal/mca/event/libevent*/libevent/libevent_pthreads.pc
opal/mca/event/libevent*/libevent/include/event2/event-config.h

opal/mca/hwloc/hwloc*/hwloc/include/hwloc/autogen/config.h
opal/mca/hwloc/hwloc*/hwloc/include/hwloc/autogen/config.h.in
opal/mca/hwloc/hwloc*/hwloc/include/private/autogen/config.h
opal/mca/hwloc/hwloc*/hwloc/include/private/autogen/config.h.in
opal/mca/hwloc/base/static-components.h.new.extern
opal/mca/hwloc/base/static-components.h.new.struct

Expand Down Expand Up @@ -361,6 +359,7 @@ orte/test/mpi/accept
orte/test/mpi/attach
orte/test/mpi/bad_exit
orte/test/mpi/bcast_loop
orte/test/mpi/binding
orte/test/mpi/concurrent_spawn
orte/test/mpi/connect
orte/test/mpi/crisscross
Expand Down
7 changes: 2 additions & 5 deletions ompi/mca/osc/rdma/osc_rdma_active_target.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
* Copyright (c) 2017 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2017 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -242,10 +243,6 @@ int ompi_osc_rdma_post_atomic (ompi_group_t *group, int assert, ompi_win_t *win)
return OMPI_SUCCESS;
}

if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}

/* translate group ranks into the communicator */
peers = ompi_osc_rdma_get_peers (module, module->pw_group);
if (OPAL_UNLIKELY(NULL == peers)) {
Expand Down Expand Up @@ -281,7 +278,7 @@ int ompi_osc_rdma_post_atomic (ompi_group_t *group, int assert, ompi_win_t *win)
do {
ompi_osc_rdma_lock_t result;

OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "attempting to post to index %d @ rank %d", post_index, peer->rank);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "attempting to post to index %d @ rank %d", (int)post_index, peer->rank);

/* try to post. if the value isn't 0 then another rank is occupying this index */
if (!ompi_osc_rdma_peer_local_state (peer)) {
Expand Down
152 changes: 118 additions & 34 deletions opal/mca/btl/openib/btl_openib_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
* Copyright (c) 2009-2012 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011-2015 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved
* Copyright (c) 2013-2016 Intel, Inc. All rights reserved.
* Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
* Copyright (c) 2014-2017 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2014 Bull SAS. All rights reserved.
Expand Down Expand Up @@ -2330,32 +2330,41 @@ static float get_ib_dev_distance(struct ibv_device *dev)
/* If we don't have hwloc, we'll default to a distance of 0,
because we have no way of measuring. */
float distance = 0;
float a, b;
int i;
hwloc_cpuset_t my_cpuset = NULL, ibv_cpuset = NULL;
hwloc_obj_t my_obj, ibv_obj, node_obj;
struct hwloc_distances_s *hwloc_distances = NULL;

#if HWLOC_API_VERSION < 0x20000
/* Override any distance logic so all devices are used */
if (0 != mca_btl_openib_component.ignore_locality ||
OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
return distance;
}

float a, b;
int i;
hwloc_cpuset_t my_cpuset = NULL, ibv_cpuset = NULL;
hwloc_obj_t my_obj, ibv_obj, node_obj;

/* Note that this struct is owned by hwloc; there's no need to
free it at the end of time */
static const struct hwloc_distances_s *hwloc_distances = NULL;
#if HWLOC_API_VERSION >= 0x20000
unsigned int j, distances_nr = 1;
int ibvindex, myindex;
#endif

if (NULL == hwloc_distances) {
hwloc_distances =
hwloc_get_whole_distance_matrix_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE);
}
#if HWLOC_API_VERSION < 0x20000
hwloc_distances =
hwloc_get_whole_distance_matrix_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE);
/* If we got no info, just return 0 */
if (NULL == hwloc_distances || NULL == hwloc_distances->latency) {
goto out;
}

/* If we got no info, just return 0 */
if (NULL == hwloc_distances || NULL == hwloc_distances->latency) {
goto out;
#else
if (0 != hwloc_distances_get_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE,
&distances_nr, &hwloc_distances,
HWLOC_DISTANCES_KIND_MEANS_LATENCY, 0) || 0 == distances_nr) {
hwloc_distances = NULL;
goto out;
}
#endif
}

/* Next, find the NUMA node where this IBV device is located */
Expand All @@ -2373,16 +2382,31 @@ static float get_ib_dev_distance(struct ibv_device *dev)

opal_output_verbose(5, opal_btl_base_framework.framework_output,
"hwloc_distances->nbobjs=%d", hwloc_distances->nbobjs);
#if HWLOC_API_VERSION < 0x20000
for (i = 0; i < (int)(2 * hwloc_distances->nbobjs); i++) {
opal_output_verbose(5, opal_btl_base_framework.framework_output,
"hwloc_distances->latency[%d]=%f", i, hwloc_distances->latency[i]);
}
#else
for (i = 0; i < (int)hwloc_distances->nbobjs; i++) {
opal_output_verbose(5, opal_btl_base_framework.framework_output,
"hwloc_distances->values[%d]=%"PRIu64, i, hwloc_distances->values[i]);
}
#endif

/* If ibv_obj is a NUMA node or below, we're good. */
switch (ibv_obj->type) {
case HWLOC_OBJ_NODE:
case HWLOC_OBJ_SOCKET:
#if HWLOC_API_VERSION < 0x20000
case HWLOC_OBJ_CACHE:
#else
case HWLOC_OBJ_L1CACHE:
case HWLOC_OBJ_L2CACHE:
case HWLOC_OBJ_L3CACHE:
case HWLOC_OBJ_L4CACHE:
case HWLOC_OBJ_L5CACHE:
#endif
case HWLOC_OBJ_CORE:
case HWLOC_OBJ_PU:
while (NULL != ibv_obj && ibv_obj->type != HWLOC_OBJ_NODE) {
Expand All @@ -2402,6 +2426,22 @@ static float get_ib_dev_distance(struct ibv_device *dev)
if (NULL == ibv_obj) {
goto out;
}
#if HWLOC_API_VERSION >= 0x20000
/* the new matrix format isn't quite as friendly, so we have to
* do an exhaustive search to find the index of this object
* in that array */
ibvindex = -1;
for (j=0; j < distances_nr; j++) {
if (ibv_obj == hwloc_distances->objs[j]) {
ibvindex = j;
break;
}
}
if (-1 == ibvindex) {
OPAL_ERROR_LOG(OPAL_ERR_NOT_FOUND);
goto out;
}
#endif

opal_output_verbose(5, opal_btl_base_framework.framework_output,
"ibv_obj->logical_index=%d", ibv_obj->logical_index);
Expand All @@ -2424,7 +2464,15 @@ static float get_ib_dev_distance(struct ibv_device *dev)
switch (my_obj->type) {
case HWLOC_OBJ_NODE:
case HWLOC_OBJ_SOCKET:
case HWLOC_OBJ_CACHE:
#if HWLOC_API_VERSION < 0x20000
case HWLOC_OBJ_CACHE:
#else
case HWLOC_OBJ_L1CACHE:
case HWLOC_OBJ_L2CACHE:
case HWLOC_OBJ_L3CACHE:
case HWLOC_OBJ_L4CACHE:
case HWLOC_OBJ_L5CACHE:
#endif
case HWLOC_OBJ_CORE:
case HWLOC_OBJ_PU:
while (NULL != my_obj && my_obj->type != HWLOC_OBJ_NODE) {
Expand All @@ -2435,12 +2483,31 @@ static float get_ib_dev_distance(struct ibv_device *dev)
"my_obj->logical_index=%d", my_obj->logical_index);
/* Distance may be asymetrical, so calculate both of them
and take the max */
a = hwloc_distances->latency[my_obj->logical_index +
(ibv_obj->logical_index *
hwloc_distances->nbobjs)];
b = hwloc_distances->latency[ibv_obj->logical_index +
(my_obj->logical_index *
hwloc_distances->nbobjs)];
#if HWLOC_API_VERSION < 0x20000
a = hwloc_distances->latency[my_obj->logical_index +
(ibv_obj->logical_index *
hwloc_distances->nbobjs)];
b = hwloc_distances->latency[ibv_obj->logical_index +
(my_obj->logical_index *
hwloc_distances->nbobjs)];
#else
/* the new matrix format isn't quite as friendly, so we have to
* do an exhaustive search to find the index of this object
* in that array */
myindex = -1;
for (j=0; j < distances_nr; j++) {
if (my_obj == hwloc_distances->objs[j]) {
myindex = j;
break;
}
}
if (-1 == myindex) {
OPAL_ERROR_LOG(OPAL_ERR_NOT_FOUND);
goto out;
}
a = (float)hwloc_distances->values[myindex + (ibvindex * hwloc_distances->nbobjs)];
b = (float)hwloc_distances->values[ibvindex + (myindex * hwloc_distances->nbobjs)];
#endif
distance = (a > b) ? a : b;
}
break;
Expand All @@ -2456,13 +2523,28 @@ static float get_ib_dev_distance(struct ibv_device *dev)
node_obj = hwloc_get_obj_inside_cpuset_by_type(opal_hwloc_topology,
ibv_obj->cpuset,
HWLOC_OBJ_NODE, ++i)) {

a = hwloc_distances->latency[node_obj->logical_index +
(ibv_obj->logical_index *
hwloc_distances->nbobjs)];
b = hwloc_distances->latency[ibv_obj->logical_index +
(node_obj->logical_index *
hwloc_distances->nbobjs)];
#if HWLOC_API_VERSION < 0x20000
a = hwloc_distances->latency[node_obj->logical_index +
(ibv_obj->logical_index *
hwloc_distances->nbobjs)];
b = hwloc_distances->latency[ibv_obj->logical_index +
(node_obj->logical_index *
hwloc_distances->nbobjs)];
#else
unsigned int j;
j = node_obj->logical_index + (ibv_obj->logical_index * hwloc_distances->nbobjs);
if (j < distances_nr) {
a = (float)hwloc_distances->values[j];
} else {
goto out;
}
j = ibv_obj->logical_index + (node_obj->logical_index * hwloc_distances->nbobjs);
if (j < distances_nr) {
b = (float)hwloc_distances->values[j];
} else {
goto out;
}
#endif
a = (a > b) ? a : b;
distance = (a > distance) ? a : distance;
}
Expand All @@ -2476,10 +2558,12 @@ static float get_ib_dev_distance(struct ibv_device *dev)
if (NULL != my_cpuset) {
hwloc_bitmap_free(my_cpuset);
}
#else
#warning FIXME get_ib_dev_distance is not implemented with hwloc v2
#endif

#if HWLOC_API_VERSION < 0x20000
if (NULL != hwloc_distances) {
hwloc_distances_release(opal_hwloc_topology, hwloc_distances);
}
#endif
return distance;
}

Expand Down
4 changes: 1 addition & 3 deletions opal/mca/btl/openib/btl_openib_proc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
* All rights reserved.
* Copyright (c) 2007-2015 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2014 Intel, Inc. All rights reserved.
* Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
* Copyright (c) 2015-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2015 Mellanox Technologies. All rights reserved.
Expand Down Expand Up @@ -77,8 +77,6 @@ void mca_btl_openib_proc_construct(mca_btl_openib_proc_t* ib_proc)

void mca_btl_openib_proc_destruct(mca_btl_openib_proc_t* ib_proc)
{
mca_btl_openib_proc_btlptr_t* elem;

/* release resources */
if(NULL != ib_proc->proc_endpoints) {
free(ib_proc->proc_endpoints);
Expand Down
57 changes: 52 additions & 5 deletions opal/mca/btl/usnic/btl_usnic_hwloc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2013-2016 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2016 Intel, Inc. All rights reserved.
* Copyright (c) 2016-2017 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand All @@ -26,22 +26,34 @@
*/
static hwloc_obj_t my_numa_node = NULL;
static int num_numa_nodes = 0;
static const struct hwloc_distances_s *matrix = NULL;
static struct hwloc_distances_s *matrix = NULL;
#if HWLOC_API_VERSION >= 0x20000
static unsigned int matrix_nr = 1;
#endif

/*
* Get the hwloc distance matrix (if we don't already have it).
*
* Note that the matrix data structure belongs to hwloc; we are not
* responsibile for freeing it.
*/
static int get_distance_matrix(void)
{
#if HWLOC_API_VERSION < 0x20000
/* Note that the matrix data structure belongs to hwloc; we are not
* responsible for freeing it. */

if (NULL == matrix) {
matrix = hwloc_get_whole_distance_matrix_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE);
}

return (NULL == matrix) ? OPAL_ERROR : OPAL_SUCCESS;
#else
if (0 != hwloc_distances_get_by_type(opal_hwloc_topology, HWLOC_OBJ_NODE,
&matrix_nr, &matrix,
HWLOC_DISTANCES_KIND_MEANS_LATENCY, 0) || 0 == matrix_nr) {
return OPAL_ERROR;
}
return OPAL_SUCCESS;
#endif
}

/*
Expand Down Expand Up @@ -219,6 +231,7 @@ int opal_btl_usnic_hwloc_distance(opal_btl_usnic_module_t *module)

/* Lookup the distance between my NUMA node and the NUMA node of
the device */
#if HWLOC_API_VERSION < 0x20000
if (NULL != dev_numa) {
module->numa_distance =
matrix->latency[dev_numa->logical_index * num_numa_nodes +
Expand All @@ -229,6 +242,40 @@ int opal_btl_usnic_hwloc_distance(opal_btl_usnic_module_t *module)
module->linux_device_name,
module->numa_distance);
}
#else
if (NULL != dev_numa) {
int myindex, devindex;
unsigned int j;
myindex = -1;
for (j=0; j < matrix_nr; j++) {
if (my_numa_node == matrix->objs[j]) {
myindex = j;
break;
}
}
if (-1 == myindex) {
return OPAL_SUCCESS;
}
devindex = -1;
for (j=0; j < matrix_nr; j++) {
if (dev_numa == matrix->objs[j]) {
devindex = j;
break;
}
}
if (-1 == devindex) {
return OPAL_SUCCESS;
}

module->numa_distance =
matrix->values[(devindex * num_numa_nodes) + myindex];

opal_output_verbose(5, USNIC_OUT,
"btl:usnic:filter_numa: %s is distance %d from me",
module->linux_device_name,
module->numa_distance);
}
#endif

return OPAL_SUCCESS;
}
Loading