From 485c60aa8cf8aaccb8e064373f3d61d54273ea48 Mon Sep 17 00:00:00 2001 From: Gilles Gouaillardet Date: Wed, 12 Jul 2017 11:48:48 +0900 Subject: [PATCH 1/6] Revert "Remove --enable-heterogeneous until fix is ready" This reverts commit open-mpi/ompi@2f4b3ab3de5f696e51733562a2d6a382413ae521. Signed-off-by: Gilles Gouaillardet --- config/opal_configure_options.m4 | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/config/opal_configure_options.m4 b/config/opal_configure_options.m4 index 26fe653396f..c7f6e7b4288 100644 --- a/config/opal_configure_options.m4 +++ b/config/opal_configure_options.m4 @@ -286,7 +286,22 @@ fi AC_DEFINE_UNQUOTED(OPAL_ENABLE_DLOPEN_SUPPORT, $OPAL_ENABLE_DLOPEN_SUPPORT, [Whether we want to enable dlopen support]) -opal_want_heterogeneous=0 +# +# Heterogeneous support +# + +AC_MSG_CHECKING([if want heterogeneous support]) +AC_ARG_ENABLE([heterogeneous], + [AC_HELP_STRING([--enable-heterogeneous], + [Enable features required for heterogeneous + platform support (default: disabled)])]) +if test "$enable_heterogeneous" = "yes" ; then + AC_MSG_RESULT([yes]) + opal_want_heterogeneous=1 +else + AC_MSG_RESULT([no]) + opal_want_heterogeneous=0 +fi AC_DEFINE_UNQUOTED([OPAL_ENABLE_HETEROGENEOUS_SUPPORT], [$opal_want_heterogeneous], [Enable features required for heterogeneous support]) From 912466bc876412a256654c88237a236a9987d23d Mon Sep 17 00:00:00 2001 From: Gilles Gouaillardet Date: Wed, 30 Mar 2016 16:32:53 +0900 Subject: [PATCH 2/6] btl/tcp: fix heterogeneous support for put / large messages Signed-off-by: Gilles Gouaillardet (cherry picked from commit open-mpi/ompi@32606ad47605c335cac5d3176754defeb7376afe) --- opal/mca/btl/tcp/btl_tcp.c | 3 ++- opal/mca/btl/tcp/btl_tcp_frag.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/opal/mca/btl/tcp/btl_tcp.c b/opal/mca/btl/tcp/btl_tcp.c index ac6289cf1f9..0800327c549 100644 --- a/opal/mca/btl/tcp/btl_tcp.c +++ b/opal/mca/btl/tcp/btl_tcp.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2006-2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Research Organization for Information Science + * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Intel, Inc. All rights reserved. * @@ -381,6 +381,7 @@ int mca_btl_tcp_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t frag->segments[1].seg_addr.lval = remote_address; frag->segments[1].seg_len = size; + if (endpoint->endpoint_nbo) MCA_BTL_BASE_SEGMENT_HTON(frag->segments[1]); frag->base.des_flags = MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_SEND_ALWAYS_CALLBACK; frag->base.des_cbfunc = fake_rdma_complete; diff --git a/opal/mca/btl/tcp/btl_tcp_frag.c b/opal/mca/btl/tcp/btl_tcp_frag.c index 08bf1536db2..75901e9c47a 100644 --- a/opal/mca/btl/tcp/btl_tcp_frag.c +++ b/opal/mca/btl/tcp/btl_tcp_frag.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2014 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2016 Research Organization for Information Science + * Copyright (c) 2015-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2015-2016 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ @@ -291,6 +291,7 @@ bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd) goto repeat; } else if (frag->iov_idx == 2) { for( i = 0; i < frag->hdr.count; i++ ) { + if (btl_endpoint->endpoint_nbo) MCA_BTL_BASE_SEGMENT_NTOH(frag->segments[i]); frag->iov[i+2].iov_base = (IOVBASE_TYPE*)frag->segments[i].seg_addr.pval; frag->iov[i+2].iov_len = frag->segments[i].seg_len; } From 5846c57fc0e2cffb77bc83ed35b330f048fef847 Mon Sep 17 00:00:00 2001 From: Gilles Gouaillardet Date: Mon, 23 Jan 2017 09:06:40 +0900 Subject: [PATCH 3/6] oob/tcp: make mca_oob_tcp_msg_type_t an uint8_t so no conversion is required when heterogeneous mode is enabled Signed-off-by: Gilles Gouaillardet (cherry picked from commit open-mpi/ompi@626e94b68912aea4a0b84c3f6dc555b79768fe7a) --- orte/mca/oob/tcp/oob_tcp_hdr.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/orte/mca/oob/tcp/oob_tcp_hdr.h b/orte/mca/oob/tcp/oob_tcp_hdr.h index 5ce87749bf9..afbd77ddd63 100644 --- a/orte/mca/oob/tcp/oob_tcp_hdr.h +++ b/orte/mca/oob/tcp/oob_tcp_hdr.h @@ -13,6 +13,8 @@ * All rights reserved. * Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * * $COPYRIGHT$ * @@ -32,12 +34,12 @@ * the message came from an external (to * this component) source */ -typedef enum { - MCA_OOB_TCP_IDENT, - MCA_OOB_TCP_PROBE, - MCA_OOB_TCP_PING, - MCA_OOB_TCP_USER -} mca_oob_tcp_msg_type_t; +typedef uint8_t mca_oob_tcp_msg_type_t; + +#define MCA_OOB_TCP_IDENT 1 +#define MCA_OOB_TCP_PROBE 2 +#define MCA_OOB_TCP_PING 3 +#define MCA_OOB_TCP_USER 4 #define ORTE_MAX_RTD_SIZE 31 @@ -54,14 +56,14 @@ typedef struct { * and let some other module try to send it */ orte_process_name_t dst; - /* type of message */ - mca_oob_tcp_msg_type_t type; /* the rml tag where this message is headed */ orte_rml_tag_t tag; /* the seq number of this message */ uint32_t seq_num; /* number of bytes in message */ uint32_t nbytes; + /* type of message */ + mca_oob_tcp_msg_type_t type; /* routed module to be used */ char routed[ORTE_MAX_RTD_SIZE+1]; } mca_oob_tcp_hdr_t; @@ -71,7 +73,6 @@ typedef struct { #define MCA_OOB_TCP_HDR_NTOH(h) \ ORTE_PROCESS_NAME_NTOH((h)->origin); \ ORTE_PROCESS_NAME_NTOH((h)->dst); \ - (h)->type = ntohl((h)->type); \ (h)->tag = ORTE_RML_TAG_NTOH((h)->tag); \ (h)->nbytes = ntohl((h)->nbytes); @@ -81,7 +82,6 @@ typedef struct { #define MCA_OOB_TCP_HDR_HTON(h) \ ORTE_PROCESS_NAME_HTON((h)->origin); \ ORTE_PROCESS_NAME_HTON((h)->dst); \ - (h)->type = htonl((h)->type); \ (h)->tag = ORTE_RML_TAG_HTON((h)->tag); \ (h)->nbytes = htonl((h)->nbytes); From c4184ae1a412dc272131361e34accc4326a855ab Mon Sep 17 00:00:00 2001 From: Gilles Gouaillardet Date: Mon, 8 May 2017 09:17:00 +0900 Subject: [PATCH 4/6] opal/ddt: use optimized description when packing contiguous datatypes Signed-off-by: Gilles Gouaillardet (cherry picked from commit open-mpi/ompi@9118777b669bd051acace871f4fa646a2551478b) --- opal/datatype/opal_convertor.c | 5 ++++- opal/datatype/opal_convertor.h | 1 + opal/datatype/opal_datatype_fake_stack.c | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/opal/datatype/opal_convertor.c b/opal/datatype/opal_convertor.c index 18cbaf9c970..166973a0bf0 100644 --- a/opal/datatype/opal_convertor.c +++ b/opal/datatype/opal_convertor.c @@ -481,7 +481,9 @@ size_t opal_convertor_compute_remote_size( opal_convertor_t* pConvertor ) pConvertor->remote_size = pConvertor->local_size; if( OPAL_UNLIKELY(datatype->bdt_used & pConvertor->master->hetero_mask) ) { pConvertor->flags &= (~CONVERTOR_HOMOGENEOUS); - pConvertor->use_desc = &(datatype->desc); + if (!(pConvertor->flags & CONVERTOR_SEND && pConvertor->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS)) { + pConvertor->use_desc = &(datatype->desc); + } if( 0 == (pConvertor->flags & CONVERTOR_HAS_REMOTE_SIZE) ) { /* This is for a single datatype, we must update it with the count */ pConvertor->remote_size = opal_datatype_compute_remote_size(datatype, @@ -570,6 +572,7 @@ int32_t opal_convertor_prepare_for_recv( opal_convertor_t* convertor, mca_cuda_convertor_init(convertor, pUserBuf); #endif + assert(! (convertor->flags & CONVERTOR_SEND)); OPAL_CONVERTOR_PREPARE( convertor, datatype, count, pUserBuf ); if( convertor->flags & CONVERTOR_WITH_CHECKSUM ) { diff --git a/opal/datatype/opal_convertor.h b/opal/datatype/opal_convertor.h index 85956af88d7..d6e164c9b62 100644 --- a/opal/datatype/opal_convertor.h +++ b/opal/datatype/opal_convertor.h @@ -218,6 +218,7 @@ static inline void opal_convertor_get_unpacked_size( const opal_convertor_t* pCo return; } if( 0 == (CONVERTOR_HAS_REMOTE_SIZE & pConv->flags) ) { + assert(! (pConv->flags & CONVERTOR_SEND)); opal_convertor_compute_remote_size( (opal_convertor_t*)pConv); } *pSize = pConv->remote_size; diff --git a/opal/datatype/opal_datatype_fake_stack.c b/opal/datatype/opal_datatype_fake_stack.c index d336f6cf76d..1cc05fe8860 100644 --- a/opal/datatype/opal_datatype_fake_stack.c +++ b/opal/datatype/opal_datatype_fake_stack.c @@ -91,6 +91,7 @@ int opal_convertor_create_stack_with_pos_general( opal_convertor_t* pConvertor, } /* remove from the main loop all the complete datatypes */ + assert (! (pConvertor->flags & CONVERTOR_SEND)); remote_size = opal_convertor_compute_remote_size( pConvertor ); count = (int32_t)(starting_point / remote_size); resting_place -= (remote_size * count); From cbaf3b65f31246cf98bbea091e676febe91efa3c Mon Sep 17 00:00:00 2001 From: Gilles Gouaillardet Date: Fri, 7 Jul 2017 09:21:39 +0900 Subject: [PATCH 5/6] opal/datatype: add minimal support to convert long double between ieee 754 quadruple precision and extended precision formats. Signed-off-by: Gilles Gouaillardet (cherry picked from commit open-mpi/ompi@8fd08b933a36cfa9d3035ce44e293fd5e8b9137a) --- configure.ac | 2 +- .../opal_copy_functions_heterogeneous.c | 87 ++++++++++++++++++- 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index e79257d2561..a0fccd9b28a 100644 --- a/configure.ac +++ b/configure.ac @@ -588,7 +588,7 @@ AC_CACHE_SAVE opal_show_title "Header file tests" AC_CHECK_HEADERS([alloca.h aio.h arpa/inet.h dirent.h \ - dlfcn.h endian.h execinfo.h err.h fcntl.h grp.h libgen.h \ + dlfcn.h endian.h execinfo.h err.h fcntl.h grp.h ieee754 libgen.h \ libutil.h memory.h netdb.h netinet/in.h netinet/tcp.h \ poll.h pthread.h pty.h pwd.h sched.h \ strings.h stropts.h linux/ethtool.h linux/sockios.h \ diff --git a/opal/datatype/opal_copy_functions_heterogeneous.c b/opal/datatype/opal_copy_functions_heterogeneous.c index 56d27b82e3e..0ad331e565b 100644 --- a/opal/datatype/opal_copy_functions_heterogeneous.c +++ b/opal/datatype/opal_copy_functions_heterogeneous.c @@ -5,6 +5,7 @@ * reserved. * Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2015-2017 Research Organization for Information Science + * Copyright (c) 2015-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -15,6 +16,10 @@ #include "opal_config.h" +#ifdef HAVE_IEEE754_H +#include +#endif + #include #include @@ -62,6 +67,64 @@ opal_dt_swap_bytes(void *to_p, const void *from_p, const size_t size, size_t cou } } +#ifdef HAVE_IEEE754_H +struct bit128 { + unsigned int mantissa3:32; + unsigned int mantissa2:32; + unsigned int mantissa1:32; + unsigned int mantissa0:16; + unsigned int exponent:15; + unsigned int negative:1; +}; + +struct bit80 { + unsigned int pad:32; + unsigned int empty:16; + unsigned int negative:1; + unsigned int exponent:15; + unsigned int mantissa0:32; + unsigned int mantissa1:32; +}; + +static inline void +opal_dt_swap_long_double(void *to_p, const void *from_p, const size_t size, size_t count, uint32_t remoteArch) +{ + size_t i; + long double*to = (long double *) to_p; + + if ((opal_local_arch&OPAL_ARCH_LDISINTEL) && !(remoteArch&OPAL_ARCH_LDISINTEL)) { +#ifdef __x86_64 + for (i=0; imantissa0 << 15) & 0x7FFF8000) | ((b->mantissa1 >> 17) & 0x00007FFF); + ld.ieee.mantissa1 = ((b->mantissa1 << 15) & 0xFFFF8000) | ((b->mantissa2 << 17) & 0x000007FFF); + ld.ieee.exponent = b->exponent; + ld.ieee.negative = b->negative; + MEMCPY( to, &ld, sizeof(long double)); + } +#endif + } else if (!(opal_local_arch&OPAL_ARCH_LDISINTEL) && (remoteArch&OPAL_ARCH_LDISINTEL)) { +#ifdef __sparcv9 + for (i=0; imantissa0 << 1) | (b->mantissa1 & 0x80000000); + ld.ieee.mantissa1 = (b->mantissa1 << 1) & 0xFFFFFFFE; + ld.ieee.exponent = b->exponent; + ld.ieee.negative = b->negative; + MEMCPY( to, &ld, sizeof(long double)); + } +#endif + } +} +#else +#define opal_dt_swap_long_double(to_p, from_p, size, count, remoteArch) +#endif + /** * BEWARE: Do not use the following macro with composed types such as * complex. As the swap is done using the entire type sizeof, the @@ -69,6 +132,9 @@ opal_dt_swap_bytes(void *to_p, const void *from_p, const size_t size, size_t cou * COPY_2SAMETYPE_HETEROGENEOUS. */ #define COPY_TYPE_HETEROGENEOUS( TYPENAME, TYPE ) \ + COPY_TYPE_HETEROGENEOUS_INTERNAL( TYPENAME, TYPE, 0 ) + +#define COPY_TYPE_HETEROGENEOUS_INTERNAL( TYPENAME, TYPE, LONG_DOUBLE ) \ static int32_t \ copy_##TYPENAME##_heterogeneous(opal_convertor_t *pConvertor, uint32_t count, \ const char* from, size_t from_len, ptrdiff_t from_extent, \ @@ -85,9 +151,15 @@ copy_##TYPENAME##_heterogeneous(opal_convertor_t *pConvertor, uint32_t count, (opal_local_arch & OPAL_ARCH_ISBIGENDIAN)) { \ if( (to_extent == from_extent) && (to_extent == sizeof(TYPE)) ) { \ opal_dt_swap_bytes(to, from, sizeof(TYPE), count); \ + if (LONG_DOUBLE) { \ + opal_dt_swap_long_double(to, from, sizeof(TYPE), count, pConvertor->remoteArch);\ + } \ } else { \ for( i = 0; i < count; i++ ) { \ opal_dt_swap_bytes(to, from, sizeof(TYPE), 1); \ + if (LONG_DOUBLE) { \ + opal_dt_swap_long_double(to, from, sizeof(TYPE), 1, pConvertor->remoteArch);\ + } \ to += to_extent; \ from += from_extent; \ } \ @@ -108,6 +180,9 @@ copy_##TYPENAME##_heterogeneous(opal_convertor_t *pConvertor, uint32_t count, } #define COPY_2SAMETYPE_HETEROGENEOUS( TYPENAME, TYPE ) \ + COPY_2SAMETYPE_HETEROGENEOUS_INTERNAL( TYPENAME, TYPE, 0) + +#define COPY_2SAMETYPE_HETEROGENEOUS_INTERNAL( TYPENAME, TYPE, LONG_DOUBLE) \ static int32_t \ copy_##TYPENAME##_heterogeneous(opal_convertor_t *pConvertor, uint32_t count, \ const char* from, size_t from_len, ptrdiff_t from_extent, \ @@ -122,11 +197,17 @@ copy_##TYPENAME##_heterogeneous(opal_convertor_t *pConvertor, uint32_t count, \ if ((pConvertor->remoteArch & OPAL_ARCH_ISBIGENDIAN) != \ (opal_local_arch & OPAL_ARCH_ISBIGENDIAN)) { \ - if( (to_extent == from_extent) && (to_extent == sizeof(TYPE)) ) { \ + if( (to_extent == from_extent) && (to_extent == (2 * sizeof(TYPE))) ) { \ opal_dt_swap_bytes(to, from, sizeof(TYPE), 2 * count); \ + if (LONG_DOUBLE) { \ + opal_dt_swap_long_double(to, from, sizeof(TYPE), 2*count, pConvertor->remoteArch);\ + } \ } else { \ for( i = 0; i < count; i++ ) { \ opal_dt_swap_bytes(to, from, sizeof(TYPE), 2); \ + if (LONG_DOUBLE) { \ + opal_dt_swap_long_double(to, from, sizeof(TYPE), 2, pConvertor->remoteArch);\ + } \ to += to_extent; \ from += from_extent; \ } \ @@ -333,7 +414,7 @@ COPY_TYPE_HETEROGENEOUS( float16, float ) #elif SIZEOF_DOUBLE == 16 COPY_TYPE_HETEROGENEOUS( float16, double ) #elif HAVE_LONG_DOUBLE && SIZEOF_LONG_DOUBLE == 16 -COPY_TYPE_HETEROGENEOUS( float16, long double ) +COPY_TYPE_HETEROGENEOUS_INTERNAL( float16, long double, 1) #else /* #error No basic type for copy function for opal_datatype_float16 found */ #define copy_float16_heterogeneous NULL @@ -354,7 +435,7 @@ COPY_2SAMETYPE_HETEROGENEOUS( double_complex, double ) #endif #if HAVE_LONG_DOUBLE__COMPLEX -COPY_2SAMETYPE_HETEROGENEOUS( long_double_complex, long double ) +COPY_2SAMETYPE_HETEROGENEOUS_INTERNAL( long_double_complex, long double, 1) #else /* #error No basic type for copy function for opal_datatype_long_double_complex found */ #define copy_long_double_complex_heterogeneous NULL From 25a8e8696ea7a7455d8c07abe14d4c35fdd51b9c Mon Sep 17 00:00:00 2001 From: Gilles Gouaillardet Date: Fri, 7 Jul 2017 09:22:34 +0900 Subject: [PATCH 6/6] opal/datatype: fix opal_dt_swap_long_double if no IEEE754_H Signed-off-by: Gilles Gouaillardet (cherry picked from commit open-mpi/ompi@a111fc8ff20ae1614cd1a2f4f914f3ce8ccb6257) --- configure.ac | 4 ++-- opal/datatype/opal_copy_functions_heterogeneous.c | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index a0fccd9b28a..804acdce231 100644 --- a/configure.ac +++ b/configure.ac @@ -588,7 +588,7 @@ AC_CACHE_SAVE opal_show_title "Header file tests" AC_CHECK_HEADERS([alloca.h aio.h arpa/inet.h dirent.h \ - dlfcn.h endian.h execinfo.h err.h fcntl.h grp.h ieee754 libgen.h \ + dlfcn.h endian.h execinfo.h err.h fcntl.h grp.h libgen.h \ libutil.h memory.h netdb.h netinet/in.h netinet/tcp.h \ poll.h pthread.h pty.h pwd.h sched.h \ strings.h stropts.h linux/ethtool.h linux/sockios.h \ @@ -599,7 +599,7 @@ AC_CHECK_HEADERS([alloca.h aio.h arpa/inet.h dirent.h \ sys/types.h sys/uio.h sys/un.h net/uio.h sys/utsname.h sys/vfs.h sys/wait.h syslog.h \ termios.h ulimit.h unistd.h util.h utmp.h malloc.h \ ifaddrs.h crt_externs.h regex.h mntent.h paths.h \ - ioLib.h sockLib.h hostLib.h shlwapi.h sys/synch.h db.h ndbm.h zlib.h]) + ioLib.h sockLib.h hostLib.h shlwapi.h sys/synch.h db.h ndbm.h zlib.h ieee754.h]) AC_CHECK_HEADERS([sys/mount.h], [], [], [AC_INCLUDES_DEFAULT diff --git a/opal/datatype/opal_copy_functions_heterogeneous.c b/opal/datatype/opal_copy_functions_heterogeneous.c index 0ad331e565b..a46e87b4dde 100644 --- a/opal/datatype/opal_copy_functions_heterogeneous.c +++ b/opal/datatype/opal_copy_functions_heterogeneous.c @@ -89,6 +89,7 @@ struct bit80 { static inline void opal_dt_swap_long_double(void *to_p, const void *from_p, const size_t size, size_t count, uint32_t remoteArch) { +#ifdef HAVE_IEEE754_H size_t i; long double*to = (long double *) to_p; @@ -120,6 +121,9 @@ opal_dt_swap_long_double(void *to_p, const void *from_p, const size_t size, size } #endif } +#else + assert(0); +#endif } #else #define opal_dt_swap_long_double(to_p, from_p, size, count, remoteArch)