diff --git a/opal/mca/btl/base/btl_base_am_rdma.c b/opal/mca/btl/base/btl_base_am_rdma.c index 9ba11bf7291..9ec9ff3e1a8 100644 --- a/opal/mca/btl/base/btl_base_am_rdma.c +++ b/opal/mca/btl/base/btl_base_am_rdma.c @@ -1138,5 +1138,28 @@ int mca_btl_base_am_rdma_init(mca_btl_base_module_t *btl) OBJ_CONSTRUCT(&default_module, mca_btl_base_am_rdma_module_t); } + /* This section check whether we can claim support of remote completion. + * + * In terms of remote completion, we are mainly interested in put and atomic ops, + * because get, atomics fops and atomic cswap support remote completion by their nature. + * + * For active message put (AM put), the target side will send a response, and the initiator + * side will wait for the response to complete the put operation. Thus if AM put is based on send, + * it support remote completion. (If AM put is based on get, it does not support remote + * completion because the target side does not wait for get's completion to send response). + * + * active message RDMA/atomics does not implement atomic ops. User was suppose to + * use atomic fops (unless the btl support atomic ops natively). + * + * In all, the conditions for AM rdma to claim support of remote completion are: + * 1. AM put is enabled (which means the btl does not support put) + * 2. AM put does not use get (so it must use send) + * 3. btl does not have native atomics ops support. + */ + if ((btl->btl_flags & MCA_BTL_FLAGS_PUT_AM) && !mca_btl_base_rdma_use_rdma_get(btl) && + !(btl->btl_flags & MCA_BTL_FLAGS_ATOMIC_OPS)) { + btl->btl_flags |= MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION; + } + return OPAL_SUCCESS; } diff --git a/opal/mca/btl/btl.h b/opal/mca/btl/btl.h index c7f8e581064..80fbc2c2afd 100644 --- a/opal/mca/btl/btl.h +++ b/opal/mca/btl/btl.h @@ -263,6 +263,16 @@ typedef uint8_t mca_btl_base_tag_t; /* The BTL has active-message based atomics */ #define MCA_BTL_FLAGS_ATOMIC_AM_FOP 0x400000 +/** Ths BTL's RDMA/atomics operation supports remote completion. + * When the BTL reported the completion of a RDMA/atomic operation + * on the initator side, the operation also finished on the target side. + * + * Note, this flag is for put and atomic write operations. Operations + * like get, atomic fetch and atomic swap support remote + * completion by nature. + */ +#define MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION 0x800000 + /* Default exclusivity levels */ #define MCA_BTL_EXCLUSIVITY_HIGH (64 * 1024) /* internal loopback */ #define MCA_BTL_EXCLUSIVITY_DEFAULT 1024 /* GM/IB/etc. */ diff --git a/opal/mca/btl/ofi/btl_ofi_module.c b/opal/mca/btl/ofi/btl_ofi_module.c index cffa0c27317..e632b5f8bd2 100644 --- a/opal/mca/btl/ofi/btl_ofi_module.c +++ b/opal/mca/btl/ofi/btl_ofi_module.c @@ -390,8 +390,10 @@ mca_btl_ofi_module_t *mca_btl_ofi_module_alloc(int mode) module->super.btl_register_mem = mca_btl_ofi_register_mem; module->super.btl_deregister_mem = mca_btl_ofi_deregister_mem; + /* btl/ofi support remote completion because it required FI_DELIVERY_COMPLETE capability + */ module->super.btl_flags |= MCA_BTL_FLAGS_ATOMIC_FOPS | MCA_BTL_FLAGS_ATOMIC_OPS - | MCA_BTL_FLAGS_RDMA; + | MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION; module->super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_SWAP | MCA_BTL_ATOMIC_SUPPORTS_CSWAP diff --git a/opal/mca/btl/self/btl_self_component.c b/opal/mca/btl/self/btl_self_component.c index 576716e4d73..875fd066bb7 100644 --- a/opal/mca/btl/self/btl_self_component.c +++ b/opal/mca/btl/self/btl_self_component.c @@ -107,6 +107,8 @@ static int mca_btl_self_component_register(void) mca_btl_self.btl_rdma_pipeline_frag_size = INT_MAX; mca_btl_self.btl_min_rdma_pipeline_size = 0; mca_btl_self.btl_flags = MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND; + /* for self, remote completion is local completion */ + mca_btl_self.btl_flags |= MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION; mca_btl_self.btl_bandwidth = 100; mca_btl_self.btl_latency = 0; mca_btl_base_param_register(&mca_btl_self_component.super.btl_version, &mca_btl_self); diff --git a/opal/mca/btl/ugni/btl_ugni_component.c b/opal/mca/btl/ugni/btl_ugni_component.c index b9441b51b0d..5af4e396507 100644 --- a/opal/mca/btl/ugni/btl_ugni_component.c +++ b/opal/mca/btl/ugni/btl_ugni_component.c @@ -469,6 +469,7 @@ static int btl_ugni_component_register(void) mca_btl_ugni_module.super.btl_flags = MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_ATOMIC_OPS | MCA_BTL_FLAGS_ATOMIC_FOPS; + mca_btl_ugni_module.super.btl_flags |= MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION; mca_btl_ugni_module.super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_AND | MCA_BTL_ATOMIC_SUPPORTS_OR