btl/sm: avoid using xpmem for fast boxes

xpmem has different behvior than other shared memory support mechanisms.
in particular, any xpmem-attached regions in a process will become invalid
once the exporting process exits.

Under certain circumstances, this behavior can result in SIGBUS errors
during mpi finalize.

Related to open-mpi#9868

Signed-off-by: Howard Pritchard <[email protected]>

Author: hppritcha

opal/mca/btl/sm/btl_sm_component.c

@@ -254,11 +254,6 @@ static int mca_btl_sm_component_close(void)
     OBJ_DESTRUCT(&mca_btl_sm_component.pending_endpoints);
     OBJ_DESTRUCT(&mca_btl_sm_component.pending_fragments);
 
-    if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)
-        && NULL != mca_btl_sm_component.my_segment) {
-        munmap(mca_btl_sm_component.my_segment, mca_btl_sm_component.segment_size);
-    }
-
     mca_btl_sm_component.my_segment = NULL;
 
     if (mca_btl_sm_component.mpool) {
@@ -276,14 +271,9 @@ static int mca_btl_base_sm_modex_send(void)
 
     modex_size = sizeof(modex) - sizeof(modex.seg_ds);
 
-    if (!mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {
         modex.seg_ds_size = opal_shmem_sizeof_shmem_ds(&mca_btl_sm_component.seg_ds);
         memmove(&modex.seg_ds, &mca_btl_sm_component.seg_ds, modex.seg_ds_size);
         modex_size += modex.seg_ds_size;
-    } else {
-        modex.segment_base = (uintptr_t) mca_btl_sm_component.my_segment;
-        modex.seg_ds_size = 0;
-    }
 
     int rc;
     OPAL_MODEX_SEND(rc, PMIX_LOCAL, &mca_btl_sm_component.super.btl_version, &modex, modex_size);
@@ -376,41 +366,29 @@ mca_btl_sm_component_init(int *num_btls, bool enable_progress_threads, bool enab
         mca_btl_sm.super.btl_put = NULL;
     }
 
-    if (!mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {
-        char *sm_file;
+    char *sm_file;
 
-        rc = opal_asprintf(&sm_file, "%s" OPAL_PATH_SEP "sm_segment.%s.%u.%x.%d",
-                           mca_btl_sm_component.backing_directory, opal_process_info.nodename,
+    rc = opal_asprintf(&sm_file, "%s" OPAL_PATH_SEP "sm_segment.%s.%u.%x.%d",
+                       mca_btl_sm_component.backing_directory, opal_process_info.nodename,
                            geteuid(), OPAL_PROC_MY_NAME.jobid, MCA_BTL_SM_LOCAL_RANK);
-        if (0 > rc) {
-            free(btls);
-            return NULL;
-        }
-        opal_pmix_register_cleanup(sm_file, false, false, false);
-
-        rc = opal_shmem_segment_create(&component->seg_ds, sm_file, component->segment_size);
-        free(sm_file);
-        if (OPAL_SUCCESS != rc) {
-            BTL_VERBOSE(("Could not create shared memory segment"));
-            free(btls);
-            return NULL;
-        }
+    if (0 > rc) {
+        free(btls);
+        return NULL;
+    }
+    opal_pmix_register_cleanup(sm_file, false, false, false);
 
-        component->my_segment = opal_shmem_segment_attach(&component->seg_ds);
-        if (NULL == component->my_segment) {
-            BTL_VERBOSE(("Could not attach to just created shared memory segment"));
-            goto failed;
-        }
-    } else {
-        /* if the shared-memory single-copy component can map memory (XPMEM) an anonymous segment
-         * can be used instead */
-        component->my_segment = mmap(NULL, component->segment_size, PROT_READ | PROT_WRITE,
-                                     MAP_ANONYMOUS | MAP_SHARED, -1, 0);
-        if ((void *) -1 == component->my_segment) {
-            BTL_VERBOSE(("Could not create anonymous memory segment"));
-            free(btls);
-            return NULL;
-        }
+    rc = opal_shmem_segment_create(&component->seg_ds, sm_file, component->segment_size);
+    free(sm_file);
+    if (OPAL_SUCCESS != rc) {
+        BTL_VERBOSE(("Could not create shared memory segment"));
+        free(btls);
+        return NULL;
+    }
+
+    component->my_segment = opal_shmem_segment_attach(&component->seg_ds);
+    if (NULL == component->my_segment) {
+        BTL_VERBOSE(("Could not attach to just created shared memory segment"));
+        goto failed;
     }
 
     /* initialize my fifo */
@@ -432,11 +410,7 @@ mca_btl_sm_component_init(int *num_btls, bool enable_progress_threads, bool enab
 
     return btls;
 failed:
-    if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {
-        munmap(component->my_segment, component->segment_size);
-    } else {
-        opal_shmem_unlink(&component->seg_ds);
-    }
+    opal_shmem_unlink(&component->seg_ds);
 
     if (btls) {
         free(btls);

opal/mca/btl/sm/btl_sm_module.c

@@ -186,12 +186,6 @@ static int init_sm_endpoint(struct mca_btl_base_endpoint_t **ep_out, struct opal
             mca_btl_sm.super.btl_put = NULL;
             mca_btl_sm.super.btl_flags &= ~MCA_BTL_FLAGS_RDMA;
         }
-        if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {
-            ep->smsc_map_context = MCA_SMSC_CALL(map_peer_region, ep->smsc_endpoint, /*flag=*/0,
-                                                 (void *) (uintptr_t) modex->segment_base,
-                                                 mca_btl_sm_component.segment_size,
-                                                 (void **) &ep->segment_base);
-        } else {
             /* store a copy of the segment information for detach */
             ep->seg_ds = malloc(modex->seg_ds_size);
             if (NULL == ep->seg_ds) {
@@ -204,7 +198,6 @@ static int init_sm_endpoint(struct mca_btl_base_endpoint_t **ep_out, struct opal
             if (NULL == ep->segment_base) {
                 return OPAL_ERROR;
             }
-        }
 
         OBJ_CONSTRUCT(&ep->lock, opal_mutex_t);
 
@@ -354,10 +347,8 @@ static int sm_finalize(struct mca_btl_base_module_t *btl)
     free(component->fbox_in_endpoints);
     component->fbox_in_endpoints = NULL;
 
-    if (!mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {
-        opal_shmem_unlink(&mca_btl_sm_component.seg_ds);
-        opal_shmem_segment_detach(&mca_btl_sm_component.seg_ds);
-    }
+    opal_shmem_unlink(&mca_btl_sm_component.seg_ds);
+    opal_shmem_segment_detach(&mca_btl_sm_component.seg_ds);
 
     return OPAL_SUCCESS;
 }
@@ -522,22 +513,18 @@ static void mca_btl_sm_endpoint_destructor(mca_btl_sm_endpoint_t *ep)
     OBJ_DESTRUCT(&ep->pending_frags);
     OBJ_DESTRUCT(&ep->pending_frags_lock);
 
-    if (!mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {
-        if (ep->seg_ds) {
-            opal_shmem_ds_t seg_ds;
-
-            /* opal_shmem_segment_detach expects a opal_shmem_ds_t and will
-             * stomp past the end of the seg_ds if it is too small (which
-             * ep->seg_ds probably is) */
-            memcpy(&seg_ds, ep->seg_ds, opal_shmem_sizeof_shmem_ds(ep->seg_ds));
-            free(ep->seg_ds);
-            ep->seg_ds = NULL;
-
-            /* disconnect from the peer's segment */
-            opal_shmem_segment_detach(&seg_ds);
-        }
-    } else if (NULL != ep->smsc_map_context) {
-        MCA_SMSC_CALL(unmap_peer_region, ep->smsc_map_context);
+    if (ep->seg_ds) {
+        opal_shmem_ds_t seg_ds;
+
+        /* opal_shmem_segment_detach expects a opal_shmem_ds_t and will
+         * stomp past the end of the seg_ds if it is too small (which
+         * ep->seg_ds probably is) */
+        memcpy(&seg_ds, ep->seg_ds, opal_shmem_sizeof_shmem_ds(ep->seg_ds));
+        free(ep->seg_ds);
+        ep->seg_ds = NULL;
+
+        /* disconnect from the peer's segment */
+        opal_shmem_segment_detach(&seg_ds);
     }
 
     if (ep->fbox_out.fbox) {