btl/vader: move backing files into /dev/shm on Linux

This commit moves the backing files to /dev/shm to avoid limitations
that may be set on /tmp. The files are registered with pmix to ensure
they are cleaned up after an erroneous exit.

Signed-off-by: Nathan Hjelm <[email protected]>
(cherry picked from commit 4810127)

Author: hjelmn

opal/mca/btl/vader/btl_vader.h

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2007 Voltaire. All rights reserved.
  * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2015 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2010-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2015      Mellanox Technologies. All rights reserved.
  *
@@ -136,6 +136,8 @@ struct mca_btl_vader_component_t {
     opal_list_t pending_endpoints;          /**< list of endpoints with pending fragments */
     opal_list_t pending_fragments;          /**< fragments pending remote completion */
 
+    char *backing_directory;                /**< directory to place shared memory backing files */
+
     /* knem stuff */
 #if OPAL_BTL_VADER_HAVE_KNEM
     unsigned int knem_dma_min;              /**< minimum size to enable DMA for knem transfers (0 disables) */

opal/mca/btl/vader/btl_vader_component.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2007 Voltaire. All rights reserved.
  * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
+ * Copyright (c) 2010-2017 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2011      NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
@@ -211,6 +211,19 @@ static int mca_btl_vader_component_register (void)
                                            OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_GROUP, &mca_btl_vader_component.single_copy_mechanism);
     OBJ_RELEASE(new_enum);
 
+    if (0 == access ("/dev/shm", W_OK)) {
+        mca_btl_vader_component.backing_directory = "/dev/shm";
+    } else {
+        mca_btl_vader_component.backing_directory = opal_process_info.proc_session_dir;
+    }
+    (void) mca_base_component_var_register (&mca_btl_vader_component.super.btl_version, "backing_directory",
+                                            "Directory to place backing files for shared memory communication. "
+                                            "This directory should be on a local filesystem such as /tmp or "
+                                            "/dev/shm (default: (linux) /dev/shm, (others) session directory)",
+                                            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_vader_component.backing_directory);
+
+
 #if OPAL_BTL_VADER_HAVE_KNEM
     /* Currently disabling DMA mode by default; it's not clear that this is useful in all applications and architectures. */
     mca_btl_vader_component.knem_dma_min = 0;
@@ -491,13 +504,17 @@ static mca_btl_base_module_t **mca_btl_vader_component_init (int *num_btls,
     if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism) {
         char *sm_file;
 
-        rc = asprintf(&sm_file, "%s" OPAL_PATH_SEP "vader_segment.%s.%d", opal_process_info.proc_session_dir,
+        rc = asprintf(&sm_file, "%s" OPAL_PATH_SEP "vader_segment.%s.%d", mca_btl_vader_component.backing_directory,
                       opal_process_info.nodename, MCA_BTL_VADER_LOCAL_RANK);
         if (0 > rc) {
             free (btls);
             return NULL;
         }
 
+        if (NULL != opal_pmix.register_cleanup) {
+            opal_pmix.register_cleanup (sm_file, false, false);
+        }
+
         rc = opal_shmem_segment_create (&component->seg_ds, sm_file, component->segment_size);
         free (sm_file);
         if (OPAL_SUCCESS != rc) {