Applying integration of NastyMPI CI build

circle.yml

@@ -21,6 +21,8 @@ test:
         parallel: true
     - bash ./dash/scripts/circleci/run-docker.sh Minimal:
         parallel: true
+    - bash ./dash/scripts/circleci/run-docker.sh Nasty:
+        parallel: true
     - grep "FAIL" ./dash-ci.log && (echo "Full log:" ; cat ./dash-ci.log ; exit 1) || exit 0:
         parallel: true
   post:

dart-impl/mpi/src/dart_communication.c

@@ -741,7 +741,8 @@ dart_ret_t dart_put_blocking(
         baseptr = dart_sharedmem_local_baseptr_set[i];
       }
       baseptr += offset;
-      DART_LOG_DEBUG("dart_put_blocking: memcpy %zu bytes", nelem * dart_mpi_sizeof_datatype(dtype));
+      DART_LOG_DEBUG("dart_put_blocking: memcpy %zu bytes",
+                     nelem * dart_mpi_sizeof_datatype(dtype));
       memcpy(baseptr, (char*)src, nelem * dart_mpi_sizeof_datatype(dtype));
       return DART_OK;
     }
@@ -784,13 +785,13 @@ dart_ret_t dart_put_blocking(
    */
   DART_LOG_DEBUG("dart_put_blocking: MPI_Put");
   if (MPI_Put(src,
-               nelem,
-               mpi_dtype,
-               target_unitid_rel,
-               disp_rel,
-               nelem,
-               mpi_dtype,
-               win)
+              nelem,
+              mpi_dtype,
+              target_unitid_rel,
+              disp_rel,
+              nelem,
+              mpi_dtype,
+              win)
       != MPI_SUCCESS) {
     DART_LOG_ERROR("dart_put_blocking ! MPI_Put failed");
     return DART_ERR_INVAL;

dart-impl/mpi/src/dart_initialization.c

@@ -283,7 +283,8 @@ dart_ret_t dart_exit()
 
 	DART_LOG_DEBUG("%2d: dart_exit()", unitid);
 	if (dart_adapt_teamlist_convert(DART_TEAM_ALL, &index) == -1) {
-    DART_LOG_ERROR("%2d: dart_exit: dart_adapt_teamlist_convert failed", unitid);
+    DART_LOG_ERROR("%2d: dart_exit: dart_adapt_teamlist_convert failed",
+                   unitid);
     return DART_ERR_OTHER;
   }
 
@@ -302,12 +303,17 @@ dart_ret_t dart_exit()
 	/* -- Free up all the resources for dart programme -- */
 	MPI_Win_free(&dart_win_local_alloc);
 #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
+  /* Has MPI shared windows: */
 	MPI_Win_free(&dart_sharedmem_win_local_alloc);
 	MPI_Comm_free(&(team_data->sharedmem_comm));
+#else
+  /* No MPI shared windows: */
+  if (dart_mempool_localalloc) {
+    MPI_Free_mem(dart_mempool_localalloc);
+  }
 #endif
   MPI_Win_free(&team_data->window);
 
-
   dart_buddy_delete(dart_localpool);
 #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
   free(team_data->sharedmem_tab);

dash/include/dash/Array.h

@@ -772,6 +772,32 @@ class Array
     allocate(m_pattern);
   }
 
+  /**
+   * Copy constructor is deleted to prevent unintentional copies of - usually
+   * huge - distributed arrays.
+   *
+   * To create a copy of a \c dash::Array instance, instantiate the copy
+   * instance explicitly and use \c dash::copy to clone elements.
+   *
+   * Example:
+   *
+   * \code
+   * dash::Array<int> a1(1024 * dash::size());
+   * dash::fill(a1.begin(), a1.end(), 123);
+   * 
+   * // create copy of array a1:
+   * dash::Array<int> a2(a1.size());
+   * dash::copy(a1.begin(), a1.end(), a2.begin());
+   * \endcode
+   */
+  Array(const self_t & other) = delete;
+
+  /**
+   * Move constructor is deleted as move semantics are non-trivial for
+   * distributed arrays.
+   */
+  Array(self_t && other) = delete;
+
   /**
    * Destructor, deallocates array elements.
    */
@@ -782,6 +808,32 @@ class Array
     DASH_LOG_TRACE_VAR("Array.~Array >", this);
   }
 
+  /**
+   * Move assignment operator is deleted as move semantics are non-trivial
+   * for distributed arrays.
+   */
+  self_t & operator=(self_t && other) = delete;
+
+  /**
+   * Assignment operator is deleted to prevent unintentional copies of
+   * - usually huge - distributed arrays.
+   *
+   * To create a copy of a \c dash::Array instance, instantiate the copy
+   * instance explicitly and use \c dash::copy to clone elements.
+   *
+   * Example:
+   *
+   * \code
+   * dash::Array<int> a1(1024 * dash::size());
+   * dash::fill(a1.begin(), a1.end(), 123);
+   * 
+   * // create copy of array a1:
+   * dash::Array<int> a2(a1.size());
+   * dash::copy(a1.begin(), a1.end(), a2.begin());
+   * \endcode
+   */
+  self_t & operator=(const self_t & rhs) = delete;
+
   /**
    * View at block at given global block offset.
    */
@@ -1014,7 +1066,12 @@ class Array
   void barrier() const
   {
     DASH_LOG_TRACE_VAR("Array.barrier()", m_team);
-    m_team->barrier();
+    if (nullptr != m_globmem) {
+      m_globmem->flush_all();
+    }
+    if (nullptr != m_team && *m_team != dash::Team::Null()) {
+      m_team->barrier();
+    }
     DASH_LOG_TRACE("Array.barrier >", "passed barrier");
   }
 
@@ -1106,7 +1163,7 @@ class Array
     }
     // Remove this function from team deallocator list to avoid
     // double-free:
-    m_pattern.team().unregister_deallocator(
+    m_team->unregister_deallocator(
       this, std::bind(&Array::deallocate, this));
     // Actual destruction of the array instance:
     DASH_LOG_TRACE_VAR("Array.deallocate()", m_globmem);
@@ -1139,7 +1196,7 @@ class Array
     // Allocate local memory of identical size on every unit:
     DASH_LOG_TRACE_VAR("Array._allocate", m_lcapacity);
     DASH_LOG_TRACE_VAR("Array._allocate", m_lsize);
-    m_globmem   = new glob_mem_type(m_lcapacity, m_pattern.team());
+    m_globmem   = new glob_mem_type(m_lcapacity, *m_team);
     // Global iterators:
     m_begin     = iterator(m_globmem, m_pattern);
     m_end       = iterator(m_begin) + m_size;
@@ -1199,7 +1256,7 @@ class Array
     // Allocate local memory of identical size on every unit:
     DASH_LOG_TRACE_VAR("Array._allocate", m_lcapacity);
     DASH_LOG_TRACE_VAR("Array._allocate", m_lsize);
-    m_globmem   = new glob_mem_type(local_elements, pattern.team());
+    m_globmem   = new glob_mem_type(local_elements, *m_team);
     // Global iterators:
     m_begin     = iterator(m_globmem, pattern);
     m_end       = iterator(m_begin) + m_size;
@@ -1213,7 +1270,7 @@ class Array
     DASH_LOG_TRACE_VAR("Array._allocate", m_lsize);
     // Register deallocator of this array instance at the team
     // instance that has been used to initialized it:
-    pattern.team().register_deallocator(
+    m_team->register_deallocator(
       this, std::bind(&Array::deallocate, this));
     // Assure all units are synchronized after allocation, otherwise
     // other units might start working on the array before allocation
@@ -1235,7 +1292,7 @@ class Array
   /// Element distribution pattern
   PatternType          m_pattern;
   /// Global memory allocation and -access
-  glob_mem_type      * m_globmem;
+  glob_mem_type      * m_globmem   = nullptr;
   /// Iterator to initial element in the array
   iterator             m_begin;
   /// Iterator to final element in the array
@@ -1247,9 +1304,9 @@ class Array
   /// Number allocated local elements in the array
   size_type            m_lcapacity;
   /// Native pointer to first local element in the array
-  ElementType        * m_lbegin;
+  ElementType        * m_lbegin    = nullptr;
   /// Native pointer past last local element in the array
-  ElementType        * m_lend;
+  ElementType        * m_lend      = nullptr;
 
 };
 

dash/include/dash/allocator/CollectiveAllocator.h

@@ -171,6 +171,9 @@ class CollectiveAllocator
    * Allocates \c num_local_elem local elements at every unit in global
    * memory space.
    *
+   * \return  Global pointer to allocated memory range, or \c DART_GPTR_NULL
+   *          if \c num_local_elem is 0 or less.
+   *
    * \see DashAllocatorConcept
    */
   pointer allocate(size_type num_local_elem)

dash/scripts/circleci/run-docker.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-MPIENVS=(mpich openmpi openmpi2)
+MPIENVS=(mpich openmpi)
 BUILD_CONFIG=$1
 DASH_ENV_EXPORTS="export DASH_MAKE_PROCS='4'; export DASH_MAX_UNITS='3'; export DASH_BUILDEX='OFF';"
 

dash/scripts/dash-ci-deploy.sh

@@ -138,7 +138,7 @@ elif [ "$BUILD_TYPE" = "Minimal" ]; then
   -DENABLE_COMPILER_WARNINGS=ON \
   -DENABLE_LT_OPTIMIZATION=OFF \
   -DENABLE_ASSERTIONS=OFF \
-  -DENABLE_SHARED_WINDOWS=ON \
+  -DENABLE_SHARED_WINDOWS=OFF \
   -DENABLE_UNIFIED_MEMORY_MODEL=ON \
   -DENABLE_DEFAULT_INDEX_TYPE_LONG=OFF \
   -DENABLE_LOGGING=OFF \
@@ -166,7 +166,7 @@ elif [ "$BUILD_TYPE" = "Nasty" ]; then
   -DINSTALL_PREFIX=$INSTALL_PATH \
   -DDART_IMPLEMENTATIONS=mpi \
   -DENABLE_ASSERTIONS=OFF \
-  -DENABLE_SHARED_WINDOWS=ON \
+  -DENABLE_SHARED_WINDOWS=OFF \
   -DENABLE_UNIFIED_MEMORY_MODEL=ON \
   -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \
   -DENABLE_LOGGING=OFF \

dash/test/AccumulateTest.h

@@ -3,11 +3,12 @@
 
 #include <gtest/gtest.h>
 #include <libdash.h>
+#include "TestBase.h"
 
 /**
  * Test fixture for class dash::accumulate
  */
-class AccumulateTest : public ::testing::Test {
+class AccumulateTest : public dash::test::TestBase {
 protected:
   size_t _dash_id;
   size_t _dash_size;
@@ -23,18 +24,13 @@ class AccumulateTest : public ::testing::Test {
   }
 
   virtual void SetUp() {
-    dash::init(&TESTENV.argc, &TESTENV.argv);
+    dash::test::TestBase::SetUp();
     _dash_id   = dash::myid();
     _dash_size = dash::size();
-    LOG_MESSAGE("===> Running test case with %d units ...",
-                _dash_size);
   }
 
   virtual void TearDown() {
-    dash::Team::All().barrier();
-    LOG_MESSAGE("<=== Finished test case with %d units",
-                _dash_size);
-    dash::finalize();
+    dash::test::TestBase::TearDown();
   }
 };
 

dash/test/ArrayLargeStructTest.h

@@ -23,7 +23,7 @@ typedef struct DGNode_s {
 /**
  * Test fixture for class dash::Array
  */
-class ArrayLargeStruct : public ::testing::Test {
+class ArrayLargeStruct : public dash::test::TestBase {
 protected:
   size_t _dash_id;
   size_t _dash_size;
@@ -39,18 +39,13 @@ class ArrayLargeStruct : public ::testing::Test {
   }
 
   virtual void SetUp() {
-    dash::init(&TESTENV.argc, &TESTENV.argv);
+    dash::test::TestBase::SetUp();
     _dash_id   = dash::myid();
     _dash_size = dash::size();
-    LOG_MESSAGE("===> Running test case with %d units ...",
-                _dash_size);
   }
 
   virtual void TearDown() {
-    dash::Team::All().barrier();
-    LOG_MESSAGE("<=== Finished test case with %d units",
-                _dash_size);
-    dash::finalize();
+    dash::test::TestBase::TearDown();
   }
 };
 

dash/test/ArrayTest.cc

@@ -1,12 +1,30 @@
 #include <libdash.h>
 #include <gtest/gtest.h>
+
 #include "TestBase.h"
 #include "ArrayTest.h"
 
+
 // global var
 dash::Array<int> array_global;
 
-TEST_F(ArrayTest, Allocation)
+TEST_F(ArrayTest, Declaration)
+{
+  dash::Array<int> array_local(19 * dash::size(), dash::BLOCKED);
+}
+
+TEST_F(ArrayTest, AllocateEmptyLocal)
+{
+  if (dash::size() < 2) {
+    SKIP_TEST_MSG("requires at least 2 units");
+  }
+
+  int block_size = 19;
+  dash::Array<int> array_local(block_size * (dash::size() - 1),
+                               dash::BLOCKCYCLIC(block_size));
+}
+
+TEST_F(ArrayTest, DelayedAllocation)
 {
   dash::Array<int> array_local;
 
@@ -52,7 +70,8 @@ TEST_F(ArrayTest, SingleWriteMultipleRead)
     ASSERT_EQ(array_size, arr6.size());
     // Fill arrays with incrementing values
     if (_dash_id == 0) {
-      LOG_MESSAGE("Assigning array values");
+      DASH_LOG_DEBUG("ArrayTest.SingleWriteMultipleRead",
+                     "writing array values");
       for (size_t i = 0; i < array_size; ++i) {
         arr1[i] = i;
         arr2[i] = i;
@@ -63,6 +82,8 @@ TEST_F(ArrayTest, SingleWriteMultipleRead)
       }
     }
     // Units waiting for value initialization
+    DASH_LOG_DEBUG("ArrayTest.SingleWriteMultipleRead",
+                   "waiting for unit 0 to write array values");
     dash::Team::All().barrier();
     // Read and assert values in arrays
     for (size_t i = 0; i < array_size; ++i) {
@@ -141,9 +162,15 @@ TEST_F(ArrayTest, PatternAllocate)
       dash::TeamSpec<1>(),
       dash::Team::All());
 
+    DASH_LOG_DEBUG("ArrayTest.PatternAllocate",
+                   "allocating array from pattern");
     array.allocate(pattern);
+    DASH_LOG_DEBUG("ArrayTest.PatternAllocate",
+                   "array pattern leaving scope");
   }
 
+  DASH_LOG_DEBUG("ArrayTest.PatternAllocate",
+                 "filling array");
   // Fill
   dash::for_each_with_index(
     array.begin(),
@@ -182,10 +209,11 @@ TEST_F(ArrayTest, TeamSplit)
   }
 
   auto & myteam = team_all.split(2);
-  auto array_a = dash::Array<double>(ext_x, myteam);
+  dash::Array<double> array_a(ext_x, myteam);
 
   array_a.barrier();
   // Check if array is allocated
   ASSERT_NE_U(array_a.lbegin(), nullptr);
   team_all.barrier();
 }
+