Merge pull request #164 from iotamudelta/ifu

Merge from upstream

Author: iotamudelta

.jenkins/pytorch/build.sh

@@ -1,15 +1,28 @@
 #!/bin/bash
 
+# For distributed, four environmental configs:
+# (1) build with only NCCL
+# (2) build with NCCL and MPI
+# (3) build with only MPI
+# (4) build with neither
+if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-* ]]; then
+  # TODO: move this to Docker
+  sudo apt-get update
+  sudo apt-get install libnccl-dev=2.2.13-1+cuda9.0 libnccl2=2.2.13-1+cuda9.0
+fi
+
+if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda8-* ]] || [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9-cudnn7-py2* ]]; then
+  # TODO: move this to Docker
+  sudo apt-get update
+  sudo apt-get install openmpi-bin libopenmpi-dev
+  sudo apt-get install -y --no-install-recommends openssh-client openssh-server
+  sudo mkdir -p /var/run/sshd
+fi
+
 if [[ "$BUILD_ENVIRONMENT" == "pytorch-linux-xenial-py3-clang5-asan" ]]; then
   exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" $*
 fi
 
-# TODO: move this to Docker
-# TODO: add both NCCL and MPI in CI test by fixing these test first
-sudo apt-get update
-sudo apt-get install libnccl-dev libnccl2
-# sudo apt-get install openmpi-bin libopenmpi-dev
-
 # Required environment variable: $BUILD_ENVIRONMENT
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.

CMakeLists.txt

@@ -306,7 +306,7 @@ if(BUILD_DOCS)
 
     if(EXISTS ${CMAKE_CURRENT_BINARY_DIR}/docs)
       file(REMOVE_RECURSE ${CMAKE_CURRENT_BINARY_DIR}/docs)
-    endif (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/docs)
+    endif()
 
     file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/docs)
     configure_file(${DOXYGEN_C_IN} ${DOXYGEN_C_OUT} @ONLY)
@@ -323,10 +323,10 @@ if(BUILD_DOCS)
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
         COMMENT "Generating Python API documentation with Doxygen"
         VERBATIM)
-  else (DOXYGEN_FOUND)
+  else()
     message(FATAL_ERROR "Doxygen needs to be installed to generate the documentation")
-  endif (DOXYGEN_FOUND)
-endif (BUILD_DOCS)
+  endif()
+endif()
 
 # ---[ CMake related files
 # Uninistall option.

aten/src/ATen/Context.cpp

@@ -9,6 +9,9 @@
 #include <stdexcept>
 
 #include "ATen/CPUGenerator.h"
+#include "ATen/RegisterCPU.h"
+
+#include "TH/TH.h"  // for USE_LAPACK
 
 #ifdef USE_SSE3
 #include <pmmintrin.h>
@@ -34,7 +37,7 @@ Context::Context()
 
   generator_registry[static_cast<int>(DeviceType::CPU)]
     .reset(new CPUGenerator(this));
-  Type::registerCPU(this);
+  register_cpu_types(this);
 }
 
 // TODO: This could be bad juju if someone calls globalContext() in the
@@ -79,6 +82,14 @@ bool Context::hasMKL() const {
 #endif
 }
 
+bool Context::hasLAPACK() const {
+#ifdef USE_LAPACK
+  return true;
+#else
+  return false;
+#endif
+}
+
 bool Context::setFlushDenormal(bool on) {
 #ifdef USE_SSE3
   // Setting flush-to-zero (FTZ) flag

aten/src/ATen/Context.h

@@ -50,6 +50,10 @@ class AT_API Context {
     return *generator;
   }
   bool hasMKL() const;
+  bool hasLAPACK() const;
+  bool hasMAGMA() const {
+    return detail::getCUDAHooks().hasMAGMA();
+  }
   bool hasCUDA() const {
     return detail::getCUDAHooks().hasCUDA();
   }
@@ -114,6 +118,7 @@ class AT_API Context {
   std::atomic<size_t> next_id;
   std::unique_ptr<THCState, void(*)(THCState*)> thc_state;
   friend struct Type;
+  friend void register_cpu_types(Context * context);
   friend void register_cuda_types(Context * context);
 };
 
@@ -157,6 +162,14 @@ static inline bool hasMKL() {
   return globalContext().hasMKL();
 }
 
+static inline bool hasLAPACK() {
+  return globalContext().hasLAPACK();
+}
+
+static inline bool hasMAGMA() {
+  return globalContext().hasMAGMA();
+}
+
 static inline int64_t current_device() {
   return globalContext().current_device();
 }

aten/src/ATen/DeviceGuard.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include <ATen/Device.h>
-#include <ATen/ScalarType.h>
+#include <ATen/core/Device.h>
+#include <ATen/core/ScalarType.h>
 #include <ATen/Tensor.h>
 #include <ATen/core/Error.h>
 #include <ATen/detail/CUDAHooksInterface.h>

aten/src/ATen/Formatting.cpp

@@ -1,6 +1,5 @@
 #include "ATen/Formatting.h"
 #include "ATen/Tensor.h"
-#include "ATen/Context.h"
 #include "ATen/TensorMethods.h"
 
 #include <cmath>

aten/src/ATen/Storage.h

@@ -26,8 +26,8 @@ struct AT_API Storage {
   template <typename T>
   T* unsafe_data() const { return storage_impl_->unsafe_data<T>(); }
 
-  size_t elementSize() const { return storage_impl_->elementSize(); }
-  ptrdiff_t size() const { return storage_impl_->size(); }
+  size_t elementSize() const { return storage_impl_->itemsize(); }
+  ptrdiff_t size() const { return storage_impl_->numel(); }
   bool resizable() const { return storage_impl_->resizable(); }
   // get() use here is to get const-correctness
   void* data() const { return storage_impl_.get()->data(); }

aten/src/ATen/StorageImpl.cpp

@@ -1,31 +1,29 @@
-#include <ATen/Context.h>
 #include <ATen/StorageImpl.h>
 
 namespace at {
 
 StorageImpl::StorageImpl(
     at::DataType data_type,
-    ptrdiff_t size,
+    int64_t numel,
     at::DataPtr data_ptr,
     at::Allocator* allocator,
     bool resizable)
     : data_type_(data_type),
       data_ptr_(std::move(data_ptr)),
-      size_(size),
+      numel_(numel),
       resizable_(resizable),
-      allocator_(allocator),
-      finalizer_(nullptr) {}
+      allocator_(allocator) {}
 
 StorageImpl::StorageImpl(
     at::DataType data_type,
-    ptrdiff_t size,
+    int64_t numel,
     at::Allocator* allocator,
     bool resizable)
     : StorageImpl(
           data_type,
-          size,
+          numel,
           allocator->allocate(
-              at::elementSize(dataTypeToScalarType(data_type)) * size),
+              at::elementSize(dataTypeToScalarType(data_type)) * numel),
           allocator,
           resizable) {}
 

aten/src/ATen/StorageImpl.h

@@ -3,7 +3,6 @@
 #include <ATen/Allocator.h>
 #include <ATen/ScalarType.h>
 #include <ATen/ScalarTypeUtils.h>
-#include <TH/THTypeConversion.hpp>
 
 #include <ATen/core/intrusive_ptr.h>
 
@@ -21,16 +20,16 @@ struct Type;
 struct AT_API StorageImpl : public c10::intrusive_ptr_target {
  public:
   StorageImpl() = delete;
-  virtual ~StorageImpl() {};
+  ~StorageImpl() {};
   StorageImpl(
       at::DataType data_type,
-      ptrdiff_t size,
+      int64_t numel,
       at::DataPtr data_ptr,
       at::Allocator* allocator,
       bool resizable);
   StorageImpl(
       at::DataType data_type,
-      ptrdiff_t size,
+      int64_t numel,
       at::Allocator* allocator,
       bool resizable);
   StorageImpl(StorageImpl&) = delete;
@@ -44,7 +43,7 @@ struct AT_API StorageImpl : public c10::intrusive_ptr_target {
   template <typename T>
   inline T* data() const {
     auto data_type_T =
-        at::scalarTypeToDataType(at::CTypeToScalarType<th::from_type<T>>::to());
+        at::scalarTypeToDataType(at::CTypeToScalarType<T>::to());
     if (dtype() != data_type_T) {
       AT_ERROR(
           "Attempt to access StorageImpl having data type ",
@@ -61,27 +60,22 @@ struct AT_API StorageImpl : public c10::intrusive_ptr_target {
   }
 
   void release_resources() override {
-    if (finalizer_) {
-      (*finalizer_)();
-    }
-    finalizer_ = nullptr;
     data_ptr_.clear();
   }
 
   void operator=(const StorageImpl&) = delete;
 
-  size_t elementSize() const {
+  size_t itemsize() const {
     return at::elementSize(dataTypeToScalarType(data_type_));
   }
 
   Type& type();
 
-  // TODO: Rename to size() and size to size_
-  ptrdiff_t size() const {
-    return size_;
+  int64_t numel() const {
+    return numel_;
   };
-  void set_size(ptrdiff_t size) {
-    size_ = size;
+  void set_numel(int64_t numel) {
+    numel_ = numel;
   };
   bool resizable() const {
     return resizable_;
@@ -132,9 +126,8 @@ struct AT_API StorageImpl : public c10::intrusive_ptr_target {
  private:
   at::DataType data_type_;
   at::DataPtr data_ptr_;
-  ptrdiff_t size_;
+  int64_t numel_;
   bool resizable_;
   at::Allocator* allocator_;
-  std::unique_ptr<THFinalizer> finalizer_;
 };
 } // namespace at

aten/src/ATen/TensorBase.h

@@ -3,8 +3,6 @@
 #include <atomic>
 #include <memory>
 
-#include "ATen/Retainable.h"
-#include "ATen/StorageImpl.h"
 #include "ATen/Storage.h"
 #include "ATen/core/optional.h"
 #include "ATen/core/TensorTypeId.h"

aten/src/ATen/TensorImpl.h

@@ -2,10 +2,10 @@
 
 #include <ATen/core/Backend.h>
 #include <ATen/Context.h>
-#include <ATen/Device.h>
+#include <ATen/core/Device.h>
 #include <ATen/DeviceGuard.h>
 #include <ATen/core/Layout.h>
-#include <ATen/ScalarType.h>
+#include <ATen/core/ScalarType.h>
 #include <ATen/Tensor.h>
 #include <ATen/Type.h>
 

aten/src/ATen/TensorOptions.h

@@ -1,5 +1,4 @@
 #include "ATen/UndefinedTensor.h"
-#include "ATen/Context.h"
 #include "ATen/core/Error.h"
 
 namespace at {

aten/src/ATen/UndefinedTensor.cpp

@@ -3,8 +3,8 @@
 
 namespace at {
 
-UndefinedType::UndefinedType(Context* context)
-    : Type(context, UndefinedTensorId(), /*is_variable=*/false, /*is_undefined=*/true) {}
+UndefinedType::UndefinedType()
+    : Type(UndefinedTensorId(), /*is_variable=*/false, /*is_undefined=*/true) {}
 ScalarType UndefinedType::scalarType() const {
   return ScalarType::Undefined;
 }

aten/src/ATen/UndefinedType.cpp

@@ -1,7 +1,6 @@
 #pragma once
 
 #include "ATen/Type.h"
-#include "ATen/Context.h"
 #include "ATen/CheckGenerator.h"
 
 #ifdef _MSC_VER
@@ -13,7 +12,7 @@
 namespace at {
 
 struct UndefinedType final : public Type {
-  explicit UndefinedType(Context* context);
+  explicit UndefinedType();
   virtual ScalarType scalarType() const override;
   virtual Backend backend() const override;
   virtual bool is_cuda() const override;