Merge from upstream master into our fork.

.jenkins/caffe2/build.sh

@@ -2,9 +2,15 @@
 
 set -ex
 
+# The INSTALL_PREFIX here must match up with test.sh
+INSTALL_PREFIX="/usr/local/caffe2"
 LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 ROOT_DIR=$(cd "$LOCAL_DIR"/../.. && pwd)
+CMAKE_ARGS=()
+
 
+# Setup SCCACHE
+###############################################################################
 # Setup sccache if SCCACHE_BUCKET is set
 if [ -n "${SCCACHE_BUCKET}" ]; then
   mkdir -p ./sccache
@@ -61,24 +67,29 @@ report_compile_cache_stats() {
   fi
 }
 
-CMAKE_ARGS=("-DBUILD_BINARY=ON")
-CMAKE_ARGS+=("-DUSE_OBSERVERS=ON")
-CMAKE_ARGS+=("-DUSE_ZSTD=ON")
-
-if [[ $BUILD_ENVIRONMENT == *-aten-* ]]; then
-  if [[ CMAKE_ARGS != *USE_ATEN* ]] && [[ CMAKE_ARGS != *BUILD_ATEN* ]]; then
-    CMAKE_ARGS+=("-DBUILD_ATEN=ON")
-  fi
+###############################################################################
+# Explicitly set Python executable.
+###############################################################################
+# On Ubuntu 16.04 the default Python is still 2.7.
+PYTHON="$(which python)"
+if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
+  PYTHON=$(which "python${BASH_REMATCH[1]}")
+  CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=${PYTHON}")
 fi
 
-# Run build script from scripts if applicable
+
+###############################################################################
+# Use special scripts for Android, conda, and setup builds
+###############################################################################
 if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
   export ANDROID_NDK=/opt/ndk
+  CMAKE_ARGS+=("-DBUILD_BINARY=ON")
+  CMAKE_ARGS+=("-DBUILD_TEST=ON")
+  CMAKE_ARGS+=("-DUSE_OBSERVERS=ON")
+  CMAKE_ARGS+=("-DUSE_ZSTD=ON")
   "${ROOT_DIR}/scripts/build_android.sh" ${CMAKE_ARGS[*]} "$@"
   exit 0
-fi
-if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
-
+elif [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
   # click (required by onnx) wants these set
   # TODO don't think this fixes the problem for conda3 yet
   export LANG=C.UTF-8
@@ -96,51 +107,60 @@ if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
   PROTOBUF_INCDIR=/opt/conda/include pip install -b /tmp/pip_install_onnx "file://${ROOT_DIR}/third_party/onnx#egg=onnx"
   report_compile_cache_stats
   exit 0
+elif [[ $BUILD_ENVIRONMENT == *setup* ]]; then
+  rm -rf $INSTALL_PREFIX && mkdir $INSTALL_PREFIX
+  PYTHONPATH=$INSTALL_PREFIX $PYTHON setup_caffe2.py develop --install-dir $INSTALL_PREFIX
+  exit 0
 fi
 
-# Run cmake from ./build_caffe2 directory so it doesn't conflict with
-# standard PyTorch build directory. Eventually these won't need to
-# be separate.
-rm -rf build_caffe2
-mkdir build_caffe2
-cd ./build_caffe2
 
-INSTALL_PREFIX="/usr/local/caffe2"
+###############################################################################
+# Set cmake args
+###############################################################################
+CMAKE_ARGS+=("-DBUILD_BINARY=ON")
+CMAKE_ARGS+=("-DBUILD_TEST=ON")
+CMAKE_ARGS+=("-DINSTALL_TEST=ON")
+CMAKE_ARGS+=("-DUSE_OBSERVERS=ON")
+CMAKE_ARGS+=("-DUSE_ZSTD=ON")
 CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}")
 
-# Explicitly set Python executable.
-# On Ubuntu 16.04 the default Python is still 2.7.
-PYTHON="$(which python)"
-if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
-  PYTHON=$(which "python${BASH_REMATCH[1]}")
-  CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=${PYTHON}")
+if [[ $BUILD_ENVIRONMENT == *-aten-* ]]; then
+  if [[ CMAKE_ARGS != *USE_ATEN* ]] && [[ CMAKE_ARGS != *BUILD_ATEN* ]]; then
+    CMAKE_ARGS+=("-DBUILD_ATEN=ON")
+  fi
+fi
+if [[ $BUILD_ENVIRONMENT == *mkl* ]]; then
+  CMAKE_ARGS+=("-DBLAS=MKL")
+fi
+if [[ $BUILD_ENVIRONMENT == *cuda* ]]; then
+  CMAKE_ARGS+=("-DUSE_CUDA=ON")
+  CMAKE_ARGS+=("-DCUDA_ARCH_NAME=Maxwell")
+  CMAKE_ARGS+=("-DUSE_NNPACK=OFF")
+
+  # Explicitly set path to NVCC such that the symlink to ccache or sccache is used
+  CMAKE_ARGS+=("-DCUDA_NVCC_EXECUTABLE=${CACHE_WRAPPER_DIR}/nvcc")
+
+  # Ensure FindCUDA.cmake can infer the right path to the CUDA toolkit.
+  # Setting PATH to resolve to the right nvcc alone isn't enough.
+  # See /usr/share/cmake-3.5/Modules/FindCUDA.cmake, block at line 589.
+  export CUDA_PATH="/usr/local/cuda"
+
+  # Ensure the ccache symlink can still find the real nvcc binary.
+  export PATH="/usr/local/cuda/bin:$PATH"
 fi
+if [[ $BUILD_ENVIRONMENT == *rocm* ]]; then
+  # TODO: This is patching the official FindHip to properly handly
+  # cmake generator expression. A PR is opened in the upstream repo here:
+  # https://github.com/ROCm-Developer-Tools/HIP/pull/516
+  # remove this hack once it's merged.
+  if [[ -f /opt/rocm/hip/cmake/FindHIP.cmake ]]; then
+    sudo sed -i 's/\ -I${dir}/\ $<$<BOOL:${dir}>:-I${dir}>/' /opt/rocm/hip/cmake/FindHIP.cmake
+  fi
 
-case "${BUILD_ENVIRONMENT}" in
-  *-mkl*)
-    CMAKE_ARGS+=("-DBLAS=MKL")
-    ;;
-  *-cuda*)
-    CMAKE_ARGS+=("-DUSE_CUDA=ON")
-    CMAKE_ARGS+=("-DCUDA_ARCH_NAME=Maxwell")
-    CMAKE_ARGS+=("-DUSE_NNPACK=OFF")
-
-    # Explicitly set path to NVCC such that the symlink to ccache or sccache is used
-    CMAKE_ARGS+=("-DCUDA_NVCC_EXECUTABLE=${CACHE_WRAPPER_DIR}/nvcc")
-
-    # Ensure FindCUDA.cmake can infer the right path to the CUDA toolkit.
-    # Setting PATH to resolve to the right nvcc alone isn't enough.
-    # See /usr/share/cmake-3.5/Modules/FindCUDA.cmake, block at line 589.
-    export CUDA_PATH="/usr/local/cuda"
-
-    # Ensure the ccache symlink can still find the real nvcc binary.
-    export PATH="/usr/local/cuda/bin:$PATH"
-    ;;
-  *-rocm*)
-    export LANG=C.UTF-8
-    export LC_ALL=C.UTF-8
-    export HCC_AMDGPU_TARGET=gfx900
-esac
+  export LANG=C.UTF-8
+  export LC_ALL=C.UTF-8
+  export HCC_AMDGPU_TARGET=gfx900
+fi
 
 # Try to include Redis support for Linux builds
 if [ "$(uname)" == "Linux" ]; then
@@ -154,14 +174,6 @@ if [ "$(uname)" == "Darwin" ]; then
   CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=ON")
 fi
 
-# We test the presence of cmake3 (for platforms like Centos and Ubuntu 14.04)
-# and use that if so.
-if [[ -x "$(command -v cmake3)" ]]; then
-    CMAKE_BINARY=cmake3
-else
-    CMAKE_BINARY=cmake
-fi
-
 # Use a speciallized onnx namespace in CI to catch hardcoded onnx namespace
 CMAKE_ARGS+=("-DONNX_NAMESPACE=ONNX_NAMESPACE_FOR_C2_CI")
 
@@ -173,17 +185,35 @@ if [[ -n "$INTEGRATED" ]]; then
     CMAKE_ARGS+=("-DCAFFE2_LINK_LOCAL_PROTOBUF=OFF")
 fi
 
-# Configure
-${CMAKE_BINARY} "${ROOT_DIR}" ${CMAKE_ARGS[*]} "$@"
-
-# Build
+# We test the presence of cmake3 (for platforms like Centos and Ubuntu 14.04)
+# and use that if so.
+if [[ -x "$(command -v cmake3)" ]]; then
+    CMAKE_BINARY=cmake3
+else
+    CMAKE_BINARY=cmake
+fi
 # sccache will fail for CUDA builds if all cores are used for compiling
 if [[ "${BUILD_ENVIRONMENT}" == *-cuda* ]] && [ -n "${SCCACHE}" ]; then
   MAX_JOBS=`expr $(nproc) - 1`
 else
   MAX_JOBS=$(nproc)
 fi
 
+
+###############################################################################
+# Configure and make
+###############################################################################
+# Run cmake from ./build_caffe2 directory so it doesn't conflict with
+# standard PyTorch build directory. Eventually these won't need to
+# be separate.
+rm -rf build_caffe2
+mkdir build_caffe2
+cd ./build_caffe2
+
+# Configure
+${CMAKE_BINARY} "${ROOT_DIR}" ${CMAKE_ARGS[*]} "$@"
+
+# Build
 if [ "$(uname)" == "Linux" ]; then
   make "-j${MAX_JOBS}" install
 else
@@ -193,6 +223,11 @@ fi
 
 report_compile_cache_stats
 
+
+###############################################################################
+# Install ONNX
+###############################################################################
+
 # Install ONNX into a local directory
 pip install --user -b /tmp/pip_install_onnx "file://${ROOT_DIR}/third_party/onnx#egg=onnx"
 

.jenkins/pytorch/build.sh

@@ -23,6 +23,24 @@ gcc --version
 # TODO: Don't run this...
 pip install -r requirements.txt || true
 
+if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
+  export HCC_AMDGPU_TARGET=gfx900
+  export LANG=C.UTF-8
+  export LC_ALL=C.UTF-8
+
+  # TODO: Install pyHIPIFY in the docker image
+  rm -rf pyHIPIFY || true
+  git clone https://github.com/ROCm-Developer-Tools/pyHIPIFY.git
+  chmod a+x pyHIPIFY/*.py
+  sudo cp -p pyHIPIFY/*.py /opt/rocm/bin
+  sudo chown -R jenkins:jenkins /usr/local
+  rm -rf "$(dirname "${BASH_SOURCE[0]}")/../../../pytorch_amd/" || true
+  python "$(dirname "${BASH_SOURCE[0]}")/../../tools/amd_build/build_pytorch_amd.py"
+  HIPCC_VERBOSE=7 VERBOSE=1 WITH_ROCM=1 python setup.py install
+  exit
+fi
+
+# TODO: Don't install this here
 if ! which conda; then
   pip install mkl mkl-devel
 fi

.jenkins/pytorch/common.sh

@@ -64,6 +64,7 @@ trap_add cleanup EXIT
 if which sccache > /dev/null; then
   # Save sccache logs to file
   sccache --stop-server || true
+  rm ~/sccache_error.log || true
   SCCACHE_ERROR_LOG=~/sccache_error.log RUST_LOG=sccache::server=error sccache --start-server
 
   # Report sccache stats for easier debugging

.jenkins/pytorch/enabled-configs.txt

@@ -30,7 +30,10 @@ pytorch-linux-trusty-pynightly-build
 pytorch-linux-trusty-pynightly-test
 pytorch-win-ws2016-cuda9-cudnn7-py3-build
 pytorch-win-ws2016-cuda9-cudnn7-py3-test
-pytorch-macos-10.13-py3-build-test
+pytorch-macos-10.13-py3-build
+pytorch-macos-10.13-py3-test
+pytorch-macos-10.13-cuda9.2-cudnn7-py3-build
 pytorch-docker-build-test
 short-perf-test-cpu
 short-perf-test-gpu
+py2-clang3.8-rocmnightly-ubuntu16.04-build

.jenkins/pytorch/macos-build.sh

@@ -1,9 +1,8 @@
 #!/bin/bash
 
-COMPACT_JOB_NAME=pytorch-macos-10.13-py3-build-test
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-
+COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-build"
 export PATH="/usr/local/bin:$PATH"
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 # Set up conda environment
 export PYTORCH_ENV_DIR="${HOME}/pytorch-ci-env"
@@ -21,10 +20,40 @@ rm -rf ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
 git submodule update --init --recursive
 export CMAKE_PREFIX_PATH=${PYTORCH_ENV_DIR}/miniconda3/
 
-# Build and test PyTorch
+# Build PyTorch
+if [[ "${JOB_BASE_NAME}" == *cuda9.2* ]]; then
+  export CUDA_VERSION=9.2
+  export TORCH_CUDA_ARCH_LIST=5.2
+  export PATH=/Developer/NVIDIA/CUDA-${CUDA_VERSION}/bin${PATH:+:${PATH}}
+  export DYLD_LIBRARY_PATH=/Developer/NVIDIA/CUDA-${CUDA_VERSION}/lib${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}}
+  export CUDA_HOME=/Developer/NVIDIA/CUDA-${CUDA_VERSION}
+  export NO_CUDA=0
+
+  # Eigen gives "explicit specialization of class must precede its first use" error
+  # when compiling with Xcode 9.1 toolchain, so we have to use Xcode 8.2 toolchain instead.
+  export DEVELOPER_DIR=/Library/Developer/CommandLineTools
+else
+  export DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
+fi
+
 export MACOSX_DEPLOYMENT_TARGET=10.9
 export CXX=clang++
 export CC=clang
+if which sccache > /dev/null; then
+  printf "#!/bin/sh\nexec sccache $(which clang++) \$*" > "${PYTORCH_ENV_DIR}/clang++"
+  chmod a+x "${PYTORCH_ENV_DIR}/clang++"
+
+  printf "#!/bin/sh\nexec sccache $(which clang) \$*" > "${PYTORCH_ENV_DIR}/clang"
+  chmod a+x "${PYTORCH_ENV_DIR}/clang"
+
+  if [[ "${JOB_BASE_NAME}" == *cuda* ]]; then
+    printf "#!/bin/sh\nexec sccache $(which nvcc) \$*" > "${PYTORCH_ENV_DIR}/nvcc"
+    chmod a+x "${PYTORCH_ENV_DIR}/nvcc"
+    export CUDA_NVCC_EXECUTABLE="${PYTORCH_ENV_DIR}/nvcc"
+  fi
+
+  export PATH="${PYTORCH_ENV_DIR}:$PATH"
+fi
 # If we run too many parallel jobs, we will OOM
 export MAX_JOBS=2
 

.jenkins/pytorch/macos-test.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-COMPACT_JOB_NAME=pytorch-macos-10.13-py3-build-test
+COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}-test"
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 export PATH="/usr/local/bin:$PATH"
@@ -21,7 +21,14 @@ rm -rf ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
 git submodule update --init --recursive
 export CMAKE_PREFIX_PATH=${PYTORCH_ENV_DIR}/miniconda3/
 
-# Build and test PyTorch
+# Test PyTorch
+if [[ "${JOB_BASE_NAME}" == *cuda9.2* ]]; then
+  # Eigen gives "explicit specialization of class must precede its first use" error
+  # when compiling with Xcode 9.1 toolchain, so we have to use Xcode 8.2 toolchain instead.
+  export DEVELOPER_DIR=/Library/Developer/CommandLineTools
+else
+  export DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
+fi
 export MACOSX_DEPLOYMENT_TARGET=10.9
 export CXX=clang++
 export CC=clang

.jenkins/pytorch/print_sccache_log.py

@@ -7,5 +7,5 @@
 
 for line in lines:
     # Ignore errors from CPU instruction set testing
-    if 'CMakeFiles/CMakeTmp/src.c' not in line:
+    if 'src.c' not in line:
         print(line)