Merge caffe2_specific into ROCm/master

.jenkins/caffe2/amd/binary_tests.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -ex
+
+LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT_DIR=$(cd "$LOCAL_DIR"/../../../ && pwd)
+
+cd "$ROOT_DIR"
+
+echo "Running C++ tests.."
+
+for file in $(find "${ROOT_DIR}/build_caffe2/bin" -executable -type f); do
+	if [[ "$file" =~ "test" ]]; then
+		case "$file" in
+		    # skip tests we know are hanging or bad
+		    */mkl_utils_test|*/aten/integer_divider_test)
+		      continue
+		      ;;
+		    */scalar_tensor_test|*/basic|*/native_test)
+				continue
+			  ;;
+			*)
+			  "$file"
+		esac
+	fi
+done

.jenkins/caffe2/amd/build_amd.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+set -ex
+
+# The INSTALL_PREFIX here must match up with test.sh
+INSTALL_PREFIX="/usr/local/caffe2"
+LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT_DIR=$(cd "$LOCAL_DIR"/../../.. && pwd)
+CMAKE_ARGS=()
+
+##############################################################################
+# Explicitly set Python executable.
+###############################################################################
+# On Ubuntu 16.04 the default Python is still 2.7.
+PYTHON="$(which python)"
+
+###############################################################################
+# Set cmake args
+###############################################################################
+CMAKE_ARGS+=("-DBUILD_BINARY=ON")
+CMAKE_ARGS+=("-DBUILD_TEST=ON")
+CMAKE_ARGS+=("-DINSTALL_TEST=ON")
+CMAKE_ARGS+=("-DUSE_OBSERVERS=ON")
+CMAKE_ARGS+=("-DUSE_ZSTD=ON")
+CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}")
+
+# TODO: This is patching the official FindHip to properly handly
+# cmake generator expression. A PR is opened in the upstream repo here:
+# https://github.com/ROCm-Developer-Tools/HIP/pull/516
+# remove this hack once it's merged.
+if [[ -f /opt/rocm/hip/cmake/FindHIP.cmake ]]; then
+  sudo sed -i 's/\ -I${dir}/\ $<$<BOOL:${dir}>:-I${dir}>/' /opt/rocm/hip/cmake/FindHIP.cmake
+fi
+
+export LANG=C.UTF-8
+export LC_ALL=C.UTF-8
+export HCC_AMDGPU_TARGET=gfx900
+export KMTHINLTO=1
+
+########## HIPIFY Caffe2 operators
+${PYTHON} "${ROOT_DIR}/tools/amd_build/build_caffe2_amd.py"
+${PYTHON} "${ROOT_DIR}/tools/amd_build/build_pytorch_amd.py"
+
+MAX_JOBS=$(nproc)
+
+###############################################################################
+# Configure and make
+###############################################################################
+# Run cmake from ./build_caffe2 directory so it doesn't conflict with
+# standard PyTorch build directory. Eventually these won't need to
+# be separate.
+rm -rf build_caffe2
+mkdir build_caffe2
+cd ./build_caffe2
+
+# Configure
+cmake "${ROOT_DIR}" ${CMAKE_ARGS[*]} "$@"
+
+# Build
+if [ "$(uname)" == "Linux" ]; then
+  make "-j${MAX_JOBS}" install
+else
+  echo "Don't know how to build on $(uname)"
+  exit 1
+fi
+
+###############################################################################
+# Install ONNX
+###############################################################################
+
+# Install ONNX into a local directory
+pip install --user -b /tmp/pip_install_onnx "file://${ROOT_DIR}/third_party/onnx#egg=onnx"
+

.jenkins/caffe2/amd/python_tests.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -ex
+
+LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT_DIR=$(cd "$LOCAL_DIR"/../../../ && pwd)
+
+cd "$ROOT_DIR"
+
+# Get the relative path to where the caffe2 python module was installed
+CAFFE2_PYPATH="$ROOT_DIR/build_caffe2/caffe2"
+
+rocm_ignore_test=()
+# need to debug
+rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/arg_ops_test.py")
+rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/piecewise_linear_transform_test.py")
+rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/softmax_ops_test.py")
+rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/unique_ops_test.py")
+rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/model_device_test.py")
+rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/data_parallel_model_test.py")
+
+# Need to go through roi ops to replace max(...) with fmaxf(...)
+rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/roi_align_rotated_op_test.py")
+
+# cuda top_k op has some asm code, the hipified version doesn't
+# compile yet, so we don't have top_k operator for now
+rocm_ignore_test+=("--ignore $CAFFE2_PYPATH/python/operator_test/top_k_test.py")
+
+
+# Python tests
+echo "Running Python tests.."
+python \
+  -m pytest \
+  -v \
+  --ignore "$CAFFE2_PYPATH/python/test/executor_test.py" \
+  --ignore "$CAFFE2_PYPATH/python/operator_test/matmul_op_test.py" \
+  --ignore "$CAFFE2_PYPATH/python/operator_test/pack_ops_test.py" \
+  --ignore "$CAFFE2_PYPATH/python/mkl/mkl_sbn_speed_test.py" \
+  ${rocm_ignore_test[@]} \
+  "$CAFFE2_PYPATH/python"