Revert "clean up the build a bit. We no longer need the separate buil… (pytorch#10285)

Summary:
…d_libtorch entrypoint (pytorch#9836)"

This reverts commit 62e23a1.
Pull Request resolved: pytorch#10285

Differential Revision: D9193107

Pulled By: ezyang

fbshipit-source-id: de96dce12fdf74410413ae18feee5caf0bed0025

Author: ezyang

.jenkins/pytorch/build.sh

@@ -74,7 +74,7 @@ fi
 WERROR=1 python setup.py install
 
 # Add the test binaries so that they won't be git clean'ed away
-git add -f build/bin build/lib
+git add -f build/bin
 
 # Testing ATen install
 if [[ "$BUILD_ENVIRONMENT" != *cuda* ]]; then
@@ -101,3 +101,11 @@ if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda8-cudnn6-py3* ]]; then
   make html
   popd
 fi
+
+# Test no-Python build
+if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
+  echo "Building libtorch"
+  # NB: Install outside of source directory (at the same level as the root
+  # pytorch folder) so that it doesn't get cleaned away prior to docker push.
+  WERROR=1 VERBOSE=1 tools/cpp_build/build_caffe2.sh "$PWD/../cpp-build"
+fi

.jenkins/pytorch/macos-build.sh

@@ -61,12 +61,6 @@ export IMAGE_COMMIT_TAG=${BUILD_ENVIRONMENT}-${IMAGE_COMMIT_ID}
 
 python setup.py install
 
-# this is a bit hacky, but not too bad. Bundle the test binaries into
-# the installation directory, so they can catch a free ride on the 7z
-# train.
-mkdir -p ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch/test_binaries/build
-mv build/{bin,lib} ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch/test_binaries/build/
-
 # Upload torch binaries when the build job is finished
 7z a ${IMAGE_COMMIT_TAG}.7z ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch*
 aws s3 cp ${IMAGE_COMMIT_TAG}.7z s3://ossci-macos-build/pytorch/${IMAGE_COMMIT_TAG}.7z --acl public-read

.jenkins/pytorch/macos-test.sh

@@ -50,13 +50,22 @@ test_python_all() {
 test_cpp_api() {
   # C++ API
 
+  # NB: Install outside of source directory (at the same level as the root
+  # pytorch folder) so that it doesn't get cleaned away prior to docker push.
+  # But still clean it before we perform our own build.
+  #
+  CPP_BUILD="$PWD/../cpp-build"
+  rm -rf $CPP_BUILD
+  mkdir -p $CPP_BUILD
+  WERROR=1 VERBOSE=1 tools/cpp_build/build_caffe2.sh "$CPP_BUILD"
+
   python tools/download_mnist.py --quiet -d test/cpp/api/mnist
 
   # Unfortunately it seems like the test can't load from miniconda3
   # without these paths being set
   export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/miniconda3/lib"
   export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/miniconda3/lib"
-  ${PYTORCH_ENV_DIR}/miniconda3/lib/python3.6/site-packages/torch/test_binaries/build/bin/test_api
+  "$CPP_BUILD"/caffe2/bin/test_api
 }
 
 if [ -z "${JOB_BASE_NAME}" ] || [[ "${JOB_BASE_NAME}" == *-test ]]; then

.jenkins/pytorch/test.sh

@@ -108,13 +108,14 @@ test_torchvision() {
 test_libtorch() {
   if [[ "$BUILD_TEST_LIBTORCH" == "1" ]]; then
      echo "Testing libtorch"
+     CPP_BUILD="$PWD/../cpp-build"
      if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
-       ./build/bin/test_jit
+       "$CPP_BUILD"/caffe2/bin/test_jit
      else
-       ./build/bin/test_jit "[cpu]"
+       "$CPP_BUILD"/caffe2/bin/test_jit "[cpu]"
      fi
      python tools/download_mnist.py --quiet -d test/cpp/api/mnist
-     OMP_NUM_THREADS=2 ./build/bin/test_api
+     OMP_NUM_THREADS=2 "$CPP_BUILD"/caffe2/bin/test_api
   fi
 }
 

tools/cpp_build/build_caffe2.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -ex
+
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+
+pushd $SCRIPTPATH
+source ./build_common.sh
+
+echo "Building Caffe2"
+
+mkdir -p $CAFFE2_BUILDPATH
+pushd $CAFFE2_BUILDPATH
+
+cmake -DUSE_CUDA:BOOL=$USE_CUDA \
+      -DBUILD_TORCH=ON \
+      -DUSE_OPENMP:BOOL=${USE_OPENMP:ON} \
+      -DBUILD_CAFFE2=OFF \
+      -DBUILD_ATEN=ON \
+      -DBUILD_PYTHON=OFF \
+      -DBUILD_BINARY=OFF \
+      -DBUILD_SHARED_LIBS=ON \
+      -DONNX_NAMESPACE=$ONNX_NAMESPACE \
+      -DCMAKE_BUILD_TYPE:STRING=$BUILD_TYPE \
+      -DCMAKE_INSTALL_PREFIX:STRING=$INSTALL_PREFIX \
+      -DCMAKE_INSTALL_MESSAGE=NEVER \
+      -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \
+      -G "$GENERATE" \
+      $PYTORCHPATH/
+$MAKE -j "$JOBS" install
+
+popd
+popd

tools/cpp_build/build_common.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+BUILD_PATH="${1:-$SCRIPTPATH/build}"
+INSTALL_PREFIX="$BUILD_PATH/install"
+PYTORCHPATH="$SCRIPTPATH/../.."
+
+USE_CUDA=0
+if [ -x "$(command -v nvcc)" ]; then
+  USE_CUDA=1
+fi
+
+CAFFE2_BUILDPATH="$BUILD_PATH/caffe2"
+NANOPB_BUILDPATH="$BUILD_PATH/nanopb"
+
+# Build with Ninja if available. It has much cleaner output.
+GENERATE="Unix Makefiles"
+MAKE=make
+if [ -x "$(command -v ninja)" ]; then
+  GENERATE=Ninja
+  MAKE=ninja
+fi
+
+# Code is developed a lot more than released, so default to Debug.
+BUILD_TYPE=${BUILD_TYPE:-Debug}
+
+# Try to build with as many threads as we have cores, default to 4 if the
+# command fails.
+set +e
+if [ -n "$MAX_JOBS" ]; then  # Use MAX_JOBS if it is set
+  JOBS=$MAX_JOBS
+elif [[ "$(uname)" == "Linux" ]]; then
+  # https://stackoverflow.com/questions/6481005/how-to-obtain-the-number-of-cpus-cores-in-linux-from-the-command-line
+  JOBS="$(grep -c '^processor' /proc/cpuinfo)"
+else # if [[ "$(uname)" == "Darwin"]]
+  # https://stackoverflow.com/questions/1715580/how-to-discover-number-of-logical-cores-on-mac-os-x
+  JOBS="$(sysctl -n hw.ncpu)"
+fi
+set -e
+if [[ $? -ne 0 ]]; then
+  JOBS=4
+fi
+
+# Make sure an ONNX namespace is set
+if [ -z "$ONNX_NAMESPACE" ]; then
+  ONNX_NAMESPACE="onnx_torch"
+fi

tools/cpp_build/build_libtorch.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -ex
+
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+
+pushd $SCRIPTPATH
+source ./build_common.sh
+
+echo "Building Torch"
+
+mkdir -p $LIBTORCH_BUILDPATH
+pushd $LIBTORCH_BUILDPATH
+
+cmake -DUSE_CUDA:BOOL=$USE_CUDA \
+      -DNO_API:BOOL=${NO_API:-0} \
+      -DCAFFE2_PATH=$PYTORCHPATH/ \
+      -DCAFFE2_BUILD_PATH=$CAFFE2_BUILDPATH \
+      -DONNX_NAMESPACE=$ONNX_NAMESPACE \
+      -DNANOPB_BUILD_PATH=$NANOPB_BUILDPATH \
+      -DINSTALL_PREFIX=$INSTALL_PREFIX \
+      -DCMAKE_BUILD_TYPE:STRING=$BUILD_TYPE \
+      -DCMAKE_INSTALL_PREFIX:STRING=$INSTALL_PREFIX \
+      -DCMAKE_INSTALL_MESSAGE=NEVER \
+      -Dnanopb_BUILD_GENERATOR:BOOL=OFF \
+      -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON \
+      -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \
+      -DVERBOSE:BOOL=${VERBOSE:-0} \
+      -G "$GENERATE" \
+      $PYTORCHPATH/torch
+$MAKE -j "$JOBS"
+
+popd
+popd

torch/CMakeLists.txt

@@ -389,7 +389,6 @@ if (BUILD_TORCH_TEST AND NOT MSVC AND NOT APPLE AND NOT USE_ROCM)
   if (USE_CUDA)
     target_link_libraries(test_jit ${CUDA_LIBRARIES})
   endif()
-
 endif()
 
 if (BUILD_TORCH_TEST AND NOT NO_API AND NOT USE_ROCM)
@@ -438,5 +437,4 @@ if (BUILD_TORCH_TEST AND NOT NO_API AND NOT USE_ROCM)
         -Wno-unused-but-set-parameter)
     endif()
   endif()
-
 endif()