Add smoke test for binary package

guyang3532 · guyang3532 · commit 6c371b89a679 · 2020-05-17T17:09:28.000+08:00
diff --git a/check_binary.sh b/check_binary.sh
@@ -256,8 +256,8 @@ build_and_run_example_cpp () {
   if [ -f "${install_root}/lib/libtorch_cuda.so" ] || [ -f "${install_root}/lib/libtorch_cuda.dylib" ]; then
     TORCH_CUDA_LINK_FLAGS="-ltorch_cuda"
   fi
-  g++ example-app.cpp -I${install_root}/include -I${install_root}/include/torch/csrc/api/include -D_GLIBCXX_USE_CXX11_ABI=$GLIBCXX_USE_CXX11_ABI -std=gnu++14 -L${install_root}/lib ${REF_LIB} ${ADDITIONAL_LINKER_FLAGS} -ltorch $TORCH_CPU_LINK_FLAGS $TORCH_CUDA_LINK_FLAGS $C10_LINK_FLAGS -o example-app
-  ./example-app
+  g++ /builder/test_example_code/$1.cpp -I${install_root}/include -I${install_root}/include/torch/csrc/api/include -D_GLIBCXX_USE_CXX11_ABI=$GLIBCXX_USE_CXX11_ABI -std=gnu++14 -L${install_root}/lib ${REF_LIB} ${ADDITIONAL_LINKER_FLAGS} -ltorch $TORCH_CPU_LINK_FLAGS $TORCH_CUDA_LINK_FLAGS $C10_LINK_FLAGS -o $1
+  ./$1
 }
 
 build_example_cpp_with_incorrect_abi () {
@@ -287,7 +287,7 @@ build_example_cpp_with_incorrect_abi () {
   if [ -f "${install_root}/lib/libtorch_cuda.so" ] || [ -f "${install_root}/lib/libtorch_cuda.dylib" ]; then
     TORCH_CUDA_LINK_FLAGS="-ltorch_cuda"
   fi
-  g++ example-app.cpp -I${install_root}/include -I${install_root}/include/torch/csrc/api/include -D_GLIBCXX_USE_CXX11_ABI=$GLIBCXX_USE_CXX11_ABI -std=gnu++14 -L${install_root}/lib ${REF_LIB} ${ADDITIONAL_LINKER_FLAGS} -ltorch $TORCH_CPU_LINK_FLAGS $TORCH_CUDA_LINK_FLAGS $C10_LINK_FLAGS -o example-app
+  g++ /builder/test_example_code/$1.cpp -I${install_root}/include -I${install_root}/include/torch/csrc/api/include -D_GLIBCXX_USE_CXX11_ABI=$GLIBCXX_USE_CXX11_ABI -std=gnu++14 -L${install_root}/lib ${REF_LIB} ${ADDITIONAL_LINKER_FLAGS} -ltorch $TORCH_CPU_LINK_FLAGS $TORCH_CUDA_LINK_FLAGS $C10_LINK_FLAGS -o $1
   ERRCODE=$?
   set -e
   if [ "$ERRCODE" -eq "0" ]; then
@@ -302,19 +302,11 @@ build_example_cpp_with_incorrect_abi () {
 # Check simple Python/C++ calls
 ###############################################################################
 if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
-  cat >example-app.cpp <<EOL
-#include <torch/torch.h>
-
-int main(int argc, const char* argv[]) {
-  TORCH_WARN("Simple test passed!");
-  return 0;
-}
-EOL
-  build_and_run_example_cpp
+  build_and_run_example_cpp simple-torch-test
   # `_GLIBCXX_USE_CXX11_ABI` is always ignored by gcc in devtoolset7, so we test
   # the expected failure case for Ubuntu 16.04 + gcc 5.4 only.
   if [[ "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* ]]; then
-    build_example_cpp_with_incorrect_abi
+    build_example_cpp_with_incorrect_abi simple-torch-test
   fi
 else
   python -c 'import torch'
@@ -328,15 +320,7 @@ fi
 
 if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
   echo "Checking that MKL is available"
-  cat >example-app.cpp <<EOL
-#include <torch/torch.h>
-
-int main(int argc, const char* argv[]) {
-  TORCH_CHECK(torch::hasMKL(), "MKL is not available");
-  return 0;
-}
-EOL
-  build_and_run_example_cpp
+  build_and_run_example_cpp check-torch-mkl
 else
   if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]]; then
     echo "Checking that MKL is available"
@@ -360,24 +344,7 @@ fi
 # Test that CUDA builds are setup correctly
 if [[ "$DESIRED_CUDA" != 'cpu' ]]; then
   if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
-    cat >example-app.cpp <<EOL
-#include <torch/torch.h>
-
-int main(int argc, const char* argv[]) {
-  std::cout << "Checking that CUDA archs are setup correctly" << std::endl;
-  TORCH_CHECK(torch::rand({3, 5}, torch::Device(torch::kCUDA)).defined(), "CUDA archs are not setup correctly");
-
-  // These have to run after CUDA is initialized
-
-  std::cout << "Checking that magma is available" << std::endl;
-  TORCH_CHECK(torch::hasMAGMA(), "MAGMA is not available");
-
-  std::cout << "Checking that CuDNN is available" << std::endl;
-  TORCH_CHECK(torch::cuda::cudnn_is_available(), "CuDNN is not available");
-  return 0;
-}
-EOL
-    build_and_run_example_cpp
+    build_and_run_example_cpp check-torch-cuda
   else
     echo "Checking that CUDA archs are setup correctly"
     timeout 20 python -c 'import torch; torch.randn([3,5]).cuda()'
diff --git a/test_example_code/check-torch-cuda.cpp b/test_example_code/check-torch-cuda.cpp
@@ -0,0 +1,15 @@
+#include <torch/torch.h>
+
+int main(int argc, const char* argv[]) {
+    std::cout << "Checking that CUDA archs are setup correctly" << std::endl;
+    TORCH_CHECK(torch::rand({ 3, 5 }, torch::Device(torch::kCUDA)).defined(), "CUDA archs are not setup correctly");
+
+    // These have to run after CUDA is initialized
+
+    std::cout << "Checking that magma is available" << std::endl;
+    TORCH_CHECK(torch::hasMAGMA(), "MAGMA is not available");
+
+    std::cout << "Checking that CuDNN is available" << std::endl;
+    TORCH_CHECK(torch::cuda::cudnn_is_available(), "CuDNN is not available");
+    return 0;
+}
diff --git a/test_example_code/check-torch-mkl.cpp b/test_example_code/check-torch-mkl.cpp
@@ -0,0 +1,6 @@
+#include <torch/torch.h>
+
+int main(int argc, const char* argv[]) {
+    TORCH_CHECK(torch::hasMKL(), "MKL is not available");
+    return 0;
+}
diff --git a/test_example_code/simple-torch-test.cpp b/test_example_code/simple-torch-test.cpp
@@ -0,0 +1,6 @@
+#include <torch/torch.h>
+
+int main(int argc, const char* argv[]) {
+    TORCH_WARN("Simple test passed!");
+    return 0;
+}
diff --git a/windows/internal/driver_update.bat b/windows/internal/driver_update.bat
@@ -0,0 +1,9 @@
+set "DRIVER_DOWNLOAD_LINK=https://s3.amazonaws.com/ossci-windows/442.50-tesla-desktop-winserver-2019-2016-international.exe"
+curl --retry 3 -kL %DRIVER_DOWNLOAD_LINK% --output 442.50-tesla-desktop-winserver-2019-2016-international.exe
+if errorlevel 1 exit /b 1
+
+start /wait 442.50-tesla-desktop-winserver-2019-2016-international.exe -s -noreboot
+if errorlevel 1 exit /b 1
+
+del 442.50-tesla-desktop-winserver-2019-2016-international.exe || ver > NUL
+
diff --git a/windows/internal/smoke_test.bat b/windows/internal/smoke_test.bat
@@ -0,0 +1,194 @@
+set SRC_DIR=%~dp0
+
+pushd %SRC_DIR%\..
+
+if "%CUDA_VERSION%" == "102" call internal\driver_update.bat
+if errorlevel 1 exit /b 1
+
+set "ORIG_PATH=%PATH%"
+
+setlocal EnableDelayedExpansion
+set NVIDIA_GPU_EXISTS=0
+for /F "delims=" %%i in ('wmic path win32_VideoController get name') do (
+    set GPUS=%%i
+    if not "x!GPUS:NVIDIA=!" == "x!GPUS!" (
+        SET NVIDIA_GPU_EXISTS=1
+        goto gpu_check_end
+    )
+)
+:gpu_check_end
+endlocal & set NVIDIA_GPU_EXISTS=%NVIDIA_GPU_EXISTS%
+
+if "%PACKAGE_TYPE%" == "wheel" goto wheel
+if "%PACKAGE_TYPE%" == "conda" goto conda
+if "%PACKAGE_TYPE%" == "libtorch" goto libtorch
+
+echo "unknown package type"
+exit /b 1
+
+:wheel
+echo "install wheel package"
+
+set PYTHON_INSTALLER_URL=
+if "%DESIRED_PYTHON%" == "3.8" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.8.2/python-3.8.2-amd64.exe"
+if "%DESIRED_PYTHON%" == "3.7" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.7.7/python-3.7.7-amd64.exe"
+if "%DESIRED_PYTHON%" == "3.6" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.6.8/python-3.6.8-amd64.exe"
+if "%PYTHON_INSTALLER_URL%" == "" (
+    echo Python %DESIRED_PYTHON% not supported yet
+)
+
+del python-amd64.exe
+curl --retry 3 -kL "%PYTHON_INSTALLER_URL%" --output python-amd64.exe
+if errorlevel 1 exit /b 1
+
+start /wait "" python-amd64.exe /quiet InstallAllUsers=1 PrependPath=1 Include_test=0 TargetDir=%CD%\Python%PYTHON_VERSION%
+if errorlevel 1 exit /b 1
+
+set "PATH=%CD%\Python%PYTHON_VERSION%\Scripts;%CD%\Python%PYTHON_VERSION%;%PATH%"
+
+pip install -q future numpy protobuf six "mkl>=2019"
+if errorlevel 1 exit /b 1
+
+for /F "delims=" %%i in ('where /R "%PYTORCH_FINAL_PACKAGE_DIR:/=\%" *.whl') do pip install "%%i"
+if errorlevel 1 exit /b 1
+
+goto smoke_test
+
+:conda
+echo "install conda package"
+
+:: Install Miniconda3
+set "CONDA_HOME=%CD%\conda"
+set "tmp_conda=%CONDA_HOME%"
+set "miniconda_exe=%CD%\miniconda.exe"
+
+rmdir /s /q conda
+del miniconda.exe
+curl -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "%miniconda_exe%"
+call ..\conda\install_conda.bat
+if ERRORLEVEL 1 exit /b 1
+
+set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%"
+
+conda create -qyn testenv python=%DESIRED_PYTHON%
+if errorlevel 1 exit /b 1
+
+call %CONDA_HOME%\condabin\activate.bat testenv
+if errorlevel 1 exit /b 1
+
+call conda install -yq future numpy protobuf six
+if ERRORLEVEL 1 exit /b 1
+
+for /F "delims=" %%i in ('where /R "%PYTORCH_FINAL_PACKAGE_DIR:/=\%" *.tar.bz2') do call conda install -y "%%i" --offline
+if ERRORLEVEL 1 exit /b 1
+
+if "%CUDA_VERSION%" == "cpu" goto install_cpu_torch
+
+set /a CUDA_VER=%CUDA_VERSION%
+set CUDA_VER_MAJOR=%CUDA_VERSION:~0,-1%
+set CUDA_VER_MINOR=%CUDA_VERSION:~-1,1%
+set CUDA_VERSION_STR=%CUDA_VER_MAJOR%.%CUDA_VER_MINOR%
+
+if "%CUDA_VERSION_STR%" == "9.2" (
+    call conda install -y "cudatoolkit=%CUDA_VERSION_STR%" -c pytorch -c defaults -c numba/label/dev
+) else (
+    call conda install -y "cudatoolkit=%CUDA_VERSION_STR%" -c pytorch
+)
+if ERRORLEVEL 1 exit /b 1
+
+goto smoke_test
+
+:install_cpu_torch
+call conda install -y cpuonly -c pytorch
+if ERRORLEVEL 1 exit /b 1
+
+:smoke_test
+python -c "import torch"
+if ERRORLEVEL 1 exit /b 1
+
+python -c "from caffe2.python import core"
+if ERRORLEVEL 1 exit /b 1
+
+echo Checking that MKL is available
+python -c "import torch; exit(0 if torch.backends.mkl.is_available() else 1)"
+if ERRORLEVEL 1 exit /b 1
+
+if "%NVIDIA_GPU_EXISTS%" == "0" (
+    echo "Skip CUDA tests for machines without a Nvidia GPU card"
+    goto end
+)
+
+echo Checking that CUDA archs are setup correctly
+python -c "import torch; torch.randn([3,5]).cuda()"
+if ERRORLEVEL 1 exit /b 1
+
+echo Checking that magma is available
+python -c "import torch; torch.rand(1).cuda(); exit(0 if torch.cuda.has_magma else 1)"
+if ERRORLEVEL 1 exit /b 1
+
+echo Checking that CuDNN is available
+python -c "import torch; exit(0 if torch.backends.cudnn.is_available() else 1)"
+if ERRORLEVEL 1 exit /b 1
+
+goto end
+
+:libtorch
+echo "install and test libtorch"
+
+powershell internal\vs_install.ps1
+if ERRORLEVEL 1 exit /b 1
+
+for /F "delims=" %%i in ('where /R "%PYTORCH_FINAL_PACKAGE_DIR:/=\%" *-latest.zip') do 7z x "%%i" -otmp
+if ERRORLEVEL 1 exit /b 1
+
+pushd tmp\libtorch
+
+for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do (
+    if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
+        set "VS15INSTALLDIR=%%i"
+        set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
+        goto vswhere
+    )
+)
+
+:vswhere
+IF "%VS15VCVARSALL%"=="" (
+    echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch test on Windows
+    exit /b 1
+)
+call "%VS15VCVARSALL%" x64
+
+set install_root=%CD%
+set INCLUDE=%INCLUDE%;%install_root%\include;%install_root%\include\torch\csrc\api\include
+set LIB=%LIB%;%install_root%\lib
+set PATH=%PATH%;%install_root%\lib
+
+cl %BUILDER_ROOT%\test_example_code\simple-torch-test.cpp c10.lib torch_cpu.lib /EHsc
+if ERRORLEVEL 1 exit /b 1
+
+.\simple-torch-test.exe
+if ERRORLEVEL 1 exit /b 1
+
+cl %BUILDER_ROOT%\test_example_code\check-torch-mkl.cpp c10.lib torch_cpu.lib /EHsc
+if ERRORLEVEL 1 exit /b 1
+
+.\check-torch-mkl.exe
+if ERRORLEVEL 1 exit /b 1
+
+if "%NVIDIA_GPU_EXISTS%" == "0" (
+    echo "Skip CUDA tests for machines without a Nvidia GPU card"
+    goto end
+)
+
+cl %BUILDER_ROOT%\test_example_code\check-torch-cuda.cpp torch_cpu.lib c10.lib torch_cuda.lib /EHsc /link /INCLUDE:?warp_size@cuda@at@@YAHXZ
+.\check-torch-cuda.exe
+if ERRORLEVEL 1 exit /b 1
+
+popd
+
+echo Cleaning temp files
+rd /s /q "tmp" || ver > nul
+
+:end
+set "PATH=%ORIG_PATH%"
+popd
