From 7ebddbf57d6e5363cd8505e2ae631e6c464ff6f1 Mon Sep 17 00:00:00 2001 From: Yuchen Huang Date: Wed, 17 Mar 2021 22:00:59 -0700 Subject: [PATCH 1/7] [iOS] added workflows for libtorchvision_ops.a binary build [ghstack-poisoned] --- .circleci/config.yml | 149 ++++++++++++- .circleci/config.yml.in | 109 ++++++++- .circleci/regenerate.py | 29 +++ .../unittest/ios/scripts/binary_ios_build.sh | 58 +++++ .../unittest/ios/scripts/binary_ios_upload.sh | 42 ++++ cmake/iOS.cmake | 207 ++++++++++++++++++ ios/CMakeLists.txt | 23 ++ ios/build_ios.sh | 30 +++ 8 files changed, 645 insertions(+), 2 deletions(-) create mode 100755 .circleci/unittest/ios/scripts/binary_ios_build.sh create mode 100644 .circleci/unittest/ios/scripts/binary_ios_upload.sh create mode 100644 cmake/iOS.cmake create mode 100644 ios/CMakeLists.txt create mode 100755 ios/build_ios.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ee1e59a1d5..78c6f2f8893 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,6 +46,58 @@ commands: fi echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + brew_update: + description: "Update Homebrew and install base formulae" + steps: + - run: + name: Update Homebrew + no_output_timeout: "10m" + command: | + set -ex + + # Update repositories manually. + # Running `brew update` produces a comparison between the + # current checkout and the updated checkout, which takes a + # very long time because the existing checkout is 2y old. + for path in $(find /usr/local/Homebrew -type d -name .git) + do + cd $path/.. + git fetch --depth=1 origin + git reset --hard origin/master + done + + export HOMEBREW_NO_AUTO_UPDATE=1 + + # Install expect and moreutils so that we can call `unbuffer` and `ts`. + # moreutils installs a `parallel` executable by default, which conflicts + # with the executable from the GNU `parallel`, so we must unlink GNU + # `parallel` first, and relink it afterwards. + brew unlink parallel + brew install moreutils + brew link parallel --overwrite + brew install expect + + brew_install: + description: "Install Homebrew formulae" + parameters: + formulae: + type: string + default: "" + steps: + - run: + name: Install << parameters.formulae >> + no_output_timeout: "10m" + command: | + set -ex + export HOMEBREW_NO_AUTO_UPDATE=1 + brew install << parameters.formulae >> + + run_brew_for_ios_build: + steps: + - brew_update + - brew_install: + formulae: libtool + binary_common: &binary_common parameters: # Edit these defaults to do a release @@ -83,6 +135,22 @@ binary_common: &binary_common UNICODE_ABI: << parameters.unicode_abi >> CU_VERSION: << parameters.cu_version >> +torchvision_ios_params: &torchvision_ios_params + parameters: + build_environment: + type: string + default: "" + ios_arch: + type: string + default: "" + ios_platform: + type: string + default: "" + environment: + BUILD_ENVIRONMENT: << parameters.build_environment >> + IOS_ARCH: << parameters.ios_arch >> + IOS_PLATFORM: << parameters.ios_platform >> + smoke_test_common: &smoke_test_common <<: *binary_common docker: @@ -278,6 +346,43 @@ jobs: paths: - "*" + binary_ios_build: + <<: *torchvision_ios_params + macos: + xcode: "12.0" + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run_brew_for_ios_build + - run: + name: Build + no_output_timeout: "1h" + command: | + script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh" + cat "$script" + source "$script" + - persist_to_workspace: + root: /Users/distiller/workspace/ + paths: ios + + binary_ios_upload: + <<: *torchvision_ios_params + macos: + xcode: "12.0" + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run_brew_for_ios_build + - run: + name: Upload + no_output_timeout: "1h" + command: | + script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh" + cat "$script" + source "$script" + binary_macos_conda: <<: *binary_common macos: @@ -595,7 +700,7 @@ jobs: keys: - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - + - run: name: Setup command: .circleci/unittest/windows/scripts/setup_env.sh @@ -1336,6 +1441,18 @@ workflows: - clang_format - torchhub_test - torch_onnx_test + - binary_ios_build: + build_environment: binary-libtorchvision_ops-ios-12.0.0-x86_64 + context: org-member + ios_arch: x86_64 + ios_platform: SIMULATOR + name: binary_libtorchvision_ops_ios_12.0.0_x86_64 + - binary_ios_build: + build_environment: binary-libtorchvision_ops-ios-12.0.0-arm64 + context: org-member + ios_arch: arm64 + ios_platform: OS + name: binary_libtorchvision_ops_ios_12.0.0_arm64 unittest: jobs: @@ -1483,6 +1600,36 @@ workflows: - clang_format - torchhub_test - torch_onnx_test + - binary_ios_build: + build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-x86_64 + context: org-member + filters: + branches: + only: + - nightly + ios_arch: x86_64 + ios_platform: SIMULATOR + name: nightly_binary_libtorchvision_ops_ios_12.0.0_x86_64 + - binary_ios_build: + build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-arm64 + context: org-member + filters: + branches: + only: + - nightly + ios_arch: arm64 + ios_platform: OS + name: nightly_binary_libtorchvision_ops_ios_12.0.0_arm64 + - binary_ios_upload: + build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-upload + context: org-member + filters: + branches: + only: + - nightly + requires: + - nightly_binary_libtorchvision_ops_ios_12.0.0_x86_64 + - nightly_binary_libtorchvision_ops_ios_12.0.0_arm64 - binary_linux_wheel: conda_docker_image: pytorch/conda-builder:cpu cu_version: cpu diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index dcd511b8f80..a5a31e8ff8d 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -46,6 +46,58 @@ commands: fi echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + brew_update: + description: "Update Homebrew and install base formulae" + steps: + - run: + name: Update Homebrew + no_output_timeout: "10m" + command: | + set -ex + + # Update repositories manually. + # Running `brew update` produces a comparison between the + # current checkout and the updated checkout, which takes a + # very long time because the existing checkout is 2y old. + for path in $(find /usr/local/Homebrew -type d -name .git) + do + cd $path/.. + git fetch --depth=1 origin + git reset --hard origin/master + done + + export HOMEBREW_NO_AUTO_UPDATE=1 + + # Install expect and moreutils so that we can call `unbuffer` and `ts`. + # moreutils installs a `parallel` executable by default, which conflicts + # with the executable from the GNU `parallel`, so we must unlink GNU + # `parallel` first, and relink it afterwards. + brew unlink parallel + brew install moreutils + brew link parallel --overwrite + brew install expect + + brew_install: + description: "Install Homebrew formulae" + parameters: + formulae: + type: string + default: "" + steps: + - run: + name: Install << parameters.formulae >> + no_output_timeout: "10m" + command: | + set -ex + export HOMEBREW_NO_AUTO_UPDATE=1 + brew install << parameters.formulae >> + + run_brew_for_ios_build: + steps: + - brew_update + - brew_install: + formulae: libtool + binary_common: &binary_common parameters: # Edit these defaults to do a release @@ -83,6 +135,22 @@ binary_common: &binary_common UNICODE_ABI: << parameters.unicode_abi >> CU_VERSION: << parameters.cu_version >> +torchvision_ios_params: &torchvision_ios_params + parameters: + build_environment: + type: string + default: "" + ios_arch: + type: string + default: "" + ios_platform: + type: string + default: "" + environment: + BUILD_ENVIRONMENT: << parameters.build_environment >> + IOS_ARCH: << parameters.ios_arch >> + IOS_PLATFORM: << parameters.ios_platform >> + smoke_test_common: &smoke_test_common <<: *binary_common docker: @@ -278,6 +346,43 @@ jobs: paths: - "*" + binary_ios_build: + <<: *torchvision_ios_params + macos: + xcode: "12.0" + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run_brew_for_ios_build + - run: + name: Build + no_output_timeout: "1h" + command: | + script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh" + cat "$script" + source "$script" + - persist_to_workspace: + root: /Users/distiller/workspace/ + paths: ios + + binary_ios_upload: + <<: *torchvision_ios_params + macos: + xcode: "12.0" + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run_brew_for_ios_build + - run: + name: Upload + no_output_timeout: "1h" + command: | + script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh" + cat "$script" + source "$script" + binary_macos_conda: <<: *binary_common macos: @@ -595,7 +700,7 @@ jobs: {% raw %} keys: - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - {% endraw %} + {% endraw %} - run: name: Setup command: .circleci/unittest/windows/scripts/setup_env.sh @@ -813,6 +918,7 @@ workflows: - clang_format - torchhub_test - torch_onnx_test + {{ ios_workflows() }} unittest: jobs: @@ -832,6 +938,7 @@ workflows: - clang_format - torchhub_test - torch_onnx_test + {{ ios_workflows(nightly=True) }} {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }} docker_build: triggers: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 6dbc0648d6f..08f9a5e0452 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -251,6 +251,34 @@ def cmake_workflows(indentation=6): jobs.append({f'cmake_{os_type}_{device}': job}) return indent(indentation, jobs) +def ios_workflows(indentation=6, nightly=False): + jobs = [] + build_job_names = [] + name_prefix = "nightly_" if nightly else "" + env_prefix = "nightly-" if nightly else "" + for arch, platform in [('x86_64', 'SIMULATOR'), ('arm64', 'OS')]: + name = f'{name_prefix}binary_libtorchvision_ops_ios_12.0.0_{arch}' + build_job_names.append(name) + build_job = { + 'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-{arch}', + 'context': 'org-member', + 'ios_arch': arch, + 'ios_platform': platform, + 'name': name, + } + if nightly: + build_job['filters'] = gen_filter_branch_tree('nightly') + jobs.append({'binary_ios_build': build_job}) + + if nightly: + upload_job = { + 'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-upload', + 'context': 'org-member', + 'filters': gen_filter_branch_tree('nightly'), + 'requires': build_job_names, + } + jobs.append({'binary_ios_upload': upload_job}) + return indent(indentation, jobs) if __name__ == "__main__": d = os.path.dirname(__file__) @@ -266,4 +294,5 @@ def cmake_workflows(indentation=6): build_workflows=build_workflows, unittest_workflows=unittest_workflows, cmake_workflows=cmake_workflows, + ios_workflows=ios_workflows, )) diff --git a/.circleci/unittest/ios/scripts/binary_ios_build.sh b/.circleci/unittest/ios/scripts/binary_ios_build.sh new file mode 100755 index 00000000000..02390cb4e5c --- /dev/null +++ b/.circleci/unittest/ios/scripts/binary_ios_build.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -ex -o pipefail + +echo "" +echo "DIR: $(pwd)" +WORKSPACE=/Users/distiller/workspace +VISION_IOS_ROOT=/Users/distiller/project/ios +export TCLLIBPATH="/usr/local/lib" + +# install conda +curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh +chmod +x ~/conda.sh +/bin/bash ~/conda.sh -b -p ~/anaconda +export PATH="~/anaconda/bin:${PATH}" +source ~/anaconda/bin/activate + +# install dependencies +conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests typing_extensions wget --yes +conda install -c conda-forge valgrind --yes +export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} + +# sync submodules +cd ${VISION_IOS_ROOT} +git submodule sync +git submodule update --init --recursive + +# clone main pytorch repo +mkdir -p ${VISION_IOS_ROOT}/lib +mkdir -p ${VISION_IOS_ROOT}/build +git clone --recursive https://github.com/pytorch/pytorch.git +TORCH_ROOT="${VISION_IOS_ROOT}/pytorch" + +# run build script +chmod a+x ${TORCH_ROOT}/scripts/build_ios.sh +echo "########################################################" +cat ${TORCH_ROOT}/scripts/build_ios.sh +echo "########################################################" +echo "IOS_ARCH: ${IOS_ARCH}" +echo "IOS_PLATFORM: ${IOS_PLATFORM}" +export IOS_ARCH=${IOS_ARCH} +export IOS_PLATFORM=${IOS_PLATFORM} +unbuffer ${TORCH_ROOT}/scripts/build_ios.sh 2>&1 | ts + +LIBTORCH_HEADER_ROOT="${TORCH_ROOT}/build_ios/install/include" +cd ${VISION_IOS_ROOT}/build +cmake -DLIBTORCH_HEADER_ROOT=${LIBTORCH_HEADER_ROOT} \ + -DCMAKE_TOOLCHAIN_FILE=${VISION_IOS_ROOT}/../cmake/iOS.cmake \ + -DIOS_ARCH=${IOS_ARCH} \ + -DIOS_PLATFORM=${IOS_PLATFORM} \ + .. +make +rm -rf ${VISION_IOS_ROOT}/build +rm -rf ${TORCH_ROOT} + +# store the binary +DEST_DIR=${WORKSPACE}/ios/${IOS_ARCH} +mkdir -p ${DEST_DIR} +cp ${VISION_IOS_ROOT}/lib/*.a ${DEST_DIR} diff --git a/.circleci/unittest/ios/scripts/binary_ios_upload.sh b/.circleci/unittest/ios/scripts/binary_ios_upload.sh new file mode 100644 index 00000000000..ce56388e5da --- /dev/null +++ b/.circleci/unittest/ios/scripts/binary_ios_upload.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -ex -o pipefail + +echo "" +echo "DIR: $(pwd)" + +WORKSPACE=/Users/distiller/workspace +PROJ_ROOT=/Users/distiller/project +ARTIFACTS_DIR=${WORKSPACE}/ios +ls ${ARTIFACTS_DIR} +ZIP_DIR=${WORKSPACE}/zip +mkdir -p ${ZIP_DIR}/install/lib + +# build a FAT bianry +cd ${ZIP_DIR}/install/lib +libs=("${ARTIFACTS_DIR}/x86_64/libtorchvision_ops.a" "${ARTIFACTS_DIR}/arm64/libtorchvision_ops.a") +lipo -create "${libs[@]}" -o ${ZIP_DIR}/install/lib/libtorchvision_ops.a +lipo -i ${ZIP_DIR}/install/lib/*.a + +# copy the license +cp ${PROJ_ROOT}/LICENSE ${ZIP_DIR}/ +# zip the library +ZIPFILE=libtorchvision_ops_ios_nightly_build.zip +cd ${ZIP_DIR} +#for testing +touch version.txt +echo $(date +%s) > version.txt +zip -r ${ZIPFILE} install version.txt LICENSE + +# upload to aws +# Install conda then 'conda install' awscli +curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh +chmod +x ~/conda.sh +/bin/bash ~/conda.sh -b -p ~/anaconda +export PATH="~/anaconda/bin:${PATH}" +source ~/anaconda/bin/activate +conda install -c conda-forge awscli --yes +set +x +export AWS_ACCESS_KEY_ID=${AWS_S3_ACCESS_KEY_FOR_PYTORCH_BINARY_UPLOAD} +export AWS_SECRET_ACCESS_KEY=${AWS_S3_ACCESS_SECRET_FOR_PYTORCH_BINARY_UPLOAD} +set -x +aws s3 cp ${ZIPFILE} s3://ossci-ios-build/ --acl public-read diff --git a/cmake/iOS.cmake b/cmake/iOS.cmake new file mode 100644 index 00000000000..d42ea4c9232 --- /dev/null +++ b/cmake/iOS.cmake @@ -0,0 +1,207 @@ +# This file is based off of the Platform/Darwin.cmake and Platform/UnixPaths.cmake +# files which are included with CMake 2.8.4 +# It has been altered for iOS development + +# Options: +# +# IOS_PLATFORM = OS (default) or SIMULATOR +# This decides if SDKS will be selected from the iPhoneOS.platform or iPhoneSimulator.platform folders +# OS - the default, used to build for iPhone and iPad physical devices, which have an arm arch. +# SIMULATOR - used to build for the Simulator platforms, which have an x86 arch. +# +# CMAKE_IOS_DEVELOPER_ROOT = automatic(default) or /path/to/platform/Developer folder +# By default this location is automatcially chosen based on the IOS_PLATFORM value above. +# If set manually, it will override the default location and force the user of a particular Developer Platform +# +# CMAKE_IOS_SDK_ROOT = automatic(default) or /path/to/platform/Developer/SDKs/SDK folder +# By default this location is automatcially chosen based on the CMAKE_IOS_DEVELOPER_ROOT value. +# In this case it will always be the most up-to-date SDK found in the CMAKE_IOS_DEVELOPER_ROOT path. +# If set manually, this will force the use of a specific SDK version + +# Macros: +# +# set_xcode_property (TARGET XCODE_PROPERTY XCODE_VALUE) +# A convenience macro for setting xcode specific properties on targets +# example: set_xcode_property (myioslib IPHONEOS_DEPLOYMENT_TARGET "3.1") +# +# find_host_package (PROGRAM ARGS) +# A macro used to find executable programs on the host system, not within the iOS environment. +# Thanks to the android-cmake project for providing the command + +# Standard settings +set(CMAKE_SYSTEM_NAME Darwin) +set(CMAKE_SYSTEM_VERSION 1) +set(UNIX True) +set(APPLE True) +set(IOS True) + +# Required as of cmake 2.8.10 +set(CMAKE_OSX_DEPLOYMENT_TARGET "" CACHE STRING "Force unset of the deployment target for iOS" FORCE) + +# Determine the cmake host system version so we know where to find the iOS SDKs +find_program(CMAKE_UNAME uname /bin /usr/bin /usr/local/bin) +if(CMAKE_UNAME) + exec_program(uname ARGS -r OUTPUT_VARIABLE CMAKE_HOST_SYSTEM_VERSION) + string(REGEX REPLACE "^([0-9]+)\\.([0-9]+).*$" "\\1" DARWIN_MAJOR_VERSION "${CMAKE_HOST_SYSTEM_VERSION}") +endif(CMAKE_UNAME) + +# Force the compilers to gcc for iOS +set(CMAKE_C_COMPILER /usr/bin/gcc CACHE STRING "") +set(CMAKE_CXX_COMPILER /usr/bin/g++ CACHE STRING "") +set(CMAKE_AR ar CACHE FILEPATH "" FORCE) +set(CMAKE_RANLIB ranlib CACHE FILEPATH "" FORCE) +set(PKG_CONFIG_EXECUTABLE pkg-config CACHE FILEPATH "" FORCE) + +# Setup iOS platform unless specified manually with IOS_PLATFORM +if(NOT DEFINED IOS_PLATFORM) + set(IOS_PLATFORM "OS") +endif(NOT DEFINED IOS_PLATFORM) +set(IOS_PLATFORM ${IOS_PLATFORM} CACHE STRING "Type of iOS Platform") + +# Check the platform selection and setup for developer root +if(${IOS_PLATFORM} STREQUAL "OS") + set(IOS_PLATFORM_LOCATION "iPhoneOS.platform") + set(XCODE_IOS_PLATFORM iphoneos) + + # This causes the installers to properly locate the output libraries + set(CMAKE_XCODE_EFFECTIVE_PLATFORMS "-iphoneos") +elseif(${IOS_PLATFORM} STREQUAL "SIMULATOR") + set(IOS_PLATFORM_LOCATION "iPhoneSimulator.platform") + set(XCODE_IOS_PLATFORM iphonesimulator) + + # This causes the installers to properly locate the output libraries + set(CMAKE_XCODE_EFFECTIVE_PLATFORMS "-iphonesimulator") +elseif(${IOS_PLATFORM} STREQUAL "WATCHOS") + set(IOS_PLATFORM_LOCATION "WatchOS.platform") + set(XCODE_IOS_PLATFORM watchos) + + # This causes the installers to properly locate the output libraries + set(CMAKE_XCODE_EFFECTIVE_PLATFORMS "-watchos") +else(${IOS_PLATFORM} STREQUAL "OS") + message(FATAL_ERROR + "Unsupported IOS_PLATFORM value selected. " + "Please choose OS, SIMULATOR, or WATCHOS.") +endif() + +# All iOS/Darwin specific settings - some may be redundant +set(CMAKE_SHARED_LIBRARY_PREFIX "lib") +set(CMAKE_SHARED_LIBRARY_SUFFIX ".dylib") +set(CMAKE_SHARED_MODULE_PREFIX "lib") +set(CMAKE_SHARED_MODULE_SUFFIX ".so") +set(CMAKE_MODULE_EXISTS 1) +set(CMAKE_DL_LIBS "") + +set(CMAKE_C_OSX_COMPATIBILITY_VERSION_FLAG "-compatibility_version ") +set(CMAKE_C_OSX_CURRENT_VERSION_FLAG "-current_version ") +set(CMAKE_CXX_OSX_COMPATIBILITY_VERSION_FLAG "${CMAKE_C_OSX_COMPATIBILITY_VERSION_FLAG}") +set(CMAKE_CXX_OSX_CURRENT_VERSION_FLAG "${CMAKE_C_OSX_CURRENT_VERSION_FLAG}") + +if(IOS_DEPLOYMENT_TARGET) + set(XCODE_IOS_PLATFORM_VERSION_FLAGS "-m${XCODE_IOS_PLATFORM}-version-min=${IOS_DEPLOYMENT_TARGET}") +endif() + +# Hidden visibilty is required for cxx on iOS +set(CMAKE_C_FLAGS_INIT "${XCODE_IOS_PLATFORM_VERSION_FLAGS}") +set(CMAKE_CXX_FLAGS_INIT "${XCODE_IOS_PLATFORM_VERSION_FLAGS} -fvisibility-inlines-hidden") + +set(CMAKE_C_LINK_FLAGS "${XCODE_IOS_PLATFORM_VERSION_FLAGS} -Wl,-search_paths_first ${CMAKE_C_LINK_FLAGS}") +set(CMAKE_CXX_LINK_FLAGS "${XCODE_IOS_PLATFORM_VERSION_FLAGS} -Wl,-search_paths_first ${CMAKE_CXX_LINK_FLAGS}") + +set(CMAKE_PLATFORM_HAS_INSTALLNAME 1) +set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "-dynamiclib -headerpad_max_install_names") +set(CMAKE_SHARED_MODULE_CREATE_C_FLAGS "-bundle -headerpad_max_install_names") +set(CMAKE_SHARED_MODULE_LOADER_C_FLAG "-Wl,-bundle_loader,") +set(CMAKE_SHARED_MODULE_LOADER_CXX_FLAG "-Wl,-bundle_loader,") +set(CMAKE_FIND_LIBRARY_SUFFIXES ".dylib" ".so" ".a") + +# hack: if a new cmake (which uses CMAKE_INSTALL_NAME_TOOL) runs on an old build tree +# (where install_name_tool was hardcoded) and where CMAKE_INSTALL_NAME_TOOL isn't in the cache +# and still cmake didn't fail in CMakeFindBinUtils.cmake (because it isn't rerun) +# hardcode CMAKE_INSTALL_NAME_TOOL here to install_name_tool, so it behaves as it did before, Alex +if(NOT DEFINED CMAKE_INSTALL_NAME_TOOL) + find_program(CMAKE_INSTALL_NAME_TOOL install_name_tool) +endif(NOT DEFINED CMAKE_INSTALL_NAME_TOOL) + +# Setup iOS deployment target +set(IOS_DEPLOYMENT_TARGET ${IOS_DEPLOYMENT_TARGET} CACHE STRING "Minimum iOS version") + +# Setup iOS developer location unless specified manually with CMAKE_IOS_DEVELOPER_ROOT +# Note Xcode 4.3 changed the installation location, choose the most recent one available +exec_program(/usr/bin/xcode-select ARGS -print-path OUTPUT_VARIABLE CMAKE_XCODE_DEVELOPER_DIR) +set(XCODE_POST_43_ROOT "${CMAKE_XCODE_DEVELOPER_DIR}/Platforms/${IOS_PLATFORM_LOCATION}/Developer") +set(XCODE_PRE_43_ROOT "/Developer/Platforms/${IOS_PLATFORM_LOCATION}/Developer") +if(NOT DEFINED CMAKE_IOS_DEVELOPER_ROOT) + if(EXISTS ${XCODE_POST_43_ROOT}) + set(CMAKE_IOS_DEVELOPER_ROOT ${XCODE_POST_43_ROOT}) + elseif(EXISTS ${XCODE_PRE_43_ROOT}) + set(CMAKE_IOS_DEVELOPER_ROOT ${XCODE_PRE_43_ROOT}) + endif(EXISTS ${XCODE_POST_43_ROOT}) +endif(NOT DEFINED CMAKE_IOS_DEVELOPER_ROOT) +set(CMAKE_IOS_DEVELOPER_ROOT ${CMAKE_IOS_DEVELOPER_ROOT} CACHE PATH "Location of iOS Platform") + +# Find and use the most recent iOS sdk unless specified manually with CMAKE_IOS_SDK_ROOT +if(NOT DEFINED CMAKE_IOS_SDK_ROOT) + file(GLOB _CMAKE_IOS_SDKS "${CMAKE_IOS_DEVELOPER_ROOT}/SDKs/*") + if(_CMAKE_IOS_SDKS) + list(SORT _CMAKE_IOS_SDKS) + list(REVERSE _CMAKE_IOS_SDKS) + list(GET _CMAKE_IOS_SDKS 0 CMAKE_IOS_SDK_ROOT) + else(_CMAKE_IOS_SDKS) + message(FATAL_ERROR "No iOS SDK's found in default search path ${CMAKE_IOS_DEVELOPER_ROOT}. Manually set CMAKE_IOS_SDK_ROOT or install the iOS SDK.") + endif(_CMAKE_IOS_SDKS) + message(STATUS "Toolchain using default iOS SDK: ${CMAKE_IOS_SDK_ROOT}") +endif(NOT DEFINED CMAKE_IOS_SDK_ROOT) +set(CMAKE_IOS_SDK_ROOT ${CMAKE_IOS_SDK_ROOT} CACHE PATH "Location of the selected iOS SDK") + +# Set the sysroot default to the most recent SDK +set(CMAKE_OSX_SYSROOT ${CMAKE_IOS_SDK_ROOT} CACHE PATH "Sysroot used for iOS support") + +# set the architecture for iOS +if(IOS_PLATFORM STREQUAL "OS") + set(DEFAULT_IOS_ARCH "arm64") +elseif(IOS_PLATFORM STREQUAL "SIMULATOR") + set(DEFAULT_IOS_ARCH "x86_64") +elseif(IOS_PLATFORM STREQUAL "WATCHOS") + set(DEFAULT_IOS_ARCH "armv7k;arm64_32") +endif() + +set(IOS_ARCH ${DEFAULT_IOS_ARCH} CACHE STRING "Build architecture for iOS") +set(CMAKE_OSX_ARCHITECTURES ${IOS_ARCH} CACHE STRING "Build architecture for iOS") + +# Set the find root to the iOS developer roots and to user defined paths +set(CMAKE_FIND_ROOT_PATH ${CMAKE_IOS_DEVELOPER_ROOT} ${CMAKE_IOS_SDK_ROOT} ${CMAKE_PREFIX_PATH} CACHE STRING "iOS find search path root") + +# default to searching for frameworks first +set(CMAKE_FIND_FRAMEWORK FIRST) + +# set up the default search directories for frameworks +set(CMAKE_SYSTEM_FRAMEWORK_PATH + ${CMAKE_IOS_SDK_ROOT}/System/Library/Frameworks + ${CMAKE_IOS_SDK_ROOT}/System/Library/PrivateFrameworks + ${CMAKE_IOS_SDK_ROOT}/Developer/Library/Frameworks +) + +# only search the iOS sdks, not the remainder of the host filesystem +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + +# This little macro lets you set any XCode specific property +macro(set_xcode_property TARGET XCODE_PROPERTY XCODE_VALUE) + set_property(TARGET ${TARGET} PROPERTY XCODE_ATTRIBUTE_${XCODE_PROPERTY} ${XCODE_VALUE}) +endmacro(set_xcode_property) + +# This macro lets you find executable programs on the host system +macro(find_host_package) + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY NEVER) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE NEVER) + set(IOS FALSE) + + find_package(${ARGN}) + + set(IOS TRUE) + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +endmacro(find_host_package) diff --git a/ios/CMakeLists.txt b/ios/CMakeLists.txt new file mode 100644 index 00000000000..6b9fd3925b2 --- /dev/null +++ b/ios/CMakeLists.txt @@ -0,0 +1,23 @@ +cmake_minimum_required(VERSION 3.4.1) +set(TARGET torchvision_ops) +project(${TARGET} CXX) +set(CMAKE_CXX_STANDARD 14) +set(LIBTORCH_HEADER_ROOT ${LIBTORCH_HEADER_ROOT}) +set(LIBRARY_OUTPUT_PATH ../lib) + +file(GLOB VISION_SRCS + ../torchvision/csrc/ops/cpu/*.h + ../torchvision/csrc/ops/cpu/*.cpp + ../torchvision/csrc/ops/*.h + ../torchvision/csrc/ops/*.cpp) + +add_library(${TARGET} STATIC + ${VISION_SRCS} +) + +file(GLOB PYTORCH_HEADERS "${LIBTORCH_HEADER_ROOT}") +file(GLOB PYTORCH_HEADERS_CSRC "${LIBTORCH_HEADER_ROOT}/torch/csrc/api/include") +target_include_directories(${TARGET} PRIVATE + ${PYTORCH_HEADERS} + ${PYTORCH_HEADERS_CSRC} +) diff --git a/ios/build_ios.sh b/ios/build_ios.sh new file mode 100755 index 00000000000..81ac2f2a218 --- /dev/null +++ b/ios/build_ios.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -ex -o pipefail +echo "" +echo "DIR: $(pwd)" +VISION_IOS_ROOT=$(dirname $(realpath $0)) + +if ! [ -n "${LIBTORCH_HEADER_ROOT:-}" ]; then + echo "Missing parameter: LIBTORCH_HEADER_ROOT" + exit 1 +fi + +if [ -n "${IOS_ARCH:-}" ]; then + if [ "${IOS_ARCH:-}" == "arm64" ]; then + IOS_PLATFORM="OS" + elif [ "${IOS_ARCH:-}" == "x86_64" ]; then + IOS_PLATFORM="SIMULATOR" + fi +fi + +mkdir -p ${VISION_IOS_ROOT}/lib +mkdir -p ${VISION_IOS_ROOT}/build +cd ${VISION_IOS_ROOT}/build +cmake -DLIBTORCH_HEADER_ROOT=${LIBTORCH_HEADER_ROOT} \ + -DCMAKE_TOOLCHAIN_FILE=${VISION_IOS_ROOT}/../cmake/iOS.cmake \ + -DIOS_ARCH=${IOS_ARCH} \ + -DIOS_PLATFORM=${IOS_PLATFORM} \ + .. +make +rm -rf ${VISION_IOS_ROOT}/build From 9e1c19cf802528730efe1b7ee45e94baf3bbe7ad Mon Sep 17 00:00:00 2001 From: Yuchen Huang Date: Fri, 26 Mar 2021 15:30:08 -0700 Subject: [PATCH 2/7] Update on "[iOS] added workflows for libtorchvision_ops.a binary build" [ghstack-poisoned] --- .circleci/regenerate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 33fede29254..4206a202814 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -260,6 +260,7 @@ def cmake_workflows(indentation=6): jobs.append({f'cmake_{os_type}_{device}': job}) return indent(indentation, jobs) + def ios_workflows(indentation=6, nightly=False): jobs = [] build_job_names = [] @@ -289,6 +290,7 @@ def ios_workflows(indentation=6, nightly=False): jobs.append({'binary_ios_upload': upload_job}) return indent(indentation, jobs) + if __name__ == "__main__": d = os.path.dirname(__file__) env = jinja2.Environment( From 8e64792dfa6ae11b6bad3b1e26309d659230de33 Mon Sep 17 00:00:00 2001 From: Yuchen Huang Date: Fri, 26 Mar 2021 15:55:06 -0700 Subject: [PATCH 3/7] Update on "[iOS] added workflows for libtorchvision_ops.a binary build" [ghstack-poisoned] --- .../unittest/ios/scripts/binary_ios_build.sh | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/.circleci/unittest/ios/scripts/binary_ios_build.sh b/.circleci/unittest/ios/scripts/binary_ios_build.sh index 02390cb4e5c..3bd348debe1 100755 --- a/.circleci/unittest/ios/scripts/binary_ios_build.sh +++ b/.circleci/unittest/ios/scripts/binary_ios_build.sh @@ -4,7 +4,7 @@ set -ex -o pipefail echo "" echo "DIR: $(pwd)" WORKSPACE=/Users/distiller/workspace -VISION_IOS_ROOT=/Users/distiller/project/ios +PROJ_ROOT_IOS=/Users/distiller/project/ios export TCLLIBPATH="/usr/local/lib" # install conda @@ -20,15 +20,15 @@ conda install -c conda-forge valgrind --yes export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} # sync submodules -cd ${VISION_IOS_ROOT} +cd ${PROJ_ROOT_IOS} git submodule sync git submodule update --init --recursive # clone main pytorch repo -mkdir -p ${VISION_IOS_ROOT}/lib -mkdir -p ${VISION_IOS_ROOT}/build +mkdir -p ${PROJ_ROOT_IOS}/lib +mkdir -p ${PROJ_ROOT_IOS}/build git clone --recursive https://github.com/pytorch/pytorch.git -TORCH_ROOT="${VISION_IOS_ROOT}/pytorch" +TORCH_ROOT="${PROJ_ROOT_IOS}/pytorch" # run build script chmod a+x ${TORCH_ROOT}/scripts/build_ios.sh @@ -42,17 +42,11 @@ export IOS_PLATFORM=${IOS_PLATFORM} unbuffer ${TORCH_ROOT}/scripts/build_ios.sh 2>&1 | ts LIBTORCH_HEADER_ROOT="${TORCH_ROOT}/build_ios/install/include" -cd ${VISION_IOS_ROOT}/build -cmake -DLIBTORCH_HEADER_ROOT=${LIBTORCH_HEADER_ROOT} \ - -DCMAKE_TOOLCHAIN_FILE=${VISION_IOS_ROOT}/../cmake/iOS.cmake \ - -DIOS_ARCH=${IOS_ARCH} \ - -DIOS_PLATFORM=${IOS_PLATFORM} \ - .. -make -rm -rf ${VISION_IOS_ROOT}/build +cd ${PROJ_ROOT_IOS} +IOS_ARCH=${IOS_ARCH} LIBTORCH_HEADER_ROOT=${LIBTORCH_HEADER_ROOT} ./build_ios.sh rm -rf ${TORCH_ROOT} # store the binary DEST_DIR=${WORKSPACE}/ios/${IOS_ARCH} mkdir -p ${DEST_DIR} -cp ${VISION_IOS_ROOT}/lib/*.a ${DEST_DIR} +cp ${PROJ_ROOT_IOS}/lib/*.a ${DEST_DIR} From 5c2f85a345abb45f17d56147f677ddbf460fa4f2 Mon Sep 17 00:00:00 2001 From: Yuchen Huang Date: Mon, 29 Mar 2021 10:24:47 -0700 Subject: [PATCH 4/7] Update on "[iOS] added workflows for libtorchvision_ops.a binary build" [ghstack-poisoned] --- .circleci/config.yml | 1 + .circleci/config.yml.in | 1 + 2 files changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d203dfe3870..16e6e161573 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,6 +73,7 @@ commands: # with the executable from the GNU `parallel`, so we must unlink GNU # `parallel` first, and relink it afterwards. brew unlink parallel + brew install coreutils brew install moreutils brew link parallel --overwrite brew install expect diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index a5a31e8ff8d..69385237994 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -73,6 +73,7 @@ commands: # with the executable from the GNU `parallel`, so we must unlink GNU # `parallel` first, and relink it afterwards. brew unlink parallel + brew install coreutils brew install moreutils brew link parallel --overwrite brew install expect From b5e7efd4b0d29c61f3bf58dd664282539a529c2a Mon Sep 17 00:00:00 2001 From: Yuchen Huang Date: Mon, 29 Mar 2021 11:27:18 -0700 Subject: [PATCH 5/7] Update on "[iOS] added workflows for libtorchvision_ops.a binary build" [ghstack-poisoned] --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 16e6e161573..4b9b7dd57a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,8 +72,8 @@ commands: # moreutils installs a `parallel` executable by default, which conflicts # with the executable from the GNU `parallel`, so we must unlink GNU # `parallel` first, and relink it afterwards. - brew unlink parallel brew install coreutils + brew unlink parallel brew install moreutils brew link parallel --overwrite brew install expect diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 69385237994..4949ec99b44 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -72,8 +72,8 @@ commands: # moreutils installs a `parallel` executable by default, which conflicts # with the executable from the GNU `parallel`, so we must unlink GNU # `parallel` first, and relink it afterwards. - brew unlink parallel brew install coreutils + brew unlink parallel brew install moreutils brew link parallel --overwrite brew install expect From 519c497bddeb854b3a855a250496d3c7249807e3 Mon Sep 17 00:00:00 2001 From: Yuchen Huang Date: Mon, 29 Mar 2021 11:48:31 -0700 Subject: [PATCH 6/7] Update on "[iOS] added workflows for libtorchvision_ops.a binary build" [ghstack-poisoned] --- .../unittest/ios/scripts/binary_ios_build.sh | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.circleci/unittest/ios/scripts/binary_ios_build.sh b/.circleci/unittest/ios/scripts/binary_ios_build.sh index 3bd348debe1..d8e3b54342f 100755 --- a/.circleci/unittest/ios/scripts/binary_ios_build.sh +++ b/.circleci/unittest/ios/scripts/binary_ios_build.sh @@ -5,6 +5,7 @@ echo "" echo "DIR: $(pwd)" WORKSPACE=/Users/distiller/workspace PROJ_ROOT_IOS=/Users/distiller/project/ios +PYTORCH_IOS_NIGHTLY_NAME=libtorch_ios_nightly_build.zip export TCLLIBPATH="/usr/local/lib" # install conda @@ -24,22 +25,16 @@ cd ${PROJ_ROOT_IOS} git submodule sync git submodule update --init --recursive -# clone main pytorch repo +# download pytorch-iOS nightly build and unzip it mkdir -p ${PROJ_ROOT_IOS}/lib mkdir -p ${PROJ_ROOT_IOS}/build -git clone --recursive https://github.com/pytorch/pytorch.git +mkdir -p ${PROJ_ROOT_IOS}/pytorch TORCH_ROOT="${PROJ_ROOT_IOS}/pytorch" -# run build script -chmod a+x ${TORCH_ROOT}/scripts/build_ios.sh -echo "########################################################" -cat ${TORCH_ROOT}/scripts/build_ios.sh -echo "########################################################" -echo "IOS_ARCH: ${IOS_ARCH}" -echo "IOS_PLATFORM: ${IOS_PLATFORM}" -export IOS_ARCH=${IOS_ARCH} -export IOS_PLATFORM=${IOS_PLATFORM} -unbuffer ${TORCH_ROOT}/scripts/build_ios.sh 2>&1 | ts +cd {TORCH_ROOT} +wget https://ossci-ios-build.s3.amazonaws.com/${PYTORCH_IOS_NIGHTLY_NAME} +mkdir -p ${PROJ_ROOT_IOS}/pytorch/build_ios +unzip -d ${PROJ_ROOT_IOS}/pytorch/build_ios ./${PYTORCH_IOS_NIGHTLY_NAME} LIBTORCH_HEADER_ROOT="${TORCH_ROOT}/build_ios/install/include" cd ${PROJ_ROOT_IOS} From 831521c6b6eb6faef47e14bab85ad9e8ffda60ad Mon Sep 17 00:00:00 2001 From: Yuchen Huang Date: Mon, 29 Mar 2021 12:30:15 -0700 Subject: [PATCH 7/7] Update on "[iOS] added workflows for libtorchvision_ops.a binary build" [ghstack-poisoned] --- .circleci/unittest/ios/scripts/binary_ios_build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/unittest/ios/scripts/binary_ios_build.sh b/.circleci/unittest/ios/scripts/binary_ios_build.sh index d8e3b54342f..e2ad7b0c55f 100755 --- a/.circleci/unittest/ios/scripts/binary_ios_build.sh +++ b/.circleci/unittest/ios/scripts/binary_ios_build.sh @@ -31,10 +31,10 @@ mkdir -p ${PROJ_ROOT_IOS}/build mkdir -p ${PROJ_ROOT_IOS}/pytorch TORCH_ROOT="${PROJ_ROOT_IOS}/pytorch" -cd {TORCH_ROOT} +cd ${TORCH_ROOT} wget https://ossci-ios-build.s3.amazonaws.com/${PYTORCH_IOS_NIGHTLY_NAME} -mkdir -p ${PROJ_ROOT_IOS}/pytorch/build_ios -unzip -d ${PROJ_ROOT_IOS}/pytorch/build_ios ./${PYTORCH_IOS_NIGHTLY_NAME} +mkdir -p ./build_ios +unzip -d ./build_ios ./${PYTORCH_IOS_NIGHTLY_NAME} LIBTORCH_HEADER_ROOT="${TORCH_ROOT}/build_ios/install/include" cd ${PROJ_ROOT_IOS}