Skip to content

Commit 5c33b92

Browse files
husthychusthyc
committed
[iOS] added workflows for libtorchvision_ops.a binary build
ghstack-source-id: 100f558 Pull Request resolved: #3582
1 parent 5fe1449 commit 5c33b92

File tree

8 files changed

+638
-2
lines changed

8 files changed

+638
-2
lines changed

‎.circleci/config.yml

Lines changed: 149 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,59 @@ commands:
4646
fi
4747
echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
4848
49+
brew_update:
50+
description: "Update Homebrew and install base formulae"
51+
steps:
52+
- run:
53+
name: Update Homebrew
54+
no_output_timeout: "10m"
55+
command: |
56+
set -ex
57+
58+
# Update repositories manually.
59+
# Running `brew update` produces a comparison between the
60+
# current checkout and the updated checkout, which takes a
61+
# very long time because the existing checkout is 2y old.
62+
for path in $(find /usr/local/Homebrew -type d -name .git)
63+
do
64+
cd $path/..
65+
git fetch --depth=1 origin
66+
git reset --hard origin/master
67+
done
68+
69+
export HOMEBREW_NO_AUTO_UPDATE=1
70+
71+
# Install expect and moreutils so that we can call `unbuffer` and `ts`.
72+
# moreutils installs a `parallel` executable by default, which conflicts
73+
# with the executable from the GNU `parallel`, so we must unlink GNU
74+
# `parallel` first, and relink it afterwards.
75+
brew install coreutils
76+
brew unlink parallel
77+
brew install moreutils
78+
brew link parallel --overwrite
79+
brew install expect
80+
81+
brew_install:
82+
description: "Install Homebrew formulae"
83+
parameters:
84+
formulae:
85+
type: string
86+
default: ""
87+
steps:
88+
- run:
89+
name: Install << parameters.formulae >>
90+
no_output_timeout: "10m"
91+
command: |
92+
set -ex
93+
export HOMEBREW_NO_AUTO_UPDATE=1
94+
brew install << parameters.formulae >>
95+
96+
run_brew_for_ios_build:
97+
steps:
98+
- brew_update
99+
- brew_install:
100+
formulae: libtool
101+
49102
binary_common: &binary_common
50103
parameters:
51104
# Edit these defaults to do a release
@@ -83,6 +136,22 @@ binary_common: &binary_common
83136
UNICODE_ABI: << parameters.unicode_abi >>
84137
CU_VERSION: << parameters.cu_version >>
85138

139+
torchvision_ios_params: &torchvision_ios_params
140+
parameters:
141+
build_environment:
142+
type: string
143+
default: ""
144+
ios_arch:
145+
type: string
146+
default: ""
147+
ios_platform:
148+
type: string
149+
default: ""
150+
environment:
151+
BUILD_ENVIRONMENT: << parameters.build_environment >>
152+
IOS_ARCH: << parameters.ios_arch >>
153+
IOS_PLATFORM: << parameters.ios_platform >>
154+
86155
smoke_test_common: &smoke_test_common
87156
<<: *binary_common
88157
docker:
@@ -278,6 +347,43 @@ jobs:
278347
paths:
279348
- "*"
280349

350+
binary_ios_build:
351+
<<: *torchvision_ios_params
352+
macos:
353+
xcode: "12.0"
354+
steps:
355+
- attach_workspace:
356+
at: ~/workspace
357+
- checkout
358+
- run_brew_for_ios_build
359+
- run:
360+
name: Build
361+
no_output_timeout: "1h"
362+
command: |
363+
script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh"
364+
cat "$script"
365+
source "$script"
366+
- persist_to_workspace:
367+
root: /Users/distiller/workspace/
368+
paths: ios
369+
370+
binary_ios_upload:
371+
<<: *torchvision_ios_params
372+
macos:
373+
xcode: "12.0"
374+
steps:
375+
- attach_workspace:
376+
at: ~/workspace
377+
- checkout
378+
- run_brew_for_ios_build
379+
- run:
380+
name: Upload
381+
no_output_timeout: "1h"
382+
command: |
383+
script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh"
384+
cat "$script"
385+
source "$script"
386+
281387
binary_macos_conda:
282388
<<: *binary_common
283389
macos:
@@ -595,7 +701,7 @@ jobs:
595701

596702
keys:
597703
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
598-
704+
599705
- run:
600706
name: Setup
601707
command: .circleci/unittest/windows/scripts/setup_env.sh
@@ -1356,6 +1462,18 @@ workflows:
13561462
- clang_format
13571463
- torchhub_test
13581464
- torch_onnx_test
1465+
- binary_ios_build:
1466+
build_environment: binary-libtorchvision_ops-ios-12.0.0-x86_64
1467+
context: org-member
1468+
ios_arch: x86_64
1469+
ios_platform: SIMULATOR
1470+
name: binary_libtorchvision_ops_ios_12.0.0_x86_64
1471+
- binary_ios_build:
1472+
build_environment: binary-libtorchvision_ops-ios-12.0.0-arm64
1473+
context: org-member
1474+
ios_arch: arm64
1475+
ios_platform: OS
1476+
name: binary_libtorchvision_ops_ios_12.0.0_arm64
13591477

13601478
unittest:
13611479
jobs:
@@ -1503,6 +1621,36 @@ workflows:
15031621
- clang_format
15041622
- torchhub_test
15051623
- torch_onnx_test
1624+
- binary_ios_build:
1625+
build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-x86_64
1626+
context: org-member
1627+
filters:
1628+
branches:
1629+
only:
1630+
- nightly
1631+
ios_arch: x86_64
1632+
ios_platform: SIMULATOR
1633+
name: nightly_binary_libtorchvision_ops_ios_12.0.0_x86_64
1634+
- binary_ios_build:
1635+
build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-arm64
1636+
context: org-member
1637+
filters:
1638+
branches:
1639+
only:
1640+
- nightly
1641+
ios_arch: arm64
1642+
ios_platform: OS
1643+
name: nightly_binary_libtorchvision_ops_ios_12.0.0_arm64
1644+
- binary_ios_upload:
1645+
build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-upload
1646+
context: org-member
1647+
filters:
1648+
branches:
1649+
only:
1650+
- nightly
1651+
requires:
1652+
- nightly_binary_libtorchvision_ops_ios_12.0.0_x86_64
1653+
- nightly_binary_libtorchvision_ops_ios_12.0.0_arm64
15061654
- binary_linux_wheel:
15071655
conda_docker_image: pytorch/conda-builder:cpu
15081656
cu_version: cpu

‎.circleci/config.yml.in

Lines changed: 109 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,59 @@ commands:
4646
fi
4747
echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
4848

49+
brew_update:
50+
description: "Update Homebrew and install base formulae"
51+
steps:
52+
- run:
53+
name: Update Homebrew
54+
no_output_timeout: "10m"
55+
command: |
56+
set -ex
57+
58+
# Update repositories manually.
59+
# Running `brew update` produces a comparison between the
60+
# current checkout and the updated checkout, which takes a
61+
# very long time because the existing checkout is 2y old.
62+
for path in $(find /usr/local/Homebrew -type d -name .git)
63+
do
64+
cd $path/..
65+
git fetch --depth=1 origin
66+
git reset --hard origin/master
67+
done
68+
69+
export HOMEBREW_NO_AUTO_UPDATE=1
70+
71+
# Install expect and moreutils so that we can call `unbuffer` and `ts`.
72+
# moreutils installs a `parallel` executable by default, which conflicts
73+
# with the executable from the GNU `parallel`, so we must unlink GNU
74+
# `parallel` first, and relink it afterwards.
75+
brew install coreutils
76+
brew unlink parallel
77+
brew install moreutils
78+
brew link parallel --overwrite
79+
brew install expect
80+
81+
brew_install:
82+
description: "Install Homebrew formulae"
83+
parameters:
84+
formulae:
85+
type: string
86+
default: ""
87+
steps:
88+
- run:
89+
name: Install << parameters.formulae >>
90+
no_output_timeout: "10m"
91+
command: |
92+
set -ex
93+
export HOMEBREW_NO_AUTO_UPDATE=1
94+
brew install << parameters.formulae >>
95+
96+
run_brew_for_ios_build:
97+
steps:
98+
- brew_update
99+
- brew_install:
100+
formulae: libtool
101+
49102
binary_common: &binary_common
50103
parameters:
51104
# Edit these defaults to do a release
@@ -83,6 +136,22 @@ binary_common: &binary_common
83136
UNICODE_ABI: << parameters.unicode_abi >>
84137
CU_VERSION: << parameters.cu_version >>
85138

139+
torchvision_ios_params: &torchvision_ios_params
140+
parameters:
141+
build_environment:
142+
type: string
143+
default: ""
144+
ios_arch:
145+
type: string
146+
default: ""
147+
ios_platform:
148+
type: string
149+
default: ""
150+
environment:
151+
BUILD_ENVIRONMENT: << parameters.build_environment >>
152+
IOS_ARCH: << parameters.ios_arch >>
153+
IOS_PLATFORM: << parameters.ios_platform >>
154+
86155
smoke_test_common: &smoke_test_common
87156
<<: *binary_common
88157
docker:
@@ -278,6 +347,43 @@ jobs:
278347
paths:
279348
- "*"
280349

350+
binary_ios_build:
351+
<<: *torchvision_ios_params
352+
macos:
353+
xcode: "12.0"
354+
steps:
355+
- attach_workspace:
356+
at: ~/workspace
357+
- checkout
358+
- run_brew_for_ios_build
359+
- run:
360+
name: Build
361+
no_output_timeout: "1h"
362+
command: |
363+
script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh"
364+
cat "$script"
365+
source "$script"
366+
- persist_to_workspace:
367+
root: /Users/distiller/workspace/
368+
paths: ios
369+
370+
binary_ios_upload:
371+
<<: *torchvision_ios_params
372+
macos:
373+
xcode: "12.0"
374+
steps:
375+
- attach_workspace:
376+
at: ~/workspace
377+
- checkout
378+
- run_brew_for_ios_build
379+
- run:
380+
name: Upload
381+
no_output_timeout: "1h"
382+
command: |
383+
script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh"
384+
cat "$script"
385+
source "$script"
386+
281387
binary_macos_conda:
282388
<<: *binary_common
283389
macos:
@@ -595,7 +701,7 @@ jobs:
595701
{% raw %}
596702
keys:
597703
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
598-
{% endraw %}
704+
{% endraw %}
599705
- run:
600706
name: Setup
601707
command: .circleci/unittest/windows/scripts/setup_env.sh
@@ -813,6 +919,7 @@ workflows:
813919
- clang_format
814920
- torchhub_test
815921
- torch_onnx_test
922+
{{ ios_workflows() }}
816923

817924
unittest:
818925
jobs:
@@ -832,6 +939,7 @@ workflows:
832939
- clang_format
833940
- torchhub_test
834941
- torch_onnx_test
942+
{{ ios_workflows(nightly=True) }}
835943
{{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
836944
docker_build:
837945
triggers:

‎.circleci/regenerate.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,36 @@ def cmake_workflows(indentation=6):
261261
return indent(indentation, jobs)
262262

263263

264+
def ios_workflows(indentation=6, nightly=False):
265+
jobs = []
266+
build_job_names = []
267+
name_prefix = "nightly_" if nightly else ""
268+
env_prefix = "nightly-" if nightly else ""
269+
for arch, platform in [('x86_64', 'SIMULATOR'), ('arm64', 'OS')]:
270+
name = f'{name_prefix}binary_libtorchvision_ops_ios_12.0.0_{arch}'
271+
build_job_names.append(name)
272+
build_job = {
273+
'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-{arch}',
274+
'context': 'org-member',
275+
'ios_arch': arch,
276+
'ios_platform': platform,
277+
'name': name,
278+
}
279+
if nightly:
280+
build_job['filters'] = gen_filter_branch_tree('nightly')
281+
jobs.append({'binary_ios_build': build_job})
282+
283+
if nightly:
284+
upload_job = {
285+
'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-upload',
286+
'context': 'org-member',
287+
'filters': gen_filter_branch_tree('nightly'),
288+
'requires': build_job_names,
289+
}
290+
jobs.append({'binary_ios_upload': upload_job})
291+
return indent(indentation, jobs)
292+
293+
264294
if __name__ == "__main__":
265295
d = os.path.dirname(__file__)
266296
env = jinja2.Environment(
@@ -275,4 +305,5 @@ def cmake_workflows(indentation=6):
275305
build_workflows=build_workflows,
276306
unittest_workflows=unittest_workflows,
277307
cmake_workflows=cmake_workflows,
308+
ios_workflows=ios_workflows,
278309
))

0 commit comments

Comments
 (0)