Skip to content

Commit 897ac9c

Browse files
husthycfmassa
husthyc
and
fmassa
authored
[iOS] added workflows for libtorchvision_ops.a binary build (#3582)
* [iOS] added workflows for libtorchvision_ops.a binary build [ghstack-poisoned] * Update on "[iOS] added workflows for libtorchvision_ops.a binary build" [ghstack-poisoned] * Update on "[iOS] added workflows for libtorchvision_ops.a binary build" [ghstack-poisoned] * Update on "[iOS] added workflows for libtorchvision_ops.a binary build" [ghstack-poisoned] * Update on "[iOS] added workflows for libtorchvision_ops.a binary build" [ghstack-poisoned] * Update on "[iOS] added workflows for libtorchvision_ops.a binary build" [ghstack-poisoned] * Update on "[iOS] added workflows for libtorchvision_ops.a binary build" [ghstack-poisoned] Co-authored-by: Francisco Massa <[email protected]>
1 parent 8485a30 commit 897ac9c

File tree

8 files changed

+638
-2
lines changed

8 files changed

+638
-2
lines changed

‎.circleci/config.yml

Lines changed: 149 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,59 @@ commands:
5454
command: |
5555
powershell .circleci/scripts/vs_install_cmath.ps1
5656
57+
brew_update:
58+
description: "Update Homebrew and install base formulae"
59+
steps:
60+
- run:
61+
name: Update Homebrew
62+
no_output_timeout: "10m"
63+
command: |
64+
set -ex
65+
66+
# Update repositories manually.
67+
# Running `brew update` produces a comparison between the
68+
# current checkout and the updated checkout, which takes a
69+
# very long time because the existing checkout is 2y old.
70+
for path in $(find /usr/local/Homebrew -type d -name .git)
71+
do
72+
cd $path/..
73+
git fetch --depth=1 origin
74+
git reset --hard origin/master
75+
done
76+
77+
export HOMEBREW_NO_AUTO_UPDATE=1
78+
79+
# Install expect and moreutils so that we can call `unbuffer` and `ts`.
80+
# moreutils installs a `parallel` executable by default, which conflicts
81+
# with the executable from the GNU `parallel`, so we must unlink GNU
82+
# `parallel` first, and relink it afterwards.
83+
brew install coreutils
84+
brew unlink parallel
85+
brew install moreutils
86+
brew link parallel --overwrite
87+
brew install expect
88+
89+
brew_install:
90+
description: "Install Homebrew formulae"
91+
parameters:
92+
formulae:
93+
type: string
94+
default: ""
95+
steps:
96+
- run:
97+
name: Install << parameters.formulae >>
98+
no_output_timeout: "10m"
99+
command: |
100+
set -ex
101+
export HOMEBREW_NO_AUTO_UPDATE=1
102+
brew install << parameters.formulae >>
103+
104+
run_brew_for_ios_build:
105+
steps:
106+
- brew_update
107+
- brew_install:
108+
formulae: libtool
109+
57110
binary_common: &binary_common
58111
parameters:
59112
# Edit these defaults to do a release
@@ -91,6 +144,22 @@ binary_common: &binary_common
91144
UNICODE_ABI: << parameters.unicode_abi >>
92145
CU_VERSION: << parameters.cu_version >>
93146

147+
torchvision_ios_params: &torchvision_ios_params
148+
parameters:
149+
build_environment:
150+
type: string
151+
default: ""
152+
ios_arch:
153+
type: string
154+
default: ""
155+
ios_platform:
156+
type: string
157+
default: ""
158+
environment:
159+
BUILD_ENVIRONMENT: << parameters.build_environment >>
160+
IOS_ARCH: << parameters.ios_arch >>
161+
IOS_PLATFORM: << parameters.ios_platform >>
162+
94163
smoke_test_common: &smoke_test_common
95164
<<: *binary_common
96165
docker:
@@ -288,6 +357,43 @@ jobs:
288357
paths:
289358
- "*"
290359

360+
binary_ios_build:
361+
<<: *torchvision_ios_params
362+
macos:
363+
xcode: "12.0"
364+
steps:
365+
- attach_workspace:
366+
at: ~/workspace
367+
- checkout
368+
- run_brew_for_ios_build
369+
- run:
370+
name: Build
371+
no_output_timeout: "1h"
372+
command: |
373+
script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh"
374+
cat "$script"
375+
source "$script"
376+
- persist_to_workspace:
377+
root: /Users/distiller/workspace/
378+
paths: ios
379+
380+
binary_ios_upload:
381+
<<: *torchvision_ios_params
382+
macos:
383+
xcode: "12.0"
384+
steps:
385+
- attach_workspace:
386+
at: ~/workspace
387+
- checkout
388+
- run_brew_for_ios_build
389+
- run:
390+
name: Upload
391+
no_output_timeout: "1h"
392+
command: |
393+
script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh"
394+
cat "$script"
395+
source "$script"
396+
291397
binary_macos_conda:
292398
<<: *binary_common
293399
macos:
@@ -607,7 +713,7 @@ jobs:
607713

608714
keys:
609715
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
610-
716+
611717
- run:
612718
name: Setup
613719
command: .circleci/unittest/windows/scripts/setup_env.sh
@@ -1370,6 +1476,18 @@ workflows:
13701476
- clang_format
13711477
- torchhub_test
13721478
- torch_onnx_test
1479+
- binary_ios_build:
1480+
build_environment: binary-libtorchvision_ops-ios-12.0.0-x86_64
1481+
context: org-member
1482+
ios_arch: x86_64
1483+
ios_platform: SIMULATOR
1484+
name: binary_libtorchvision_ops_ios_12.0.0_x86_64
1485+
- binary_ios_build:
1486+
build_environment: binary-libtorchvision_ops-ios-12.0.0-arm64
1487+
context: org-member
1488+
ios_arch: arm64
1489+
ios_platform: OS
1490+
name: binary_libtorchvision_ops_ios_12.0.0_arm64
13731491

13741492
unittest:
13751493
jobs:
@@ -1517,6 +1635,36 @@ workflows:
15171635
- clang_format
15181636
- torchhub_test
15191637
- torch_onnx_test
1638+
- binary_ios_build:
1639+
build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-x86_64
1640+
context: org-member
1641+
filters:
1642+
branches:
1643+
only:
1644+
- nightly
1645+
ios_arch: x86_64
1646+
ios_platform: SIMULATOR
1647+
name: nightly_binary_libtorchvision_ops_ios_12.0.0_x86_64
1648+
- binary_ios_build:
1649+
build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-arm64
1650+
context: org-member
1651+
filters:
1652+
branches:
1653+
only:
1654+
- nightly
1655+
ios_arch: arm64
1656+
ios_platform: OS
1657+
name: nightly_binary_libtorchvision_ops_ios_12.0.0_arm64
1658+
- binary_ios_upload:
1659+
build_environment: nightly-binary-libtorchvision_ops-ios-12.0.0-upload
1660+
context: org-member
1661+
filters:
1662+
branches:
1663+
only:
1664+
- nightly
1665+
requires:
1666+
- nightly_binary_libtorchvision_ops_ios_12.0.0_x86_64
1667+
- nightly_binary_libtorchvision_ops_ios_12.0.0_arm64
15201668
- binary_linux_wheel:
15211669
conda_docker_image: pytorch/conda-builder:cpu
15221670
cu_version: cpu

‎.circleci/config.yml.in

Lines changed: 109 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,59 @@ commands:
5454
command: |
5555
powershell .circleci/scripts/vs_install_cmath.ps1
5656

57+
brew_update:
58+
description: "Update Homebrew and install base formulae"
59+
steps:
60+
- run:
61+
name: Update Homebrew
62+
no_output_timeout: "10m"
63+
command: |
64+
set -ex
65+
66+
# Update repositories manually.
67+
# Running `brew update` produces a comparison between the
68+
# current checkout and the updated checkout, which takes a
69+
# very long time because the existing checkout is 2y old.
70+
for path in $(find /usr/local/Homebrew -type d -name .git)
71+
do
72+
cd $path/..
73+
git fetch --depth=1 origin
74+
git reset --hard origin/master
75+
done
76+
77+
export HOMEBREW_NO_AUTO_UPDATE=1
78+
79+
# Install expect and moreutils so that we can call `unbuffer` and `ts`.
80+
# moreutils installs a `parallel` executable by default, which conflicts
81+
# with the executable from the GNU `parallel`, so we must unlink GNU
82+
# `parallel` first, and relink it afterwards.
83+
brew install coreutils
84+
brew unlink parallel
85+
brew install moreutils
86+
brew link parallel --overwrite
87+
brew install expect
88+
89+
brew_install:
90+
description: "Install Homebrew formulae"
91+
parameters:
92+
formulae:
93+
type: string
94+
default: ""
95+
steps:
96+
- run:
97+
name: Install << parameters.formulae >>
98+
no_output_timeout: "10m"
99+
command: |
100+
set -ex
101+
export HOMEBREW_NO_AUTO_UPDATE=1
102+
brew install << parameters.formulae >>
103+
104+
run_brew_for_ios_build:
105+
steps:
106+
- brew_update
107+
- brew_install:
108+
formulae: libtool
109+
57110
binary_common: &binary_common
58111
parameters:
59112
# Edit these defaults to do a release
@@ -91,6 +144,22 @@ binary_common: &binary_common
91144
UNICODE_ABI: << parameters.unicode_abi >>
92145
CU_VERSION: << parameters.cu_version >>
93146

147+
torchvision_ios_params: &torchvision_ios_params
148+
parameters:
149+
build_environment:
150+
type: string
151+
default: ""
152+
ios_arch:
153+
type: string
154+
default: ""
155+
ios_platform:
156+
type: string
157+
default: ""
158+
environment:
159+
BUILD_ENVIRONMENT: << parameters.build_environment >>
160+
IOS_ARCH: << parameters.ios_arch >>
161+
IOS_PLATFORM: << parameters.ios_platform >>
162+
94163
smoke_test_common: &smoke_test_common
95164
<<: *binary_common
96165
docker:
@@ -288,6 +357,43 @@ jobs:
288357
paths:
289358
- "*"
290359

360+
binary_ios_build:
361+
<<: *torchvision_ios_params
362+
macos:
363+
xcode: "12.0"
364+
steps:
365+
- attach_workspace:
366+
at: ~/workspace
367+
- checkout
368+
- run_brew_for_ios_build
369+
- run:
370+
name: Build
371+
no_output_timeout: "1h"
372+
command: |
373+
script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh"
374+
cat "$script"
375+
source "$script"
376+
- persist_to_workspace:
377+
root: /Users/distiller/workspace/
378+
paths: ios
379+
380+
binary_ios_upload:
381+
<<: *torchvision_ios_params
382+
macos:
383+
xcode: "12.0"
384+
steps:
385+
- attach_workspace:
386+
at: ~/workspace
387+
- checkout
388+
- run_brew_for_ios_build
389+
- run:
390+
name: Upload
391+
no_output_timeout: "1h"
392+
command: |
393+
script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh"
394+
cat "$script"
395+
source "$script"
396+
291397
binary_macos_conda:
292398
<<: *binary_common
293399
macos:
@@ -607,7 +713,7 @@ jobs:
607713
{% raw %}
608714
keys:
609715
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
610-
{% endraw %}
716+
{% endraw %}
611717
- run:
612718
name: Setup
613719
command: .circleci/unittest/windows/scripts/setup_env.sh
@@ -827,6 +933,7 @@ workflows:
827933
- clang_format
828934
- torchhub_test
829935
- torch_onnx_test
936+
{{ ios_workflows() }}
830937

831938
unittest:
832939
jobs:
@@ -846,6 +953,7 @@ workflows:
846953
- clang_format
847954
- torchhub_test
848955
- torch_onnx_test
956+
{{ ios_workflows(nightly=True) }}
849957
{{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
850958
docker_build:
851959
triggers:

‎.circleci/regenerate.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,36 @@ def cmake_workflows(indentation=6):
261261
return indent(indentation, jobs)
262262

263263

264+
def ios_workflows(indentation=6, nightly=False):
265+
jobs = []
266+
build_job_names = []
267+
name_prefix = "nightly_" if nightly else ""
268+
env_prefix = "nightly-" if nightly else ""
269+
for arch, platform in [('x86_64', 'SIMULATOR'), ('arm64', 'OS')]:
270+
name = f'{name_prefix}binary_libtorchvision_ops_ios_12.0.0_{arch}'
271+
build_job_names.append(name)
272+
build_job = {
273+
'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-{arch}',
274+
'context': 'org-member',
275+
'ios_arch': arch,
276+
'ios_platform': platform,
277+
'name': name,
278+
}
279+
if nightly:
280+
build_job['filters'] = gen_filter_branch_tree('nightly')
281+
jobs.append({'binary_ios_build': build_job})
282+
283+
if nightly:
284+
upload_job = {
285+
'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-upload',
286+
'context': 'org-member',
287+
'filters': gen_filter_branch_tree('nightly'),
288+
'requires': build_job_names,
289+
}
290+
jobs.append({'binary_ios_upload': upload_job})
291+
return indent(indentation, jobs)
292+
293+
264294
if __name__ == "__main__":
265295
d = os.path.dirname(__file__)
266296
env = jinja2.Environment(
@@ -275,4 +305,5 @@ def cmake_workflows(indentation=6):
275305
build_workflows=build_workflows,
276306
unittest_workflows=unittest_workflows,
277307
cmake_workflows=cmake_workflows,
308+
ios_workflows=ios_workflows,
278309
))

0 commit comments

Comments
 (0)