Skip to content

Commit bb821d4

Browse files
author
Mike Schneider
authored
[aarch64][CICD]Add aarch64 docker image build. (#1472)
* Add aarch64 docker image build * removing ulimit for PT workflow * set aarch64 worker for docker build
1 parent 14851d9 commit bb821d4

File tree

6 files changed

+147
-54
lines changed

6 files changed

+147
-54
lines changed

‎.github/workflows/build-manywheel-images.yml

+17
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,15 @@ on:
1212
paths:
1313
- .github/workflows/build-manywheel-images.yml
1414
- manywheel/Dockerfile
15+
- manywheel/Dockerfile_aarch64
1516
- manywheel/Dockerfile_cxx11-abi
1617
- manywheel/build_docker.sh
1718
- 'common/*'
1819
pull_request:
1920
paths:
2021
- .github/workflows/build-manywheel-images.yml
2122
- manywheel/Dockerfile
23+
- manywheel/Dockerfile_aarch64
2224
- manywheel/Dockerfile_cxx11-abi
2325
- 'common/*'
2426
- manywheel/build_docker.sh
@@ -82,6 +84,21 @@ jobs:
8284
- name: Build Docker Image
8385
run: |
8486
manywheel/build_docker.sh
87+
build-docker-cpu-aarch64:
88+
runs-on: linux.t4g.2xlarge
89+
env:
90+
GPU_ARCH_TYPE: cpu-aarch64
91+
steps:
92+
- name: Checkout PyTorch
93+
uses: actions/checkout@v3
94+
- name: Authenticate if WITH_PUSH
95+
run: |
96+
if [[ "${WITH_PUSH}" == true ]]; then
97+
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin
98+
fi
99+
- name: Build Docker Image
100+
run: |
101+
manywheel/build_docker.sh
85102
build-docker-cpu-cxx11-abi:
86103
runs-on: ubuntu-22.04
87104
env:

‎aarch64_linux/aarch64_ci_setup.sh

+1-20
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,6 @@ CONDA_EXE=/opt/conda/bin/conda
99
PATH=/opt/conda/bin:$PATH
1010
LD_LIBRARY_PATH=/opt/conda/lib:$LD_LIBRARY_PATH
1111

12-
###############################################################################
13-
# Install OS dependent packages
14-
###############################################################################
15-
yum -y install epel-release
16-
yum -y install less zstd libgomp
17-
1812
###############################################################################
1913
# Install conda
2014
# disable SSL_verify due to getting "Could not find a suitable TLS CA certificate bundle, invalid path"
@@ -26,19 +20,6 @@ chmod +x /mambaforge.sh
2620
/mambaforge.sh -b -p /opt/conda
2721
rm /mambaforge.sh
2822
/opt/conda/bin/conda config --set ssl_verify False
29-
/opt/conda/bin/conda install -y -c conda-forge python=${DESIRED_PYTHON} numpy pyyaml setuptools patchelf pygit2 openblas
23+
/opt/conda/bin/conda install -y -c conda-forge python=${DESIRED_PYTHON} numpy pyyaml setuptools patchelf pygit2 openblas ninja scons
3024
python --version
3125
conda --version
32-
33-
###############################################################################
34-
# Exec libglfortran.a hack
35-
#
36-
# libgfortran.a from quay.io/pypa/manylinux2014_aarch64 is not compiled with -fPIC.
37-
# This causes __stack_chk_guard@@GLIBC_2.17 on pytorch build. To solve, get
38-
# ubuntu's libgfortran.a which is compiled with -fPIC
39-
###############################################################################
40-
cd ~/
41-
curl -L -o ~/libgfortran-10-dev.deb http://ports.ubuntu.com/ubuntu-ports/pool/universe/g/gcc-10/libgfortran-10-dev_10.5.0-1ubuntu1_arm64.deb
42-
ar x ~/libgfortran-10-dev.deb
43-
tar --use-compress-program=unzstd -xvf data.tar.zst -C ~/
44-
cp -f ~/usr/lib/gcc/aarch64-linux-gnu/10/libgfortran.a /opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10/

‎aarch64_linux/aarch64_wheel_ci_build.py

+33-34
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,42 @@
11
#!/usr/bin/env python3
2+
# encoding: UTF-8
23

34
import os
45
import subprocess
56
from pygit2 import Repository
67
from typing import List
78

89

9-
''''
10-
Helper for getting paths for Python
11-
'''
1210
def list_dir(path: str) -> List[str]:
13-
return subprocess.check_output(["ls", "-1", path]).decode().split("\n")
11+
''''
12+
Helper for getting paths for Python
13+
'''
14+
return subprocess.check_output(["ls", "-1", path]).decode().split("\n")
1415

1516

16-
'''
17-
Using ArmComputeLibrary for aarch64 PyTorch
18-
'''
1917
def build_ArmComputeLibrary(git_clone_flags: str = "") -> None:
18+
'''
19+
Using ArmComputeLibrary for aarch64 PyTorch
20+
'''
2021
print('Building Arm Compute Library')
2122
os.system("cd / && mkdir /acl")
2223
os.system(f"git clone https://github.com/ARM-software/ComputeLibrary.git -b v23.05.1 {git_clone_flags}")
2324
os.system('sed -i -e \'s/"armv8.2-a"/"armv8-a"/g\' ComputeLibrary/SConscript; '
2425
'sed -i -e \'s/-march=armv8.2-a+fp16/-march=armv8-a/g\' ComputeLibrary/SConstruct; '
2526
'sed -i -e \'s/"-march=armv8.2-a"/"-march=armv8-a"/g\' ComputeLibrary/filedefs.json')
26-
os.system(f"cd ComputeLibrary; export acl_install_dir=/acl; " \
27-
f"scons Werror=1 -j8 debug=0 neon=1 opencl=0 os=linux openmp=1 cppthreads=0 arch=armv8.2-a multi_isa=1 build=native build_dir=$acl_install_dir/build; " \
28-
f"cp -r arm_compute $acl_install_dir; " \
29-
f"cp -r include $acl_install_dir; " \
30-
f"cp -r utils $acl_install_dir; " \
31-
f"cp -r support $acl_install_dir; " \
32-
f"cp -r src $acl_install_dir; cd /")
33-
34-
35-
'''
36-
Complete wheel build and put in artifact location
37-
'''
27+
os.system("cd ComputeLibrary; export acl_install_dir=/acl; "
28+
"scons Werror=1 -j8 debug=0 neon=1 opencl=0 os=linux openmp=1 cppthreads=0 arch=armv8.2-a multi_isa=1 build=native build_dir=$acl_install_dir/build; "
29+
"cp -r arm_compute $acl_install_dir; "
30+
"cp -r include $acl_install_dir; "
31+
"cp -r utils $acl_install_dir; "
32+
"cp -r support $acl_install_dir; "
33+
"cp -r src $acl_install_dir; cd /")
34+
35+
3836
def complete_wheel(folder: str):
37+
'''
38+
Complete wheel build and put in artifact location
39+
'''
3940
wheel_name = list_dir(f"/{folder}/dist")[0]
4041

4142
if "pytorch" in folder:
@@ -54,10 +55,10 @@ def complete_wheel(folder: str):
5455
return repaired_wheel_name
5556

5657

57-
'''
58-
Parse inline arguments
59-
'''
6058
def parse_arguments():
59+
'''
60+
Parse inline arguments
61+
'''
6162
from argparse import ArgumentParser
6263
parser = ArgumentParser("AARCH64 wheels python CD")
6364
parser.add_argument("--debug", action="store_true")
@@ -67,11 +68,10 @@ def parse_arguments():
6768
return parser.parse_args()
6869

6970

70-
'''
71-
Entry Point
72-
'''
7371
if __name__ == '__main__':
74-
72+
'''
73+
Entry Point
74+
'''
7575
args = parse_arguments()
7676
enable_mkldnn = args.enable_mkldnn
7777
repo = Repository('/pytorch')
@@ -80,26 +80,25 @@ def parse_arguments():
8080
branch = 'master'
8181

8282
git_clone_flags = " --depth 1 --shallow-submodules"
83-
os.system(f"conda install -y ninja scons")
8483

8584
print('Building PyTorch wheel')
8685
build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 "
87-
os.system(f"python setup.py clean")
86+
os.system("python setup.py clean")
8887

8988
if branch == 'nightly' or branch == 'master':
90-
build_date = subprocess.check_output(['git','log','--pretty=format:%cs','-1'], cwd='/pytorch').decode().replace('-','')
91-
version = subprocess.check_output(['cat','version.txt'], cwd='/pytorch').decode().strip()[:-2]
89+
build_date = subprocess.check_output(['git', 'log', '--pretty=format:%cs', '-1'], cwd='/pytorch').decode().replace('-', '')
90+
version = subprocess.check_output(['cat', 'version.txt'], cwd='/pytorch').decode().strip()[:-2]
9291
build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={version}.dev{build_date} PYTORCH_BUILD_NUMBER=1 "
9392
if branch.startswith("v1.") or branch.startswith("v2."):
9493
build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={branch[1:branch.find('-')]} PYTORCH_BUILD_NUMBER=1 "
9594
if enable_mkldnn:
9695
build_ArmComputeLibrary(git_clone_flags)
9796
print("build pytorch with mkldnn+acl backend")
9897
build_vars += "USE_MKLDNN=ON USE_MKLDNN_ACL=ON " \
99-
"ACL_ROOT_DIR=/acl " \
100-
"LD_LIBRARY_PATH=/pytorch/build/lib:/acl/build:$LD_LIBRARY_PATH " \
101-
"ACL_INCLUDE_DIR=/acl/build " \
102-
"ACL_LIBRARY=/acl/build "
98+
"ACL_ROOT_DIR=/acl " \
99+
"LD_LIBRARY_PATH=/pytorch/build/lib:/acl/build:$LD_LIBRARY_PATH " \
100+
"ACL_INCLUDE_DIR=/acl/build " \
101+
"ACL_LIBRARY=/acl/build "
103102
else:
104103
print("build pytorch without mkldnn backend")
105104

‎manywheel/Dockerfile_aarch64

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
FROM quay.io/pypa/manylinux2014_aarch64 as base
2+
3+
4+
# Graviton needs GCC 10 for the build
5+
ARG DEVTOOLSET_VERSION=10
6+
7+
# Language variabes
8+
ENV LC_ALL=en_US.UTF-8
9+
ENV LANG=en_US.UTF-8
10+
ENV LANGUAGE=en_US.UTF-8
11+
12+
# Installed needed OS packages. This is to support all
13+
# the binary builds (torch, vision, audio, text, data)
14+
RUN yum -y install epel-release
15+
RUN yum -y update
16+
RUN yum install -y \
17+
autoconf \
18+
automake \
19+
bison \
20+
bzip2 \
21+
curl \
22+
diffutils \
23+
file \
24+
git \
25+
make \
26+
patch \
27+
perl \
28+
unzip \
29+
util-linux \
30+
wget \
31+
which \
32+
xz \
33+
yasm \
34+
less \
35+
zstd \
36+
libgomp \
37+
devtoolset-${DEVTOOLSET_VERSION}-gcc \
38+
devtoolset-${DEVTOOLSET_VERSION}-gcc-c++ \
39+
devtoolset-${DEVTOOLSET_VERSION}-gcc-gfortran \
40+
devtoolset-${DEVTOOLSET_VERSION}-binutils
41+
42+
# Ensure the expected devtoolset is used
43+
ENV PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
44+
ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
45+
46+
47+
# git236+ would refuse to run git commands in repos owned by other users
48+
# Which causes version check to fail, as pytorch repo is bind-mounted into the image
49+
# Override this behaviour by treating every folder as safe
50+
# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327
51+
RUN git config --global --add safe.directory "*"
52+
53+
54+
###############################################################################
55+
# libglfortran.a hack
56+
#
57+
# libgfortran.a from quay.io/pypa/manylinux2014_aarch64 is not compiled with -fPIC.
58+
# This causes __stack_chk_guard@@GLIBC_2.17 on pytorch build. To solve, get
59+
# ubuntu's libgfortran.a which is compiled with -fPIC
60+
# NOTE: Need a better way to get this library as Ubuntu's package can be removed by the vender, or changed
61+
###############################################################################
62+
RUN cd ~/ \
63+
&& curl -L -o ~/libgfortran-10-dev.deb http://ports.ubuntu.com/ubuntu-ports/pool/universe/g/gcc-10/libgfortran-10-dev_10.5.0-1ubuntu1_arm64.deb \
64+
&& ar x ~/libgfortran-10-dev.deb \
65+
&& tar --use-compress-program=unzstd -xvf data.tar.zst -C ~/ \
66+
&& cp -f ~/usr/lib/gcc/aarch64-linux-gnu/10/libgfortran.a /opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10/
67+
68+
# install cmake
69+
RUN yum install -y cmake3 && \
70+
ln -s /usr/bin/cmake3 /usr/bin/cmake
71+
72+
FROM base as openssl
73+
# Install openssl (this must precede `build python` step)
74+
# (In order to have a proper SSL module, Python is compiled
75+
# against a recent openssl [see env vars above], which is linked
76+
# statically. We delete openssl afterwards.)
77+
ADD ./common/install_openssl.sh install_openssl.sh
78+
RUN bash ./install_openssl.sh && rm install_openssl.sh
79+
ENV SSL_CERT_FILE=/opt/_internal/certs.pem
80+
81+
FROM openssl as final
82+
# remove unncessary python versions
83+
RUN rm -rf /opt/python/cp26-cp26m /opt/_internal/cpython-2.6.9-ucs2
84+
RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4
85+
RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6
86+
RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6

‎manywheel/build_all_docker.sh

+2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ TOPDIR=$(git rev-parse --show-toplevel)
77
GPU_ARCH_TYPE=cpu "${TOPDIR}/manywheel/build_docker.sh"
88
MANYLINUX_VERSION=2014 GPU_ARCH_TYPE=cpu "${TOPDIR}/manywheel/build_docker.sh"
99

10+
GPU_ARCH_TYPE=cpu-aarch64 "${TOPDIR}/manywheel/build_docker.sh"
11+
1012
GPU_ARCH_TYPE=cpu-cxx11-abi "${TOPDIR}/manywheel/build_docker.sh"
1113

1214
for cuda_version in 12.1 11.8; do

‎manywheel/build_docker.sh

+8
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,14 @@ case ${GPU_ARCH_TYPE} in
2020
GPU_IMAGE=centos:7
2121
DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=9"
2222
;;
23+
cpu-aarch64)
24+
TARGET=final
25+
DOCKER_TAG=cpu-aarch64
26+
LEGACY_DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/manylinux-cpu-aarch64
27+
GPU_IMAGE=arm64v8/centos:7
28+
DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=10"
29+
MANY_LINUX_VERSION="aarch64"
30+
;;
2331
cpu-cxx11-abi)
2432
TARGET=final
2533
DOCKER_TAG=cpu-cxx11-abi

0 commit comments

Comments
 (0)