Skip to content

Commit a56c0ea

Browse files
pjh5pjh5
authored
Merge pull request pytorch#78 from pjh5/notoolkit
Removing cudatoolkit and most sed calls
2 parents b38ca42 + b2bc806 commit a56c0ea

File tree

3 files changed

+106
-95
lines changed

3 files changed

+106
-95
lines changed

‎conda/build_pytorch.sh

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ fi
1414

1515
if [ "$#" -ne 3 ]; then
1616
echo "Illegal number of parameters. Pass cuda version, pytorch version, build number"
17+
echo "CUDA version should be Mm with no dot, e.g. '80'"
18+
echo "DESIRED_PYTHON should be M.m, e.g. '2.7'"
1719
exit 1
1820
fi
1921

@@ -35,6 +37,9 @@ else
3537
PYTORCH_BRANCH="v$build_version"
3638
fi
3739

40+
# Don't upload the packages until we've verified that they're correct
41+
conda config --set anaconda_upload no
42+
3843
# Fill in missing env variables
3944
if [ -z "$ANACONDA_TOKEN" ]; then
4045
# Token needed to upload to the conda channel above
@@ -47,15 +52,6 @@ fi
4752
if [[ -z "$GITHUB_ORG" ]]; then
4853
GITHUB_ORG='pytorch'
4954
fi
50-
51-
if [[ "$OSTYPE" == "darwin"* ]]; then
52-
DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
53-
fi
54-
55-
# Don't upload the packages until we've verified that they're correct
56-
conda config --set anaconda_upload no
57-
58-
# Keep an array of cmake variables to add to
5955
if [[ -z "$CMAKE_ARGS" ]]; then
6056
# These are passed to tools/build_pytorch_libs.sh::build()
6157
CMAKE_ARGS=()
@@ -64,74 +60,77 @@ if [[ -z "$EXTRA_CAFFE2_CMAKE_FLAGS" ]]; then
6460
# These are passed to tools/build_pytorch_libs.sh::build_caffe2()
6561
EXTRA_CAFFE2_CMAKE_FLAGS=()
6662
fi
67-
68-
# Build for a specified Python, or if none were given then all of them
6963
if [[ -z "$DESIRED_PYTHON" ]]; then
7064
DESIRED_PYTHON=('2.7' '3.5' '3.6' '3.7')
7165
fi
66+
if [[ "$OSTYPE" == "darwin"* ]]; then
67+
DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
68+
fi
69+
if [[ "$desired_cuda" == 'cpu' ]]; then
70+
cpu_only=1
71+
else
72+
# Switch desired_cuda to be M.m to be consistent with other scripts in
73+
# pytorch/builder
74+
cuda_nodot="$desired_cuda"
75+
desired_cuda="${desired_cuda:0:1}.${desired_cuda:1:1}"
76+
fi
77+
78+
7279
echo "Will build for all Pythons: ${DESIRED_PYTHON[@]}"
73-
echo "Will build for all CUDA versions: ${desired_cuda[@]}"
80+
echo "Will build for CUDA version: ${desired_cuda}"
7481

7582
# Determine which build folder to use, if not given it directly
7683
if [[ -n "$TORCH_CONDA_BUILD_FOLDER" ]]; then
7784
build_folder="$TORCH_CONDA_BUILD_FOLDER"
7885
else
79-
if [[ "$OSTYPE" == 'darwin'* || "$desired_cuda" == '90' ]]; then
86+
if [[ "$OSTYPE" == 'darwin'* || "$desired_cuda" == '9.0' ]]; then
8087
build_folder='pytorch'
81-
elif [[ "$desired_cuda" == 'cpu' ]]; then
88+
elif [[ -n "$cpu_only" ]]; then
8289
build_folder='pytorch-cpu'
8390
else
84-
build_folder="pytorch-$desired_cuda"
91+
build_folder="pytorch-$cuda_nodot"
8592
fi
8693
build_folder="$build_folder-$build_version"
8794
fi
8895
meta_yaml="$build_folder/meta.yaml"
8996
echo "Using conda-build folder $build_folder"
9097

9198
# Clone the Pytorch repo, so that we can call run_test.py in it
92-
pytorch_rootdir="root_${GITHUB_ORG}pytorch${PYTORCH_BRANCH}"
99+
pytorch_rootdir="$(pwd)/root_${GITHUB_ORG}pytorch${PYTORCH_BRANCH}"
93100
rm -rf "$pytorch_rootdir"
94-
git clone "https://github.com/$GITHUB_ORG/pytorch.git" "$pytorch_rootdir"
101+
git clone --recursive "https://github.com/$GITHUB_ORG/pytorch.git" "$pytorch_rootdir"
95102
pushd "$pytorch_rootdir"
96103
git checkout "$PYTORCH_BRANCH"
97104
popd
98105

99106
# Switch between CPU or CUDA configerations
100-
build_string="$PYTORCH_BUILD_NUMBER"
101-
if [[ "$desired_cuda" == 'cpu' ]]; then
107+
build_string_suffix="$PYTORCH_BUILD_NUMBER"
108+
if [[ -n "$cpu_only" ]]; then
102109
export NO_CUDA=1
103110
export CUDA_VERSION="0.0"
104111
export CUDNN_VERSION="0.0"
105-
$portable_sed '/cudatoolkit/d' "$meta_yaml"
106112
if [[ "$OSTYPE" != "darwin"* ]]; then
107-
build_string="_cpu_${build_string}"
113+
build_string_suffix="cpu_${build_string_suffix}"
108114
fi
115+
$portable_sed "/magma-cuda.*/d" "$meta_yaml"
109116
else
110117
# Switch the CUDA version that /usr/local/cuda points to. This script also
111118
# sets CUDA_VERSION and CUDNN_VERSION
112-
echo "Switching to CUDA version ${desired_cuda:0:1}.${desired_cuda:1:1}"
113-
. ./switch_cuda_version.sh "${desired_cuda:0:1}.${desired_cuda:1:1}"
114-
$portable_sed "s/cudatoolkit =[0-9]/cudatoolkit =${desired_cuda:0:1}/g" "$meta_yaml"
115-
build_string="_${CUDA_VERSION}_${CUDNN_VERSION}_${build_string}"
116-
if [[ "$desired_cuda" == 92 ]]; then
119+
echo "Switching to CUDA version $desired_cuda"
120+
. ./switch_cuda_version.sh "$desired_cuda"
121+
build_string_suffix="cuda${CUDA_VERSION}_cudnn${CUDNN_VERSION}_${build_string_suffix}"
122+
if [[ "$desired_cuda" == '9.2' ]]; then
117123
# ATen tests can't build with CUDA 9.2 and the old compiler used here
118124
EXTRA_CAFFE2_CMAKE_FLAGS+=("-DATEN_NO_TEST=ON")
119125
fi
120126
ALLOW_DISTRIBUTED_TEST_ERRORS=1
121127
fi
122128

123-
# Alter the meta.yaml to use passed in Github repo/branch
124-
$portable_sed "s#git_url:.*#git_url: https://github.com/$GITHUB_ORG/pytorch#g" "$meta_yaml"
125-
$portable_sed "s#git_rev:.*#git_rev: $PYTORCH_BRANCH#g" "$meta_yaml"
126-
127129
# Loop through all Python versions to build a package for each
128130
for py_ver in "${DESIRED_PYTHON[@]}"; do
129-
build_string="py${py_ver:0:1}${py_ver:2:1}_${build_string}"
131+
build_string="py${py_ver}_${build_string_suffix}"
130132
folder_tag="${build_string}_$(date +'%Y%m%d')"
131133

132-
# Rewrite the build string in meta.yaml to handle cpu-only packages
133-
$portable_sed "s/string:.*/string: ${build_string}/g" "$meta_yaml"
134-
135134
# Create the conda package into this temporary folder. This is so we can find
136135
# the package afterwards, as there's no easy way to extract the final filename
137136
# from conda-build
@@ -143,6 +142,9 @@ for py_ver in "${DESIRED_PYTHON[@]}"; do
143142
echo "Build $build_folder for Python version $py_ver"
144143
time CMAKE_ARGS=${CMAKE_ARGS[@]} \
145144
EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]} \
145+
PYTORCH_GITHUB_ROOT_DIR="$pytorch_rootdir" \
146+
PYTORCH_BUILD_STRING="$build_string" \
147+
PYTORCH_MAGMA_CUDA_VERSION="$cuda_nodot" \
146148
conda build -c "$ANACONDA_USER" \
147149
--no-anaconda-upload \
148150
--python "$py_ver" \
@@ -156,9 +158,6 @@ for py_ver in "${DESIRED_PYTHON[@]}"; do
156158
conda create -yn "$test_env" python="$py_ver"
157159
source activate "$test_env"
158160
conda install -y numpy>=1.11 mkl>=2018 cffi ninja
159-
if [[ "$desired_cuda" == 80 || "$desired_cuda" == 90 ]]; then
160-
conda install -y cudatoolkit=="${desired_cuda:0:1}"
161-
fi
162161

163162
# Extract the package for testing
164163
ls -lah "$output_folder"
@@ -185,8 +184,9 @@ for py_ver in "${DESIRED_PYTHON[@]}"; do
185184
# Clean up test folder
186185
source deactivate
187186
conda env remove -yn "$test_env"
188-
# rm -rf "$output_folder"
187+
rm -rf "$output_folder"
189188
done
189+
rm -rf "$pytorch_rootdir"
190190

191191
unset PYTORCH_BUILD_VERSION
192192
unset PYTORCH_BUILD_NUMBER

‎conda/pytorch-nightly/build.sh

Lines changed: 63 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,20 @@ export TH_BINARY_BUILD=1
77
export PYTORCH_BUILD_VERSION=$PKG_VERSION
88
export PYTORCH_BUILD_NUMBER=$PKG_BUILDNUM
99

10+
# MacOS build is simple, and will not be for CUDA
11+
if [[ "$OSTYPE" == "darwin"* ]]; then
12+
MACOSX_DEPLOYMENT_TARGET=10.9 \
13+
CXX=clang++ \
14+
CC=clang \
15+
python setup.py install
16+
exit 0
17+
fi
18+
19+
1020
if [[ -z "$NO_CUDA" || "$NO_CUDA" == 0 ]]; then
21+
build_with_cuda=1
22+
fi
23+
if [[ -n "$build_with_cuda" ]]; then
1124
# compile for Kepler, Kepler+Tesla, Maxwell, Volta
1225
export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;6.1"
1326
export TORCH_NVCC_FLAGS="-Xfatbin -compress-all"
@@ -31,64 +44,62 @@ fname_with_sha256() {
3144
fi
3245
}
3346

34-
DEPS_LIST=(
35-
)
47+
DEPS_LIST=()
48+
if [[ -n "$build_with_cuda" ]]; then
49+
cuda_majmin="${CUDA_VERSION:0:3}"
50+
DEPS_LIST+=("/usr/local/cuda/lib64/libcudart.so.$cuda_majmin")
51+
DEPS_LIST+=("/usr/local/cuda/lib64/libnvToolsExt.so.1")
52+
DEPS_LIST+=("/usr/local/cuda/lib64/libnvrtc.so.$cuda_majmin")
53+
DEPS_LIST+=("/usr/local/cuda/lib64/libnvrtc-builtins.so")
54+
fi
3655

37-
DEPS_SONAME=(
38-
)
3956

57+
# install
58+
python setup.py install
4059

41-
if [[ "$OSTYPE" == "darwin"* ]]; then
42-
MACOSX_DEPLOYMENT_TARGET=10.9 \
43-
CXX=clang++ \
44-
CC=clang \
45-
python setup.py install
46-
else
47-
# install
48-
python setup.py install
60+
# copy over needed dependent .so files over and tag them with their hash
61+
patched=()
62+
for filepath in "${DEPS_LIST[@]}"; do
63+
filename=$(basename $filepath)
64+
destpath=$SP_DIR/torch/lib/$filename
65+
cp $filepath $destpath
4966

50-
# copy over needed dependent .so files over and tag them with their hash
51-
patched=()
52-
for filepath in "${DEPS_LIST[@]}"; do
53-
filename=$(basename $filepath)
54-
destpath=$SP_DIR/torch/lib/$filename
55-
cp $filepath $destpath
67+
patchedpath=$(fname_with_sha256 $destpath)
68+
patchedname=$(basename $patchedpath)
69+
if [[ "$destpath" != "$patchedpath" ]]; then
70+
mv $destpath $patchedpath
71+
fi
5672

57-
patchedpath=$(fname_with_sha256 $destpath)
58-
patchedname=$(basename $patchedpath)
59-
mv $destpath $patchedpath
73+
patched+=("$patchedname")
74+
echo "Copied $filepath to $patchedpath"
75+
done
6076

61-
patched+=("$patchedname")
62-
echo "Copied $filepath to $patchedpath"
77+
# run patchelf to fix the so names to the hashed names
78+
for ((i=0;i<${#DEPS_LIST[@]};++i)); do
79+
find $SP_DIR/torch -name '*.so*' | while read sofile; do
80+
origname="$(basename ${DEPS_LIST[i]})"
81+
patchedname=${patched[i]}
82+
set +e
83+
patchelf --print-needed $sofile | grep $origname 2>&1 >/dev/null
84+
ERRCODE=$?
85+
set -e
86+
if [ "$ERRCODE" -eq "0" ]; then
87+
echo "patching $sofile entry $origname to $patchedname"
88+
patchelf --replace-needed $origname $patchedname $sofile
89+
fi
6390
done
91+
done
6492

65-
# run patchelf to fix the so names to the hashed names
66-
for ((i=0;i<${#DEPS_LIST[@]};++i)); do
67-
find $SP_DIR/torch -name '*.so*' | while read sofile; do
68-
origname=${DEPS_SONAME[i]}
69-
patchedname=${patched[i]}
70-
set +e
71-
patchelf --print-needed $sofile | grep $origname 2>&1 >/dev/null
72-
ERRCODE=$?
73-
set -e
74-
if [ "$ERRCODE" -eq "0" ]; then
75-
echo "patching $sofile entry $origname to $patchedname"
76-
patchelf --replace-needed $origname $patchedname $sofile
77-
fi
78-
done
79-
done
93+
# set RPATH of _C.so and similar to $ORIGIN, $ORIGIN/lib and conda/lib
94+
find $SP_DIR/torch -name "*.so*" -maxdepth 1 -type f | while read sofile; do
95+
echo "Setting rpath of $sofile to " '$ORIGIN:$ORIGIN/lib:$ORIGIN/../../..'
96+
patchelf --set-rpath '$ORIGIN:$ORIGIN/lib:$ORIGIN/../../..' $sofile
97+
patchelf --print-rpath $sofile
98+
done
8099

81-
# set RPATH of _C.so and similar to $ORIGIN, $ORIGIN/lib and conda/lib
82-
find $SP_DIR/torch -name "*.so*" -maxdepth 1 -type f | while read sofile; do
83-
echo "Setting rpath of $sofile to " '$ORIGIN:$ORIGIN/lib:$ORIGIN/../../..'
84-
patchelf --set-rpath '$ORIGIN:$ORIGIN/lib:$ORIGIN/../../..' $sofile
85-
patchelf --print-rpath $sofile
86-
done
87-
88-
# set RPATH of lib/ files to $ORIGIN and conda/lib
89-
find $SP_DIR/torch/lib -name "*.so*" -maxdepth 1 -type f | while read sofile; do
90-
echo "Setting rpath of $sofile to " '$ORIGIN:$ORIGIN/lib:$ORIGIN/../../../..'
91-
patchelf --set-rpath '$ORIGIN:$ORIGIN/../../../..' $sofile
92-
patchelf --print-rpath $sofile
93-
done
94-
fi
100+
# set RPATH of lib/ files to $ORIGIN and conda/lib
101+
find $SP_DIR/torch/lib -name "*.so*" -maxdepth 1 -type f | while read sofile; do
102+
echo "Setting rpath of $sofile to " '$ORIGIN:$ORIGIN/lib:$ORIGIN/../../../..'
103+
patchelf --set-rpath '$ORIGIN:$ORIGIN/../../../..' $sofile
104+
patchelf --print-rpath $sofile
105+
done

‎conda/pytorch-nightly/meta.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@ package:
33
version: "{{ environ.get('PYTORCH_BUILD_VERSION') }}"
44

55
source:
6-
git_url: https://github.com/pytorch/pytorch
7-
git_rev: "master"
6+
path: "{{ environ.get('PYTORCH_GITHUB_ROOT_DIR') }}"
87

98
requirements:
109
build:
@@ -18,21 +17,22 @@ requirements:
1817
- mkl-include
1918
- typing
2019
- ninja
21-
- magma-cuda80 # [not osx]
20+
- magma-cuda{{ environ.get('PYTORCH_MAGMA_CUDA_VERSION') }} # [not osx]
2221

2322
run:
2423
- python
2524
- numpy >=1.11
2625
- mkl >=2018
2726
- cffi
2827
- ninja
29-
- cudatoolkit =8 # [not osx]
3028

3129
build:
3230
number: {{ environ.get('PYTORCH_BUILD_NUMBER') }}
3331
detect_binary_files_with_prefix: False
34-
string: py{{py}}_cuda{{ environ['CUDA_VERSION'] }}_cudnn{{ environ['CUDNN_VERSION'] }}_{{environ.get('PYTORCH_BUILD_NUMBER')}}
32+
string: "{{ environ.get('PYTORCH_BUILD_STRING') }}"
3533
script_env:
34+
- CUDA_VERSION
35+
- CUDNN_VERSION
3636
- NO_CUDA
3737
- CMAKE_ARGS
3838
- EXTRA_CAFFE2_CMAKE_FLAGS

0 commit comments

Comments
 (0)