Skip to content

Commit ead2e21

Browse files
atalmanatalman
authored
Revert "[BE] Remove unused files and dead code" (#1821)
This reverts commit bebc062.
1 parent 129544c commit ead2e21

File tree

5 files changed

+193
-0
lines changed

5 files changed

+193
-0
lines changed

‎.circleci/scripts/binary_checkout.sh

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/bin/bash
2+
# Derived from https://github.com/pytorch/pytorch/blob/2c7df1360aa17d4a6d6726998eede3671bcb36ee/.circleci/scripts/binary_populate_env.sh
3+
4+
set -eux -o pipefail
5+
6+
retry () {
7+
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
8+
}
9+
10+
11+
# This step runs on multiple executors with different envfile locations
12+
if [[ "$OSTYPE" == "msys" ]]; then
13+
# windows executor (builds and tests)
14+
rm -rf /c/w
15+
ln -s "${HOME}" /c/w
16+
WORK_DIR="/c/w"
17+
elif [[ -d "/home/circleci/project" ]]; then
18+
# machine executor (binary tests)
19+
WORK_DIR="${HOME}/project"
20+
else
21+
# macos executor (builds and tests)
22+
# docker executor (binary builds)
23+
WORK_DIR="${HOME}"
24+
fi
25+
26+
if [[ "$OSTYPE" == "msys" ]]; then
27+
# We need to make the paths as short as possible on Windows
28+
PYTORCH_ROOT="$WORK_DIR/p"
29+
BUILDER_ROOT="$WORK_DIR/b"
30+
else
31+
PYTORCH_ROOT="$WORK_DIR/pytorch"
32+
BUILDER_ROOT="$WORK_DIR/builder"
33+
fi
34+
35+
# Persist these variables for the subsequent steps
36+
echo "export WORK_DIR=${WORK_DIR}" >> ${BASH_ENV}
37+
echo "export PYTORCH_ROOT=${PYTORCH_ROOT}" >> ${BASH_ENV}
38+
echo "export BUILDER_ROOT=${BUILDER_ROOT}" >> ${BASH_ENV}
39+
40+
# Clone the Pytorch branch
41+
retry git clone --depth 1 https://github.com/pytorch/pytorch.git "$PYTORCH_ROOT"
42+
# Removed checking out pytorch/pytorch using CIRCLE_PR_NUMBER and CIRCLE_SHA1 as
43+
# those environment variables are tied to the host repo where the build is being
44+
# triggered.
45+
retry git submodule update --init --recursive
46+
pushd "$PYTORCH_ROOT"
47+
echo "Using Pytorch from "
48+
git --no-pager log --max-count 1
49+
popd
50+
51+
# Clone the Builder master repo
52+
retry git clone -q https://github.com/pytorch/builder.git "$BUILDER_ROOT"
53+
pushd "$BUILDER_ROOT"
54+
if [[ -n "${CIRCLE_SHA1:-}" ]]; then
55+
# Check out a specific commit (typically the latest) from pytorch/builder
56+
git reset --hard "${CIRCLE_SHA1}"
57+
git checkout -q -B main
58+
fi
59+
echo "Using builder from "
60+
git --no-pager log --max-count 1
61+
popd

‎.circleci/scripts/binary_populate_env.sh

+1
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ export BUILD_JNI=$BUILD_JNI
143143
export PIP_UPLOAD_FOLDER="$PIP_UPLOAD_FOLDER"
144144
export DOCKER_IMAGE="$DOCKER_IMAGE"
145145
146+
# Remove WORKD_DIR, PYTORCH_ROOT, BUILDER_ROOT defined & persisted in binary_checkout.sh
146147
export MAC_PACKAGE_WORK_DIR="$WORK_DIR"
147148
export MINICONDA_ROOT="$WORK_DIR/miniconda"
148149
export PYTORCH_FINAL_PACKAGE_DIR="$WORK_DIR/final_pkgs"

‎cron/update_s3_htmls.sh

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
# Update the html links file in the s3 bucket Pip uses this html file to look
6+
# through all the wheels and pick the most recently uploaded one (by the
7+
# version, not the actual date of upload). There is one html file per cuda/cpu
8+
# version
9+
10+
# Upload for all CUDA/cpu versions if not given one to use
11+
if [[ -z "$CUDA_VERSIONS" ]]; then
12+
export CUDA_VERSIONS=('cpu' 'cu92' 'cu100' 'cu101' 'cu102' 'cu110' 'rocm5.0' 'rocm5.1.1')
13+
fi
14+
15+
if [[ -z "$HTML_NAME" ]]; then
16+
export HTML_NAME='torch_nightly.html'
17+
fi
18+
19+
# Dry run disabled by default for legacy purposes
20+
DRY_RUN=${DRY_RUN:-disabled}
21+
DRY_RUN_FLAG=""
22+
if [[ "${DRY_RUN}" != disabled ]]; then
23+
DRY_RUN_FLAG="--dryrun"
24+
fi
25+
26+
# NB: includes trailing slash (from PIP_UPLOAD_FOLDER)
27+
s3_base="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}"
28+
29+
# Pull all existing whls in this directory and turn them into html links
30+
# N.B. we use the .dev as a hacky way to exclude all wheels with old
31+
# 'yyyy.mm.dd' versions
32+
#
33+
# NB: replacing + with %2B is to fix old versions of pip which don't
34+
# this transform automatically. This makes the display a little
35+
# ugly but whatever
36+
function generate_html() {
37+
# Trailing slash required in both cases
38+
dir="$1"
39+
url_prefix="$2"
40+
aws s3 ls "${s3_base}${dir}" | grep --only-matching '\S*\.whl' | sed 's#+#%2B#g' | sed 's#.*#<a href="'"${url_prefix}"'&">'"${url_prefix}"'&</a><br>#g'
41+
}
42+
43+
# This will be included in all the sub-indices
44+
generate_html '' '../' > "root-$HTML_NAME"
45+
generate_html '' '' > "$HTML_NAME"
46+
47+
for cuda_ver in "${CUDA_VERSIONS[@]}"; do
48+
generate_html "${cuda_ver}/" "" > "${cuda_ver}-$HTML_NAME"
49+
cat "root-$HTML_NAME" >> "${cuda_ver}-$HTML_NAME"
50+
generate_html "${cuda_ver}/" "${cuda_ver}/" >> "$HTML_NAME"
51+
52+
# Check your work every once in a while
53+
echo "Setting ${cuda_ver}/$HTML_NAME to:"
54+
cat "${cuda_ver}-$HTML_NAME"
55+
(
56+
set -x
57+
aws s3 cp ${DRY_RUN_FLAG} "${cuda_ver}-$HTML_NAME" "s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${cuda_ver}/$HTML_NAME" --acl public-read --cache-control 'no-cache,no-store,must-revalidate'
58+
)
59+
60+
done
61+
62+
# Check your work every once in a while
63+
echo "Setting $HTML_NAME to:"
64+
cat "$HTML_NAME"
65+
(
66+
set -x
67+
68+
# Upload the html file back up
69+
# Note the lack of a / b/c duplicate / do cause problems in s3
70+
aws s3 cp ${DRY_RUN_FLAG} "$HTML_NAME" "$s3_base$HTML_NAME" --acl public-read --cache-control 'no-cache,no-store,must-revalidate'
71+
)

‎manywheel/build_scripts/ssl-check.py

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# cf. https://github.com/pypa/manylinux/issues/53
2+
3+
GOOD_SSL = "https://google.com"
4+
BAD_SSL = "https://self-signed.badssl.com"
5+
6+
import sys
7+
8+
print("Testing SSL certificate checking for Python:", sys.version)
9+
10+
if (sys.version_info[:2] < (2, 7)
11+
or sys.version_info[:2] < (3, 4)):
12+
print("This version never checks SSL certs; skipping tests")
13+
sys.exit(0)
14+
15+
if sys.version_info[0] >= 3:
16+
from urllib.request import urlopen
17+
EXC = OSError
18+
else:
19+
from urllib import urlopen
20+
EXC = IOError
21+
22+
print("Connecting to %s should work" % (GOOD_SSL,))
23+
urlopen(GOOD_SSL)
24+
print("...it did, yay.")
25+
26+
print("Connecting to %s should fail" % (BAD_SSL,))
27+
try:
28+
urlopen(BAD_SSL)
29+
# If we get here then we failed:
30+
print("...it DIDN'T!!!!!11!!1one!")
31+
sys.exit(1)
32+
except EXC:
33+
print("...it did, yay.")

‎manywheel/test_wheel.sh

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
yum install -y wget git
5+
6+
rm -rf /usr/local/cuda*
7+
8+
# Install Anaconda
9+
if ! ls /py
10+
then
11+
echo "Miniconda needs to be installed"
12+
wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
13+
bash ~/miniconda.sh -b -p /py
14+
else
15+
echo "Miniconda is already installed"
16+
fi
17+
18+
export PATH="/py/bin:$PATH"
19+
20+
# Anaconda token
21+
if ls /remote/token
22+
then
23+
source /remote/token
24+
fi
25+
26+
conda install -y conda-build anaconda-client
27+

0 commit comments

Comments
 (0)