From 323e1c87326cc8ad3a8bb7691e113a22f7c4242b Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 5 Apr 2023 08:23:03 -0700 Subject: [PATCH 1/2] Remove special case for Python 3.11 --- .github/scripts/install_torch.sh | 19 +++++++++ .github/scripts/validate_binaries.sh | 64 ++++++++++------------------ aarch64_linux/build_aarch64_wheel.py | 6 +-- conda/build_pytorch.sh | 6 --- 4 files changed, 43 insertions(+), 52 deletions(-) create mode 100644 .github/scripts/install_torch.sh diff --git a/.github/scripts/install_torch.sh b/.github/scripts/install_torch.sh new file mode 100644 index 000000000..ddfc0002c --- /dev/null +++ b/.github/scripts/install_torch.sh @@ -0,0 +1,19 @@ +conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy +conda activate ${ENV_NAME} +export MATRIX_INSTALLATION="${MATRIX_INSTALLATION/torchvision}" +export MATRIX_INSTALLATION="${MATRIX_INSTALLATION/torchaudio}" +eval $MATRIX_INSTALLATION + +export PYTORCH_PIP_PREFIX="" + +if [[ ${MATRIX_CHANNEL} = "nightly" ]]; then + export PYTORCH_PIP_PREFIX="--pre" +fi + +if [[ ${MATRIX_CHANNEL} = "nightly" || ${MATRIX_CHANNEL} = "test" ]]; then + export PYTORCH_PIP_DOWNLOAD_URL="https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}" + export PYTORCH_CONDA_CHANNEL="pytorch-${MATRIX_CHANNEL}" +else + export PYTORCH_CONDA_CHANNEL="pytorch" + export PYTORCH_PIP_DOWNLOAD_URL="https://download.pytorch.org/whl/${MATRIX_DESIRED_CUDA}" +fi diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index 6ce3dd70b..c4de395c8 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -2,49 +2,31 @@ if [[ ${MATRIX_PACKAGE_TYPE} == "libtorch" ]]; then curl ${MATRIX_INSTALLATION} -o libtorch.zip unzip libtorch.zip else - #special case for Python 3.11 - if [[ ${MATRIX_PYTHON_VERSION} == '3.11' ]]; then - conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} - conda activate ${ENV_NAME} - - INSTALLATION=${MATRIX_INSTALLATION/"-c pytorch"/"-c malfet -c pytorch"} - INSTALLATION=${INSTALLATION/"pytorch-cuda"/"pytorch-${MATRIX_CHANNEL}::pytorch-cuda"} - INSTALLATION=${INSTALLATION/"conda install"/"conda install -y"} - - eval $INSTALLATION - python ./test/smoke_test/smoke_test.py + # Special case Pypi installation package, only applicable to linux nightly CUDA 11.7 builds, wheel package + if [[ ${TARGET_OS} == 'linux' && ${MATRIX_GPU_ARCH_VERSION} == '11.7' && ${MATRIX_PACKAGE_TYPE} == 'manywheel' && ${MATRIX_CHANNEL} != 'nightly' ]]; then + conda create -yp ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg + INSTALLATION_PYPI=${MATRIX_INSTALLATION/"cu117"/"cu117_pypi_cudnn"} + INSTALLATION_PYPI=${INSTALLATION_PYPI/"torchvision torchaudio"/""} + INSTALLATION_PYPI=${INSTALLATION_PYPI/"index-url"/"extra-index-url"} + conda run -p ${ENV_NAME}_pypi ${INSTALLATION_PYPI} + conda run -p ${ENV_NAME}_pypi python ./test/smoke_test/smoke_test.py --package torchonly conda deactivate - conda env remove -n ${ENV_NAME} - else - - - - # Special case Pypi installation package, only applicable to linux nightly CUDA 11.7 builds, wheel package - if [[ ${TARGET_OS} == 'linux' && ${MATRIX_GPU_ARCH_VERSION} == '11.7' && ${MATRIX_PACKAGE_TYPE} == 'manywheel' && ${MATRIX_CHANNEL} != 'nightly' ]]; then - conda create -yp ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg - INSTALLATION_PYPI=${MATRIX_INSTALLATION/"cu117"/"cu117_pypi_cudnn"} - INSTALLATION_PYPI=${INSTALLATION_PYPI/"torchvision torchaudio"/""} - INSTALLATION_PYPI=${INSTALLATION_PYPI/"index-url"/"extra-index-url"} - conda run -p ${ENV_NAME}_pypi ${INSTALLATION_PYPI} - conda run -p ${ENV_NAME}_pypi python ./test/smoke_test/smoke_test.py --package torchonly - conda deactivate - conda env remove -p ${ENV_NAME}_pypi - fi - - # Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159 - conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg - conda activate ${ENV_NAME} - INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"} - eval $INSTALLATION + conda env remove -p ${ENV_NAME}_pypi + fi - if [[ ${TARGET_OS} == 'linux' ]]; then - export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib" - export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH - ${PWD}/check_binary.sh - fi + # Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159 + conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg + conda activate ${ENV_NAME} + INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"} + eval $INSTALLATION - python ./test/smoke_test/smoke_test.py - conda deactivate - conda env remove -n ${ENV_NAME} + if [[ ${TARGET_OS} == 'linux' ]]; then + export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib" + export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH + ${PWD}/check_binary.sh fi + + python ./test/smoke_test/smoke_test.py + conda deactivate + conda env remove -n ${ENV_NAME} fi diff --git a/aarch64_linux/build_aarch64_wheel.py b/aarch64_linux/build_aarch64_wheel.py index f7b70208a..5bf8450ee 100755 --- a/aarch64_linux/build_aarch64_wheel.py +++ b/aarch64_linux/build_aarch64_wheel.py @@ -211,14 +211,10 @@ def install_condaforge_python(host: RemoteHost, python_version="3.8") -> None: # Python-3.6 EOLed and not compatible with conda-4.11 install_condaforge(host, suffix="download/4.10.3-10/Miniforge3-4.10.3-10-Linux-aarch64.sh") host.run_cmd(f"conda install -y python={python_version} numpy pyyaml") - elif python_version == "3.11": - install_condaforge(host, suffix="download/4.11.0-4/Miniforge3-4.11.0-4-Linux-aarch64.sh") - # Pytorch-1.10 or older are not compatible with setuptools=59.6 or newer - host.run_cmd(f"conda install -y python={python_version} numpy pyyaml setuptools=59.8.0 -c malfet") else: install_condaforge(host, suffix="download/4.11.0-4/Miniforge3-4.11.0-4-Linux-aarch64.sh") # Pytorch-1.10 or older are not compatible with setuptools=59.6 or newer - host.run_cmd(f"conda install -y python={python_version} numpy pyyaml setuptools=59.5.0") + host.run_cmd(f"conda install -y python={python_version} numpy pyyaml setuptools>=59.5.0") def build_OpenBLAS(host: RemoteHost, git_clone_flags: str = "") -> None: diff --git a/conda/build_pytorch.sh b/conda/build_pytorch.sh index adde9fb27..b9bdcf89b 100755 --- a/conda/build_pytorch.sh +++ b/conda/build_pytorch.sh @@ -299,12 +299,6 @@ else export CONDA_BUILD_EXTRA_ARGS="" fi -if [[ "$DESIRED_PYTHON" == "3.11" ]]; then - # TODO: Remove me when numpy is available in default channel - # or copy numpy to pytorch channel - export CONDA_BUILD_EXTRA_ARGS="-c malfet ${CONDA_BUILD_EXTRA_ARGS}" -fi - # Build PyTorch with Gloo's TCP_TLS transport if [[ "$(uname)" == 'Linux' ]]; then export USE_GLOO_WITH_OPENSSL=1 From fb72091106b3fa734e3d55daa78a7134cf391106 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 5 Apr 2023 08:26:17 -0700 Subject: [PATCH 2/2] Remove install torch script --- .github/scripts/install_torch.sh | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 .github/scripts/install_torch.sh diff --git a/.github/scripts/install_torch.sh b/.github/scripts/install_torch.sh deleted file mode 100644 index ddfc0002c..000000000 --- a/.github/scripts/install_torch.sh +++ /dev/null @@ -1,19 +0,0 @@ -conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy -conda activate ${ENV_NAME} -export MATRIX_INSTALLATION="${MATRIX_INSTALLATION/torchvision}" -export MATRIX_INSTALLATION="${MATRIX_INSTALLATION/torchaudio}" -eval $MATRIX_INSTALLATION - -export PYTORCH_PIP_PREFIX="" - -if [[ ${MATRIX_CHANNEL} = "nightly" ]]; then - export PYTORCH_PIP_PREFIX="--pre" -fi - -if [[ ${MATRIX_CHANNEL} = "nightly" || ${MATRIX_CHANNEL} = "test" ]]; then - export PYTORCH_PIP_DOWNLOAD_URL="https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}" - export PYTORCH_CONDA_CHANNEL="pytorch-${MATRIX_CHANNEL}" -else - export PYTORCH_CONDA_CHANNEL="pytorch" - export PYTORCH_PIP_DOWNLOAD_URL="https://download.pytorch.org/whl/${MATRIX_DESIRED_CUDA}" -fi