Add poetry and pypi tests (#1419)

* smoke test poetry

Add a little more tests

test

Test poetry

test

Test poetry on python 3.10

Add more poetry tests

Test en us

test

test

Try verboose

testing

testing

try quiet install

Code refactooring

test

move linux pipy validation to workflow

test

test

Fix path

try test pipy

More torch installations

test

testing

test

test

test new

fix install 2

try poetry nightly

test nightly

test

test

Test poetry validation

test

test_new

test

* Put back executing this on pull

Author: atalman

.github/scripts/validate_binaries.sh

@@ -2,18 +2,6 @@ if [[ ${MATRIX_PACKAGE_TYPE} == "libtorch" ]]; then
     curl ${MATRIX_INSTALLATION} -o libtorch.zip
     unzip libtorch.zip
 else
-    # Special case Pypi installation package, only applicable to linux nightly CUDA 11.7 builds, wheel package
-    if [[ ${TARGET_OS} == 'linux' && ${MATRIX_GPU_ARCH_VERSION} == '11.7' && ${MATRIX_PACKAGE_TYPE} == 'manywheel' && ${MATRIX_CHANNEL} != 'nightly' ]]; then
-        conda create -yp ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
-        INSTALLATION_PYPI=${MATRIX_INSTALLATION/"cu117"/"cu117_pypi_cudnn"}
-        INSTALLATION_PYPI=${INSTALLATION_PYPI/"torchvision torchaudio"/""}
-        INSTALLATION_PYPI=${INSTALLATION_PYPI/"index-url"/"extra-index-url"}
-        conda run -p ${ENV_NAME}_pypi ${INSTALLATION_PYPI}
-        conda run -p ${ENV_NAME}_pypi python ./test/smoke_test/smoke_test.py --package torchonly
-        conda deactivate
-        conda env remove -p ${ENV_NAME}_pypi
-    fi
-
     # Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159
     conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
     conda activate ${ENV_NAME}

.github/scripts/validate_pipy.sh

@@ -0,0 +1,12 @@
+conda create -yp ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
+
+if [[ ${MATRIX_CHANNEL} != "release" ]]; then
+    conda run -p ${ENV_NAME}_pypi pip3 install --pre torch --index-url "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}_pypi_cudnn"
+    conda run -p ${ENV_NAME}_pypi pip3 install --pre torchvision torchaudio --index-url "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}"
+else
+    conda run -p ${ENV_NAME}_pypi pip3 install torch torchvision torchaudio
+fi
+
+conda run -p ${ENV_NAME}_pypi python ./test/smoke_test/smoke_test.py
+conda deactivate
+conda env remove -p ${ENV_NAME}_pypi

.github/scripts/validate_poetry.sh

@@ -0,0 +1,25 @@
+
+conda create -y -n ${ENV_NAME}_poetry python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
+conda activate ${ENV_NAME}_poetry
+curl -sSL https://install.python-poetry.org | python3 - --git https://github.com/python-poetry/poetry.git@master
+export PATH="/root/.local/bin:$PATH"
+
+poetry --version
+poetry new test_poetry
+cd test_poetry
+
+if [[ ${MATRIX_CHANNEL} != "release" ]]; then
+    # Installing poetry from our custom repo. We need to configure it before use and disable authentication
+    export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
+    poetry source add --priority=explicit domains "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}"
+    poetry source add --priority=explicit pytorch "https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}_pypi_cudnn"
+    poetry --quiet add --source pytorch torch
+    poetry --quiet add --source domains torchvision torchaudio
+else
+    poetry --quiet add torch torchaudio torchvision
+fi
+
+python ../test/smoke_test/smoke_test.py
+conda deactivate
+conda env remove -p ${ENV_NAME}_poetry
+cd ..

.github/workflows/validate-linux-binaries.yml

@@ -55,4 +55,13 @@ jobs:
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TARGET_OS="linux"
         eval "$(conda shell.bash hook)"
+
+        # Special case PyPi installation package. And Install of PyPi package via poetry
+        if [[ ${MATRIX_PACKAGE_TYPE} == "manywheel" ]] && \
+           {[${MATRIX_GPU_ARCH_VERSION} == "12.1" && ${MATRIX_CHANNEL} != "release"] || \
+            [${MATRIX_GPU_ARCH_VERSION} == "11.7" && ${MATRIX_CHANNEL} == "release"]}; then
+          source ./.github/scripts/validate_pipy.sh
+          source ./.github/scripts/validate_poetry.sh
+        fi
+
         source ./.github/scripts/validate_binaries.sh