From 0f0f21819333027f217c0d68db376e344862a1ba Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 8 Nov 2022 13:17:10 -0800 Subject: [PATCH 1/2] Testing new windows runners test Testing Testing testing testing test Test Testing testing Testing Testing test Test test testing testing Test testing test testing testing testing testing testing testing test test testing testing testing testing Test test test testing testing testing testing testing testing testing testing testing Refactor code --- .../validate-windows-binary/action.yml | 47 ---------- .../workflows/validate-windows-binaries.yml | 89 ++++++------------- test/smoke_test/smoke_test.py | 3 +- 3 files changed, 28 insertions(+), 111 deletions(-) delete mode 100644 .github/actions/validate-windows-binary/action.yml diff --git a/.github/actions/validate-windows-binary/action.yml b/.github/actions/validate-windows-binary/action.yml deleted file mode 100644 index be8c7a443..000000000 --- a/.github/actions/validate-windows-binary/action.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: 'validate-windows-binary' -description: 'Windows Binary Conda or Wheel Validation' -inputs: - gpu_arch_type: - description: 'GPU arch type' - required: true - default: 'cpu' - gpu_arch_ver: - description: 'GPU arch version' - required: true - default: 'cpu' - installation: - description: 'Installation instructions' - required: true - default: '' - python_version: - description: 'Python version' - required: true - default: '3.9' -runs: - using: "composite" - steps: - - name: Check nvidia smi - if: ${{ inputs.gpu_arch_type == 'cuda' }} - shell: powershell - run: | - nvidia-smi - - name: Install conda - if: ${{ inputs.gpu_arch_type == 'cpu' }} - uses: conda-incubator/setup-miniconda@v2 - with: - python-version: ${{ inputs.python_version }} - auto-update-conda: true - miniconda-version: "latest" - activate-environment: conda-env-${{ github.run_id }} - - name: Conda Install pytorch and smoke test - shell: powershell - env: - GPU_ARCH_VER: ${{ inputs.gpu_arch_ver }} - GPU_ARCH_TYPE: ${{ inputs.gpu_arch_type }} - INSTALLATION: ${{ inputs.installation }} - CUDA_VER: ${{ inputs.desired_cuda }} - run: | - conda install numpy pillow python=${{ inputs.python_version }} - $install = '${{ inputs.installation }}' - Invoke-Expression $install - python ./test/smoke_test/smoke_test.py diff --git a/.github/workflows/validate-windows-binaries.yml b/.github/workflows/validate-windows-binaries.yml index abf8f5586..cc06b68e1 100644 --- a/.github/workflows/validate-windows-binaries.yml +++ b/.github/workflows/validate-windows-binaries.yml @@ -20,74 +20,37 @@ on: - all jobs: - generate-windows-conda-matrix: + generate-windows-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main with: - package-type: conda + package-type: all os: windows channel: ${{ inputs.channel }} - generate-windows-wheel-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - channel: ${{ inputs.channel }} - generate-windows-libtorch-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: libtorch - os: windows - channel: ${{ inputs.channel }} - - win-conda: - needs: generate-windows-conda-matrix - strategy: - matrix: - ${{ fromJson(needs.generate-windows-conda-matrix.outputs.matrix) }} - fail-fast: false - runs-on: ${{ matrix.validation_runner }} - steps: - - name: Checkout PyTorch builder - uses: actions/checkout@v2 - - name: Validate binary conda - uses: ./.github/actions/validate-windows-binary - with: - gpu_arch_type: ${{ matrix.gpu_arch_type }} - gpu_arch_ver: ${{ matrix.gpu_arch_version }} - installation: ${{ matrix.installation }} - python_version: ${{ matrix.python_version }} - win-wheel: - needs: generate-windows-wheel-matrix + win: + needs: generate-windows-matrix strategy: - matrix: - ${{ fromJson(needs.generate-windows-wheel-matrix.outputs.matrix) }} + matrix: ${{ fromJson(needs.generate-windows-matrix.outputs.matrix) }} fail-fast: false - runs-on: ${{ matrix.validation_runner }} - steps: - - name: Checkout PyTorch builder - uses: actions/checkout@v2 - - name: Validate binary wheel - uses: ./.github/actions/validate-windows-binary - with: - gpu_arch_type: ${{ matrix.gpu_arch_type }} - gpu_arch_ver: ${{ matrix.gpu_arch_version }} - installation: ${{ matrix.installation }} - python_version: ${{ matrix.python_version }} + uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + with: + runner: ${{ matrix.validation_runner }} + repository: "pytorch/builder" + job-name: ${{ matrix.build_name }} + script: | + set -ex + export ENV_NAME="conda-env-${{ github.run_id }}" + export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}" + export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}" + export INSTALLATION="${{ matrix.installation }}" + export CUDA_VER="${{ matrix.desired_cuda }}" - win-libt: - needs: generate-windows-libtorch-matrix - strategy: - matrix: - ${{ fromJson(needs.generate-windows-libtorch-matrix.outputs.matrix) }} - fail-fast: false - runs-on: "windows-2019" - env: - PYTHON_VERSION: ${{ matrix.python_version }} - steps: - - name: Install pytorch and smoke test - shell: powershell - run: | - $install = '${{ matrix.installation }}' - Invoke-WebRequest -Uri $install -OutFile 'libtorch.zip' - Expand-Archive -Force libtorch.zip . + if [[ ${{ matrix.package_type }} == "libtorch" ]]; then + curl ${{ matrix.installation }} -o libtorch.zip + unzip libtorch.zip + else + conda create -y -n ${ENV_NAME} python=${{ matrix.python_version }} numpy pillow + conda activate ${ENV_NAME} + eval $INSTALLATION + python ./test/smoke_test/smoke_test.py + fi diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index 0ddcd267a..2e66eb508 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -4,6 +4,7 @@ from pathlib import Path import argparse import torch +import platform gpu_arch_ver = os.getenv("GPU_ARCH_VER") gpu_arch_type = os.getenv("GPU_ARCH_TYPE") @@ -78,7 +79,7 @@ def smoke_test_cuda(package: str) -> None: if(package == 'all'): import torchaudio import torchvision - if installation_str.find("nightly") != -1: + if installation_str.find("nightly") != -1 or platform.system() == "Windows" : # just print out cuda version, as version check were already performed during import print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}") print(f"torchaudio cuda: {torch.ops.torchaudio.cuda_version()}") From 2f902ae53b3e36535b50dae8895496cbe8d67c4e Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 9 Nov 2022 13:58:29 -0800 Subject: [PATCH 2/2] Adding details for the test-infra issue --- test/smoke_test/smoke_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index 2e66eb508..2c80aad04 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -79,6 +79,8 @@ def smoke_test_cuda(package: str) -> None: if(package == 'all'): import torchaudio import torchvision + # There is an issue with current windows runners calling conda from python + # https://github.com/pytorch/test-infra/issues/1054 if installation_str.find("nightly") != -1 or platform.system() == "Windows" : # just print out cuda version, as version check were already performed during import print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}")