diff --git a/.github/actions/validate-windows-binary/action.yml b/.github/actions/validate-windows-binary/action.yml
deleted file mode 100644
index be8c7a443..000000000
--- a/.github/actions/validate-windows-binary/action.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-name: 'validate-windows-binary'
-description: 'Windows Binary Conda or Wheel Validation'
-inputs:
-  gpu_arch_type:
-    description: 'GPU arch type'
-    required: true
-    default: 'cpu'
-  gpu_arch_ver:
-    description: 'GPU arch version'
-    required: true
-    default: 'cpu'
-  installation:
-    description: 'Installation instructions'
-    required: true
-    default: ''
-  python_version:
-    description: 'Python version'
-    required: true
-    default: '3.9'
-runs:
-  using: "composite"
-  steps:
-    - name: Check nvidia smi
-      if: ${{ inputs.gpu_arch_type == 'cuda' }}
-      shell: powershell
-      run: |
-        nvidia-smi
-    - name: Install conda
-      if: ${{ inputs.gpu_arch_type == 'cpu' }}
-      uses: conda-incubator/setup-miniconda@v2
-      with:
-        python-version: ${{ inputs.python_version }}
-        auto-update-conda: true
-        miniconda-version: "latest"
-        activate-environment: conda-env-${{ github.run_id }}
-    - name: Conda Install pytorch and smoke test
-      shell: powershell
-      env:
-        GPU_ARCH_VER: ${{ inputs.gpu_arch_ver }}
-        GPU_ARCH_TYPE: ${{ inputs.gpu_arch_type }}
-        INSTALLATION: ${{ inputs.installation }}
-        CUDA_VER: ${{ inputs.desired_cuda }}
-      run: |
-        conda install numpy pillow python=${{ inputs.python_version }}
-        $install = '${{ inputs.installation }}'
-        Invoke-Expression $install
-        python  ./test/smoke_test/smoke_test.py
diff --git a/.github/workflows/validate-windows-binaries.yml b/.github/workflows/validate-windows-binaries.yml
index abf8f5586..cc06b68e1 100644
--- a/.github/workflows/validate-windows-binaries.yml
+++ b/.github/workflows/validate-windows-binaries.yml
@@ -20,74 +20,37 @@ on:
           - all
 
 jobs:
-  generate-windows-conda-matrix:
+  generate-windows-matrix:
     uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
     with:
-      package-type: conda
+      package-type: all
       os: windows
       channel: ${{ inputs.channel }}
-  generate-windows-wheel-matrix:
-    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
-    with:
-      package-type: wheel
-      os: windows
-      channel: ${{ inputs.channel }}
-  generate-windows-libtorch-matrix:
-    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
-    with:
-      package-type: libtorch
-      os: windows
-      channel: ${{ inputs.channel }}
-
-  win-conda:
-    needs: generate-windows-conda-matrix
-    strategy:
-      matrix:
-        ${{ fromJson(needs.generate-windows-conda-matrix.outputs.matrix) }}
-      fail-fast: false
-    runs-on: ${{ matrix.validation_runner }}
-    steps:
-      - name: Checkout PyTorch builder
-        uses: actions/checkout@v2
-      - name: Validate binary conda
-        uses: ./.github/actions/validate-windows-binary
-        with:
-          gpu_arch_type: ${{ matrix.gpu_arch_type }}
-          gpu_arch_ver: ${{ matrix.gpu_arch_version }}
-          installation: ${{ matrix.installation }}
-          python_version: ${{ matrix.python_version }}
 
-  win-wheel:
-    needs: generate-windows-wheel-matrix
+  win:
+    needs: generate-windows-matrix
     strategy:
-      matrix:
-        ${{ fromJson(needs.generate-windows-wheel-matrix.outputs.matrix) }}
+      matrix: ${{ fromJson(needs.generate-windows-matrix.outputs.matrix) }}
       fail-fast: false
-    runs-on: ${{ matrix.validation_runner }}
-    steps:
-      - name: Checkout PyTorch builder
-        uses: actions/checkout@v2
-      - name: Validate binary wheel
-        uses: ./.github/actions/validate-windows-binary
-        with:
-          gpu_arch_type: ${{ matrix.gpu_arch_type }}
-          gpu_arch_ver: ${{ matrix.gpu_arch_version }}
-          installation: ${{ matrix.installation }}
-          python_version: ${{ matrix.python_version }}
+    uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
+    with:
+      runner: ${{ matrix.validation_runner }}
+      repository: "pytorch/builder"
+      job-name: ${{ matrix.build_name }}
+      script: |
+        set -ex
+        export ENV_NAME="conda-env-${{ github.run_id }}"
+        export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
+        export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
+        export INSTALLATION="${{ matrix.installation }}"
+        export CUDA_VER="${{ matrix.desired_cuda }}"
 
-  win-libt:
-    needs: generate-windows-libtorch-matrix
-    strategy:
-      matrix:
-        ${{ fromJson(needs.generate-windows-libtorch-matrix.outputs.matrix) }}
-      fail-fast: false
-    runs-on: "windows-2019"
-    env:
-      PYTHON_VERSION: ${{ matrix.python_version }}
-    steps:
-      - name: Install pytorch and smoke test
-        shell: powershell
-        run: |
-          $install = '${{ matrix.installation }}'
-          Invoke-WebRequest -Uri $install -OutFile 'libtorch.zip'
-          Expand-Archive -Force libtorch.zip .
+        if [[ ${{ matrix.package_type }} == "libtorch" ]]; then
+          curl ${{ matrix.installation }} -o libtorch.zip
+          unzip libtorch.zip
+        else
+          conda create -y -n ${ENV_NAME} python=${{ matrix.python_version }} numpy pillow
+          conda activate ${ENV_NAME}
+          eval $INSTALLATION
+          python ./test/smoke_test/smoke_test.py
+        fi
diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py
index 0ddcd267a..2c80aad04 100644
--- a/test/smoke_test/smoke_test.py
+++ b/test/smoke_test/smoke_test.py
@@ -4,6 +4,7 @@
 from pathlib import Path
 import argparse
 import torch
+import platform
 
 gpu_arch_ver = os.getenv("GPU_ARCH_VER")
 gpu_arch_type = os.getenv("GPU_ARCH_TYPE")
@@ -78,7 +79,9 @@ def smoke_test_cuda(package: str) -> None:
     if(package == 'all'):
         import torchaudio
         import torchvision
-        if installation_str.find("nightly") != -1:
+        # There is an issue with current windows runners calling conda from python
+        # https://github.com/pytorch/test-infra/issues/1054
+        if installation_str.find("nightly") != -1 or platform.system() == "Windows" :
             # just print out cuda version, as version check were already performed during import
             print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}")
             print(f"torchaudio cuda: {torch.ops.torchaudio.cuda_version()}")