diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh
index 97d94ffc2..f05d9b20a 100755
--- a/.github/scripts/validate_binaries.sh
+++ b/.github/scripts/validate_binaries.sh
@@ -25,7 +25,7 @@ else
             INSTALLATION_PYPI=${INSTALLATION_PYPI/"index-url"/"extra-index-url"}
             conda run -p ${ENV_NAME}_pypi ${INSTALLATION_PYPI}
             conda run -p ${ENV_NAME}_pypi python ./test/smoke_test/smoke_test.py --package torchonly
-            conda activate base
+            conda deactivate
             conda env remove -p ${ENV_NAME}_pypi
         fi
 
@@ -41,5 +41,7 @@ else
         fi
 
         python  ./test/smoke_test/smoke_test.py
+        conda deactivate
+        conda env remove -n ${ENV_NAME}
     fi
 fi
diff --git a/.github/workflows/validate-binaries.yml b/.github/workflows/validate-binaries.yml
index 2a6106a20..4ae260538 100644
--- a/.github/workflows/validate-binaries.yml
+++ b/.github/workflows/validate-binaries.yml
@@ -22,6 +22,10 @@ on:
         default: ""
         required: false
         type: string
+      limit-win-builds:
+        description: "Limit windows builds to single python/cuda config"
+        default: "disable"
+        type: string
   workflow_dispatch:
     inputs:
       os:
@@ -49,6 +53,11 @@ on:
         default: ""
         required: false
         type: string
+      limit-win-builds:
+        description: "Limit windows builds to single python/cuda config"
+        default: "disable"
+        required: false
+        type: string
 
 jobs:
   win:
@@ -57,6 +66,7 @@ jobs:
     with:
       channel: ${{ inputs.channel }}
       ref: ${{ inputs.ref || github.ref }}
+      limit-win-builds: ${{ inputs.limit-win-builds }}
 
   linux:
     if:  inputs.os == 'linux' || inputs.os == 'all'
diff --git a/.github/workflows/validate-nightly-binaries.yml b/.github/workflows/validate-nightly-binaries.yml
index 32c6781ba..c252e0433 100644
--- a/.github/workflows/validate-nightly-binaries.yml
+++ b/.github/workflows/validate-nightly-binaries.yml
@@ -32,3 +32,4 @@ jobs:
     with:
       channel: nightly
       os: all
+      limit-win-builds: enable
diff --git a/.github/workflows/validate-release-binaries.yml b/.github/workflows/validate-release-binaries.yml
index cc77561ed..c0e239d03 100644
--- a/.github/workflows/validate-release-binaries.yml
+++ b/.github/workflows/validate-release-binaries.yml
@@ -32,3 +32,4 @@ jobs:
     with:
       channel: release
       os: all
+      limit-win-builds: enable
diff --git a/.github/workflows/validate-windows-binaries.yml b/.github/workflows/validate-windows-binaries.yml
index 497f9849d..7424feebb 100644
--- a/.github/workflows/validate-windows-binaries.yml
+++ b/.github/workflows/validate-windows-binaries.yml
@@ -12,6 +12,10 @@ on:
         default: ""
         required: false
         type: string
+      limit-win-builds:
+        description: "Limit windows builds to single python/cuda config"
+        default: "disable"
+        type: string
   workflow_dispatch:
     inputs:
       channel:
@@ -28,6 +32,11 @@ on:
         default: ""
         required: false
         type: string
+      limit-win-builds:
+        description: "Limit windows builds to single python/cuda config"
+        default: "disable"
+        required: false
+        type: string
 
 jobs:
   generate-windows-matrix:
@@ -36,6 +45,7 @@ jobs:
       package-type: all
       os: windows
       channel: ${{ inputs.channel }}
+      limit-win-builds: ${{ inputs.limit-win-builds }}
 
   win:
     needs: generate-windows-matrix