[Validations] Add option to use meta cdn for pypi index

.github/scripts/validate_binaries.sh

@@ -17,9 +17,9 @@ else
     # Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159
     conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
     conda activate ${ENV_NAME}
-    
+
     # Remove when https://github.com/pytorch/builder/issues/1985 is fixed
-    if [[ ${MATRIX_GPU_ARCH_TYPE} == 'cuda-aarch64' ]]; then 
+    if [[ ${MATRIX_GPU_ARCH_TYPE} == 'cuda-aarch64' ]]; then
         pip3 install numpy --force-reinstall
     fi
 
@@ -35,6 +35,12 @@ else
         INSTALLATION=${INSTALLATION/"--index-url"/"--extra-index-url"}
     fi
 
+    # use-meta-cdn: use meta cdn for pypi download
+    if [[ ${USE_META_CDN} == 'true' ]]; then
+        INSTALLATION=${INSTALLATION/"download.pytorch.org"/"d3kup0pazkvub8.cloudfront.net"}
+    fi
+
+
     if [[ ${TORCH_ONLY} == 'true' ]]; then
         INSTALLATION=${INSTALLATION/"torchvision torchaudio"/""}
         TEST_SUFFIX=" --package torchonly"

.github/workflows/validate-aarch64-linux-binaries.yml

@@ -7,11 +7,6 @@ on:
         description: "Channel to use (nightly, test, release, all)"
         required: true
         type: string
-      ref:
-        description: 'Reference to checkout, defaults to empty'
-        default: ""
-        required: false
-        type: string
       torchonly:
         description: 'Validate torchonly'
         default: false
@@ -42,6 +37,11 @@ on:
         default: false
         required: false
         type: boolean
+      use-meta-cdn:
+        description: 'Use meta cdn for installing pip binaries'
+        default: false
+        required: false
+        type: boolean
       use-extra-index-url:
         description: 'Use extra-index url for pip tests'
         default: false
@@ -58,11 +58,6 @@ on:
           - nightly
           - test
           - all
-      ref:
-        description: 'Reference to checkout, defaults to empty'
-        default: ""
-        required: false
-        type: string
       torchonly:
         description: 'Validate torchonly'
         default: false
@@ -88,6 +83,11 @@ on:
         default: false
         required: false
         type: boolean
+      use-meta-cdn:
+        description: 'Use meta cdn for installing pip binaries'
+        default: false
+        required: false
+        type: boolean
       use-extra-index-url:
         description: 'Use extra-index url for pip tests'
         default: false
@@ -114,7 +114,7 @@ jobs:
     with:
       runner: ${{ matrix.validation_runner }}
       repository: "pytorch/builder"
-      ref: ${{ inputs.ref || github.ref }}
+      ref: ${{ github.ref }}
       job-name: ${{ matrix.build_name }}
       docker-image: ${{ matrix.container_image }}
       binary-matrix: ${{ toJSON(matrix) }}
@@ -131,6 +131,7 @@ jobs:
         export RELEASE_VERSION=${{ inputs.version }}
         export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }}
         export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }}
+        export USE_META_CDN=${{ inputs.use-meta-cdn }}
         export USE_VERSION_SET=${{ inputs.use-version-set }}
         if [[ ${USE_VERSION_SET} == 'true' ]]; then
           export VISION_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchvision }}

.github/workflows/validate-binaries.yml

@@ -17,11 +17,6 @@ on:
         description: "Channel to use (nightly, test, release, all)"
         required: true
         type: string
-      ref:
-        description: 'Reference to checkout, defaults to empty'
-        default: ""
-        required: false
-        type: string
       torchonly:
         description: 'Validate torchonly'
         default: false
@@ -47,6 +42,11 @@ on:
         default: false
         required: false
         type: boolean
+      use-meta-cdn:
+        description: 'Use meta cdn for installing pip binaries'
+        default: false
+        required: false
+        type: boolean
       use_split_build:
         description: |
           [Experimental] Use Split Build
@@ -81,11 +81,6 @@ on:
           - nightly
           - test
           - all
-      ref:
-        description: 'Reference to checkout, defaults to empty'
-        default: ""
-        required: false
-        type: string
       torchonly:
         description: 'Validate torchonly'
         default: false
@@ -111,6 +106,11 @@ on:
         default: false
         required: false
         type: boolean
+      use-meta-cdn:
+        description: 'Use meta cdn for installing pip binaries'
+        default: false
+        required: false
+        type: boolean
       use_split_build:
         description: |
           [Experimental] Use Split Build
@@ -136,21 +136,20 @@ jobs:
     uses: ./.github/workflows/validate-windows-binaries.yml
     with:
       channel: ${{ inputs.channel }}
-      ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
       version: ${{ inputs.version }}
       release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
       use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }}
       use-force-reinstall: ${{ inputs.use-force-reinstall }}
       use-extra-index-url: ${{ inputs.use-extra-index-url }}
+      use-meta-cdn: ${{ inputs.use-meta-cdn }}
 
   linux:
     if:  inputs.os == 'linux' || inputs.os == 'all'
     needs: generate-release-matrix
     uses: ./.github/workflows/validate-linux-binaries.yml
     with:
       channel: ${{ inputs.channel }}
-      ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
       version: ${{ inputs.version }}
       release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
@@ -159,31 +158,32 @@ jobs:
       use-force-reinstall: ${{ inputs.use-force-reinstall }}
       use_split_build: ${{ inputs.use_split_build }}
       use-extra-index-url: ${{ inputs.use-extra-index-url }}
+      use-meta-cdn: ${{ inputs.use-meta-cdn }}
 
   linux-aarch64:
     if:  inputs.os == 'linux-aarch64' || inputs.os == 'all'
     needs: generate-release-matrix
     uses: ./.github/workflows/validate-aarch64-linux-binaries.yml
     with:
       channel: ${{ inputs.channel }}
-      ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
       version: ${{ inputs.version }}
       release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
       use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }}
       use-force-reinstall: ${{ inputs.use-force-reinstall }}
       use-extra-index-url: ${{ inputs.use-extra-index-url }}
+      use-meta-cdn: ${{ inputs.use-meta-cdn }}
 
   mac-arm64:
     if:  inputs.os == 'macos' || inputs.os == 'all'
     needs: generate-release-matrix
     uses: ./.github/workflows/validate-macos-arm64-binaries.yml
     with:
       channel: ${{ inputs.channel }}
-      ref: ${{ inputs.ref || github.ref }}
       torchonly: ${{ inputs.torchonly }}
       version: ${{ inputs.version }}
       release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
       use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }}
       use-force-reinstall: ${{ inputs.use-force-reinstall }}
       use-extra-index-url: ${{ inputs.use-extra-index-url }}
+      use-meta-cdn: ${{ inputs.use-meta-cdn }}

.github/workflows/validate-linux-binaries.yml

@@ -7,11 +7,6 @@ on:
         description: "Channel to use (nightly, test, release, all)"
         required: true
         type: string
-      ref:
-        description: 'Reference to checkout, defaults to empty'
-        default: ""
-        required: false
-        type: string
       torchonly:
         description: 'Validate torchonly'
         default: false
@@ -47,6 +42,11 @@ on:
         default: false
         required: false
         type: boolean
+      use-meta-cdn:
+        description: 'Use meta cdn for installing pip binaries'
+        default: false
+        required: false
+        type: boolean
       use_split_build:
         description: |
           [Experimental] Use split build
@@ -69,11 +69,6 @@ on:
           - nightly
           - test
           - all
-      ref:
-        description: 'Reference to checkout, defaults to empty'
-        default: ""
-        required: false
-        type: string
       torchonly:
         description: 'Validate torchonly'
         default: false
@@ -104,6 +99,11 @@ on:
         default: false
         required: false
         type: boolean
+      use-meta-cdn:
+        description: 'Use meta cdn for installing pip binaries'
+        default: false
+        required: false
+        type: boolean
       use_split_build:
         description: |
           [Experimental] Use split build
@@ -137,7 +137,7 @@ jobs:
     with:
       runner: ${{ matrix.validation_runner }}
       repository: "pytorch/builder"
-      ref: ${{ inputs.ref || github.ref }}
+      ref: ${{ github.ref }}
       job-name: ${{ matrix.build_name }}
       docker-image: ${{ matrix.gpu_arch_type == 'xpu' && matrix.container_image || 'pytorch/conda-builder' }}
       binary-matrix: ${{ toJSON(matrix) }}
@@ -149,6 +149,7 @@ jobs:
         export INCLUDE_TEST_OPS=${{ inputs.include-test-ops }}
         export USE_ONLY_DL_PYTORCH_ORG=${{ inputs.use-only-dl-pytorch-org }}
         export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }}
+        export USE_META_CDN=${{ inputs.use-meta-cdn }}
         export RELEASE_VERSION=${{ inputs.version }}
         export USE_VERSION_SET=${{ inputs.use-version-set }}
         if [[ ${USE_VERSION_SET} == 'true' ]]; then

.github/workflows/validate-macos-arm64-binaries.yml

@@ -7,11 +7,6 @@ on:
         description: "Channel to use (nightly, test, release, all)"
         required: true
         type: string
-      ref:
-        description: 'Reference to checkout, defaults to empty'
-        default: ""
-        required: false
-        type: string
       torchonly:
         description: 'Validate torchonly'
         default: false
@@ -42,6 +37,11 @@ on:
         default: false
         required: false
         type: boolean
+      use-meta-cdn:
+        description: 'Use meta cdn for installing pip binaries'
+        default: false
+        required: false
+        type: boolean
       use-extra-index-url:
         description: 'Use extra-index url for pip tests'
         default: false
@@ -58,11 +58,6 @@ on:
           - nightly
           - test
           - all
-      ref:
-        description: 'Reference to checkout, defaults to empty'
-        default: ""
-        required: false
-        type: string
       torchonly:
         description: 'Validate torchonly'
         default: false
@@ -88,6 +83,11 @@ on:
         default: false
         required: false
         type: boolean
+      use-meta-cdn:
+        description: 'Use meta cdn for installing pip binaries'
+        default: false
+        required: false
+        type: boolean
       use-extra-index-url:
         description: 'Use extra-index url for pip tests'
         default: false
@@ -113,7 +113,7 @@ jobs:
     with:
       runner: ${{ matrix.validation_runner }}
       repository: "pytorch/builder"
-      ref: ${{ inputs.ref || github.ref }}
+      ref: ${{ github.ref }}
       job-name: ${{ matrix.build_name }}
       binary-matrix: ${{ toJSON(matrix) }}
       script: |
@@ -124,6 +124,7 @@ jobs:
         export RELEASE_VERSION=${{ inputs.version }}
         export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }}
         export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }}
+        export USE_META_CDN=${{ inputs.use-meta-cdn }}
         export USE_VERSION_SET=${{ inputs.use-version-set }}
 
         if [[ ${{ matrix.package_type }} == "conda" ]]; then

.github/workflows/validate-windows-binaries.yml

@@ -7,11 +7,6 @@ on:
         description: "Channel to use (nightly, test, release, all)"
         required: true
         type: string
-      ref:
-        description: 'Reference to checkout, defaults to empty'
-        default: ""
-        required: false
-        type: string
       torchonly:
         description: 'Validate torchonly'
         default: false
@@ -42,6 +37,11 @@ on:
         default: false
         required: false
         type: boolean
+      use-meta-cdn:
+        description: 'Use meta cdn for installing pip binaries'
+        default: false
+        required: false
+        type: boolean
       use-extra-index-url:
         description: 'Use extra-index url for pip tests'
         default: false
@@ -58,11 +58,6 @@ on:
           - nightly
           - test
           - all
-      ref:
-        description: 'Reference to checkout, defaults to empty'
-        default: ""
-        required: false
-        type: string
       torchonly:
         description: 'Validate torchonly'
         default: false
@@ -88,6 +83,11 @@ on:
         default: false
         required: false
         type: boolean
+      use-meta-cdn:
+        description: 'Use meta cdn for installing pip binaries'
+        default: false
+        required: false
+        type: boolean
       use-extra-index-url:
         description: 'Use extra-index url for pip tests'
         default: false
@@ -113,7 +113,7 @@ jobs:
     with:
       runner: ${{ matrix.package_type == 'libtorch' && 'windows.4xlarge' || matrix.validation_runner }}
       repository: "pytorch/builder"
-      ref: ${{ inputs.ref || github.ref }}
+      ref: ${{ github.ref }}
       job-name: ${{ matrix.build_name }}
       binary-matrix: ${{ toJSON(matrix) }}
       timeout: 60
@@ -125,6 +125,7 @@ jobs:
         export RELEASE_VERSION=${{ inputs.version }}
         export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }}
         export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }}
+        export USE_META_CDN=${{ inputs.use-meta-cdn }}
         export USE_VERSION_SET=${{ inputs.use-version-set }}
         if [[ ${USE_VERSION_SET} == 'true' ]]; then
           export VISION_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchvision }}