Use python venv for SPMI collection / jit rolling build (#95714)

Use python venv for SPMI collection and jit rolling build

Since it needs to install packages.

Author: AndyAyersMS

eng/pipelines/coreclr/templates/build-jit-job.yml

@@ -57,15 +57,19 @@ jobs:
       value: ''
 
     - ${{ if eq(parameters.osGroup, 'windows') }}:
+      - name: PythonSetupScript
+        value: 'py -3 -m venv $(Build.SourcesDirectory)\venv'
       - name: PythonScript
-        value: 'py -3'
+        value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe'
       - name: PipScript
-        value: 'py -3 -m pip'
+        value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe -m pip'
     - ${{ if ne(parameters.osGroup, 'windows') }}:
+      - name: PythonSetupScript
+        value: 'python3 -m venv $(Build.SourcesDirectory)/venv'
       - name: PythonScript
-        value: 'python3'
+        value: '$(Build.SourcesDirectory)/venv/bin/python3'
       - name: PipScript
-        value: 'pip3'
+        value: '$(Build.SourcesDirectory)/venv/bin/pip3'
 
     - ${{ parameters.variables }}
 
@@ -106,8 +110,11 @@ jobs:
           artifactFeeds: public/dotnet-public-pypi
           onlyAddExtraIndex: false
 
+      - script: $(PythonSetupScript)
+        displayName: Enable python venv
+
       # Ensure the Python azure-storage-blob package is installed before doing the upload.
-      - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall
+      - script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall
         displayName: Upgrade Pip to latest and install azure-storage-blob Python package
 
       - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(buildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) --use_latest_jit_change

eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml

@@ -55,10 +55,12 @@ jobs:
     - CollectionName: ${{ parameters.collectionName }}
 
     - ${{ if eq(parameters.osGroup, 'windows') }}:
+      - name: PythonSetupScript
+        value: 'py -3 -m venv $(Build.SourcesDirectory)\venv'
       - name: PythonScript
-        value: 'py -3'
+        value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe'
       - name: PipScript
-        value: 'py -3 -m pip'
+        value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe -m pip'
       - name: Core_Root_Dir
         value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root'
       - name: MchFilesLocation
@@ -70,10 +72,12 @@ jobs:
       - name: PayloadLocation
         value: '$(Build.SourcesDirectory)\payload'
     - ${{ if ne(parameters.osGroup, 'windows') }}:
+      - name: PythonSetupScript
+        value: 'python3 -m venv $(Build.SourcesDirectory)/venv'
       - name: PythonScript
-        value: 'python3'
+        value: '$(Build.SourcesDirectory)/venv/bin/python3'
       - name: PipScript
-        value: 'pip3'
+        value: '$(Build.SourcesDirectory)/venv/bin/pip3'
       - name: Core_Root_Dir
         value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Tests/Core_Root'
       - name: MchFilesLocation
@@ -112,6 +116,9 @@ jobs:
     steps:
     - ${{ parameters.steps }}
 
+    - script: $(PythonSetupScript)
+      displayName: Enable python venv
+
     - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB
       displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }}
 
@@ -173,7 +180,7 @@ jobs:
         onlyAddExtraIndex: false
 
     # Ensure the Python azure-storage-blob package is installed before doing the upload.
-    - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall
+    - script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall
       displayName: Upgrade Pip to latest and install azure-storage-blob Python package
 
     - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)