Experiment to improve CI

.azure-pipelines/jobs/test-windows.yml

@@ -2,67 +2,85 @@ parameters:
   vmImage:
 
 jobs:
-- job: Test_Primary
-  displayName: Test Primary
+- job: Test_Primary_one
+  displayName: Tests / Windows /
 
   pool:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      Python27-x86:
+      "2.7-x86 / 1":
         python.version: '2.7'
         python.architecture: x86
-      Python27-x64:
+      "2.7 / 1":
         python.version: '2.7'
         python.architecture: x64
-        useVenv: true
-      Python35-x64:
-        python.version: '3.5'
-        python.architecture: x64
-      Python36-x64:
-        python.version: '3.6'
+      "3.8 / 1":
+        python.version: '3.8'
         python.architecture: x64
-        useVenv: true
-      Python37-x64:
-        python.version: '3.7'
+
+  steps:
+  - template: ../steps/run-tests-windows.yml
+    parameters:
+      testGroup: one
+
+- job: Test_Primary_two
+  displayName: Tests / Windows /
+
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+  strategy:
+    matrix:
+      "2.7-x86 / 2":
+        python.version: '2.7'
+        python.architecture: x86
+      "2.7 / 2":
+        python.version: '2.7'
         python.architecture: x64
-      Python38-x64:
+      "3.8 / 2":
         python.version: '3.8'
         python.architecture: x64
-    maxParallel: 6
 
   steps:
   - template: ../steps/run-tests-windows.yml
     parameters:
-      runIntegrationTests: true
-      useVenv: '$(useVenv)'
+      testGroup: two
 
 - job: Test_Secondary
-  displayName: Test Secondary
+  displayName: Tests / Windows /
   # Don't run integration tests for these runs
-  # Run after Test_Primary so we don't devour time and jobs if tests are going to fail
-  dependsOn: Test_Primary
+  # Run after Test_Primary_* so we don't devour time and jobs.
+  dependsOn:
+  - Test_Primary_one
+  - Test_Primary_two
 
   pool:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      # This is for Windows, so test x86 builds
-      Python35-x86:
+      3.5:
+        python.version: '3.5'
+        python.architecture: x64
+      3.6:
+        python.version: '3.6'
+        python.architecture: x64
+      3.7:
+        python.version: '3.7'
+        python.architecture: x64
+      3.5-x86:
         python.version: '3.5'
         python.architecture: x86
-      Python36-x86:
+      3.6-x86:
         python.version: '3.6'
         python.architecture: x86
-      Python37-x86:
+      3.7-x86:
         python.version: '3.7'
         python.architecture: x86
-      Python38-x86:
+      3.8-x86:
         python.version: '3.8'
         python.architecture: x86
-    maxParallel: 6
 
   steps:
   - template: ../steps/run-tests-windows.yml
     parameters:
-      runIntegrationTests: false
+      testGroup: unit

.azure-pipelines/steps/run-tests-windows.yml

@@ -1,6 +1,10 @@
 parameters:
-  runIntegrationTests:
-  useVenv: false
+- name: testGroup
+  type: string
+  values:
+  - unit
+  - one
+  - two
 
 steps:
 - task: UsePythonVersion@0
@@ -28,27 +32,32 @@ steps:
 - bash: pip install --upgrade 'virtualenv<20' setuptools tox
   displayName: Install Tox
 
-- script: tox -e py -- -m unit -n auto --junit-xml=junit/unit-test.xml
-  env:
-    TEMP: "R:\\Temp"
-  displayName: Tox run unit tests
+# Lots of fun logic, for... running the correct tests.
+- ${{ if in(parameters.testGroup, 'unit', 'one') }}:
+  - script: tox -e py -- -m unit -n auto --junit-xml=junit/unit-test.xml
+    env:
+      TEMP: "R:\\Temp"
+    displayName: Run unit tests
 
-- ${{ if eq(parameters.runIntegrationTests, 'true') }}:
+- ${{ if in(parameters.testGroup, 'one', 'two') }}:
   - powershell: |
       # Fix Git SSL errors
-      pip install certifi tox
+      pip install certifi
       python -m certifi > cacert.txt
       $env:GIT_SSL_CAINFO = $(Get-Content cacert.txt)
 
       # Shorten paths to get under MAX_PATH or else integration tests will fail
       # https://bugs.python.org/issue18199
       $env:TEMP = "R:\Temp"
 
-      tox -e py -- $env:USE_VENV_ARG -m integration -n auto --duration=5 --junit-xml=junit/integration-test.xml
-    displayName: Tox run integration tests
+      pip install tox
+      tox -e py -- -k $env:SELECTOR -m integration -n auto --duration=5 --junit-xml=junit/integration-test.xml
     env:
-      ${{ if eq(parameters.useVenv, 'true') }}:
-        USE_VENV_ARG: "--use-venv"
+      ${{ if eq(parameters.testGroup, 'one') }}:
+        SELECTOR: "not test_install"
+      ${{ if eq(parameters.testGroup, 'two') }}:
+        SELECTOR: "test_install"
+    displayName: Run integration tests
 
 - task: PublishTestResults@2
   displayName: Publish Test Results

.azure-pipelines/windows.yml

@@ -2,7 +2,3 @@ jobs:
 - template: jobs/test-windows.yml
   parameters:
     vmImage: vs2017-win2016
-
-- template: jobs/package.yml
-  parameters:
-    vmImage: vs2017-win2016