gh-263: Run Python 3.12 task as the experimental task (#274)

Co-authored-by: Hugo van Kemenade <[email protected]>

Author: corona10

.github/workflows/main.yml

@@ -14,25 +14,37 @@ jobs:
 
   test:
     runs-on: ${{ matrix.os }}
+    permissions:
+      pull-requests: write
     name: ${{ matrix.os }} - ${{ matrix.python }}
     strategy:
       fail-fast: false
       matrix:
         # Test all supported versions on Ubuntu:
         os: [ubuntu-latest]
-        python: ["3.7", "3.8", "3.9", "3.10", 3.11-dev, 3.12-dev]
+        python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        experimental: [false]
         include:
+          # As the experimental task for the dev version.
+          - os: ubuntu-latest
+            python: "3.12-dev"
+            experimental: true
           # Also test PyPy, macOS, and Windows:
           - os: ubuntu-latest
             python: pypy-3.9
+            experimental: false
           - os: ubuntu-latest
             python: pypy-3.8
+            experimental: false
           - os: ubuntu-latest
             python: pypy-3.7
+            experimental: false
           - os: macos-latest
             python: "3.10"
+            experimental: false
           - os: windows-latest
             python: "3.10"
+            experimental: false
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python }}
@@ -54,4 +66,18 @@ jobs:
     - name: Display Python version
       run: python -c "import sys; print(sys.version)"
     - name: Run Tests
+      id: pyperformance
       run: python -u -m pyperformance.tests
+      continue-on-error: ${{ matrix.experimental }}
+    - name: Notify result for experimental tasks (Failure)
+      uses: marocchino/sticky-pull-request-comment@v2
+      if: ${{ steps.pyperformance.outcome != 'success' && matrix.experimental }}
+      with:
+        message: |
+          ❌: ${{ matrix.os }} - ${{ matrix.python }} has failed, but allowed as the experimental task.
+    - name: Notify result for experimental tasks (Success)
+      uses: marocchino/sticky-pull-request-comment@v2
+      if: ${{ steps.pyperformance.outcome == 'success' && matrix.experimental }}
+      with:
+        message: |
+          ✅: ${{ matrix.os }} - ${{ matrix.python }} has passed, now we can disable the experimental flag.