Make building pipeline less confusing

.github/workflows/build-pre-release.yml

@@ -10,19 +10,12 @@ on:
 
       target:
         type: string
-        description: "target os to build for: linux,macos,windows"
-        default: "linux,macos,windows"
-
-      cibw-skip:
-        type: string
-        description: 'CIBUILDWHEEL builds pattern to skip, goes to CIBW_SKIP env'
-        required: false
-        default: 'cp36* cp37* pp*i686 *musllinux*'
+        description: "target os to build for: linux,macos-x86,macos-arm,windows,linux-aarch64"
+        default: "linux,macos-x86,macos-arm,windows,linux-aarch64"
 
 jobs:
   build-and-publish:
     uses: ./.github/workflows/lib-build-and-push.yml
     with:
       python-version: ${{ inputs.python-version }}
       target: ${{ inputs.target }}
-      cibw-skip: ${{ inputs.cibw-skip }}

.github/workflows/build-push.yml

@@ -26,4 +26,3 @@ jobs:
     with:
       upload: ${{ github.event_name == 'push' && endsWith(github.event.ref, 'scylla') }}
       python-version: '3.13'
-      target: linux,macos,windows

.github/workflows/lib-build-and-push.yml

@@ -6,40 +6,20 @@ on:
       upload:
         description: 'Upload to PyPI'
         type: boolean
+        required: false
         default: false
 
       python-version:
-        description: 'Python version to run against'
-        required: true
+        description: 'Python version to run on'
         type: string
-        default: "1.13"
+        required: true
+        default: "3.13"
 
       target:
-        type: string
         description: "target os to build for: linux,macos,windows"
-        default: "linux,macos,windows"
-
-      cibw-skip:
         type: string
-        description: 'CIBUILDWHEEL builds pattern to skip, goes to CIBW_SKIP env'
         required: false
-        default: 'cp36* cp37* pp*i686 *musllinux*'
-
-env:
- CIBW_TEST_COMMAND_LINUX: > 
-    pytest {project}/tests/unit && 
-    EVENT_LOOP_MANAGER=gevent pytest {project}/tests/unit/io/test_geventreactor.py
-
- CIBW_TEST_COMMAND_MACOS: "pytest {project}/tests/unit -k 'not (test_multi_timer_validation or test_empty_connections or test_timer_cancellation)' "
- CIBW_TEST_COMMAND_WINDOWS: "pytest {project}/tests/unit  -k \"not (test_deserialize_date_range_year or test_datetype or test_libevreactor)\" "
- CIBW_BEFORE_TEST: "pip install -r {project}/test-requirements.txt"
- CIBW_BEFORE_BUILD_LINUX: "rm -rf ~/.pyxbld && rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux && yum install -y libffi-devel libev libev-devel openssl openssl-devel"
- CIBW_ENVIRONMENT: "CASS_DRIVER_BUILD_CONCURRENCY=2 CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST=yes CFLAGS='-g0 -O3'"
- CIBW_SKIP: cp36* cp37* pp*i686 *musllinux*
- CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
- CIBW_MANYLINUX_PYPY_X86_64_IMAGE: manylinux_2_28
- CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
- CIBW_MANYLINUX_PYPY_AARCH64_IMAGE: manylinux_2_28
+        default: "linux,macos-x86,macos-arm,windows,linux-aarch64"
 
 jobs:
   prepare-matrix:
@@ -53,19 +33,27 @@ jobs:
         run: |
           echo -n "[" > /tmp/matrix.json
           was_added=""
-          for os in $(echo "${{ inputs.target }}" | tr -d " " | tr "," "\n")
+          for target in $(echo "${{ inputs.target }}" | tr -d " " | tr "," "\n")
           do
-           if [[ "${os}" == "linux" ]]; then
+           if [[ "${target}" == "linux" ]]; then
+             [ -n "$was_added" ] && echo -n ","  >> /tmp/matrix.json
+             echo -n '{"os":"ubuntu-20.04", "target": "linux"}' >> /tmp/matrix.json
+             was_added=1
+           elif [[ "${target}" == "linux-aarch64" ]]; then
              [ -n "$was_added" ] && echo -n ","  >> /tmp/matrix.json
-             echo -n '{"os":"ubuntu-20.04","platform":"x86_64"},{"os":"ubuntu-20.04","platform":"PyPy"}' >> /tmp/matrix.json
+             echo -n  '{"os":"ubuntu-20.04", "target": "linux-aarch64"}' >> /tmp/matrix.json
              was_added=1
-           elif [[ "${os}" == "windows" ]]; then
+           elif [[ "${target}" == "windows" ]]; then
              [ -n "$was_added" ] && echo -n ","  >> /tmp/matrix.json
-             echo -n  '{"os":"windows-latest","platform":"win64"},{"os":"windows-latest","platform":"PyPy"}' >> /tmp/matrix.json
+             echo -n  '{"os":"windows-2022", "target": "windows"}' >> /tmp/matrix.json
              was_added=1
-           elif [[ "${os}" == "macos" ]]; then
+           elif [[ "${target}" == "macos-x86" ]]; then
              [ -n "$was_added" ] && echo -n ","  >> /tmp/matrix.json
-             echo -n  '{"os":"macos-latest","platform":"all"},{"os":"macos-13","platform":"all"},{"os":"macos-latest","platform":"PyPy"}' >> /tmp/matrix.json
+             echo -n  '{"os":"macos-13", "target": "macos-x86"}' >> /tmp/matrix.json
+             was_added=1
+           elif [[ "${target}" == "macos-arm" ]]; then
+             [ -n "$was_added" ] && echo -n ","  >> /tmp/matrix.json
+             echo -n  '{"os":"macos-14", "target": "macos-arm"}' >> /tmp/matrix.json
              was_added=1
            fi
           done
@@ -74,32 +62,13 @@ jobs:
           echo "matrix=$(cat /tmp/matrix.json)" >> $GITHUB_OUTPUT
 
   build-wheels:
-    name: Build wheels ${{ matrix.os }} (${{ matrix.platform }})
+    name: Build wheels for ${{ matrix.target }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    needs: prepare-matrix
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-24.04
-            platform: x86_64
-
-          - os: ubuntu-24.04
-            platform: PyPy
-
-          - os: windows-2022
-            platform: win64
-
-          - os: windows-latest
-            platform: PyPy
-
-          - os: macos-14
-            platform: all
-
-          - os: macos-13
-            platform: all
-
-          - os: macos-latest
-            platform: PyPy
+        include: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }}
 
     steps:
       - uses: actions/checkout@v4
@@ -122,13 +91,20 @@ jobs:
       - name: Install OpenSSL for Windows
         if: runner.os == 'Windows'
         run: |
-          choco install openssl --version=3.4.1 -f -y
+          choco install openssl --version=3.4.1 -f -y --no-progress
+
+      - name: Set up QEMU
+        if: matrix.target == 'linux-aarch64'
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: all
+          image: 'docker.io/tonistiigi/binfmt:desktop-v8.1.5'
 
       - name: Install Conan
         if: runner.os == 'Windows'
         uses: turtlebrowser/get-conan@main
 
-      - name: configure libev for Windows
+      - name: Configure libev for Windows
         if: runner.os == 'Windows'
         run: |
           conan profile detect
@@ -139,57 +115,31 @@ jobs:
         run: |
           brew install libev
 
-      - name: Overwrite for Linux 64
-        if: runner.os == 'Linux' && matrix.platform == 'x86_64'
-        run: |
-          echo "CIBW_BUILD=cp3*_x86_64" >> $GITHUB_ENV
-
-      - name: Overwrite for Linux PyPy
-        if: runner.os == 'Linux' && matrix.platform == 'PyPy'
+      - name: Overwrite for MacOS
+        if: runner.os == 'MacOS'
         run: |
-          echo "CIBW_BUILD=pp*" >> $GITHUB_ENV
-          echo "CIBW_TEST_COMMAND_LINUX=" >> $GITHUB_ENV
-
-      - name: Overwrite for Windows 64
-        if: runner.os == 'Windows' && matrix.platform == 'win64'
-        run: |
-          echo "CIBW_BUILD=cp*win_amd64" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
-          echo "CIBW_ENVIRONMENT_WINDOWS= CC=clang CXX=clang++" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
-
-      - name: Overwrite for Windows PyPY
-        if: runner.os == 'Windows' && matrix.platform == 'PyPy'
-        run: |
-          echo "CIBW_BUILD=pp*" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
-          echo "CIBW_TEST_COMMAND_WINDOWS=" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
-
-      - name: Overwrite for MacOs
-        if: runner.os == 'MacOs' && matrix.platform == 'all'
-        run: |
-          echo "CIBW_BUILD=cp39* cp310* cp311* cp312* cp313*" >> $GITHUB_ENV
-          echo "CIBW_BEFORE_TEST_MACOS=pip install -r {project}/test-requirements.txt pytest" >> $GITHUB_ENV
+          ##### Set MACOSX_DEPLOYMENT_TARGET
           if [ "${{ matrix.os }}" == "macos-13" ]; then
             echo "MACOSX_DEPLOYMENT_TARGET=13.0" >> $GITHUB_ENV;
             echo "Enforcing target deployment for 13.0"
           elif [ "${{ matrix.os }}" == "macos-14" ]; then
             echo "MACOSX_DEPLOYMENT_TARGET=14.0" >> $GITHUB_ENV;
             echo "Enforcing target deployment for 14.0"
-          else
-            echo "Unknown macos version" && false;
           fi
-      - name: Overwrite for MacOs PyPy
-        if: runner.os == 'MacOs' && matrix.platform == 'PyPy'
-        run: |
-          echo "CIBW_BUILD=pp*" >> $GITHUB_ENV
-          echo "CIBW_BEFORE_TEST_MACOS=pip install -r {project}/test-requirements.txt pytest" >> $GITHUB_ENV
-          echo "CIBW_TEST_COMMAND_MACOS=" >> $GITHUB_ENV
 
       - name: Build wheels
+        if: matrix.target != 'linux-aarch64'
         run: |
           python3 -m cibuildwheel --output-dir wheelhouse
 
+      - name: Build wheels for linux aarch64
+        if: matrix.target == 'linux-aarch64'
+        run: |
+          CIBW_BUILD='cp3*' python -m cibuildwheel --archs aarch64 --output-dir wheelhouse
+
       - uses: actions/upload-artifact@v4
         with:
-          name: wheels-${{ matrix.os }}-${{ matrix.platform }}
+          name: wheels-${{ matrix.target }}-${{ matrix.os }}
           path: ./wheelhouse/*.whl
 
   build-sdist:
@@ -211,56 +161,13 @@ jobs:
           name: source-dist
           path: dist/*.tar.gz
 
-  build-wheels-extra-arch:
-    # The host should always be linux
-    runs-on: ubuntu-24.04
-    name: Build extra arch ${{ matrix.archs }} wheels
-    strategy:
-      fail-fast: false
-      matrix:
-        archs: [ aarch64,]  # ppc64le ]
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        id: qemu
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: all
-          image: 'docker.io/tonistiigi/binfmt:desktop-v8.1.5'
-        if: runner.os == 'Linux'
-
-      - uses: actions/setup-python@v5
-        name: Install Python
-        with:
-          python-version: ${{ inputs.python-version }}
-
-      - name: Install cibuildwheel
-        run: |
-          python -m pip install cibuildwheel==2.22.0
-
-      - name: Build wheels
-        env:
-          CIBW_BUILD: "cp39* cp310* cp311* cp312* cp313*" # limit to specific version since it take much more time than jobs limit
-        run: |
-          python -m cibuildwheel --archs  ${{ matrix.archs }} --output-dir wheelhouse
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: wheels-${{ matrix.archs }}
-          path: ./wheelhouse/*.whl
-
   upload_pypi:
     if: inputs.upload
-    needs: [build-wheels, build-wheels-extra-arch, build-sdist]
+    needs: [build-wheels, build-sdist]
     runs-on: ubuntu-24.04
     permissions:
       id-token: write
 
-    # upload to PyPI on every tag starting with 'v'
-    # alternatively, to publish when a GitHub Release is created, use the following rule:
-    # if: github.event_name == 'release' && github.event.action == 'published'
     steps:
       - uses: actions/download-artifact@v4
         with:

pyproject.toml

@@ -78,4 +78,50 @@ requires = [
     "Cython",
 ]
 
-build-backend = "setuptools.build_meta"
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+version_file = "cassandra/_version.py"
+tag_regex = '(?P<version>\d*?\.\d*?\.\d*?)-scylla'
+
+#### CI BUILDWHEEL CONFIG ####
+
+[tool.cibuildwheel]
+build-frontend = "build[uv]"
+environment = { CASS_DRIVER_BUILD_CONCURRENCY = "2", CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST = "yes", CFLAGS = "-g0 -O3" }
+skip = ["cp2*", "cp36*", "pp36*", "cp37*", "pp37*", "*i686", "*musllinux*"]
+build = ["cp3*", "pp3*"]
+
+before-test = "pip install -r {project}/test-requirements.txt"
+
+manylinux-x86_64-image = "manylinux_2_28"
+manylinux-aarch64-image = "manylinux_2_28"
+manylinux-pypy_x86_64-image = "manylinux_2_28"
+manylinux-pypy_aarch64-image = "manylinux_2_28"
+
+[tool.cibuildwheel.linux]
+
+before-build = "rm -rf ~/.pyxbld && rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux && yum install -y libffi-devel libev libev-devel openssl openssl-devel"
+test-command = [
+  "pytest {package}/tests/unit",
+  "EVENT_LOOP_MANAGER=gevent pytest {package}/tests/unit/io/test_geventreactor.py",
+]
+
+[tool.cibuildwheel.macos]
+build-frontend = "build"
+test-command = [
+    "pytest {project}/tests/unit -k 'not (test_multi_timer_validation or test_empty_connections or test_timer_cancellation)'"
+]
+
+[tool.cibuildwheel.windows]
+build-frontend = "build"
+test-command = [
+    "pytest {project}/tests/unit  -k \"not (test_deserialize_date_range_year or test_datetype or test_libevreactor)\""
+]
+
+# TODO: set CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST to yes when https://github.com/scylladb/python-driver/issues/429 is fixed
+environment = { CASS_DRIVER_BUILD_CONCURRENCY = "2", CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST = "no"}
+
+[[tool.cibuildwheel.overrides]]
+select = "pp*"
+test-command = []

setup.py

@@ -14,24 +14,22 @@
 
 from __future__ import print_function
 
-import fnmatch
 import os
 import shutil
 import sys
 import json
 import warnings
 from pathlib import Path
-from sysconfig import get_config_vars
 
-if __name__ == '__main__' and sys.argv[1] == "gevent_nosetests":
-    print("Running gevent tests")
-    from gevent.monkey import patch_all
-    patch_all()
-
-if __name__ == '__main__' and sys.argv[1] == "eventlet_nosetests":
-    print("Running eventlet tests")
-    from eventlet import monkey_patch
-    monkey_patch()
+if __name__ == '__main__' and len(sys.argv) > 1:
+    if sys.argv[1] == "gevent_nosetests":
+        print("Running gevent tests")
+        from gevent.monkey import patch_all
+        patch_all()
+    elif sys.argv[1] == "eventlet_nosetests":
+        print("Running eventlet tests")
+        from eventlet import monkey_patch
+        monkey_patch()
 
 from setuptools.command.build_ext import build_ext
 from setuptools import Extension, Command, setup
@@ -57,7 +55,7 @@ class eventlet_nosetests(nosetests):
         description = "run nosetests with eventlet monkey patching"
 
 has_cqlengine = False
-if __name__ == '__main__' and sys.argv[1] == "install":
+if __name__ == '__main__' and len(sys.argv) > 1 and sys.argv[1] == "install":
     try:
         import cqlengine
         has_cqlengine = True
@@ -453,9 +451,7 @@ def run_setup(extensions):
         else:
             sys.stderr.write("Bypassing Cython setup requirement\n")
 
-    setup(
-        tests_require=['nose', 'mock>=2.0.0', 'PyYAML', 'pytz', 'sure'],
-        **kw)
+    setup(**kw)
 
 
 run_setup(None)