From 8518cc30b29464c4ed7f5169fc4d9e5f910332cd Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:27:19 +0530 Subject: [PATCH 1/6] Revamp wheels publishing + add Anaconda deployment --- .github/workflows/wheel_tests_and_release.yml | 132 +++++++++++------- 1 file changed, 79 insertions(+), 53 deletions(-) diff --git a/.github/workflows/wheel_tests_and_release.yml b/.github/workflows/wheel_tests_and_release.yml index 11813904e..2855eaaa7 100644 --- a/.github/workflows/wheel_tests_and_release.yml +++ b/.github/workflows/wheel_tests_and_release.yml @@ -4,6 +4,18 @@ on: tags: - "v*" - "buildwheels*" + # Make it possible to upload wheels manually if needed + workflow_dispatch: + inputs: + push_wheels: + description: > + 'Push wheels to Anaconda if "true". Default is "false". Warning: this will overwrite existing wheels.' + required: false + default: "false" + # Upload wheels to anaconda.org on a schedule + schedule: + # Run at 0300 hours on days 3 and 17 of the month + - cron: "0 3 3,17 * *" env: CIBW_BUILD_VERBOSITY: 2 # CIBW_BEFORE_BUILD: pip install cython @@ -23,28 +35,26 @@ jobs: cibw_manylinux: [manylinux2014] cibw_arch: ["x86_64"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 - uses: actions/setup-python@v4 name: Install Python with: python-version: "3.9" - - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel - name: Build the wheel - run: | - python -m cibuildwheel --output-dir dist + uses: pypa/cibuildwheel@8d945475ac4b1aac4ae08b2fd27db9917158b6ce # 2.17.0 + with: + output-dir: dist env: CIBW_BUILD: ${{ matrix.cibw_python }} CIBW_ARCHS_LINUX: ${{ matrix.cibw_arch }} CIBW_SKIP: "*-musllinux_*" CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.cibw_manylinux }} CIBW_MANYLINUX_I686_IMAGE: "manylinux2010" - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: - name: wheels + name: linux_x86_64_wheels path: ./dist/*.whl build_linux_aarch64_wheels: @@ -57,7 +67,7 @@ jobs: cibw_python: ["cp39-*", "cp310-*", "cp311-*", "cp312-*"] cibw_manylinux: [manylinux2014] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 - uses: actions/setup-python@v4 @@ -68,20 +78,18 @@ jobs: uses: docker/setup-qemu-action@v3 with: platforms: arm64 - - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel - name: Build the wheel - run: | - python -m cibuildwheel --output-dir dist + uses: pypa/cibuildwheel@8d945475ac4b1aac4ae08b2fd27db9917158b6ce # 2.17.0 + with: + output-dir: dist env: CIBW_BUILD: ${{ matrix.cibw_python }} CIBW_ARCHS_LINUX: aarch64 CIBW_SKIP: "*-musllinux_*" CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.cibw_manylinux }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: - name: wheels + name: linux_aarch64_wheels path: ./dist/*.whl build_macos_wheels: @@ -94,7 +102,7 @@ jobs: cibw_python: ["cp39-*", "cp310-*", "cp311-*", "cp312-*"] cibw_arch: ["x86_64", "arm64"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 @@ -103,32 +111,17 @@ jobs: with: python-version: "3.9" - - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel - - - name: Build wheels for CPython (MacOS) - run: | - # We need to set both MACOS_DEPLOYMENT_TARGET and MACOSX_DEPLOYMENT_TARGET - # until there is a new release with this commit: - # https://github.com/mesonbuild/meson-python/pull/309 (should be in 0.13.0) - if [[ "$CIBW_ARCHS_MACOS" == arm64 ]]; then - export MACOSX_DEPLOYMENT_TARGET=11.0 - export MACOS_DEPLOYMENT_TARGET=11.0 - else - export MACOSX_DEPLOYMENT_TARGET=10.13 - export MACOS_DEPLOYMENT_TARGET=10.13 - fi - echo MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} - - python -m cibuildwheel --output-dir dist + - name: Build wheels for CPython (macOS) + uses: pypa/cibuildwheel@8d945475ac4b1aac4ae08b2fd27db9917158b6ce # 2.17.0 + with: + output-dir: dist env: CIBW_BUILD: ${{ matrix.cibw_python }} CIBW_ARCHS_MACOS: ${{ matrix.cibw_arch }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: - name: wheels + name: macos_wheels path: ./dist/*.whl build_windows_wheels: @@ -141,7 +134,7 @@ jobs: cibw_arch: ["AMD64", "x86"] cibw_python: ["cp39-*", "cp310-*", "cp311-*", "cp312-*"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 @@ -150,10 +143,6 @@ jobs: with: python-version: "3.9" - - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel - - name: Setup MSVC (32-bit) if: matrix.cibw_arch == 'x86' uses: bus1/cabuild/action/msdevshell@e22aba57d6e74891d059d66501b6b5aed8123c4d # v1 @@ -167,19 +156,20 @@ jobs: architecture: x64 - name: Build Windows wheels for CPython - run: | - python -m cibuildwheel --output-dir dist + uses: pypa/cibuildwheel@8d945475ac4b1aac4ae08b2fd27db9917158b6ce # 2.17.0 + with: + output-dir: dist env: CIBW_BUILD: ${{ matrix.cibw_python }} CIBW_ARCHS_WINDOWS: ${{ matrix.cibw_arch }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: - name: wheels + name: windows_wheels path: ./dist/*.whl - deploy: - name: Release + deploy_pypi: + name: Release (PyPI) needs: [ build_linux_x86_64_wheels, @@ -190,7 +180,7 @@ jobs: if: github.repository_owner == 'PyWavelets' && startsWith(github.ref, 'refs/tags/v') && always() runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 @@ -204,11 +194,13 @@ jobs: python -m pip install --upgrade pip pip install twine pip install cython numpy build - - uses: actions/download-artifact@v3 + + - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 id: download with: - name: wheels + pattern: "*_wheels" path: ./dist + merge-multiple: true - name: Publish the source distribution on PyPI run: | @@ -224,7 +216,41 @@ jobs: TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }} - name: Github release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@d99959edae48b5ffffd7b00da66dcdb0a33a52ee # v2.0.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} + + deploy_anaconda: + name: Release (Anaconda) + needs: + [ + build_linux_x86_64_wheels, + build_linux_aarch64_wheels, + build_macos_wheels, + build_windows_wheels, + ] + # Only run on the master branch, on schedule, or when triggered manually + if: >- + github.repository_owner == 'PyWavelets' && + (github.event_name == 'push' && github.ref == 'refs/heads/master') || + (github.event_name == 'workflow_dispatch' && github.event.inputs.push_wheels == 'true') || + (github.event_name == 'schedule') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + fetch-depth: 0 + + - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 + id: download + with: + pattern: "*_wheels" + path: dist/ + merge-multiple: true + + - name: Push to Anaconda PyPI index + uses: scientific-python/upload-nightly-action@b67d7fcc0396e1128a474d1ab2b48aa94680f9fc # v0.5.0 + with: + artifacts_path: dist/ + anaconda_nightly_upload_token: ${{ secrets.ANACONDA_ORG_UPLOAD_TOKEN }} From 984429f65d75a3dca6a30c2df2b7c2679e792aab Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:40:21 +0530 Subject: [PATCH 2/6] Refine workflow run contexts and conditions --- .github/workflows/wheel_tests_and_release.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel_tests_and_release.yml b/.github/workflows/wheel_tests_and_release.yml index a07d62162..b97be1fc0 100644 --- a/.github/workflows/wheel_tests_and_release.yml +++ b/.github/workflows/wheel_tests_and_release.yml @@ -4,6 +4,10 @@ on: tags: - "v*" - "buildwheels*" + branches: + # Runs on every merge to master to upload .dev0 wheels to anaconda.org + - master + - v1.** # Make it possible to upload wheels manually if needed workflow_dispatch: inputs: @@ -177,7 +181,8 @@ jobs: build_macos_wheels, build_windows_wheels, ] - if: github.repository_owner == 'PyWavelets' && startsWith(github.ref, 'refs/tags/v') && always() + # Run only on tags pushed to the repository + if: github.repository_owner == 'PyWavelets' && startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest steps: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 @@ -230,7 +235,7 @@ jobs: build_macos_wheels, build_windows_wheels, ] - # Only run on the master branch, on schedule, or when triggered manually + # Run only on pushes to the master branch, on schedule, or when triggered manually if: >- github.repository_owner == 'PyWavelets' && (github.event_name == 'push' && github.ref == 'refs/heads/master') || From 89c9ea1273bf0d61e9e4ca90605784d40bb3178f Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 12 Mar 2024 19:38:01 +0530 Subject: [PATCH 3/6] Allow PyPI publishing job to be triggered manually --- .github/workflows/wheel_tests_and_release.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel_tests_and_release.yml b/.github/workflows/wheel_tests_and_release.yml index b97be1fc0..611b35e1b 100644 --- a/.github/workflows/wheel_tests_and_release.yml +++ b/.github/workflows/wheel_tests_and_release.yml @@ -181,8 +181,10 @@ jobs: build_macos_wheels, build_windows_wheels, ] - # Run only on tags pushed to the repository - if: github.repository_owner == 'PyWavelets' && startsWith(github.ref, 'refs/tags/v') + # Run only on tags pushed to the repository or when triggered manually + if: >- + github.repository == 'PyWavelets/pywt' && (startsWith(github.ref, 'refs/tags/v') || + (github.event_name == 'workflow_dispatch' && github.event.inputs.push_wheels == 'true')) runs-on: ubuntu-latest steps: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 @@ -237,7 +239,7 @@ jobs: ] # Run only on pushes to the master branch, on schedule, or when triggered manually if: >- - github.repository_owner == 'PyWavelets' && + github.repository == 'PyWavelets/pywt' && (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'workflow_dispatch' && github.event.inputs.push_wheels == 'true') || (github.event_name == 'schedule') From e8e4d86f42aab0fe1a011f8496b9cb53e1a74033 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 12 Mar 2024 21:20:20 +0530 Subject: [PATCH 4/6] Do not allow triggering PyPI releases manually --- .github/workflows/wheel_tests_and_release.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wheel_tests_and_release.yml b/.github/workflows/wheel_tests_and_release.yml index 611b35e1b..6108c3a5b 100644 --- a/.github/workflows/wheel_tests_and_release.yml +++ b/.github/workflows/wheel_tests_and_release.yml @@ -181,10 +181,8 @@ jobs: build_macos_wheels, build_windows_wheels, ] - # Run only on tags pushed to the repository or when triggered manually - if: >- - github.repository == 'PyWavelets/pywt' && (startsWith(github.ref, 'refs/tags/v') || - (github.event_name == 'workflow_dispatch' && github.event.inputs.push_wheels == 'true')) + # Run only on tags pushed to the repository + if: github.repository == 'PyWavelets/pywt' && startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest steps: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 From 90ec5e4cbf8d4bbf0aaf8c224df0ef2abe714f3d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 12 Mar 2024 22:35:59 +0530 Subject: [PATCH 5/6] Ensure unique artifact names, via identifiers --- .github/workflows/wheel_tests_and_release.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/wheel_tests_and_release.yml b/.github/workflows/wheel_tests_and_release.yml index 6108c3a5b..374eb549a 100644 --- a/.github/workflows/wheel_tests_and_release.yml +++ b/.github/workflows/wheel_tests_and_release.yml @@ -58,8 +58,9 @@ jobs: CIBW_MANYLINUX_I686_IMAGE: "manylinux2010" - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: - name: linux_x86_64_wheels + name: wheels_linux_${{ matrix.cibw_arch }}_${{ matrix.cibw_python }}_manylinux2014 path: ./dist/*.whl + if-no-files-found: error build_linux_aarch64_wheels: name: Build python ${{ matrix.cibw_python }} aarch64 wheels on ${{ matrix.os }} @@ -70,6 +71,7 @@ jobs: os: [ubuntu-latest] cibw_python: ["cp39-*", "cp310-*", "cp311-*", "cp312-*"] cibw_manylinux: [manylinux2014] + cibw_arch: ["aarch64"] steps: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: @@ -88,12 +90,12 @@ jobs: output-dir: dist env: CIBW_BUILD: ${{ matrix.cibw_python }} - CIBW_ARCHS_LINUX: aarch64 + CIBW_ARCHS_LINUX: ${{ matrix.cibw_arch }} CIBW_SKIP: "*-musllinux_*" CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.cibw_manylinux }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: - name: linux_aarch64_wheels + name: wheels_linux_${{ matrix.cibw_arch }}_${{ matrix.cibw_python }}_manylinux2014 path: ./dist/*.whl build_macos_wheels: @@ -125,8 +127,9 @@ jobs: - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: - name: macos_wheels + name: wheels_macos_${{ matrix.cibw_arch }}_${{ matrix.cibw_python }} path: ./dist/*.whl + if-no-files-found: error build_windows_wheels: name: Build ${{ matrix.cibw_python }} ${{ matrix.cibw_arch }} wheels on ${{ matrix.os }} @@ -169,8 +172,9 @@ jobs: - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: - name: windows_wheels + name: wheels_windows_${{ matrix.cibw_arch }}_${{ matrix.cibw_python }} path: ./dist/*.whl + if-no-files-found: error deploy_pypi: name: Release (PyPI) @@ -203,7 +207,7 @@ jobs: - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 id: download with: - pattern: "*_wheels" + pattern: "wheels_*" path: ./dist merge-multiple: true From 83ba782221bb115111258246b02c994d5041710b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 12 Mar 2024 22:43:45 +0530 Subject: [PATCH 6/6] Rename matrix `cibw_python` matrix variable --- .github/workflows/wheel_tests_and_release.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/wheel_tests_and_release.yml b/.github/workflows/wheel_tests_and_release.yml index 374eb549a..6bf07dc6c 100644 --- a/.github/workflows/wheel_tests_and_release.yml +++ b/.github/workflows/wheel_tests_and_release.yml @@ -35,7 +35,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - cibw_python: ["cp39-*", "cp310-*", "cp311-*", "cp312-*"] + cibw_python: ["cp39", "cp310", "cp311", "cp312"] cibw_manylinux: [manylinux2014] cibw_arch: ["x86_64"] steps: @@ -51,7 +51,7 @@ jobs: with: output-dir: dist env: - CIBW_BUILD: ${{ matrix.cibw_python }} + CIBW_BUILD: ${{ matrix.cibw_python }}-* CIBW_ARCHS_LINUX: ${{ matrix.cibw_arch }} CIBW_SKIP: "*-musllinux_*" CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.cibw_manylinux }} @@ -69,7 +69,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - cibw_python: ["cp39-*", "cp310-*", "cp311-*", "cp312-*"] + cibw_python: ["cp39", "cp310", "cp311", "cp312"] cibw_manylinux: [manylinux2014] cibw_arch: ["aarch64"] steps: @@ -89,7 +89,7 @@ jobs: with: output-dir: dist env: - CIBW_BUILD: ${{ matrix.cibw_python }} + CIBW_BUILD: ${{ matrix.cibw_python }}-* CIBW_ARCHS_LINUX: ${{ matrix.cibw_arch }} CIBW_SKIP: "*-musllinux_*" CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.cibw_manylinux }} @@ -105,7 +105,7 @@ jobs: fail-fast: false matrix: os: [macos-latest] - cibw_python: ["cp39-*", "cp310-*", "cp311-*", "cp312-*"] + cibw_python: ["cp39", "cp310", "cp311", "cp312"] cibw_arch: ["x86_64", "arm64"] steps: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 @@ -122,7 +122,7 @@ jobs: with: output-dir: dist env: - CIBW_BUILD: ${{ matrix.cibw_python }} + CIBW_BUILD: ${{ matrix.cibw_python }}-* CIBW_ARCHS_MACOS: ${{ matrix.cibw_arch }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 @@ -139,7 +139,7 @@ jobs: matrix: os: [windows-latest] cibw_arch: ["AMD64", "x86"] - cibw_python: ["cp39-*", "cp310-*", "cp311-*", "cp312-*"] + cibw_python: ["cp39", "cp310", "cp311", "cp312"] steps: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: @@ -167,7 +167,7 @@ jobs: with: output-dir: dist env: - CIBW_BUILD: ${{ matrix.cibw_python }} + CIBW_BUILD: ${{ matrix.cibw_python }}-* CIBW_ARCHS_WINDOWS: ${{ matrix.cibw_arch }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1