chore(ci): introduce daily pre-releases

.github/workflows/build_changelog.yml

@@ -17,8 +17,8 @@ on:
 #    branches:
 #      - develop
   schedule:
-    # Note: run daily at 7am UTC time until upstream git-chlog uses stable sorting
-    - cron: "0 7 * * *"
+    # Note: run daily at 10am UTC time until upstream git-chlog uses stable sorting
+    - cron: "0 10 * * *"
 
 permissions:
   contents: read

.github/workflows/pre-release.yml

@@ -0,0 +1,275 @@
+name: Pre-Release
+
+# PRE-RELEASE PROCESS
+#
+# === Automated activities ===
+#
+# 1.  [Seal] Bump to release version and export source code with integrity hash
+# 2.  [Quality check] Restore sealed source code, run tests, linting, security and complexity base line
+# 3.  [Build] Restore sealed source code, create and export hashed build artifact for PyPi release (wheel, tarball)
+# 4.  [Provenance] Generates provenance for build, signs attestation with GitHub OIDC claims to confirm it came from this release pipeline, commit, org, repo, branch, hash, etc.
+# 5.  [Release] Restore built artifact, and publish package to PyPi prod repository
+# 6.  [PR to bump version] Restore sealed source code, and create a PR to update trunk with latest released project metadata
+
+# NOTE
+#
+# See MAINTAINERS.md "Releasing a new version" for release mechanisms
+#
+# Every job is isolated and starts a new fresh container.
+
+env:
+  RELEASE_COMMIT: ${{ github.sha }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      skip_code_quality:
+        description: "Skip tests, linting, and baseline. Only use if release fail for reasons beyond our control and you need a quick release."
+        default: false
+        type: boolean
+        required: false
+      skip_pypi:
+        description: "Skip publishing to PyPi. Used for testing release steps."
+        default: false
+        type: boolean
+        required: false
+  schedule:
+    # Note: run daily on weekdays at 8am UTC time
+    - cron: "0 8 * * 1-5"
+
+permissions:
+  contents: read
+
+jobs:
+
+  # This job bumps the package version to the pre-release version
+  # creates an integrity hash from the source code
+  # uploads the artifact with the integrity hash as the key name
+  # so subsequent jobs can restore from a trusted point in time to prevent tampering
+  seal:
+    # ignore forks
+    if: github.repository == 'aws-powertools/powertools-lambda-python'
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+        integrity_hash: ${{ steps.seal_source_code.outputs.integrity_hash }}
+        artifact_name: ${{ steps.seal_source_code.outputs.artifact_name }}
+        RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }}
+    steps:
+      # NOTE: Different from prod release, we need both poetry and source code available in earlier steps to bump and verify.
+
+      # We use a pinned version of Poetry to be certain it won't modify source code before we create a hash
+      - name: Install poetry
+        run: |
+          pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0
+          pipx inject poetry git+https://github.com/monim67/poetry-bumpversion@315fe3324a699fa12ec20e202eb7375d4327d1c4 # v0.3.1
+
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+        with:
+          ref: ${{ env.RELEASE_COMMIT }}
+
+      - name: Bump and export release version
+        id: release_version
+        run: |
+          RELEASE_VERSION="$(poetry version prerelease --short | head -n1 | tr -d '\n')"
+
+          echo "RELEASE_VERSION=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT"
+
+      - name: Verifies pre-release version semantics
+        # verify pre-release semantics before proceeding to avoid versioning pollution
+        # e.g., 2.40.0a1 and 2.40.0b2 are valid while 2.40.0 is not
+        # NOTE. we do it in a separate step to handle edge cases like
+        # `poetry` CLI uses immutable install, versioning behaviour could change even in a minor version (we had breaking changes before)
+        # a separate step allows us to pinpoint what happened (before/after)
+        run: |
+          if [[ ! "$RELEASE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+[a-b].*$ ]]; then
+            echo "Version $VERSION doesn't look like a pre-release version; aborting"
+            exit 1
+          fi
+        env:
+          RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION}}
+
+      - name: Seal and upload
+        id: seal_source_code
+        uses: ./.github/actions/seal
+        with:
+          artifact_name_prefix: "source"
+
+  # This job runs our automated test suite, complexity and security baselines
+  # it ensures previously merged have been tested as part of the pull request process
+  #
+  # NOTE
+  #
+  # we don't upload the artifact after testing to prevent any tampering of our source code dependencies
+  quality_check:
+    needs: seal
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      # NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev)
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+        with:
+          ref: ${{ env.RELEASE_COMMIT }}
+
+      - name: Restore sealed source code
+        uses: ./.github/actions/seal-restore
+        with:
+          integrity_hash: ${{ needs.seal.outputs.integrity_hash }}
+          artifact_name: ${{ needs.seal.outputs.artifact_name }}
+
+      - name: Debug cache restore
+        run: cat pyproject.toml
+
+      - name: Install poetry
+        run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0
+      - name: Set up Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: "3.12"
+          cache: "poetry"
+      - name: Install dependencies
+        run: make dev
+      - name: Run all tests, linting and baselines
+        run: make pr
+
+  # This job creates a release artifact (tar.gz, wheel)
+  # it checks out code from release commit for custom actions to work
+  # then restores the sealed source code (overwrites any potential tampering)
+  # it's done separately from release job to enforce least privilege.
+  # We export just the final build artifact for release
+  build:
+    runs-on: ubuntu-latest
+    needs: [quality_check, seal]
+    permissions:
+      contents: read
+    outputs:
+      integrity_hash: ${{ steps.seal_build.outputs.integrity_hash }}
+      artifact_name: ${{ steps.seal_build.outputs.artifact_name }}
+      attestation_hashes: ${{ steps.encoded_hash.outputs.attestation_hashes }}
+    steps:
+      # NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev)
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+        with:
+          ref: ${{ env.RELEASE_COMMIT }}
+
+      - name: Restore sealed source code
+        uses: ./.github/actions/seal-restore
+        with:
+          integrity_hash: ${{ needs.seal.outputs.integrity_hash }}
+          artifact_name: ${{ needs.seal.outputs.artifact_name }}
+
+      - name: Install poetry
+        run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0
+      - name: Set up Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: "3.12"
+          cache: "poetry"
+
+      - name: Build python package and wheel
+        run: poetry build
+
+      - name: Seal and upload
+        id: seal_build
+        uses: ./.github/actions/seal
+        with:
+          artifact_name_prefix: "build"
+          files: "dist/"
+
+      # NOTE: SLSA retraces our build to its artifact to ensure it wasn't tampered
+      # coupled with GitHub OIDC, SLSA can then confidently sign it came from this release pipeline+commit+branch+org+repo+actor+integrity hash
+      - name: Create attestation encoded hash for provenance
+        id: encoded_hash
+        working-directory: dist
+        run: echo "attestation_hashes=$(sha256sum ./* | base64 -w0)" >> "$GITHUB_OUTPUT"
+
+  # This job creates a provenance file that describes how our release was built (all steps)
+  # after it verifies our build is reproducible within the same pipeline
+  # it confirms that its own software and the CI build haven't been tampered with (Trust but verify)
+  # lastly, it creates and sign an attestation (multiple.intoto.jsonl) that confirms
+  # this build artifact came from this GitHub org, branch, actor, commit ID, inputs that triggered this pipeline, and matches its integrity hash
+  # NOTE: supply chain threats review (we protect against all of them now): https://slsa.dev/spec/v1.0/threats-overview
+  provenance:
+    needs: [seal, build]
+    permissions:
+      contents: write # nested job explicitly require despite upload assets being set to false
+      actions: read # To read the workflow path.
+      id-token: write # To sign the provenance.
+    # NOTE: provenance fails if we use action pinning... it's a Github limitation
+    # because SLSA needs to trace & attest it came from a given branch; pinning doesn't expose that information
+    # https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/generic/README.md#referencing-the-slsa-generator
+    uses: slsa-framework/slsa-github-generator/.github/workflows/[email protected]
+    with:
+      base64-subjects: ${{ needs.build.outputs.attestation_hashes }}
+      upload-assets: false  # we upload its attestation in create_tag job, otherwise it creates a new release
+
+  # This job uses release artifact to publish to PyPi
+  # it exchanges JWT tokens with GitHub to obtain PyPi credentials
+  # since it's already registered as a Trusted Publisher.
+  # It uses the sealed build artifact (.whl, .tar.gz) to release it
+  release:
+    needs: [build, seal, provenance]
+    environment: pre-release
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write # OIDC for PyPi Trusted Publisher feature
+    env:
+      RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }}
+    steps:
+      # NOTE: we need actions/checkout in order to use our local actions (e.g., ./.github/actions)
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+        with:
+          ref: ${{ env.RELEASE_COMMIT }}
+
+      - name: Restore sealed source code
+        uses: ./.github/actions/seal-restore
+        with:
+          integrity_hash: ${{ needs.build.outputs.integrity_hash }}
+          artifact_name: ${{ needs.build.outputs.artifact_name }}
+
+      - name: Upload to PyPi prod
+        if: ${{ !inputs.skip_pypi }}
+        uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
+
+  # Creates a PR with the latest version we've just released
+  # since our trunk is protected against any direct pushes from automation
+  bump_version:
+    needs: [release, seal, provenance]
+    permissions:
+      contents: write  # create-pr action creates a temporary branch
+      pull-requests: write # create-pr action creates a PR using the temporary branch
+    runs-on: ubuntu-latest
+    steps:
+      # NOTE: we need actions/checkout to authenticate and configure git first
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+        with:
+          ref: ${{ env.RELEASE_COMMIT }}
+
+      - name: Restore sealed source code
+        uses: ./.github/actions/seal-restore
+        with:
+          integrity_hash: ${{ needs.seal.outputs.integrity_hash }}
+          artifact_name: ${{ needs.seal.outputs.artifact_name }}
+
+      - name: Download provenance
+        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+        with:
+          name: ${{needs.provenance.outputs.provenance-name}}
+
+      - name: Update provenance
+        run: mkdir -p "${PROVENANCE_DIR}" && mv "${PROVENANCE_FILE}" "${PROVENANCE_DIR}/"
+        env:
+          PROVENANCE_FILE: ${{ needs.provenance.outputs.provenance-name }}
+          PROVENANCE_DIR: provenance/${{ needs.seal.outputs.RELEASE_VERSION}}
+
+      - name: Create PR
+        id: create-pr
+        uses: ./.github/actions/create-pr
+        with:
+          files: "pyproject.toml aws_lambda_powertools/shared/version.py provenance/"
+          temp_branch_prefix: "ci-bump"
+          pull_request_title: "chore(ci): new pre-release ${{ needs.seal.outputs.RELEASE_VERSION }}"
+          github_token: ${{ secrets.GITHUB_TOKEN }}

docs/maintainers.md

@@ -154,9 +154,9 @@ Firstly, make sure the commit history in the `develop` branch **(1)** it's up to
 
 **Looks good, what's next?**
 
-Kickoff the `Release` workflow with the intended version - this might take around 25m-30m to complete.
+Kickoff the [`Release` workflow](https://github.com/aws-powertools/powertools-lambda-python/blob/6db9079d21698b72f5d36d72c993c1aad7276db6/.github/workflows/release.yml#L3) with the intended version - this might take around 25m-30m to complete.
 
-Once complete, you can start drafting the release notes to let customers know **what changed and what's in it for them (a.k.a why they should care)**. We have guidelines in the release notes section so you know what good looks like.
+Once complete, you can start drafting the release notes to let customers know **what changed and what's in it for them (a.k.a why they should care)**. We have guidelines in the [release notes section](#drafting-release-notes) so you know what good looks like.
 
 > **NOTE**: Documentation might take a few minutes to reflect the latest version due to caching and CDN invalidations.
 
@@ -231,33 +231,43 @@ Release complete : milestone, m6, 10:31,2m
 
 #### Drafting release notes
 
+!!! info "Make sure the release workflow completed before you edit release notes."
+
 Visit the [Releases page](https://github.com/aws-powertools/powertools-lambda-python/releases) and choose the edit pencil button.
 
 Make sure the `tag` field reflects the new version you're releasing, the target branch field is set to `develop`, and `release title` matches your tag e.g., `v1.26.0`.
 
 You'll notice we group all changes based on their [labels](#labels) like `feature`, `bug`, `documentation`, etc.
+!!! question inline end "Spotted a typo?"
 
-**I spotted a typo or incorrect grouping - how do I fix it?**
+    Edit the respective PR title/labels and run the [Release Drafter workflow](https://github.com/aws-powertools/powertools-lambda-python/actions/workflows/release-drafter.yml).
 
-Edit the respective PR title and update their [labels](#labels). Then run the [Release Drafter workflow](https://github.com/aws-powertools/powertools-lambda-python/actions/workflows/release-drafter.yml) to update the Draft release.
+!!! question "All good, what's next?"
 
-> **NOTE**: This won't change the CHANGELOG as the merge commit is immutable. Don't worry about it. We'd only rewrite git history only if this can lead to confusion and we'd pair with another maintainer.
+The best part comes now!
 
-**All looking good, what's next?**
+Replace the placeholder `[Human readable summary of changes]` with what you'd like to communicate to customers what this release is all about.
 
-The best part comes now. Replace the placeholder `[Human readable summary of changes]` with what you'd like to communicate to customers what this release is all about. Rule of thumb: always put yourself in the customers shoes.
+!!! tip inline end "Always put yourself in the customers shoes. Most read the first sentence only to know whether this is for them."
 
 These are some questions to keep in mind when drafting your first or future release notes:
 
-- Can customers understand at a high level what changed in this release?
-- Is there a link to the documentation where they can read more about each main change?
-- Are there any graphics or [code snippets](https://carbon.now.sh/) that can enhance readability?
-- Are we calling out any key contributor(s) to this release?
-    - All contributors are automatically credited, use this as an exceptional case to feature them
+- **Can customers briefly understand the main changes in less than 30s?**
+    - _tip: first paragraph is punchy and optimizes for dependabot-like notifications._
+- **Are we calling out key contributor(s) to this release?**
+- **Is it clear what each change enables/unlocks and before?**
+    - _tip: use present and active voice; lead with the answer._
+- **Does it include a link to the documentation for each main change?**
+    - _tip: release explains what a change unblocks/enables (before/after), docs go in details_
+- **Is code snippet better in text or [graphic](https://carbon.now.sh)?**
+- **Does code snippet focus on the change only?**
+    - _tip: release snippets highlight functionality, no need to be functional (that's docs)_
 
 Once you're happy, hit `Publish release` 🎉🎉🎉.
 
-This will kick off the [Publishing workflow](https://github.com/aws-powertools/powertools-lambda-python/actions/workflows/release.yml) and within a few minutes you should see the latest version in PyPi, and all issues labeled as `pending-release` will be closed and notified.
+### Releasing an alpha release
+
+We publish alpha releases _(`prerelease`)_ every morning during business days (~8am UTC). You can also manually trigger `pre-release` workflow when needed.
 
 ### Run end to end tests
 

examples/homepage/install/sar/cdk_sar.py

@@ -3,7 +3,7 @@
 
 POWERTOOLS_BASE_NAME = "AWSLambdaPowertools"
 # Find latest from github.com/aws-powertools/powertools-lambda-python/releases
-POWERTOOLS_VER = "2.37.0"
+POWERTOOLS_VER = "2.39.1"
 POWERTOOLS_ARN = "arn:aws:serverlessrepo:eu-west-1:057560766410:applications/aws-lambda-powertools-python-layer"