CI: Merge workflows into build.yml

.ci/retrofit-worktree.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+if [ $# != 2 ]; then
+    echo >&2 "usage: $0 WORKTREE_NAME WORKTREE_DIRECTORY"
+    echo >&2 "Ensures that the current working directory is a git repository,"
+    echo >&2 "then makes WORKTREE_DIRECTORY a git worktree named WORKTREE_NAME."
+fi
+WORKTREE_NAME="$1"
+WORKTREE_DIRECTORY="$2"
+
+export GIT_AUTHOR_NAME="ci-sage workflow"
+export GIT_AUTHOR_EMAIL="[email protected]"
+export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
+export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
+
+set -ex
+
+# If actions/checkout downloaded our source tree using the GitHub REST API
+# instead of with git (because do not have git installed in our image),
+# we first make the source tree a repo.
+if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
+
+# Tag this state of the source tree "new". This is what we want to build and test.
+git tag -f new
+
+# Our container image contains a source tree in $WORKTREE_DIRECTORY with a full build of Sage.
+# But $WORKTREE_DIRECTORY is not a git repository.
+# We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new".
+# We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
+# Then we update worktree and index with "git reset --hard new".
+# (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
+# Finally we reset the index to "old". (This keeps all mtimes unchanged.)
+# The changed files now show up as uncommitted changes.
+# The final "git add -N" makes sure that files that were added in "new" do not show
+# as untracked files, which would be removed by "git clean -fx".
+git worktree add --detach $WORKTREE_NAME
+rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
+rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME
+if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi
+(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)

.devcontainer/onCreate-conda.sh

@@ -4,7 +4,7 @@ set -e
 # Create conda environment
 ./bootstrap-conda
 conda install mamba -n base -c conda-forge -y
-mamba env create --file src/environment-dev.yml || mamba env update --file src/environment-dev.yml
+mamba env create --file src/environment-dev-3.11.yml || mamba env update --file src/environment-dev-3.11.yml
 conda init bash
 
 # Build sage

.github/workflows/build.yml

@@ -49,6 +49,31 @@ jobs:
           path: upstream
           name: upstream
 
+  changed_files:
+    runs-on: ubuntu-latest
+    name: List changed packages
+    outputs:
+      uninstall_targets: ${{ steps.build-targets.outputs.uninstall_targets }}
+      build_targets:     ${{ steps.build-targets.outputs.build_targets }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get all packages that have changed
+        id: changed-packages
+        uses: tj-actions/changed-files@v38
+        with:
+          files_yaml: |
+            configures:
+              - 'build/pkgs/*/spkg-configure.m4'
+            pkgs:
+              - 'build/pkgs/**'
+              - 'pkgs/**'
+      - name: Determine targets to build
+        id: build-targets
+        run: |
+          echo "uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))" >> $GITHUB_OUTPUT
+          echo "build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([-_.a-z0-9]*)/[^ ]* *,\2-ensure,;'; done | sort -u))" >> $GITHUB_OUTPUT
+          cat $GITHUB_OUTPUT
+
   build:
     runs-on: ubuntu-latest
     container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
@@ -69,30 +94,8 @@ jobs:
         id: worktree
         run: |
           set -ex
-          git config --global user.email "[email protected]"
-          git config --global user.name "Build & Test workflow"
           git config --global --add safe.directory $(pwd)
-          # If actions/checkout downloaded our source tree using the GitHub REST API
-          # instead of with git (because do not have git installed in our image),
-          # we first make the source tree a repo.
-          if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
-          # Tag this state of the source tree "new". This is what we want to build and test.
-          git tag -f new
-          # Our container image contains a source tree in /sage with a full build of Sage.
-          # But /sage is not a git repository.
-          # We make /sage a worktree whose index is at tag "new".
-          # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
-          # Then we update worktree and index with "git reset --hard new".
-          # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
-          # Finally we reset the index to "old". (This keeps all mtimes unchanged.)
-          # The changed files now show up as uncommitted changes.
-          # The final "git add -N" makes sure that files that were added in "new" do not show
-          # as untracked files, which would be removed by "git clean -fx".
-          git worktree add --detach worktree-image
-          rm -rf /sage/.git && mv worktree-image/.git /sage/
-          rm -rf worktree-image && ln -s /sage worktree-image
-          if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
-          (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
+          .ci/retrofit-worktree.sh worktree-image /sage
 
       - name: Download upstream artifact
         uses: actions/download-artifact@v3
@@ -126,20 +129,9 @@ jobs:
           MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
-      - name: Set up node to install pyright
-        if: always() && steps.worktree.outcome == 'success'
-        uses: actions/setup-node@v3
-        with:
-          node-version: '12'
-
-      - name: Install pyright
-        if: always() && steps.worktree.outcome == 'success'
-        # Fix to v232 due to bug https://github.com/microsoft/pyright/issues/3239
-        run: npm install -g [email protected]
-
       - name: Static code check with pyright
         if: always() && steps.worktree.outcome == 'success'
-        run: pyright
+        run: ./sage -tox -e pyright
         working-directory: ./worktree-image
 
       - name: Clean (fallback to non-incremental)
@@ -195,3 +187,255 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           files: ./worktree-image/coverage.xml
+
+  linux-standard:
+    needs: [changed_files]
+    if: "${{needs.changed_files.outputs.uninstall_targets}}${{needs.changed_files.outputs.build_targets}}"
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build incrementally from published Docker image
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/sagemath/sage/
+      from_docker_target: "with-targets"
+      from_docker_tag: "dev"
+      docker_targets: "with-targets"
+      targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
+      tox_system_factors: >-
+        ["ubuntu-focal", "ubuntu-jammy", "ubuntu-mantic", "debian-bullseye", "debian-bookworm",
+         "fedora-30", "fedora-38", "gentoo-python3.11", "debian-bullseye-i386"]
+      tox_packages_factors: >-
+          ["standard"]
+      docker_push_repository: ghcr.io/${{ github.repository }}/
+    permissions:
+      packages: write
+
+  linux-standard-sitepackages:
+    needs: [changed_files]
+    if: "${{needs.changed_files.outputs.uninstall_targets}}${{needs.changed_files.outputs.build_targets}}"
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build incrementally from published Docker image
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/sagemath/sage/
+      from_docker_target: "with-targets"
+      from_docker_tag: "dev"
+      docker_targets: "with-targets"
+      targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
+      # Only test systems with a usable system python (>= 3.9)
+      tox_system_factors: >-
+        ["ubuntu-jammy", "ubuntu-mantic", "debian-bullseye", "debian-bookworm",
+         "fedora-33", "fedora-38", "gentoo-python3.11", "archlinux", "debian-bullseye-i386"]
+      tox_packages_factors: >-
+          ["standard-sitepackages"]
+      docker_push_repository: ghcr.io/${{ github.repository }}/
+    permissions:
+      packages: write
+
+  linux-minimal:
+    needs: [changed_files]
+    if: "${{needs.changed_files.outputs.uninstall_targets}}${{needs.changed_files.outputs.build_targets}}"
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build incrementally from published Docker image
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/sagemath/sage/
+      from_docker_target: "with-targets"
+      from_docker_tag: "dev"
+      docker_targets: "with-targets"
+      targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
+      tox_system_factors: >-
+        ["ubuntu-focal", "ubuntu-jammy", "ubuntu-mantic", "debian-bullseye", "debian-bookworm",
+         "fedora-30", "fedora-38", "gentoo-python3.11", "debian-bullseye-i386"]
+      tox_packages_factors: >-
+          ["minimal"]
+      docker_push_repository: ghcr.io/${{ github.repository }}/
+    permissions:
+      packages: write
+
+  build-docs:
+    runs-on: ubuntu-latest
+    container: ghcr.io/sagemath/sage/sage-ubuntu-focal-standard-with-targets:dev
+    needs: [get_ci_fixes]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Update system packages
+        run: |
+          apt-get update && apt-get install -y git zip
+
+      - name: Add prebuilt tree as a worktree
+        id: worktree
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "Build & Test workflow"
+          git config --global --add safe.directory $(pwd)
+          .ci/retrofit-worktree.sh worktree-image /sage
+          # Keep track of changes to built HTML
+          new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old")
+
+      - name: Download upstream artifact
+        uses: actions/download-artifact@v3
+        with:
+          path: upstream
+          name: upstream
+
+      - name: Apply CI fixes from sagemath/sage
+        # After applying the fixes, make sure all changes are marked as uncommitted changes.
+        run: |
+          if [ -r upstream/ci_fixes.patch ]; then
+            (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
+          fi
+
+      - name: Incremental build
+        id: incremental
+        run: |
+          # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps.
+          ./bootstrap && make build
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2 --output-sync=recurse
+          SAGE_NUM_THREADS: 2
+
+      - name: Build (fallback to non-incremental)
+        id: build
+        if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
+        run: |
+          set -ex
+          make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2 --output-sync=recurse
+          SAGE_NUM_THREADS: 2
+
+      - name: Build docs
+        id: docbuild
+        if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
+        # Always non-incremental because of the concern that
+        # incremental docbuild may introduce broken links (inter-file references) though build succeeds
+        run: |
+          set -ex
+          export SAGE_USE_CDNS=yes
+          mv /sage/local/share/doc/sage/html/en/.git /sage/.git-doc
+          make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage
+          mkdir -p /sage/local/share/doc/sage/html/en/ && mv /sage/.git-doc /sage/local/share/doc/sage/html/en/.git
+          ./config.status && make doc-html
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2 --output-sync=recurse
+          SAGE_NUM_THREADS: 2
+
+      - name: Copy docs
+        id: copy
+        if: always() && steps.docbuild.outcome == 'success'
+        run: |
+          set -ex
+          mkdir -p ./docs
+          # Create changelog
+          echo '## Preview of CHANGES.html'
+          (cd /sage/local/share/doc/sage/html/en && git diff --name-only) | tee ./docs/CHANGES.txt
+          (cd /sage/local/share/doc/sage/html/en && git diff; rm -rf .git) > ./docs/html.diff
+          echo '## Preview of html.diff'; head -n 400 ./docs/html.diff
+          (echo '<p><a href="html.diff">HTML diff</a>'; sed -E 's,(.*),<p><a href="\1">\1</a>,' ./docs/CHANGES.txt) > ./docs/CHANGES.html
+          # For some reason the deploy step below cannot find /sage/...
+          # So copy everything from there to local folder
+          # We also need to replace the symlinks because netlify is not following them
+          cp -r -L /sage/local/share/doc/sage/html/en/* ./docs
+          # Zip everything for increased performance
+          zip -r docs.zip docs
+
+      - name: Upload docs
+        if: always() && steps.copy.outcome == 'success'
+        uses: actions/upload-artifact@v3
+        with:
+          name: docs
+          path: docs.zip
+
+  build-docs-pdf:
+    runs-on: ubuntu-latest
+    container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
+    needs: [get_ci_fixes]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Update system packages
+        run: |
+          export PATH="build/bin:$PATH"
+          eval $(sage-print-system-package-command auto update)
+          eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip)
+          eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive)
+
+
+      - name: Add prebuilt tree as a worktree
+        id: worktree
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "Build & Test workflow"
+          git config --global --add safe.directory $(pwd)
+          .ci/retrofit-worktree.sh worktree-image /sage
+          # Keep track of changes to built HTML
+          new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old")
+
+      - name: Download upstream artifact
+        uses: actions/download-artifact@v3
+        with:
+          path: upstream
+          name: upstream
+
+      - name: Apply CI fixes from sagemath/sage
+        # After applying the fixes, make sure all changes are marked as uncommitted changes.
+        run: |
+          if [ -r upstream/ci_fixes.patch ]; then
+            (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
+          fi
+
+      - name: Incremental build
+        id: incremental
+        run: |
+          # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps.
+          ./bootstrap && make build
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2
+          SAGE_NUM_THREADS: 2
+
+      - name: Build (fallback to non-incremental)
+        id: build
+        if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
+        run: |
+          set -ex
+          make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2
+          SAGE_NUM_THREADS: 2
+
+      - name: Build docs (PDF)
+        id: docbuild
+        if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
+        run: make build V=0 && make doc-pdf
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2
+          SAGE_NUM_THREADS: 2
+
+      - name: Copy docs
+        if: always() && steps.docbuild.outcome == 'success'
+        run: |
+          # For some reason the deploy step below cannot find /sage/...
+          # So copy everything from there to local folder
+          mkdir -p ./docs
+          cp -r -L /sage/local/share/doc/sage/pdf/en/* ./docs
+          # Zip everything for increased performance
+          zip -r docs-pdf.zip docs
+
+      - name: Upload docs
+        if: always() && steps.copy.outcome == 'success'
+        uses: actions/upload-artifact@v3
+        with:
+          name: docs-pdf
+          path: docs-pdf.zip