From dd0575d813b5d2bca5016cee883ee60dafef650a Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 22 Feb 2023 12:34:23 -0800 Subject: [PATCH 01/13] pkgs/*/tox.ini, src/doc/en/developer: Update instructions for testing --- pkgs/sagemath-categories/tox.ini | 6 ++--- .../en/developer/packaging_sage_library.rst | 24 +++++++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index a240b091e30..01826d43d2d 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -1,10 +1,10 @@ # To build and test in the tox environment: # -# ./sage -sh -c '(cd pkgs/sagemath-categories && tox -v -v -v -e sagepython)' +# make wheels && ./sage -sh -c '(cd pkgs/sagemath-categories && tox -v -v -v -e sagepython-sagewheels-nopypi-norequirements)' # -# To test interactively: +# After this, to test interactively: # -# pkgs/sagemath-categories/.tox/sagepython/bin/python +# pkgs/sagemath-categories/.tox/sagepython-sagewheels-nopypi-norequirements/bin/sage # [tox] envlist = diff --git a/src/doc/en/developer/packaging_sage_library.rst b/src/doc/en/developer/packaging_sage_library.rst index 7e7a5cde56f..69189d61bf5 100644 --- a/src/doc/en/developer/packaging_sage_library.rst +++ b/src/doc/en/developer/packaging_sage_library.rst @@ -582,16 +582,14 @@ distribution to be tested (and its Python dependencies). Let's try it out first with the entire Sage library, represented by the distribution **sagemath-standard**. Note that after Sage has been -built normally, a set of wheels for all installed Python packages is -available in ``SAGE_VENV/var/lib/sage/wheels/``:: +built normally, a set of wheels for most installed Python distribution +packages is available in ``SAGE_VENV/var/lib/sage/wheels/``:: $ ls venv/var/lib/sage/wheels Babel-2.9.1-py2.py3-none-any.whl Cython-0.29.24-cp39-cp39-macosx_11_0_x86_64.whl Jinja2-2.11.2-py2.py3-none-any.whl ... - sage_conf-9.5b6-py3-none-any.whl - ... scipy-1.7.2-cp39-cp39-macosx_11_0_x86_64.whl setuptools-58.2.0-py3-none-any.whl ... @@ -599,6 +597,22 @@ available in ``SAGE_VENV/var/lib/sage/wheels/``:: widgetsnbextension-3.5.1-py2.py3-none-any.whl zipp-3.5.0-py3-none-any.whl +However, in a build of Sage with the default configuration +``configure --enable-editable``, there will be no wheels for the +distributions ``sage_*`` and ``sagemath-*``. + +To create these wheels, use the command ``make wheels``:: + + $ make wheels + ... + $ ls venv/var/lib/sage/wheels/sage* + ... + sage_conf-10.0b2-py3-none-any.whl + ... + +(You can also use ``./configure --enable-wheels`` to ensure that +these wheels are always available and up to date.) + Note in particular the wheel for **sage-conf**, which provides configuration variable settings and the connection to the non-Python packages installed in ``SAGE_LOCAL``. @@ -639,7 +653,7 @@ without depending on optional packages, but without the packages Again we can run the test with ``tox`` in a separate virtual environment:: - $ ./bootstrap && ./sage -sh -c '(cd pkgs/sagemath-standard-no-symbolics && SAGE_NUM_THREADS=16 tox -v -v -v -e sagepython-sagewheels-nopypi)' + $ ./bootstrap && make wheels && ./sage -sh -c '(cd pkgs/sagemath-standard-no-symbolics && SAGE_NUM_THREADS=16 tox -v -v -v -e sagepython-sagewheels-nopypi-norequirements)' Some small distributions, for example the ones providing the two lowest levels, `sagemath-objects `_ From 829d567fefb04fa2a9cb21a5622dd54e92d1c3dc Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 22 Feb 2023 23:12:36 -0800 Subject: [PATCH 02/13] pkgs/sagemath-categories/tox.ini: Change default toxenv to sagepython-sagewheels-nopypi-norequirements --- pkgs/sagemath-categories/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 01826d43d2d..b12013547c1 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -8,7 +8,7 @@ # [tox] envlist = - sagepython-norequirements + sagepython-sagewheels-nopypi-norequirements [testenv] deps = From 379c6a45564164082af5abb47dfeef59d81fa236 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 25 Feb 2023 14:48:57 -0800 Subject: [PATCH 03/13] pkgs/sagemath-polyhedra/tox.ini: Update for tox 4 --- pkgs/sagemath-categories/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index b12013547c1..989a2582094 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -1,6 +1,6 @@ # To build and test in the tox environment: # -# make wheels && ./sage -sh -c '(cd pkgs/sagemath-categories && tox -v -v -v -e sagepython-sagewheels-nopypi-norequirements)' +# make SAGE_WHEELS=yes sagemath_categories-build-deps && ./sage -sh -c '(cd pkgs/sagemath-categories && SAGE_NUM_THREADS=8 tox -v -v -v -e sagepython-sagewheels-nopypi-norequirements)' # # After this, to test interactively: # From 513eac050880510e6874cd3a3348240684ffec1f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 25 Feb 2023 16:22:39 -0800 Subject: [PATCH 04/13] pkgs/sagemath-*/tox.ini: Update for tox 4 --- pkgs/sagemath-categories/tox.ini | 56 ++++++++++++++++++------ pkgs/sagemath-environment/tox.ini | 57 ++++++++++++++++++++----- pkgs/sagemath-objects/tox.ini | 55 +++++++++++++++++++----- pkgs/sagemath-repl/tox.ini | 59 +++++++++++++++++++------ pkgs/sagemath-standard/tox.ini | 71 ++++++++++++++++++++----------- 5 files changed, 228 insertions(+), 70 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 989a2582094..c60217f633c 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -10,14 +10,14 @@ envlist = sagepython-sagewheels-nopypi-norequirements -[testenv] -deps = - !norequirements: -rrequirements.txt - # tox 3.x does not handle extras when using --installpkg. https://github.com/tox-dev/tox/issues/1576 - sagemath-repl - -extras = test +requires = + # Auto-provision a modern tox. + # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance + tox>=4.2 +[pkgenv] +# Environment in which to build the sdist. +# https://tox.wiki/en/latest/upgrading.html#packaging-environments passenv = # Variables set by .homebrew-build-env CPATH @@ -32,36 +32,68 @@ passenv = sagewheels: SAGE_SPKG_WHEELS setenv = - # Sage scripts such as sage-runtests like to use $HOME/.sage - HOME={envdir} # We supply pip options by environment variables so that they # apply both to the installation of the dependencies and of the package sagewheels: PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels} nopypi: PIP_NO_INDEX=true +[testenv] +deps = + !norequirements: -rrequirements.txt + +extras = test + +passenv = {[pkgenv]passenv} + +setenv = {[pkgenv]setenv} + # Sage scripts such as sage-runtests like to use $HOME/.sage + HOME={envdir} + allowlist_externals = bash commands = - # Beware of the treacherous non-src layout. "./sage/" shadows the install sage package. + # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.cpython.builtin_types, sage.cpython.cython_metaclass, sage.cpython.debug, sage.structure.all, sage.categories.all' # Test that importing sage.categories.all initializes categories {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' - bash -c 'cd {temp_dir} && SAGE_SRC=$(python -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --initial --environment=sage.all__sagemath_categories --optional=sage $SAGE_SRC/sage/structure || echo "(lots of doctest failures are expected)"' + bash -c 'cd $(python -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") \ + && sage-runtests -p --initial --environment=sage.all__sagemath_categories --optional=sage $SAGE_SRC/sage/structure || echo "(lots of doctest failures are expected)"' + +[testenv:.pkg-sagepython] +# Environment in which to build the sdist. +# inherits from [pkgenv] - https://tox.wiki/en/latest/upgrading.html#packaging-environments +basepython = {env:SAGE_VENV}/bin/python3 + +[testenv:.pkg-sagepython-sagewheels-nopypi] +passenv = {[pkgenv]passenv} + SAGE_VENV + SAGE_SPKG_WHEELS + +setenv = {[pkgenv]setenv} + PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels} + PIP_NO_INDEX=true + +basepython = {env:SAGE_VENV}/bin/python3 [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython [testenv:sagepython-sagewheels-nopypi] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi [testenv:sagepython-sagewheels-nopypi-norequirements] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi [testenv:sagepython-sagewheels] -basepython = {env:SAGE_VENV}/bin/python3 +basepython = {env:SAGE_VENV}/bin/python +package_env = .pkg-sagepython [testenv:sagepython-norequirements] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index 46077bd8e27..d602725b5a0 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -1,22 +1,28 @@ # To build and test in the tox environment: # -# ./sage -sh -c '(cd pkgs/sagemath-environment && tox -v -v -e sagepython)' +# ./sage -sh -c '(cd pkgs/sagemath-environment && tox -v -v -e sagepython-norequirements)' # # To test interactively: # -# pkgs/sagemath-environment/.tox/sagepython/bin/python +# pkgs/sagemath-environment/.tox/sagepython-norequirements/bin/python # [tox] envlist = sagepython-norequirements -isolated_build = True - -[testenv] -deps = - !norequirements: -rrequirements.txt +requires = + # Auto-provision a modern tox. + # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance + tox>=4.2 +[pkgenv] +# Environment in which to build the sdist. +# https://tox.wiki/en/latest/upgrading.html#packaging-environments passenv = + # Variables set by .homebrew-build-env + CPATH + LIBRARY_PATH + PKG_CONFIG_PATH # Parallel build SAGE_NUM_THREADS SAGE_NUM_THREADS_PARALLEL @@ -26,13 +32,23 @@ passenv = sagewheels: SAGE_SPKG_WHEELS setenv = - # Sage scripts such as sage-runtests like to use $HOME/.sage - HOME={envdir} # We supply pip options by environment variables so that they # apply both to the installation of the dependencies and of the package sagewheels: PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels} nopypi: PIP_NO_INDEX=true +[testenv] +deps = + !norequirements: -rrequirements.txt + +extras = test + +passenv = {[pkgenv]passenv} + +setenv = {[pkgenv]setenv} + # Sage scripts such as sage-runtests like to use $HOME/.sage + HOME={envdir} + allowlist_externals = bash @@ -40,17 +56,38 @@ commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.features.all import all_features; print(sorted(all_features(), key=lambda x: x.name)); import sage.misc.package' +[testenv:.pkg-sagepython] +# Environment in which to build the sdist. +# inherits from [pkgenv] - https://tox.wiki/en/latest/upgrading.html#packaging-environments +basepython = {env:SAGE_VENV}/bin/python3 + +[testenv:.pkg-sagepython-sagewheels-nopypi] +passenv = {[pkgenv]passenv} + SAGE_VENV + SAGE_SPKG_WHEELS + +setenv = {[pkgenv]setenv} + PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels} + PIP_NO_INDEX=true + +basepython = {env:SAGE_VENV}/bin/python3 + [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython [testenv:sagepython-sagewheels-nopypi] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi [testenv:sagepython-sagewheels-nopypi-norequirements] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi [testenv:sagepython-sagewheels] -basepython = {env:SAGE_VENV}/bin/python3 +basepython = {env:SAGE_VENV}/bin/python +package_env = .pkg-sagepython [testenv:sagepython-norequirements] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index e34531467e5..4bd3a96419c 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -1,21 +1,23 @@ # To build and test in the tox environment: # -# ./sage -sh -c '(cd pkgs/sagemath-objects && tox -v -v -e sagepython)' +# ./sage -sh -c '(cd pkgs/sagemath-objects && SAGE_NUM_THREADS=8 tox -v -v -e sagepython-norequirements)' # -# To test interactively: +# After this, to test interactively: # -# pkgs/sagemath-objects/.tox/sagepython/bin/python +# pkgs/sagemath-objects/.tox/sagepython-norequirements/bin/python # [tox] envlist = sagepython-norequirements -[testenv] -deps = - !norequirements: -rrequirements.txt - -extras = test +requires = + # Auto-provision a modern tox. + # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance + tox>=4.2 +[pkgenv] +# Environment in which to build the sdist. +# https://tox.wiki/en/latest/upgrading.html#packaging-environments passenv = # Variables set by .homebrew-build-env CPATH @@ -30,13 +32,23 @@ passenv = sagewheels: SAGE_SPKG_WHEELS setenv = - # Sage scripts such as sage-runtests like to use $HOME/.sage - HOME={envdir} # We supply pip options by environment variables so that they # apply both to the installation of the dependencies and of the package sagewheels: PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels} nopypi: PIP_NO_INDEX=true +[testenv] +deps = + !norequirements: -rrequirements.txt + +extras = test + +passenv = {[pkgenv]passenv} + +setenv = {[pkgenv]setenv} + # Sage scripts such as sage-runtests like to use $HOME/.sage + HOME={envdir} + allowlist_externals = bash @@ -48,17 +60,38 @@ commands = #bash -c 'cd {temp_dir} && SAGE_SRC=$(python -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --environment=sage.all__sagemath_objects --initial --optional=sage $SAGE_SRC/sage/structure || echo "(lots of doctest failures are expected)"' +[testenv:.pkg-sagepython] +# Environment in which to build the sdist. +# inherits from [pkgenv] - https://tox.wiki/en/latest/upgrading.html#packaging-environments +basepython = {env:SAGE_VENV}/bin/python3 + +[testenv:.pkg-sagepython-sagewheels-nopypi] +passenv = {[pkgenv]passenv} + SAGE_VENV + SAGE_SPKG_WHEELS + +setenv = {[pkgenv]setenv} + PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels} + PIP_NO_INDEX=true + +basepython = {env:SAGE_VENV}/bin/python3 + [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython [testenv:sagepython-sagewheels-nopypi] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi [testenv:sagepython-sagewheels-nopypi-norequirements] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi [testenv:sagepython-sagewheels] -basepython = {env:SAGE_VENV}/bin/python3 +basepython = {env:SAGE_VENV}/bin/python +package_env = .pkg-sagepython [testenv:sagepython-norequirements] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index 084016ce768..eafb1d13d77 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -1,21 +1,23 @@ # To build and test in the tox environment: # -# ./sage -sh -c '(cd pkgs/sagemath-repl && tox -v -v -e sagepython)' +# make SAGE_WHEELS=yes sagemath_repl-build-deps && ./sage -sh -c '(cd pkgs/sagemath-repl && SAGE_NUM_THREADS=8 tox -v -v -e sagepython-sagewheels-nopypi-norequirements)' # -# To test interactively: +# After this, to test interactively: # -# pkgs/sagemath-repl/.tox/sagepython/bin/python +# pkgs/sagemath-repl/.tox/sagepython-sagewheels-nopypi-norequirements/bin/python # [tox] envlist = - sagepython-norequirements + sagepython-sagewheels-nopypi-norequirements -isolated_build = True - -[testenv] -deps = - !norequirements: -rrequirements.txt +requires = + # Auto-provision a modern tox. + # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance + tox>=4.2 +[pkgenv] +# Environment in which to build the sdist. +# https://tox.wiki/en/latest/upgrading.html#packaging-environments passenv = # Variables set by .homebrew-build-env CPATH @@ -30,13 +32,23 @@ passenv = sagewheels: SAGE_SPKG_WHEELS setenv = - # Sage scripts such as sage-runtests like to use $HOME/.sage - HOME={envdir} # We supply pip options by environment variables so that they # apply both to the installation of the dependencies and of the package sagewheels: PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels} nopypi: PIP_NO_INDEX=true +[testenv] +deps = + !norequirements: -rrequirements.txt + +extras = test + +passenv = {[pkgenv]passenv} + +setenv = {[pkgenv]setenv} + # Sage scripts such as sage-runtests like to use $HOME/.sage + HOME={envdir} + allowlist_externals = bash @@ -44,19 +56,40 @@ commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.repl.all; import sage.doctest.all' - bash -c 'cd bin && SAGE_SRC=$({envpython} -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --environment=sage.all__sagemath_repl --initial --optional=sage $SAGE_SRC/sage/repl $SAGE_SRC/sage/doctest $SAGE_SRC/sage/misc/sage_input.py $SAGE_SRC/sage/misc/sage_eval.py || echo "(lots of doctest failures are expected)"' + bash -c 'cd $({envpython} -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py || echo "(lots of doctest failures are expected)"' + +[testenv:.pkg-sagepython] +# Environment in which to build the sdist. +# inherits from [pkgenv] - https://tox.wiki/en/latest/upgrading.html#packaging-environments +basepython = {env:SAGE_VENV}/bin/python3 + +[testenv:.pkg-sagepython-sagewheels-nopypi] +passenv = {[pkgenv]passenv} + SAGE_VENV + SAGE_SPKG_WHEELS + +setenv = {[pkgenv]setenv} + PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels} + PIP_NO_INDEX=true + +basepython = {env:SAGE_VENV}/bin/python3 [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython [testenv:sagepython-sagewheels-nopypi] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi [testenv:sagepython-sagewheels-nopypi-norequirements] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi [testenv:sagepython-sagewheels] -basepython = {env:SAGE_VENV}/bin/python3 +basepython = {env:SAGE_VENV}/bin/python +package_env = .pkg-sagepython [testenv:sagepython-norequirements] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython diff --git a/pkgs/sagemath-standard/tox.ini b/pkgs/sagemath-standard/tox.ini index 305eddd7586..c2703e2b0e7 100644 --- a/pkgs/sagemath-standard/tox.ini +++ b/pkgs/sagemath-standard/tox.ini @@ -1,7 +1,3 @@ -# First install tox: -# -# ./sage -i tox -# # All tests require an installation of the non-Python components of the Sage distribution # in SAGE_LOCAL. # @@ -67,14 +63,14 @@ envlist = # sagepython-sagewheels-pipenv -[testenv] -deps = - pipenv: pipenv - !pipenv-!norequirements: -rrequirements.txt - ## Needed for fpylll - norequirements: Cython - norequirements: cysignals +requires = + # Auto-provision a modern tox. + # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance + tox>=4.2 +[pkgenv] +# Environment in which to build the sdist. +# https://tox.wiki/en/latest/upgrading.html#packaging-environments passenv = # Variables set by .homebrew-build-env CPATH @@ -82,14 +78,13 @@ passenv = PKG_CONFIG_PATH # Parallel build SAGE_NUM_THREADS + SAGE_NUM_THREADS_PARALLEL # SAGE_VENV only for referring to the basepython or finding the wheels sagepython, sagewheels: SAGE_VENV # Location of the wheels sagewheels: SAGE_SPKG_WHEELS setenv = - # Sage scripts such as sage-runtests like to use $HOME/.sage - HOME={envdir} # We supply pip options by environment variables so that they # apply both to the installation of the dependencies and of the package sagewheels: PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels} @@ -103,23 +98,28 @@ setenv = # so we cannot isolate it in the tox environment. pipenv: PIPENV_SKIP_LOCK=true +[testenv] +deps = + pipenv: pipenv + !pipenv-!norequirements: -rrequirements.txt + ## Needed for fpylll + norequirements: Cython + norequirements: cysignals + sitepackages = sitepackages: True !sitepackages: False -allowlist_externals = - bash +extras = test -skip_install = - pipenv: True - !pipenv: False +passenv = {[pkgenv]passenv} -commands_pre = - # Use Pipenv to install according to Pipfile into the virtual environment created by tox. - # https://pipenv-searchable.readthedocs.io/advanced.html#tox-automation-project - pipenv-!dist: pipenv install -v -v -v - # Same, but use $SAGE_ROOT/Pipfile - pipenv-dist: bash -c '(cd ../../../.. && pipenv install -v -v -v)' +setenv = {[pkgenv]setenv} + # Sage scripts such as sage-runtests like to use $HOME/.sage + HOME={envdir} + +allowlist_externals = + bash commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. @@ -130,23 +130,46 @@ commands = sage -t -p --all +[testenv:.pkg-sagepython] +# Environment in which to build the sdist. +# inherits from [pkgenv] - https://tox.wiki/en/latest/upgrading.html#packaging-environments +basepython = {env:SAGE_VENV}/bin/python3 + +[testenv:.pkg-sagepython-sagewheels-nopypi] +passenv = {[pkgenv]passenv} + SAGE_VENV + SAGE_SPKG_WHEELS + +setenv = {[pkgenv]setenv} + PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels} + PIP_NO_INDEX=true + +basepython = {env:SAGE_VENV}/bin/python3 + [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython [testenv:sagepython-sagewheels-nopypi] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi [testenv:sagepython-sagewheels-nopypi-norequirements] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi [testenv:sagepython-sagewheels] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython [testenv:sagepython-sagewheels-pipenv-dist] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython [testenv:sagepython-sagewheels-norequirements] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython [testenv:sagepython-sagewheels-pipenv] basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython From 4a35130033c3eba8ad3a03829e064e2994580bff Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 26 Feb 2023 01:38:23 -0800 Subject: [PATCH 05/13] pkgs/sagemath-categories/tox.ini: Fix up --- pkgs/sagemath-categories/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index c60217f633c..c644b953880 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -60,7 +60,7 @@ commands = {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' bash -c 'cd $(python -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") \ - && sage-runtests -p --initial --environment=sage.all__sagemath_categories --optional=sage $SAGE_SRC/sage/structure || echo "(lots of doctest failures are expected)"' + && sage-runtests -p --initial --environment=sage.all__sagemath_categories --optional=sage sage/structure || echo "(lots of doctest failures are expected)"' [testenv:.pkg-sagepython] # Environment in which to build the sdist. From a156e2a3a71eca90e982771228d350e596a86fc8 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 26 Feb 2023 19:03:00 -0800 Subject: [PATCH 06/13] pkgs/sagemath-*/tox.ini: Configure testenv:.tox --- pkgs/sagemath-categories/tox.ini | 7 ++++++- pkgs/sagemath-environment/tox.ini | 5 +++++ pkgs/sagemath-objects/tox.ini | 5 +++++ pkgs/sagemath-repl/tox.ini | 5 +++++ pkgs/sagemath-standard/tox.ini | 5 +++++ 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index c644b953880..6f798bbd00d 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -13,7 +13,7 @@ envlist = requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance - tox>=4.2 + tox>=4.8 [pkgenv] # Environment in which to build the sdist. @@ -62,6 +62,11 @@ commands = bash -c 'cd $(python -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") \ && sage-runtests -p --initial --environment=sage.all__sagemath_categories --optional=sage sage/structure || echo "(lots of doctest failures are expected)"' +[testenv:.tox] +# Allow access to PyPI for auto-provisioning a suitable tox version +passenv = +setenv = + [testenv:.pkg-sagepython] # Environment in which to build the sdist. # inherits from [pkgenv] - https://tox.wiki/en/latest/upgrading.html#packaging-environments diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index d602725b5a0..4ae9cabf595 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -56,6 +56,11 @@ commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.features.all import all_features; print(sorted(all_features(), key=lambda x: x.name)); import sage.misc.package' +[testenv:.tox] +# Allow access to PyPI for auto-provisioning a suitable tox version +passenv = +setenv = + [testenv:.pkg-sagepython] # Environment in which to build the sdist. # inherits from [pkgenv] - https://tox.wiki/en/latest/upgrading.html#packaging-environments diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index 4bd3a96419c..5221c8613c8 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -60,6 +60,11 @@ commands = #bash -c 'cd {temp_dir} && SAGE_SRC=$(python -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --environment=sage.all__sagemath_objects --initial --optional=sage $SAGE_SRC/sage/structure || echo "(lots of doctest failures are expected)"' +[testenv:.tox] +# Allow access to PyPI for auto-provisioning a suitable tox version +passenv = +setenv = + [testenv:.pkg-sagepython] # Environment in which to build the sdist. # inherits from [pkgenv] - https://tox.wiki/en/latest/upgrading.html#packaging-environments diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index eafb1d13d77..bdc705c9504 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -58,6 +58,11 @@ commands = bash -c 'cd $({envpython} -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py || echo "(lots of doctest failures are expected)"' +[testenv:.tox] +# Allow access to PyPI for auto-provisioning a suitable tox version +passenv = +setenv = + [testenv:.pkg-sagepython] # Environment in which to build the sdist. # inherits from [pkgenv] - https://tox.wiki/en/latest/upgrading.html#packaging-environments diff --git a/pkgs/sagemath-standard/tox.ini b/pkgs/sagemath-standard/tox.ini index c2703e2b0e7..c2a254b84c7 100644 --- a/pkgs/sagemath-standard/tox.ini +++ b/pkgs/sagemath-standard/tox.ini @@ -130,6 +130,11 @@ commands = sage -t -p --all +[testenv:.tox] +# Allow access to PyPI for auto-provisioning a suitable tox version +passenv = +setenv = + [testenv:.pkg-sagepython] # Environment in which to build the sdist. # inherits from [pkgenv] - https://tox.wiki/en/latest/upgrading.html#packaging-environments From 353e6254ef4022d9b1f7bc4ec5c5cfb1a481b10d Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 26 Feb 2023 21:04:59 -0800 Subject: [PATCH 07/13] pkgs/sagemath-*/tox.ini: Configure testenv:.tox (fixup) --- pkgs/sagemath-categories/tox.ini | 2 +- pkgs/sagemath-environment/tox.ini | 2 +- pkgs/sagemath-objects/tox.ini | 2 +- pkgs/sagemath-repl/tox.ini | 2 +- pkgs/sagemath-standard/tox.ini | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 6f798bbd00d..930271ccf47 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -65,7 +65,7 @@ commands = [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version passenv = -setenv = +setenv = PIP_NO_INDEX=false [testenv:.pkg-sagepython] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index 4ae9cabf595..65bcdc902e0 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -59,7 +59,7 @@ commands = [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version passenv = -setenv = +setenv = PIP_NO_INDEX=false [testenv:.pkg-sagepython] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index 5221c8613c8..e5bf1e6dac7 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -63,7 +63,7 @@ commands = [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version passenv = -setenv = +setenv = PIP_NO_INDEX=false [testenv:.pkg-sagepython] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index bdc705c9504..1637788be39 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -61,7 +61,7 @@ commands = [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version passenv = -setenv = +setenv = PIP_NO_INDEX=false [testenv:.pkg-sagepython] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-standard/tox.ini b/pkgs/sagemath-standard/tox.ini index c2a254b84c7..1204ea9df09 100644 --- a/pkgs/sagemath-standard/tox.ini +++ b/pkgs/sagemath-standard/tox.ini @@ -133,7 +133,7 @@ commands = [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version passenv = -setenv = +setenv = PIP_NO_INDEX=false [testenv:.pkg-sagepython] # Environment in which to build the sdist. From b48a24c725ac54f3342aee74cc709cac3bccbaba Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 26 Feb 2023 21:25:59 -0800 Subject: [PATCH 08/13] pkgs/sagemath-categories/tox.ini: fix up --- pkgs/sagemath-categories/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 930271ccf47..bc5213f1730 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -13,7 +13,7 @@ envlist = requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance - tox>=4.8 + tox>=4.2 [pkgenv] # Environment in which to build the sdist. From 9597136e60545a9233446eeba65f98036106e8c9 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 17 Mar 2023 19:12:42 -0700 Subject: [PATCH 09/13] pkgs/sagemath-categories/tox.ini: Update instructions to include building sagemath_repl wheel --- pkgs/sagemath-categories/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index bc5213f1730..79bd309f0e3 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -1,6 +1,6 @@ # To build and test in the tox environment: # -# make SAGE_WHEELS=yes sagemath_categories-build-deps && ./sage -sh -c '(cd pkgs/sagemath-categories && SAGE_NUM_THREADS=8 tox -v -v -v -e sagepython-sagewheels-nopypi-norequirements)' +# make SAGE_WHEELS=yes sagemath_categories-build-deps sagemath_repl && ./sage -sh -c '(cd pkgs/sagemath-categories && SAGE_NUM_THREADS=8 tox -v -v -v -e sagepython-sagewheels-nopypi-norequirements)' # # After this, to test interactively: # From cf8f92e6dc6a18925afc6e711588a2fc2663c891 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 30 Apr 2023 23:30:08 -0700 Subject: [PATCH 10/13] src/sage/misc/package.py: Make doctests work within venvs --- src/sage/misc/package.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/sage/misc/package.py b/src/sage/misc/package.py index 772007c6910..a7de52331b2 100644 --- a/src/sage/misc/package.py +++ b/src/sage/misc/package.py @@ -384,12 +384,19 @@ def installed_packages(exclude_pip=True): - ``exclude_pip`` -- (optional, default: ``True``) whether "pip" packages are excluded from the list - EXAMPLES:: + EXAMPLES: + + Below we test for a standard package without ``spkg-configure.m4`` script + that should be installed in ``SAGE_LOCAL``. When Sage is installed by + the Sage distribution (indicated by feature ``sage_spkg``), we should have + the installation record for this package. (We do not test for installation + records of Python packages. Our ``SAGE_VENV`` is not necessarily the + main Sage venv; it could be a user-created venv or a venv created by tox.) - sage: sorted(installed_packages().keys()) # optional - sage_spkg - [...'gmpy2', ...'sage_conf', ...] - sage: installed_packages()['gmpy2'] # optional - sage_spkg, random - '2.1.0b5' + sage: sorted(installed_packages().keys()) # optional - sage_spkg + [...'conway_polynomials', ...] + sage: installed_packages()['conway_polynomials'] # optional - sage_spkg, random + '0.5' .. SEEALSO:: @@ -424,12 +431,12 @@ def is_package_installed(package, exclude_pip=True): EXAMPLES:: - sage: is_package_installed('gap') # optional - sage_spkg + sage: is_package_installed('conway_polynomials') # optional - sage_spkg True Giving just the beginning of the package name is not good enough:: - sage: is_package_installed('matplotli') # optional - sage_spkg + sage: is_package_installed('conway_poly') # optional - sage_spkg False Otherwise, installing "pillow" would cause this function to think From c350f06caf08af9d8a6addf376496efee946ff2a Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 1 May 2023 00:32:53 -0700 Subject: [PATCH 11/13] pkgs/sagemath-standard/tox.ini: change_dir to fix shadowing by sage/ in a sub-python --- pkgs/sagemath-standard/tox.ini | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkgs/sagemath-standard/tox.ini b/pkgs/sagemath-standard/tox.ini index 1204ea9df09..0430bcda5cc 100644 --- a/pkgs/sagemath-standard/tox.ini +++ b/pkgs/sagemath-standard/tox.ini @@ -121,9 +121,12 @@ setenv = {[pkgenv]setenv} allowlist_externals = bash +# Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. +# So we change to another directory when running commands. +change_dir = {env_tmp_dir} + commands = - # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - python -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.all; print(sage.all.__file__)' + python -c 'import sage.all; print(sage.all.__file__)' # We check that the "sage" script invokes the correct Python. sage -c 'import sys; print("sys.path =", sys.path); import sage.all; print(sage.all.__file__)' From b46a2d3f66a0c9977431541a7b379568ab1ac495 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 1 May 2023 00:52:43 -0700 Subject: [PATCH 12/13] src/sage/misc/package.py: Make doctests work within venvs, part 2 --- src/sage/misc/package.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sage/misc/package.py b/src/sage/misc/package.py index a7de52331b2..26ddebdeda5 100644 --- a/src/sage/misc/package.py +++ b/src/sage/misc/package.py @@ -628,15 +628,15 @@ def package_manifest(package): EXAMPLES:: sage: from sage.misc.package import package_manifest - sage: sagetex_manifest = package_manifest('sagetex') # optional - sage_spkg - sage: sagetex_manifest['package_name'] == 'sagetex' # optional - sage_spkg + sage: manifest = package_manifest('conway_polynomials') # optional - sage_spkg + sage: manifest['package_name'] == 'conway_polynomials' # optional - sage_spkg True - sage: 'files' in sagetex_manifest # optional - sage_spkg + sage: 'files' in manifest # optional - sage_spkg True Test a nonexistent package:: - sage: package_manifest('dummy-package') # optional - sage_spkg + sage: package_manifest('dummy-package') # optional - sage_spkg Traceback (most recent call last): ... KeyError: 'dummy-package' From 0c8bb55180037fe043f5bc450b0f072e3bc87c03 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 2 May 2023 13:10:12 -0700 Subject: [PATCH 13/13] Fixups --- pkgs/sagemath-categories/tox.ini | 2 +- src/sage/misc/package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 79bd309f0e3..c7dbc3fb5dd 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -4,7 +4,7 @@ # # After this, to test interactively: # -# pkgs/sagemath-categories/.tox/sagepython-sagewheels-nopypi-norequirements/bin/sage +# pkgs/sagemath-categories/.tox/sagepython-sagewheels-nopypi-norequirements/bin/sage # [tox] envlist = diff --git a/src/sage/misc/package.py b/src/sage/misc/package.py index 26ddebdeda5..d0b5808e215 100644 --- a/src/sage/misc/package.py +++ b/src/sage/misc/package.py @@ -391,7 +391,7 @@ def installed_packages(exclude_pip=True): the Sage distribution (indicated by feature ``sage_spkg``), we should have the installation record for this package. (We do not test for installation records of Python packages. Our ``SAGE_VENV`` is not necessarily the - main Sage venv; it could be a user-created venv or a venv created by tox.) + main Sage venv; it could be a user-created venv or a venv created by tox.):: sage: sorted(installed_packages().keys()) # optional - sage_spkg [...'conway_polynomials', ...]