From 7ed806861c89745c5d9e0ed8d28cd2a45dc5a527 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 12 Mar 2024 14:30:23 +0100 Subject: [PATCH 01/12] install upstream-dev h5py and enable h5netcdf again --- ci/install-upstream-wheels.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index b1db58b7d03..3bbecd38ba0 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -5,7 +5,7 @@ micromamba install "cython>=0.29.20" py-cpuinfo # temporarily (?) remove numbagg and numba micromamba remove -y numba numbagg # temporarily remove backends -micromamba remove -y cf_units h5py hdf5 netcdf4 +micromamba remove -y cf_units netcdf4 # forcibly remove packages to avoid artifacts conda uninstall -y --force \ numpy \ @@ -28,6 +28,7 @@ python -m pip install \ --upgrade \ numpy \ scipy \ + h5py \ matplotlib \ pandas # without build isolation for packages compiling against numpy @@ -60,4 +61,4 @@ python -m pip install \ git+https://github.com/SciTools/nc-time-axis \ git+https://github.com/xarray-contrib/flox \ git+https://github.com/dgasmith/opt_einsum - # git+https://github.com/h5netcdf/h5netcdf + git+https://github.com/h5netcdf/h5netcdf From 9949bc3f833a12e0850eb6c197457714f3d873ea Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 12 Mar 2024 14:32:50 +0100 Subject: [PATCH 02/12] try installing nightly `pyarrow` [skip-ci] --- ci/install-upstream-wheels.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index 3bbecd38ba0..c8155486893 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -31,6 +31,15 @@ python -m pip install \ h5py \ matplotlib \ pandas +# for some reason pandas depends on pyarrow already. +# Remove once a `pyarrow` version compiled with `numpy>=2.0` is on `conda-forge` +python -m pip install \ + -i https://pypi.fury.io/arrow-nightlies/ \ + --prefer-binary \ + --no-deps \ + --pre \ + --upgrade \ + pyarrow # without build isolation for packages compiling against numpy # TODO: remove once there are `numpy>=2.0` builds for numcodecs and cftime python -m pip install \ From 899ae7f26a0c7c6ff6ff2a859310d23f2daebb5f Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 12 Mar 2024 14:38:57 +0100 Subject: [PATCH 03/12] use `micromamba` to speed up the force-removal of packages --- ci/install-upstream-wheels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index c8155486893..6b044561d0b 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -7,7 +7,7 @@ micromamba remove -y numba numbagg # temporarily remove backends micromamba remove -y cf_units netcdf4 # forcibly remove packages to avoid artifacts -conda uninstall -y --force \ +micromamba remove -y --force \ numpy \ scipy \ pandas \ From f14a69fecc7eb5c8164219b0e2c599542468a43e Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 12 Mar 2024 14:41:02 +0100 Subject: [PATCH 04/12] add a missing trailing backslash [skip-ci] --- ci/install-upstream-wheels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index 6b044561d0b..9eb1ee2f248 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -69,5 +69,5 @@ python -m pip install \ git+https://github.com/intake/filesystem_spec \ git+https://github.com/SciTools/nc-time-axis \ git+https://github.com/xarray-contrib/flox \ - git+https://github.com/dgasmith/opt_einsum + git+https://github.com/dgasmith/opt_einsum \ git+https://github.com/h5netcdf/h5netcdf From 126820ee94b7c74657db49070b5f0a6121ccfe85 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 12 Mar 2024 14:46:49 +0100 Subject: [PATCH 05/12] revert the enabling of upstream-dev h5netcdf [skip-ci] --- ci/install-upstream-wheels.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index 9eb1ee2f248..b86dee26a22 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -5,7 +5,7 @@ micromamba install "cython>=0.29.20" py-cpuinfo # temporarily (?) remove numbagg and numba micromamba remove -y numba numbagg # temporarily remove backends -micromamba remove -y cf_units netcdf4 +micromamba remove -y cf_units hdf5 h5py netcdf4 # forcibly remove packages to avoid artifacts micromamba remove -y --force \ numpy \ @@ -69,5 +69,5 @@ python -m pip install \ git+https://github.com/intake/filesystem_spec \ git+https://github.com/SciTools/nc-time-axis \ git+https://github.com/xarray-contrib/flox \ - git+https://github.com/dgasmith/opt_einsum \ - git+https://github.com/h5netcdf/h5netcdf + git+https://github.com/dgasmith/opt_einsum + # git+https://github.com/h5netcdf/h5netcdf From 4b28dc1dac9cc0e1cc0bacd4b8b57a0d117a1ded Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 12 Mar 2024 14:48:57 +0100 Subject: [PATCH 06/12] also build `numexpr` --- ci/install-upstream-wheels.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index b86dee26a22..c4f2823b835 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -41,7 +41,7 @@ python -m pip install \ --upgrade \ pyarrow # without build isolation for packages compiling against numpy -# TODO: remove once there are `numpy>=2.0` builds for numcodecs and cftime +# TODO: remove once there are `numpy>=2.0` builds for these python -m pip install \ --no-deps \ --upgrade \ @@ -52,6 +52,11 @@ python -m pip install \ --upgrade \ --no-build-isolation \ git+https://github.com/zarr-developers/numcodecs +python -m pip install \ + --no-deps \ + --upgrade \ + --no-build-isolation \ + git+https://github.com/pydata/numexpr python -m pip install \ --no-deps \ --upgrade \ From 9f6f765723dfe43beba0e5ae4947a64fcc9c1310 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 12 Mar 2024 15:23:09 +0100 Subject: [PATCH 07/12] replace the removed `numpy.array_api` with `array-api-strict` --- ci/requirements/all-but-dask.yml | 1 + ci/requirements/environment-3.12.yml | 1 + ci/requirements/environment-windows-3.12.yml | 1 + ci/requirements/environment-windows.yml | 1 + ci/requirements/environment.yml | 1 + ci/requirements/min-all-deps.yml | 1 + xarray/tests/test_array_api.py | 8 ++------ 7 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ci/requirements/all-but-dask.yml b/ci/requirements/all-but-dask.yml index c16c174ff96..2f47643cc87 100644 --- a/ci/requirements/all-but-dask.yml +++ b/ci/requirements/all-but-dask.yml @@ -5,6 +5,7 @@ channels: dependencies: - black - aiobotocore + - array-api-strict - boto3 - bottleneck - cartopy diff --git a/ci/requirements/environment-3.12.yml b/ci/requirements/environment-3.12.yml index 805f71c0023..99b260a69a3 100644 --- a/ci/requirements/environment-3.12.yml +++ b/ci/requirements/environment-3.12.yml @@ -4,6 +4,7 @@ channels: - nodefaults dependencies: - aiobotocore + - array-api-strict - boto3 - bottleneck - cartopy diff --git a/ci/requirements/environment-windows-3.12.yml b/ci/requirements/environment-windows-3.12.yml index a1c78c64bbb..5b3034b7a20 100644 --- a/ci/requirements/environment-windows-3.12.yml +++ b/ci/requirements/environment-windows-3.12.yml @@ -2,6 +2,7 @@ name: xarray-tests channels: - conda-forge dependencies: + - array-api-strict - boto3 - bottleneck - cartopy diff --git a/ci/requirements/environment-windows.yml b/ci/requirements/environment-windows.yml index 1fd2cf54dba..cc361bac5e9 100644 --- a/ci/requirements/environment-windows.yml +++ b/ci/requirements/environment-windows.yml @@ -2,6 +2,7 @@ name: xarray-tests channels: - conda-forge dependencies: + - array-api-strict - boto3 - bottleneck - cartopy diff --git a/ci/requirements/environment.yml b/ci/requirements/environment.yml index c96f3fd717b..d3dbc088867 100644 --- a/ci/requirements/environment.yml +++ b/ci/requirements/environment.yml @@ -4,6 +4,7 @@ channels: - nodefaults dependencies: - aiobotocore + - array-api-strict - boto3 - bottleneck - cartopy diff --git a/ci/requirements/min-all-deps.yml b/ci/requirements/min-all-deps.yml index 775c98b83b7..d2965fb3fc5 100644 --- a/ci/requirements/min-all-deps.yml +++ b/ci/requirements/min-all-deps.yml @@ -8,6 +8,7 @@ dependencies: # When upgrading python, numpy, or pandas, must also change # doc/user-guide/installing.rst, doc/user-guide/plotting.rst and setup.py. - python=3.9 + - array-api-strict=1.0 # dependency for testing the array api compat - boto3=1.24 - bottleneck=1.3 - cartopy=0.21 diff --git a/xarray/tests/test_array_api.py b/xarray/tests/test_array_api.py index 03dcfd9b20f..9d6b183a880 100644 --- a/xarray/tests/test_array_api.py +++ b/xarray/tests/test_array_api.py @@ -1,18 +1,14 @@ from __future__ import annotations -import warnings - import pytest import xarray as xr from xarray.testing import assert_equal np = pytest.importorskip("numpy", minversion="1.22") +xp = pytest.importorskip("array_api_strict") -with warnings.catch_warnings(): - warnings.simplefilter("ignore") - import numpy.array_api as xp # isort:skip - from numpy.array_api._array_object import Array # isort:skip +from array_api_strict._array_object import Array # isort:skip @pytest.fixture From d5344a7749b564564c1d1e67813f64da50d8cb9e Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 12 Mar 2024 15:27:43 +0100 Subject: [PATCH 08/12] don't install upstream-dev `h5py` --- ci/install-upstream-wheels.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index c4f2823b835..511f072cee1 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -28,7 +28,6 @@ python -m pip install \ --upgrade \ numpy \ scipy \ - h5py \ matplotlib \ pandas # for some reason pandas depends on pyarrow already. From 0c23613aade3753febcea246290cb7465da738f5 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 12 Mar 2024 15:31:46 +0100 Subject: [PATCH 09/12] remove `numexpr` from the environment instead [skip-ci] --- ci/install-upstream-wheels.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index 511f072cee1..68e841b4181 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -4,6 +4,8 @@ micromamba install "cython>=0.29.20" py-cpuinfo # temporarily (?) remove numbagg and numba micromamba remove -y numba numbagg +# temporarily remove numexpr +micromamba remove -y numexpr # temporarily remove backends micromamba remove -y cf_units hdf5 h5py netcdf4 # forcibly remove packages to avoid artifacts @@ -51,11 +53,6 @@ python -m pip install \ --upgrade \ --no-build-isolation \ git+https://github.com/zarr-developers/numcodecs -python -m pip install \ - --no-deps \ - --upgrade \ - --no-build-isolation \ - git+https://github.com/pydata/numexpr python -m pip install \ --no-deps \ --upgrade \ From 8efb1c80a2448ae28da71a67fa0b39e929c66fe2 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 12 Mar 2024 15:43:33 +0100 Subject: [PATCH 10/12] ignore the missing typing in `array_api_strict` --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4b5f6b31a43..d2a5c6b8748 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,6 +138,7 @@ module = [ "toolz.*", "zarr.*", "numpy.exceptions.*", # remove once support for `numpy<2.0` has been dropped + "array_api_strict.*", ] # Gradually we want to add more modules to this list, ratcheting up our total From 2be5514067b319af26bfc13d847ea92ab9f2c048 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 12 Mar 2024 15:56:45 +0100 Subject: [PATCH 11/12] restore the old `numpy.array_api` import for `numpy<2` --- xarray/tests/test_array_api.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_array_api.py b/xarray/tests/test_array_api.py index 9d6b183a880..506019bee97 100644 --- a/xarray/tests/test_array_api.py +++ b/xarray/tests/test_array_api.py @@ -6,9 +6,20 @@ from xarray.testing import assert_equal np = pytest.importorskip("numpy", minversion="1.22") -xp = pytest.importorskip("array_api_strict") -from array_api_strict._array_object import Array # isort:skip +try: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + + import numpy.array_api as xp # isort:skip + from numpy.array_api._array_object import Array # isort:skip +except ImportError: + # for `numpy>=2.0` + xp = pytest.importorskip("array_api_strict") + + from array_api_strict._array_object import Array # isort:skip @pytest.fixture From 705ab7767a0b22307dc1534ac04503c18cc65dc4 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 12 Mar 2024 16:19:11 +0100 Subject: [PATCH 12/12] ignore the redefinition of `Array` --- xarray/tests/test_array_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/tests/test_array_api.py b/xarray/tests/test_array_api.py index 506019bee97..a5ffb37a109 100644 --- a/xarray/tests/test_array_api.py +++ b/xarray/tests/test_array_api.py @@ -13,13 +13,13 @@ with warnings.catch_warnings(): warnings.simplefilter("ignore") - import numpy.array_api as xp # isort:skip - from numpy.array_api._array_object import Array # isort:skip + import numpy.array_api as xp + from numpy.array_api._array_object import Array except ImportError: # for `numpy>=2.0` xp = pytest.importorskip("array_api_strict") - from array_api_strict._array_object import Array # isort:skip + from array_api_strict._array_object import Array # type: ignore[no-redef] @pytest.fixture