diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8d43de7b1d5..1da9a06b1f3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,6 +20,8 @@ jobs: conda_env: py37 py38: conda_env: py38 + py38-all-but-dask: + conda_env: py38-all-but-dask py38-upstream-dev: conda_env: py38 upstream_dev: true diff --git a/ci/requirements/py38-all-but-dask.yml b/ci/requirements/py38-all-but-dask.yml new file mode 100644 index 00000000000..a375d9e1e5a --- /dev/null +++ b/ci/requirements/py38-all-but-dask.yml @@ -0,0 +1,44 @@ +name: xarray-tests +channels: + - conda-forge +dependencies: + - python=3.8 + - black + - boto3 + - bottleneck + - cartopy + - cdms2 + - cfgrib + - cftime + - coveralls + - flake8 + - h5netcdf + - h5py + - hdf5 + - hypothesis + - isort + - lxml # Optional dep of pydap + - matplotlib + - mypy=0.761 # Must match .pre-commit-config.yaml + - nc-time-axis + - netcdf4 + - numba + - numpy + - pandas + - pint + - pip + - pseudonetcdf + - pydap + - pynio + - pytest + - pytest-cov + - pytest-env + - rasterio + - scipy + - seaborn + - setuptools + - sparse + - toolz + - zarr + - pip: + - numbagg diff --git a/doc/whats-new.rst b/doc/whats-new.rst index c70dfd4f3f6..f811d996076 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -63,6 +63,9 @@ Internal Changes - Run the ``isort`` pre-commit hook only on python source files and update the ``flake8`` version. (:issue:`3750`, :pull:`3711`) By `Justus Magin `_. +- Add a CI job that runs the tests with every optional dependency + except ``dask``. (:issue:`3794`, :pull:`3919`) + By `Justus Magin `_. .. _whats-new.0.15.1: diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index cdc74e06882..c262dae2811 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -56,7 +56,7 @@ def __getitem__(self, key): return array[key.tuple] elif isinstance(key, indexing.VectorizedIndexer): return array.vindex[ - indexing._arrayize_vectorized_indexer(key.tuple, self.shape).tuple + indexing._arrayize_vectorized_indexer(key, self.shape).tuple ] else: assert isinstance(key, indexing.OuterIndexer) @@ -586,6 +586,12 @@ def open_zarr( "Instead found %s. " % chunks ) + if chunks == "auto": + try: + import dask.array # noqa + except ImportError: + chunks = None + if not decode_cf: mask_and_scale = False decode_times = False diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 82fe1b38149..a611e05599f 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -581,6 +581,10 @@ def test_orthogonal_indexing(self): actual = on_disk.isel(**indexers) assert_identical(expected, actual) + @pytest.mark.xfail( + not has_dask, + reason="the code for indexing without dask handles negative steps in slices incorrectly", + ) def test_vectorized_indexing(self): in_memory = create_test_data() with self.roundtrip(in_memory) as on_disk: @@ -1539,6 +1543,7 @@ def test_roundtrip_consolidated(self): self.check_dtypes_roundtripped(expected, actual) assert_identical(expected, actual) + @requires_dask def test_auto_chunk(self): original = create_test_data().chunk() @@ -1556,6 +1561,7 @@ def test_auto_chunk(self): # chunk size should be the same as original assert v.chunks == original[k].chunks + @requires_dask @pytest.mark.filterwarnings("ignore:Specified Dask chunks") def test_manual_chunk(self): original = create_test_data().chunk({"dim1": 3, "dim2": 4, "dim3": 3}) @@ -1598,6 +1604,7 @@ def test_manual_chunk(self): assert_identical(actual, auto) assert_identical(actual.load(), auto.load()) + @requires_dask def test_warning_on_bad_chunks(self): original = create_test_data().chunk({"dim1": 4, "dim2": 3, "dim3": 5}) @@ -1620,6 +1627,7 @@ def test_warning_on_bad_chunks(self): assert v._in_memory == (k in actual.dims) assert len(record) == 0 + @requires_dask def test_deprecate_auto_chunk(self): original = create_test_data().chunk() with pytest.warns(FutureWarning): @@ -1638,6 +1646,7 @@ def test_deprecate_auto_chunk(self): # there should be no chunks assert v.chunks is None + @requires_dask def test_write_uneven_dask_chunks(self): # regression for GH#2225 original = create_test_data().chunk({"dim1": 3, "dim2": 4, "dim3": 3}) @@ -1662,6 +1671,7 @@ def test_chunk_encoding(self): with self.roundtrip(data) as actual: pass + @requires_dask def test_chunk_encoding_with_dask(self): # These datasets DO have dask chunks. Need to check for various # interactions between dask and zarr chunks @@ -1896,6 +1906,7 @@ def test_append_with_new_variable(self): combined["new_var"] = ds_with_new_var["new_var"] assert_identical(combined, xr.open_zarr(store_target)) + @requires_dask def test_to_zarr_compute_false_roundtrip(self): from dask.delayed import Delayed @@ -1915,6 +1926,7 @@ def test_to_zarr_compute_false_roundtrip(self): with self.open(store) as actual: assert_identical(original, actual) + @requires_dask def test_to_zarr_append_compute_false_roundtrip(self): from dask.delayed import Delayed @@ -1951,6 +1963,7 @@ def test_to_zarr_append_compute_false_roundtrip(self): with self.open(store) as actual: assert_identical(xr.concat([ds, ds_to_append], dim="time"), actual) + @requires_dask def test_encoding_chunksizes(self): # regression test for GH2278 # see also test_encoding_chunksizes_unlimited @@ -3513,6 +3526,7 @@ def test_uamiv_format_read(self): assert_allclose(expected, actual) camxfile.close() + @requires_dask def test_uamiv_format_mfread(self): """ Open a CAMx file and test data variables @@ -3939,6 +3953,9 @@ def test_chunks(self): ex = expected.sel(band=1).mean(dim="x") assert_allclose(ac, ex) + @pytest.mark.xfail( + not has_dask, reason="without dask, a non-serializable lock is used" + ) def test_pickle_rasterio(self): # regression test for https://github.com/pydata/xarray/issues/2121 with create_tmp_geotiff() as (tmp_file, expected): @@ -4012,6 +4029,7 @@ def test_geotiff_tags(self): with xr.open_rasterio(tmp_file) as rioda: assert isinstance(rioda.attrs["AREA_OR_POINT"], str) + @requires_dask def test_no_mftime(self): # rasterio can accept "filename" urguments that are actually urls, # including paths to remote files. diff --git a/xarray/tests/test_sparse.py b/xarray/tests/test_sparse.py index 09ab1be9af9..f3c09ba6a5f 100644 --- a/xarray/tests/test_sparse.py +++ b/xarray/tests/test_sparse.py @@ -837,6 +837,7 @@ def test_sparse_coords(self): ) +@requires_dask def test_chunk(): s = sparse.COO.from_numpy(np.array([0, 0, 1, 2])) a = DataArray(s)