Merge remote-tracking branch 'upstream/master' into v3-store-part1

Author: grlee77

.github/workflows/minimal.yml

@@ -26,3 +26,10 @@ jobs:
         conda activate minimal
         python -m pip install .
         pytest -svx
+    - name: Fixture generation
+      shell: "bash -l {0}"
+      run: |
+        conda activate minimal
+        rm -rf fixture/
+        pytest -svx zarr/tests/test_dim_separator.py zarr/tests/test_storage.py
+        # This simulates fixture-less tests in conda and debian packaging

docs/contributing.rst

@@ -152,7 +152,7 @@ the tests will be skipped. To install all optional dependencies, run::
     $ pip install -r requirements_dev_optional.txt
 
 To also run the doctests within docstrings (requires optional
-depencies to be installed), run::
+dependencies to be installed), run::
 
     $ pytest -v --doctest-plus zarr
 

docs/release.rst

@@ -9,6 +9,8 @@ Unreleased
 Enhancements
 ~~~~~~~~~~~~
 
+* Allow to assign array ``fill_values`` and update metadata accordingly. :issue:`662`
+
 * array indexing with [] (getitem and setitem) now supports fancy indexing.
   By :user:`Juan Nunez-Iglesias <jni>`; :issue:`725`.
 
@@ -21,12 +23,26 @@ Enhancements
 * Create a Base store class for Zarr Store.
   By :user:`Greggory Lee <grlee77>`; :issue:`789`.
 
+.. _release_2.10.3:
+
+2.10.3
+------
+
 Bug fixes
 ~~~~~~~~~
 
 * N5 keywords now emit UserWarning instead of raising a ValueError.
   By :user:`Boaz Mohar <boazmohar>`; :issue:`860`.
 
+* blocks_to_decompress not used in read_part function.
+  By :user:`Boaz Mohar <boazmohar>`; :issue:`861`.
+
+* defines blocksize for array, updates hexdigest values.
+  By :user:`Andrew Fulton <andrewfulton9>`; :issue:`867`.
+
+* Fix test failure on Debian and conda-forge builds.
+  By :user:`Josh Moore <joshmoore>`; :issue:`871`.
+
 .. _release_2.10.2:
 
 2.10.2
@@ -328,7 +344,7 @@ See `this link <https://github.com/zarr-developers/zarr-python/milestone/11?clos
 merged PR tagged with the 2.6 milestone.
 
 * Add ability to partially read and decompress arrays, see :issue:`667`. It is
-  only available to chunks stored using fs-spec and using bloc as a compressor.
+  only available to chunks stored using fsspec and using Blosc as a compressor.
 
   For certain analysis case when only a small portion of chunks is needed it can
   be advantageous to only access and decompress part of the chunks. Doing
@@ -359,7 +375,7 @@ This release will be the last to support Python 3.5, next version of Zarr will b
   without ``ipytree`` installed.
   By :user:`Zain Patel <mzjp2>`; :issue:`537`
 
-* Add typing informations to many of the core functions :issue:`589`
+* Add typing information to many of the core functions :issue:`589`
 
 * Explicitly close stores during testing.
   By :user:`Elliott Sales de Andrade <QuLogic>`; :issue:`442`

docs/spec/v1.rst

@@ -150,7 +150,7 @@ and columns 4000-5000 and is stored under the key '2.4'; etc.
 
 There is no need for all chunks to be present within an array
 store. If a chunk is not present then it is considered to be in an
-uninitialized state.  An unitialized chunk MUST be treated as if it
+uninitialized state.  An uninitialized chunk MUST be treated as if it
 was uniformly filled with the value of the 'fill_value' field in the
 array metadata. If the 'fill_value' field is ``null`` then the
 contents of the chunk are undefined.

docs/spec/v2.rst

@@ -81,7 +81,7 @@ filters
 The following keys MAY be present within the object:
 
 dimension_separator
-    If present, either the string ``"."`` or ``"/""`` definining the separator placed
+    If present, either the string ``"."`` or ``"/""`` defining the separator placed
     between the dimensions of a chunk. If the value is not set, then the
     default MUST be assumed to be ``"."``, leading to chunk keys of the form "0.0".
     Arrays defined with ``"/"`` as the dimension separator can be considered to have
@@ -222,7 +222,7 @@ columns 4000-5000 and is stored under the key "2.4"; etc.
 
 There is no need for all chunks to be present within an array store. If a chunk
 is not present then it is considered to be in an uninitialized state.  An
-unitialized chunk MUST be treated as if it was uniformly filled with the value
+uninitialized chunk MUST be treated as if it was uniformly filled with the value
 of the "fill_value" field in the array metadata. If the "fill_value" field is
 ``null`` then the contents of the chunk are undefined.
 

docs/tutorial.rst

@@ -1297,7 +1297,7 @@ ratios, depending on the correlation structure within the data. E.g.::
     Chunks initialized : 100/100
 
 In the above example, Fortran order gives a better compression ratio. This is an
-artifical example but illustrates the general point that changing the order of
+artificial example but illustrates the general point that changing the order of
 bytes within chunks of an array may improve the compression ratio, depending on
 the structure of the data, the compression algorithm used, and which compression
 filters (e.g., byte-shuffle) have been applied.

requirements_dev_optional.txt

@@ -7,7 +7,7 @@ ipytree==0.2.1
 # don't let pyup change pinning for azure-storage-blob, need to pin to older
 # version to get compatibility with azure storage emulator on appveyor (FIXME)
 azure-storage-blob==12.8.1 # pyup: ignore
-redis==4.0.0
+redis==4.0.2
 types-redis
 types-setuptools
 pymongo==3.12.1
@@ -16,8 +16,8 @@ tox==3.24.4
 coverage
 flake8==4.0.1
 pytest-cov==3.0.0
-pytest-doctestplus==0.11.0
+pytest-doctestplus==0.11.1
 pytest-timeout==2.0.1
-h5py==3.5.0
-fsspec[s3]==2021.11.0
+h5py==3.6.0
+fsspec[s3]==2021.11.1
 moto[server]>=1.3.14

setup.cfg

@@ -0,0 +1,3 @@
+[codespell]
+skip = ./.git
+ignore-words-list = ba, ihs, kake, nd, noe, nwo, te

zarr/_storage/absstore.py

@@ -76,7 +76,8 @@ def __init__(self, container=None, prefix='', account_name=None, account_key=Non
         self._account_name = account_name
         self._account_key = account_key
 
-    def _warn_deprecated(self, property_):
+    @staticmethod
+    def _warn_deprecated(property_):
         msg = ("The {} property is deprecated and will be removed in a future "
                "version. Get the property from 'ABSStore.client' instead.")
         warnings.warn(msg.format(property_), FutureWarning, stacklevel=3)

zarr/convenience.py

@@ -428,7 +428,7 @@ def tree(grp, expand=False, level=None):
     return TreeViewer(grp, expand=expand, level=level)
 
 
-class _LogWriter(object):
+class _LogWriter:
 
     def __init__(self, log):
         self.log_func = None

zarr/core.py

@@ -74,7 +74,7 @@ class Array:
         operations. If False, user attributes are reloaded from the store prior
         to all attribute read operations.
     partial_decompress : bool, optional
-        If True and while the chunk_store is a FSStore and the compresion used
+        If True and while the chunk_store is a FSStore and the compression used
         is Blosc, when getting data from the array chunks will be partially
         read and decompressed when possible.
 
@@ -347,6 +347,11 @@ def fill_value(self):
         """A value used for uninitialized portions of the array."""
         return self._fill_value
 
+    @fill_value.setter
+    def fill_value(self, new):
+        self._fill_value = new
+        self._flush_metadata_nosync()
+
     @property
     def order(self):
         """A string indicating the order in which bytes are arranged within
@@ -454,7 +459,7 @@ def nchunks_initialized(self):
         # count chunk keys
         return sum(1 for k in listdir(self.chunk_store, self._path) if prog.match(k))
 
-    # backwards compability
+    # backwards compatibility
     initialized = nchunks_initialized
 
     @property
@@ -1103,7 +1108,7 @@ def get_mask_selection(self, selection, out=None, fields=None):
             >>> import numpy as np
             >>> z = zarr.array(np.arange(100).reshape(10, 10))
 
-        Retrieve items by specifying a maks::
+        Retrieve items by specifying a mask::
 
             >>> sel = np.zeros_like(z, dtype=bool)
             >>> sel[1, 1] = True
@@ -1953,17 +1958,15 @@ def _chunk_delitems(self, ckeys):
             # that will trigger this condition, but it's possible that they
             # will be developed in the future.
             tuple(map(self._chunk_delitem, ckeys))
-        return None
 
     def _chunk_delitem(self, ckey):
         """
         Attempt to delete the value associated with ckey.
         """
         try:
             del self.chunk_store[ckey]
-            return
         except KeyError:
-            return
+            pass
 
     def _chunk_setitem(self, chunk_coords, chunk_selection, value, fields=None):
         """Replace part or whole of a chunk.
@@ -2065,7 +2068,7 @@ def _decode_chunk(self, cdata, start=None, nitems=None, expected_shape=None):
         if self._compressor:
             # only decode requested items
             if (
-                all([x is not None for x in [start, nitems]])
+                all(x is not None for x in [start, nitems])
                 and self._compressor.codec_id == "blosc"
             ) and hasattr(self._compressor, "decode_partial"):
                 chunk = self._compressor.decode_partial(cdata, start, nitems)

zarr/creation.py

@@ -136,7 +136,7 @@ def create(shape, chunks=True, dtype=None, compressor='default',
         store_separator = getattr(store, "_dimension_separator", None)
         if store_separator not in (None, dimension_separator):
             raise ValueError(
-                f"Specified dimension_separtor: {dimension_separator}"
+                f"Specified dimension_separator: {dimension_separator}"
                 f"conflicts with store's separator: "
                 f"{store_separator}")
     dimension_separator = normalize_dimension_separator(dimension_separator)
@@ -439,7 +439,7 @@ def open_array(
         If using an fsspec URL to create the store, these will be passed to
         the backend implementation. Ignored otherwise.
     partial_decompress : bool, optional
-        If True and while the chunk_store is a FSStore and the compresion used
+        If True and while the chunk_store is a FSStore and the compression used
         is Blosc, when getting data from the array chunks will be partially
         read and decompressed when possible.
     write_empty_chunks : bool, optional

zarr/indexing.py

@@ -132,7 +132,7 @@ def normalize_integer_selection(dim_sel, dim_len):
 """
 
 
-class IntDimIndexer(object):
+class IntDimIndexer:
 
     def __init__(self, dim_sel, dim_len, dim_chunk_len):
 
@@ -157,7 +157,7 @@ def ceildiv(a, b):
     return math.ceil(a / b)
 
 
-class SliceDimIndexer(object):
+class SliceDimIndexer:
 
     def __init__(self, dim_sel, dim_len, dim_chunk_len):
 
@@ -308,19 +308,19 @@ def is_positive_slice(s):
 
 def is_contiguous_selection(selection):
     selection = ensure_tuple(selection)
-    return all([
+    return all(
         (is_integer_array(s) or is_contiguous_slice(s) or s == Ellipsis)
         for s in selection
-    ])
+    )
 
 
 def is_basic_selection(selection):
     selection = ensure_tuple(selection)
-    return all([is_integer(s) or is_positive_slice(s) for s in selection])
+    return all(is_integer(s) or is_positive_slice(s) for s in selection)
 
 
 # noinspection PyProtectedMember
-class BasicIndexer(object):
+class BasicIndexer:
 
     def __init__(self, selection, array):
 
@@ -361,7 +361,7 @@ def __iter__(self):
             yield ChunkProjection(chunk_coords, chunk_selection, out_selection)
 
 
-class BoolArrayDimIndexer(object):
+class BoolArrayDimIndexer:
 
     def __init__(self, dim_sel, dim_len, dim_chunk_len):
 
@@ -451,7 +451,7 @@ def boundscheck_indices(x, dim_len):
         raise BoundsCheckError(dim_len)
 
 
-class IntArrayDimIndexer(object):
+class IntArrayDimIndexer:
     """Integer array selection against a single dimension."""
 
     def __init__(self, dim_sel, dim_len, dim_chunk_len, wraparound=True, boundscheck=True,
@@ -579,7 +579,7 @@ def oindex_set(a, selection, value):
 
 
 # noinspection PyProtectedMember
-class OrthogonalIndexer(object):
+class OrthogonalIndexer:
 
     def __init__(self, selection, array):
 
@@ -649,7 +649,7 @@ def __iter__(self):
             yield ChunkProjection(chunk_coords, chunk_selection, out_selection)
 
 
-class OIndex(object):
+class OIndex:
 
     def __init__(self, array):
         self.array = array
@@ -671,8 +671,8 @@ def __setitem__(self, selection, value):
 def is_coordinate_selection(selection, array):
     return (
         (len(selection) == len(array._shape)) and
-        all([is_integer(dim_sel) or is_integer_array(dim_sel)
-             for dim_sel in selection])
+        all(is_integer(dim_sel) or is_integer_array(dim_sel)
+            for dim_sel in selection)
     )
 
 
@@ -686,7 +686,7 @@ def is_mask_selection(selection, array):
 
 
 # noinspection PyProtectedMember
-class CoordinateIndexer(object):
+class CoordinateIndexer:
 
     def __init__(self, selection, array):
 
@@ -805,7 +805,7 @@ def __init__(self, selection, array):
         super().__init__(selection, array)
 
 
-class VIndex(object):
+class VIndex:
 
     def __init__(self, array):
         self.array = array
@@ -905,7 +905,7 @@ def make_slice_selection(selection):
     return ls
 
 
-class PartialChunkIterator(object):
+class PartialChunkIterator:
     """Iterator to retrieve the specific coordinates of requested data
     from within a compressed chunk.