numcodecs.zfpy is ready

.github/workflows/ci-osx.yaml

@@ -46,3 +46,4 @@ jobs:
         run: |
           conda activate env
           pytest -v --pyargs numcodecs
+

docs/conf.py

@@ -24,7 +24,7 @@ def __getattr__(cls, name):
         return Mock()
 
 
-MOCK_MODULES = ['msgpack']
+MOCK_MODULES = ['msgpack', 'zfpy']
 sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
 

docs/index.rst

@@ -62,6 +62,7 @@ Contents
     registry
     blosc
     lz4
+    zfpy
     zstd
     zlib
     gzip

docs/release.rst

@@ -1,6 +1,14 @@
 Release notes
 =============
 
+
+Upcoming Release
+----------------
+
+* The :class:`numcodecs.zfpy.ZFPY` codec is now supported on Python 3.8 if
+  `zfpy==0.5.5 <https://pypi.org/project/zfpy/>`_ is installed
+
+
 .. _unreleased:
 
 Unreleased
@@ -15,6 +23,7 @@ Unreleased
   By :user:`Jackson Maxfield Brown <JacksonMaxfield>`, :issue:`276`.
   Help from :user:`Oleg Höfling <hoefling>`, :issue:`273`.
 
+
 .. _release_0.7.3:
 
 0.7.3
@@ -43,6 +52,7 @@ Unreleased
 * Update docs regarding wheels.
   By :user:`Josh Moore <joshmoore>`, :issue:`250`.
 
+
 .. _release_0.7.1:
 
 0.7.1

docs/zfpy.rst

@@ -0,0 +1,11 @@
+ZFPY
+====
+.. automodule:: numcodecs.zfpy
+
+.. autoclass:: ZFPY
+
+    .. autoattribute:: codec_id
+    .. automethod:: encode
+    .. automethod:: decode
+    .. automethod:: get_config
+    .. automethod:: from_config

fixture/zfpy/codec.00/config.json

@@ -0,0 +1,10 @@
+{
+    "compression_kwargs": {
+        "rate": -1
+    },
+    "id": "zfpy",
+    "mode": 2,
+    "precision": -1,
+    "rate": -1,
+    "tolerance": -1
+}

fixture/zfpy/codec.01/config.json

@@ -0,0 +1,10 @@
+{
+    "compression_kwargs": {
+        "tolerance": -1
+    },
+    "id": "zfpy",
+    "mode": 4,
+    "precision": -1,
+    "rate": -1,
+    "tolerance": -1
+}

fixture/zfpy/codec.02/config.json

@@ -0,0 +1,10 @@
+{
+    "compression_kwargs": {
+        "tolerance": -1
+    },
+    "id": "zfpy",
+    "mode": 4,
+    "precision": -1,
+    "rate": -1,
+    "tolerance": -1
+}

fixture/zfpy/codec.03/config.json

@@ -0,0 +1,10 @@
+{
+    "compression_kwargs": {
+        "precision": -1
+    },
+    "id": "zfpy",
+    "mode": 3,
+    "precision": -1,
+    "rate": -1,
+    "tolerance": -1
+}

numcodecs/__init__.py

@@ -3,7 +3,7 @@
 transformation codecs for use in data storage and communication
 applications. These include:
 
-* Compression codecs, e.g., Zlib, BZ2, LZMA and Blosc.
+* Compression codecs, e.g., Zlib, BZ2, LZMA, ZFPY and Blosc.
 * Pre-compression filters, e.g., Delta, Quantize, FixedScaleOffset,
   PackBits, Categorize.
 * Integrity checks, e.g., CRC32, Adler32.
@@ -16,7 +16,6 @@
 <https://github.com/alimanfoo/numcodecs/issues>`_.
 
 """
-
 import multiprocessing
 import atexit
 
@@ -68,6 +67,12 @@
 except ImportError:  # pragma: no cover
     pass
 
+try:
+    from numcodecs.zfpy import ZFPY
+    register_codec(ZFPY)
+except ImportError:  # pragma: no cover
+    pass
+
 from numcodecs.astype import AsType
 register_codec(AsType)
 

numcodecs/tests/common.py

@@ -268,12 +268,10 @@ def check_backwards_compatibility(codec_id, arrays, codecs, precision=None, pref
 
             # file with codec configuration information
             codec_fn = os.path.join(codec_dir, 'config.json')
-
             # one time save config
             if not os.path.exists(codec_fn):  # pragma: no cover
                 with open(codec_fn, mode='w') as cf:
                     _json.dump(codec.get_config(), cf, sort_keys=True, indent=4)
-
             # load config and compare with expectation
             with open(codec_fn, mode='r') as cf:
                 config = _json.load(cf)

numcodecs/tests/test_zfpy.py

@@ -0,0 +1,86 @@
+import pytest
+
+
+import numpy as np
+
+
+try:
+    # noinspection PyProtectedMember
+    from numcodecs.zfpy import ZFPY, _zfpy
+except ImportError:  # pragma: no cover
+    pytest.skip("ZFPY not available", allow_module_level=True)
+
+
+from numcodecs.tests.common import (
+    check_encode_decode_array,
+    check_config,
+    check_repr,
+    check_backwards_compatibility,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+)
+
+
+codecs = [
+    ZFPY(mode=_zfpy.mode_fixed_rate, rate=-1),
+    ZFPY(),
+    ZFPY(mode=_zfpy.mode_fixed_accuracy, tolerance=-1),
+    ZFPY(mode=_zfpy.mode_fixed_precision, precision=-1),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.linspace(1000, 1001, 1000, dtype="f4"),
+    np.linspace(1000, 1001, 1000, dtype="f8"),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.normal(loc=1000, scale=1, size=(10, 10, 10)),
+    np.random.normal(loc=1000, scale=1, size=(2, 5, 10, 10)),
+    np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(5, 10, 20))),
+    np.random.randint(-(2 ** 31), -(2 ** 31) + 20, size=1000, dtype="i4").reshape(
+        100, 10
+    ),
+    np.random.randint(-(2 ** 63), -(2 ** 63) + 20, size=1000, dtype="i8").reshape(
+        10, 10, 10
+    ),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        if arr.dtype == np.int32 or arr.dtype == np.int64:
+            codec = [codecs[-1]]
+        else:
+            codec = codecs
+        for code in codec:
+            check_encode_decode_array(arr, code)
+
+
+def test_config():
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("ZFPY(mode=4, tolerance=0.001, rate=-1, precision=-1)")
+
+
+def test_backwards_compatibility():
+    for i, code in enumerate(codecs):
+        if code.mode == _zfpy.mode_fixed_rate:
+            codec = [code]
+            check_backwards_compatibility(ZFPY.codec_id, arrays, codec)
+        else:
+            check_backwards_compatibility(
+                ZFPY.codec_id, arrays[: len(arrays) - 2], codecs
+            )
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(ZFPY())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(ZFPY())

numcodecs/zfpy.py

@@ -0,0 +1,89 @@
+_zfpy = None
+try:
+    import zfpy as _zfpy
+except ImportError:  # pragma: no cover
+    pass
+
+
+if _zfpy:
+
+    from .abc import Codec
+    from .compat import ndarray_copy, ensure_contiguous_ndarray, ensure_bytes
+
+    # noinspection PyShadowingBuiltins
+    class ZFPY(Codec):
+        """Codec providing compression using zfpy via the Python standard
+        library.
+
+        Parameters
+        ----------
+        mode : integer
+            One of the zfpy mode choice, e.g., ``zfpy.mode_fixed_accuracy``.
+        tolerance : double, optional
+            A double-precision number, specifying the compression accuracy needed.
+        rate : double, optional
+            A double-precision number, specifying the compression rate needed.
+        precision : int, optional
+            A integer number, specifying the compression precision needed.
+
+        """
+
+        codec_id = "zfpy"
+
+        def __init__(
+            self,
+            mode=_zfpy.mode_fixed_accuracy,
+            tolerance=-1,
+            rate=-1,
+            precision=-1,
+            compression_kwargs=None,
+        ):
+            self.mode = mode
+            if mode == _zfpy.mode_fixed_accuracy:
+                self.compression_kwargs = {"tolerance": tolerance}
+            elif mode == _zfpy.mode_fixed_rate:
+                self.compression_kwargs = {"rate": rate}
+            elif mode == _zfpy.mode_fixed_precision:
+                self.compression_kwargs = {"precision": precision}
+            else:
+                pass
+
+            self.tolerance = tolerance
+            self.rate = rate
+            self.precision = precision
+
+        def encode(self, buf):
+
+            # normalise inputs
+            buf = ensure_contiguous_ndarray(buf)
+
+            # do compression
+            return _zfpy.compress_numpy(
+                buf, write_header=True, **self.compression_kwargs
+            )
+
+        def decode(self, buf, out=None):
+
+            # normalise inputs
+            buf = ensure_bytes(buf)
+            if out is not None:
+                out = ensure_contiguous_ndarray(out)
+
+            # do decompression
+            dec = _zfpy.decompress_numpy(buf)
+
+            # handle destination
+            if out is not None:
+                return ndarray_copy(dec, out)
+            else:
+                return dec
+
+        def __repr__(self):
+            r = "%s(mode=%r, tolerance=%s, rate=%s, precision=%s)" % (
+                type(self).__name__,
+                self.mode,
+                self.tolerance,
+                self.rate,
+                self.precision,
+            )
+            return r

requirements_dev.txt

@@ -1,3 +1,5 @@
 Cython==0.29.21
 msgpack==1.0.2
 numpy==1.19.0
+zfpy==0.5.5; python_version < '3.9'
+

requirements_rtfd.txt

@@ -6,3 +6,4 @@ numpydoc
 mock
 numpy
 cython
+zfpy==0.5.5; python_version < '3.9'

setup.py

@@ -76,6 +76,8 @@ def blosc_extension():
                      if os.path.isdir(d)]
     include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*')
                      if os.path.isdir(d)]
+    include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*/*')
+                     if os.path.isdir(d)]
     define_macros += [('HAVE_LZ4', 1),
                       ('HAVE_SNAPPY', 1),
                       ('HAVE_ZLIB', 1),
@@ -315,6 +317,7 @@ def run_setup(with_extensions):
     if with_extensions:
         ext_modules = (blosc_extension() + zstd_extension() + lz4_extension() +
                        compat_extension() + shuffle_extension() + vlen_extension())
+
         cmdclass = dict(build_ext=ve_build_ext)
     else:
         ext_modules = []