numcodecs.zfpy is ready (#229)

* added zfp compressor

* modified zfp in setup.py for new zfp054 directory

* changed print statement to raise,also changed zfp source path

* added set_stride, changed zfp source path

* added zfp source as a submodule

* removed zfp source as a submodule

* added zfp module

* added zfp docs

* used zfp submodule and define type by _type

* removed blanked lines

* added white space after mode

* modified comments to meet numpy docstring

* format setup.py

* added zfp.c

* add path to bitstream.h

* updated include path

* update zfp.c with include path

* changed include path again

* Removed zfp submodule

* Removed zfp_extension

* Remove test_zfp.py

* added zfp.py

* removed zfp.py

* added zfpy.py

* add pip install zfpy

* add pip install modified

* add pip install modified 2

* add pip install modified 3

* add zfpy.py

* add zfpy in requirements_dev

* add zfpy finxture

* add zfpy with index-url

* remove zfp.rst

* get 100% coverage

* change for coverage

* change for PEP 8

* add ensure_contiguous_ndarray(buf)

* skip tests at py38 and osx

* fixed format issue

* skip tests at py38

* remove variable comp_arr

* skip test on darwin

* remove extra blank line

* removed import sys

* removed zfp.pyx

* change zfpy repr

* change the zfpy repr test

* use zfpy at PyPI

* update zfpy in release.rst

* updating submodule to latest

* Revert "updating submodule to latest"

This reverts commit 0d5e320.

* Revert "Revert "updating submodule to latest""

This reverts commit 0eab0ff.

* Revert "Revert "Revert "updating submodule to latest"""

This reverts commit 2f4fe41.

* update c-blosc to 1.18.1

* Update release.rst

* Revert unrelated C file changes

* Use `pytest.skip` to skip testing

* Run `black` on zfpy

* Update requirements_rtfd.txt

* Update requirements_dev.txt

* Update requirements_dev.txt

* Update requirements_rtfd.txt

* Update requirements_rtfd.txt

* Update requirements_dev.txt

* Update conf.py

* Update conf.py

* Update conf.py

* Update conf.py

* Update conf.py

Co-authored-by: John Kirkham <[email protected]>

Author: halehawk

.github/workflows/ci-osx.yaml

@@ -46,3 +46,4 @@ jobs:
         run: |
           conda activate env
           pytest -v --pyargs numcodecs
+

docs/conf.py

@@ -24,7 +24,7 @@ def __getattr__(cls, name):
         return Mock()
 
 
-MOCK_MODULES = ['msgpack']
+MOCK_MODULES = ['msgpack', 'zfpy']
 sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
 

docs/index.rst

@@ -0,0 +1,10 @@
+{
+    "compression_kwargs": {
+        "rate": -1
+    },
+    "id": "zfpy",
+    "mode": 2,
+    "precision": -1,
+    "rate": -1,
+    "tolerance": -1
+}

docs/release.rst

@@ -0,0 +1,10 @@
+{
+    "compression_kwargs": {
+        "tolerance": -1
+    },
+    "id": "zfpy",
+    "mode": 4,
+    "precision": -1,
+    "rate": -1,
+    "tolerance": -1
+}

docs/zfpy.rst

@@ -0,0 +1,10 @@
+{
+    "compression_kwargs": {
+        "tolerance": -1
+    },
+    "id": "zfpy",
+    "mode": 4,
+    "precision": -1,
+    "rate": -1,
+    "tolerance": -1
+}

fixture/zfpy/array.00.npy

@@ -0,0 +1,10 @@
+{
+    "compression_kwargs": {
+        "precision": -1
+    },
+    "id": "zfpy",
+    "mode": 3,
+    "precision": -1,
+    "rate": -1,
+    "tolerance": -1
+}

fixture/zfpy/array.01.npy

@@ -3,7 +3,7 @@
 transformation codecs for use in data storage and communication
 applications. These include:
 
-* Compression codecs, e.g., Zlib, BZ2, LZMA and Blosc.
+* Compression codecs, e.g., Zlib, BZ2, LZMA, ZFPY and Blosc.
 * Pre-compression filters, e.g., Delta, Quantize, FixedScaleOffset,
   PackBits, Categorize.
 * Integrity checks, e.g., CRC32, Adler32.
@@ -16,7 +16,6 @@
 <https://github.com/alimanfoo/numcodecs/issues>`_.
 
 """
-
 import multiprocessing
 import atexit
 
@@ -68,6 +67,12 @@
 except ImportError:  # pragma: no cover
     pass
 
+try:
+    from numcodecs.zfpy import ZFPY
+    register_codec(ZFPY)
+except ImportError:  # pragma: no cover
+    pass
+
 from numcodecs.astype import AsType
 register_codec(AsType)
 

fixture/zfpy/array.02.npy

@@ -268,12 +268,10 @@ def check_backwards_compatibility(codec_id, arrays, codecs, precision=None, pref
 
             # file with codec configuration information
             codec_fn = os.path.join(codec_dir, 'config.json')
-
             # one time save config
             if not os.path.exists(codec_fn):  # pragma: no cover
                 with open(codec_fn, mode='w') as cf:
                     _json.dump(codec.get_config(), cf, sort_keys=True, indent=4)
-
             # load config and compare with expectation
             with open(codec_fn, mode='r') as cf:
                 config = _json.load(cf)

fixture/zfpy/array.03.npy

@@ -0,0 +1,86 @@
+import pytest
+
+
+import numpy as np
+
+
+try:
+    # noinspection PyProtectedMember
+    from numcodecs.zfpy import ZFPY, _zfpy
+except ImportError:  # pragma: no cover
+    pytest.skip("ZFPY not available", allow_module_level=True)
+
+
+from numcodecs.tests.common import (
+    check_encode_decode_array,
+    check_config,
+    check_repr,
+    check_backwards_compatibility,
+    check_err_decode_object_buffer,
+    check_err_encode_object_buffer,
+)
+
+
+codecs = [
+    ZFPY(mode=_zfpy.mode_fixed_rate, rate=-1),
+    ZFPY(),
+    ZFPY(mode=_zfpy.mode_fixed_accuracy, tolerance=-1),
+    ZFPY(mode=_zfpy.mode_fixed_precision, precision=-1),
+]
+
+
+# mix of dtypes: integer, float, bool, string
+# mix of shapes: 1D, 2D, 3D
+# mix of orders: C, F
+arrays = [
+    np.linspace(1000, 1001, 1000, dtype="f4"),
+    np.linspace(1000, 1001, 1000, dtype="f8"),
+    np.random.normal(loc=1000, scale=1, size=(100, 10)),
+    np.random.normal(loc=1000, scale=1, size=(10, 10, 10)),
+    np.random.normal(loc=1000, scale=1, size=(2, 5, 10, 10)),
+    np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(5, 10, 20))),
+    np.random.randint(-(2 ** 31), -(2 ** 31) + 20, size=1000, dtype="i4").reshape(
+        100, 10
+    ),
+    np.random.randint(-(2 ** 63), -(2 ** 63) + 20, size=1000, dtype="i8").reshape(
+        10, 10, 10
+    ),
+]
+
+
+def test_encode_decode():
+    for arr in arrays:
+        if arr.dtype == np.int32 or arr.dtype == np.int64:
+            codec = [codecs[-1]]
+        else:
+            codec = codecs
+        for code in codec:
+            check_encode_decode_array(arr, code)
+
+
+def test_config():
+    for codec in codecs:
+        check_config(codec)
+
+
+def test_repr():
+    check_repr("ZFPY(mode=4, tolerance=0.001, rate=-1, precision=-1)")
+
+
+def test_backwards_compatibility():
+    for i, code in enumerate(codecs):
+        if code.mode == _zfpy.mode_fixed_rate:
+            codec = [code]
+            check_backwards_compatibility(ZFPY.codec_id, arrays, codec)
+        else:
+            check_backwards_compatibility(
+                ZFPY.codec_id, arrays[: len(arrays) - 2], codecs
+            )
+
+
+def test_err_decode_object_buffer():
+    check_err_decode_object_buffer(ZFPY())
+
+
+def test_err_encode_object_buffer():
+    check_err_encode_object_buffer(ZFPY())

fixture/zfpy/array.04.npy

@@ -0,0 +1,89 @@
+_zfpy = None
+try:
+    import zfpy as _zfpy
+except ImportError:  # pragma: no cover
+    pass
+
+
+if _zfpy:
+
+    from .abc import Codec
+    from .compat import ndarray_copy, ensure_contiguous_ndarray, ensure_bytes
+
+    # noinspection PyShadowingBuiltins
+    class ZFPY(Codec):
+        """Codec providing compression using zfpy via the Python standard
+        library.
+
+        Parameters
+        ----------
+        mode : integer
+            One of the zfpy mode choice, e.g., ``zfpy.mode_fixed_accuracy``.
+        tolerance : double, optional
+            A double-precision number, specifying the compression accuracy needed.
+        rate : double, optional
+            A double-precision number, specifying the compression rate needed.
+        precision : int, optional
+            A integer number, specifying the compression precision needed.
+
+        """
+
+        codec_id = "zfpy"
+
+        def __init__(
+            self,
+            mode=_zfpy.mode_fixed_accuracy,
+            tolerance=-1,
+            rate=-1,
+            precision=-1,
+            compression_kwargs=None,
+        ):
+            self.mode = mode
+            if mode == _zfpy.mode_fixed_accuracy:
+                self.compression_kwargs = {"tolerance": tolerance}
+            elif mode == _zfpy.mode_fixed_rate:
+                self.compression_kwargs = {"rate": rate}
+            elif mode == _zfpy.mode_fixed_precision:
+                self.compression_kwargs = {"precision": precision}
+            else:
+                pass
+
+            self.tolerance = tolerance
+            self.rate = rate
+            self.precision = precision
+
+        def encode(self, buf):
+
+            # normalise inputs
+            buf = ensure_contiguous_ndarray(buf)
+
+            # do compression
+            return _zfpy.compress_numpy(
+                buf, write_header=True, **self.compression_kwargs
+            )
+
+        def decode(self, buf, out=None):
+
+            # normalise inputs
+            buf = ensure_bytes(buf)
+            if out is not None:
+                out = ensure_contiguous_ndarray(out)
+
+            # do decompression
+            dec = _zfpy.decompress_numpy(buf)
+
+            # handle destination
+            if out is not None:
+                return ndarray_copy(dec, out)
+            else:
+                return dec
+
+        def __repr__(self):
+            r = "%s(mode=%r, tolerance=%s, rate=%s, precision=%s)" % (
+                type(self).__name__,
+                self.mode,
+                self.tolerance,
+                self.rate,
+                self.precision,
+            )
+            return r

fixture/zfpy/array.05.npy

@@ -1,3 +1,5 @@
 Cython==0.29.21
 msgpack==1.0.2
 numpy==1.19.0
+zfpy==0.5.5; python_version < '3.9'
+

fixture/zfpy/array.06.npy

@@ -6,3 +6,4 @@ numpydoc
 mock
 numpy
 cython
+zfpy==0.5.5; python_version < '3.9'

fixture/zfpy/array.07.npy

@@ -76,6 +76,8 @@ def blosc_extension():
                      if os.path.isdir(d)]
     include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*')
                      if os.path.isdir(d)]
+    include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*/*')
+                     if os.path.isdir(d)]
     define_macros += [('HAVE_LZ4', 1),
                       ('HAVE_SNAPPY', 1),
                       ('HAVE_ZLIB', 1),
@@ -315,6 +317,7 @@ def run_setup(with_extensions):
     if with_extensions:
         ext_modules = (blosc_extension() + zstd_extension() + lz4_extension() +
                        compat_extension() + shuffle_extension() + vlen_extension())
+
         cmdclass = dict(build_ext=ve_build_ext)
     else:
         ext_modules = []