From 2ae3636f578786bbf96bbc43a23036219dc64c2e Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Tue, 28 Dec 2021 16:03:29 -0500 Subject: [PATCH 1/8] Allow codec registry via entrypoints --- numcodecs/registry.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/numcodecs/registry.py b/numcodecs/registry.py index 0ff8dc57..5d2a617a 100644 --- a/numcodecs/registry.py +++ b/numcodecs/registry.py @@ -3,6 +3,12 @@ codec_registry = dict() +try: + raise ImportError + import entrypoints + entries = entrypoints.get_group_named("numcodecs.codecs") +except ImportError: + entries = {} def get_codec(config): @@ -30,8 +36,11 @@ def get_codec(config): codec_id = config.pop('id', None) cls = codec_registry.get(codec_id) if cls is None: - raise ValueError('codec not available: %r' % codec_id) - return cls.from_config(config) + if codec_id in entries: + cls = entries[codec_id].load() + if cls: + return cls.from_config(config) + raise ValueError('codec not available: %r' % codec_id) def register_codec(cls, codec_id=None): From 3587325c8edf01f0665113e1a94eb972576dabed Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Tue, 28 Dec 2021 16:03:51 -0500 Subject: [PATCH 2/8] remove debug --- numcodecs/registry.py | 1 - 1 file changed, 1 deletion(-) diff --git a/numcodecs/registry.py b/numcodecs/registry.py index 5d2a617a..852ae51c 100644 --- a/numcodecs/registry.py +++ b/numcodecs/registry.py @@ -4,7 +4,6 @@ codec_registry = dict() try: - raise ImportError import entrypoints entries = entrypoints.get_group_named("numcodecs.codecs") except ImportError: From 0e8d61f22052200111999bb6c9f2151f5b43f946 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 29 Dec 2021 16:30:08 -0500 Subject: [PATCH 3/8] Add release note --- docs/release.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 63f4bbc1..a483ed64 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -8,6 +8,9 @@ Unreleased .. _release_0.9.1: +* Add ability to find codecs via entrypoints + By :user:`Martin Durant `, :issue:`290`. + 0.9.1 ----- From a07a427b45a5cee8ad5762df27869d460024624b Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Thu, 30 Dec 2021 13:01:15 -0500 Subject: [PATCH 4/8] Add test --- numcodecs/registry.py | 18 ++++++++++---- .../entry_points.txt | 2 ++ .../tests/package_with_entrypoint/__init__.py | 12 ++++++++++ numcodecs/tests/test_entrypoints.py | 24 +++++++++++++++++++ requirements_test.txt | 1 + 5 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 numcodecs/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt create mode 100644 numcodecs/tests/package_with_entrypoint/__init__.py create mode 100644 numcodecs/tests/test_entrypoints.py diff --git a/numcodecs/registry.py b/numcodecs/registry.py index 852ae51c..d02ea853 100644 --- a/numcodecs/registry.py +++ b/numcodecs/registry.py @@ -3,11 +3,20 @@ codec_registry = dict() -try: +entries = {} + + +def run_entrypoints(): import entrypoints - entries = entrypoints.get_group_named("numcodecs.codecs") -except ImportError: - entries = {} + entries.clear() + entries.update(entrypoints.get_group_named("numcodecs.codecs")) + + +try: + run_entrypoints() +except ImportError: # pragma: no cover + # marked "no cover" since we will include entrypoints in test env + pass def get_codec(config): @@ -37,6 +46,7 @@ def get_codec(config): if cls is None: if codec_id in entries: cls = entries[codec_id].load() + register_codec(cls, codec_id=codec_id) if cls: return cls.from_config(config) raise ValueError('codec not available: %r' % codec_id) diff --git a/numcodecs/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt b/numcodecs/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt new file mode 100644 index 00000000..bdc6ec5d --- /dev/null +++ b/numcodecs/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[numcodecs.codecs] +test = package_with_entrypoint:TestCodec diff --git a/numcodecs/tests/package_with_entrypoint/__init__.py b/numcodecs/tests/package_with_entrypoint/__init__.py new file mode 100644 index 00000000..13d4ec78 --- /dev/null +++ b/numcodecs/tests/package_with_entrypoint/__init__.py @@ -0,0 +1,12 @@ +from numcodecs.abc import Codec + + +class TestCodec(Codec): + + codec_id = "test" + + def encode(self, buf): + pass + + def decode(self, buf, out=None): + pass diff --git a/numcodecs/tests/test_entrypoints.py b/numcodecs/tests/test_entrypoints.py new file mode 100644 index 00000000..0d017f2d --- /dev/null +++ b/numcodecs/tests/test_entrypoints.py @@ -0,0 +1,24 @@ +import os.path +import sys + +import pytest + +import numcodecs.registry + + +here = os.path.abspath(os.path.dirname(__file__)) + + +@pytest.fixture() +def set_path(): + sys.path.append(here) + numcodecs.registry.run_entrypoints() + yield + sys.path.remove(here) + numcodecs.registry.run_entrypoints() + numcodecs.registry.codec_registry.pop("test") + + +def test_entrypoint_codec(set_path): + cls = numcodecs.registry.get_codec({"id": "test"}) + assert cls.codec_id == "test" diff --git a/requirements_test.txt b/requirements_test.txt index abdc250c..e6c4d294 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,5 +1,6 @@ coverage coveralls +entrypoints flake8 pytest pytest-cov From 23f1a4fc96d4a596887a7356a44cc1a08d254e64 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Thu, 30 Dec 2021 13:02:27 -0500 Subject: [PATCH 5/8] Mark required but unused test methods as no cover --- numcodecs/tests/package_with_entrypoint/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numcodecs/tests/package_with_entrypoint/__init__.py b/numcodecs/tests/package_with_entrypoint/__init__.py index 13d4ec78..a44b9808 100644 --- a/numcodecs/tests/package_with_entrypoint/__init__.py +++ b/numcodecs/tests/package_with_entrypoint/__init__.py @@ -5,8 +5,8 @@ class TestCodec(Codec): codec_id = "test" - def encode(self, buf): + def encode(self, buf): # pragma: no cover pass - def decode(self, buf, out=None): + def decode(self, buf, out=None): # pragma: no cover pass From 34c08b1dcbd7c4f1c8cf4804e2e1db991ed4ae03 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Thu, 30 Dec 2021 13:08:26 -0500 Subject: [PATCH 6/8] better exception --- numcodecs/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numcodecs/registry.py b/numcodecs/registry.py index d02ea853..e15dd45e 100644 --- a/numcodecs/registry.py +++ b/numcodecs/registry.py @@ -14,7 +14,7 @@ def run_entrypoints(): try: run_entrypoints() -except ImportError: # pragma: no cover +except (ImportError, ModuleNotFoundError): # pragma: no cover # marked "no cover" since we will include entrypoints in test env pass From f180e269ce729c6bd67c7476a69952063391bb92 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Thu, 30 Dec 2021 15:19:31 -0500 Subject: [PATCH 7/8] Skip test is no entryoints --- numcodecs/tests/test_entrypoints.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numcodecs/tests/test_entrypoints.py b/numcodecs/tests/test_entrypoints.py index 0d017f2d..e7c4bd4d 100644 --- a/numcodecs/tests/test_entrypoints.py +++ b/numcodecs/tests/test_entrypoints.py @@ -7,6 +7,7 @@ here = os.path.abspath(os.path.dirname(__file__)) +pytest.importorskip("entrypoints") @pytest.fixture() From 43d28592e709fef915a0411c7f9a75da5b8ea372 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Mon, 10 Jan 2022 10:24:53 -0500 Subject: [PATCH 8/8] Add logging --- numcodecs/registry.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/numcodecs/registry.py b/numcodecs/registry.py index e15dd45e..a4dba8de 100644 --- a/numcodecs/registry.py +++ b/numcodecs/registry.py @@ -1,7 +1,8 @@ """The registry module provides some simple convenience functions to enable applications to dynamically register and look-up codec classes.""" +import logging - +logger = logging.getLogger("numcodecs") codec_registry = dict() entries = {} @@ -45,6 +46,7 @@ def get_codec(config): cls = codec_registry.get(codec_id) if cls is None: if codec_id in entries: + logger.debug("Auto loading codec '%s' from entrypoint", codec_id) cls = entries[codec_id].load() register_codec(cls, codec_id=codec_id) if cls: @@ -68,4 +70,5 @@ def register_codec(cls, codec_id=None): """ if codec_id is None: codec_id = cls.codec_id + logger.debug("Registering codec '%s'", codec_id) codec_registry[codec_id] = cls