From 2c7b7af6a7a743292b3a5bd0907e07de06b285b8 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Mon, 1 Feb 2021 14:16:50 +0100 Subject: [PATCH 1/6] style --- xarray/backends/plugins.py | 40 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 6d3ec7e7da5..c158df07300 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -9,18 +9,18 @@ from .common import BACKEND_ENTRYPOINTS -def remove_duplicates(backend_entrypoints): +def remove_duplicates(pkg_entrypoints): # sort and group entrypoints by name - backend_entrypoints = sorted(backend_entrypoints, key=lambda ep: ep.name) - backend_entrypoints_grouped = itertools.groupby( - backend_entrypoints, key=lambda ep: ep.name + pkg_entrypoints = sorted(pkg_entrypoints, key=lambda ep: ep.name) + pkg_entrypoints_grouped = itertools.groupby( + pkg_entrypoints, key=lambda ep: ep.name ) # check if there are multiple entrypoints for the same name - unique_backend_entrypoints = [] - for name, matches in backend_entrypoints_grouped: + unique_pkg_entrypoints = [] + for name, matches in pkg_entrypoints_grouped: matches = list(matches) - unique_backend_entrypoints.append(matches[0]) + unique_pkg_entrypoints.append(matches[0]) matches_len = len(matches) if matches_len > 1: selected_module_name = matches[0].module_name @@ -30,7 +30,7 @@ def remove_duplicates(backend_entrypoints): f"\n {all_module_names}.\n It will be used: {selected_module_name}.", RuntimeWarning, ) - return unique_backend_entrypoints + return unique_pkg_entrypoints def detect_parameters(open_dataset): @@ -48,13 +48,13 @@ def detect_parameters(open_dataset): return tuple(parameters) -def create_engines_dict(backend_entrypoints): - engines = {} - for backend_ep in backend_entrypoints: - name = backend_ep.name - backend = backend_ep.load() - engines[name] = backend - return engines +def create_backends_dict(pkg_entrypoints): + backend_entrypoints = {} + for pkg_ep in pkg_entrypoints: + name = pkg_ep.name + backend = pkg_ep.load() + backend_entrypoints[name] = backend + return backend_entrypoints def set_missing_parameters(engines): @@ -64,10 +64,10 @@ def set_missing_parameters(engines): backend.open_dataset_parameters = detect_parameters(open_dataset) -def build_engines(entrypoints): +def build_engines(pkg_entrypoints): backend_entrypoints = BACKEND_ENTRYPOINTS.copy() - pkg_entrypoints = remove_duplicates(entrypoints) - external_backend_entrypoints = create_engines_dict(pkg_entrypoints) + pkg_entrypoints = remove_duplicates(pkg_entrypoints) + external_backend_entrypoints = create_backends_dict(pkg_entrypoints) backend_entrypoints.update(external_backend_entrypoints) set_missing_parameters(backend_entrypoints) return backend_entrypoints @@ -75,8 +75,8 @@ def build_engines(entrypoints): @functools.lru_cache(maxsize=1) def list_engines(): - entrypoints = pkg_resources.iter_entry_points("xarray.backends") - return build_engines(entrypoints) + pkg_entrypoints = pkg_resources.iter_entry_points("xarray.backends") + return build_engines(pkg_entrypoints) def guess_engine(store_spec): From 31f6afe4fd7660a57800584b9992716ab7c1f60d Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Tue, 2 Feb 2021 02:08:00 +0100 Subject: [PATCH 2/6] order the plugins and tests --- xarray/backends/plugins.py | 23 +++++++++++++++++++---- xarray/tests/test_plugins.py | 27 ++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index c158df07300..247cfe0e58b 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -9,6 +9,9 @@ from .common import BACKEND_ENTRYPOINTS +standard_backends_order = ["netcdf4", "h5netcdf", "scipy"] + + def remove_duplicates(pkg_entrypoints): # sort and group entrypoints by name @@ -48,7 +51,7 @@ def detect_parameters(open_dataset): return tuple(parameters) -def create_backends_dict(pkg_entrypoints): +def backends_dict_from_pkg(pkg_entrypoints): backend_entrypoints = {} for pkg_ep in pkg_entrypoints: name = pkg_ep.name @@ -64,11 +67,23 @@ def set_missing_parameters(engines): backend.open_dataset_parameters = detect_parameters(open_dataset) +def merge_backends_dict(backend_entrypoints,external_backend_entrypoints): + backend_entrypoints = backend_entrypoints.copy() + backend_entrypoints.update(external_backend_entrypoints) + ordered_backends_entrypoints = {} + for be_name in standard_backends_order: + if backend_entrypoints.get(be_name, None) is not None: + ordered_backends_entrypoints[be_name] = backend_entrypoints.pop(be_name) + ordered_backends_entrypoints.update( + {name: backend_entrypoints[name] for name in sorted(backend_entrypoints)} + ) + return ordered_backends_entrypoints + + def build_engines(pkg_entrypoints): - backend_entrypoints = BACKEND_ENTRYPOINTS.copy() pkg_entrypoints = remove_duplicates(pkg_entrypoints) - external_backend_entrypoints = create_backends_dict(pkg_entrypoints) - backend_entrypoints.update(external_backend_entrypoints) + external_backend_entrypoints = backends_dict_from_pkg(pkg_entrypoints) + backend_entrypoints = merge_backends_dict(BACKEND_ENTRYPOINTS, external_backend_entrypoints) set_missing_parameters(backend_entrypoints) return backend_entrypoints diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 38ebce6da1a..3936b67c9a6 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -53,13 +53,13 @@ def test_remove_duplicates_warnings(dummy_duplicated_entrypoints): @mock.patch("pkg_resources.EntryPoint.load", mock.MagicMock(return_value=None)) -def test_create_engines_dict(): +def test_backends_dict_from_pkg(): specs = [ "engine1 = xarray.tests.test_plugins:backend_1", "engine2 = xarray.tests.test_plugins:backend_2", ] entrypoints = [pkg_resources.EntryPoint.parse(spec) for spec in specs] - engines = plugins.create_engines_dict(entrypoints) + engines = plugins.backends_dict_from_pkg(entrypoints) assert len(engines) == 2 assert engines.keys() == set(("engine1", "engine2")) @@ -101,7 +101,7 @@ def test_set_missing_parameters_raise_error(): @mock.patch("pkg_resources.EntryPoint.load", mock.MagicMock(return_value=dummy_cfgrib)) def test_build_engines(): dummy_cfgrib_pkg_entrypoint = pkg_resources.EntryPoint.parse( - "cfgrib = xarray.tests.test_plugins:backend_1" + "cfgrib = xarray.tests.test_plugins:backend_1", ) backend_entrypoints = plugins.build_engines([dummy_cfgrib_pkg_entrypoint]) assert backend_entrypoints["cfgrib"] is dummy_cfgrib @@ -109,3 +109,24 @@ def test_build_engines(): "filename_or_obj", "decoder", ) + + +@mock.patch("pkg_resources.EntryPoint.load", mock.MagicMock(return_value=dummy_cfgrib)) +def test_build_engines_sorted(): + dummy_pkg_entrypoints = [ + pkg_resources.EntryPoint.parse("dummy2 = xarray.tests.test_plugins:backend_1",), + pkg_resources.EntryPoint.parse("dummy1 = xarray.tests.test_plugins:backend_1",) + ] + backend_entrypoints = plugins.build_engines(dummy_pkg_entrypoints) + backend_entrypoints = list(backend_entrypoints) + + indices = [] + for be in plugins.standard_backends_order: + index = backend_entrypoints.index(be) + backend_entrypoints.pop(index) + indices.append(index) + + assert set(indices) < {0, -1} + assert list(backend_entrypoints) == sorted(backend_entrypoints) + + From 5c121d57a3fbe3082b2450d940d27ae4350ceb66 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 11 Feb 2021 05:16:38 +0100 Subject: [PATCH 3/6] style --- xarray/backends/plugins.py | 11 +++++------ xarray/tests/test_plugins.py | 12 +++++++----- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 247cfe0e58b..f10fdcef163 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -8,7 +8,6 @@ from .common import BACKEND_ENTRYPOINTS - standard_backends_order = ["netcdf4", "h5netcdf", "scipy"] @@ -16,9 +15,7 @@ def remove_duplicates(pkg_entrypoints): # sort and group entrypoints by name pkg_entrypoints = sorted(pkg_entrypoints, key=lambda ep: ep.name) - pkg_entrypoints_grouped = itertools.groupby( - pkg_entrypoints, key=lambda ep: ep.name - ) + pkg_entrypoints_grouped = itertools.groupby(pkg_entrypoints, key=lambda ep: ep.name) # check if there are multiple entrypoints for the same name unique_pkg_entrypoints = [] for name, matches in pkg_entrypoints_grouped: @@ -67,7 +64,7 @@ def set_missing_parameters(engines): backend.open_dataset_parameters = detect_parameters(open_dataset) -def merge_backends_dict(backend_entrypoints,external_backend_entrypoints): +def merge_backends_dict(backend_entrypoints, external_backend_entrypoints): backend_entrypoints = backend_entrypoints.copy() backend_entrypoints.update(external_backend_entrypoints) ordered_backends_entrypoints = {} @@ -83,7 +80,9 @@ def merge_backends_dict(backend_entrypoints,external_backend_entrypoints): def build_engines(pkg_entrypoints): pkg_entrypoints = remove_duplicates(pkg_entrypoints) external_backend_entrypoints = backends_dict_from_pkg(pkg_entrypoints) - backend_entrypoints = merge_backends_dict(BACKEND_ENTRYPOINTS, external_backend_entrypoints) + backend_entrypoints = merge_backends_dict( + BACKEND_ENTRYPOINTS, external_backend_entrypoints + ) set_missing_parameters(backend_entrypoints) return backend_entrypoints diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 3936b67c9a6..67038eb8dfd 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -101,7 +101,7 @@ def test_set_missing_parameters_raise_error(): @mock.patch("pkg_resources.EntryPoint.load", mock.MagicMock(return_value=dummy_cfgrib)) def test_build_engines(): dummy_cfgrib_pkg_entrypoint = pkg_resources.EntryPoint.parse( - "cfgrib = xarray.tests.test_plugins:backend_1", + "cfgrib = xarray.tests.test_plugins:backend_1" ) backend_entrypoints = plugins.build_engines([dummy_cfgrib_pkg_entrypoint]) assert backend_entrypoints["cfgrib"] is dummy_cfgrib @@ -114,8 +114,12 @@ def test_build_engines(): @mock.patch("pkg_resources.EntryPoint.load", mock.MagicMock(return_value=dummy_cfgrib)) def test_build_engines_sorted(): dummy_pkg_entrypoints = [ - pkg_resources.EntryPoint.parse("dummy2 = xarray.tests.test_plugins:backend_1",), - pkg_resources.EntryPoint.parse("dummy1 = xarray.tests.test_plugins:backend_1",) + pkg_resources.EntryPoint.parse( + "dummy2 = xarray.tests.test_plugins:backend_1", + ), + pkg_resources.EntryPoint.parse( + "dummy1 = xarray.tests.test_plugins:backend_1", + ), ] backend_entrypoints = plugins.build_engines(dummy_pkg_entrypoints) backend_entrypoints = list(backend_entrypoints) @@ -128,5 +132,3 @@ def test_build_engines_sorted(): assert set(indices) < {0, -1} assert list(backend_entrypoints) == sorted(backend_entrypoints) - - From 4759fcf71d00123776fa2bdff8ee91066a9c578a Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 11 Feb 2021 05:42:56 +0100 Subject: [PATCH 4/6] fix post merge --- xarray/backends/plugins.py | 10 ++++------ xarray/tests/test_plugins.py | 5 ++++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 389bb790922..0fdddfee906 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -67,9 +67,7 @@ def set_missing_parameters(backend_entrypoints): backend.open_dataset_parameters = detect_parameters(open_dataset) -def merge_backends_dict(backend_entrypoints, external_backend_entrypoints): - backend_entrypoints = backend_entrypoints.copy() - backend_entrypoints.update(external_backend_entrypoints) +def sort_backends(backend_entrypoints): ordered_backends_entrypoints = {} for be_name in standard_backends_order: if backend_entrypoints.get(be_name, None) is not None: @@ -81,11 +79,11 @@ def merge_backends_dict(backend_entrypoints, external_backend_entrypoints): def build_engines(pkg_entrypoints): + backend_entrypoints = BACKEND_ENTRYPOINTS.copy() pkg_entrypoints = remove_duplicates(pkg_entrypoints) external_backend_entrypoints = backends_dict_from_pkg(pkg_entrypoints) - backend_entrypoints = merge_backends_dict( - BACKEND_ENTRYPOINTS, external_backend_entrypoints - ) + backend_entrypoints.update(external_backend_entrypoints) + backend_entrypoints = sort_backends(backend_entrypoints) set_missing_parameters(backend_entrypoints) engines = {} for name, backend in backend_entrypoints.items(): diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 33650faa56c..04274a500de 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -118,7 +118,10 @@ def test_build_engines(): ) -@mock.patch("pkg_resources.EntryPoint.load", mock.MagicMock(return_value=dummy_cfgrib)) +@mock.patch( + "pkg_resources.EntryPoint.load", + mock.MagicMock(return_value=DummyBackendEntrypoint1), +) def test_build_engines_sorted(): dummy_pkg_entrypoints = [ pkg_resources.EntryPoint.parse( From 56577c268ebb689495019b107f507a5f7a9fd5b7 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 11 Feb 2021 06:15:07 +0100 Subject: [PATCH 5/6] fix plugin_test --- xarray/tests/test_plugins.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 04274a500de..b4b9c1cfb64 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -111,6 +111,7 @@ def test_build_engines(): "cfgrib = xarray.tests.test_plugins:backend_1" ) backend_entrypoints = plugins.build_engines([dummy_pkg_entrypoint]) + assert isinstance(backend_entrypoints["cfgrib"], DummyBackendEntrypoint1) assert backend_entrypoints["cfgrib"].open_dataset_parameters == ( "filename_or_obj", @@ -136,9 +137,12 @@ def test_build_engines_sorted(): indices = [] for be in plugins.standard_backends_order: - index = backend_entrypoints.index(be) - backend_entrypoints.pop(index) - indices.append(index) + try: + index = backend_entrypoints.index(be) + backend_entrypoints.pop(index) + indices.append(index) + except ValueError: + pass assert set(indices) < {0, -1} assert list(backend_entrypoints) == sorted(backend_entrypoints) From b8fa58442d2f64bebb1157ad2982b94e5ef36a60 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 11 Feb 2021 14:25:23 +0100 Subject: [PATCH 6/6] capitalize global variable --- xarray/backends/plugins.py | 4 ++-- xarray/tests/test_plugins.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 0fdddfee906..88c29306d18 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -8,7 +8,7 @@ from .common import BACKEND_ENTRYPOINTS -standard_backends_order = ["netcdf4", "h5netcdf", "scipy"] +STANDARD_BACKENDS_ORDER = ["netcdf4", "h5netcdf", "scipy"] def remove_duplicates(pkg_entrypoints): @@ -69,7 +69,7 @@ def set_missing_parameters(backend_entrypoints): def sort_backends(backend_entrypoints): ordered_backends_entrypoints = {} - for be_name in standard_backends_order: + for be_name in STANDARD_BACKENDS_ORDER: if backend_entrypoints.get(be_name, None) is not None: ordered_backends_entrypoints[be_name] = backend_entrypoints.pop(be_name) ordered_backends_entrypoints.update( diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index b4b9c1cfb64..0cda2901cee 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -136,7 +136,7 @@ def test_build_engines_sorted(): backend_entrypoints = list(backend_entrypoints) indices = [] - for be in plugins.standard_backends_order: + for be in plugins.STANDARD_BACKENDS_ORDER: try: index = backend_entrypoints.index(be) backend_entrypoints.pop(index)