From 52169da55b33bfc727ed20a6cd7cd503d9ac8006 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 7 Jan 2021 10:49:29 +0100 Subject: [PATCH 01/17] fix list_engine --- xarray/backends/plugins.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 6e9045c1c7f..19760199c90 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -30,6 +30,17 @@ "pynio": pynio_backend, } +BACKEND_DEPENDENCIES = { + "netcdf4": "netcdf4", + "h5netcdf": "h5netcdf", + "scipy": "scipy", + "pseudonetcdf": "PseudoNetCDF", + "zarr": "zarr", + "cfgrib": "cfgrib", + "pydap": "pydap", + "pynio": "Nio" +} + def remove_duplicates(backend_entrypoints): @@ -86,8 +97,21 @@ def set_missing_parameters(engines): backend.open_dataset_parameters = detect_parameters(open_dataset) +def internal_available_entrypoints(): + backend_entrypoints = {} + for entrypoint in BACKEND_ENTRYPOINTS: + if entrypoint != "store": + try: + __import__(BACKEND_DEPENDENCIES[entrypoint]) + except ImportError: + pass + else: + backend_entrypoints[entrypoint] = BACKEND_ENTRYPOINTS[entrypoint] + return backend_entrypoints + + def build_engines(entrypoints): - backend_entrypoints = BACKEND_ENTRYPOINTS.copy() + backend_entrypoints = internal_available_entrypoints() pkg_entrypoints = remove_duplicates(entrypoints) external_backend_entrypoints = create_engines_dict(pkg_entrypoints) backend_entrypoints.update(external_backend_entrypoints) @@ -105,10 +129,6 @@ def guess_engine(store_spec): engines = list_engines() # use the pre-defined selection order for netCDF files - for engine in ["netcdf4", "h5netcdf", "scipy"]: - if engine in engines and engines[engine].guess_can_open(store_spec): - return engine - for engine, backend in engines.items(): try: if backend.guess_can_open and backend.guess_can_open(store_spec): From 4d641fdfba7b45136869d4f953ad20222d6606df Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 7 Jan 2021 12:36:37 +0100 Subject: [PATCH 02/17] fix store engine and netcdf4 --- xarray/backends/plugins.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 19760199c90..1ece9e6d992 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -31,7 +31,7 @@ } BACKEND_DEPENDENCIES = { - "netcdf4": "netcdf4", + "netcdf4": "netCDF4", "h5netcdf": "h5netcdf", "scipy": "scipy", "pseudonetcdf": "PseudoNetCDF", @@ -98,15 +98,13 @@ def set_missing_parameters(engines): def internal_available_entrypoints(): - backend_entrypoints = {} + backend_entrypoints = BACKEND_ENTRYPOINTS.copy() for entrypoint in BACKEND_ENTRYPOINTS: if entrypoint != "store": try: __import__(BACKEND_DEPENDENCIES[entrypoint]) except ImportError: - pass - else: - backend_entrypoints[entrypoint] = BACKEND_ENTRYPOINTS[entrypoint] + backend_entrypoints.pop(entrypoint) return backend_entrypoints From 30fb6c24c6cf2a0638697af18e4bfb3956d517a1 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 7 Jan 2021 14:59:33 +0100 Subject: [PATCH 03/17] reve --- xarray/backends/plugins.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 1ece9e6d992..8f5d11a0ead 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -127,6 +127,10 @@ def guess_engine(store_spec): engines = list_engines() # use the pre-defined selection order for netCDF files + for engine in ["netcdf4", "h5netcdf", "scipy"]: + if engine in engines and engines[engine].guess_can_open(store_spec): + return engine + for engine, backend in engines.items(): try: if backend.guess_can_open and backend.guess_can_open(store_spec): From d4722b3de0903ecafc78acbabaef13ec783acdf0 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 7 Jan 2021 14:59:33 +0100 Subject: [PATCH 04/17] revert changes in guess_engine --- xarray/backends/plugins.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 1ece9e6d992..e38bf7de1e4 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -38,7 +38,7 @@ "zarr": "zarr", "cfgrib": "cfgrib", "pydap": "pydap", - "pynio": "Nio" + "pynio": "Nio", } @@ -127,6 +127,10 @@ def guess_engine(store_spec): engines = list_engines() # use the pre-defined selection order for netCDF files + for engine in ["netcdf4", "h5netcdf", "scipy"]: + if engine in engines and engines[engine].guess_can_open(store_spec): + return engine + for engine, backend in engines.items(): try: if backend.guess_can_open and backend.guess_can_open(store_spec): From bcab954d2aa15431257d6051179ba37cfc72d573 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Mon, 11 Jan 2021 09:24:07 +0100 Subject: [PATCH 05/17] add resister of backend if dependencies aere instralled --- xarray/backends/cfgrib_.py | 12 +++++++- xarray/backends/common.py | 4 +++ xarray/backends/h5netcdf_.py | 12 +++++++- xarray/backends/netCDF4_.py | 6 ++++ xarray/backends/plugins.py | 48 ++------------------------------ xarray/backends/pseudonetcdf_.py | 13 ++++++++- xarray/backends/pydap_.py | 14 +++++++++- xarray/backends/pynio_.py | 13 ++++++++- xarray/backends/scipy_.py | 12 +++++++- xarray/backends/store.py | 5 +++- xarray/backends/zarr.py | 6 ++++ 11 files changed, 92 insertions(+), 53 deletions(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 57cc4d6f366..37ad0d230dd 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -1,3 +1,4 @@ +import importlib import os import numpy as np @@ -5,7 +6,12 @@ from ..core import indexing from ..core.utils import Frozen, FrozenDict, close_on_error from ..core.variable import Variable -from .common import AbstractDataStore, BackendArray, BackendEntrypoint +from .common import ( + BACKEND_ENTRYPOINTS, + AbstractDataStore, + BackendArray, + BackendEntrypoint, +) from .locks import SerializableLock, ensure_lock from .store import open_backend_dataset_store @@ -129,3 +135,7 @@ def open_backend_dataset_cfgrib( cfgrib_backend = BackendEntrypoint( open_dataset=open_backend_dataset_cfgrib, guess_can_open=guess_can_open_cfgrib ) + + +if importlib.util.find_spec("cfgrib"): + BACKEND_ENTRYPOINTS["cfgrib"] = cfgrib_backend diff --git a/xarray/backends/common.py b/xarray/backends/common.py index 72a63957662..5c45eb8de18 100644 --- a/xarray/backends/common.py +++ b/xarray/backends/common.py @@ -1,6 +1,7 @@ import logging import time import traceback +import typing as T import numpy as np @@ -349,3 +350,6 @@ def __init__(self, open_dataset, open_dataset_parameters=None, guess_can_open=No self.open_dataset = open_dataset self.open_dataset_parameters = open_dataset_parameters self.guess_can_open = guess_can_open + + +BACKEND_ENTRYPOINTS: T.Dict[str, BackendEntrypoint] = {} diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index b2996369ee7..0d02d009f10 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -1,4 +1,5 @@ import functools +import importlib import io import os from distutils.version import LooseVersion @@ -8,7 +9,12 @@ from ..core import indexing from ..core.utils import FrozenDict, is_remote_uri, read_magic_number from ..core.variable import Variable -from .common import BackendEntrypoint, WritableCFDataStore, find_root_and_group +from .common import ( + BACKEND_ENTRYPOINTS, + BackendEntrypoint, + WritableCFDataStore, + find_root_and_group, +) from .file_manager import CachingFileManager, DummyFileManager from .locks import HDF5_LOCK, combine_locks, ensure_lock, get_write_lock from .netCDF4_ import ( @@ -375,3 +381,7 @@ def open_backend_dataset_h5netcdf( h5netcdf_backend = BackendEntrypoint( open_dataset=open_backend_dataset_h5netcdf, guess_can_open=guess_can_open_h5netcdf ) + + +if importlib.util.find_spec("h5netcdf"): + BACKEND_ENTRYPOINTS["h5netcdf"] = h5netcdf_backend diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 0e35270ea9a..bc1b13d7e64 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -1,4 +1,5 @@ import functools +import importlib import operator import os import pathlib @@ -12,6 +13,7 @@ from ..core.utils import FrozenDict, close_on_error, is_remote_uri from ..core.variable import Variable from .common import ( + BACKEND_ENTRYPOINTS, BackendArray, BackendEntrypoint, WritableCFDataStore, @@ -563,3 +565,7 @@ def open_backend_dataset_netcdf4( netcdf4_backend = BackendEntrypoint( open_dataset=open_backend_dataset_netcdf4, guess_can_open=guess_can_open_netcdf4 ) + + +if importlib.util.find_spec("netCDF4"): + BACKEND_ENTRYPOINTS["netcdf4"] = netcdf4_backend diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index e38bf7de1e4..9cd8cc1ce2e 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -2,44 +2,11 @@ import inspect import itertools import logging -import typing as T import warnings import pkg_resources -from .cfgrib_ import cfgrib_backend -from .common import BackendEntrypoint -from .h5netcdf_ import h5netcdf_backend -from .netCDF4_ import netcdf4_backend -from .pseudonetcdf_ import pseudonetcdf_backend -from .pydap_ import pydap_backend -from .pynio_ import pynio_backend -from .scipy_ import scipy_backend -from .store import store_backend -from .zarr import zarr_backend - -BACKEND_ENTRYPOINTS: T.Dict[str, BackendEntrypoint] = { - "store": store_backend, - "netcdf4": netcdf4_backend, - "h5netcdf": h5netcdf_backend, - "scipy": scipy_backend, - "pseudonetcdf": pseudonetcdf_backend, - "zarr": zarr_backend, - "cfgrib": cfgrib_backend, - "pydap": pydap_backend, - "pynio": pynio_backend, -} - -BACKEND_DEPENDENCIES = { - "netcdf4": "netCDF4", - "h5netcdf": "h5netcdf", - "scipy": "scipy", - "pseudonetcdf": "PseudoNetCDF", - "zarr": "zarr", - "cfgrib": "cfgrib", - "pydap": "pydap", - "pynio": "Nio", -} +from .common import BACKEND_ENTRYPOINTS def remove_duplicates(backend_entrypoints): @@ -97,19 +64,8 @@ def set_missing_parameters(engines): backend.open_dataset_parameters = detect_parameters(open_dataset) -def internal_available_entrypoints(): - backend_entrypoints = BACKEND_ENTRYPOINTS.copy() - for entrypoint in BACKEND_ENTRYPOINTS: - if entrypoint != "store": - try: - __import__(BACKEND_DEPENDENCIES[entrypoint]) - except ImportError: - backend_entrypoints.pop(entrypoint) - return backend_entrypoints - - def build_engines(entrypoints): - backend_entrypoints = internal_available_entrypoints() + backend_entrypoints = BACKEND_ENTRYPOINTS.copy() pkg_entrypoints = remove_duplicates(entrypoints) external_backend_entrypoints = create_engines_dict(pkg_entrypoints) backend_entrypoints.update(external_backend_entrypoints) diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index d9128d1d503..d928f94f8c2 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -1,9 +1,16 @@ +import importlib + import numpy as np from ..core import indexing from ..core.utils import Frozen, FrozenDict, close_on_error from ..core.variable import Variable -from .common import AbstractDataStore, BackendArray, BackendEntrypoint +from .common import ( + BACKEND_ENTRYPOINTS, + AbstractDataStore, + BackendArray, + BackendEntrypoint, +) from .file_manager import CachingFileManager from .locks import HDF5_LOCK, NETCDFC_LOCK, combine_locks, ensure_lock from .store import open_backend_dataset_store @@ -138,3 +145,7 @@ def open_backend_dataset_pseudonetcdf( open_dataset=open_backend_dataset_pseudonetcdf, open_dataset_parameters=open_dataset_parameters, ) + + +if importlib.util.find_spec("PseudoNetCDF"): + BACKEND_ENTRYPOINTS["pseudonetcdf"] = pseudonetcdf_backend diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index 4995045a739..17706a54e42 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -1,10 +1,18 @@ +import importlib + import numpy as np from ..core import indexing from ..core.pycompat import integer_types from ..core.utils import Frozen, FrozenDict, close_on_error, is_dict_like, is_remote_uri from ..core.variable import Variable -from .common import AbstractDataStore, BackendArray, BackendEntrypoint, robust_getitem +from .common import ( + BACKEND_ENTRYPOINTS, + AbstractDataStore, + BackendArray, + BackendEntrypoint, + robust_getitem, +) from .store import open_backend_dataset_store @@ -133,3 +141,7 @@ def open_backend_dataset_pydap( pydap_backend = BackendEntrypoint( open_dataset=open_backend_dataset_pydap, guess_can_open=guess_can_open_pydap ) + + +if importlib.util.find_spec("pydap"): + BACKEND_ENTRYPOINTS["pydap"] = pydap_backend diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index dc6c47935e8..af68f556b37 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -1,9 +1,16 @@ +import importlib + import numpy as np from ..core import indexing from ..core.utils import Frozen, FrozenDict, close_on_error from ..core.variable import Variable -from .common import AbstractDataStore, BackendArray, BackendEntrypoint +from .common import ( + BACKEND_ENTRYPOINTS, + AbstractDataStore, + BackendArray, + BackendEntrypoint, +) from .file_manager import CachingFileManager from .locks import HDF5_LOCK, NETCDFC_LOCK, SerializableLock, combine_locks, ensure_lock from .store import open_backend_dataset_store @@ -119,3 +126,7 @@ def open_backend_dataset_pynio( pynio_backend = BackendEntrypoint(open_dataset=open_backend_dataset_pynio) + + +if importlib.util.find_spec("Nio"): + BACKEND_ENTRYPOINTS["pynio"] = pynio_backend diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index 873a91f9c07..9b25823ff61 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -1,3 +1,4 @@ +import importlib import io import os @@ -6,7 +7,12 @@ from ..core.indexing import NumpyIndexingAdapter from ..core.utils import Frozen, FrozenDict, close_on_error, read_magic_number from ..core.variable import Variable -from .common import BackendArray, BackendEntrypoint, WritableCFDataStore +from .common import ( + BACKEND_ENTRYPOINTS, + BackendArray, + BackendEntrypoint, + WritableCFDataStore, +) from .file_manager import CachingFileManager, DummyFileManager from .locks import ensure_lock, get_write_lock from .netcdf3 import encode_nc3_attr_value, encode_nc3_variable, is_valid_nc3_name @@ -271,3 +277,7 @@ def open_backend_dataset_scipy( scipy_backend = BackendEntrypoint( open_dataset=open_backend_dataset_scipy, guess_can_open=guess_can_open_scipy ) + + +if importlib.util.find_spec("scipy"): + BACKEND_ENTRYPOINTS["scipy"] = scipy_backend diff --git a/xarray/backends/store.py b/xarray/backends/store.py index d314a9c3ca9..f720eb36e19 100644 --- a/xarray/backends/store.py +++ b/xarray/backends/store.py @@ -1,6 +1,6 @@ from .. import conventions from ..core.dataset import Dataset -from .common import AbstractDataStore, BackendEntrypoint +from .common import BACKEND_ENTRYPOINTS, AbstractDataStore, BackendEntrypoint def guess_can_open_store(store_spec): @@ -45,3 +45,6 @@ def open_backend_dataset_store( store_backend = BackendEntrypoint( open_dataset=open_backend_dataset_store, guess_can_open=guess_can_open_store ) + + +BACKEND_ENTRYPOINTS["store"] = store_backend diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 3b4b3a3d9d5..b8fa9a9f82f 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -1,3 +1,4 @@ +import importlib import os import pathlib @@ -9,6 +10,7 @@ from ..core.utils import FrozenDict, HiddenKeyDict, close_on_error from ..core.variable import Variable from .common import ( + BACKEND_ENTRYPOINTS, AbstractWritableDataStore, BackendArray, BackendEntrypoint, @@ -705,3 +707,7 @@ def open_backend_dataset_zarr( zarr_backend = BackendEntrypoint(open_dataset=open_backend_dataset_zarr) + + +if importlib.util.find_spec("zarr"): + BACKEND_ENTRYPOINTS["zarr"] = zarr_backend From 37913325e49a98a02a767f98e670612229cecc0a Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Fri, 15 Jan 2021 11:22:29 +0100 Subject: [PATCH 06/17] style mypy --- xarray/backends/cfgrib_.py | 2 +- xarray/backends/h5netcdf_.py | 2 +- xarray/backends/netCDF4_.py | 2 +- xarray/backends/pseudonetcdf_.py | 2 +- xarray/backends/pydap_.py | 2 +- xarray/backends/pynio_.py | 2 +- xarray/backends/scipy_.py | 2 +- xarray/backends/zarr.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 348e9937c6e..884461942ad 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -1,4 +1,4 @@ -import importlib +import importlib.util import os import numpy as np diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 0d02d009f10..c686fe79b51 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -1,5 +1,5 @@ import functools -import importlib +import importlib.util import io import os from distutils.version import LooseVersion diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index bc1b13d7e64..19e2583b422 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -1,5 +1,5 @@ import functools -import importlib +import importlib.util import operator import os import pathlib diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index d928f94f8c2..7af79b4456d 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -1,4 +1,4 @@ -import importlib +import importlib.util import numpy as np diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index 17706a54e42..b65e672cea4 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -1,4 +1,4 @@ -import importlib +import importlib.util import numpy as np diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index af68f556b37..69ba31544d6 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -1,4 +1,4 @@ -import importlib +import importlib.util import numpy as np diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index 9b25823ff61..fd491dac701 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -1,4 +1,4 @@ -import importlib +import importlib.util import io import os diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index b8fa9a9f82f..5add5e67726 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -1,4 +1,4 @@ -import importlib +import importlib.util import os import pathlib From 194479724731b8b8b430d9474b594c6c338e6ee1 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Fri, 15 Jan 2021 11:38:17 +0100 Subject: [PATCH 07/17] fix import --- xarray/tests/test_plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 110ef47209f..38ebce6da1a 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -92,7 +92,7 @@ def test_set_missing_parameters_raise_error(): with pytest.raises(TypeError): plugins.set_missing_parameters({"engine": backend}) - backend = plugins.BackendEntrypoint( + backend = common.BackendEntrypoint( dummy_open_dataset_kwargs, ("filename_or_obj", "decoder") ) plugins.set_missing_parameters({"engine": backend}) From 6e62442f8c5f36c1f909959b271656ca6a04080f Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Fri, 15 Jan 2021 13:50:09 +0100 Subject: [PATCH 08/17] use import instead of importlib --- xarray/backends/apiv2.py | 5 +---- xarray/backends/cfgrib_.py | 10 ++++++---- xarray/backends/h5netcdf_.py | 12 ++++++------ xarray/backends/netCDF4_.py | 10 ++++++---- xarray/backends/pseudonetcdf_.py | 15 +++++++++------ xarray/backends/pydap_.py | 11 ++++++----- xarray/backends/pynio_.py | 10 ++++++---- xarray/backends/zarr.py | 10 ++++++---- 8 files changed, 46 insertions(+), 37 deletions(-) diff --git a/xarray/backends/apiv2.py b/xarray/backends/apiv2.py index 0f98291983d..1e31f10d2fb 100644 --- a/xarray/backends/apiv2.py +++ b/xarray/backends/apiv2.py @@ -262,10 +262,7 @@ def open_dataset( overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None) backend_ds = backend.open_dataset( - filename_or_obj, - drop_variables=drop_variables, - **decoders, - **kwargs, + filename_or_obj, drop_variables=drop_variables, **decoders, **kwargs, ) ds = _dataset_from_backend_dataset( backend_ds, diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 884461942ad..9d533d61285 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -1,4 +1,3 @@ -import importlib.util import os import numpy as np @@ -44,7 +43,6 @@ class CfGribDataStore(AbstractDataStore): """ def __init__(self, filename, lock=None, **backend_kwargs): - import cfgrib if lock is None: lock = ECCODES_LOCK @@ -137,5 +135,9 @@ def open_backend_dataset_cfgrib( ) -if importlib.util.find_spec("cfgrib"): - BACKEND_ENTRYPOINTS["cfgrib"] = cfgrib_backend +try: + import cfgrib +except ImportError: + pass +else: + BACKEND_ENTRYPOINTS["pynio"] = cfgrib_backend diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index c686fe79b51..e53beda2864 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -1,5 +1,4 @@ import functools -import importlib.util import io import os from distutils.version import LooseVersion @@ -91,8 +90,6 @@ class H5NetCDFStore(WritableCFDataStore): def __init__(self, manager, group=None, mode=None, lock=HDF5_LOCK, autoclose=False): - import h5netcdf - if isinstance(manager, (h5netcdf.File, h5netcdf.Group)): if group is None: root, group = find_root_and_group(manager) @@ -128,7 +125,6 @@ def open( invalid_netcdf=None, phony_dims=None, ): - import h5netcdf if isinstance(filename, bytes): raise ValueError( @@ -383,5 +379,9 @@ def open_backend_dataset_h5netcdf( ) -if importlib.util.find_spec("h5netcdf"): - BACKEND_ENTRYPOINTS["h5netcdf"] = h5netcdf_backend +try: + import h5netcdf +except ImportError: + pass +else: + BACKEND_ENTRYPOINTS["pynio"] = h5netcdf_backend diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 19e2583b422..75a3d54c1a1 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -1,5 +1,4 @@ import functools -import importlib.util import operator import os import pathlib @@ -300,7 +299,6 @@ class NetCDF4DataStore(WritableCFDataStore): def __init__( self, manager, group=None, mode=None, lock=NETCDF4_PYTHON_LOCK, autoclose=False ): - import netCDF4 if isinstance(manager, netCDF4.Dataset): if group is None: @@ -567,5 +565,9 @@ def open_backend_dataset_netcdf4( ) -if importlib.util.find_spec("netCDF4"): - BACKEND_ENTRYPOINTS["netcdf4"] = netcdf4_backend +try: + import netCDF4 +except ImportError: + pass +else: + BACKEND_ENTRYPOINTS["pynio"] = netcdf4_backend diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index 7af79b4456d..0d4cfe979d0 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -1,5 +1,3 @@ -import importlib.util - import numpy as np from ..core import indexing @@ -47,7 +45,6 @@ class PseudoNetCDFDataStore(AbstractDataStore): @classmethod def open(cls, filename, lock=None, mode=None, **format_kwargs): - from PseudoNetCDF import pncopen keywords = {"kwargs": format_kwargs} # only include mode if explicitly passed @@ -57,7 +54,9 @@ def open(cls, filename, lock=None, mode=None, **format_kwargs): if lock is None: lock = PNETCDF_LOCK - manager = CachingFileManager(pncopen, filename, lock=lock, **keywords) + manager = CachingFileManager( + PseudoNetCDF.pncopen, filename, lock=lock, **keywords + ) return cls(manager, lock) def __init__(self, manager, lock=None): @@ -147,5 +146,9 @@ def open_backend_dataset_pseudonetcdf( ) -if importlib.util.find_spec("PseudoNetCDF"): - BACKEND_ENTRYPOINTS["pseudonetcdf"] = pseudonetcdf_backend +try: + import PseudoNetCDF +except ImportError: + pass +else: + BACKEND_ENTRYPOINTS["pynio"] = pseudonetcdf_backend diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index b65e672cea4..5f3a7038e28 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -1,5 +1,3 @@ -import importlib.util - import numpy as np from ..core import indexing @@ -82,7 +80,6 @@ def __init__(self, ds): @classmethod def open(cls, url, session=None): - import pydap.client ds = pydap.client.open_url(url, session=session) return cls(ds) @@ -143,5 +140,9 @@ def open_backend_dataset_pydap( ) -if importlib.util.find_spec("pydap"): - BACKEND_ENTRYPOINTS["pydap"] = pydap_backend +try: + import pydap.client +except ImportError: + pass +else: + BACKEND_ENTRYPOINTS["pynio"] = pydap_backend diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index 69ba31544d6..49ef93eae10 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -1,5 +1,3 @@ -import importlib.util - import numpy as np from ..core import indexing @@ -52,7 +50,7 @@ class NioDataStore(AbstractDataStore): """Store for accessing datasets via PyNIO""" def __init__(self, filename, mode="r", lock=None, **kwargs): - import Nio + # import Nio if lock is None: lock = PYNIO_LOCK @@ -128,5 +126,9 @@ def open_backend_dataset_pynio( pynio_backend = BackendEntrypoint(open_dataset=open_backend_dataset_pynio) -if importlib.util.find_spec("Nio"): +try: + import Nio +except ImportError: + pass +else: BACKEND_ENTRYPOINTS["pynio"] = pynio_backend diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 5add5e67726..43f1ab3bfe3 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -1,4 +1,3 @@ -import importlib.util import os import pathlib @@ -291,7 +290,6 @@ def open_group( append_dim=None, write_region=None, ): - import zarr # zarr doesn't support pathlib.Path objects yet. zarr-python#601 if isinstance(store, pathlib.Path): @@ -709,5 +707,9 @@ def open_backend_dataset_zarr( zarr_backend = BackendEntrypoint(open_dataset=open_backend_dataset_zarr) -if importlib.util.find_spec("zarr"): - BACKEND_ENTRYPOINTS["zarr"] = zarr_backend +try: + import zarr +except ImportError: + pass +else: + BACKEND_ENTRYPOINTS["pynio"] = zarr_backend From 05f4d4752a6742e832fbb1d727df74c117516cb7 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Fri, 15 Jan 2021 14:01:33 +0100 Subject: [PATCH 09/17] black --- xarray/backends/apiv2.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xarray/backends/apiv2.py b/xarray/backends/apiv2.py index 1e31f10d2fb..0f98291983d 100644 --- a/xarray/backends/apiv2.py +++ b/xarray/backends/apiv2.py @@ -262,7 +262,10 @@ def open_dataset( overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None) backend_ds = backend.open_dataset( - filename_or_obj, drop_variables=drop_variables, **decoders, **kwargs, + filename_or_obj, + drop_variables=drop_variables, + **decoders, + **kwargs, ) ds = _dataset_from_backend_dataset( backend_ds, From 52da30447dcc6588b7ae9ac7b75fb51b3f0eef89 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Fri, 15 Jan 2021 14:32:59 +0100 Subject: [PATCH 10/17] replace ImportError with ModuleNotFoundError --- xarray/backends/cfgrib_.py | 2 +- xarray/backends/h5netcdf_.py | 2 +- xarray/backends/netCDF4_.py | 2 +- xarray/backends/pseudonetcdf_.py | 2 +- xarray/backends/pydap_.py | 2 +- xarray/backends/pynio_.py | 2 +- xarray/backends/zarr.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 9d533d61285..8e76275641e 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -137,7 +137,7 @@ def open_backend_dataset_cfgrib( try: import cfgrib -except ImportError: +except ModuleNotFoundError: pass else: BACKEND_ENTRYPOINTS["pynio"] = cfgrib_backend diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index e53beda2864..fb5867fbac9 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -381,7 +381,7 @@ def open_backend_dataset_h5netcdf( try: import h5netcdf -except ImportError: +except c: pass else: BACKEND_ENTRYPOINTS["pynio"] = h5netcdf_backend diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 75a3d54c1a1..a101ef21bf1 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -567,7 +567,7 @@ def open_backend_dataset_netcdf4( try: import netCDF4 -except ImportError: +except ModuleNotFoundError: pass else: BACKEND_ENTRYPOINTS["pynio"] = netcdf4_backend diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index 0d4cfe979d0..51323f8461f 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -148,7 +148,7 @@ def open_backend_dataset_pseudonetcdf( try: import PseudoNetCDF -except ImportError: +except ModuleNotFoundError: pass else: BACKEND_ENTRYPOINTS["pynio"] = pseudonetcdf_backend diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index 5f3a7038e28..b44a3edf01f 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -142,7 +142,7 @@ def open_backend_dataset_pydap( try: import pydap.client -except ImportError: +except ModuleNotFoundError: pass else: BACKEND_ENTRYPOINTS["pynio"] = pydap_backend diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index 49ef93eae10..bea294ea265 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -128,7 +128,7 @@ def open_backend_dataset_pynio( try: import Nio -except ImportError: +except ModuleNotFoundError: pass else: BACKEND_ENTRYPOINTS["pynio"] = pynio_backend diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 43f1ab3bfe3..5ce9653720a 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -709,7 +709,7 @@ def open_backend_dataset_zarr( try: import zarr -except ImportError: +except ModuleNotFoundError: pass else: BACKEND_ENTRYPOINTS["pynio"] = zarr_backend From 82bf6fd1007bd2d80ffe915cca25f3f69180df51 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Fri, 15 Jan 2021 14:41:18 +0100 Subject: [PATCH 11/17] fix typo --- xarray/backends/h5netcdf_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index fb5867fbac9..e9eab42585c 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -381,7 +381,7 @@ def open_backend_dataset_h5netcdf( try: import h5netcdf -except c: +except ModuleNotFoundError: pass else: BACKEND_ENTRYPOINTS["pynio"] = h5netcdf_backend From 0fb9b61ce7768f8c426db7aac7b42656019f54e5 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Fri, 15 Jan 2021 14:52:51 +0100 Subject: [PATCH 12/17] fix typos --- xarray/backends/cfgrib_.py | 2 +- xarray/backends/h5netcdf_.py | 2 +- xarray/backends/netCDF4_.py | 2 +- xarray/backends/pseudonetcdf_.py | 2 +- xarray/backends/pydap_.py | 2 +- xarray/backends/zarr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 8e76275641e..9c6cc9c66e6 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -140,4 +140,4 @@ def open_backend_dataset_cfgrib( except ModuleNotFoundError: pass else: - BACKEND_ENTRYPOINTS["pynio"] = cfgrib_backend + BACKEND_ENTRYPOINTS["cfgrib"] = cfgrib_backend diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index e9eab42585c..99bb8d2427c 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -384,4 +384,4 @@ def open_backend_dataset_h5netcdf( except ModuleNotFoundError: pass else: - BACKEND_ENTRYPOINTS["pynio"] = h5netcdf_backend + BACKEND_ENTRYPOINTS["h5netcdf"] = h5netcdf_backend diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index a101ef21bf1..08ef2fefe65 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -570,4 +570,4 @@ def open_backend_dataset_netcdf4( except ModuleNotFoundError: pass else: - BACKEND_ENTRYPOINTS["pynio"] = netcdf4_backend + BACKEND_ENTRYPOINTS["netcdf4"] = netcdf4_backend diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index 51323f8461f..369974a298d 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -151,4 +151,4 @@ def open_backend_dataset_pseudonetcdf( except ModuleNotFoundError: pass else: - BACKEND_ENTRYPOINTS["pynio"] = pseudonetcdf_backend + BACKEND_ENTRYPOINTS["pseudonetcdf"] = pseudonetcdf_backend diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index b44a3edf01f..de65d0daf09 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -145,4 +145,4 @@ def open_backend_dataset_pydap( except ModuleNotFoundError: pass else: - BACKEND_ENTRYPOINTS["pynio"] = pydap_backend + BACKEND_ENTRYPOINTS["pydap"] = pydap_backend diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 5ce9653720a..159b087901c 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -712,4 +712,4 @@ def open_backend_dataset_zarr( except ModuleNotFoundError: pass else: - BACKEND_ENTRYPOINTS["pynio"] = zarr_backend + BACKEND_ENTRYPOINTS["zarr"] = zarr_backend From 9f3044fef9129dc9d46070afd9ff8476a0c0d34d Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Fri, 15 Jan 2021 21:54:58 +0100 Subject: [PATCH 13/17] remove else --- xarray/backends/cfgrib_.py | 4 ++-- xarray/backends/h5netcdf_.py | 4 ++-- xarray/backends/netCDF4_.py | 4 ++-- xarray/backends/pseudonetcdf_.py | 4 ++-- xarray/backends/pydap_.py | 4 ++-- xarray/backends/pynio_.py | 4 ++-- xarray/backends/scipy_.py | 9 +++++---- xarray/backends/zarr.py | 4 ++-- 8 files changed, 19 insertions(+), 18 deletions(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 9c6cc9c66e6..803d941551a 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -137,7 +137,7 @@ def open_backend_dataset_cfgrib( try: import cfgrib + + BACKEND_ENTRYPOINTS["cfgrib"] = cfgrib_backend except ModuleNotFoundError: pass -else: - BACKEND_ENTRYPOINTS["cfgrib"] = cfgrib_backend diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 99bb8d2427c..04fe365b496 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -381,7 +381,7 @@ def open_backend_dataset_h5netcdf( try: import h5netcdf + + BACKEND_ENTRYPOINTS["h5netcdf"] = h5netcdf_backend except ModuleNotFoundError: pass -else: - BACKEND_ENTRYPOINTS["h5netcdf"] = h5netcdf_backend diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 08ef2fefe65..5ad4f25c760 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -567,7 +567,7 @@ def open_backend_dataset_netcdf4( try: import netCDF4 + + BACKEND_ENTRYPOINTS["netcdf4"] = netcdf4_backend except ModuleNotFoundError: pass -else: - BACKEND_ENTRYPOINTS["netcdf4"] = netcdf4_backend diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index 369974a298d..a83ae0ffc5c 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -148,7 +148,7 @@ def open_backend_dataset_pseudonetcdf( try: import PseudoNetCDF + + BACKEND_ENTRYPOINTS["pseudonetcdf"] = pseudonetcdf_backend except ModuleNotFoundError: pass -else: - BACKEND_ENTRYPOINTS["pseudonetcdf"] = pseudonetcdf_backend diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index de65d0daf09..52865d05144 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -142,7 +142,7 @@ def open_backend_dataset_pydap( try: import pydap.client + + BACKEND_ENTRYPOINTS["pydap"] = pydap_backend except ModuleNotFoundError: pass -else: - BACKEND_ENTRYPOINTS["pydap"] = pydap_backend diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index bea294ea265..713a02617f7 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -128,7 +128,7 @@ def open_backend_dataset_pynio( try: import Nio + + BACKEND_ENTRYPOINTS["pynio"] = pynio_backend except ModuleNotFoundError: pass -else: - BACKEND_ENTRYPOINTS["pynio"] = pynio_backend diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index fd491dac701..5a5f041ca6d 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -1,4 +1,3 @@ -import importlib.util import io import os @@ -67,8 +66,6 @@ def __setitem__(self, key, value): def _open_scipy_netcdf(filename, mode, mmap, version): import gzip - import scipy.io - # if the string ends with .gz, then gunzip and open as netcdf file if isinstance(filename, str) and filename.endswith(".gz"): try: @@ -279,5 +276,9 @@ def open_backend_dataset_scipy( ) -if importlib.util.find_spec("scipy"): +try: + import scipy.io + BACKEND_ENTRYPOINTS["scipy"] = scipy_backend +except ModuleNotFoundError: + pass diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 159b087901c..01e58e22468 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -709,7 +709,7 @@ def open_backend_dataset_zarr( try: import zarr + + BACKEND_ENTRYPOINTS["zarr"] = zarr_backend except ModuleNotFoundError: pass -else: - BACKEND_ENTRYPOINTS["zarr"] = zarr_backend From 0454f48447eb84772d112d649cb4579354b62cbc Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Mon, 18 Jan 2021 14:49:08 +0100 Subject: [PATCH 14/17] Revert remove imports inside backends functions --- xarray/backends/common.py | 4 ++-- xarray/backends/h5netcdf_.py | 4 +++- xarray/backends/netCDF4_.py | 5 +++-- xarray/backends/pseudonetcdf_.py | 7 +++---- xarray/backends/pydap_.py | 3 ++- xarray/backends/pynio_.py | 4 ++-- xarray/backends/scipy_.py | 4 +++- xarray/backends/zarr.py | 5 +++-- 8 files changed, 21 insertions(+), 15 deletions(-) diff --git a/xarray/backends/common.py b/xarray/backends/common.py index 5c45eb8de18..adb70658fab 100644 --- a/xarray/backends/common.py +++ b/xarray/backends/common.py @@ -1,7 +1,7 @@ import logging import time import traceback -import typing as T +from typing import Dict import numpy as np @@ -352,4 +352,4 @@ def __init__(self, open_dataset, open_dataset_parameters=None, guess_can_open=No self.guess_can_open = guess_can_open -BACKEND_ENTRYPOINTS: T.Dict[str, BackendEntrypoint] = {} +BACKEND_ENTRYPOINTS: Dict[str, BackendEntrypoint] = {} diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 04fe365b496..c26b5b2e6b9 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -90,6 +90,8 @@ class H5NetCDFStore(WritableCFDataStore): def __init__(self, manager, group=None, mode=None, lock=HDF5_LOCK, autoclose=False): + import h5netcdf # noqa: F811 + if isinstance(manager, (h5netcdf.File, h5netcdf.Group)): if group is None: root, group = find_root_and_group(manager) @@ -380,7 +382,7 @@ def open_backend_dataset_h5netcdf( try: - import h5netcdf + import h5netcdf # noqa: F401 BACKEND_ENTRYPOINTS["h5netcdf"] = h5netcdf_backend except ModuleNotFoundError: diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 5ad4f25c760..9a37e3c2cc3 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -299,6 +299,7 @@ class NetCDF4DataStore(WritableCFDataStore): def __init__( self, manager, group=None, mode=None, lock=NETCDF4_PYTHON_LOCK, autoclose=False ): + import netCDF4 # noqa: F811 if isinstance(manager, netCDF4.Dataset): if group is None: @@ -335,7 +336,7 @@ def open( lock_maker=None, autoclose=False, ): - import netCDF4 + import netCDF4 # noqa: F811 if isinstance(filename, pathlib.Path): filename = os.fspath(filename) @@ -566,7 +567,7 @@ def open_backend_dataset_netcdf4( try: - import netCDF4 + import netCDF4 # noqa: F401 BACKEND_ENTRYPOINTS["netcdf4"] = netcdf4_backend except ModuleNotFoundError: diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index a83ae0ffc5c..2b9e79ff530 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -45,6 +45,7 @@ class PseudoNetCDFDataStore(AbstractDataStore): @classmethod def open(cls, filename, lock=None, mode=None, **format_kwargs): + from PseudoNetCDF import pncopen keywords = {"kwargs": format_kwargs} # only include mode if explicitly passed @@ -54,9 +55,7 @@ def open(cls, filename, lock=None, mode=None, **format_kwargs): if lock is None: lock = PNETCDF_LOCK - manager = CachingFileManager( - PseudoNetCDF.pncopen, filename, lock=lock, **keywords - ) + manager = CachingFileManager(pncopen, filename, lock=lock, **keywords) return cls(manager, lock) def __init__(self, manager, lock=None): @@ -147,7 +146,7 @@ def open_backend_dataset_pseudonetcdf( try: - import PseudoNetCDF + import PseudoNetCDF # noqa: F401 BACKEND_ENTRYPOINTS["pseudonetcdf"] = pseudonetcdf_backend except ModuleNotFoundError: diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index 52865d05144..b82a141c35d 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -80,6 +80,7 @@ def __init__(self, ds): @classmethod def open(cls, url, session=None): + import pydap.client # noqa F811 ds = pydap.client.open_url(url, session=session) return cls(ds) @@ -141,7 +142,7 @@ def open_backend_dataset_pydap( try: - import pydap.client + import pydap.client # noqa F401 BACKEND_ENTRYPOINTS["pydap"] = pydap_backend except ModuleNotFoundError: diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index 713a02617f7..ab980acd3c0 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -50,7 +50,7 @@ class NioDataStore(AbstractDataStore): """Store for accessing datasets via PyNIO""" def __init__(self, filename, mode="r", lock=None, **kwargs): - # import Nio + import Nio # noqa: F811 if lock is None: lock = PYNIO_LOCK @@ -127,7 +127,7 @@ def open_backend_dataset_pynio( try: - import Nio + import Nio # noqa: F401 BACKEND_ENTRYPOINTS["pynio"] = pynio_backend except ModuleNotFoundError: diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index 5a5f041ca6d..c3284a4516f 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -66,6 +66,8 @@ def __setitem__(self, key, value): def _open_scipy_netcdf(filename, mode, mmap, version): import gzip + import scipy.io # noqa: F811 + # if the string ends with .gz, then gunzip and open as netcdf file if isinstance(filename, str) and filename.endswith(".gz"): try: @@ -277,7 +279,7 @@ def open_backend_dataset_scipy( try: - import scipy.io + import scipy.io # noqa: F401 BACKEND_ENTRYPOINTS["scipy"] = scipy_backend except ModuleNotFoundError: diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 01e58e22468..8eaa44a22c3 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -290,6 +290,7 @@ def open_group( append_dim=None, write_region=None, ): + import zarr # noqa: F811 # zarr doesn't support pathlib.Path objects yet. zarr-python#601 if isinstance(store, pathlib.Path): @@ -409,7 +410,7 @@ def store( dimension on which the zarray will be appended only needed in append mode """ - import zarr + import zarr # noqa: F811 existing_variables = { vn for vn in variables if _encode_variable_name(vn) in self.ds @@ -708,7 +709,7 @@ def open_backend_dataset_zarr( try: - import zarr + import zarr # noqa: F401 BACKEND_ENTRYPOINTS["zarr"] = zarr_backend except ModuleNotFoundError: From 2f68f24bcfaa05b6666a5fed83e19b246463d80e Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Mon, 18 Jan 2021 14:55:26 +0100 Subject: [PATCH 15/17] Revert remove imports inside cfgrib --- xarray/backends/cfgrib_.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 803d941551a..465a0c9041a 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -43,6 +43,7 @@ class CfGribDataStore(AbstractDataStore): """ def __init__(self, filename, lock=None, **backend_kwargs): + import cfgrib # noqa: F811 if lock is None: lock = ECCODES_LOCK @@ -136,7 +137,7 @@ def open_backend_dataset_cfgrib( try: - import cfgrib + import cfgrib # noqa: F401 BACKEND_ENTRYPOINTS["cfgrib"] = cfgrib_backend except ModuleNotFoundError: From b67072590bfc40c1104fd0e7ef1b031524057f62 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Tue, 19 Jan 2021 09:30:39 +0100 Subject: [PATCH 16/17] modify check on imports inside the backends --- xarray/backends/cfgrib_.py | 15 +++++++++------ xarray/backends/h5netcdf_.py | 14 ++++++++------ xarray/backends/netCDF4_.py | 15 +++++++++------ xarray/backends/pseudonetcdf_.py | 15 +++++++++------ xarray/backends/pydap_.py | 14 ++++++++------ xarray/backends/pynio_.py | 15 +++++++++------ xarray/backends/scipy_.py | 15 ++++++++------- xarray/backends/zarr.py | 16 +++++++++------- 8 files changed, 69 insertions(+), 50 deletions(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 465a0c9041a..4a0ac7d67f9 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -14,6 +14,14 @@ from .locks import SerializableLock, ensure_lock from .store import open_backend_dataset_store +try: + import cfgrib + + has_cfgrib = True +except ModuleNotFoundError: + has_cfgrib = False + + # FIXME: Add a dedicated lock, even if ecCodes is supposed to be thread-safe # in most circumstances. See: # https://confluence.ecmwf.int/display/ECC/Frequently+Asked+Questions @@ -43,7 +51,6 @@ class CfGribDataStore(AbstractDataStore): """ def __init__(self, filename, lock=None, **backend_kwargs): - import cfgrib # noqa: F811 if lock is None: lock = ECCODES_LOCK @@ -136,9 +143,5 @@ def open_backend_dataset_cfgrib( ) -try: - import cfgrib # noqa: F401 - +if has_cfgrib: BACKEND_ENTRYPOINTS["cfgrib"] = cfgrib_backend -except ModuleNotFoundError: - pass diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index c26b5b2e6b9..53c39c5945a 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -25,6 +25,13 @@ ) from .store import open_backend_dataset_store +try: + import h5netcdf + + has_h5netcdf = True +except ModuleNotFoundError: + has_h5netcdf = False + class H5NetCDFArrayWrapper(BaseNetCDF4Array): def get_array(self, needs_lock=True): @@ -380,10 +387,5 @@ def open_backend_dataset_h5netcdf( open_dataset=open_backend_dataset_h5netcdf, guess_can_open=guess_can_open_h5netcdf ) - -try: - import h5netcdf # noqa: F401 - +if has_h5netcdf: BACKEND_ENTRYPOINTS["h5netcdf"] = h5netcdf_backend -except ModuleNotFoundError: - pass diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 9a37e3c2cc3..a659493ce72 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -24,6 +24,14 @@ from .netcdf3 import encode_nc3_attr_value, encode_nc3_variable from .store import open_backend_dataset_store +try: + import netCDF4 + + has_netcdf4 = True +except ModuleNotFoundError: + has_netcdf4 = False + + # This lookup table maps from dtype.byteorder to a readable endian # string used by netCDF4. _endian_lookup = {"=": "native", ">": "big", "<": "little", "|": "native"} @@ -299,7 +307,6 @@ class NetCDF4DataStore(WritableCFDataStore): def __init__( self, manager, group=None, mode=None, lock=NETCDF4_PYTHON_LOCK, autoclose=False ): - import netCDF4 # noqa: F811 if isinstance(manager, netCDF4.Dataset): if group is None: @@ -566,9 +573,5 @@ def open_backend_dataset_netcdf4( ) -try: - import netCDF4 # noqa: F401 - +if has_netcdf4: BACKEND_ENTRYPOINTS["netcdf4"] = netcdf4_backend -except ModuleNotFoundError: - pass diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index 2b9e79ff530..c2bfd519bed 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -13,6 +13,14 @@ from .locks import HDF5_LOCK, NETCDFC_LOCK, combine_locks, ensure_lock from .store import open_backend_dataset_store +try: + from PseudoNetCDF import pncopen + + has_pseudonetcdf = True +except ModuleNotFoundError: + has_pseudonetcdf = False + + # psuedonetcdf can invoke netCDF libraries internally PNETCDF_LOCK = combine_locks([HDF5_LOCK, NETCDFC_LOCK]) @@ -45,7 +53,6 @@ class PseudoNetCDFDataStore(AbstractDataStore): @classmethod def open(cls, filename, lock=None, mode=None, **format_kwargs): - from PseudoNetCDF import pncopen keywords = {"kwargs": format_kwargs} # only include mode if explicitly passed @@ -145,9 +152,5 @@ def open_backend_dataset_pseudonetcdf( ) -try: - import PseudoNetCDF # noqa: F401 - +if has_pseudonetcdf: BACKEND_ENTRYPOINTS["pseudonetcdf"] = pseudonetcdf_backend -except ModuleNotFoundError: - pass diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index b82a141c35d..c5ce943a10a 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -13,6 +13,13 @@ ) from .store import open_backend_dataset_store +try: + import pydap.client + + has_pydap = True +except ModuleNotFoundError: + has_pydap = False + class PydapArrayWrapper(BackendArray): def __init__(self, array): @@ -80,7 +87,6 @@ def __init__(self, ds): @classmethod def open(cls, url, session=None): - import pydap.client # noqa F811 ds = pydap.client.open_url(url, session=session) return cls(ds) @@ -141,9 +147,5 @@ def open_backend_dataset_pydap( ) -try: - import pydap.client # noqa F401 - +if has_pydap: BACKEND_ENTRYPOINTS["pydap"] = pydap_backend -except ModuleNotFoundError: - pass diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index ab980acd3c0..261daa69880 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -13,6 +13,14 @@ from .locks import HDF5_LOCK, NETCDFC_LOCK, SerializableLock, combine_locks, ensure_lock from .store import open_backend_dataset_store +try: + import Nio + + has_pynio = True +except ModuleNotFoundError: + has_pynio = False + + # PyNIO can invoke netCDF libraries internally # Add a dedicated lock just in case NCL as well isn't thread-safe. NCL_LOCK = SerializableLock() @@ -50,7 +58,6 @@ class NioDataStore(AbstractDataStore): """Store for accessing datasets via PyNIO""" def __init__(self, filename, mode="r", lock=None, **kwargs): - import Nio # noqa: F811 if lock is None: lock = PYNIO_LOCK @@ -126,9 +133,5 @@ def open_backend_dataset_pynio( pynio_backend = BackendEntrypoint(open_dataset=open_backend_dataset_pynio) -try: - import Nio # noqa: F401 - +if has_pynio: BACKEND_ENTRYPOINTS["pynio"] = pynio_backend -except ModuleNotFoundError: - pass diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index c3284a4516f..df51d07d686 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -17,6 +17,13 @@ from .netcdf3 import encode_nc3_attr_value, encode_nc3_variable, is_valid_nc3_name from .store import open_backend_dataset_store +try: + import scipy.io + + has_scipy = True +except ModuleNotFoundError: + has_scipy = False + def _decode_string(s): if isinstance(s, bytes): @@ -66,8 +73,6 @@ def __setitem__(self, key, value): def _open_scipy_netcdf(filename, mode, mmap, version): import gzip - import scipy.io # noqa: F811 - # if the string ends with .gz, then gunzip and open as netcdf file if isinstance(filename, str) and filename.endswith(".gz"): try: @@ -278,9 +283,5 @@ def open_backend_dataset_scipy( ) -try: - import scipy.io # noqa: F401 - +if has_scipy: BACKEND_ENTRYPOINTS["scipy"] = scipy_backend -except ModuleNotFoundError: - pass diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 8eaa44a22c3..ceeb23cac9b 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -17,6 +17,14 @@ ) from .store import open_backend_dataset_store +try: + import zarr + + has_zarr = True +except ModuleNotFoundError: + has_zarr = False + + # need some special secret attributes to tell us the dimensions DIMENSION_KEY = "_ARRAY_DIMENSIONS" @@ -290,7 +298,6 @@ def open_group( append_dim=None, write_region=None, ): - import zarr # noqa: F811 # zarr doesn't support pathlib.Path objects yet. zarr-python#601 if isinstance(store, pathlib.Path): @@ -410,7 +417,6 @@ def store( dimension on which the zarray will be appended only needed in append mode """ - import zarr # noqa: F811 existing_variables = { vn for vn in variables if _encode_variable_name(vn) in self.ds @@ -708,9 +714,5 @@ def open_backend_dataset_zarr( zarr_backend = BackendEntrypoint(open_dataset=open_backend_dataset_zarr) -try: - import zarr # noqa: F401 - +if has_zarr: BACKEND_ENTRYPOINTS["zarr"] = zarr_backend -except ModuleNotFoundError: - pass From c42d45e95f34648d28c0545895c304fbb095c539 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Tue, 19 Jan 2021 09:50:31 +0100 Subject: [PATCH 17/17] remove not used import --- xarray/backends/h5netcdf_.py | 2 -- xarray/backends/netCDF4_.py | 1 - 2 files changed, 3 deletions(-) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 53c39c5945a..562600de4b6 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -97,8 +97,6 @@ class H5NetCDFStore(WritableCFDataStore): def __init__(self, manager, group=None, mode=None, lock=HDF5_LOCK, autoclose=False): - import h5netcdf # noqa: F811 - if isinstance(manager, (h5netcdf.File, h5netcdf.Group)): if group is None: root, group = find_root_and_group(manager) diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index a659493ce72..5bb4eec837b 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -343,7 +343,6 @@ def open( lock_maker=None, autoclose=False, ): - import netCDF4 # noqa: F811 if isinstance(filename, pathlib.Path): filename = os.fspath(filename)