Port all the engines to apiv2

setup.cfg

@@ -86,6 +86,12 @@ xarray.backends =
     zarr = xarray.backends.zarr:zarr_backend
     h5netcdf = xarray.backends.h5netcdf_:h5netcdf_backend
     cfgrib = xarray.backends.cfgrib_:cfgrib_backend
+    scipy = xarray.backends.scipy_:scipy_backend
+    pynio = xarray.backends.pynio_:pynio_backend
+    pseudonetcdf = xarray.backends.pseudonetcdf_:pseudonetcdf_backend
+    netcdf4 = xarray.backends.netCDF4_:netcdf4_backend
+    store = xarray.backends.store:store_backend
+
 
 [options.extras_require]
 io =

xarray/backends/api.py

@@ -157,7 +157,9 @@ def _get_default_engine(path, allow_remote=False):
 
 
 def _autodetect_engine(filename_or_obj):
-    if isinstance(filename_or_obj, str):
+    if isinstance(filename_or_obj, AbstractDataStore):
+        engine = "store"
+    elif isinstance(filename_or_obj, str):
         engine = _get_default_engine(filename_or_obj, allow_remote=True)
     else:
         engine = _get_engine_from_magic_number(filename_or_obj)
@@ -436,11 +438,10 @@ def open_dataset(
     open_mfdataset
     """
     if os.environ.get("XARRAY_BACKEND_API", "v1") == "v2":
-        kwargs = locals().copy()
-        from . import apiv2, plugins
+        kwargs = locals()
+        from . import apiv2
 
-        if engine in plugins.list_engines():
-            return apiv2.open_dataset(**kwargs)
+        return apiv2.open_dataset(**kwargs)
 
     if autoclose is not None:
         warnings.warn(

xarray/backends/apiv2.py

@@ -1,4 +1,5 @@
 import os
+import warnings
 
 from ..core.dataset import _get_chunk, _maybe_chunk
 from ..core.utils import is_remote_uri
@@ -112,6 +113,7 @@ def open_dataset(
     concat_characters=None,
     decode_coords=None,
     drop_variables=None,
+    autoclose=None,
     backend_kwargs=None,
     **kwargs,
 ):
@@ -226,6 +228,16 @@ def open_dataset(
     --------
     open_mfdataset
     """
+    if autoclose is not None:
+        warnings.warn(
+            "The autoclose argument is no longer used by "
+            "xarray.open_dataset() and is now ignored; it will be removed in "
+            "a future version of xarray. If necessary, you can control the "
+            "maximum number of simultaneous open files with "
+            "xarray.set_options(file_cache_maxsize=...).",
+            FutureWarning,
+            stacklevel=2,
+        )
 
     if cache is None:
         cache = chunks is None

xarray/backends/cfgrib_.py

@@ -1,13 +1,12 @@
 import numpy as np
 
-from .. import conventions
 from ..core import indexing
-from ..core.dataset import Dataset
-from ..core.utils import Frozen, FrozenDict, close_on_error
+from ..core.utils import Frozen, FrozenDict
 from ..core.variable import Variable
 from .common import AbstractDataStore, BackendArray
 from .locks import SerializableLock, ensure_lock
 from .plugins import BackendEntrypoint
+from .store import open_backend_dataset_store
 
 # FIXME: Add a dedicated lock, even if ecCodes is supposed to be thread-safe
 #   in most circumstances. See:
@@ -104,28 +103,16 @@ def open_backend_dataset_cfgrib(
         lock=lock,
     )
 
-    with close_on_error(store):
-        vars, attrs = store.load()
-        file_obj = store
-        encoding = store.get_encoding()
-
-        vars, attrs, coord_names = conventions.decode_cf_variables(
-            vars,
-            attrs,
-            mask_and_scale=mask_and_scale,
-            decode_times=decode_times,
-            concat_characters=concat_characters,
-            decode_coords=decode_coords,
-            drop_variables=drop_variables,
-            use_cftime=use_cftime,
-            decode_timedelta=decode_timedelta,
-        )
-
-        ds = Dataset(vars, attrs=attrs)
-        ds = ds.set_coords(coord_names.intersection(vars))
-        ds._file_obj = file_obj
-        ds.encoding = encoding
-
+    ds = open_backend_dataset_store(
+        store,
+        mask_and_scale=mask_and_scale,
+        decode_times=decode_times,
+        concat_characters=concat_characters,
+        decode_coords=decode_coords,
+        drop_variables=drop_variables,
+        use_cftime=use_cftime,
+        decode_timedelta=decode_timedelta,
+    )
     return ds
 
 

xarray/backends/h5netcdf_.py

@@ -3,10 +3,8 @@
 
 import numpy as np
 
-from .. import conventions
 from ..core import indexing
-from ..core.dataset import Dataset
-from ..core.utils import FrozenDict, close_on_error, is_remote_uri
+from ..core.utils import FrozenDict, is_remote_uri
 from ..core.variable import Variable
 from .common import WritableCFDataStore, find_root_and_group
 from .file_manager import CachingFileManager, DummyFileManager
@@ -19,6 +17,7 @@
     _nc4_require_group,
 )
 from .plugins import BackendEntrypoint
+from .store import open_backend_dataset_store
 
 
 class H5NetCDFArrayWrapper(BaseNetCDF4Array):
@@ -352,28 +351,16 @@ def open_backend_dataset_h5netcdf(
         phony_dims=phony_dims,
     )
 
-    with close_on_error(store):
-        vars, attrs = store.load()
-        file_obj = store
-        encoding = store.get_encoding()
-
-        vars, attrs, coord_names = conventions.decode_cf_variables(
-            vars,
-            attrs,
-            mask_and_scale=mask_and_scale,
-            decode_times=decode_times,
-            concat_characters=concat_characters,
-            decode_coords=decode_coords,
-            drop_variables=drop_variables,
-            use_cftime=use_cftime,
-            decode_timedelta=decode_timedelta,
-        )
-
-        ds = Dataset(vars, attrs=attrs)
-        ds = ds.set_coords(coord_names.intersection(vars))
-        ds._file_obj = file_obj
-        ds.encoding = encoding
-
+    ds = open_backend_dataset_store(
+        store,
+        mask_and_scale=mask_and_scale,
+        decode_times=decode_times,
+        concat_characters=concat_characters,
+        decode_coords=decode_coords,
+        drop_variables=drop_variables,
+        use_cftime=use_cftime,
+        decode_timedelta=decode_timedelta,
+    )
     return ds
 
 

xarray/backends/netCDF4_.py

@@ -18,6 +18,8 @@
 from .file_manager import CachingFileManager, DummyFileManager
 from .locks import HDF5_LOCK, NETCDFC_LOCK, combine_locks, ensure_lock, get_write_lock
 from .netcdf3 import encode_nc3_attr_value, encode_nc3_variable
+from .plugins import BackendEntrypoint
+from .store import open_backend_dataset_store
 
 # This lookup table maps from dtype.byteorder to a readable endian
 # string used by netCDF4.
@@ -496,3 +498,50 @@ def sync(self):
 
     def close(self, **kwargs):
         self._manager.close(**kwargs)
+
+
+def open_backend_dataset_netcdf4(
+    filename_or_obj,
+    mask_and_scale=True,
+    decode_times=None,
+    concat_characters=None,
+    decode_coords=None,
+    drop_variables=None,
+    use_cftime=None,
+    decode_timedelta=None,
+    group=None,
+    mode="r",
+    format="NETCDF4",
+    clobber=True,
+    diskless=False,
+    persist=False,
+    lock=None,
+    autoclose=False,
+):
+
+    store = NetCDF4DataStore.open(
+        filename_or_obj,
+        mode=mode,
+        format=format,
+        group=group,
+        clobber=clobber,
+        diskless=diskless,
+        persist=persist,
+        lock=lock,
+        autoclose=autoclose,
+    )
+
+    ds = open_backend_dataset_store(
+        store,
+        mask_and_scale=mask_and_scale,
+        decode_times=decode_times,
+        concat_characters=concat_characters,
+        decode_coords=decode_coords,
+        drop_variables=drop_variables,
+        use_cftime=use_cftime,
+        decode_timedelta=decode_timedelta,
+    )
+    return ds
+
+
+netcdf4_backend = BackendEntrypoint(open_dataset=open_backend_dataset_netcdf4)

xarray/backends/pseudonetcdf_.py

@@ -6,6 +6,8 @@
 from .common import AbstractDataStore, BackendArray
 from .file_manager import CachingFileManager
 from .locks import HDF5_LOCK, NETCDFC_LOCK, combine_locks, ensure_lock
+from .plugins import BackendEntrypoint
+from .store import open_backend_dataset_store
 
 # psuedonetcdf can invoke netCDF libraries internally
 PNETCDF_LOCK = combine_locks([HDF5_LOCK, NETCDFC_LOCK])
@@ -85,3 +87,54 @@ def get_encoding(self):
 
     def close(self):
         self._manager.close()
+
+
+def open_backend_dataset_pseudonetcdf(
+    filename_or_obj,
+    mask_and_scale=False,
+    decode_times=None,
+    concat_characters=None,
+    decode_coords=None,
+    drop_variables=None,
+    use_cftime=None,
+    decode_timedelta=None,
+    mode=None,
+    lock=None,
+    **format_kwargs,
+):
+
+    store = PseudoNetCDFDataStore.open(
+        filename_or_obj, lock=lock, mode=mode, **format_kwargs
+    )
+
+    ds = open_backend_dataset_store(
+        store,
+        mask_and_scale=mask_and_scale,
+        decode_times=decode_times,
+        concat_characters=concat_characters,
+        decode_coords=decode_coords,
+        drop_variables=drop_variables,
+        use_cftime=use_cftime,
+        decode_timedelta=decode_timedelta,
+    )
+    return ds
+
+
+# *args and **kwargs are not allowed in open_backend_dataset_ kwargs,
+# unless the open_dataset_parameters are explicity defined like this:
+open_dataset_parameters = (
+    "filename_or_obj",
+    "mask_and_scale",
+    "decode_times",
+    "concat_characters",
+    "decode_coords",
+    "drop_variables",
+    "use_cftime",
+    "decode_timedelta",
+    "mode",
+    "lock",
+)
+pseudonetcdf_backend = BackendEntrypoint(
+    open_dataset=open_backend_dataset_pseudonetcdf,
+    open_dataset_parameters=open_dataset_parameters,
+)

xarray/backends/pydap_.py

@@ -5,6 +5,8 @@
 from ..core.utils import Frozen, FrozenDict, is_dict_like
 from ..core.variable import Variable
 from .common import AbstractDataStore, BackendArray, robust_getitem
+from .plugins import BackendEntrypoint
+from .store import open_backend_dataset_store
 
 
 class PydapArrayWrapper(BackendArray):
@@ -92,3 +94,36 @@ def get_attrs(self):
 
     def get_dimensions(self):
         return Frozen(self.ds.dimensions)
+
+
+def open_backend_dataset_pydap(
+    filename_or_obj,
+    mask_and_scale=True,
+    decode_times=None,
+    concat_characters=None,
+    decode_coords=None,
+    drop_variables=None,
+    use_cftime=None,
+    decode_timedelta=None,
+    session=None,
+):
+
+    store = PydapDataStore.open(
+        filename_or_obj,
+        session=session,
+    )
+
+    ds = open_backend_dataset_store(
+        store,
+        mask_and_scale=mask_and_scale,
+        decode_times=decode_times,
+        concat_characters=concat_characters,
+        decode_coords=decode_coords,
+        drop_variables=drop_variables,
+        use_cftime=use_cftime,
+        decode_timedelta=decode_timedelta,
+    )
+    return ds
+
+
+pydap_backend = BackendEntrypoint(open_dataset=open_backend_dataset_pydap)