Merge pull request #532 from markelg/master

Add a --drop-variables flag to xray.open_dataset to exclude certain variables

Author: shoyer

doc/whats-new.rst

@@ -87,7 +87,9 @@ v0.5.3 (unreleased)
     da = xray.DataArray(np.random.random_sample(size=(5, 4)))
     da.where(da < 0.5)
     da.where(da < 0.5).to_masked_array(copy=True)
-
+- Added new flag "drop_variables" to :py:meth:`~xray.open_dataset` for
+  excluding variables from being parsed. This may be useful to drop 
+  variables with problems or inconsistent values.
 
 Bug fixes
 ~~~~~~~~~

xray/backends/api.py

@@ -64,7 +64,7 @@ def _default_lock(filename, engine):
 def open_dataset(filename_or_obj, group=None, decode_cf=True,
                  mask_and_scale=True, decode_times=True,
                  concat_characters=True, decode_coords=True, engine=None,
-                 chunks=None, lock=None):
+                 chunks=None, lock=None, drop_variables=None):
     """Load and decode a dataset from a file or file-like object.
 
     Parameters
@@ -114,6 +114,10 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True,
         used when reading data from netCDF files with the netcdf4 and h5netcdf
         engines to avoid issues with concurrent access when using dask's
         multithreaded backend.
+    drop_variables: string or iterable, optional 
+        A variable or list of variables to exclude from being parsed from the
+        dataset. This may be useful to drop variables with problems or
+        inconsistent values. 
 
     Returns
     -------
@@ -133,7 +137,8 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True,
     def maybe_decode_store(store, lock=False):
         ds = conventions.decode_cf(
             store, mask_and_scale=mask_and_scale, decode_times=decode_times,
-            concat_characters=concat_characters, decode_coords=decode_coords)
+            concat_characters=concat_characters, decode_coords=decode_coords,
+            drop_variables=drop_variables)
         if chunks is not None:
             ds = ds.chunk(chunks, lock=lock)
         return ds

xray/conventions.py

@@ -12,7 +12,7 @@
 from .core.formatting import format_timestamp, first_n_items
 from .core.variable import as_variable, Variable
 from .core.pycompat import (iteritems, bytes_type, unicode_type, OrderedDict,
-                            PY3)
+                            PY3, basestring)
 
 
 # standard calendars recognized by netcdftime
@@ -783,7 +783,7 @@ def decode_cf_variable(var, concat_characters=True, mask_and_scale=True,
 
 def decode_cf_variables(variables, attributes, concat_characters=True,
                         mask_and_scale=True, decode_times=True,
-                        decode_coords=True):
+                        decode_coords=True, drop_variables=None):
     """
     Decode a several CF encoded variables.
 
@@ -805,8 +805,16 @@ def stackable(dim):
 
     coord_names = set()
 
+    if isinstance(drop_variables, basestring):
+        drop_variables = [drop_variables]
+    elif drop_variables is None:
+        drop_variables = []
+    drop_variables = set(drop_variables)
+
     new_vars = OrderedDict()
     for k, v in iteritems(variables):
+        if k in drop_variables:
+            continue
         concat = (concat_characters and v.dtype.kind == 'S' and v.ndim > 0 and
                   stackable(v.dims[-1]))
         new_vars[k] = decode_cf_variable(
@@ -828,7 +836,7 @@ def stackable(dim):
 
 
 def decode_cf(obj, concat_characters=True, mask_and_scale=True,
-              decode_times=True, decode_coords=True):
+              decode_times=True, decode_coords=True, drop_variables=None):
     """Decode the given Dataset or Datastore according to CF conventions into
     a new Dataset.
 
@@ -848,7 +856,11 @@ def decode_cf(obj, concat_characters=True, mask_and_scale=True,
     decode_coords : bool, optional
         Use the 'coordinates' attribute on variable (or the dataset itself) to
         identify coordinates.
-
+    drop_variables: string or iterable, optional 
+        A variable or list of variables to exclude from being parsed from the
+        dataset.This may be useful to drop variables with problems or
+        inconsistent values. 
+    
     Returns
     -------
     decoded : Dataset
@@ -870,7 +882,7 @@ def decode_cf(obj, concat_characters=True, mask_and_scale=True,
 
     vars, attrs, coord_names = decode_cf_variables(
         vars, attrs, concat_characters, mask_and_scale, decode_times,
-        decode_coords)
+        decode_coords, drop_variables=drop_variables)
     ds = Dataset(vars, attrs=attrs)
     ds = ds.set_coords(coord_names.union(extra_coords))
     ds._file_obj = file_obj

xray/test/test_conventions.py

@@ -491,7 +491,23 @@ def test_decode_cf_with_multiple_missing_values(self):
             actual = conventions.decode_cf_variable(original)
             self.assertDatasetIdentical(expected, actual)
             self.assertIn('variable has multiple fill', str(w[0].message))
-
+    def test_decode_cf_with_drop_variables(self):
+        original = Dataset({
+            't': ('t', [0, 1, 2], {'units': 'days since 2000-01-01'}),
+            'x' : ("x", [9, 8, 7], {'units' : 'km'}),
+            'foo': (('t', 'x'), [[0, 0, 0], [1, 1, 1], [2, 2, 2]], {'units': 'bar'}),
+            'y': ('t', [5, 10, -999], {'_FillValue': -999})
+        })
+        expected = Dataset({
+            't': pd.date_range('2000-01-01', periods=3),
+            'x' : ("x", [0, 1, 2]),
+            'foo': (('t', 'x'), [[0, 0, 0], [1, 1, 1], [2, 2, 2]], {'units': 'bar'}),
+            'y': ('t', [5, 10, np.nan])
+        })
+        actual = conventions.decode_cf(original, drop_variables=("x",))
+        actual2 = conventions.decode_cf(original, drop_variables="x")
+        self.assertDatasetIdentical(expected, actual)
+        self.assertDatasetIdentical(expected, actual2)
 
 class CFEncodedInMemoryStore(InMemoryDataStore):
     def store(self, variables, attributes):