Fixes for compat='no_conflicts' and open_mfdataset (#1007)

* Fixes for compat='no_conflicts' and open_mfdataset

- `xarray.merge` now uses `compat='no_conflicts'` by default.
- `compat='no_conflicts'` preserves attributes.
- `Variable.no_conflicts` broadcasts arguments, making it a strictly more
  relaxed check than `broadcast_equals`.
- Allow `concat_dim=None` in `open_mfdataset` and `auto_combine` to disable
  trying to infer a dimension for concatenating.
- Add `compat` as an argument to `open_mfdataset` to allow more control over
  merging.

* Fix what's new

* Updates to tests/doc for auto_combine

Author: shoyer

doc/combining.rst

@@ -193,6 +193,8 @@ numpy):
 Note that ``NaN`` does not compare equal to ``NaN`` in element-wise comparison;
 you may need to deal with missing values explicitly.
 
+.. _combining.no_conflicts:
+
 Merging with 'no_conflicts'
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/whats-new.rst

@@ -21,6 +21,11 @@ v0.9.0 (unreleased)
 Breaking changes
 ~~~~~~~~~~~~~~~~
 
+- The default behavior of ``merge`` is now ``compat='no_conflicts'``, so some
+  merges will now succeed in cases that previously raised
+  ``xarray.MergeError``. Set ``compat='broadcast_equals'`` to restore the
+  previous default.
+
 Deprecations
 ~~~~~~~~~~~~
 
@@ -59,8 +64,14 @@ By `Robin Wilson <https://github.com/robintw>`_.
 
 - Added the ``compat`` option ``'no_conflicts'`` to ``merge``, allowing the
   combination of xarray objects with disjoint (:issue:`742`) or
-  overlapping (:issue:`835`) coordinates as long as any present data agrees.
-  By `Johnnie Gray <https://github.com/jcmgray>`_.
+  overlapping (:issue:`835`) coordinates as long as all present data agrees.
+  By `Johnnie Gray <https://github.com/jcmgray>`_. See
+  :ref:`combining.no_conflicts` for more details.
+
+- It is now possible to set ``concat_dim=None`` explicitly in
+  :py:func:`~xarray.open_mfdataset` to disable inferring a dimension along
+  which to concatenate.
+  By `Stephan Hoyer <https://github.com/shoyer>`_.
 
 Bug fixes
 ~~~~~~~~~

xarray/backends/api.py

@@ -379,8 +379,12 @@ def close(self):
             f.close()
 
 
-def open_mfdataset(paths, chunks=None, concat_dim=None, preprocess=None,
-                   engine=None, lock=None, **kwargs):
+_CONCAT_DIM_DEFAULT = '__infer_concat_dim__'
+
+
+def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT,
+                   compat='no_conflicts', preprocess=None, engine=None,
+                   lock=None, **kwargs):
     """Open multiple files as a single dataset.
 
     Experimental. Requires dask to be installed.
@@ -397,12 +401,28 @@ def open_mfdataset(paths, chunks=None, concat_dim=None, preprocess=None,
         By default, chunks will be chosen to load entire input files into
         memory at once. This has a major impact on performance: please see the
         full documentation for more details.
-    concat_dim : str or DataArray or Index, optional
+    concat_dim : None, str, DataArray or Index, optional
         Dimension to concatenate files along. This argument is passed on to
         :py:func:`xarray.auto_combine` along with the dataset objects. You only
         need to provide this argument if the dimension along which you want to
         concatenate is not a dimension in the original datasets, e.g., if you
         want to stack a collection of 2D arrays along a third dimension.
+        By default, xarray attempts to infer this argument by examining
+        component files. Set ``concat_dim=None`` explicitly to disable
+        concatenation.
+    compat : {'identical', 'equals', 'broadcast_equals',
+              'no_conflicts'}, optional
+        String indicating how to compare variables of the same name for
+        potential conflicts when merging:
+
+        - 'broadcast_equals': all values must be equal when variables are
+          broadcast against each other to ensure common dimensions.
+        - 'equals': all values and dimensions must be the same.
+        - 'identical': all values, dimensions and attributes must be the
+          same.
+        - 'no_conflicts': only values which are not null in both datasets
+          must be equal. The returned dataset then contains the combination
+          of all non-null values.
     preprocess : callable, optional
         If provided, call this function on each dataset prior to concatenation.
     engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio'}, optional
@@ -440,7 +460,10 @@ def open_mfdataset(paths, chunks=None, concat_dim=None, preprocess=None,
     if preprocess is not None:
         datasets = [preprocess(ds) for ds in datasets]
 
-    combined = auto_combine(datasets, concat_dim=concat_dim)
+    if concat_dim is _CONCAT_DIM_DEFAULT:
+        combined = auto_combine(datasets, compat=compat)
+    else:
+        combined = auto_combine(datasets, concat_dim=concat_dim, compat=compat)
     combined._file_obj = _MultiFileCloser(file_objs)
     return combined
 

xarray/core/combine.py

@@ -329,28 +329,25 @@ def _auto_concat(datasets, dim=None):
         return concat(datasets, dim=dim)
 
 
-def auto_combine(datasets, concat_dim=None):
+_CONCAT_DIM_DEFAULT = '__infer_concat_dim__'
+
+
+def auto_combine(datasets,
+                 concat_dim=_CONCAT_DIM_DEFAULT,
+                 compat='no_conflicts'):
     """Attempt to auto-magically combine the given datasets into one.
 
     This method attempts to combine a list of datasets into a single entity by
     inspecting metadata and using a combination of concat and merge.
 
-    It does not concatenate along more than one dimension or align or sort data
-    under any circumstances. It will fail in complex cases, for which you
-    should use ``concat`` and ``merge`` explicitly.
-
-    When ``auto_combine`` may succeed:
+    It does not concatenate along more than one dimension or sort data under any
+    circumstances. It does align coordinates, but different variables on
+    datasets can cause it to fail under some scenarios. In complex cases, you
+    may need to clean up your data and use ``concat``/``merge`` explicitly.
 
-    * You have N years of data and M data variables. Each combination of a
-      distinct time period and test of data variables is saved its own dataset.
-
-    Examples of when ``auto_combine`` fails:
-
-    * In the above scenario, one file is missing, containing the data for one
-      year's data for one variable.
-    * In the most recent year, there is an additional data variable.
-    * Your data includes "time" and "station" dimensions, and each year's data
-      has a different set of stations.
+    ``auto_combine`` works well if you have N years of data and M data
+    variables, and each combination of a distinct time period and set of data
+    variables is saved its own dataset.
 
     Parameters
     ----------
@@ -362,6 +359,22 @@ def auto_combine(datasets, concat_dim=None):
         dimension along which you want to concatenate is not a dimension in
         the original datasets, e.g., if you want to stack a collection of
         2D arrays along a third dimension.
+        By default, xarray attempts to infer this argument by examining
+        component files. Set ``concat_dim=None`` explicitly to disable
+        concatenation.
+    compat : {'identical', 'equals', 'broadcast_equals',
+              'no_conflicts'}, optional
+        String indicating how to compare variables of the same name for
+        potential conflicts:
+
+        - 'broadcast_equals': all values must be equal when variables are
+          broadcast against each other to ensure common dimensions.
+        - 'equals': all values and dimensions must be the same.
+        - 'identical': all values, dimensions and attributes must be the
+          same.
+        - 'no_conflicts': only values which are not null in both datasets
+          must be equal. The returned dataset then contains the combination
+          of all non-null values.
 
     Returns
     -------
@@ -373,8 +386,12 @@ def auto_combine(datasets, concat_dim=None):
     Dataset.merge
     """
     from toolz import itertoolz
-    grouped = itertoolz.groupby(lambda ds: tuple(sorted(ds.data_vars)),
-                                datasets).values()
-    concatenated = [_auto_concat(ds, dim=concat_dim) for ds in grouped]
-    merged = merge(concatenated)
+    if concat_dim is not None:
+        dim = None if concat_dim is _CONCAT_DIM_DEFAULT else concat_dim
+        grouped = itertoolz.groupby(lambda ds: tuple(sorted(ds.data_vars)),
+                                    datasets).values()
+        concatenated = [_auto_concat(ds, dim=dim) for ds in grouped]
+    else:
+        concatenated = datasets
+    merged = merge(concatenated, compat=compat)
     return merged

xarray/core/dataset.py

@@ -1439,8 +1439,8 @@ def update(self, other, inplace=True):
         return self._replace_vars_and_dims(variables, coord_names, dims,
                                            inplace=inplace)
 
-    def merge(self, other, inplace=False, overwrite_vars=set(),
-              compat='broadcast_equals', join='outer'):
+    def merge(self, other, inplace=False, overwrite_vars=frozenset(),
+              compat='no_conflicts', join='outer'):
         """Merge the arrays of two datasets into a single dataset.
 
         This method generally not allow for overriding data, with the exception
@@ -1490,7 +1490,8 @@ def merge(self, other, inplace=False, overwrite_vars=set(),
             If any variables conflict (see ``compat``).
         """
         variables, coord_names, dims = dataset_merge_method(
-            self, other, overwrite_vars, compat=compat, join=join)
+            self, other, overwrite_vars=overwrite_vars, compat=compat,
+            join=join)
 
         return self._replace_vars_and_dims(variables, coord_names, dims,
                                            inplace=inplace)

xarray/core/merge.py

@@ -82,7 +82,9 @@ def unique_variable(name, variables, compat='broadcast_equals'):
                                  'first value: %r\nsecond value: %r'
                                  % (name, out, var))
             if combine_method:
+                # TODO: add preservation of attrs into fillna
                 out = getattr(out, combine_method)(var)
+                out.attrs = var.attrs
 
     return out
 
@@ -397,8 +399,12 @@ def merge_data_and_coords(data, coords, compat='broadcast_equals',
     return merge_core(objs, compat, join, explicit_coords=explicit_coords)
 
 
-def merge_core(objs, compat='broadcast_equals', join='outer', priority_arg=None,
-               explicit_coords=None, indexes=None):
+def merge_core(objs,
+               compat='broadcast_equals',
+               join='outer',
+               priority_arg=None,
+               explicit_coords=None,
+               indexes=None):
     """Core logic for merging labeled objects.
 
     This is not public API.
@@ -466,7 +472,7 @@ def merge_core(objs, compat='broadcast_equals', join='outer', priority_arg=None,
     return variables, coord_names, dict(dims)
 
 
-def merge(objects, compat='broadcast_equals', join='outer'):
+def merge(objects, compat='no_conflicts', join='outer'):
     """Merge any number of xarray objects into a single Dataset as variables.
 
     Parameters
@@ -476,7 +482,17 @@ def merge(objects, compat='broadcast_equals', join='outer'):
         DataArray objects, they must have a name.
     compat : {'identical', 'equals', 'broadcast_equals',
               'no_conflicts'}, optional
-        Compatibility checks to use when merging variables.
+        String indicating how to compare variables of the same name for
+        potential conflicts:
+
+        - 'broadcast_equals': all values must be equal when variables are
+          broadcast against each other to ensure common dimensions.
+        - 'equals': all values and dimensions must be the same.
+        - 'identical': all values, dimensions and attributes must be the
+          same.
+        - 'no_conflicts': only values which are not null in both datasets
+          must be equal. The returned dataset then contains the combination
+          of all non-null values.
     join : {'outer', 'inner', 'left', 'right'}, optional
         How to combine objects with different indexes.
 
@@ -521,8 +537,7 @@ def merge(objects, compat='broadcast_equals', join='outer'):
     return merged
 
 
-def dataset_merge_method(dataset, other, overwrite_vars=frozenset(),
-                         compat='broadcast_equals', join='outer'):
+def dataset_merge_method(dataset, other, overwrite_vars, compat, join):
     """Guts of the Dataset.merge method."""
 
     # we are locked into supporting overwrite_vars for the Dataset.merge

xarray/core/variable.py

@@ -974,11 +974,7 @@ def concat(cls, variables, dim='concat_dim', positions=None,
 
         return cls(dims, data, attrs)
 
-    def _data_equals(self, other):
-        return (self._data is other._data or
-                ops.array_equiv(self.data, other.data))
-
-    def equals(self, other):
+    def equals(self, other, equiv=ops.array_equiv):
         """True if two Variables have the same dimensions and values;
         otherwise False.
 
@@ -990,11 +986,13 @@ def equals(self, other):
         """
         other = getattr(other, 'variable', other)
         try:
-            return (self.dims == other.dims and self._data_equals(other))
+            return (self.dims == other.dims and
+                    (self._data is other._data or
+                     equiv(self.data, other.data)))
         except (TypeError, AttributeError):
             return False
 
-    def broadcast_equals(self, other):
+    def broadcast_equals(self, other, equiv=ops.array_equiv):
         """True if two Variables have the values after being broadcast against
         each other; otherwise False.
 
@@ -1005,7 +1003,7 @@ def broadcast_equals(self, other):
             self, other = broadcast_variables(self, other)
         except (ValueError, AttributeError):
             return False
-        return self.equals(other)
+        return self.equals(other, equiv=equiv)
 
     def identical(self, other):
         """Like equals, but also checks attributes.
@@ -1016,23 +1014,14 @@ def identical(self, other):
         except (TypeError, AttributeError):
             return False
 
-    def _data_no_conflicts(self, other):
-        return (self._data is other._data or
-                ops.array_notnull_equiv(self.data, other.data))
-
     def no_conflicts(self, other):
         """True if the intersection of two Variable's non-null data is
         equal; otherwise false.
 
         Variables can thus still be equal if there are locations where either,
         or both, contain NaN values.
         """
-        other = getattr(other, 'variable', other)
-        try:
-            return (self.dims == other.dims and
-                    self._data_no_conflicts(other))
-        except (TypeError, AttributeError):
-            return False
+        return self.broadcast_equals(other, equiv=ops.array_notnull_equiv)
 
     @property
     def real(self):

xarray/test/test_backends.py

@@ -942,6 +942,15 @@ def test_open_and_do_math(self):
                 actual = 1.0 * ds
                 self.assertDatasetAllClose(original, actual)
 
+    def test_open_mfdataset_concat_dim_none(self):
+        with create_tmp_file() as tmp1:
+            with create_tmp_file() as tmp2:
+                data = Dataset({'x': 0})
+                data.to_netcdf(tmp1)
+                Dataset({'x': np.nan}).to_netcdf(tmp2)
+                with open_mfdataset([tmp1, tmp2], concat_dim=None) as actual:
+                    self.assertDatasetIdentical(data, actual)
+
     def test_open_dataset(self):
         original = Dataset({'foo': ('x', np.random.randn(10))})
         with create_tmp_file() as tmp: