Raise errors on new warnings from within xarray (#8974)

* Raise an error on new warnings from within xarray

Author: max-sixty

.github/workflows/ci.yaml

@@ -81,10 +81,15 @@ jobs:
             if [[ "${{ matrix.env }}" == "flaky" ]] ;
             then
               echo "CONDA_ENV_FILE=ci/requirements/environment.yml" >> $GITHUB_ENV
-              echo "PYTEST_EXTRA_FLAGS=--run-flaky --run-network-tests" >> $GITHUB_ENV
+              echo "PYTEST_EXTRA_FLAGS=--run-flaky --run-network-tests -W default" >> $GITHUB_ENV
             else
               echo "CONDA_ENV_FILE=ci/requirements/${{ matrix.env }}.yml" >> $GITHUB_ENV
             fi
+            if [[ "${{ matrix.env }}" == "min-all-deps" ]] ;
+            then
+              # Don't raise on warnings
+              echo "PYTEST_EXTRA_FLAGS=-W default" >> $GITHUB_ENV
+            fi
           else
             if [[ ${{ matrix.python-version }} != "3.12" ]]; then
               echo "CONDA_ENV_FILE=ci/requirements/environment.yml" >> $GITHUB_ENV

pyproject.toml

@@ -282,9 +282,53 @@ ban-relative-imports = "all"
 
 [tool.pytest.ini_options]
 addopts = ["--strict-config", "--strict-markers"]
+
+# We want to forbid warnings from within xarray in our tests — instead we should
+# fix our own code, or mark the test itself as expecting a warning. So this:
+# - Converts any warning from xarray into an error
+# - Allows some warnings ("default") which the test suite currently raises,
+#   since it wasn't practical to fix them all before merging this config. The
+#   arnings are still listed in CI (since it uses `default`, not `ignore`).
+#
+# We can remove these rules allowing warnings; a valued contribution is removing
+# a line, seeing what breaks, and then fixing the library code or tests so that
+# it doesn't raise warnings.
+#
+# While we only raise an error on warnings from within xarray, if dependency
+# raises a warning with a stacklevel such that it's interpreted to be raised
+# from xarray, please feel free to add a rule switching it to `default` here.
+#
+# If these settings get in the way of making progress, it's also acceptable to
+# temporarily add additional ignores.
+
 filterwarnings = [
-  "ignore:Using a non-tuple sequence for multidimensional indexing is deprecated:FutureWarning",
+  "error:::xarray.*",
+  "default:No index created:UserWarning:xarray.core.dataset",
+  "default::UserWarning:xarray.tests.test_coding_times",
+  "default::UserWarning:xarray.tests.test_computation",
+  "default::UserWarning:xarray.tests.test_dataset",
+  "default:`ancestors` has been deprecated:DeprecationWarning:xarray.core.treenode",
+  "default:`iter_lineage` has been deprecated:DeprecationWarning:xarray.core.treenode",
+  "default:`lineage` has been deprecated:DeprecationWarning:xarray.core.treenode",
+  "default:coords should be an ndarray:DeprecationWarning:xarray.tests.test_variable",
+  "default:deallocating CachingFileManager:RuntimeWarning:xarray.backends.*",
+  "default:deallocating CachingFileManager:RuntimeWarning:xarray.backends.netCDF4_",
+  "default:deallocating CachingFileManager:RuntimeWarning:xarray.core.indexing",
+  "default:Failed to decode variable.*NumPy will stop allowing conversion of out-of-bound Python integers to integer arrays:DeprecationWarning",
+  "default:dropping variables using `drop` is deprecated; use drop_vars:DeprecationWarning:xarray.tests.test_groupby",
+  "default:The `interpolation` argument to quantile was renamed to `method`:FutureWarning:xarray.*",
+  "default:invalid value encountered in cast:RuntimeWarning:xarray.core.duck_array_ops",
+  "default:invalid value encountered in cast:RuntimeWarning:xarray.conventions",
+  "default:invalid value encountered in cast:RuntimeWarning:xarray.tests.test_units",
+  "default:invalid value encountered in cast:RuntimeWarning:xarray.tests.test_array_api",
+  "default:NumPy will stop allowing conversion of:DeprecationWarning",
+  "default:shape should be provided:DeprecationWarning:xarray.tests.test_variable",
+  "default:the `pandas.MultiIndex` object:FutureWarning:xarray.tests.test_variable",
+  "default:Using a non-tuple sequence for multidimensional indexing is deprecated:FutureWarning",
+  "default:Duplicate dimension names present:UserWarning:xarray.namedarray.core",
+  "default:::xarray.tests.test_strategies",
 ]
+
 log_cli_level = "INFO"
 markers = [
   "flaky: flaky tests",