Adds marked tests as slow, flakey, and optionalci

Tests are marked using the following py.test options:

 * @slow which is run on default, not run if `--skip-slow`
 * @optionalci is run on default, not run if `--skip-optional-ci`
 * @flakey which is not run on default, run if `--run-flakey`

Note, the `optionalci` category is needed in order to mark
tests that are environment dependent when run on continuous
integration (CI) hardware.

Author: pwolfram

.travis.yml

@@ -12,14 +12,24 @@ matrix:
   include:
   - python: 2.7
     env: CONDA_ENV=py27-min
+  - python: 2.7
+    env: CONDA_ENV=py27-min SKIP_CI=--skip-optional-ci
   - python: 2.7
     env: CONDA_ENV=py27-cdat+pynio
+  - python: 2.7
+    env: CONDA_ENV=py27-cdat+pynio SKIP_CI=--skip-optional-ci
   - python: 3.4
     env: CONDA_ENV=py34
+  - python: 3.4
+    env: CONDA_ENV=py34 SKIP_CI=--skip-optional-ci
   - python: 3.5
     env: CONDA_ENV=py35
+  - python: 3.5
+    env: CONDA_ENV=py35 SKIP_CI=--skip-optional-ci
   - python: 3.6
     env: CONDA_ENV=py36
+  - python: 3.6
+    env: CONDA_ENV=py36 SKIP_CI=--skip-optional-ci
   - python: 3.6
     env: CONDA_ENV=py36-pydap
   - python: 3.6
@@ -51,6 +61,28 @@ matrix:
     env: CONDA_ENV=py36-pandas-dev
   - python: 3.6
     env: CONDA_ENV=py36-condaforge-rc
+  - python: 2.7
+    env: CONDA_ENV=py27-min
+  - python: 2.7
+    env: CONDA_ENV=py27-cdat+pynio
+  - python: 3.4
+    env: CONDA_ENV=py34
+  - python: 3.5
+    env: CONDA_ENV=py35
+  - python: 3.6
+    env: CONDA_ENV=py36
+  - python: 3.6
+    env: CONDA_ENV=py36-pydap
+  - python: 2.7
+    env: CONDA_ENV=py27-min
+  - python: 2.7
+    env: CONDA_ENV=py27-cdat+pynio
+  - python: 3.4
+    env: CONDA_ENV=py34
+  - python: 3.5
+    env: CONDA_ENV=py35
+  - python: 3.6
+    env: CONDA_ENV=py36
 
 before_install:
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
@@ -76,7 +108,7 @@ install:
   - python setup.py install
 
 script:
-  - py.test xarray --cov=xarray --cov-report term-missing --verbose
+  - py.test xarray --cov=xarray --cov-report term-missing --verbose $SKIP_CI
 
 after_success:
   - coveralls

appveyor.yml

@@ -38,4 +38,4 @@ install:
 build: false
 
 test_script:
-  - "py.test xarray --verbose"
+  - "py.test xarray --verbose --skip-optional-ci"

conftest.py

@@ -0,0 +1,9 @@
+import pytest
+
+def pytest_addoption(parser):
+    parser.addoption("--run-flakey", action="store_true",
+                     help="runs flakey tests")
+    parser.addoption("--skip-optional-ci", action="store_true",
+                     help="skips optional tests continuous integration (CI)")
+    parser.addoption("--skip-slow", action="store_true",
+                     help="skips slow tests")

xarray/tests/__init__.py

@@ -114,6 +114,24 @@ def requires_bottleneck(test):
     return test if has_bottleneck else pytest.mark.skip('requires bottleneck')(test)
 
 
+flakey = pytest.mark.skipif(
+    not pytest.config.getoption("--run-flakey"),
+    reason="set --run-flakey option to run flakey tests"
+)
+
+optionalci = pytest.mark.skipif(
+    pytest.config.getoption("--skip-optional-ci"),
+    reason=("set --skip-optional-ci option to skip tests in CI, "
+            "e.g., due to travis CI resource issues")
+)
+
+
+slow = pytest.mark.skipif(
+    pytest.config.getoption("--skip-slow"),
+    reason="set --skip-slow option to run slow tests"
+)
+
+
 class TestCase(unittest.TestCase):
     if PY3:
         # Python 3 assertCountEqual is roughly equivalent to Python 2

xarray/tests/test_backends.py

@@ -26,7 +26,8 @@
 
 from . import (TestCase, requires_scipy, requires_netCDF4, requires_pydap,
                requires_scipy_or_netCDF4, requires_dask, requires_h5netcdf,
-               requires_pynio, has_netCDF4, has_scipy, assert_allclose)
+               requires_pynio, has_netCDF4, has_scipy, assert_allclose,
+               flakey, optionalci, slow)
 from .test_dataset import create_test_data
 
 try:
@@ -1059,8 +1060,9 @@ def test_encoding_unlimited_dims(self):
                 ds.to_netcdf(tmp_file, engine='h5netcdf', unlimited_dims=['y'])
 
 # tests pending h5netcdf fix
-# class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest):
-#     autoclose = True
+@flakey
+class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest):
+    autoclose = True
 
 
 class OpenMFDatasetManyFilesTest(TestCase):
@@ -1105,32 +1107,42 @@ def test_3_autoclose_pynio(self):
 
     # use of autoclose=True with h5netcdf broken because of
     # probable h5netcdf error, uncomment when fixed to test
-    # @requires_dask
-    # @requires_h5netcdf
-    # def test_4_autoclose_h5netcdf(self):
-    #     self.validate_open_mfdataset_autoclose(engine=['h5netcdf'])
+    @requires_dask
+    @requires_h5netcdf
+    @flakey
+    def test_4_autoclose_h5netcdf(self):
+        self.validate_open_mfdataset_autoclose(engine=['h5netcdf'])
 
     @requires_dask
     @requires_netCDF4
+    @optionalci
+    @slow
     def test_1_open_large_num_files_netcdf4(self):
         self.validate_open_mfdataset_large_num_files(engine=['netcdf4'])
 
     @requires_dask
     @requires_scipy
+    @optionalci
+    @slow
     def test_2_open_large_num_files_scipy(self):
         self.validate_open_mfdataset_large_num_files(engine=['scipy'])
 
     @requires_dask
     @requires_pynio
+    @optionalci
+    @slow
     def test_3_open_large_num_files_pynio(self):
         self.validate_open_mfdataset_large_num_files(engine=['pynio'])
 
     # use of autoclose=True with h5netcdf broken because of
     # probable h5netcdf error, uncomment when fixed to test
-    # @requires_dask
-    # @requires_h5netcdf
-    # def test_4_open_large_num_files_h5netcdf(self):
-    #     self.validate_open_mfdataset_large_num_files(engine=['h5netcdf'])
+    @requires_dask
+    @requires_h5netcdf
+    @flakey
+    @optionalci
+    @slow
+    def test_4_open_large_num_files_h5netcdf(self):
+        self.validate_open_mfdataset_large_num_files(engine=['h5netcdf'])
 
 
 @requires_dask