tests: more xfails, nicer system

Author: henryiii

tests/conftest.py

@@ -197,25 +197,40 @@ def pytest_configure():
 
 # Platform Markers
 
-PLAT = sys.platform.lower()
-IMPL = platform.python_implementation().lower()
-PYMAJ = str(sys.version_info.major)
-PYNAME = "py" + PYMAJ
-
-CURRENT = {PLAT, IMPL, PYNAME, IMPL + PYMAJ}
-START = {"xfail", "skip"}
+PLAT = sys.platform.lower()  # linux, osx, or win32
+IMPL = platform.python_implementation().lower()  # cpython or pypy
 
 
 def pytest_collection_modifyitems(items):
+    """
+    This will find the markers listed in pytest.ini and add
+    skip or xfail as needed. The second part can be a platform
+    marker (linux, osx, or win32), or python, cpython, or pypy.
+
+    You can add also add Python version numbers - either 2 or 3.
+
+    Keyword arguments are passed on.
+
+    # Will skip on Python 2
+    pytest.mark.skip_python(2)
+
+    # Will xfail on pypy as long as TypeError is raised
+    pytest.mark.xfail_pypy(reason="Not supported", raises=TypeError)
+    """
     for item in items:
         for mark in tuple(item.iter_markers()):
+            # Check for recognised name
             parts = mark.name.split("_")
-            if len(parts) == 2 and parts[0] in START and parts[1] in CURRENT:
+            if len(parts) == 2 and parts[0] in {"xfail", "skip"}:
                 marker = getattr(pytest.mark, parts[0])
-                reason = "expected to fail on {}".format(parts[1])
 
-                item.add_marker(
-                    marker(**mark.kwargs)
-                    if "reason" in mark.kwargs
-                    else marker(reason=reason, **mark.kwargs)
-                )
+                if parts[1] in {PLAT, IMPL, "python"}:
+                    # args lets you remove only Py 2 or 3
+                    if mark.args:
+                        (ver,) = mark.args  # Only single argument supported
+                        assert isinstance(ver, int), "should be a version number"
+                        if ver == sys.version_info.major:
+                            item.add_marker(marker(**mark.kwargs))
+                    else:
+                        assert parts[1] != "python", "version required (otherwise use mark.skip)"
+                        item.add_marker(marker(**mark.kwargs))

tests/pytest.ini

@@ -4,8 +4,8 @@ norecursedirs = test_cmake_build test_embed
 addopts =
     # show summary of skipped tests
     -rs
-    # Report xfails
-    -rx
+    # Report xfails (uncomment to use
+    # -rx
     # capture only Python print and C++ py::print, but not C output (low-level Python errors)
     --capture=sys
     # warnings for deprecated features
@@ -26,18 +26,12 @@ markers =
     xfail_linux: expected failure on Linux
     xfail_darwin: expected failure on MacOS
     xfail_win32: expected failure on Windows
+    xfail_python: expected failure on Python (2 or 3 required)
     xfail_cpython: expected failure on CPython
-    xfail_pypy: expected failure on PyPy (2+3)
-    xfail_pypy2: expected failure on PyPy 2
-    xfail_pypy3: expected failure on PyPy 3
-    xfail_py2: expected failure on Python 2
-    xfail_py3: expected failure on Python 3
-    skip_linux: does not run on Linux
-    skip_darwin: does not run on MacOS
-    skip_win32: does not run on Windows
-    skip_cpython: does not run on CPython
-    skip_pypy: does not run on PyPy (2+3)
-    skip_pypy2: does not run on PyPy 2
-    skip_pypy3: does not run on PyPy 3
-    skip_py2: does not run on Python 2
-    skip_py3: does not run on Python 3
+    xfail_pypy: not valid on PyPy (2+3)
+    skip_linux: not valid on Linux
+    skip_darwin: not valid on MacOS
+    skip_win32: not valid on Windows
+    skip_python: not valid on Python (2 or 3 required)
+    skip_cpython: not valid on CPython
+    skip_pypy: not valid on PyPy (2+3)

tests/test_buffers.py

@@ -35,9 +35,8 @@ def test_from_python():
     assert cstats.move_assignments == 0
 
 
-# PyPy: Memory leak in the "np.array(m, copy=False)" call
-# https://bitbucket.org/pypy/pypy/issues/2444
-@pytest.mark.skip_pypy
+# https://foss.heptapod.net/pypy/pypy/-/issues/2444
+@pytest.mark.xfail_pypy(reason="memory leak in np.array(m, copy=False)")
 def test_to_python():
     mat = m.Matrix(5, 4)
     assert memoryview(mat).shape == (5, 4)
@@ -72,7 +71,7 @@ def test_to_python():
     assert cstats.move_assignments == 0
 
 
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 def test_inherited_protocol():
     """SquareMatrix is derived from Matrix and inherits the buffer protocol"""
 
@@ -81,7 +80,7 @@ def test_inherited_protocol():
     assert np.asarray(matrix).shape == (5, 5)
 
 
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 def test_pointer_to_member_fn():
     for cls in [m.Buffer, m.ConstBuffer, m.DerivedBuffer]:
         buf = cls()
@@ -90,15 +89,15 @@ def test_pointer_to_member_fn():
         assert value == 0x12345678
 
 
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 def test_readonly_buffer():
     buf = m.BufferReadOnly(0x64)
     view = memoryview(buf)
     assert view[0] == b'd' if six.PY2 else 0x64
     assert view.readonly
 
 
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 def test_selective_readonly_buffer():
     buf = m.BufferReadOnlySelect()
 

tests/test_call_policies.py

@@ -70,8 +70,8 @@ def test_keep_alive_return_value(capture):
     """
 
 
-# https://bitbucket.org/pypy/pypy/issues/2447
-@pytest.mark.skip_pypy
+# https://foss.heptapod.net/pypy/pypy/-/issues/2447
+@pytest.mark.xfail_pypy(reason="_PyObject_GetDictPtr is unimplemented")
 def test_alive_gc(capture):
     n_inst = ConstructorStats.detail_reg_inst()
     p = m.ParentGC()

tests/test_class.py

@@ -261,7 +261,7 @@ def test_brace_initialization():
     assert b.vec == [123, 456]
 
 
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 def test_class_refcount():
     """Instances must correctly increase/decrease the reference count of their types (#1029)"""
     from sys import getrefcount
@@ -307,17 +307,17 @@ def test_aligned():
         assert p % 1024 == 0
 
 
-# https://bitbucket.org/pypy/pypy/issues/2742
-@pytest.mark.skip_pypy
+# https://foss.heptapod.net/pypy/pypy/-/issues/2742
+@pytest.mark.xfail_pypy
 def test_final():
     with pytest.raises(TypeError) as exc_info:
         class PyFinalChild(m.IsFinal):
             pass
     assert str(exc_info.value).endswith("is not an acceptable base type")
 
 
-# https://bitbucket.org/pypy/pypy/issues/2742
-@pytest.mark.skip_pypy
+# https://foss.heptapod.net/pypy/pypy/-/issues/2742
+@pytest.mark.xfail_pypy
 def test_non_final_final():
     with pytest.raises(TypeError) as exc_info:
         class PyNonFinalFinalChild(m.IsNonFinalFinal):

tests/test_eval.py

@@ -15,7 +15,7 @@ def test_evals(capture):
     assert m.test_eval_failure()
 
 
-@pytest.mark.xfail_pypy3(raises=RuntimeError)
+@pytest.mark.xfail_pypy(3, raises=RuntimeError)
 def test_eval_file():
     filename = os.path.join(os.path.dirname(__file__), "test_eval_call.py")
     assert m.test_eval_file(filename)

tests/test_factory_constructors.py

@@ -418,7 +418,7 @@ def test_reallocations(capture, msg):
     """)
 
 
-@pytest.mark.skip_py2
+@pytest.mark.skip_python(2)
 def test_invalid_self():
     """Tests invocation of the pybind-registered base class with an invalid `self` argument.  You
     can only actually do this on Python 3: Python 2 raises an exception itself if you try."""

tests/test_kwargs_and_defaults.py

@@ -146,8 +146,7 @@ def test_keyword_only_args(msg):
     """
 
 
-# PyPy2 doesn't seem to double count
-@pytest.mark.xfail_pypy2
+@pytest.mark.xfail_pypy(2, reason="PyPy2 doesn't seem to double count")
 def test_args_refcount():
     """Issue/PR #1216 - py::args elements get double-inc_ref()ed when combined with regular
     arguments"""

tests/test_methods_and_attributes.py

@@ -258,7 +258,7 @@ def test_property_rvalue_policy():
 
 
 # https://bitbucket.org/pypy/pypy/issues/2447
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 def test_dynamic_attributes():
     instance = m.DynamicClass()
     assert not hasattr(instance, "foo")
@@ -299,8 +299,8 @@ class PythonDerivedDynamicClass(m.DynamicClass):
         assert cstats.alive() == 0
 
 
-# https://bitbucket.org/pypy/pypy/issues/2447
-@pytest.mark.skip_pypy
+# https://foss.heptapod.net/pypy/pypy/-/issues/2447
+@pytest.mark.xfail_pypy
 def test_cyclic_gc():
     # One object references itself
     instance = m.DynamicClass()

tests/test_multiple_inheritance.py

@@ -11,8 +11,8 @@ def test_multiple_inheritance_cpp():
     assert mt.bar() == 4
 
 
-@pytest.mark.skip_pypy2
-@pytest.mark.xfail_pypy3
+@pytest.mark.skip_pypy(2)
+@pytest.mark.xfail_pypy(3)
 def test_multiple_inheritance_mix1():
     class Base1:
         def __init__(self, i):
@@ -51,8 +51,8 @@ def __init__(self, i, j):
     assert mt.bar() == 4
 
 
-@pytest.mark.skip_pypy2
-@pytest.mark.xfail_pypy3
+@pytest.mark.skip_pypy(2)
+@pytest.mark.xfail_pypy(3)
 def test_multiple_inheritance_python():
 
     class MI1(m.Base1, m.Base2):

tests/test_numpy_array.py

@@ -240,7 +240,7 @@ def test_numpy_view(capture):
     """
 
 
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 def test_cast_numpy_int64_to_uint64():
     m.function_taking_uint64(123)
     m.function_taking_uint64(np.uint64(123))
@@ -421,7 +421,7 @@ def test_array_resize(msg):
     assert(b.shape == (8, 8))
 
 
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 def test_array_create_and_resize(msg):
     a = m.create_and_resize(2)
     assert(a.size == 4)
@@ -433,7 +433,7 @@ def test_index_using_ellipsis():
     assert a.shape == (6,)
 
 
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 def test_dtype_refcount_leak():
     from sys import getrefcount
     dtype = np.dtype(np.float_)

tests/test_numpy_dtypes.py

@@ -291,7 +291,7 @@ def test_register_dtype():
     assert 'dtype is already registered' in str(excinfo.value)
 
 
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 def test_str_leak():
     from sys import getrefcount
     fmt = "f4"

tests/test_pickling.py

@@ -22,7 +22,7 @@ def test_roundtrip(cls_name):
     assert p2.extra2() == p.extra2()
 
 
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 @pytest.mark.parametrize("cls_name", ["PickleableWithDict", "PickleableWithDictNew"])
 def test_roundtrip_with_dict(cls_name):
     cls = getattr(m, cls_name)

tests/test_pytypes.py

@@ -373,7 +373,7 @@ def test_test_memoryview_from_buffer_nullptr():
             m.test_memoryview_from_buffer_nullptr()
 
 
-@pytest.mark.skip_py2
+@pytest.mark.skip_python(2)
 def test_memoryview_from_memory():
     view = m.test_memoryview_from_memory()
     assert isinstance(view, memoryview)

tests/test_stl_binders.py

@@ -68,8 +68,7 @@ def test_vector_int():
     assert len(v_int2) == 0
 
 
-# related to the PyPy's buffer protocol.
-@pytest.mark.skip_pypy
+# Older PyPy's failed here, related to the PyPy's buffer protocol.
 def test_vector_buffer():
     b = bytearray([1, 2, 3, 4])
     v = m.VectorUChar(b)
@@ -94,7 +93,7 @@ def test_vector_buffer():
     assert "NumPy type info missing for " in str(excinfo.value)
 
 
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 def test_vector_buffer_numpy():
     np = pytest.importorskip("numpy")
     a = np.array([1, 2, 3, 4], dtype=np.int32)

tests/test_virtual_functions.py

@@ -160,7 +160,7 @@ def f(self):
 
 # PyPy: Reference count > 1 causes call with noncopyable instance
 # to fail in ncv1.print_nc()
-@pytest.mark.skip_pypy
+@pytest.mark.xfail_pypy
 @pytest.mark.skipif(not hasattr(m, "NCVirt"), reason="NCVirt test broken on ICPC")
 def test_move_support():
     class NCVirtExt(m.NCVirt):