From 9279ea2882b96be6f7de3f8fbaa38062b1f2353e Mon Sep 17 00:00:00 2001 From: Reagan Lee <96998476+reaganjlee@users.noreply.github.com> Date: Fri, 30 Jun 2023 15:29:02 -0700 Subject: [PATCH 1/4] Emit unmatched warnings from pytest.warns() --- AUTHORS | 1 + changelog/9288.breaking.rst | 1 + src/_pytest/recwarn.py | 33 ++++++++++++++++++++++++++++++ testing/test_recwarn.py | 40 +++++++++++++++++++++++++++++++++---- 4 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 changelog/9288.breaking.rst diff --git a/AUTHORS b/AUTHORS index e0ed531b07b..28116c3c834 100644 --- a/AUTHORS +++ b/AUTHORS @@ -311,6 +311,7 @@ Raphael Pierzina Rafal Semik Raquel Alegre Ravi Chandra +Reagan Lee Robert Holt Roberto Aldera Roberto Polli diff --git a/changelog/9288.breaking.rst b/changelog/9288.breaking.rst new file mode 100644 index 00000000000..7f101123d48 --- /dev/null +++ b/changelog/9288.breaking.rst @@ -0,0 +1 @@ +Set :func:`warns` to re-emit unmatched warnings when the context closes diff --git a/src/_pytest/recwarn.py b/src/_pytest/recwarn.py index b422f362730..334b3b850df 100644 --- a/src/_pytest/recwarn.py +++ b/src/_pytest/recwarn.py @@ -149,6 +149,10 @@ def warns( # noqa: F811 This could be achieved in the same way as with exceptions, see :ref:`parametrizing_conditional_raising` for an example. + .. note:: + Unlike the stdlib :func:`warnings.catch_warnings` context manager, + unmatched warnings will be re-emitted when the context closes. + """ __tracebackhide__ = True if not args: @@ -290,6 +294,32 @@ def __exit__( def found_str(): return pformat([record.message for record in self], indent=2) + def re_emit() -> None: + for r in self: + if matches(r): + continue + + assert issubclass(r.message.__class__, Warning) + + warnings.warn_explicit( + str(r.message), + r.message.__class__, + r.filename, + r.lineno, + module=r.__module__, + source=r.source, + ) + + def matches(warning) -> bool: + if self.expected_warning is not None: + if issubclass(warning.category, self.expected_warning): + if self.match_expr is not None: + if re.compile(self.match_expr).search(str(warning.message)): + return True + return False + return True + return False + # only check if we're not currently handling an exception if exc_type is None and exc_val is None and exc_tb is None: if self.expected_warning is not None: @@ -303,6 +333,7 @@ def found_str(): for r in self: if issubclass(r.category, self.expected_warning): if re.compile(self.match_expr).search(str(r.message)): + re_emit() break else: fail( @@ -311,3 +342,5 @@ def found_str(): Regex: {self.match_expr} Emitted warnings: {found_str()}""" ) + else: + re_emit() diff --git a/testing/test_recwarn.py b/testing/test_recwarn.py index 7e0f836a634..d9cd1d778e2 100644 --- a/testing/test_recwarn.py +++ b/testing/test_recwarn.py @@ -376,10 +376,12 @@ def test_match_regex(self) -> None: warnings.warn("value must be 42", UserWarning) def test_one_from_multiple_warns(self) -> None: - with pytest.warns(UserWarning, match=r"aaa"): - warnings.warn("cccccccccc", UserWarning) - warnings.warn("bbbbbbbbbb", UserWarning) - warnings.warn("aaaaaaaaaa", UserWarning) + with pytest.raises(pytest.fail.Exception): + with pytest.warns(UserWarning, match=r"aaa"): + with pytest.warns(UserWarning, match=r"aaa"): + warnings.warn("cccccccccc", UserWarning) + warnings.warn("bbbbbbbbbb", UserWarning) + warnings.warn("aaaaaaaaaa", UserWarning) def test_none_of_multiple_warns(self) -> None: with pytest.raises(pytest.fail.Exception): @@ -403,3 +405,33 @@ def test_warns_context_manager_with_kwargs(self) -> None: with pytest.warns(UserWarning, foo="bar"): # type: ignore pass assert "Unexpected keyword arguments" in str(excinfo.value) + + def test_re_emit_single(self) -> None: + with pytest.warns(DeprecationWarning): + with pytest.warns(UserWarning): + warnings.warn("user warning", UserWarning) + warnings.warn("some deprecation warning", DeprecationWarning) + + def test_re_emit_multiple(self) -> None: + with pytest.warns(UserWarning): + warnings.warn("first warning", UserWarning) + warnings.warn("second warning", UserWarning) + + def test_re_emit_match_single(self) -> None: + with pytest.warns(DeprecationWarning): + with pytest.warns(UserWarning, match="user warning"): + warnings.warn("user warning", UserWarning) + warnings.warn("some deprecation warning", DeprecationWarning) + + def test_re_emit_match_multiple(self) -> None: + # with pytest.warns(UserWarning): + with pytest.warns(UserWarning, match="user warning"): + warnings.warn("first user warning", UserWarning) + warnings.warn("second user warning", UserWarning) + + def test_re_emit_non_match_single(self) -> None: + # with pytest.warns(UserWarning): + with pytest.warns(UserWarning, match="v2 warning"): + with pytest.warns(UserWarning, match="v1 warning"): + warnings.warn("v1 warning", UserWarning) + warnings.warn("non-matching v2 warning", UserWarning) From a1b37022afd4aec31009a3e56872c435b05e3c45 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Fri, 30 Jun 2023 15:29:02 -0700 Subject: [PATCH 2/4] Refactor warns() exit logic --- src/_pytest/recwarn.py | 85 ++++++++++++++++++----------------------- testing/test_recwarn.py | 14 +++---- 2 files changed, 44 insertions(+), 55 deletions(-) diff --git a/src/_pytest/recwarn.py b/src/_pytest/recwarn.py index 334b3b850df..5452b7cb9c0 100644 --- a/src/_pytest/recwarn.py +++ b/src/_pytest/recwarn.py @@ -281,6 +281,12 @@ def __init__( self.expected_warning = expected_warning_tup self.match_expr = match_expr + def matches(self, warning: warnings.WarningMessage) -> bool: + assert self.expected_warning is not None + return issubclass(warning.category, self.expected_warning) and bool( + self.match_expr is None or re.search(self.match_expr, str(warning.message)) + ) + def __exit__( self, exc_type: Optional[Type[BaseException]], @@ -291,56 +297,39 @@ def __exit__( __tracebackhide__ = True - def found_str(): - return pformat([record.message for record in self], indent=2) + if self.expected_warning is None: + # nothing to do in this deprecated case, see WARNS_NONE_ARG above + return - def re_emit() -> None: - for r in self: - if matches(r): - continue + if not (exc_type is None and exc_val is None and exc_tb is None): + # We currently ignore missing warnings if an exception is active. + # TODO: fix this, because it means things are surprisingly order-sensitive. + return - assert issubclass(r.message.__class__, Warning) + def found_str(): + return pformat([record.message for record in self], indent=2) - warnings.warn_explicit( - str(r.message), - r.message.__class__, - r.filename, - r.lineno, - module=r.__module__, - source=r.source, + try: + if not any(issubclass(w.category, self.expected_warning) for w in self): + fail( + f"DID NOT WARN. No warnings of type {self.expected_warning} were emitted.\n" + f" Emitted warnings: {found_str()}." ) - - def matches(warning) -> bool: - if self.expected_warning is not None: - if issubclass(warning.category, self.expected_warning): - if self.match_expr is not None: - if re.compile(self.match_expr).search(str(warning.message)): - return True - return False - return True - return False - - # only check if we're not currently handling an exception - if exc_type is None and exc_val is None and exc_tb is None: - if self.expected_warning is not None: - if not any(issubclass(r.category, self.expected_warning) for r in self): - __tracebackhide__ = True - fail( - f"DID NOT WARN. No warnings of type {self.expected_warning} were emitted.\n" - f"The list of emitted warnings is: {found_str()}." + elif not any(self.matches(w) for w in self): + fail( + f"DID NOT WARN. No warnings of type {self.expected_warning} matching the regex were emitted.\n" + f" Regex: {self.match_expr}\n" + f" Emitted warnings: {found_str()}." + ) + finally: + # Whether or not any warnings matched, we want to re-emit all unmatched warnings. + for w in self: + if not self.matches(w): + warnings.warn_explicit( + str(w.message), + w.message.__class__, + w.filename, + w.lineno, + module=w.__module__, + source=w.source, ) - elif self.match_expr is not None: - for r in self: - if issubclass(r.category, self.expected_warning): - if re.compile(self.match_expr).search(str(r.message)): - re_emit() - break - else: - fail( - f"""\ -DID NOT WARN. No warnings of type {self.expected_warning} matching the regex were emitted. - Regex: {self.match_expr} - Emitted warnings: {found_str()}""" - ) - else: - re_emit() diff --git a/testing/test_recwarn.py b/testing/test_recwarn.py index d9cd1d778e2..a423b20012a 100644 --- a/testing/test_recwarn.py +++ b/testing/test_recwarn.py @@ -376,7 +376,7 @@ def test_match_regex(self) -> None: warnings.warn("value must be 42", UserWarning) def test_one_from_multiple_warns(self) -> None: - with pytest.raises(pytest.fail.Exception): + with pytest.raises(pytest.fail.Exception, match="DID NOT WARN"): with pytest.warns(UserWarning, match=r"aaa"): with pytest.warns(UserWarning, match=r"aaa"): warnings.warn("cccccccccc", UserWarning) @@ -384,7 +384,7 @@ def test_one_from_multiple_warns(self) -> None: warnings.warn("aaaaaaaaaa", UserWarning) def test_none_of_multiple_warns(self) -> None: - with pytest.raises(pytest.fail.Exception): + with pytest.raises(pytest.fail.Exception, match="DID NOT WARN"): with pytest.warns(UserWarning, match=r"aaa"): warnings.warn("bbbbbbbbbb", UserWarning) warnings.warn("cccccccccc", UserWarning) @@ -424,13 +424,13 @@ def test_re_emit_match_single(self) -> None: warnings.warn("some deprecation warning", DeprecationWarning) def test_re_emit_match_multiple(self) -> None: - # with pytest.warns(UserWarning): - with pytest.warns(UserWarning, match="user warning"): - warnings.warn("first user warning", UserWarning) - warnings.warn("second user warning", UserWarning) + with warnings.catch_warnings(): + warnings.simplefilter("error") # if anything is re-emitted + with pytest.warns(UserWarning, match="user warning"): + warnings.warn("first user warning", UserWarning) + warnings.warn("second user warning", UserWarning) def test_re_emit_non_match_single(self) -> None: - # with pytest.warns(UserWarning): with pytest.warns(UserWarning, match="v2 warning"): with pytest.warns(UserWarning, match="v1 warning"): warnings.warn("v1 warning", UserWarning) From 7022fb455d5c5853cb839cd20e53bbbacaa46d28 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Fri, 30 Jun 2023 15:29:03 -0700 Subject: [PATCH 3/4] Update tests for re-emitted warnings --- src/_pytest/recwarn.py | 7 +-- testing/test_recwarn.py | 103 ++++++++++++++++++++++------------------ 2 files changed, 61 insertions(+), 49 deletions(-) diff --git a/src/_pytest/recwarn.py b/src/_pytest/recwarn.py index 5452b7cb9c0..efe2a3dc6ca 100644 --- a/src/_pytest/recwarn.py +++ b/src/_pytest/recwarn.py @@ -135,8 +135,9 @@ def warns( # noqa: F811 >>> with pytest.warns(UserWarning, match=r'must be \d+$'): ... warnings.warn("value must be 42", UserWarning) - >>> with pytest.warns(UserWarning, match=r'must be \d+$'): - ... warnings.warn("this is not here", UserWarning) + >>> with pytest.warns(UserWarning): # catch re-emitted warning + ... with pytest.warns(UserWarning, match=r'must be \d+$'): + ... warnings.warn("this is not here", UserWarning) Traceback (most recent call last): ... Failed: DID NOT WARN. No warnings of type ...UserWarning... were emitted... @@ -327,7 +328,7 @@ def found_str(): if not self.matches(w): warnings.warn_explicit( str(w.message), - w.message.__class__, + w.message.__class__, # type: ignore w.filename, w.lineno, module=w.__module__, diff --git a/testing/test_recwarn.py b/testing/test_recwarn.py index a423b20012a..6b3856bd924 100644 --- a/testing/test_recwarn.py +++ b/testing/test_recwarn.py @@ -203,19 +203,21 @@ def test_deprecated_call_specificity(self) -> None: def f(): warnings.warn(warning("hi")) - with pytest.raises(pytest.fail.Exception): - pytest.deprecated_call(f) - with pytest.raises(pytest.fail.Exception): - with pytest.deprecated_call(): - f() + with pytest.warns(warning): + with pytest.raises(pytest.fail.Exception): + pytest.deprecated_call(f) + with pytest.raises(pytest.fail.Exception): + with pytest.deprecated_call(): + f() def test_deprecated_call_supports_match(self) -> None: with pytest.deprecated_call(match=r"must be \d+$"): warnings.warn("value must be 42", DeprecationWarning) - with pytest.raises(pytest.fail.Exception): - with pytest.deprecated_call(match=r"must be \d+$"): - warnings.warn("this is not here", DeprecationWarning) + with pytest.deprecated_call(): + with pytest.raises(pytest.fail.Exception, match="DID NOT WARN"): + with pytest.deprecated_call(match=r"must be \d+$"): + warnings.warn("this is not here", DeprecationWarning) class TestWarns: @@ -227,8 +229,9 @@ def test_check_callable(self) -> None: def test_several_messages(self) -> None: # different messages, b/c Python suppresses multiple identical warnings pytest.warns(RuntimeWarning, lambda: warnings.warn("w1", RuntimeWarning)) - with pytest.raises(pytest.fail.Exception): - pytest.warns(UserWarning, lambda: warnings.warn("w2", RuntimeWarning)) + with pytest.warns(RuntimeWarning): + with pytest.raises(pytest.fail.Exception): + pytest.warns(UserWarning, lambda: warnings.warn("w2", RuntimeWarning)) pytest.warns(RuntimeWarning, lambda: warnings.warn("w3", RuntimeWarning)) def test_function(self) -> None: @@ -243,13 +246,14 @@ def test_warning_tuple(self) -> None: pytest.warns( (RuntimeWarning, SyntaxWarning), lambda: warnings.warn("w2", SyntaxWarning) ) - pytest.raises( - pytest.fail.Exception, - lambda: pytest.warns( - (RuntimeWarning, SyntaxWarning), - lambda: warnings.warn("w3", UserWarning), - ), - ) + with pytest.warns(): + pytest.raises( + pytest.fail.Exception, + lambda: pytest.warns( + (RuntimeWarning, SyntaxWarning), + lambda: warnings.warn("w3", UserWarning), + ), + ) def test_as_contextmanager(self) -> None: with pytest.warns(RuntimeWarning): @@ -258,20 +262,22 @@ def test_as_contextmanager(self) -> None: with pytest.warns(UserWarning): warnings.warn("user", UserWarning) - with pytest.raises(pytest.fail.Exception) as excinfo: - with pytest.warns(RuntimeWarning): - warnings.warn("user", UserWarning) + with pytest.warns(): + with pytest.raises(pytest.fail.Exception) as excinfo: + with pytest.warns(RuntimeWarning): + warnings.warn("user", UserWarning) excinfo.match( r"DID NOT WARN. No warnings of type \(.+RuntimeWarning.+,\) were emitted.\n" - r"The list of emitted warnings is: \[UserWarning\('user',?\)\]." + r" Emitted warnings: \[UserWarning\('user',?\)\]." ) - with pytest.raises(pytest.fail.Exception) as excinfo: - with pytest.warns(UserWarning): - warnings.warn("runtime", RuntimeWarning) + with pytest.warns(): + with pytest.raises(pytest.fail.Exception) as excinfo: + with pytest.warns(UserWarning): + warnings.warn("runtime", RuntimeWarning) excinfo.match( r"DID NOT WARN. No warnings of type \(.+UserWarning.+,\) were emitted.\n" - r"The list of emitted warnings is: \[RuntimeWarning\('runtime',?\)]." + r" Emitted warnings: \[RuntimeWarning\('runtime',?\)]." ) with pytest.raises(pytest.fail.Exception) as excinfo: @@ -279,19 +285,20 @@ def test_as_contextmanager(self) -> None: pass excinfo.match( r"DID NOT WARN. No warnings of type \(.+UserWarning.+,\) were emitted.\n" - r"The list of emitted warnings is: \[\]." + r" Emitted warnings: \[\]." ) warning_classes = (UserWarning, FutureWarning) - with pytest.raises(pytest.fail.Exception) as excinfo: - with pytest.warns(warning_classes) as warninfo: - warnings.warn("runtime", RuntimeWarning) - warnings.warn("import", ImportWarning) + with pytest.warns(): + with pytest.raises(pytest.fail.Exception) as excinfo: + with pytest.warns(warning_classes) as warninfo: + warnings.warn("runtime", RuntimeWarning) + warnings.warn("import", ImportWarning) messages = [each.message for each in warninfo] expected_str = ( f"DID NOT WARN. No warnings of type {warning_classes} were emitted.\n" - f"The list of emitted warnings is: {messages}." + f" Emitted warnings: {messages}." ) assert str(excinfo.value) == expected_str @@ -367,27 +374,31 @@ def test_match_regex(self) -> None: with pytest.warns(UserWarning, match=r"must be \d+$"): warnings.warn("value must be 42", UserWarning) - with pytest.raises(pytest.fail.Exception): - with pytest.warns(UserWarning, match=r"must be \d+$"): - warnings.warn("this is not here", UserWarning) + with pytest.warns(): + with pytest.raises(pytest.fail.Exception): + with pytest.warns(UserWarning, match=r"must be \d+$"): + warnings.warn("this is not here", UserWarning) - with pytest.raises(pytest.fail.Exception): - with pytest.warns(FutureWarning, match=r"must be \d+$"): - warnings.warn("value must be 42", UserWarning) + with pytest.warns(): + with pytest.raises(pytest.fail.Exception): + with pytest.warns(FutureWarning, match=r"must be \d+$"): + warnings.warn("value must be 42", UserWarning) def test_one_from_multiple_warns(self) -> None: - with pytest.raises(pytest.fail.Exception, match="DID NOT WARN"): - with pytest.warns(UserWarning, match=r"aaa"): + with pytest.warns(): + with pytest.raises(pytest.fail.Exception, match="DID NOT WARN"): with pytest.warns(UserWarning, match=r"aaa"): - warnings.warn("cccccccccc", UserWarning) - warnings.warn("bbbbbbbbbb", UserWarning) - warnings.warn("aaaaaaaaaa", UserWarning) + with pytest.warns(UserWarning, match=r"aaa"): + warnings.warn("cccccccccc", UserWarning) + warnings.warn("bbbbbbbbbb", UserWarning) + warnings.warn("aaaaaaaaaa", UserWarning) def test_none_of_multiple_warns(self) -> None: - with pytest.raises(pytest.fail.Exception, match="DID NOT WARN"): - with pytest.warns(UserWarning, match=r"aaa"): - warnings.warn("bbbbbbbbbb", UserWarning) - warnings.warn("cccccccccc", UserWarning) + with pytest.warns(): + with pytest.raises(pytest.fail.Exception, match="DID NOT WARN"): + with pytest.warns(UserWarning, match=r"aaa"): + warnings.warn("bbbbbbbbbb", UserWarning) + warnings.warn("cccccccccc", UserWarning) @pytest.mark.filterwarnings("ignore") def test_can_capture_previously_warned(self) -> None: From 2d48171e88e861bde0e7cf3d3d087ebbe203c103 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Fri, 30 Jun 2023 15:29:03 -0700 Subject: [PATCH 4/4] Tweak docs on review --- changelog/9288.breaking.rst | 8 +++++++- src/_pytest/recwarn.py | 12 ++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/changelog/9288.breaking.rst b/changelog/9288.breaking.rst index 7f101123d48..053af8013ec 100644 --- a/changelog/9288.breaking.rst +++ b/changelog/9288.breaking.rst @@ -1 +1,7 @@ -Set :func:`warns` to re-emit unmatched warnings when the context closes +:func:`pytest.warns ` now re-emits unmatched warnings when the context +closes -- previously it would consume all warnings, hiding those that were not +matched by the function. + +While this is a new feature, we decided to announce this as a breaking change +because many test suites are configured to error-out on warnings, and will +therefore fail on the newly-re-emitted warnings. diff --git a/src/_pytest/recwarn.py b/src/_pytest/recwarn.py index efe2a3dc6ca..127a7a856a4 100644 --- a/src/_pytest/recwarn.py +++ b/src/_pytest/recwarn.py @@ -117,10 +117,10 @@ def warns( # noqa: F811 warning of that class or classes. This helper produces a list of :class:`warnings.WarningMessage` objects, one for - each warning raised (regardless of whether it is an ``expected_warning`` or not). + each warning emitted (regardless of whether it is an ``expected_warning`` or not). + Since pytest 8.0, unmatched warnings are also re-emitted when the context closes. - This function can be used as a context manager, which will capture all the raised - warnings inside it:: + This function can be used as a context manager:: >>> import pytest >>> with pytest.warns(RuntimeWarning): @@ -150,10 +150,6 @@ def warns( # noqa: F811 This could be achieved in the same way as with exceptions, see :ref:`parametrizing_conditional_raising` for an example. - .. note:: - Unlike the stdlib :func:`warnings.catch_warnings` context manager, - unmatched warnings will be re-emitted when the context closes. - """ __tracebackhide__ = True if not args: @@ -328,7 +324,7 @@ def found_str(): if not self.matches(w): warnings.warn_explicit( str(w.message), - w.message.__class__, # type: ignore + w.message.__class__, # type: ignore[arg-type] w.filename, w.lineno, module=w.__module__,