From 00b6c4df3fa46e18bb6a6a18dacd510f75d6ef4b Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 15:45:53 -0500 Subject: [PATCH 01/16] Addressing creation and implementation of new flaky resrouce --- Lib/test/_test_multiprocessing.py | 1 + Lib/test/libregrtest/utils.py | 2 +- Lib/test/test_glob.py | 3 ++- Lib/test/test_signal.py | 2 ++ Lib/test/test_smtplib.py | 2 ++ 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 4b7c3e7fa8bdd7..3463f7caaf856d 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -39,6 +39,7 @@ from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper +from test.support import requires_resource # Skip tests if _multiprocessing wasn't built. diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 3eff9e753b6d84..baf831a341777e 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -33,7 +33,7 @@ ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime') + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime', 'flaky') # Other resources excluded from --use=all: # diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index a45b30599d5309..6a341db4c0c962 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -8,6 +8,7 @@ from test import support from test.support import is_wasi, Py_DEBUG +from test.support import requires_resource from test.support.os_helper import (TESTFN, skip_unless_symlink, can_symlink, create_empty_file, change_cwd) @@ -521,7 +522,7 @@ class SymlinkLoopGlobTests(unittest.TestCase): # gh-109959: On Linux, glob._isdir() and glob._lexists() can return False # randomly when checking the "link/" symbolic link. # https://github.com/python/cpython/issues/109959#issuecomment-2577550700 - @unittest.skip("flaky test") + @support.requires_resource('flaky') def test_selflink(self): tempdir = TESTFN + "_dir" os.makedirs(tempdir) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 72a01cd1e451f4..eb178668a4faea 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -13,6 +13,7 @@ import time import unittest from test import support +from test.support import requires_resource from test.support import ( is_apple, is_apple_mobile, os_helper, threading_helper ) @@ -1353,6 +1354,7 @@ def handler(signum, frame): @unittest.skipUnless(hasattr(signal, "SIGUSR1"), "test needs SIGUSR1") @threading_helper.requires_working_threading() + @support.requires_resource('flaky') def test_stress_modifying_handlers(self): # bpo-43406: race condition between trip_signal() and signal.signal signum = signal.SIGUSR1 diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 4c9fc14bd43f54..15f2c59d8bf7e1 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -23,6 +23,7 @@ from test.support import threading_helper from test.support import asyncore from test.support import smtpd +from test.support import requires_resource from unittest.mock import Mock @@ -374,6 +375,7 @@ def testHELP(self): b'RCPT DATA RSET NOOP QUIT VRFY') smtp.quit() + @support.requires_resource('flaky') def testSend(self): # connect and send mail m = 'A test message' From 31c5118f8b5f074961e49660f7fcb04701e94b9c Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sun, 23 Feb 2025 21:20:53 +0000 Subject: [PATCH 02/16] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst diff --git a/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst b/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst new file mode 100644 index 00000000000000..69bd0df5760dd1 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst @@ -0,0 +1 @@ +Created flaky resource for flagging tests with flaky outcomes using the support.required_resource() decorator. Flagged some initial tests based on-inline comments. This resource can be used to automate re-running certain tests in CI. From 820842f7564a79f7373530d0411790e50e07a474 Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 17:45:09 -0500 Subject: [PATCH 03/16] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_glob.py | 3 +-- .../next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 6a341db4c0c962..3e6b00c88c5920 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -7,8 +7,7 @@ import warnings from test import support -from test.support import is_wasi, Py_DEBUG -from test.support import requires_resource +from test.support import is_wasi, Py_DEBUG, requires_resource from test.support.os_helper import (TESTFN, skip_unless_symlink, can_symlink, create_empty_file, change_cwd) diff --git a/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst b/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst index 69bd0df5760dd1..93e80a96092d8a 100644 --- a/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst +++ b/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst @@ -1 +1 @@ -Created flaky resource for flagging tests with flaky outcomes using the support.required_resource() decorator. Flagged some initial tests based on-inline comments. This resource can be used to automate re-running certain tests in CI. +Add the ``flaky`` resource for flagging tests with flaky outcomes using the :func:`test.support.required_resource` decorator. From 505e25f0380963596597a3c93e0abd653bb6bfd5 Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 18:07:10 -0500 Subject: [PATCH 04/16] update the command-line of libregretest --- Lib/test/libregrtest/cmdline.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 81c7106ac0c0f4..ffc08747a3c4e8 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -122,6 +122,8 @@ tzdata - Run tests that require timezone data. + flaky - Run tests that are known to be unreliable/unstable. + To enable all resources except one, use '-uall,-'. For example, to run all the tests except for the gui tests, give the option '-uall,-gui'. From 34a1557b2e0f936c171d7d2306e72226d34b9b29 Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 18:08:20 -0500 Subject: [PATCH 05/16] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst b/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst index 93e80a96092d8a..6735508feb0a6d 100644 --- a/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst +++ b/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst @@ -1 +1 @@ -Add the ``flaky`` resource for flagging tests with flaky outcomes using the :func:`test.support.required_resource` decorator. +Add the ``flaky`` resource for flagging tests with flaky outcomes using the :func:`!test.support.required_resource` decorator. From 81cc32c9503e71ccd50cae1786f1cbae8b14502b Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 18:14:44 -0500 Subject: [PATCH 06/16] Change flaky to outer-decorator --- Lib/test/test_signal.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index eb178668a4faea..78ec0b0b1f2a2a 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1348,13 +1348,12 @@ def handler(signum, frame): # All ITIMER_REAL signals should have been delivered to the # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") - + @support.requires_resource('flaky') @support.requires_gil_enabled("gh-121065: test is flaky on free-threaded build") @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") @unittest.skipUnless(hasattr(signal, "SIGUSR1"), "test needs SIGUSR1") @threading_helper.requires_working_threading() - @support.requires_resource('flaky') def test_stress_modifying_handlers(self): # bpo-43406: race condition between trip_signal() and signal.signal signum = signal.SIGUSR1 From 0c7f3fca3034168c8c22bd3d7fa53de1ad47ad1a Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 18:20:54 -0500 Subject: [PATCH 07/16] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_signal.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 78ec0b0b1f2a2a..e1591f797423c1 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1348,6 +1348,7 @@ def handler(signum, frame): # All ITIMER_REAL signals should have been delivered to the # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") + @support.requires_resource('flaky') @support.requires_gil_enabled("gh-121065: test is flaky on free-threaded build") @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") From 1657217453d5d286487863ead73f61bd2c7b17d7 Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 18:21:37 -0500 Subject: [PATCH 08/16] Remove unecessary import --- Lib/test/_test_multiprocessing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 3463f7caaf856d..4b7c3e7fa8bdd7 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -39,7 +39,6 @@ from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper -from test.support import requires_resource # Skip tests if _multiprocessing wasn't built. From 711cfdd5c3b486f782e98d2df723189a91d9ac9b Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 19:26:17 -0500 Subject: [PATCH 09/16] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_signal.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index e1591f797423c1..624cf6a5ee1e9e 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1349,6 +1349,7 @@ def handler(signum, frame): # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") + @support.requires_resource('flaky') @support.requires_gil_enabled("gh-121065: test is flaky on free-threaded build") @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") From 4fb099c64ec80656d7bc8426e1b20e8038eec4fe Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 19:26:51 -0500 Subject: [PATCH 10/16] Condense imports --- Lib/test/test_signal.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 624cf6a5ee1e9e..a4ce0caf9f2a49 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -13,9 +13,8 @@ import time import unittest from test import support -from test.support import requires_resource from test.support import ( - is_apple, is_apple_mobile, os_helper, threading_helper + is_apple, is_apple_mobile, os_helper, threading_helper, requires_resource ) from test.support.script_helper import assert_python_ok, spawn_python try: From 9fee4db8013ca20ba1bef8b1057e10eab1bfb9f9 Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 19:28:39 -0500 Subject: [PATCH 11/16] Remove import and flaky decorator in test_smtplib --- Lib/test/test_smtplib.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 15f2c59d8bf7e1..4c9fc14bd43f54 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -23,7 +23,6 @@ from test.support import threading_helper from test.support import asyncore from test.support import smtpd -from test.support import requires_resource from unittest.mock import Mock @@ -375,7 +374,6 @@ def testHELP(self): b'RCPT DATA RSET NOOP QUIT VRFY') smtp.quit() - @support.requires_resource('flaky') def testSend(self): # connect and send mail m = 'A test message' From 47c2fea58e94d73c52c63e889012527aa08d56f9 Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 19:41:36 -0500 Subject: [PATCH 12/16] Revert test_signal --- Lib/test/test_signal.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index a4ce0caf9f2a49..0ed99215c2461f 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -14,7 +14,7 @@ import unittest from test import support from test.support import ( - is_apple, is_apple_mobile, os_helper, threading_helper, requires_resource + is_apple, is_apple_mobile, os_helper, threading_helper ) from test.support.script_helper import assert_python_ok, spawn_python try: @@ -1349,7 +1349,6 @@ def handler(signum, frame): self.assertEqual(len(sigs), N, "Some signals were lost") - @support.requires_resource('flaky') @support.requires_gil_enabled("gh-121065: test is flaky on free-threaded build") @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") @unittest.skipUnless(hasattr(signal, "SIGUSR1"), From 806df62496227ce4383aa40dcb1b32a4948ab979 Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 19:46:43 -0500 Subject: [PATCH 13/16] Revert test_glob --- Lib/test/test_glob.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 3e6b00c88c5920..7bd9df4a61a350 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -521,7 +521,6 @@ class SymlinkLoopGlobTests(unittest.TestCase): # gh-109959: On Linux, glob._isdir() and glob._lexists() can return False # randomly when checking the "link/" symbolic link. # https://github.com/python/cpython/issues/109959#issuecomment-2577550700 - @support.requires_resource('flaky') def test_selflink(self): tempdir = TESTFN + "_dir" os.makedirs(tempdir) From 0dc248702a5bd3feafa8527579b470779db24ff2 Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 20:10:04 -0500 Subject: [PATCH 14/16] Re-decorate function to flaky in test_glob --- Lib/test/test_glob.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 7bd9df4a61a350..270d9f3b82380e 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -521,6 +521,7 @@ class SymlinkLoopGlobTests(unittest.TestCase): # gh-109959: On Linux, glob._isdir() and glob._lexists() can return False # randomly when checking the "link/" symbolic link. # https://github.com/python/cpython/issues/109959#issuecomment-2577550700 + @requires_resource('flaky') def test_selflink(self): tempdir = TESTFN + "_dir" os.makedirs(tempdir) From 3298da3c1b6279948647575dc43126b6cce12b9c Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 20:10:28 -0500 Subject: [PATCH 15/16] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_signal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 0ed99215c2461f..72a01cd1e451f4 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1348,7 +1348,6 @@ def handler(signum, frame): # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") - @support.requires_gil_enabled("gh-121065: test is flaky on free-threaded build") @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") @unittest.skipUnless(hasattr(signal, "SIGUSR1"), From 1b12e5cbd3cb71d23f2fd9cdbebc053cb5ec20db Mon Sep 17 00:00:00 2001 From: floor-licker Date: Sun, 23 Feb 2025 20:43:21 -0500 Subject: [PATCH 16/16] Apply suggestions from code review Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- .../next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst b/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst index 6735508feb0a6d..4f6261f3fef950 100644 --- a/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst +++ b/Misc/NEWS.d/next/Tests/2025-02-23-21-20-52.gh-issue-130474.dEckCL.rst @@ -1 +1,2 @@ -Add the ``flaky`` resource for flagging tests with flaky outcomes using the :func:`!test.support.required_resource` decorator. +Add the ``flaky`` resource for flagging tests with unreliable outcomes +using the :py:deco:`!test.support.required_resource` decorator.