From e96c92ac1b6201429a79069c30bd8c5103f6b0e2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 31 Aug 2025 18:56:27 +0100 Subject: [PATCH 1/3] Change CI arguments: remove `--randomize`, add `--fail-rerun` --- Android/android.py | 5 +---- Lib/test/libregrtest/cmdline.py | 4 ++-- Lib/test/test_regrtest.py | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Android/android.py b/Android/android.py index 85874ad9b60f3d..79d4ea2cfb7b51 100755 --- a/Android/android.py +++ b/Android/android.py @@ -739,10 +739,7 @@ def ci(context): # Arguments are similar to --fast-ci, but in single-process mode. launcher_args = ["--managed", "maxVersion", "-v"] - test_args = [ - "--single-process", "--fail-env-changed", "--rerun", "--slowest", - "--verbose3", "-u", "all,-cpu", "--timeout=600" - ] + test_args = ["--fast-ci", "--single-process"] run( ["./android.py", "test", *launcher_args, "--", *test_args], cwd=temp_dir diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 07681d75448e24..a82a586b6f4188 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -453,13 +453,13 @@ def _parse_args(args, **kwargs): # Continuous Integration (CI): common options for fast/slow CI modes if ns.slow_ci or ns.fast_ci: # Similar to options: - # -j0 --randomize --fail-env-changed --rerun --slowest --verbose3 + # -j0 --fail-env-changed --rerun --fail-rerun --slowest --verbose3 if ns.use_mp is None: ns.use_mp = 0 - ns.randomize = True ns.fail_env_changed = True if ns.python is None: ns.rerun = True + ns.fail_rerun = True ns.print_slow = True ns.verbose3 = True else: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 5bc3c5924b07fb..4408aeca521d12 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -432,7 +432,7 @@ def check_ci_mode(self, args, use_resources, rerun=True): regrtest = self.create_regrtest(args) self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) - self.assertTrue(regrtest.randomize) + self.assertEqual(regrtest.fail_rerun, rerun) self.assertIsInstance(regrtest.random_seed, int) self.assertTrue(regrtest.fail_env_changed) self.assertTrue(regrtest.print_slowest) From f69095c3e20d96353542acbfe2ed98d413cedc1e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 31 Aug 2025 19:22:18 +0100 Subject: [PATCH 2/3] Remove outdated comment --- Android/android.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Android/android.py b/Android/android.py index 79d4ea2cfb7b51..41e212366ef08a 100755 --- a/Android/android.py +++ b/Android/android.py @@ -737,7 +737,6 @@ def ci(context): # Prove the package is self-contained by using it to run the tests. shutil.unpack_archive(package_path, temp_dir) - # Arguments are similar to --fast-ci, but in single-process mode. launcher_args = ["--managed", "maxVersion", "-v"] test_args = ["--fast-ci", "--single-process"] run( From b64cacef900cc83f5b7aedf332e0c806923b5c84 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 8 Sep 2025 10:40:21 +0100 Subject: [PATCH 3/3] Revert changes to default args; add a `--no-randomize` option instead --- Android/android.py | 4 +++- Lib/test/libregrtest/cmdline.py | 12 ++++++++---- Lib/test/libregrtest/main.py | 2 +- Lib/test/test_regrtest.py | 33 +++++++++++++++++++++++++++++++-- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/Android/android.py b/Android/android.py index 41e212366ef08a..15046b6fe1e1c0 100755 --- a/Android/android.py +++ b/Android/android.py @@ -737,8 +737,10 @@ def ci(context): # Prove the package is self-contained by using it to run the tests. shutil.unpack_archive(package_path, temp_dir) + # Randomization is disabled because order-dependent failures are + # much less likely to pass on a rerun in single-process mode. launcher_args = ["--managed", "maxVersion", "-v"] - test_args = ["--fast-ci", "--single-process"] + test_args = ["--fast-ci", "--single-process", "--no-randomize"] run( ["./android.py", "test", *launcher_args, "--", *test_args], cwd=temp_dir diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index a82a586b6f4188..6d5220ef797ca3 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -155,7 +155,7 @@ def __init__(self, **kwargs) -> None: self.list_cases = False self.list_tests = False self.single = False - self.randomize = False + self.randomize = None self.fromfile = None self.fail_env_changed = False self.use_resources: list[str] = [] @@ -271,6 +271,9 @@ def _create_parser(): group = parser.add_argument_group('Selecting tests') group.add_argument('-r', '--randomize', action='store_true', help='randomize test execution order.' + more_details) + group.add_argument('--no-randomize', dest='randomize', action='store_false', + help='do not randomize test execution order, even if ' + 'it would be implied by another option') group.add_argument('--prioritize', metavar='TEST1,TEST2,...', action='append', type=priority_list, help='select these tests first, even if the order is' @@ -453,13 +456,14 @@ def _parse_args(args, **kwargs): # Continuous Integration (CI): common options for fast/slow CI modes if ns.slow_ci or ns.fast_ci: # Similar to options: - # -j0 --fail-env-changed --rerun --fail-rerun --slowest --verbose3 + # -j0 --randomize --fail-env-changed --rerun --slowest --verbose3 if ns.use_mp is None: ns.use_mp = 0 + if ns.randomize is None: + ns.randomize = True ns.fail_env_changed = True if ns.python is None: ns.rerun = True - ns.fail_rerun = True ns.print_slow = True ns.verbose3 = True else: @@ -537,7 +541,7 @@ def _parse_args(args, **kwargs): ns.use_resources.remove(r) elif r not in ns.use_resources: ns.use_resources.append(r) - if ns.random_seed is not None: + if ns.random_seed is not None and ns.randomize is None: ns.randomize = True if ns.verbose: ns.header = True diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index a2d01b157ac89b..88d402e93e6631 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -128,7 +128,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self._tmp_dir: StrPath | None = ns.tempdir # Randomize - self.randomize: bool = ns.randomize + self.randomize: bool = bool(ns.randomize) if ('SOURCE_DATE_EPOCH' in os.environ # don't use the variable if empty and os.environ['SOURCE_DATE_EPOCH'] diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 4408aeca521d12..94284eebc5b3c3 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -182,6 +182,22 @@ def test_randomize(self): self.assertTrue(regrtest.randomize) self.assertIsInstance(regrtest.random_seed, int) + def test_no_randomize(self): + ns = self.parse_args([]) + self.assertIsNone(ns.randomize) + + ns = self.parse_args(["--randomize"]) + self.assertIs(ns.randomize, True) + + ns = self.parse_args(["--no-randomize"]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--randomize", "--no-randomize"]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--no-randomize", "--randomize"]) + self.assertIs(ns.randomize, True) + def test_randseed(self): ns = self.parse_args(['--randseed', '12345']) self.assertEqual(ns.random_seed, 12345) @@ -189,6 +205,10 @@ def test_randseed(self): self.checkError(['--randseed'], 'expected one argument') self.checkError(['--randseed', 'foo'], 'invalid int value') + ns = self.parse_args(['--randseed', '12345', '--no-randomize']) + self.assertEqual(ns.random_seed, 12345) + self.assertFalse(ns.randomize) + def test_fromfile(self): for opt in '-f', '--fromfile': with self.subTest(opt=opt): @@ -428,11 +448,11 @@ def create_regrtest(self, args): return regrtest - def check_ci_mode(self, args, use_resources, rerun=True): + def check_ci_mode(self, args, use_resources, *, rerun=True, randomize=True): regrtest = self.create_regrtest(args) self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) - self.assertEqual(regrtest.fail_rerun, rerun) + self.assertEqual(regrtest.randomize, randomize) self.assertIsInstance(regrtest.random_seed, int) self.assertTrue(regrtest.fail_env_changed) self.assertTrue(regrtest.print_slowest) @@ -469,6 +489,15 @@ def test_slow_ci(self): regrtest = self.check_ci_mode(args, use_resources) self.assertEqual(regrtest.timeout, 20 * 60) + def test_ci_no_randomize(self): + all_resources = set(cmdline.ALL_RESOURCES) + self.check_ci_mode( + ["--slow-ci", "--no-randomize"], all_resources, randomize=False + ) + self.check_ci_mode( + ["--fast-ci", "--no-randomize"], all_resources - {'cpu'}, randomize=False + ) + def test_dont_add_python_opts(self): args = ['--dont-add-python-opts'] ns = cmdline._parse_args(args)