From 70108e3f9365c7c42b14a72f22a24bc3cfd9ae3d Mon Sep 17 00:00:00 2001 From: Joshua Herman Date: Tue, 25 Apr 2023 15:41:40 -0500 Subject: [PATCH] gh-82054: Implementing unittest sharding for asyncio --- Lib/test/libregrtest/runtest.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index e9bb72a7d77ee1..09f56b5a08e580 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -143,6 +143,15 @@ def __str__(self) -> str: # set of tests that we don't want to be executed when using regrtest NOTTESTS = set() +#If these test directories are encountered recurse into them and treat each +# test_ .py or dir as a separate test module. This can increase parallelism. +# Beware this can't generally be done for any directory with sub-tests as the +# __init__.py may do things which alter what tests are to be run. + +SPLITTESTDIRS = { + "test_asyncio", + "test_compiler", +} # Storage of uncollectable objects FOUND_GARBAGE = [] @@ -158,7 +167,7 @@ def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): +def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, splittestdirs=SPLITTESTDIRS, base_mod=""): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) names = os.listdir(testdir) @@ -166,8 +175,16 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): others = set(stdtests) | nottests for name in names: mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and ext in (".py", "") and mod not in others: - tests.append(mod) + if mod[:5] == "test_" and mod not in others: + if mod in splittestdirs: + subdir = os.path.join(testdir, mod) + if len(base_mod): + mod = f"{base_mod}.{mod}" + else: + mod = f"test.{mod}" + tests.extend(findtests(subdir, [], nottests, splittestdirs, mod)) + elif ext in (".py", ""): + tests.append(f"{base_mod}.{mod}" if len(base_mod) else mod) return stdtests + sorted(tests)