gh-109566: regrtest reexecutes the process (#109909)

When --fast-ci or --slow-ci option is used, regrtest now replaces the
current process with a new process to add "-u -W default -bb -E"
options to Python.

Changes:

* PCbuild/rt.bat and Tools/scripts/run_tests.py no longer need to add
  "-u -W default -bb -E" options to Python: it's now done by
  regrtest.
* Fix Tools/scripts/run_tests.py: flush stdout before replacing the
  process. Previously, buffered messages were lost.

Author: vstinner

Lib/test/__main__.py

@@ -1,2 +1,2 @@
 from test.libregrtest.main import main
-main()
+main(reexec=True)

Lib/test/libregrtest/cmdline.py

@@ -184,6 +184,7 @@ def __init__(self, **kwargs) -> None:
         self.threshold = None
         self.fail_rerun = False
         self.tempdir = None
+        self.no_reexec = False
 
         super().__init__(**kwargs)
 
@@ -343,6 +344,8 @@ def _create_parser():
                        help='override the working directory for the test run')
     group.add_argument('--cleanup', action='store_true',
                        help='remove old test_python_* directories')
+    group.add_argument('--no-reexec', action='store_true',
+                       help="internal option, don't use it")
     return parser
 
 
@@ -421,6 +424,8 @@ def _parse_args(args, **kwargs):
         ns.verbose3 = True
         if MS_WINDOWS:
             ns.nowindows = True  # Silence alerts under Windows
+    else:
+        ns.no_reexec = True
 
     # When both --slow-ci and --fast-ci options are present,
     # --slow-ci has the priority

Lib/test/libregrtest/main.py

@@ -1,6 +1,7 @@
 import os
 import random
 import re
+import shlex
 import sys
 import time
 
@@ -20,7 +21,7 @@
     StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple,
     strip_py_suffix, count, format_duration,
     printlist, get_temp_dir, get_work_dir, exit_timeout,
-    display_header, cleanup_temp_dir,
+    display_header, cleanup_temp_dir, print_warning,
     MS_WINDOWS)
 
 
@@ -47,7 +48,7 @@ class Regrtest:
     directly to set the values that would normally be set by flags
     on the command line.
     """
-    def __init__(self, ns: Namespace):
+    def __init__(self, ns: Namespace, reexec: bool = False):
         # Log verbosity
         self.verbose: int = int(ns.verbose)
         self.quiet: bool = ns.quiet
@@ -69,6 +70,7 @@ def __init__(self, ns: Namespace):
         self.want_cleanup: bool = ns.cleanup
         self.want_rerun: bool = ns.rerun
         self.want_run_leaks: bool = ns.runleaks
+        self.want_reexec: bool = (reexec and not ns.no_reexec)
 
         # Select tests
         if ns.match_tests:
@@ -95,6 +97,7 @@ def __init__(self, ns: Namespace):
         self.worker_json: StrJSON | None = ns.worker_json
 
         # Options to run tests
+        self.ci_mode: bool = (ns.fast_ci or ns.slow_ci)
         self.fail_fast: bool = ns.failfast
         self.fail_env_changed: bool = ns.fail_env_changed
         self.fail_rerun: bool = ns.fail_rerun
@@ -483,7 +486,37 @@ def run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
                 # processes.
                 return self._run_tests(selected, tests)
 
+    def _reexecute_python(self):
+        if self.python_cmd:
+            # Do nothing if --python=cmd option is used
+            return
+
+        python_opts = [
+            '-u',                 # Unbuffered stdout and stderr
+            '-W', 'default',      # Add warnings filter 'default'
+            '-bb',                # Error on bytes/str comparison
+            '-E',                 # Ignore PYTHON* environment variables
+        ]
+
+        cmd = [*sys.orig_argv, "--no-reexec"]
+        cmd[1:1] = python_opts
+
+        # Make sure that messages before execv() are logged
+        sys.stdout.flush()
+        sys.stderr.flush()
+
+        try:
+            os.execv(cmd[0], cmd)
+            # execv() do no return and so we don't get to this line on success
+        except OSError as exc:
+            cmd_text = shlex.join(cmd)
+            print_warning(f"Failed to reexecute Python: {exc!r}\n"
+                          f"Command: {cmd_text}")
+
     def main(self, tests: TestList | None = None):
+        if self.want_reexec and self.ci_mode:
+            self._reexecute_python()
+
         if self.junit_filename and not os.path.isabs(self.junit_filename):
             self.junit_filename = os.path.abspath(self.junit_filename)
 
@@ -515,7 +548,7 @@ def main(self, tests: TestList | None = None):
         sys.exit(exitcode)
 
 
-def main(tests=None, **kwargs):
+def main(tests=None, reexec=False, **kwargs):
     """Run the Python suite."""
     ns = _parse_args(sys.argv[1:], **kwargs)
-    Regrtest(ns).main(tests=tests)
+    Regrtest(ns, reexec=reexec).main(tests=tests)

Lib/test/test_regrtest.py

@@ -382,7 +382,8 @@ def check_ci_mode(self, args, use_resources):
         # Check Regrtest attributes which are more reliable than Namespace
         # which has an unclear API
         regrtest = main.Regrtest(ns)
-        self.assertNotEqual(regrtest.num_workers, 0)
+        self.assertTrue(regrtest.ci_mode)
+        self.assertEqual(regrtest.num_workers, -1)
         self.assertTrue(regrtest.want_rerun)
         self.assertTrue(regrtest.randomize)
         self.assertIsNone(regrtest.random_seed)
@@ -1960,6 +1961,61 @@ def test_dev_mode(self):
         self.check_executed_tests(output, tests,
                                   stats=len(tests), parallel=True)
 
+    def check_reexec(self, option):
+        # --fast-ci and --slow-ci add "-u -W default -bb -E" options to Python
+        code = textwrap.dedent(r"""
+            import sys
+            import unittest
+            try:
+                from _testinternalcapi import get_config
+            except ImportError:
+                get_config = None
+
+            class WorkerTests(unittest.TestCase):
+                @unittest.skipUnless(get_config is None, 'need get_config()')
+                def test_config(self):
+                    config = get_config()['config']
+                    # -u option
+                    self.assertEqual(config['buffered_stdio'], 0)
+                    # -W default option
+                    self.assertTrue(config['warnoptions'], ['default'])
+                    # -bb option
+                    self.assertTrue(config['bytes_warning'], 2)
+                    # -E option
+                    self.assertTrue(config['use_environment'], 0)
+
+                # test if get_config() is not available
+                def test_unbuffered(self):
+                    # -u option
+                    self.assertFalse(sys.stdout.line_buffering)
+                    self.assertFalse(sys.stderr.line_buffering)
+
+                def test_python_opts(self):
+                    # -W default option
+                    self.assertTrue(sys.warnoptions, ['default'])
+                    # -bb option
+                    self.assertEqual(sys.flags.bytes_warning, 2)
+                    # -E option
+                    self.assertTrue(sys.flags.ignore_environment)
+        """)
+        testname = self.create_test(code=code)
+
+        cmd = [sys.executable,
+               "-m", "test", option,
+               f'--testdir={self.tmptestdir}',
+               testname]
+        proc = subprocess.run(cmd,
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.STDOUT,
+                              text=True)
+        self.assertEqual(proc.returncode, 0, proc)
+
+    def test_reexec_fast_ci(self):
+        self.check_reexec("--fast-ci")
+
+    def test_reexec_slow_ci(self):
+        self.check_reexec("--slow-ci")
+
 
 class TestUtils(unittest.TestCase):
     def test_format_duration(self):

Misc/NEWS.d/next/Tests/2023-09-26-18-12-01.gh-issue-109566.CP0Vhf.rst

@@ -0,0 +1,3 @@
+regrtest: When ``--fast-ci`` or ``--slow-ci`` option is used, regrtest now
+replaces the current process with a new process to add ``-u -W default -bb -E``
+options to Python. Patch by Victor Stinner.

PCbuild/rt.bat

@@ -48,7 +48,7 @@ if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts
 
 if not defined prefix set prefix=%pcbuild%amd64
 set exe=%prefix%\python%suffix%.exe
-set cmd="%exe%" %dashO% -u -Wd -E -bb -m test %regrtestargs%
+set cmd="%exe%" %dashO% -m test %regrtestargs%
 if defined qmode goto Qmode
 
 echo Deleting .pyc files ...

Tools/scripts/run_tests.py

@@ -23,11 +23,7 @@ def is_python_flag(arg):
 
 
 def main(regrtest_args):
-    args = [sys.executable,
-            '-u',                 # Unbuffered stdout and stderr
-            '-W', 'default',      # Warnings set to 'default'
-            '-bb',                # Warnings about bytes/bytearray
-            ]
+    args = [sys.executable]
 
     cross_compile = '_PYTHON_HOST_PLATFORM' in os.environ
     if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None:
@@ -47,7 +43,6 @@ def main(regrtest_args):
         }
     else:
         environ = os.environ.copy()
-        args.append("-E")
 
     # Allow user-specified interpreter options to override our defaults.
     args.extend(test.support.args_from_interpreter_flags())
@@ -70,7 +65,8 @@ def main(regrtest_args):
 
     args.extend(regrtest_args)
 
-    print(shlex.join(args))
+    print(shlex.join(args), flush=True)
+
     if sys.platform == 'win32':
         from subprocess import call
         sys.exit(call(args))