src/sage/doctest/forker.py: Show '# [failed in baseline]' earlier

src/bin/sage-runtests

@@ -141,7 +141,8 @@ if __name__ == "__main__":
             in_filenames = True
             new_arguments.append('--')
         new_arguments.append(arg)
-        afterlog = bool(arg == '--logfile')
+        afterlog = arg in ['--logfile', '--stats_path', '--stats-path',
+                           '--baseline_stats_path', '--baseline-stats-path']
 
     args = parser.parse_args(new_arguments)
 

src/sage/doctest/control.py

@@ -1097,6 +1097,35 @@ def sort_key(source):
                 return -self.stats.get(basename, default).get('walltime', 0), basename
             self.sources = sorted(self.sources, key=sort_key)
 
+    def source_baseline(self, source):
+        r"""
+        Return the ``baseline_stats`` value of ``source``.
+
+        INPUT:
+
+        - ``source`` -- a :class:`DocTestSource` instance
+
+        OUTPUT:
+
+        A dictionary.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults, DocTestController
+            sage: from sage.env import SAGE_SRC
+            sage: import os
+            sage: filename = os.path.join(SAGE_SRC,'sage','doctest','util.py')
+            sage: DD = DocTestDefaults()
+            sage: DC = DocTestController(DD, [filename])
+            sage: DC.expand_files_into_sources()
+            sage: DC.source_baseline(DC.sources[0])
+            {}
+        """
+        if self.baseline_stats:
+            basename = source.basename
+            return self.baseline_stats.get(basename, {})
+        return {}
+
     def run_doctests(self):
         """
         Actually runs the doctests.
@@ -1142,6 +1171,8 @@ def run_doctests(self):
             iterations = ", ".join(iterations)
             if iterations:
                 iterations = " (%s)" % (iterations)
+            if self.baseline_stats:
+                self.log(f"Using --baseline-stats-path={self.options.baseline_stats_path}")
             self.log("Doctesting %s%s%s." % (filestr, threads, iterations))
             self.reporter = DocTestReporter(self)
             self.dispatcher = DocTestDispatcher(self)

src/sage/doctest/forker.py

@@ -513,7 +513,7 @@ def __init__(self, *args, **kwds):
 
         - ``stdout`` -- an open file to restore for debugging
 
-        - ``checker`` -- None, or an instance of
+        - ``checker`` -- ``None``, or an instance of
           :class:`doctest.OutputChecker`
 
         - ``verbose`` -- boolean, determines whether verbose printing
@@ -522,6 +522,8 @@ def __init__(self, *args, **kwds):
         - ``optionflags`` -- Controls the comparison with the expected
           output.  See :mod:`testmod` for more information.
 
+        - ``baseline`` -- dictionary, the ``baseline_stats`` value
+
         EXAMPLES::
 
             sage: from sage.doctest.parsing import SageOutputChecker
@@ -535,6 +537,7 @@ def __init__(self, *args, **kwds):
         O = kwds.pop('outtmpfile', None)
         self.msgfile = kwds.pop('msgfile', None)
         self.options = kwds.pop('sage_options')
+        self.baseline = kwds.pop('baseline', {})
         doctest.DocTestRunner.__init__(self, *args, **kwds)
         self._fakeout = SageSpoofInOut(O)
         if self.msgfile is None:
@@ -1721,12 +1724,14 @@ def serial_dispatch(self):
         """
         for source in self.controller.sources:
             heading = self.controller.reporter.report_head(source)
+            baseline = self.controller.source_baseline(source)
             if not self.controller.options.only_errors:
                 self.controller.log(heading)
 
             with tempfile.TemporaryFile() as outtmpfile:
                 result = DocTestTask(source)(self.controller.options,
-                                             outtmpfile, self.controller.logger)
+                                             outtmpfile, self.controller.logger,
+                                             baseline=baseline)
                 outtmpfile.seek(0)
                 output = bytes_to_str(outtmpfile.read())
 
@@ -1985,9 +1990,10 @@ def sel_exit():
                             # Start a new worker.
                             import copy
                             worker_options = copy.copy(opt)
+                            baseline = self.controller.source_baseline(source)
                             if target_endtime is not None:
                                 worker_options.target_walltime = (target_endtime - now) / (max(1, pending_tests / opt.nthreads))
-                            w = DocTestWorker(source, options=worker_options, funclist=[sel_exit])
+                            w = DocTestWorker(source, options=worker_options, funclist=[sel_exit], baseline=baseline)
                             heading = self.controller.reporter.report_head(w.source)
                             if not self.controller.options.only_errors:
                                 w.messages = heading + "\n"
@@ -2128,6 +2134,8 @@ class should be accessed by the child process.
     - ``funclist`` -- a list of callables to be called at the start of
       the child process.
 
+    - ``baseline`` -- dictionary, the ``baseline_stats`` value
+
     EXAMPLES::
 
         sage: from sage.doctest.forker import DocTestWorker, DocTestTask
@@ -2147,7 +2155,7 @@ class should be accessed by the child process.
         sage: reporter.report(FDS, False, W.exitcode, result, "")
             [... tests, ... s]
     """
-    def __init__(self, source, options, funclist=[]):
+    def __init__(self, source, options, funclist=[], baseline=None):
         """
         Initialization.
 
@@ -2171,6 +2179,7 @@ def __init__(self, source, options, funclist=[]):
         self.source = source
         self.options = options
         self.funclist = funclist
+        self.baseline = baseline
 
         # Open pipe for messages. These are raw file descriptors,
         # not Python file objects!
@@ -2242,7 +2251,8 @@ def run(self):
         os.close(self.rmessages)
         msgpipe = os.fdopen(self.wmessages, "w")
         try:
-            task(self.options, self.outtmpfile, msgpipe, self.result_queue)
+            task(self.options, self.outtmpfile, msgpipe, self.result_queue,
+                 baseline=self.baseline)
         finally:
             msgpipe.close()
             self.outtmpfile.close()
@@ -2508,7 +2518,8 @@ def __init__(self, source):
         """
         self.source = source
 
-    def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None):
+    def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None, *,
+                 baseline=None):
         """
         Calling the task does the actual work of running the doctests.
 
@@ -2525,6 +2536,8 @@ def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None):
         - ``result_queue`` -- an instance of :class:`multiprocessing.Queue`
           to store the doctest result. For testing, this can also be None.
 
+        - ``baseline`` -- a dictionary, the ``baseline_stats`` value.
+
         OUTPUT:
 
         - ``(doctests, result_dict)`` where ``doctests`` is the number of
@@ -2560,7 +2573,8 @@ def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None):
                 outtmpfile=outtmpfile,
                 msgfile=msgfile,
                 sage_options=options,
-                optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS)
+                optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS,
+                baseline=baseline)
             runner.basename = self.source.basename
             runner.filename = self.source.path
             N = options.file_iterations

src/sage/doctest/reporting.py

@@ -162,14 +162,16 @@ def were_doctests_with_optional_tag_run(self, tag):
             return True
         return False
 
-    def report_head(self, source):
+    def report_head(self, source, fail_msg=None):
         """
-        Return the "sage -t [options] file.py" line as string.
+        Return the ``sage -t [options] file.py`` line as string.
 
         INPUT:
 
         - ``source`` -- a source from :mod:`sage.doctest.sources`
 
+        - ``fail_msg`` -- ``None`` or a string
+
         EXAMPLES::
 
             sage: from sage.doctest.reporting import DocTestReporter
@@ -190,6 +192,8 @@ def report_head(self, source):
             sage: DD.long = True
             sage: print(DTR.report_head(FDS))
             sage -t --long .../sage/doctest/reporting.py
+            sage: print(DTR.report_head(FDS, "Failed by self-sabotage"))
+            sage -t --long .../sage/doctest/reporting.py  # Failed by self-sabotage
         """
         cmd = "sage -t"
         if self.controller.options.long:
@@ -206,6 +210,13 @@ def report_head(self, source):
         if environment != "sage.repl.ipython_kernel.all_jupyter":
             cmd += f" --environment={environment}"
         cmd += " " + source.printpath
+        baseline = self.controller.source_baseline(source)
+        if fail_msg:
+            cmd += "  # " + fail_msg
+        if baseline.get('failed', False):
+            if not fail_msg:
+                cmd += "  #"
+            cmd += " [failed in baseline]"
         return cmd
 
     def report(self, source, timeout, return_code, results, output, pid=None):
@@ -399,10 +410,7 @@ def report(self, source, timeout, return_code, results, output, pid=None):
             postscript = self.postscript
             stats = self.stats
             basename = source.basename
-            if self.controller.baseline_stats:
-                the_baseline_stats = self.controller.baseline_stats.get(basename, {})
-            else:
-                the_baseline_stats = {}
+            baseline = self.controller.source_baseline(source)
             cmd = self.report_head(source)
             try:
                 ntests, result_dict = results
@@ -423,14 +431,12 @@ def report(self, source, timeout, return_code, results, output, pid=None):
                         fail_msg += " (and interrupt failed)"
                     else:
                         fail_msg += " (with %s after interrupt)" % signal_name(sig)
-                if the_baseline_stats.get('failed', False):
-                    fail_msg += " [failed in baseline]"
                 log("    %s\n%s\nTests run before %s timed out:" % (fail_msg, "*"*70, process_name))
                 log(output)
                 log("*"*70)
-                postscript['lines'].append(cmd + "  # %s" % fail_msg)
+                postscript['lines'].append(self.report_head(source, fail_msg))
                 stats[basename] = {"failed": True, "walltime": 1e6, "ntests": ntests}
-                if not the_baseline_stats.get('failed', False):
+                if not baseline.get('failed', False):
                     self.error_status |= 4
             elif return_code:
                 if return_code > 0:
@@ -439,14 +445,12 @@ def report(self, source, timeout, return_code, results, output, pid=None):
                     fail_msg = "Killed due to %s" % signal_name(-return_code)
                 if ntests > 0:
                     fail_msg += " after testing finished"
-                if the_baseline_stats.get('failed', False):
-                    fail_msg += " [failed in baseline]"
                 log("    %s\n%s\nTests run before %s failed:" % (fail_msg,"*"*70, process_name))
                 log(output)
                 log("*"*70)
-                postscript['lines'].append(cmd + "  # %s" % fail_msg)
+                postscript['lines'].append(self.report_head(source, fail_msg))
                 stats[basename] = {"failed": True, "walltime": 1e6, "ntests": ntests}
-                if not the_baseline_stats.get('failed', False):
+                if not baseline.get('failed', False):
                     self.error_status |= (8 if return_code > 0 else 16)
             else:
                 if hasattr(result_dict, 'walltime') and hasattr(result_dict.walltime, '__len__') and len(result_dict.walltime) > 0:
@@ -461,13 +465,13 @@ def report(self, source, timeout, return_code, results, output, pid=None):
                     log("    Error in doctesting framework (bad result returned)\n%s\nTests run before error:" % ("*"*70))
                     log(output)
                     log("*"*70)
-                    postscript['lines'].append(cmd + "  # Testing error: bad result")
+                    postscript['lines'].append(self.report_head(source, "Testing error: bad result"))
                     self.error_status |= 64
                 elif result_dict.err == 'noresult':
                     log("    Error in doctesting framework (no result returned)\n%s\nTests run before error:" % ("*"*70))
                     log(output)
                     log("*"*70)
-                    postscript['lines'].append(cmd + "  # Testing error: no result")
+                    postscript['lines'].append(self.report_head(source, "Testing error: no result"))
                     self.error_status |= 64
                 elif result_dict.err == 'tab':
                     if len(result_dict.tab_linenos) > 5:
@@ -476,11 +480,11 @@ def report(self, source, timeout, return_code, results, output, pid=None):
                     if len(result_dict.tab_linenos) > 1:
                         tabs = "s" + tabs
                     log("    Error: TAB character found at line%s" % (tabs))
-                    postscript['lines'].append(cmd + "  # Tab character found")
+                    postscript['lines'].append(self.report_head(source, "Tab character found"))
                     self.error_status |= 32
                 elif result_dict.err == 'line_number':
                     log("    Error: Source line number found")
-                    postscript['lines'].append(cmd + "  # Source line number found")
+                    postscript['lines'].append(self.report_head(source, "Source line number found"))
                     self.error_status |= 256
                 elif result_dict.err is not None:
                     # This case should not occur
@@ -497,22 +501,25 @@ def report(self, source, timeout, return_code, results, output, pid=None):
                     if output:
                         log("Tests run before doctest exception:\n" + output)
                         log("*"*70)
-                    postscript['lines'].append(cmd + "  # %s" % fail_msg)
+                    postscript['lines'].append(self.report_head(source, fail_msg))
                     if hasattr(result_dict, 'tb'):
                         log(result_dict.tb)
                     if hasattr(result_dict, 'walltime'):
                         stats[basename] = {"failed": True, "walltime": wall, "ntests": ntests}
                     else:
                         stats[basename] = {"failed": True, "walltime": 1e6, "ntests": ntests}
-                    self.error_status |= 64
+                    # This codepath is triggered by doctests that test some timeout
+                    # ("AlarmInterrupt in doctesting framework") or other signal handling
+                    # behavior. This is why we handle the baseline in this codepath,
+                    # in contrast to other "Error in doctesting framework" codepaths.
+                    if not baseline.get('failed', False):
+                        self.error_status |= 64
                 if result_dict.err is None or result_dict.err == 'tab':
                     f = result_dict.failures
                     if f:
                         fail_msg = "%s failed" % (count_noun(f, "doctest"))
-                        if the_baseline_stats.get('failed', False):
-                            fail_msg += " [failed in baseline]"
-                        postscript['lines'].append(cmd + "  # %s" % fail_msg)
-                        if not the_baseline_stats.get('failed', False):
+                        postscript['lines'].append(self.report_head(source, fail_msg))
+                        if not baseline.get('failed', False):
                             self.error_status |= 1
                     if f or result_dict.err == 'tab':
                         stats[basename] = {"failed": True, "walltime": wall, "ntests": ntests}