diff --git a/src/bin/sage-runtests b/src/bin/sage-runtests index e25cebb48a8..4a8b05cfab6 100755 --- a/src/bin/sage-runtests +++ b/src/bin/sage-runtests @@ -141,7 +141,8 @@ if __name__ == "__main__": in_filenames = True new_arguments.append('--') new_arguments.append(arg) - afterlog = bool(arg == '--logfile') + afterlog = arg in ['--logfile', '--stats_path', '--stats-path', + '--baseline_stats_path', '--baseline-stats-path'] args = parser.parse_args(new_arguments) diff --git a/src/sage/doctest/control.py b/src/sage/doctest/control.py index fd92dc386bf..1a128983187 100644 --- a/src/sage/doctest/control.py +++ b/src/sage/doctest/control.py @@ -1097,6 +1097,35 @@ def sort_key(source): return -self.stats.get(basename, default).get('walltime', 0), basename self.sources = sorted(self.sources, key=sort_key) + def source_baseline(self, source): + r""" + Return the ``baseline_stats`` value of ``source``. + + INPUT: + + - ``source`` -- a :class:`DocTestSource` instance + + OUTPUT: + + A dictionary. + + EXAMPLES:: + + sage: from sage.doctest.control import DocTestDefaults, DocTestController + sage: from sage.env import SAGE_SRC + sage: import os + sage: filename = os.path.join(SAGE_SRC,'sage','doctest','util.py') + sage: DD = DocTestDefaults() + sage: DC = DocTestController(DD, [filename]) + sage: DC.expand_files_into_sources() + sage: DC.source_baseline(DC.sources[0]) + {} + """ + if self.baseline_stats: + basename = source.basename + return self.baseline_stats.get(basename, {}) + return {} + def run_doctests(self): """ Actually runs the doctests. @@ -1142,6 +1171,8 @@ def run_doctests(self): iterations = ", ".join(iterations) if iterations: iterations = " (%s)" % (iterations) + if self.baseline_stats: + self.log(f"Using --baseline-stats-path={self.options.baseline_stats_path}") self.log("Doctesting %s%s%s." % (filestr, threads, iterations)) self.reporter = DocTestReporter(self) self.dispatcher = DocTestDispatcher(self) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index efd28d10abe..ea0d49e0e4e 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -513,7 +513,7 @@ def __init__(self, *args, **kwds): - ``stdout`` -- an open file to restore for debugging - - ``checker`` -- None, or an instance of + - ``checker`` -- ``None``, or an instance of :class:`doctest.OutputChecker` - ``verbose`` -- boolean, determines whether verbose printing @@ -522,6 +522,8 @@ def __init__(self, *args, **kwds): - ``optionflags`` -- Controls the comparison with the expected output. See :mod:`testmod` for more information. + - ``baseline`` -- dictionary, the ``baseline_stats`` value + EXAMPLES:: sage: from sage.doctest.parsing import SageOutputChecker @@ -535,6 +537,7 @@ def __init__(self, *args, **kwds): O = kwds.pop('outtmpfile', None) self.msgfile = kwds.pop('msgfile', None) self.options = kwds.pop('sage_options') + self.baseline = kwds.pop('baseline', {}) doctest.DocTestRunner.__init__(self, *args, **kwds) self._fakeout = SageSpoofInOut(O) if self.msgfile is None: @@ -1721,12 +1724,14 @@ def serial_dispatch(self): """ for source in self.controller.sources: heading = self.controller.reporter.report_head(source) + baseline = self.controller.source_baseline(source) if not self.controller.options.only_errors: self.controller.log(heading) with tempfile.TemporaryFile() as outtmpfile: result = DocTestTask(source)(self.controller.options, - outtmpfile, self.controller.logger) + outtmpfile, self.controller.logger, + baseline=baseline) outtmpfile.seek(0) output = bytes_to_str(outtmpfile.read()) @@ -1985,9 +1990,10 @@ def sel_exit(): # Start a new worker. import copy worker_options = copy.copy(opt) + baseline = self.controller.source_baseline(source) if target_endtime is not None: worker_options.target_walltime = (target_endtime - now) / (max(1, pending_tests / opt.nthreads)) - w = DocTestWorker(source, options=worker_options, funclist=[sel_exit]) + w = DocTestWorker(source, options=worker_options, funclist=[sel_exit], baseline=baseline) heading = self.controller.reporter.report_head(w.source) if not self.controller.options.only_errors: w.messages = heading + "\n" @@ -2128,6 +2134,8 @@ class should be accessed by the child process. - ``funclist`` -- a list of callables to be called at the start of the child process. + - ``baseline`` -- dictionary, the ``baseline_stats`` value + EXAMPLES:: sage: from sage.doctest.forker import DocTestWorker, DocTestTask @@ -2147,7 +2155,7 @@ class should be accessed by the child process. sage: reporter.report(FDS, False, W.exitcode, result, "") [... tests, ... s] """ - def __init__(self, source, options, funclist=[]): + def __init__(self, source, options, funclist=[], baseline=None): """ Initialization. @@ -2171,6 +2179,7 @@ def __init__(self, source, options, funclist=[]): self.source = source self.options = options self.funclist = funclist + self.baseline = baseline # Open pipe for messages. These are raw file descriptors, # not Python file objects! @@ -2242,7 +2251,8 @@ def run(self): os.close(self.rmessages) msgpipe = os.fdopen(self.wmessages, "w") try: - task(self.options, self.outtmpfile, msgpipe, self.result_queue) + task(self.options, self.outtmpfile, msgpipe, self.result_queue, + baseline=self.baseline) finally: msgpipe.close() self.outtmpfile.close() @@ -2508,7 +2518,8 @@ def __init__(self, source): """ self.source = source - def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None): + def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None, *, + baseline=None): """ Calling the task does the actual work of running the doctests. @@ -2525,6 +2536,8 @@ def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None): - ``result_queue`` -- an instance of :class:`multiprocessing.Queue` to store the doctest result. For testing, this can also be None. + - ``baseline`` -- a dictionary, the ``baseline_stats`` value. + OUTPUT: - ``(doctests, result_dict)`` where ``doctests`` is the number of @@ -2560,7 +2573,8 @@ def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None): outtmpfile=outtmpfile, msgfile=msgfile, sage_options=options, - optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS) + optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS, + baseline=baseline) runner.basename = self.source.basename runner.filename = self.source.path N = options.file_iterations diff --git a/src/sage/doctest/reporting.py b/src/sage/doctest/reporting.py index a86153ce326..993ebeaa7a1 100644 --- a/src/sage/doctest/reporting.py +++ b/src/sage/doctest/reporting.py @@ -162,14 +162,16 @@ def were_doctests_with_optional_tag_run(self, tag): return True return False - def report_head(self, source): + def report_head(self, source, fail_msg=None): """ - Return the "sage -t [options] file.py" line as string. + Return the ``sage -t [options] file.py`` line as string. INPUT: - ``source`` -- a source from :mod:`sage.doctest.sources` + - ``fail_msg`` -- ``None`` or a string + EXAMPLES:: sage: from sage.doctest.reporting import DocTestReporter @@ -190,6 +192,8 @@ def report_head(self, source): sage: DD.long = True sage: print(DTR.report_head(FDS)) sage -t --long .../sage/doctest/reporting.py + sage: print(DTR.report_head(FDS, "Failed by self-sabotage")) + sage -t --long .../sage/doctest/reporting.py # Failed by self-sabotage """ cmd = "sage -t" if self.controller.options.long: @@ -206,6 +210,13 @@ def report_head(self, source): if environment != "sage.repl.ipython_kernel.all_jupyter": cmd += f" --environment={environment}" cmd += " " + source.printpath + baseline = self.controller.source_baseline(source) + if fail_msg: + cmd += " # " + fail_msg + if baseline.get('failed', False): + if not fail_msg: + cmd += " #" + cmd += " [failed in baseline]" return cmd def report(self, source, timeout, return_code, results, output, pid=None): @@ -399,10 +410,7 @@ def report(self, source, timeout, return_code, results, output, pid=None): postscript = self.postscript stats = self.stats basename = source.basename - if self.controller.baseline_stats: - the_baseline_stats = self.controller.baseline_stats.get(basename, {}) - else: - the_baseline_stats = {} + baseline = self.controller.source_baseline(source) cmd = self.report_head(source) try: ntests, result_dict = results @@ -423,14 +431,12 @@ def report(self, source, timeout, return_code, results, output, pid=None): fail_msg += " (and interrupt failed)" else: fail_msg += " (with %s after interrupt)" % signal_name(sig) - if the_baseline_stats.get('failed', False): - fail_msg += " [failed in baseline]" log(" %s\n%s\nTests run before %s timed out:" % (fail_msg, "*"*70, process_name)) log(output) log("*"*70) - postscript['lines'].append(cmd + " # %s" % fail_msg) + postscript['lines'].append(self.report_head(source, fail_msg)) stats[basename] = {"failed": True, "walltime": 1e6, "ntests": ntests} - if not the_baseline_stats.get('failed', False): + if not baseline.get('failed', False): self.error_status |= 4 elif return_code: if return_code > 0: @@ -439,14 +445,12 @@ def report(self, source, timeout, return_code, results, output, pid=None): fail_msg = "Killed due to %s" % signal_name(-return_code) if ntests > 0: fail_msg += " after testing finished" - if the_baseline_stats.get('failed', False): - fail_msg += " [failed in baseline]" log(" %s\n%s\nTests run before %s failed:" % (fail_msg,"*"*70, process_name)) log(output) log("*"*70) - postscript['lines'].append(cmd + " # %s" % fail_msg) + postscript['lines'].append(self.report_head(source, fail_msg)) stats[basename] = {"failed": True, "walltime": 1e6, "ntests": ntests} - if not the_baseline_stats.get('failed', False): + if not baseline.get('failed', False): self.error_status |= (8 if return_code > 0 else 16) else: if hasattr(result_dict, 'walltime') and hasattr(result_dict.walltime, '__len__') and len(result_dict.walltime) > 0: @@ -461,13 +465,13 @@ def report(self, source, timeout, return_code, results, output, pid=None): log(" Error in doctesting framework (bad result returned)\n%s\nTests run before error:" % ("*"*70)) log(output) log("*"*70) - postscript['lines'].append(cmd + " # Testing error: bad result") + postscript['lines'].append(self.report_head(source, "Testing error: bad result")) self.error_status |= 64 elif result_dict.err == 'noresult': log(" Error in doctesting framework (no result returned)\n%s\nTests run before error:" % ("*"*70)) log(output) log("*"*70) - postscript['lines'].append(cmd + " # Testing error: no result") + postscript['lines'].append(self.report_head(source, "Testing error: no result")) self.error_status |= 64 elif result_dict.err == 'tab': if len(result_dict.tab_linenos) > 5: @@ -476,11 +480,11 @@ def report(self, source, timeout, return_code, results, output, pid=None): if len(result_dict.tab_linenos) > 1: tabs = "s" + tabs log(" Error: TAB character found at line%s" % (tabs)) - postscript['lines'].append(cmd + " # Tab character found") + postscript['lines'].append(self.report_head(source, "Tab character found")) self.error_status |= 32 elif result_dict.err == 'line_number': log(" Error: Source line number found") - postscript['lines'].append(cmd + " # Source line number found") + postscript['lines'].append(self.report_head(source, "Source line number found")) self.error_status |= 256 elif result_dict.err is not None: # This case should not occur @@ -497,22 +501,25 @@ def report(self, source, timeout, return_code, results, output, pid=None): if output: log("Tests run before doctest exception:\n" + output) log("*"*70) - postscript['lines'].append(cmd + " # %s" % fail_msg) + postscript['lines'].append(self.report_head(source, fail_msg)) if hasattr(result_dict, 'tb'): log(result_dict.tb) if hasattr(result_dict, 'walltime'): stats[basename] = {"failed": True, "walltime": wall, "ntests": ntests} else: stats[basename] = {"failed": True, "walltime": 1e6, "ntests": ntests} - self.error_status |= 64 + # This codepath is triggered by doctests that test some timeout + # ("AlarmInterrupt in doctesting framework") or other signal handling + # behavior. This is why we handle the baseline in this codepath, + # in contrast to other "Error in doctesting framework" codepaths. + if not baseline.get('failed', False): + self.error_status |= 64 if result_dict.err is None or result_dict.err == 'tab': f = result_dict.failures if f: fail_msg = "%s failed" % (count_noun(f, "doctest")) - if the_baseline_stats.get('failed', False): - fail_msg += " [failed in baseline]" - postscript['lines'].append(cmd + " # %s" % fail_msg) - if not the_baseline_stats.get('failed', False): + postscript['lines'].append(self.report_head(source, fail_msg)) + if not baseline.get('failed', False): self.error_status |= 1 if f or result_dict.err == 'tab': stats[basename] = {"failed": True, "walltime": wall, "ntests": ntests}