Implement # sage.doctest: flaky marker

src/sage/algebras/fusion_rings/fusion_ring.py

@@ -1,3 +1,4 @@
+# sage.doctest: flaky (:issue:`39538`)
 """
 Fusion rings
 """

src/sage/doctest/forker.py

@@ -1740,6 +1740,21 @@
     """
     Create parallel :class:`DocTestWorker` processes and dispatches
     doctesting tasks.
+
+    .. NOTE::
+
+        If this is run directly in the normal Sage command-line,
+        it calls :func:`init_sage` which in turn calls
+        :meth:`~sage.repl.rich_output.display_manager.DisplayManager.switch_backend`
+        to the doctest backend, which is incompatible with the IPython-based
+        command-line.  As such, if an error such as
+        ``TypeError: cannot unpack non-iterable NoneType object`` is seen,
+        a workaround is to run the following::
+
+            sage: # not tested
+            sage: from sage.repl.rich_output.backend_ipython import BackendIPythonCommandline
+            sage: backend = BackendIPythonCommandline()
+            sage: get_ipython().display_formatter.dm.switch_backend(backend, shell=get_ipython())
     """
     def __init__(self, controller: DocTestController):
         """
@@ -1867,6 +1882,119 @@
                1 of   1 in ...
                 [1 test, 1 failure, ...s wall]
             Killing test ...
+
+        TESTS:
+
+        Test flaky files. This test should fail the first time and success the second time::
+
+            sage: # long time
+            sage: with NTF(suffix='.py', mode='w+t') as f1:
+            ....:     t = walltime()
+            ....:     _ = f1.write(f"# sage.doctest: flaky\n'''\nsage: sleep(10 if walltime() < {t+1} else 0)\n'''")
+            ....:     f1.flush()
+            ....:     DC = DocTestController(DocTestDefaults(timeout=2),
+            ....:                            [f1.name])
+            ....:     DC.expand_files_into_sources()
+            ....:     DD = DocTestDispatcher(DC)
+            ....:     DR = DocTestReporter(DC)
+            ....:     DC.reporter = DR
+            ....:     DC.dispatcher = DD
+            ....:     DC.timer = Timer().start()
+            ....:     DD.parallel_dispatch()
+                sage -t ...
+                sage -t ...
+                    [1 test, ...s wall]
+
+        Segmentation fault and abort are also handled::
+
+            sage: # long time
+            sage: with NTF(suffix='.py', mode='w+t') as f1:
+            ....:     t = walltime()
+            ....:     _ = f1.write(f"# sage.doctest: flaky\n"
+            ....:                  f"'''\n"
+            ....:                  f"sage: from cysignals.tests import unguarded_abort\n"
+            ....:                  f"sage: if walltime() < {t+0.5r}: sleep(1r); unguarded_abort()\n"
+            ....:                  f"'''\n")
+            ....:     f1.flush()
+            ....:     DC = DocTestController(DocTestDefaults(timeout=10),
+            ....:                            [f1.name])
+            ....:     DC.expand_files_into_sources()
+            ....:     DD = DocTestDispatcher(DC)
+            ....:     DR = DocTestReporter(DC)
+            ....:     DC.reporter = DR
+            ....:     DC.dispatcher = DD
+            ....:     DC.timer = Timer().start()
+            ....:     DD.parallel_dispatch()
+                sage -t ...
+                sage -t ...
+                    [2 tests, ...s wall]
+
+        This test always fail, so even flaky can't help it::
+
+            sage: # long time
+            sage: with NTF(suffix='.py', mode='w+t') as f1:
+            ....:     _ = f1.write(f"# sage.doctest: flaky\n'''\nsage: sleep(10)\n'''")
+            ....:     f1.flush()
+            ....:     DC = DocTestController(DocTestDefaults(timeout=2),
+            ....:                            [f1.name])
+            ....:     DC.expand_files_into_sources()
+            ....:     DD = DocTestDispatcher(DC)
+            ....:     DR = DocTestReporter(DC)
+            ....:     DC.reporter = DR
+            ....:     DC.dispatcher = DD
+            ....:     DC.timer = Timer().start()
+            ....:     DD.parallel_dispatch()
+            sage -t ...
+            sage -t ...
+                Timed out
+            **********************************************************************
+            ...
+
+        Of course without flaky, the test should fail (since it timeouts the first time)::
+
+            sage: # long time
+            sage: with NTF(suffix='.py', mode='w+t') as f1:
+            ....:     t = walltime()
+            ....:     _ = f1.write(f"'''\nsage: sleep(10 if walltime() < {t+1} else 0)\n'''")
+            ....:     f1.flush()
+            ....:     DC = DocTestController(DocTestDefaults(timeout=2),
+            ....:                            [f1.name])
+            ....:     DC.expand_files_into_sources()
+            ....:     DD = DocTestDispatcher(DC)
+            ....:     DR = DocTestReporter(DC)
+            ....:     DC.reporter = DR
+            ....:     DC.dispatcher = DD
+            ....:     DC.timer = Timer().start()
+            ....:     DD.parallel_dispatch()
+            sage -t ...
+                Timed out
+            **********************************************************************
+            ...
+
+        If it doesn't fail the first time, it must not be retried::
+
+            sage: from contextlib import redirect_stdout
+            sage: from io import StringIO
+            sage: with NTF(suffix='.py', mode='w+t') as f1, StringIO() as f, redirect_stdout(f):
+            ....:     t = walltime()
+            ....:     _ = f1.write(f"'''\nsage: sleep(0.5)\n'''")
+            ....:     f1.flush()
+            ....:     DC = DocTestController(DocTestDefaults(timeout=2),
+            ....:                            [f1.name])
+            ....:     DC.expand_files_into_sources()
+            ....:     DD = DocTestDispatcher(DC)
+            ....:     DR = DocTestReporter(DC)
+            ....:     DC.reporter = DR
+            ....:     DC.dispatcher = DD
+            ....:     DC.timer = Timer().start()
+            ....:     DD.parallel_dispatch()
+            ....:     s = f.getvalue()
+            sage: print(s)
+            sage -t ...
+            **********************************************************************
+            ...
+            sage: s.count("sage -t")
+            1
         """
         opt = self.controller.options
 
@@ -1911,11 +2039,11 @@
         # List of alive DocTestWorkers (child processes). Workers which
         # are done but whose messages have not been read are also
         # considered alive.
-        workers = []
+        workers: list[DocTestWorker] = []
 
         # List of DocTestWorkers which have finished running but
         # whose results have not been reported yet.
-        finished = []
+        finished: list[DocTestWorker] = []
 
         # If exitfirst is set and we got a failure.
         abort_now = False
@@ -1953,6 +2081,28 @@
                     # precision.
                     now = time.time()
 
+                    def start_new_worker(source: DocTestSource, num_retries_left: typing.Optional[int] = None) -> DocTestWorker:
+                        nonlocal opt, target_endtime, now, pending_tests, sel_exit
+                        import copy
+                        worker_options = copy.copy(opt)
+                        baseline = self.controller.source_baseline(source)
+                        if target_endtime is not None:
+                            worker_options.target_walltime = (target_endtime - now) / (max(1, pending_tests / opt.nthreads))
+                        w = DocTestWorker(source, options=worker_options, funclist=[sel_exit], baseline=baseline)
+                        if num_retries_left is not None:
+                            w.num_retries_left = num_retries_left
+                        elif 'flaky' in source.file_optional_tags:
+                            w.num_retries_left = 2
+                        heading = self.controller.reporter.report_head(w.source)
+                        if not self.controller.options.only_errors:
+                            w.messages = heading + "\n"
+                        # Store length of heading to detect if the
+                        # worker has something interesting to report.
+                        w.heading_len = len(w.messages)
+                        w.start()  # This might take some time
+                        w.deadline = time.time() + opt.timeout
+                        return w
+
                     # If there were any substantial changes in the state
                     # (new worker started or finished worker reported),
                     # restart this while loop instead of calling pselect().
@@ -1965,7 +2115,7 @@
                     # "finished" list.
                     # Create a new list "new_workers" containing the active
                     # workers (to avoid updating "workers" in place).
-                    new_workers = []
+                    new_workers: list[DocTestWorker] = []
                     for w in workers:
                         if w.rmessages is not None or w.is_alive():
                             if now >= w.deadline:
@@ -2008,8 +2158,19 @@
                     workers = new_workers
 
                     # Similarly, process finished workers.
-                    new_finished = []
+                    new_finished: list[DocTestWorker] = []
                     for w in finished:
+                        if (w.killed or w.result[1].err) and w.num_retries_left > 0:
+                            # in this case, the messages from w should be suppressed
+                            # (better handling could be implemented later)
+                            # should also check w.killed or w.result if want to only retry
+                            # on timeout/segmentation fault
+                            # in that case w.source.file_optional_tags['flaky'] can be checked
+                            if follow is w:
+                                follow = None
+                            workers.append(start_new_worker(w.source, w.num_retries_left - 1))
+                            continue
+
                         if opt.exitfirst and w.result[1].failures:
                             abort_now = True
                         elif follow is not None and follow is not w:
@@ -2042,28 +2203,13 @@
                     while (source_iter is not None and len(workers) < opt.nthreads
                            and (not job_client or job_client.acquire())):
                         try:
-                            source = next(source_iter)
+                            source: DocTestSource = next(source_iter)
                         except StopIteration:
                             source_iter = None
                             if job_client:
                                 job_client.release()
                         else:
-                            # Start a new worker.
-                            import copy
-                            worker_options = copy.copy(opt)
-                            baseline = self.controller.source_baseline(source)
-                            if target_endtime is not None:
-                                worker_options.target_walltime = (target_endtime - now) / (max(1, pending_tests / opt.nthreads))
-                            w = DocTestWorker(source, options=worker_options, funclist=[sel_exit], baseline=baseline)
-                            heading = self.controller.reporter.report_head(w.source)
-                            if not self.controller.options.only_errors:
-                                w.messages = heading + "\n"
-                            # Store length of heading to detect if the
-                            # worker has something interesting to report.
-                            w.heading_len = len(w.messages)
-                            w.start()  # This might take some time
-                            w.deadline = time.time() + opt.timeout
-                            workers.append(w)
+                            workers.append(start_new_worker(source))
                             restart = True
 
                     # Recompute state if needed
@@ -2240,13 +2386,20 @@
         self.funclist = funclist
         self.baseline = baseline
 
+        # This is not used by this class in any way, but DocTestDispatcher
+        # uses this to keep track of reruns for flaky tests.
+        self.num_retries_left = 0
+
         # Open pipe for messages. These are raw file descriptors,
         # not Python file objects!
         self.rmessages, self.wmessages = os.pipe()
 
         # Create Queue for the result. Since we're running only one
         # doctest, this "queue" will contain only 1 element.
-        self.result_queue = multiprocessing.Queue(1)
+        self.result_queue: "Queue[tuple[int, DictAsObject]]" = multiprocessing.Queue(1)
+
+        # See :meth:`DocTestTask.__call__` for more information of this.
+        self.result: tuple[int, DictAsObject]
 
         # Temporary file for stdout/stderr of the child process.
         # Normally, this isn't used in the master process except to
@@ -2528,6 +2681,7 @@
 
     EXAMPLES::
 
+        sage: # long time
         sage: from sage.doctest.forker import DocTestTask
         sage: from sage.doctest.sources import FileDocTestSource
         sage: from sage.doctest.control import DocTestDefaults, DocTestController
@@ -2587,6 +2741,11 @@
           doctests and ``result_dict`` is a dictionary annotated with
           timings and error information.
 
+          ``result_dict`` contains the key ``'err'`` (possibly ``None``),
+          and optionally contain the keys ``'walltime'``, ``'cputime'``,
+          ``'walltime_skips'``, ``'tb'``, ``'tab_linenos'``, ``'optionals'``,
+          ``'failures'``.
+
         - Also put ``(doctests, result_dict)`` onto the ``result_queue``
           if the latter isn't None.
 

src/sage/doctest/parsing.py

@@ -60,7 +60,7 @@
 )
 
 special_optional_regex_raw = (
-    "py2|long time|not implemented|not tested|optional|needs|known bug"
+    "py2|long time|not implemented|not tested|optional|needs|known bug|flaky"
 )
 tag_with_explanation_regex_raw = r"((?:!?\w|[.])*)\s*(?:\((?P<cmd_explanation>.*?)\))?"
 optional_regex: Pattern[str] = re.compile(
@@ -105,6 +105,7 @@ def parse_optional_tags(
     - ``'not tested'``
     - ``'known bug'`` (possible values are ``None``, ``linux`` and ``macos``)
     - ``'py2'``
+    - ``'flaky'``
     - ``'optional -- FEATURE...'`` or ``'needs FEATURE...'`` --
       the dictionary will just have the key ``'FEATURE'``
 
@@ -174,6 +175,17 @@ def parse_optional_tags(
         sage: parse_optional_tags("sage: #this is not #needs scipy\n....: import scipy",
         ....:                     return_string_sans_tags=True)
         ({'scipy': None}, 'sage: #this is not \n....: import scipy', False)
+
+    TESTS::
+
+        sage: parse_optional_tags("# flaky")
+        {'flaky': None}
+
+    Remember to update the documentation above whenever the following changes::
+
+        sage: from sage.doctest.parsing import special_optional_regex
+        sage: special_optional_regex.pattern
+        'py2|long time|not implemented|not tested|optional|needs|known bug|flaky'
     """
     safe, literals, _ = strip_string_literals(string)
     split = safe.split('\n', 1)
@@ -268,6 +280,11 @@ def parse_file_optional_tags(lines) -> dict[str, str | None]:
         sage: with open(filename, "r") as f:
         ....:     parse_file_optional_tags(enumerate(f))
         {'xyz': None}
+        sage: with open(filename, "w") as f:
+        ....:     _ = f.write("# sage.doctest: flaky")
+        sage: with open(filename, "r") as f:
+        ....:     parse_file_optional_tags(enumerate(f))
+        {'flaky': None}
     """
     tags: dict[str, str | None] = {}
     for line_count, line in lines:
@@ -998,6 +1015,14 @@ def parse(self, string, *args) -> list[doctest.Example | str]:
             'C.minimum_distance(algorithm="guava")  # optional - guava\n'
             sage: dte.want
             '...\n24\n'
+
+        Test putting flaky tag in a single test::
+
+            sage: example5 = 'sage: 1 # flaky\n1'
+            sage: parsed5 = DTP.parse(example5)
+            Traceback (most recent call last):
+            ...
+            NotImplementedError: 'flaky' tag is only implemented for whole file, not for single test
         """
         # Regular expressions
         find_sage_prompt = re.compile(r"^(\s*)sage: ", re.M)
@@ -1071,6 +1096,8 @@ def check_and_clear_tag_counts():
         for item in res:
             if isinstance(item, doctest.Example):
                 optional_tags_with_values, _, is_persistent = parse_optional_tags(item.source, return_string_sans_tags=True)
+                if "flaky" in optional_tags_with_values:
+                    raise NotImplementedError("'flaky' tag is only implemented for whole file, not for single test")
                 optional_tags = set(optional_tags_with_values)
                 if is_persistent:
                     check_and_clear_tag_counts()
@@ -1095,6 +1122,11 @@ def check_and_clear_tag_counts():
                             ('not tested' in optional_tags)):
                         continue
 
+                    if 'flaky' in optional_tags:
+                        # since a single test cannot have the 'flaky' tag,
+                        # this must come from file_optional_tags
+                        optional_tags.remove('flaky')
+
                     if 'long time' in optional_tags:
                         if self.long:
                             optional_tags.remove('long time')

src/sage/libs/singular/function.pyx

@@ -1,3 +1,4 @@
+# sage.doctest: flaky (:issue:`29528`)
 """
 libSingular: Functions
 

src/sage/rings/polynomial/multi_polynomial_libsingular.pyx

@@ -1,3 +1,4 @@
+# sage.doctest: flaky
 r"""
 Multivariate Polynomials via libSINGULAR
 

src/sage/rings/polynomial/plural.pyx

@@ -1,3 +1,4 @@
+# sage.doctest: flaky (:issue:`29528`)
 r"""
 Noncommutative polynomials via libSINGULAR/Plural
 

src/sage/rings/polynomial/polynomial_element.pyx

@@ -1,3 +1,4 @@
+# sage.doctest: flaky (:issue:`39183`)
 """
 Univariate polynomial base class