Enable test reruns on failed fragiled tests

Added ability to set per suite in suite.ini configuration file
'retries' option, which sets the number of accepted reruns of
the tests failed from 'fragile' list:

  fragile = {
    "retries": 10,
    "tests": {
        "bitset.test.lua": {
            "issues": [ "gh-4095" ],
        }
    }}

Part of #189

Author: avtikhon

lib/test_suite.py

@@ -81,7 +81,7 @@ def __init__(self, suite_path, args):
         self.args = args
         self.tests = []
         self.ini = {}
-        self.fragile = {'tests': {}}
+        self.fragile = {'retries': 0, 'tests': {}}
         self.suite_path = suite_path
         self.ini["core"] = "tarantool"
 
@@ -265,5 +265,8 @@ def run_test(self, test, server, inspector):
     def is_parallel(self):
         return self.ini['is_parallel']
 
+    def fragile_retries(self):
+        return self.fragile.get('retries', 0)
+
     def show_reproduce_content(self):
         return self.ini['show_reproduce_content']

lib/worker.py

@@ -326,8 +326,21 @@ def run_loop(self, task_queue, result_queue):
                 self.stop_worker(task_queue, result_queue)
                 break
 
+            short_status = None
             result_queue.put(self.current_task(task_id))
-            short_status = self.run_task(task_id)
+            retries_left = self.suite.fragile_retries()
+            # let's run till short_status became 'pass'
+            while short_status != 'pass' and retries_left >= 0:
+                # print message only after some fails occurred
+                if short_status == 'fail':
+                    color_stdout(
+                        'Test "%s", conf: "%s"\n'
+                        '\tfrom "fragile" list failed, rerunning ...\n'
+                        % (task_id[0], task_id[1]), schema='error')
+                # run task and save the result to short_status
+                short_status = self.run_task(task_id)
+                retries_left = retries_left - 1
+
             result_queue.put(self.wrap_result(task_id, short_status))
             if not lib.Options().args.is_force and short_status == 'fail':
                 color_stdout(