Add timeout for starting tarantool server

Function wait_until_started in TarantoolServer contains seek_wait,
which waits pattern in logfile. If there is no pattern, server is
hanging. Was added start-server-time (by default equals to 90 secs).
The pattern is sought until the time runs out and wait_until_started
returns True if the pattern was found (else False) and
TarantoolServer.start() returns same. Was added new logging that the
instance wasn't started.

Fixes: #276

Author: VitaliyaIoffe

lib/options.py

@@ -211,6 +211,14 @@ def __init__(self):
                 help="""Break the test process with kill signal if the test runs
                 longer than this amount of seconds. Default: 110 [seconds].""")
 
+        parser.add_argument(
+            "--server-start-timeout",
+            dest="server_start_timeout",
+            default=env_int('SERVER_START_TIMEOUT', 90),
+            type=int,
+            help="""Break the test process with kill signal if the test starts
+            longer than this amount of seconds. Default: 90 [seconds].""")
+
         parser.add_argument(
                 "--no-output-timeout",
                 dest="no_output_timeout",

lib/preprocessor.py

@@ -7,6 +7,7 @@
 import yaml
 from gevent import socket
 
+from lib import Options
 from lib.admin_connection import AdminAsyncConnection
 from lib.colorer import color_log
 from lib.utils import signum
@@ -206,8 +207,13 @@ def server_start(self, ctype, sname, opts):
             if crash_expected:
                 # disable crash detector
                 self.servers[sname].crash_expected = True
-            self.servers[sname].start(silent=True, rais=True, wait=wait,
-                                      wait_load=wait_load, args=args)
+            if not self.servers[sname].start(
+                    silent=True, rais=True, wait=wait,
+                    wait_load=wait_load, args=args,
+                    server_start_timeout=Options().args.server_start_timeout):
+                color_log('\n[Instance "{0}"] Failed to start tarantool '
+                          'instance\n'.format(sname),
+                          schema='error')
         except Exception as e:
             crash_occured = True
             if not (crash_expected and

lib/tarantool_server.py

@@ -445,7 +445,7 @@ def seek_once(self, msg):
                 if pos != -1:
                     return pos
 
-    def seek_wait(self, msg, proc=None, name=None):
+    def seek_wait(self, msg, proc=None, name=None, timeout=90):
         while True:
             if os.path.exists(self.path):
                 break
@@ -454,7 +454,8 @@ def seek_wait(self, msg, proc=None, name=None):
         with open(self.path, 'r') as f:
             f.seek(self.log_begin, os.SEEK_SET)
             cur_pos = self.log_begin
-            while True:
+            current_time = time.time()
+            while current_time + timeout > time.time():
                 if not (proc is None):
                     if not (proc.poll() is None):
                         raise TarantoolStartError(name)
@@ -464,8 +465,9 @@ def seek_wait(self, msg, proc=None, name=None):
                     f.seek(cur_pos, os.SEEK_SET)
                     continue
                 if re.findall(msg, log_str):
-                    return
+                    return True
                 cur_pos = f.tell()
+        return False
 
 
 class TarantoolServer(Server):
@@ -756,7 +758,8 @@ def install(self, silent=True):
 
     def deploy(self, silent=True, **kwargs):
         self.install(silent)
-        self.start(silent=silent, **kwargs)
+        if not self.start(silent=silent, **kwargs):
+            return False
 
     def copy_files(self):
         if self.script:
@@ -829,7 +832,7 @@ def cleanup(self, *args, **kwargs):
                                              *args, **kwargs)
 
     def start(self, silent=True, wait=True, wait_load=True, rais=True, args=[],
-              **kwargs):
+              server_start_timeout=90, **kwargs):
         if self._start_against_running:
             return
         if self.status == 'started':
@@ -880,7 +883,9 @@ def start(self, silent=True, wait=True, wait_load=True, rais=True, args=[],
 
         if wait:
             try:
-                self.wait_until_started(wait_load)
+                if not self.wait_until_started(
+                        wait_load, server_start_timeout=server_start_timeout):
+                    return False
             except TarantoolStartError:
                 # Python tests expect we raise an exception when non-default
                 # server fails
@@ -1090,22 +1095,26 @@ def kill_old_server(self, silent=True):
         self.wait_until_stopped(pid)
         return True
 
-    def wait_until_started(self, wait_load=True):
+    def wait_until_started(self, wait_load=True, server_start_timeout=90):
         """ Wait until server is started.
 
         Server consists of two parts:
         1) wait until server is listening on sockets
         2) wait until server tells us his status
 
         """
-        color_log('DEBUG: [Instance {}] Waiting until started '
-                  '(wait_load={})\n'.format(self.name, str(wait_load)),
-                  schema='info')
+        color_log(
+            'DEBUG: [Instance {}] Waiting until started '
+            '(wait_load={}, start_timeout={})\n'.format(
+                self.name, str(wait_load), str(server_start_timeout)),
+            schema='info')
 
         if wait_load:
             msg = 'entering the event loop|will retry binding|hot standby mode'
             p = self.process if not self.gdb and not self.lldb else None
-            self.logfile_pos.seek_wait(msg, p, self.name)
+            if not self.logfile_pos.seek_wait(
+                    msg, p, self.name, server_start_timeout):
+                return False
         while True:
             try:
                 temp = AdminConnection('localhost', self.admin.port)