Windows test fixes and CI with AppVeyor

appveyor.yml

@@ -0,0 +1,19 @@
+environment:
+  matrix:
+  - PYTHON: C:\\Python33
+  - PYTHON: C:\\Python34
+  - PYTHON: C:\\Python35
+  - PYTHON: C:\\Python33-x64
+  - PYTHON: C:\\Python34-x64
+  - PYTHON: C:\\Python35-x64
+install:
+  - "git submodule update --init typeshed"
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;C:\\Python27;%PATH%"
+  - "REN C:\\Python27\\python.exe python2.exe"
+  - "python --version"
+  - "python2 --version"
+build_script:
+  - "pip install -r test-requirements.txt"
+  - "python setup.py install"
+test_script:
+- cmd: python runtests.py -v

mypy/build.py

@@ -762,10 +762,14 @@ def write_cache(id: str, path: str, tree: MypyFile,
     with open(meta_json_tmp, 'w') as f:
         json.dump(meta, f, sort_keys=True)
         f.write('\n')
-    # TODO: On Windows, os.rename() may not be atomic, and we could
-    # use os.replace().  However that's new in Python 3.3.
-    os.rename(data_json_tmp, data_json)
-    os.rename(meta_json_tmp, meta_json)
+    # TODO: This is a temporary change until Python 3.2 support is dropped, see #1504
+    # os.rename will raise an exception rather than replace files on Windows
+    try:
+        replace = os.replace
+    except AttributeError:
+        replace = os.rename
+    replace(data_json_tmp, data_json)
+    replace(meta_json_tmp, meta_json)
 
 
 """Dependency manager.

mypy/test/data.py

@@ -65,6 +65,8 @@ def parse_test_cases(
             tcout = []  # type: List[str]
             if i < len(p) and p[i].id == 'out':
                 tcout = p[i].data
+                if p[i].arg == 'pathfix':
+                    tcout = [s.replace('/', os.path.sep) for s in tcout]
                 ok = True
                 i += 1
             elif optional_out:

mypy/test/data/cmdline.test

@@ -18,7 +18,7 @@ undef
 [file pkg/subpkg/a.py]
 undef
 import pkg.subpkg.a
-[out]
+[out pathfix]
 pkg/a.py:1: error: Name 'undef' is not defined
 pkg/subpkg/a.py:1: error: Name 'undef' is not defined
 
@@ -31,7 +31,7 @@ undef
 [file pkg/subpkg/a.py]
 undef
 import pkg.subpkg.a
-[out]
+[out pathfix]
 pkg/a.py:1: error: Name 'undef' is not defined
 pkg/subpkg/a.py:1: error: Name 'undef' is not defined
 
@@ -41,7 +41,7 @@ pkg/subpkg/a.py:1: error: Name 'undef' is not defined
 undef
 [file dir/subdir/a.py]
 undef
-[out]
+[out pathfix]
 dir/a.py:1: error: Name 'undef' is not defined
 
 [case testCmdlineNonPackageSlash]
@@ -50,7 +50,7 @@ dir/a.py:1: error: Name 'undef' is not defined
 undef
 [file dir/subdir/a.py]
 undef
-[out]
+[out pathfix]
 dir/a.py:1: error: Name 'undef' is not defined
 
 [case testCmdlinePackageContainingSubdir]
@@ -60,7 +60,7 @@ dir/a.py:1: error: Name 'undef' is not defined
 undef
 [file pkg/subdir/a.py]
 undef
-[out]
+[out pathfix]
 pkg/a.py:1: error: Name 'undef' is not defined
 
 [case testCmdlineNonPackageContainingPackage]
@@ -71,6 +71,6 @@ import subpkg.a
 [file dir/subpkg/__init__.py]
 [file dir/subpkg/a.py]
 undef
-[out]
+[out pathfix]
 dir/subpkg/a.py:1: error: Name 'undef' is not defined
 dir/a.py:1: error: Name 'undef' is not defined

mypy/test/data/pythoneval.test

@@ -16,7 +16,7 @@ import re
 from typing import Sized, Sequence, Iterator, Iterable, Mapping, AbstractSet
 
 def check(o, t):
-    rep = re.sub('0x[0-9a-f]+', '0x...', repr(o))
+    rep = re.sub('0x[0-9a-fA-F]+', '0x...', repr(o))
     rep = rep.replace('sequenceiterator', 'str_iterator')
     trep = str(t).replace('_abcoll.Sized', 'collections.abc.Sized')
     print(rep, trep, isinstance(o, t))

mypy/test/data/semanal-modules.test

@@ -770,7 +770,7 @@ import m.x
 [file m/__init__.py]
 [file m/x.py]
 from .x import nonexistent
-[out]
+[out pathfix]
 main:1: note: In module imported here:
 tmp/m/x.py:1: error: Module has no attribute 'nonexistent'
 
@@ -779,7 +779,7 @@ import m.x
 [file m/__init__.py]
 [file m/x.py]
 from m.x import nonexistent
-[out]
+[out pathfix]
 main:1: note: In module imported here:
 tmp/m/x.py:1: error: Module has no attribute 'nonexistent'
 
@@ -846,7 +846,7 @@ import m
 x
 [file m.py]
 y
-[out]
+[out pathfix]
 main:1: note: In module imported here:
 tmp/m.py:1: error: Name 'y' is not defined
 main:2: error: Name 'x' is not defined

mypy/test/testcheck.py

@@ -152,7 +152,8 @@ def find_error_paths(self, a: List[str]) -> Set[str]:
         for line in a:
             m = re.match(r'([^\s:]+):\d+: error:', line)
             if m:
-                hits.add(m.group(1))
+                p = m.group(1).replace('/', os.path.sep)
+                hits.add(p)
         return hits
 
     def find_module_files(self) -> Dict[str, str]:

mypy/test/teststubgen.py

@@ -108,34 +108,35 @@ def test_stubgen(testcase):
         sys.path.insert(0, 'stubgen-test-path')
     os.mkdir('stubgen-test-path')
     source = '\n'.join(testcase.input)
-    handle = tempfile.NamedTemporaryFile(prefix='prog_', suffix='.py', dir='stubgen-test-path')
+    handle = tempfile.NamedTemporaryFile(prefix='prog_', suffix='.py', dir='stubgen-test-path',
+                                         delete=False)
     assert os.path.isabs(handle.name)
     path = os.path.basename(handle.name)
     name = path[:-3]
     path = os.path.join('stubgen-test-path', path)
     out_dir = '_out'
     os.mkdir(out_dir)
     try:
-        with open(path, 'w') as file:
-            file.write(source)
-            file.close()
-            # Without this we may sometimes be unable to import the module below, as importlib
-            # caches os.listdir() results in Python 3.3+ (Guido explained this to me).
-            reset_importlib_caches()
-            try:
-                if testcase.name.endswith('_import'):
-                    generate_stub_for_module(name, out_dir, quiet=True)
-                else:
-                    generate_stub(path, out_dir)
-                a = load_output(out_dir)
-            except CompileError as e:
-                a = e.messages
-            assert_string_arrays_equal(testcase.output, a,
-                                       'Invalid output ({}, line {})'.format(
-                                           testcase.file, testcase.line))
+        handle.write(bytes(source, 'ascii'))
+        handle.close()
+        # Without this we may sometimes be unable to import the module below, as importlib
+        # caches os.listdir() results in Python 3.3+ (Guido explained this to me).
+        reset_importlib_caches()
+        try:
+            if testcase.name.endswith('_import'):
+                generate_stub_for_module(name, out_dir, quiet=True)
+            else:
+                generate_stub(path, out_dir)
+            a = load_output(out_dir)
+        except CompileError as e:
+            a = e.messages
+        assert_string_arrays_equal(testcase.output, a,
+                                   'Invalid output ({}, line {})'.format(
+                                       testcase.file, testcase.line))
     finally:
-        shutil.rmtree(out_dir)
         handle.close()
+        os.unlink(handle.name)
+        shutil.rmtree(out_dir)
 
 
 def reset_importlib_caches():

mypy/waiter.py

@@ -31,42 +31,23 @@ def __init__(self, name: str, args: List[str], *, cwd: str = None,
         self.end_time = None  # type: float
 
     def start(self) -> None:
-        self.outfile = tempfile.NamedTemporaryFile()
+        self.outfile = tempfile.TemporaryFile()
         self.start_time = time.time()
         self.process = Popen(self.args, cwd=self.cwd, env=self.env,
                              stdout=self.outfile, stderr=STDOUT)
         self.pid = self.process.pid
 
-    def handle_exit_status(self, status: int) -> None:
-        """Update process exit status received via an external os.waitpid() call."""
-        # Inlined subprocess._handle_exitstatus, it's not a public API.
-        # TODO(jukka): I'm not quite sure why this is implemented like this.
-        self.end_time = time.time()
-        process = self.process
-        assert process.returncode is None
-        if os.WIFSIGNALED(status):
-            process.returncode = -os.WTERMSIG(status)
-        elif os.WIFEXITED(status):
-            process.returncode = os.WEXITSTATUS(status)
-        else:
-            # Should never happen
-            raise RuntimeError("Unknown child exit status!")
-        assert process.returncode is not None
-
     def wait(self) -> int:
         return self.process.wait()
 
     def status(self) -> Optional[int]:
         return self.process.returncode
 
     def read_output(self) -> str:
-        with open(self.outfile.name, 'rb') as file:
-            # Assume it's ascii to avoid unicode headaches (and portability issues).
-            return file.read().decode('ascii')
-
-    def cleanup(self) -> None:
-        self.outfile.close()
-        assert not os.path.exists(self.outfile.name)
+        file = self.outfile
+        file.seek(0)
+        # Assume it's ascii to avoid unicode headaches (and portability issues).
+        return file.read().decode('ascii')
 
     @property
     def elapsed_time(self) -> float:
@@ -178,17 +159,25 @@ def _record_time(self, name: str, elapsed_time: float) -> None:
         name2 = re.sub('( .*?) .*', r'\1', name)  # First two words.
         self.times2[name2] = elapsed_time + self.times2.get(name2, 0)
 
+    def _poll_current(self) -> Tuple[int, int]:
+        while True:
+            time.sleep(.25)
+            for pid in self.current:
+                cmd = self.current[pid][1]
+                code = cmd.process.poll()
+                if code is not None:
+                    cmd.end_time = time.time()
+                    return pid, code
+
     def _wait_next(self) -> Tuple[List[str], int, int]:
         """Wait for a single task to finish.
 
         Return tuple (list of failed tasks, number test cases, number of failed tests).
         """
-        pid, status = os.waitpid(-1, 0)
+        pid, status = self._poll_current()
         num, cmd = self.current.pop(pid)
         name = cmd.name
 
-        cmd.handle_exit_status(status)
-
         self._record_time(cmd.name, cmd.elapsed_time)
 
         rc = cmd.wait()
@@ -223,7 +212,6 @@ def _wait_next(self) -> Tuple[List[str], int, int]:
 
         # Get task output.
         output = cmd.read_output()
-        cmd.cleanup()
         num_tests, num_tests_failed = parse_test_stats_from_output(output, fail_type)
 
         if fail_type is not None or self.verbosity >= 1: