From 77de99a815d7357094d7680fa4ac274841fc46a6 Mon Sep 17 00:00:00 2001 From: James Tatum Date: Fri, 6 May 2016 23:57:51 -0700 Subject: [PATCH 1/4] Make waiter cross-platform Waiter depended on a couple of posix tricks. The first was a clever use of os.waitpid(-1, 0) to wait for any spawned processes to return. On Windows, this raises an exception - Windows has no concept of a process group. The second was the use of NamedTempFile(). On anything but Windows, a file can be open for writing and for reading at the same time. This trick actually isn't necessary the way NamedTempFile is used here. It exposes a file-like object pointing to a file already opened in binary mode. All we have to do is seek and read from it. --- mypy/waiter.py | 44 ++++++++++++++++---------------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/mypy/waiter.py b/mypy/waiter.py index 50a949fbed45..4bb66654ca92 100644 --- a/mypy/waiter.py +++ b/mypy/waiter.py @@ -31,28 +31,12 @@ def __init__(self, name: str, args: List[str], *, cwd: str = None, self.end_time = None # type: float def start(self) -> None: - self.outfile = tempfile.NamedTemporaryFile() + self.outfile = tempfile.TemporaryFile() self.start_time = time.time() self.process = Popen(self.args, cwd=self.cwd, env=self.env, stdout=self.outfile, stderr=STDOUT) self.pid = self.process.pid - def handle_exit_status(self, status: int) -> None: - """Update process exit status received via an external os.waitpid() call.""" - # Inlined subprocess._handle_exitstatus, it's not a public API. - # TODO(jukka): I'm not quite sure why this is implemented like this. - self.end_time = time.time() - process = self.process - assert process.returncode is None - if os.WIFSIGNALED(status): - process.returncode = -os.WTERMSIG(status) - elif os.WIFEXITED(status): - process.returncode = os.WEXITSTATUS(status) - else: - # Should never happen - raise RuntimeError("Unknown child exit status!") - assert process.returncode is not None - def wait(self) -> int: return self.process.wait() @@ -60,13 +44,10 @@ def status(self) -> Optional[int]: return self.process.returncode def read_output(self) -> str: - with open(self.outfile.name, 'rb') as file: - # Assume it's ascii to avoid unicode headaches (and portability issues). - return file.read().decode('ascii') - - def cleanup(self) -> None: - self.outfile.close() - assert not os.path.exists(self.outfile.name) + file = self.outfile + file.seek(0) + # Assume it's ascii to avoid unicode headaches (and portability issues). + return file.read().decode('ascii') @property def elapsed_time(self) -> float: @@ -178,17 +159,25 @@ def _record_time(self, name: str, elapsed_time: float) -> None: name2 = re.sub('( .*?) .*', r'\1', name) # First two words. self.times2[name2] = elapsed_time + self.times2.get(name2, 0) + def _poll_current(self) -> Tuple[int, int]: + while True: + time.sleep(.25) + for pid in self.current: + cmd = self.current[pid][1] + code = cmd.process.poll() + if code is not None: + cmd.end_time = time.time() + return pid, code + def _wait_next(self) -> Tuple[List[str], int, int]: """Wait for a single task to finish. Return tuple (list of failed tasks, number test cases, number of failed tests). """ - pid, status = os.waitpid(-1, 0) + pid, status = self._poll_current() num, cmd = self.current.pop(pid) name = cmd.name - cmd.handle_exit_status(status) - self._record_time(cmd.name, cmd.elapsed_time) rc = cmd.wait() @@ -223,7 +212,6 @@ def _wait_next(self) -> Tuple[List[str], int, int]: # Get task output. output = cmd.read_output() - cmd.cleanup() num_tests, num_tests_failed = parse_test_stats_from_output(output, fail_type) if fail_type is not None or self.verbosity >= 1: From addd9f4e8573807df2d4b1f767583a2802cae794 Mon Sep 17 00:00:00 2001 From: James Tatum Date: Sat, 7 May 2016 00:06:06 -0700 Subject: [PATCH 2/4] Add appveyor config for Windows CI The only tricky bit of this is renaming python.exe to python2.exe. This is due to util.try_find_python2_interpreter(), which may well need work for Windows since the version symlinks don't exist on Windows. --- appveyor.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000000..4d1dc1161331 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,19 @@ +environment: + matrix: + - PYTHON: C:\\Python33 + - PYTHON: C:\\Python34 + - PYTHON: C:\\Python35 + - PYTHON: C:\\Python33-x64 + - PYTHON: C:\\Python34-x64 + - PYTHON: C:\\Python35-x64 +install: + - "git submodule update --init typeshed" + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;C:\\Python27;%PATH%" + - "REN C:\\Python27\\python.exe python2.exe" + - "python --version" + - "python2 --version" +build_script: + - "pip install -r test-requirements.txt" + - "python setup.py install" +test_script: +- cmd: python runtests.py -v From 81511fca91ee8dc6dfae7d57147851c9f986f0b4 Mon Sep 17 00:00:00 2001 From: James Tatum Date: Sat, 7 May 2016 09:38:20 -0700 Subject: [PATCH 3/4] Fixing Windows tests Most of these fixes revolve around the path separator and the way Windows handles files and locking. There is one bug fix in here - build.write_cache() was using os.rename to replace a file, which fails on Windows. I was only able to fix that for Python 3.3 and up. --- mypy/build.py | 12 ++++++--- mypy/test/data.py | 2 ++ mypy/test/data/cmdline.test | 12 ++++----- mypy/test/data/pythoneval.test | 2 +- mypy/test/data/semanal-modules.test | 6 ++--- mypy/test/testcheck.py | 3 ++- mypy/test/teststubgen.py | 39 +++++++++++++++-------------- 7 files changed, 42 insertions(+), 34 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 1c3c0ff2432d..a73fa9684dcd 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -762,10 +762,14 @@ def write_cache(id: str, path: str, tree: MypyFile, with open(meta_json_tmp, 'w') as f: json.dump(meta, f, sort_keys=True) f.write('\n') - # TODO: On Windows, os.rename() may not be atomic, and we could - # use os.replace(). However that's new in Python 3.3. - os.rename(data_json_tmp, data_json) - os.rename(meta_json_tmp, meta_json) + # TODO: This is a temporary change until Python 3.2 support is dropped, see #1504 + # os.rename will raise an exception rather than replace files on Windows + try: + replace = os.replace + except AttributeError: + replace = os.rename + replace(data_json_tmp, data_json) + replace(meta_json_tmp, meta_json) """Dependency manager. diff --git a/mypy/test/data.py b/mypy/test/data.py index 07f6d00f3e5e..bbda65c11cd3 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -65,6 +65,8 @@ def parse_test_cases( tcout = [] # type: List[str] if i < len(p) and p[i].id == 'out': tcout = p[i].data + if p[i].arg == 'pathfix': + tcout = [s.replace('/', os.path.sep) for s in tcout] ok = True i += 1 elif optional_out: diff --git a/mypy/test/data/cmdline.test b/mypy/test/data/cmdline.test index 276300b49173..ed6d41a1f98b 100644 --- a/mypy/test/data/cmdline.test +++ b/mypy/test/data/cmdline.test @@ -18,7 +18,7 @@ undef [file pkg/subpkg/a.py] undef import pkg.subpkg.a -[out] +[out pathfix] pkg/a.py:1: error: Name 'undef' is not defined pkg/subpkg/a.py:1: error: Name 'undef' is not defined @@ -31,7 +31,7 @@ undef [file pkg/subpkg/a.py] undef import pkg.subpkg.a -[out] +[out pathfix] pkg/a.py:1: error: Name 'undef' is not defined pkg/subpkg/a.py:1: error: Name 'undef' is not defined @@ -41,7 +41,7 @@ pkg/subpkg/a.py:1: error: Name 'undef' is not defined undef [file dir/subdir/a.py] undef -[out] +[out pathfix] dir/a.py:1: error: Name 'undef' is not defined [case testCmdlineNonPackageSlash] @@ -50,7 +50,7 @@ dir/a.py:1: error: Name 'undef' is not defined undef [file dir/subdir/a.py] undef -[out] +[out pathfix] dir/a.py:1: error: Name 'undef' is not defined [case testCmdlinePackageContainingSubdir] @@ -60,7 +60,7 @@ dir/a.py:1: error: Name 'undef' is not defined undef [file pkg/subdir/a.py] undef -[out] +[out pathfix] pkg/a.py:1: error: Name 'undef' is not defined [case testCmdlineNonPackageContainingPackage] @@ -71,6 +71,6 @@ import subpkg.a [file dir/subpkg/__init__.py] [file dir/subpkg/a.py] undef -[out] +[out pathfix] dir/subpkg/a.py:1: error: Name 'undef' is not defined dir/a.py:1: error: Name 'undef' is not defined diff --git a/mypy/test/data/pythoneval.test b/mypy/test/data/pythoneval.test index 9559509c9dbf..63198420b7fc 100644 --- a/mypy/test/data/pythoneval.test +++ b/mypy/test/data/pythoneval.test @@ -16,7 +16,7 @@ import re from typing import Sized, Sequence, Iterator, Iterable, Mapping, AbstractSet def check(o, t): - rep = re.sub('0x[0-9a-f]+', '0x...', repr(o)) + rep = re.sub('0x[0-9a-fA-F]+', '0x...', repr(o)) rep = rep.replace('sequenceiterator', 'str_iterator') trep = str(t).replace('_abcoll.Sized', 'collections.abc.Sized') print(rep, trep, isinstance(o, t)) diff --git a/mypy/test/data/semanal-modules.test b/mypy/test/data/semanal-modules.test index a7cbef578085..86131a345aed 100644 --- a/mypy/test/data/semanal-modules.test +++ b/mypy/test/data/semanal-modules.test @@ -770,7 +770,7 @@ import m.x [file m/__init__.py] [file m/x.py] from .x import nonexistent -[out] +[out pathfix] main:1: note: In module imported here: tmp/m/x.py:1: error: Module has no attribute 'nonexistent' @@ -779,7 +779,7 @@ import m.x [file m/__init__.py] [file m/x.py] from m.x import nonexistent -[out] +[out pathfix] main:1: note: In module imported here: tmp/m/x.py:1: error: Module has no attribute 'nonexistent' @@ -846,7 +846,7 @@ import m x [file m.py] y -[out] +[out pathfix] main:1: note: In module imported here: tmp/m.py:1: error: Name 'y' is not defined main:2: error: Name 'x' is not defined diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 16f60648a41a..2a9d7b45545d 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -152,7 +152,8 @@ def find_error_paths(self, a: List[str]) -> Set[str]: for line in a: m = re.match(r'([^\s:]+):\d+: error:', line) if m: - hits.add(m.group(1)) + p = m.group(1).replace('/', os.path.sep) + hits.add(p) return hits def find_module_files(self) -> Dict[str, str]: diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 893659d2f6cf..a74525f14787 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -108,7 +108,8 @@ def test_stubgen(testcase): sys.path.insert(0, 'stubgen-test-path') os.mkdir('stubgen-test-path') source = '\n'.join(testcase.input) - handle = tempfile.NamedTemporaryFile(prefix='prog_', suffix='.py', dir='stubgen-test-path') + handle = tempfile.NamedTemporaryFile(prefix='prog_', suffix='.py', dir='stubgen-test-path', + delete=False) assert os.path.isabs(handle.name) path = os.path.basename(handle.name) name = path[:-3] @@ -116,26 +117,26 @@ def test_stubgen(testcase): out_dir = '_out' os.mkdir(out_dir) try: - with open(path, 'w') as file: - file.write(source) - file.close() - # Without this we may sometimes be unable to import the module below, as importlib - # caches os.listdir() results in Python 3.3+ (Guido explained this to me). - reset_importlib_caches() - try: - if testcase.name.endswith('_import'): - generate_stub_for_module(name, out_dir, quiet=True) - else: - generate_stub(path, out_dir) - a = load_output(out_dir) - except CompileError as e: - a = e.messages - assert_string_arrays_equal(testcase.output, a, - 'Invalid output ({}, line {})'.format( - testcase.file, testcase.line)) + handle.write(bytes(source, 'ascii')) + handle.close() + # Without this we may sometimes be unable to import the module below, as importlib + # caches os.listdir() results in Python 3.3+ (Guido explained this to me). + reset_importlib_caches() + try: + if testcase.name.endswith('_import'): + generate_stub_for_module(name, out_dir, quiet=True) + else: + generate_stub(path, out_dir) + a = load_output(out_dir) + except CompileError as e: + a = e.messages + assert_string_arrays_equal(testcase.output, a, + 'Invalid output ({}, line {})'.format( + testcase.file, testcase.line)) finally: - shutil.rmtree(out_dir) handle.close() + os.unlink(handle.name) + shutil.rmtree(out_dir) def reset_importlib_caches(): From e843a347ae04c482884e884aa702d270aa206f68 Mon Sep 17 00:00:00 2001 From: James Tatum Date: Sat, 7 May 2016 13:40:27 -0700 Subject: [PATCH 4/4] Update typeshed to latest --- typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typeshed b/typeshed index 292447bd6270..33fe6a0685b5 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 292447bd627041ec055b633cac1f5895790b73f1 +Subproject commit 33fe6a0685b5647477801373914b137aecb8c4b8