modify pytype_test to run from within pytype too, and support python3

.travis.yml

@@ -11,12 +11,13 @@ matrix:
       env: TEST_CMD="./tests/mypy_test.py --no-implicit-optional"
     - python: "2.7"
       env: TEST_CMD="./tests/pytype_test.py --num-parallel=4"
+      sudo: true
 
 install:
   # pytype needs py-2.7, mypy needs py-3.3+. Additional logic in runtests.py
   - if [[ $TRAVIS_PYTHON_VERSION == '3.6-dev' ]]; then pip install -U flake8==3.3.0 flake8-bugbear>=17.3.0 flake8-pyi>=17.1.0; fi
   - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install -U git+git://github.com/python/mypy git+git://github.com/python/typed_ast; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install -U git+git://github.com/google/pytype; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install -U git+git://github.com/google/pytype; wget https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/14.04/x86_64/python-3.6.tar.bz2; sudo tar xjf python-3.6.tar.bz2 --directory /; fi
 
 script:
   - $TEST_CMD

README.md

@@ -127,7 +127,7 @@ invoking:
 (Note that flake8 only works with Python 3.6 or higher.)
 
 To run the pytype tests, you need a separate virtual environment with
-Python 2.7. Run:
+Python 2.7, and a Python 3.6 interpreter somewhere you can point to. Run:
 ```
 $ virtualenv --python=python2.7 .venv2
 $ source .venv2/bin/activate
@@ -136,7 +136,7 @@ $ source .venv2/bin/activate
 This will install pytype from its GitHub repo. You can then run pytype
 tests by running:
 ```
-(.venv2)$ python tests/pytype_test.py
+(.venv2)$ python tests/pytype_test.py --python36-exe=/path/to/python3.6
 ```
 
 For mypy, if you are in the typeshed repo that is submodule of the

tests/pytype_blacklist.txt

@@ -79,3 +79,13 @@ stdlib/3/tokenize.pyi # parse only
 stdlib/3/types.pyi # parse only
 stdlib/3/urllib/error.pyi # parse only
 stdlib/3/urllib/request.pyi # parse only
+stdlib/3/collections/abc.pyi # parse only
+stdlib/3/signal.pyi # parse only
+stdlib/3/shutil.pyi # parse only
+stdlib/3/re.pyi # parse only
+stdlib/3/posix.pyi # parse only
+stdlib/3/platform.pyi # parse only
+stdlib/3/fcntl.pyi # parse only
+stdlib/3/configparser.pyi # parse only
+stdlib/3/compileall.pyi # parse only
+stdlib/3.4/pathlib.pyi # parse only

tests/pytype_test.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-"""Test runner for typeshed.
+r"""Test runner for typeshed.
 
 Depends on mypy and pytype being installed.
 
@@ -15,17 +15,33 @@
 also discover incorrect usage of imported modules.
 """
 
+import argparse
+import collections
 import os
 import re
-import sys
-import argparse
 import subprocess
-import collections
+import sys
 
-parser = argparse.ArgumentParser(description="Pytype tests.")
-parser.add_argument('-n', '--dry-run', action='store_true', help="Don't actually run tests")
+parser = argparse.ArgumentParser(description='Pytype/typeshed tests.')
+parser.add_argument('-n', '--dry-run', action='store_true',
+                    help="Don't actually run tests")
 parser.add_argument('--num-parallel', type=int, default=1,
-                    help="Number of test processes to spawn")
+                    help='Number of test processes to spawn')
+# Default to '' so that symlinking typeshed/stdlib in cwd will work.
+parser.add_argument('--typeshed-location', type=str, default='',
+                    help='Path to typeshed installation.')
+# Default to '' so that finding pytype in path will work.
+parser.add_argument('--pytype-bin-dir', type=str, default='',
+                    help='Path to directory with pytype and pytd executables.')
+# Set to true to print a stack trace every time an exception is thrown.
+parser.add_argument('--print-stderr', type=bool, default=False,
+                    help='Print stderr every time an error is encountered.')
+# We need to invoke python3.6. The default here works with our travis tests.
+parser.add_argument('--python36-exe', type=str,
+                    default='/opt/python/3.6/bin/python3.6',
+                    help='Path to a python 3.6 interpreter.')
+
+Dirs = collections.namedtuple('Dirs', ['pytype', 'typeshed'])
 
 
 def main():
@@ -40,8 +56,27 @@ def main():
         sys.exit(1)
 
 
-def load_blacklist():
-    filename = os.path.join(os.path.dirname(__file__), "pytype_blacklist.txt")
+def get_project_dirs(args):
+    """Top-level project directories for pytype executables and typeshed."""
+    typeshed_location = args.typeshed_location or os.getcwd()
+    return Dirs(args.pytype_bin_dir, typeshed_location)
+
+
+class PathMatcher(object):
+    def __init__(self, patterns):
+        if patterns:
+            self.matcher = re.compile('(%s)$' % '|'.join(patterns))
+        else:
+            self.matcher = None
+
+    def search(self, path):
+        if not self.matcher:
+            return False
+        return self.matcher.search(path)
+
+
+def load_blacklist(dirs):
+    filename = os.path.join(dirs.typeshed, 'tests', 'pytype_blacklist.txt')
     skip_re = re.compile(r'^\s*([^\s#]+)\s*(?:#.*)?$')
     parse_only_re = re.compile(r'^\s*([^\s#]+)\s*#\s*parse only\s*')
     skip = []
@@ -87,34 +122,60 @@ def communicate(self):
         return self.results
 
 
+def _get_relative(filename):
+    top = filename.find('stdlib/')
+    return filename[top:]
+
+
 def _get_module_name(filename):
-    """Converts a filename stdblib/m.n/module/foo to module.foo."""
-    return '.'.join(filename.split(os.path.sep)[2:]).replace(
+    """Converts a filename stdlib/m.n/module/foo to module.foo."""
+    return '.'.join(_get_relative(filename).split(os.path.sep)[2:]).replace(
         '.pyi', '').replace('.__init__', '')
 
 
-def pytype_test(args):
+def can_run(path, exe, *args):
+    exe = os.path.join(path, exe)
     try:
-        BinaryRun(['pytd', '-h']).communicate()
+        BinaryRun([exe] + list(args)).communicate()
+        return True
     except OSError:
+        return False
+
+def pytype_test(args):
+    dirs = get_project_dirs(args)
+    pytype_exe = os.path.join(dirs.pytype, 'pytype')
+    stdlib_path = os.path.join(dirs.typeshed, 'stdlib')
+
+    if not os.path.isdir(stdlib_path):
+        print('Cannot find typeshed stdlib at %s '
+            '(specify parent dir via --typeshed_location)' % stdlib_path)
+        return 0, 0
+
+    if can_run(dirs.pytype, 'pytd', '-h'):
+        pytd_exe = os.path.join(dirs.pytype, 'pytd')
+    elif can_run(dirs.pytype, 'pytd_tool', '-h'):
+        pytd_exe = os.path.join(dirs.pytype, 'pytd_tool')
+    else:
         print('Cannot run pytd. Did you install pytype?')
         return 0, 0
 
-    skip, parse_only = load_blacklist()
     wanted = re.compile(r'stdlib/.*\.pyi$')
-    skipped = re.compile('(%s)$' % '|'.join(skip))
-    parse_only = re.compile('(%s)$' % '|'.join(parse_only))
+    skip, parse_only = load_blacklist(dirs)
+    skipped = PathMatcher(skip)
+    parse_only = PathMatcher(parse_only)
 
     pytype_run = []
     pytd_run = []
+    bad = []
 
-    for root, _, filenames in os.walk('stdlib'):
+    for root, _, filenames in os.walk(stdlib_path):
         for f in sorted(filenames):
             f = os.path.join(root, f)
-            if wanted.search(f):
-                if parse_only.search(f):
+            rel = _get_relative(f)
+            if wanted.search(rel):
+                if parse_only.search(rel):
                     pytd_run.append(f)
-                elif not skipped.search(f):
+                elif not skipped.search(rel):
                     pytype_run.append(f)
 
     running_tests = collections.deque()
@@ -124,15 +185,22 @@ def pytype_test(args):
         while files and len(running_tests) < args.num_parallel:
             f = files.pop()
             if f in pytype_run:
+                run_cmd = [
+                    pytype_exe,
+                    '--module-name=%s' % _get_module_name(f),
+                    '--parse-pyi'
+                ]
+                if 'stdlib/3' in f:
+                    run_cmd += [
+                        '-V 3.6',
+                        '--python_exe=%s' % args.python36_exe
+                    ]
                 test_run = BinaryRun(
-                    ['pytype',
-                     '--module-name=%s' % _get_module_name(f),
-                     '--parse-pyi',
-                     f],
+                    run_cmd + [f],
                     dry_run=args.dry_run,
-                    env={"TYPESHED_HOME": os.getcwd()})
+                    env={"TYPESHED_HOME": dirs.typeshed})
             elif f in pytd_run:
-                test_run = BinaryRun(['pytd', f], dry_run=args.dry_run)
+                test_run = BinaryRun([pytd_exe, f], dry_run=args.dry_run)
             else:
                 raise ValueError('Unknown action for file: %s' % f)
             running_tests.append(test_run)
@@ -141,15 +209,22 @@ def pytype_test(args):
             break
 
         test_run = running_tests.popleft()
-        code, stdout, stderr = test_run.communicate()
+        code, _, stderr = test_run.communicate()
         max_code = max(max_code, code)
         runs += 1
 
         if code:
-            print(stderr)
+            if args.print_stderr:
+                print(stderr)
             errors += 1
+            # We strip off the stack trace and just leave the last line with the
+            # actual error; to see the stack traces use --print_stderr.
+            bad.append((_get_relative(test_run.args[-1]),
+                        stderr.rstrip().rsplit('\n', 1)[-1]))
 
     print('Ran pytype with %d pyis, got %d errors.' % (runs, errors))
+    for f, err in bad:
+        print('%s: %s' % (f, err))
     return max_code, runs
 
 if __name__ == '__main__':