Enhance mypy support.

.travis.yml

@@ -8,9 +8,7 @@ env: TEST_SUITE=test-main
 matrix:
   include:
   - python: "3.6"
-    env: TEST_SUITE=lint
-  - python: "3.6"
-    env: TEST_SUITE=run-mypy
+    env: TEST_SUITE=test-static-analysis
 install:
   - tools/provision
   - source zulip-api-py*-venv/bin/activate

tools/run-mypy

@@ -8,7 +8,9 @@ import sys
 import argparse
 import subprocess
 
-import lister
+from collections import OrderedDict
+from pathlib import PurePath
+from server_lib import lister
 from typing import cast, Dict, List
 
 TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -17,13 +19,33 @@ os.chdir(os.path.dirname(TOOLS_DIR))
 sys.path.append(os.path.dirname(TOOLS_DIR))
 
 exclude = """
-""".split()
+zulip/integrations/git/zulip_git_config.py
+zulip/integrations/irc/irc_mirror_backend.py
+zulip/integrations/log2zulip/log2zulip
+zulip/integrations/perforce/zulip_perforce_config.py
+zulip/integrations/perforce/git_p4.py
+zulip/integrations/svn/zulip_svn_config.py
+zulip/integrations/zephyr/process_ccache
+zulip/tests/__init__.py
+zulip/tests/test_default_arguments.py
+
+zulip_bots/zulip_bots/bots
+zulip_bots/generate_manifest.py
+zulip_bots/setup.py
+zulip_bots/zulip_bots/lib.py
+zulip_bots/zulip_bots/provision.py
+zulip_bots/zulip_bots/run.py
+zulip_bots/zulip_bots/test_lib.py
+zulip_bots/zulip_bots/test_run.py
+zulip_bots/zulip_bots/zulip_bot_output.py
 
-default_targets = ['zulip/zulip',
-                   'zulip/setup.py']
+zulip_botserver/tests/__init__.py
+zulip_botserver/zulip_botserver/server.py
+zulip_botserver/setup.py
+""".split()
 
 parser = argparse.ArgumentParser(description="Run mypy on files tracked by git.")
-parser.add_argument('targets', nargs='*', default=default_targets,
+parser.add_argument('targets', nargs='*', default=[],
                     help="""files and directories to include in the result.
                     If this is not specified, the current directory is used""")
 parser.add_argument('-m', '--modified', action='store_true', default=False, help='list only modified files')
@@ -57,6 +79,12 @@ pyi_files = set(files_dict['pyi'])
 python_files = [fpath for fpath in files_dict['py']
                 if not fpath.endswith('.py') or fpath + 'i' not in pyi_files]
 
+repo_python_files = OrderedDict([('zulip', []), ('zulip_bots', []), ('zulip_botserver', [])])
+for file_path in python_files:
+    repo = PurePath(file_path).parts[0]
+    if repo in repo_python_files:
+        repo_python_files[repo].append(file_path)
+
 mypy_command = "mypy"
 
 extra_args = ["--check-untyped-defs",
@@ -75,8 +103,12 @@ if args.quick:
     extra_args.append("--quick")
 
 # run mypy
-if python_files:
-    rc = subprocess.call([mypy_command] + extra_args + python_files)
-    sys.exit(rc)
-else:
-    print("There are no files to run mypy on.")
+status = 0
+for repo, python_files in repo_python_files.items():
+    print("Running mypy for `{}`.".format(repo))
+    if python_files:
+        print(python_files)
+        status |= subprocess.call([mypy_command] + extra_args + python_files)
+    else:
+        print("There are no files to run mypy on.")
+sys.exit(status)

tools/server_lib/lister.py

@@ -3,7 +3,6 @@
 from __future__ import absolute_import
 
 import os
-from os.path import abspath
 import sys
 import subprocess
 import re
@@ -54,7 +53,7 @@ def list_files(targets=[], ftypes=[], use_shebang=True, modified_only=False,
         If ftypes is [], all files are included.
     use_shebang - Determine file type of extensionless files from their shebang.
     modified_only - Only include files which have been modified.
-    exclude - List of paths to be excluded, relative to repository root.
+    exclude - List of files or directories to be excluded, relative to repository root.
     group_by_ftype - If True, returns a dict of lists keyed by file type.
         If False, returns a flat list of files.
     extless_only - Only include extensionless files in output.
@@ -66,7 +65,7 @@ def list_files(targets=[], ftypes=[], use_shebang=True, modified_only=False,
     # sys.argv as str, so that battle is already lost.  Settle for hoping
     # everything is UTF-8.
     repository_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).strip().decode('utf-8')
-    exclude_abspaths = [os.path.normpath(os.path.join(repository_root, fpath)) for fpath in exclude]
+    exclude_abspaths = [os.path.abspath(os.path.join(repository_root, fpath)) for fpath in exclude]
 
     cmdline = ['git', 'ls-files'] + targets
     if modified_only:
@@ -84,8 +83,8 @@ def list_files(targets=[], ftypes=[], use_shebang=True, modified_only=False,
         ext = os.path.splitext(fpath)[1]
         if extless_only and ext:
             continue
-        absfpath = abspath(fpath)
-        if any(absfpath == expath or absfpath.startswith(expath + '/')
+        absfpath = os.path.abspath(fpath)
+        if any(absfpath == expath or absfpath.startswith(os.path.abspath(expath) + os.sep)
                for expath in exclude_abspaths):
             continue
 

tools/test-static-analysis

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ev
+
+tools/lint
+tools/run-mypy