diff --git a/.travis.yml b/.travis.yml index af37e44a5..d75502da4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,7 @@ env: TEST_SUITE=test-main matrix: include: - python: "3.6" - env: TEST_SUITE=lint - - python: "3.6" - env: TEST_SUITE=run-mypy + env: TEST_SUITE=test-static-analysis install: - tools/provision - source zulip-api-py*-venv/bin/activate diff --git a/tools/lister.py b/tools/lister.py deleted file mode 100644 index 6e8e547b5..000000000 --- a/tools/lister.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function -from __future__ import absolute_import - -import os -from os.path import abspath -import sys -import subprocess -import re -from collections import defaultdict -import argparse -from six.moves import filter - -from typing import Union, List, Dict - -def get_ftype(fpath, use_shebang): - # type: (str, bool) -> str - ext = os.path.splitext(fpath)[1] - if ext: - return ext[1:] - elif use_shebang: - # opening a file may throw an OSError - with open(fpath) as f: - first_line = f.readline() - if re.search(r'^#!.*\bpython', first_line): - return 'py' - elif re.search(r'^#!.*sh', first_line): - return 'sh' - elif re.search(r'^#!.*\bperl', first_line): - return 'pl' - elif re.search(r'^#!.*\bnode', first_line): - return 'js' - elif re.search(r'^#!.*\bruby', first_line): - return 'rb' - elif re.search(r'^#!', first_line): - print('Error: Unknown shebang in file "%s":\n%s' % (fpath, first_line), file=sys.stderr) - return '' - else: - return '' - else: - return '' - -def list_files(targets=[], ftypes=[], use_shebang=True, modified_only=False, - exclude=[], group_by_ftype=False, extless_only=False): - # type: (List[str], List[str], bool, bool, List[str], bool, bool) -> Union[Dict[str, List[str]], List[str]] - """ - List files tracked by git. - - Returns a list of files which are either in targets or in directories in targets. - If targets is [], list of all tracked files in current directory is returned. - - Other arguments: - ftypes - List of file types on which to filter the search. - If ftypes is [], all files are included. - use_shebang - Determine file type of extensionless files from their shebang. - modified_only - Only include files which have been modified. - exclude - List of paths to be excluded, relative to repository root. - group_by_ftype - If True, returns a dict of lists keyed by file type. - If False, returns a flat list of files. - extless_only - Only include extensionless files in output. - """ - ftypes = [x.strip('.') for x in ftypes] - ftypes_set = set(ftypes) - - # Really this is all bytes -- it's a file path -- but we get paths in - # sys.argv as str, so that battle is already lost. Settle for hoping - # everything is UTF-8. - repository_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).strip().decode('utf-8') - exclude_abspaths = [os.path.normpath(os.path.join(repository_root, fpath)) for fpath in exclude] - - cmdline = ['git', 'ls-files'] + targets - if modified_only: - cmdline.append('-m') - - files_gen = (x.strip() for x in subprocess.check_output(cmdline, universal_newlines=True).split('\n')) - # throw away empty lines and non-files (like symlinks) - files = list(filter(os.path.isfile, files_gen)) - - result_dict = defaultdict(list) # type: Dict[str, List[str]] - result_list = [] # type: List[str] - - for fpath in files: - # this will take a long time if exclude is very large - ext = os.path.splitext(fpath)[1] - if extless_only and ext: - continue - absfpath = abspath(fpath) - if any(absfpath == expath or absfpath.startswith(expath + '/') - for expath in exclude_abspaths): - continue - - if ftypes or group_by_ftype: - try: - filetype = get_ftype(fpath, use_shebang) - except (OSError, UnicodeDecodeError) as e: - etype = e.__class__.__name__ - print('Error: %s while determining type of file "%s":' % (etype, fpath), file=sys.stderr) - print(e, file=sys.stderr) - filetype = '' - if ftypes and filetype not in ftypes_set: - continue - - if group_by_ftype: - result_dict[filetype].append(fpath) - else: - result_list.append(fpath) - - if group_by_ftype: - return result_dict - else: - return result_list - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="List files tracked by git and optionally filter by type") - parser.add_argument('targets', nargs='*', default=[], - help='''files and directories to include in the result. - If this is not specified, the current directory is used''') - parser.add_argument('-m', '--modified', action='store_true', default=False, help='list only modified files') - parser.add_argument('-f', '--ftypes', nargs='+', default=[], - help="list of file types to filter on. All files are included if this option is absent") - parser.add_argument('--ext-only', dest='extonly', action='store_true', default=False, - help='only use extension to determine file type') - parser.add_argument('--exclude', nargs='+', default=[], - help='list of files and directories to exclude from results, relative to repo root') - parser.add_argument('--extless-only', dest='extless_only', action='store_true', default=False, - help='only include extensionless files in output') - args = parser.parse_args() - listing = list_files(targets=args.targets, ftypes=args.ftypes, use_shebang=not args.extonly, - modified_only=args.modified, exclude=args.exclude, extless_only=args.extless_only) - for l in listing: - print(l) diff --git a/tools/run-mypy b/tools/run-mypy index 0f530431b..25f3a14bd 100755 --- a/tools/run-mypy +++ b/tools/run-mypy @@ -8,7 +8,9 @@ import sys import argparse import subprocess -import lister +from collections import OrderedDict +from pathlib import PurePath +from server_lib import lister from typing import cast, Dict, List TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -17,13 +19,33 @@ os.chdir(os.path.dirname(TOOLS_DIR)) sys.path.append(os.path.dirname(TOOLS_DIR)) exclude = """ -""".split() +zulip/integrations/git/zulip_git_config.py +zulip/integrations/irc/irc_mirror_backend.py +zulip/integrations/log2zulip/log2zulip +zulip/integrations/perforce/zulip_perforce_config.py +zulip/integrations/perforce/git_p4.py +zulip/integrations/svn/zulip_svn_config.py +zulip/integrations/zephyr/process_ccache +zulip/tests/__init__.py +zulip/tests/test_default_arguments.py + +zulip_bots/zulip_bots/bots +zulip_bots/generate_manifest.py +zulip_bots/setup.py +zulip_bots/zulip_bots/lib.py +zulip_bots/zulip_bots/provision.py +zulip_bots/zulip_bots/run.py +zulip_bots/zulip_bots/test_lib.py +zulip_bots/zulip_bots/test_run.py +zulip_bots/zulip_bots/zulip_bot_output.py -default_targets = ['zulip/zulip', - 'zulip/setup.py'] +zulip_botserver/tests/__init__.py +zulip_botserver/zulip_botserver/server.py +zulip_botserver/setup.py +""".split() parser = argparse.ArgumentParser(description="Run mypy on files tracked by git.") -parser.add_argument('targets', nargs='*', default=default_targets, +parser.add_argument('targets', nargs='*', default=[], help="""files and directories to include in the result. If this is not specified, the current directory is used""") parser.add_argument('-m', '--modified', action='store_true', default=False, help='list only modified files') @@ -57,6 +79,12 @@ pyi_files = set(files_dict['pyi']) python_files = [fpath for fpath in files_dict['py'] if not fpath.endswith('.py') or fpath + 'i' not in pyi_files] +repo_python_files = OrderedDict([('zulip', []), ('zulip_bots', []), ('zulip_botserver', [])]) +for file_path in python_files: + repo = PurePath(file_path).parts[0] + if repo in repo_python_files: + repo_python_files[repo].append(file_path) + mypy_command = "mypy" extra_args = ["--check-untyped-defs", @@ -75,8 +103,12 @@ if args.quick: extra_args.append("--quick") # run mypy -if python_files: - rc = subprocess.call([mypy_command] + extra_args + python_files) - sys.exit(rc) -else: - print("There are no files to run mypy on.") +status = 0 +for repo, python_files in repo_python_files.items(): + print("Running mypy for `{}`.".format(repo)) + if python_files: + print(python_files) + status |= subprocess.call([mypy_command] + extra_args + python_files) + else: + print("There are no files to run mypy on.") +sys.exit(status) diff --git a/tools/server_lib/lister.py b/tools/server_lib/lister.py index 6e8e547b5..3b82026f2 100755 --- a/tools/server_lib/lister.py +++ b/tools/server_lib/lister.py @@ -3,7 +3,6 @@ from __future__ import absolute_import import os -from os.path import abspath import sys import subprocess import re @@ -54,7 +53,7 @@ def list_files(targets=[], ftypes=[], use_shebang=True, modified_only=False, If ftypes is [], all files are included. use_shebang - Determine file type of extensionless files from their shebang. modified_only - Only include files which have been modified. - exclude - List of paths to be excluded, relative to repository root. + exclude - List of files or directories to be excluded, relative to repository root. group_by_ftype - If True, returns a dict of lists keyed by file type. If False, returns a flat list of files. extless_only - Only include extensionless files in output. @@ -66,7 +65,7 @@ def list_files(targets=[], ftypes=[], use_shebang=True, modified_only=False, # sys.argv as str, so that battle is already lost. Settle for hoping # everything is UTF-8. repository_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).strip().decode('utf-8') - exclude_abspaths = [os.path.normpath(os.path.join(repository_root, fpath)) for fpath in exclude] + exclude_abspaths = [os.path.abspath(os.path.join(repository_root, fpath)) for fpath in exclude] cmdline = ['git', 'ls-files'] + targets if modified_only: @@ -84,8 +83,8 @@ def list_files(targets=[], ftypes=[], use_shebang=True, modified_only=False, ext = os.path.splitext(fpath)[1] if extless_only and ext: continue - absfpath = abspath(fpath) - if any(absfpath == expath or absfpath.startswith(expath + '/') + absfpath = os.path.abspath(fpath) + if any(absfpath == expath or absfpath.startswith(os.path.abspath(expath) + os.sep) for expath in exclude_abspaths): continue diff --git a/tools/test-static-analysis b/tools/test-static-analysis new file mode 100755 index 000000000..6e152e6f6 --- /dev/null +++ b/tools/test-static-analysis @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -ev + +tools/lint +tools/run-mypy