From 8298af027a7cd931256b6fa39d1f84c11f6940d4 Mon Sep 17 00:00:00 2001 From: Matthias Kramm Date: Tue, 13 Oct 2015 08:30:15 -0700 Subject: [PATCH 1/7] add git (submodule) integrity check If mypy is a git repository, check that the submodules are up-to-date (and abort if not) in runtests.py, scripts/mypy, setup.py. New option --dirty-stubs for runtests.py + scripts/mypy, for bypassing said check. --- mypy/git.py | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++ mypy/main.py | 7 ++++ runtests.py | 8 ++++ setup.py | 3 ++ 4 files changed, 127 insertions(+) create mode 100644 mypy/git.py diff --git a/mypy/git.py b/mypy/git.py new file mode 100644 index 000000000000..94d7d1204cff --- /dev/null +++ b/mypy/git.py @@ -0,0 +1,109 @@ +"""Utilities for verifying git integrity.""" + +# Used also from setup.py, so don't pull in anything additional here (like mypy or typing): +import os +import pipes +import subprocess +import sys + + +def is_git_repo(dir: str) -> bool: + """Is the passed directory version controlled with git?""" + return os.path.exists(os.path.join(dir, ".git")) + + +def have_git() -> bool: + """Can we run the git executable?""" + try: + subprocess.check_output(["git", "config", "-l"]) + return True + except subprocess.CalledProcessError: + return False + + +def run_in_dir(dir, command) -> bytes: + """Convenience function: Run a command in a directory.""" + return subprocess.check_output("cd " + pipes.quote(dir) + "; " + command, + shell=True) + + +def get_submodules(dir: str): + """Return a list of all git top-level submodules in a given directory.""" + # It would be nicer to do + # "git submodule foreach 'echo MODULE $name $path $sha1 $toplevel'" + # but that wouldn't work on Windows. + output = run_in_dir(dir, "git submodule status") + for line in output.splitlines(): + status = line[0] + sha5, name, *_ = line[1:].split(b" ") + yield (status, sha5, name.decode(sys.getfilesystemencoding())) + + +def git_revision(dir: str) -> bytes: + """Get the SHA-1 of the HEAD of a git repository.""" + return run_in_dir(dir, "git rev-parse HEAD").strip() + + +def is_dirty(dir: str) -> bool: + """Check whether a git repository has uncommitted changes.""" + return run_in_dir(dir, "git status -uno --porcelain").strip() != b"" + + +def has_extra_files(dir: str) -> bool: + """Check whether a git repository has untracked files.""" + return run_in_dir(dir, "git clean --dry-run -d").strip() != b"" + + +def warn_no_git_executable() -> None: + print("Warning: Couldn't check git integrity. " + "git executable not in path.", file=sys.stderr) + + +def warn_dirty(dir) -> None: + print("Warning: git module '{}' has uncommitted changes.".format(dir), + file=sys.stderr) + + +def warn_extra_files(dir) -> None: + print("Warning: git module '{}' has untracked files.".format(dir), + file=sys.stderr) + + +def error_submodule_not_initialized(name: str, dir: str) -> None: + print("Submodule '{}' not initialized.".format(name), file=sys.stderr) + print("Please run:", file=sys.stderr) + print(" cd {}".format(pipes.quote(dir)), file=sys.stderr) + print(" git submodule init {}".format(name), file=sys.stderr) + + +def error_submodule_not_updated(name: str, dir: str) -> None: + print("Submodule '{}' not updated.".format(name), file=sys.stderr) + print("Please run:", file=sys.stderr) + print(" cd {}".format(pipes.quote(dir)), file=sys.stderr) + print(" git submodule update {}".format(name), file=sys.stderr) + + +def verify_git_integrity_or_abort(datadir: str) -> None: + """Verify the (submodule) integrity of a git repository. + + Potentially output warnings/errors (to stderr), and exit with status 1 + if we detected a severe problem. + """ + + if not is_git_repo(datadir): + return + if not have_git(): + warn_no_git_executable() + return + for _, revision, submodule in get_submodules(datadir): + submodule_path = os.path.join(datadir, submodule) + if not is_git_repo(submodule_path): + error_submodule_not_initialized(submodule, datadir) + sys.exit(1) + elif revision != git_revision(submodule_path): + error_submodule_not_updated(submodule, datadir) + sys.exit(1) + elif is_dirty(submodule_path): + warn_dirty(submodule) + elif has_extra_files(submodule_path): + warn_extra_files(submodule) diff --git a/mypy/main.py b/mypy/main.py index cc9f126b39bb..2f50e52cb962 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -12,6 +12,7 @@ from mypy import build from mypy import defaults +from mypy import git from mypy.errors import CompileError from mypy.version import __version__ @@ -26,6 +27,7 @@ def __init__(self) -> None: self.custom_typing_module = None # type: str self.report_dirs = {} # type: Dict[str, str] self.python_path = False + self.dirty_stubs = False def main(script_path: str) -> None: @@ -39,6 +41,8 @@ def main(script_path: str) -> None: else: bin_dir = None path, module, program_text, options = process_options(sys.argv[1:]) + if not options.dirty_stubs: + git.verify_git_integrity_or_abort(build.default_data_dir(bin_dir)) try: if options.target == build.TYPE_CHECK: type_check_only(path, module, program_text, bin_dir, options) @@ -117,6 +121,9 @@ def process_options(args: List[str]) -> Tuple[str, str, str, Options]: args[1])) options.pyversion = (int(version_components[0]), int(version_components[1])) args = args[2:] + elif args[0] == '-f' or args[0] == '--dirty-stubs': + options.dirty_stubs = True + args = args[2:] elif args[0] == '-m' and args[1:]: options.build_flags.append(build.MODULE) return None, args[1], None, options diff --git a/runtests.py b/runtests.py index 462fa1491369..d64925a6bb8d 100755 --- a/runtests.py +++ b/runtests.py @@ -26,6 +26,7 @@ def get_versions(): # type: () -> typing.List[str] from typing import Dict, List, Optional, Set from mypy.waiter import Waiter, LazySubprocess +from mypy import git import itertools import os @@ -293,6 +294,7 @@ def main() -> None: blacklist = [] # type: List[str] arglist = [] # type: List[str] list_only = False + dirty_stubs = False allow_opts = True curlist = whitelist @@ -312,6 +314,8 @@ def main() -> None: curlist = arglist elif a == '-l' or a == '--list': list_only = True + elif a == '-f' or a == '--dirty-stubs': + dirty_stubs = True elif a == '-h' or a == '--help': usage(0) else: @@ -329,6 +333,10 @@ def main() -> None: driver = Driver(whitelist=whitelist, blacklist=blacklist, arglist=arglist, verbosity=verbosity, xfail=[]) + + if not dirty_stubs: + git.verify_git_integrity_or_abort(driver.cwd) + driver.prepend_path('PATH', [join(driver.cwd, 'scripts')]) driver.prepend_path('MYPYPATH', [driver.cwd]) driver.prepend_path('PYTHONPATH', [driver.cwd]) diff --git a/setup.py b/setup.py index 0a0caa15075a..d7d1bc374519 100644 --- a/setup.py +++ b/setup.py @@ -7,11 +7,14 @@ from distutils.core import setup from mypy.version import __version__ +from mypy import git if sys.version_info < (3, 2, 0): sys.stderr.write("ERROR: You need Python 3.2 or later to use mypy.\n") exit(1) +git.verify_git_integrity_or_abort(".") + version = __version__ description = 'Optional static typing for Python' long_description = ''' From 7bb878b6e6a4d920fedb0881c4e61b88e6adbbcf Mon Sep 17 00:00:00 2001 From: Matthias Kramm Date: Tue, 13 Oct 2015 13:20:23 -0700 Subject: [PATCH 2/7] Use 'cwd' parameter of subprocess functions --- mypy/git.py | 28 ++++++++++++++++------------ mypy/main.py | 2 +- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/mypy/git.py b/mypy/git.py index 94d7d1204cff..2a27e36a4766 100644 --- a/mypy/git.py +++ b/mypy/git.py @@ -21,37 +21,33 @@ def have_git() -> bool: return False -def run_in_dir(dir, command) -> bytes: - """Convenience function: Run a command in a directory.""" - return subprocess.check_output("cd " + pipes.quote(dir) + "; " + command, - shell=True) - - def get_submodules(dir: str): """Return a list of all git top-level submodules in a given directory.""" # It would be nicer to do # "git submodule foreach 'echo MODULE $name $path $sha1 $toplevel'" # but that wouldn't work on Windows. - output = run_in_dir(dir, "git submodule status") + output = subprocess.check_output(["git", "submodule", "status"], cwd=dir) for line in output.splitlines(): status = line[0] - sha5, name, *_ = line[1:].split(b" ") - yield (status, sha5, name.decode(sys.getfilesystemencoding())) + revision, name, *_ = line[1:].split(b" ") + yield (status, revision, name.decode(sys.getfilesystemencoding())) def git_revision(dir: str) -> bytes: """Get the SHA-1 of the HEAD of a git repository.""" - return run_in_dir(dir, "git rev-parse HEAD").strip() + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=dir).strip() def is_dirty(dir: str) -> bool: """Check whether a git repository has uncommitted changes.""" - return run_in_dir(dir, "git status -uno --porcelain").strip() != b"" + output = subprocess.check_output(["git", "status", "-uno", "--porcelain"], cwd=dir) + return output.strip() != b"" def has_extra_files(dir: str) -> bool: """Check whether a git repository has untracked files.""" - return run_in_dir(dir, "git clean --dry-run -d").strip() != b"" + output = subprocess.check_output(["git", "clean", "--dry-run", "-d"], cwd=dir) + return output.strip() != b"" def warn_no_git_executable() -> None: @@ -62,11 +58,17 @@ def warn_no_git_executable() -> None: def warn_dirty(dir) -> None: print("Warning: git module '{}' has uncommitted changes.".format(dir), file=sys.stderr) + print("Got to the directory", file=sys.stderr) + print(" {}".format(dir), file=sys.stderr) + print("and commit or reset your changes", file=sys.stderr) def warn_extra_files(dir) -> None: print("Warning: git module '{}' has untracked files.".format(dir), file=sys.stderr) + print("Got to the directory", file=sys.stderr) + print(" {}".format(dir), file=sys.stderr) + print("and add & commit your new files.", file=sys.stderr) def error_submodule_not_initialized(name: str, dir: str) -> None: @@ -81,6 +83,8 @@ def error_submodule_not_updated(name: str, dir: str) -> None: print("Please run:", file=sys.stderr) print(" cd {}".format(pipes.quote(dir)), file=sys.stderr) print(" git submodule update {}".format(name), file=sys.stderr) + print("(If you got this message because you updated {}".format(name), file=sys.stderr) + print(" then run \"git add {}\" to silence this check)") def verify_git_integrity_or_abort(datadir: str) -> None: diff --git a/mypy/main.py b/mypy/main.py index 2f50e52cb962..ef744a934f04 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -123,7 +123,7 @@ def process_options(args: List[str]) -> Tuple[str, str, str, Options]: args = args[2:] elif args[0] == '-f' or args[0] == '--dirty-stubs': options.dirty_stubs = True - args = args[2:] + args = args[1:] elif args[0] == '-m' and args[1:]: options.build_flags.append(build.MODULE) return None, args[1], None, options From 1370d93d046e612da01a0d252463aed5ce7964db Mon Sep 17 00:00:00 2001 From: Matthias Kramm Date: Tue, 13 Oct 2015 13:47:24 -0700 Subject: [PATCH 3/7] Fix bug in submodule revision check. 'git submodule status' returns the current revision, not the revision in the superproject's index. Do an explicit retrieval of the latter using 'git ls-files'. --- mypy/git.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/mypy/git.py b/mypy/git.py index 2a27e36a4766..e6207c9268f4 100644 --- a/mypy/git.py +++ b/mypy/git.py @@ -28,9 +28,9 @@ def get_submodules(dir: str): # but that wouldn't work on Windows. output = subprocess.check_output(["git", "submodule", "status"], cwd=dir) for line in output.splitlines(): - status = line[0] - revision, name, *_ = line[1:].split(b" ") - yield (status, revision, name.decode(sys.getfilesystemencoding())) + unused_status = line[0] + unused_revision, name, *_ = line[1:].split(b" ") + yield name.decode(sys.getfilesystemencoding()) def git_revision(dir: str) -> bytes: @@ -38,6 +38,12 @@ def git_revision(dir: str) -> bytes: return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=dir).strip() +def submodule_revision(dir: str, submodule: str) -> bytes: + """Get the SHA-1 a submodule is supposed to have.""" + output = subprocess.check_output(["git", "ls-files", "-s", submodule], cwd=dir).strip() + # E.g.: "160000 e4a7edb949e0b920b16f61aeeb19fc3d328f3012 0 typeshed" + return output.split()[1] + def is_dirty(dir: str) -> bool: """Check whether a git repository has uncommitted changes.""" output = subprocess.check_output(["git", "status", "-uno", "--porcelain"], cwd=dir) @@ -84,7 +90,7 @@ def error_submodule_not_updated(name: str, dir: str) -> None: print(" cd {}".format(pipes.quote(dir)), file=sys.stderr) print(" git submodule update {}".format(name), file=sys.stderr) print("(If you got this message because you updated {}".format(name), file=sys.stderr) - print(" then run \"git add {}\" to silence this check)") + print(" then run \"git add {}\" to silence this check)".format(name), file=sys.stderr) def verify_git_integrity_or_abort(datadir: str) -> None: @@ -99,12 +105,12 @@ def verify_git_integrity_or_abort(datadir: str) -> None: if not have_git(): warn_no_git_executable() return - for _, revision, submodule in get_submodules(datadir): + for submodule in get_submodules(datadir): submodule_path = os.path.join(datadir, submodule) if not is_git_repo(submodule_path): error_submodule_not_initialized(submodule, datadir) sys.exit(1) - elif revision != git_revision(submodule_path): + elif submodule_revision(datadir, submodule) != git_revision(submodule_path): error_submodule_not_updated(submodule, datadir) sys.exit(1) elif is_dirty(submodule_path): From ede09690c17ff912b79909d756f34366fbc6c32c Mon Sep 17 00:00:00 2001 From: Matthias Kramm Date: Tue, 13 Oct 2015 16:56:26 -0700 Subject: [PATCH 4/7] fix wording/spelling and linter errors --- mypy/git.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mypy/git.py b/mypy/git.py index e6207c9268f4..704c7e402db3 100644 --- a/mypy/git.py +++ b/mypy/git.py @@ -8,14 +8,14 @@ def is_git_repo(dir: str) -> bool: - """Is the passed directory version controlled with git?""" + """Is the given directory version-controlled with git?""" return os.path.exists(os.path.join(dir, ".git")) def have_git() -> bool: """Can we run the git executable?""" try: - subprocess.check_output(["git", "config", "-l"]) + subprocess.check_output(["git", "--help"]) return True except subprocess.CalledProcessError: return False @@ -27,9 +27,12 @@ def get_submodules(dir: str): # "git submodule foreach 'echo MODULE $name $path $sha1 $toplevel'" # but that wouldn't work on Windows. output = subprocess.check_output(["git", "submodule", "status"], cwd=dir) + # " name desc" + # status='-': not initialized + # status='+': changed + # status='u': merge conflicts for line in output.splitlines(): - unused_status = line[0] - unused_revision, name, *_ = line[1:].split(b" ") + name = line.split(b" ")[1] yield name.decode(sys.getfilesystemencoding()) @@ -44,6 +47,7 @@ def submodule_revision(dir: str, submodule: str) -> bytes: # E.g.: "160000 e4a7edb949e0b920b16f61aeeb19fc3d328f3012 0 typeshed" return output.split()[1] + def is_dirty(dir: str) -> bool: """Check whether a git repository has uncommitted changes.""" output = subprocess.check_output(["git", "status", "-uno", "--porcelain"], cwd=dir) @@ -64,7 +68,7 @@ def warn_no_git_executable() -> None: def warn_dirty(dir) -> None: print("Warning: git module '{}' has uncommitted changes.".format(dir), file=sys.stderr) - print("Got to the directory", file=sys.stderr) + print("Go to the directory", file=sys.stderr) print(" {}".format(dir), file=sys.stderr) print("and commit or reset your changes", file=sys.stderr) @@ -72,7 +76,7 @@ def warn_dirty(dir) -> None: def warn_extra_files(dir) -> None: print("Warning: git module '{}' has untracked files.".format(dir), file=sys.stderr) - print("Got to the directory", file=sys.stderr) + print("Go to the directory", file=sys.stderr) print(" {}".format(dir), file=sys.stderr) print("and add & commit your new files.", file=sys.stderr) From 0456a922abc334027cfa1e582da0db1926763f79 Mon Sep 17 00:00:00 2001 From: Matthias Kramm Date: Wed, 14 Oct 2015 06:02:30 -0700 Subject: [PATCH 5/7] fix typeshed directory in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d7d1bc374519..853aad57ac78 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ def find_data_files(base, globs): data_files = [] -data_files += find_data_files('stubs', ['*.py', '*.pyi']) +data_files += find_data_files('typeshed', ['*.py', '*.pyi']) data_files += find_data_files('xml', ['*.xsd', '*.xslt', '*.css']) From 373dda3446f39a9278abba9f653805cdd076f0ad Mon Sep 17 00:00:00 2001 From: Matthias Kramm Date: Wed, 14 Oct 2015 06:12:20 -0700 Subject: [PATCH 6/7] Revert "fix typeshed directory in setup.py" This reverts commit 0456a922abc334027cfa1e582da0db1926763f79. (I did this in the wrong branch) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 853aad57ac78..d7d1bc374519 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ def find_data_files(base, globs): data_files = [] -data_files += find_data_files('typeshed', ['*.py', '*.pyi']) +data_files += find_data_files('stubs', ['*.py', '*.pyi']) data_files += find_data_files('xml', ['*.xsd', '*.xslt', '*.css']) From 5ad7a450ca078d6c5b423500f22dac2c5915603c Mon Sep 17 00:00:00 2001 From: Matthias Kramm Date: Wed, 14 Oct 2015 06:24:34 -0700 Subject: [PATCH 7/7] Omit "cd $dir" message if $dir is "." --- mypy/git.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mypy/git.py b/mypy/git.py index 704c7e402db3..e1233af629cb 100644 --- a/mypy/git.py +++ b/mypy/git.py @@ -81,19 +81,27 @@ def warn_extra_files(dir) -> None: print("and add & commit your new files.", file=sys.stderr) +def chdir_prefix(dir) -> str: + """Return the command to change to the target directory, plus '&&'.""" + if os.path.relpath(dir) != ".": + return "cd " + pipes.quote(dir) + " && " + else: + return "" + + def error_submodule_not_initialized(name: str, dir: str) -> None: print("Submodule '{}' not initialized.".format(name), file=sys.stderr) print("Please run:", file=sys.stderr) - print(" cd {}".format(pipes.quote(dir)), file=sys.stderr) - print(" git submodule init {}".format(name), file=sys.stderr) + print(" {}git submodule update --init {}".format( + chdir_prefix(dir), name), file=sys.stderr) def error_submodule_not_updated(name: str, dir: str) -> None: print("Submodule '{}' not updated.".format(name), file=sys.stderr) print("Please run:", file=sys.stderr) - print(" cd {}".format(pipes.quote(dir)), file=sys.stderr) - print(" git submodule update {}".format(name), file=sys.stderr) - print("(If you got this message because you updated {}".format(name), file=sys.stderr) + print(" {}git submodule update {}".format( + chdir_prefix(dir), name), file=sys.stderr) + print("(If you got this message because you updated {} yourself".format(name), file=sys.stderr) print(" then run \"git add {}\" to silence this check)".format(name), file=sys.stderr)