add git (submodule) integrity check

mypy/git.py

@@ -0,0 +1,131 @@
+"""Utilities for verifying git integrity."""
+
+# Used also from setup.py, so don't pull in anything additional here (like mypy or typing):
+import os
+import pipes
+import subprocess
+import sys
+
+
+def is_git_repo(dir: str) -> bool:
+    """Is the given directory version-controlled with git?"""
+    return os.path.exists(os.path.join(dir, ".git"))
+
+
+def have_git() -> bool:
+    """Can we run the git executable?"""
+    try:
+        subprocess.check_output(["git", "--help"])
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+
+def get_submodules(dir: str):
+    """Return a list of all git top-level submodules in a given directory."""
+    # It would be nicer to do
+    # "git submodule foreach 'echo MODULE $name $path $sha1 $toplevel'"
+    # but that wouldn't work on Windows.
+    output = subprocess.check_output(["git", "submodule", "status"], cwd=dir)
+    # "<status><sha1> name desc"
+    # status='-': not initialized
+    # status='+': changed
+    # status='u': merge conflicts
+    for line in output.splitlines():
+        name = line.split(b" ")[1]
+        yield name.decode(sys.getfilesystemencoding())
+
+
+def git_revision(dir: str) -> bytes:
+    """Get the SHA-1 of the HEAD of a git repository."""
+    return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=dir).strip()
+
+
+def submodule_revision(dir: str, submodule: str) -> bytes:
+    """Get the SHA-1 a submodule is supposed to have."""
+    output = subprocess.check_output(["git", "ls-files", "-s", submodule], cwd=dir).strip()
+    # E.g.: "160000 e4a7edb949e0b920b16f61aeeb19fc3d328f3012 0       typeshed"
+    return output.split()[1]
+
+
+def is_dirty(dir: str) -> bool:
+    """Check whether a git repository has uncommitted changes."""
+    output = subprocess.check_output(["git", "status", "-uno", "--porcelain"], cwd=dir)
+    return output.strip() != b""
+
+
+def has_extra_files(dir: str) -> bool:
+    """Check whether a git repository has untracked files."""
+    output = subprocess.check_output(["git", "clean", "--dry-run", "-d"], cwd=dir)
+    return output.strip() != b""
+
+
+def warn_no_git_executable() -> None:
+    print("Warning: Couldn't check git integrity. "
+          "git executable not in path.", file=sys.stderr)
+
+
+def warn_dirty(dir) -> None:
+    print("Warning: git module '{}' has uncommitted changes.".format(dir),
+          file=sys.stderr)
+    print("Go to the directory", file=sys.stderr)
+    print("  {}".format(dir), file=sys.stderr)
+    print("and commit or reset your changes", file=sys.stderr)
+
+
+def warn_extra_files(dir) -> None:
+    print("Warning: git module '{}' has untracked files.".format(dir),
+          file=sys.stderr)
+    print("Go to the directory", file=sys.stderr)
+    print("  {}".format(dir), file=sys.stderr)
+    print("and add & commit your new files.", file=sys.stderr)
+
+
+def chdir_prefix(dir) -> str:
+    """Return the command to change to the target directory, plus '&&'."""
+    if os.path.relpath(dir) != ".":
+        return "cd " + pipes.quote(dir) + " && "
+    else:
+        return ""
+
+
+def error_submodule_not_initialized(name: str, dir: str) -> None:
+    print("Submodule '{}' not initialized.".format(name), file=sys.stderr)
+    print("Please run:", file=sys.stderr)
+    print("  {}git submodule update --init {}".format(
+        chdir_prefix(dir), name), file=sys.stderr)
+
+
+def error_submodule_not_updated(name: str, dir: str) -> None:
+    print("Submodule '{}' not updated.".format(name), file=sys.stderr)
+    print("Please run:", file=sys.stderr)
+    print("  {}git submodule update {}".format(
+        chdir_prefix(dir), name), file=sys.stderr)
+    print("(If you got this message because you updated {} yourself".format(name), file=sys.stderr)
+    print(" then run \"git add {}\" to silence this check)".format(name), file=sys.stderr)
+
+
+def verify_git_integrity_or_abort(datadir: str) -> None:
+    """Verify the (submodule) integrity of a git repository.
+
+    Potentially output warnings/errors (to stderr), and exit with status 1
+    if we detected a severe problem.
+    """
+
+    if not is_git_repo(datadir):
+        return
+    if not have_git():
+        warn_no_git_executable()
+        return
+    for submodule in get_submodules(datadir):
+        submodule_path = os.path.join(datadir, submodule)
+        if not is_git_repo(submodule_path):
+            error_submodule_not_initialized(submodule, datadir)
+            sys.exit(1)
+        elif submodule_revision(datadir, submodule) != git_revision(submodule_path):
+            error_submodule_not_updated(submodule, datadir)
+            sys.exit(1)
+        elif is_dirty(submodule_path):
+            warn_dirty(submodule)
+        elif has_extra_files(submodule_path):
+            warn_extra_files(submodule)

mypy/main.py

@@ -12,6 +12,7 @@
 
 from mypy import build
 from mypy import defaults
+from mypy import git
 from mypy.errors import CompileError
 
 from mypy.version import __version__
@@ -26,6 +27,7 @@ def __init__(self) -> None:
         self.custom_typing_module = None  # type: str
         self.report_dirs = {}  # type: Dict[str, str]
         self.python_path = False
+        self.dirty_stubs = False
 
 
 def main(script_path: str) -> None:
@@ -39,6 +41,8 @@ def main(script_path: str) -> None:
     else:
         bin_dir = None
     path, module, program_text, options = process_options(sys.argv[1:])
+    if not options.dirty_stubs:
+        git.verify_git_integrity_or_abort(build.default_data_dir(bin_dir))
     try:
         if options.target == build.TYPE_CHECK:
             type_check_only(path, module, program_text, bin_dir, options)
@@ -117,6 +121,9 @@ def process_options(args: List[str]) -> Tuple[str, str, str, Options]:
                     args[1]))
             options.pyversion = (int(version_components[0]), int(version_components[1]))
             args = args[2:]
+        elif args[0] == '-f' or args[0] == '--dirty-stubs':
+            options.dirty_stubs = True
+            args = args[1:]
         elif args[0] == '-m' and args[1:]:
             options.build_flags.append(build.MODULE)
             return None, args[1], None, options

runtests.py

@@ -26,6 +26,7 @@ def get_versions():  # type: () -> typing.List[str]
 from typing import Dict, List, Optional, Set
 
 from mypy.waiter import Waiter, LazySubprocess
+from mypy import git
 
 import itertools
 import os
@@ -293,6 +294,7 @@ def main() -> None:
     blacklist = []  # type: List[str]
     arglist = []  # type: List[str]
     list_only = False
+    dirty_stubs = False
 
     allow_opts = True
     curlist = whitelist
@@ -312,6 +314,8 @@ def main() -> None:
                 curlist = arglist
             elif a == '-l' or a == '--list':
                 list_only = True
+            elif a == '-f' or a == '--dirty-stubs':
+                dirty_stubs = True
             elif a == '-h' or a == '--help':
                 usage(0)
             else:
@@ -329,6 +333,10 @@ def main() -> None:
 
     driver = Driver(whitelist=whitelist, blacklist=blacklist, arglist=arglist,
             verbosity=verbosity, xfail=[])
+
+    if not dirty_stubs:
+        git.verify_git_integrity_or_abort(driver.cwd)
+
     driver.prepend_path('PATH', [join(driver.cwd, 'scripts')])
     driver.prepend_path('MYPYPATH', [driver.cwd])
     driver.prepend_path('PYTHONPATH', [driver.cwd])

setup.py

@@ -7,11 +7,14 @@
 
 from distutils.core import setup
 from mypy.version import __version__
+from mypy import git
 
 if sys.version_info < (3, 2, 0):
     sys.stderr.write("ERROR: You need Python 3.2 or later to use mypy.\n")
     exit(1)
 
+git.verify_git_integrity_or_abort(".")
+
 version = __version__
 description = 'Optional static typing for Python'
 long_description = '''