From c7bd732238e8b5c9c38758cdfcc87130fb5d696f Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 28 Jul 2016 14:16:49 -0700 Subject: [PATCH 1/4] Make modify invalidate cache if version id changes This commit modifies build.py to add an extra "version_id" attribute to the cache metadata. If the version id specified in mypy/version.py differs from the id stored in the cached data, mypy will automatically invalidate that cached data. --- mypy/build.py | 17 +++++++++++++---- mypy/test/testgraph.py | 4 +++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 49aba4616e57..1b9e792525bf 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -137,7 +137,7 @@ def build(sources: List[BuildSource], # Use stub builtins (to speed up test cases and to make them easier to # debug). This is a test-only feature, so assume our files are laid out # as in the source tree. - root_dir = os.path.dirname(os.path.dirname(__file__)) + root_dir = dirname(dirname(__file__)) lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) else: for source in sources: @@ -172,7 +172,8 @@ def build(sources: List[BuildSource], ignore_prefix=os.getcwd(), source_set=source_set, reports=reports, - options=options) + options=options, + version_id=__version__) try: dispatch(sources, manager) @@ -285,6 +286,7 @@ def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]: ('child_modules', List[str]), # all submodules of the given module ('options', Optional[Dict[str, bool]]), # build options ('dep_prios', List[int]), + ('version_id', str), # mypy version for cache invalidation ]) # NOTE: dependencies + suppressed == all reachable imports; # suppressed contains those reachable imports that were prevented by @@ -320,6 +322,7 @@ class BuildManager: options: Build options missing_modules: Set of modules that could not be imported encountered so far stale_modules: Set of modules that needed to be rechecked + version_id: The current mypy version (based on commit id when possible) """ def __init__(self, data_dir: str, @@ -327,7 +330,8 @@ def __init__(self, data_dir: str, ignore_prefix: str, source_set: BuildSourceSet, reports: Reports, - options: Options) -> None: + options: Options, + version_id: str) -> None: self.start_time = time.time() self.data_dir = data_dir self.errors = Errors(options.suppress_error_context) @@ -336,6 +340,7 @@ def __init__(self, data_dir: str, self.source_set = source_set self.reports = reports self.options = options + self.version_id = version_id self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors, options=options) self.modules = self.semantic_analyzer.modules self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) @@ -717,6 +722,7 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache meta.get('child_modules', []), meta.get('options'), meta.get('dep_prios', []), + meta.get('version_id') ) if (m.id != id or m.path != path or m.mtime is None or m.size is None or @@ -724,7 +730,9 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache return None # Ignore cache if generated by an older mypy version. - if m.options is None or len(m.dependencies) != len(m.dep_prios): + if (m.version_id != manager.version_id + or m.options is None + or len(m.dependencies) != len(m.dep_prios)): return None # Ignore cache if (relevant) options aren't the same. @@ -810,6 +818,7 @@ def write_cache(id: str, path: str, tree: MypyFile, 'child_modules': child_modules, 'options': select_options_affecting_cache(manager.options), 'dep_prios': dep_prios, + 'version_id': manager.version_id, } with open(meta_json_tmp, 'w') as f: json.dump(meta, f, sort_keys=True) diff --git a/mypy/test/testgraph.py b/mypy/test/testgraph.py index 8ac099490220..efbec771cb7d 100644 --- a/mypy/test/testgraph.py +++ b/mypy/test/testgraph.py @@ -5,6 +5,7 @@ from mypy.myunit import Suite, assert_equal from mypy.build import BuildManager, State from mypy.build import topsort, strongly_connected_components, sorted_components, order_ascc +from mypy.version import __version__ from mypy.options import Options @@ -38,7 +39,8 @@ def _make_manager(self): ignore_prefix='', source_set=None, reports=None, - options=Options()) + options=Options(), + version_id=__version__) return manager def test_sorted_components(self) -> None: From 5e391ed49fa630f4e32f31f3ff4f2c5620a3cc93 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 28 Jul 2016 14:18:26 -0700 Subject: [PATCH 2/4] Include git hash in version.py during setup This commit modifies setup.py to automatically edit the installed mypy/version.py file during installation to include the git hash if applicable. If setup.py is not being run from a git repo, or if a git executable can't be found in the system, mypy will default to using the original version id. --- setup.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/setup.py b/setup.py index 10da82951979..f344ccec0e2d 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ exit(1) from distutils.core import setup +from distutils.command.build_py import build_py from mypy.version import __version__ from mypy import git @@ -30,6 +31,19 @@ '''.lstrip() +def cache_version_id(): + """Returns the version id to use for the incremental hash. + + If setup.py is run from a git repo, the git commit hash will be + included if possible. If not, then this function will fall back to + using the default version id from mypy/version.py.""" + if git.is_git_repo('.') and git.have_git(): + return __version__ + '-' + git.git_revision('.').decode('utf-8') + else: + # Default fallback + return __version__ + + def find_data_files(base, globs): """Find all interesting data files, for setup(data_files=) @@ -51,6 +65,19 @@ def find_data_files(base, globs): return rv + +class CustomPythonBuild(build_py): + def pin_version(self): + path = os.path.join(self.build_lib, 'mypy') + self.mkpath(path) + with open(os.path.join(path, 'version.py'), 'w') as stream: + stream.write('__version__ = "{}"\n'.format(cache_version_id())) + + def run(self): + self.execute(self.pin_version, ()) + build_py.run(self) + + data_files = [] data_files += find_data_files('typeshed', ['*.py', '*.pyi']) @@ -93,4 +120,5 @@ def find_data_files(base, globs): scripts=scripts, data_files=data_files, classifiers=classifiers, + cmdclass={'build_py': CustomPythonBuild}, ) From addfb95810e123299175fbda0ca5c95a5faad129 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Fri, 29 Jul 2016 15:44:26 -0700 Subject: [PATCH 3/4] Add trailing commas --- mypy/build.py | 2 +- mypy/test/testgraph.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 1b9e792525bf..01ee1681ea55 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -722,7 +722,7 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache meta.get('child_modules', []), meta.get('options'), meta.get('dep_prios', []), - meta.get('version_id') + meta.get('version_id'), ) if (m.id != id or m.path != path or m.mtime is None or m.size is None or diff --git a/mypy/test/testgraph.py b/mypy/test/testgraph.py index efbec771cb7d..00a8b85b3995 100644 --- a/mypy/test/testgraph.py +++ b/mypy/test/testgraph.py @@ -40,7 +40,8 @@ def _make_manager(self): source_set=None, reports=None, options=Options(), - version_id=__version__) + version_id=__version__, + ) return manager def test_sorted_components(self) -> None: From d75f9622ddd4c87207d4d50b8b9118fe1b67cbeb Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Fri, 29 Jul 2016 15:49:34 -0700 Subject: [PATCH 4/4] Another comma --- mypy/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index 01ee1681ea55..8ca0e5f4db4f 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -173,7 +173,8 @@ def build(sources: List[BuildSource], source_set=source_set, reports=reports, options=options, - version_id=__version__) + version_id=__version__, + ) try: dispatch(sources, manager)