From 96041b16ff2d3ca56c0ed5f68d8fb91c1be67104 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 19:25:34 -0800 Subject: [PATCH 01/47] Add docs for python-executable --- docs/source/command_line.rst | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index a72390879911..01b6036f6993 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -360,11 +360,27 @@ Here are some more useful flags: updates the cache, but regular incremental mode ignores cache files written by quick mode. +- ``--python-executable EXECUTABLE`` will have mypy collect type information + from `PEP 561`_ compliant packages installed for the Python executable + ``EXECUTABLE``. If not provided, mypy will use PEP 561 compliant packages + installed for the Python executable running mypy. See + :ref:`installed-packages` for more on making PEP 561 compliant packages. This + flag will attempt to set ``--python-version`` if not already set. + - ``--python-version X.Y`` will make mypy typecheck your code as if it were run under Python version X.Y. Without this option, mypy will default to using whatever version of Python is running mypy. Note that the ``-2`` and ``--py2`` flags are aliases for ``--python-version 2.7``. See - :ref:`version_and_platform_checks` for more about this feature. + :ref:`version_and_platform_checks` for more about this feature. This flag + will attempt to find a Python executable of the corresponding version to + search for `PEP 561`_ compliant packages. If you'd like to disable this, see + ``--no-site-packages`` below. + +- ``--no-site-packages`` will disable searching for `PEP 561`_ compliant + packages. This will also disable searching for a usable Python executable. + Use this flag if mypy cannot find a Python executable for the version of + Python being checked, and you don't need to use PEP 561 typed packages. + Otherwise, use ``--python-executable``. - ``--platform PLATFORM`` will make mypy typecheck your code as if it were run under the the given operating system. Without this option, mypy will @@ -447,6 +463,8 @@ For the remaining flags you can read the full ``mypy -h`` output. Command line flags are liable to change between releases. +.. _PEP 561: https://www.python.org/dev/peps/pep-0561/ + .. _integrating-mypy: Integrating mypy into another Python application From 4483b3eb3119bfa3f86d467c503804fe27dba2b1 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 22:53:15 -0800 Subject: [PATCH 02/47] Add --python-executable and --no-site-packages --- mypy/main.py | 79 ++++++++++++++++++++++++++++++++++++++++++-- mypy/options.py | 1 + mypy/test/helpers.py | 1 + 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index b7c1117ea029..512297b18fbe 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1,10 +1,12 @@ """Mypy type checker command line tool.""" import argparse +import ast import configparser import fnmatch import os import re +import subprocess import sys import time @@ -205,6 +207,45 @@ def invert_flag_name(flag: str) -> str: return '--no-{}'.format(flag[2:]) +class PythonExecutableInferenceError(Exception): + """Represents a failure to infer the version or executable while searching.""" + + +if sys.platform == 'win32': + def python_executable_prefix(v: str) -> List[str]: + return ['py', '-{}'.format(v)] +else: + def python_executable_prefix(v: str) -> List[str]: + return ['python{}'.format(v)] + + +def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: + try: + check = subprocess.check_output([python_executable, '-c', + 'import sys; print(repr(sys.version_info[:2]))'], + stderr=subprocess.STDOUT).decode() + return ast.literal_eval(check) + except (subprocess.CalledProcessError, FileNotFoundError): + raise PythonExecutableInferenceError( + 'Error: invalid Python executable {}'.format(python_executable)) + + +def _python_executable_from_version(python_version: Tuple[int, int]) -> str: + if sys.version_info[:2] == python_version: + return sys.executable + str_ver = '.'.join(map(str, python_version)) + print(str_ver) + try: + sys_exe = subprocess.check_output(python_executable_prefix(str_ver) + + ['-c', 'import sys; print(sys.executable)'], + stderr=subprocess.STDOUT).decode().strip() + return sys_exe + except (subprocess.CalledProcessError, FileNotFoundError): + raise PythonExecutableInferenceError( + 'Error: failed to find a Python executable matching version {},' + ' perhaps try --python-executable, or --no-site-packages?'.format(python_version)) + + def process_options(args: List[str], require_targets: bool = True, server_options: bool = False, @@ -255,10 +296,16 @@ def add_invertible_flag(flag: str, parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__) parser.add_argument('--python-version', type=parse_version, metavar='x.y', - help='use Python x.y') + help='use Python x.y', dest='special-opts:python_version') + parser.add_argument('--python-executable', action='store', metavar='EXECUTABLE', + help="Python executable whose installed packages will be" + " used in typechecking.", dest='special-opts:python_executable') + parser.add_argument('--no-site-packages', action='store_true', + dest='special-opts:no_site_packages', + help="Do not search for PEP 561 packages in the package directory.") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " - "(defaults to sys.platform).") + "(defaults to sys.platform).") parser.add_argument('-2', '--py2', dest='python_version', action='store_const', const=defaults.PYTHON2_VERSION, help="use Python 2 mode") parser.add_argument('--ignore-missing-imports', action='store_true', @@ -482,6 +529,34 @@ def add_invertible_flag(flag: str, print("Warning: --no-fast-parser no longer has any effect. The fast parser " "is now mypy's default and only parser.") + try: + # Infer Python version and/or executable if one is not given + if special_opts.python_executable is not None and special_opts.python_version is not None: + py_exe_ver = _python_version_from_executable(special_opts.python_executable) + if py_exe_ver != special_opts.python_version: + parser.error( + 'Python version {} did not match executable {}, got version {}.'.format( + special_opts.python_version, special_opts.python_executable, py_exe_ver + )) + else: + options.python_version = special_opts.python_version + options.python_executable = special_opts.python_executable + elif special_opts.python_executable is None and special_opts.python_version is not None: + options.python_version = special_opts.python_version + py_exe = None + if not special_opts.no_site_packages: + py_exe = _python_executable_from_version(special_opts.python_version) + options.python_executable = py_exe + elif special_opts.python_version is None and special_opts.python_executable is not None: + options.python_version = _python_version_from_executable( + special_opts.python_executable) + options.python_executable = special_opts.python_executable + except PythonExecutableInferenceError as e: + parser.error(str(e)) + + if special_opts.no_site_packages: + options.python_executable = None + # Check for invalid argument combinations. if require_targets: code_methods = sum(bool(c) for c in [special_opts.modules, diff --git a/mypy/options.py b/mypy/options.py index 5ea251df2c9d..c3d07df08191 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -53,6 +53,7 @@ def __init__(self) -> None: # -- build options -- self.build_type = BuildType.STANDARD self.python_version = sys.version_info[:2] # type: Tuple[int, int] + self.python_executable = sys.executable # type: Optional[str] self.platform = sys.platform self.custom_typing_module = None # type: Optional[str] self.custom_typeshed_dir = None # type: Optional[str] diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 267f99e5586b..ad17d8387ace 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -316,6 +316,7 @@ def parse_options(program_text: str, testcase: DataDrivenTestCase, flag_list = None if flags: flag_list = flags.group(1).split() + flag_list.append('--no-site-packages') # the tests shouldn't need an installed Python targets, options = process_options(flag_list, require_targets=False) if targets: # TODO: support specifying targets via the flags pragma From 53ff42c936fa47dd6db6cd200ce4a0eea022df8e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 23:09:28 -0800 Subject: [PATCH 03/47] Split inference out of process_options --- mypy/main.py | 49 ++++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 512297b18fbe..1b5863b226ee 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -246,6 +246,33 @@ def _python_executable_from_version(python_version: Tuple[int, int]) -> str: ' perhaps try --python-executable, or --no-site-packages?'.format(python_version)) +def infer_python_version_and_executable(options: Options, + special_opts: argparse.Namespace + ) -> Options: + # Infer Python version and/or executable if one is not given + if special_opts.python_executable is not None and special_opts.python_version is not None: + py_exe_ver = _python_version_from_executable(special_opts.python_executable) + if py_exe_ver != special_opts.python_version: + raise PythonExecutableInferenceError( + 'Python version {} did not match executable {}, got version {}.'.format( + special_opts.python_version, special_opts.python_executable, py_exe_ver + )) + else: + options.python_version = special_opts.python_version + options.python_executable = special_opts.python_executable + elif special_opts.python_executable is None and special_opts.python_version is not None: + options.python_version = special_opts.python_version + py_exe = None + if not special_opts.no_site_packages: + py_exe = _python_executable_from_version(special_opts.python_version) + options.python_executable = py_exe + elif special_opts.python_version is None and special_opts.python_executable is not None: + options.python_version = _python_version_from_executable( + special_opts.python_executable) + options.python_executable = special_opts.python_executable + return options + + def process_options(args: List[str], require_targets: bool = True, server_options: bool = False, @@ -530,27 +557,7 @@ def add_invertible_flag(flag: str, "is now mypy's default and only parser.") try: - # Infer Python version and/or executable if one is not given - if special_opts.python_executable is not None and special_opts.python_version is not None: - py_exe_ver = _python_version_from_executable(special_opts.python_executable) - if py_exe_ver != special_opts.python_version: - parser.error( - 'Python version {} did not match executable {}, got version {}.'.format( - special_opts.python_version, special_opts.python_executable, py_exe_ver - )) - else: - options.python_version = special_opts.python_version - options.python_executable = special_opts.python_executable - elif special_opts.python_executable is None and special_opts.python_version is not None: - options.python_version = special_opts.python_version - py_exe = None - if not special_opts.no_site_packages: - py_exe = _python_executable_from_version(special_opts.python_version) - options.python_executable = py_exe - elif special_opts.python_version is None and special_opts.python_executable is not None: - options.python_version = _python_version_from_executable( - special_opts.python_executable) - options.python_executable = special_opts.python_executable + options = infer_python_version_and_executable(options, special_opts) except PythonExecutableInferenceError as e: parser.error(str(e)) From f523efa7ab7b8146fad4d55787f066c1be8a8391 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 23:25:26 -0800 Subject: [PATCH 04/47] Change --no-site-packages to --no-infer-executable --- docs/source/command_line.rst | 20 +++++++------------- mypy/main.py | 14 +++++++------- mypy/test/helpers.py | 2 +- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 01b6036f6993..0dda5af9c637 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -360,27 +360,21 @@ Here are some more useful flags: updates the cache, but regular incremental mode ignores cache files written by quick mode. -- ``--python-executable EXECUTABLE`` will have mypy collect type information - from `PEP 561`_ compliant packages installed for the Python executable - ``EXECUTABLE``. If not provided, mypy will use PEP 561 compliant packages - installed for the Python executable running mypy. See - :ref:`installed-packages` for more on making PEP 561 compliant packages. This - flag will attempt to set ``--python-version`` if not already set. +- ``--python-executable EXECUTABLE`` This flag will attempt to set + ``--python-version`` if not already set based on the interpreter given. - ``--python-version X.Y`` will make mypy typecheck your code as if it were run under Python version X.Y. Without this option, mypy will default to using whatever version of Python is running mypy. Note that the ``-2`` and ``--py2`` flags are aliases for ``--python-version 2.7``. See :ref:`version_and_platform_checks` for more about this feature. This flag - will attempt to find a Python executable of the corresponding version to - search for `PEP 561`_ compliant packages. If you'd like to disable this, see - ``--no-site-packages`` below. + will attempt to find a Python executable of the corresponding version. If + you'd like to disable this, see ``--no-infer-executable`` below. -- ``--no-site-packages`` will disable searching for `PEP 561`_ compliant - packages. This will also disable searching for a usable Python executable. +- ``--no-infer-executable`` will disable searching for a usable Python + executable based on the Python version mypy is using to type check code. Use this flag if mypy cannot find a Python executable for the version of - Python being checked, and you don't need to use PEP 561 typed packages. - Otherwise, use ``--python-executable``. + Python being checked, and don't need mypy to use an executable. - ``--platform PLATFORM`` will make mypy typecheck your code as if it were run under the the given operating system. Without this option, mypy will diff --git a/mypy/main.py b/mypy/main.py index 1b5863b226ee..887dd106e8d5 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -263,7 +263,7 @@ def infer_python_version_and_executable(options: Options, elif special_opts.python_executable is None and special_opts.python_version is not None: options.python_version = special_opts.python_version py_exe = None - if not special_opts.no_site_packages: + if not special_opts.no_executable: py_exe = _python_executable_from_version(special_opts.python_version) options.python_executable = py_exe elif special_opts.python_version is None and special_opts.python_executable is not None: @@ -325,11 +325,11 @@ def add_invertible_flag(flag: str, parser.add_argument('--python-version', type=parse_version, metavar='x.y', help='use Python x.y', dest='special-opts:python_version') parser.add_argument('--python-executable', action='store', metavar='EXECUTABLE', - help="Python executable whose installed packages will be" - " used in typechecking.", dest='special-opts:python_executable') - parser.add_argument('--no-site-packages', action='store_true', - dest='special-opts:no_site_packages', - help="Do not search for PEP 561 packages in the package directory.") + help="Python executable which will be used in typechecking.", + dest='special-opts:python_executable') + parser.add_argument('--no-infer-executable', action='store_true', + dest='special-opts:no_executable', + help="Do not infer a Python executable based on the version.") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " "(defaults to sys.platform).") @@ -561,7 +561,7 @@ def add_invertible_flag(flag: str, except PythonExecutableInferenceError as e: parser.error(str(e)) - if special_opts.no_site_packages: + if special_opts.no_executable: options.python_executable = None # Check for invalid argument combinations. diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index ad17d8387ace..adce623d8af0 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -316,7 +316,7 @@ def parse_options(program_text: str, testcase: DataDrivenTestCase, flag_list = None if flags: flag_list = flags.group(1).split() - flag_list.append('--no-site-packages') # the tests shouldn't need an installed Python + flag_list.append('--no-infer-executable') # the tests shouldn't need an installed Python targets, options = process_options(flag_list, require_targets=False) if targets: # TODO: support specifying targets via the flags pragma From 4d668272d918251c543e96ebca9747fbf43073b4 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 23:43:53 -0800 Subject: [PATCH 05/47] Add --no-infer-executable to subprocessed tests --- mypy/main.py | 2 +- mypy/test/testcmdline.py | 1 + mypy/test/testpythoneval.py | 2 +- runtests.py | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 887dd106e8d5..4f45aba8e3a9 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -243,7 +243,7 @@ def _python_executable_from_version(python_version: Tuple[int, int]) -> str: except (subprocess.CalledProcessError, FileNotFoundError): raise PythonExecutableInferenceError( 'Error: failed to find a Python executable matching version {},' - ' perhaps try --python-executable, or --no-site-packages?'.format(python_version)) + ' perhaps try --python-executable, or --no-infer-executable?'.format(python_version)) def infer_python_version_and_executable(options: Options, diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py index 57910c1a1dc0..7a3016991cce 100644 --- a/mypy/test/testcmdline.py +++ b/mypy/test/testcmdline.py @@ -47,6 +47,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None: file.write('{}\n'.format(s)) args = parse_args(testcase.input[0]) args.append('--show-traceback') + args.append('--no-infer-executable') # Type check the program. fixed = [python3_path, os.path.join(testcase.old_cwd, 'scripts', 'mypy')] diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index 0634442f172a..871753b833de 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -49,7 +49,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None: version. """ assert testcase.old_cwd is not None, "test was not properly set up" - mypy_cmdline = ['--show-traceback'] + mypy_cmdline = ['--show-traceback', '--no-infer-executable'] py2 = testcase.name.lower().endswith('python2') if py2: mypy_cmdline.append('--py2') diff --git a/runtests.py b/runtests.py index a2a24c29a7ca..0714cf88cabf 100755 --- a/runtests.py +++ b/runtests.py @@ -73,6 +73,7 @@ def add_mypy_cmd(self, name: str, mypy_args: List[str], cwd: Optional[str] = Non return args = [sys.executable, self.mypy] + mypy_args args.append('--show-traceback') + args.append('--no-infer-executable') self.waiter.add(LazySubprocess(full_name, args, cwd=cwd, env=self.env)) def add_mypy(self, name: str, *args: str, cwd: Optional[str] = None) -> None: From 1582692c54fb303e4580097cb0f50197e865d295 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 7 Mar 2018 00:24:18 -0800 Subject: [PATCH 06/47] Add test for python-executable and no-infer-executable --- mypy/test/testargs.py | 51 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index 20db610cda18..337b591fb0dd 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -4,10 +4,15 @@ defaults, and that argparse doesn't assign any new members to the Options object it creates. """ +import argparse +import sys + +import pytest # type: ignore from mypy.test.helpers import Suite, assert_equal from mypy.options import Options -from mypy.main import process_options +from mypy.main import (process_options, PythonExecutableInferenceError, + infer_python_version_and_executable) class ArgSuite(Suite): @@ -17,3 +22,47 @@ def test_coherence(self) -> None: # FIX: test this too. Requires changing working dir to avoid finding 'setup.cfg' options.config_file = parsed_options.config_file assert_equal(options, parsed_options) + + def test_executable_inference(self) -> None: + """Test the --python-executable flag with --python-version""" + sys_ver_str = '.'.join(map(str, sys.version_info[:2])) + + base = ['file.py'] # dummy file + + # test inference given one (infer the other) + matching_version = base + ['--python-version={}'.format(sys_ver_str)] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + matching_version = base + ['--python-executable={}'.format(sys.executable)] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + # test inference given both + matching_version = base + ['--python-version={}'.format(sys_ver_str), + '--python-executable={}'.format(sys.executable)] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + # test that we error if the version mismatch + # argparse sys.exits on a parser.error, we need to check the raw inference function + options = Options() + + special_opts = argparse.Namespace() + special_opts.python_executable = sys.executable + special_opts.python_version = (2, 10) # obviously wrong + special_opts.no_executable = None + with pytest.raises(PythonExecutableInferenceError) as e: + options = infer_python_version_and_executable(options, special_opts) + assert str(e.value) == 'Python version (2, 10) did not match executable {}, got' \ + ' version {}.'.format(sys.executable, str(sys.version_info[:2])) + + # test that --no-infer-executable will disable executable inference + matching_version = base + ['--python-version={}'.format(sys_ver_str), + '--no-infer-executable'] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable is None From 65ebca9b37fafab8566ab412fd457ff30e82b360 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 7 Mar 2018 02:40:43 -0800 Subject: [PATCH 07/47] Change no-infer-executable back to no-site-packages --- docs/source/command_line.rst | 24 +++++++++++++++--------- mypy/main.py | 6 +++--- mypy/test/helpers.py | 2 +- mypy/test/testargs.py | 4 ++-- mypy/test/testcmdline.py | 2 +- mypy/test/testpythoneval.py | 2 +- runtests.py | 2 +- 7 files changed, 24 insertions(+), 18 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 0dda5af9c637..29df9dfb19d0 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -360,21 +360,27 @@ Here are some more useful flags: updates the cache, but regular incremental mode ignores cache files written by quick mode. -- ``--python-executable EXECUTABLE`` This flag will attempt to set - ``--python-version`` if not already set based on the interpreter given. +- ``--python-executable EXECUTABLE`` will have mypy collect type information + from `PEP 561`_ compliant packages installed for the Python executable + ``EXECUTABLE``. If not provided, mypy will use PEP 561 compliant packages + installed for the Python executable running mypy. See + :ref:`installed-packages` for more on making PEP 561 compliant packages. + This flag will attempt to set ``--python-version`` if not already set. - ``--python-version X.Y`` will make mypy typecheck your code as if it were run under Python version X.Y. Without this option, mypy will default to using whatever version of Python is running mypy. Note that the ``-2`` and ``--py2`` flags are aliases for ``--python-version 2.7``. See :ref:`version_and_platform_checks` for more about this feature. This flag - will attempt to find a Python executable of the corresponding version. If - you'd like to disable this, see ``--no-infer-executable`` below. - -- ``--no-infer-executable`` will disable searching for a usable Python - executable based on the Python version mypy is using to type check code. - Use this flag if mypy cannot find a Python executable for the version of - Python being checked, and don't need mypy to use an executable. + will attempt to find a Python executable of the corresponding version to + search for `PEP 561`_ compliant packages. If you'd like to disable this, + see ``--no-site-packages`` below. + +- ``--no-site-packages`` will disable searching for `PEP 561`_ compliant + packages. This will also disable searching for a usable Python executable. + Use this flag if mypy cannot find a Python executable for the version of + Python being checked, and you don't need to use PEP 561 typed packages. + Otherwise, use ``--python-executable``. - ``--platform PLATFORM`` will make mypy typecheck your code as if it were run under the the given operating system. Without this option, mypy will diff --git a/mypy/main.py b/mypy/main.py index 4f45aba8e3a9..d34ca0e01cbe 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -243,7 +243,7 @@ def _python_executable_from_version(python_version: Tuple[int, int]) -> str: except (subprocess.CalledProcessError, FileNotFoundError): raise PythonExecutableInferenceError( 'Error: failed to find a Python executable matching version {},' - ' perhaps try --python-executable, or --no-infer-executable?'.format(python_version)) + ' perhaps try --python-executable, or --no-site-packages?'.format(python_version)) def infer_python_version_and_executable(options: Options, @@ -327,9 +327,9 @@ def add_invertible_flag(flag: str, parser.add_argument('--python-executable', action='store', metavar='EXECUTABLE', help="Python executable which will be used in typechecking.", dest='special-opts:python_executable') - parser.add_argument('--no-infer-executable', action='store_true', + parser.add_argument('--no-site-packages', action='store_true', dest='special-opts:no_executable', - help="Do not infer a Python executable based on the version.") + help="Do not search for installed PEP 561 compliant packages.") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " "(defaults to sys.platform).") diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index adce623d8af0..ad17d8387ace 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -316,7 +316,7 @@ def parse_options(program_text: str, testcase: DataDrivenTestCase, flag_list = None if flags: flag_list = flags.group(1).split() - flag_list.append('--no-infer-executable') # the tests shouldn't need an installed Python + flag_list.append('--no-site-packages') # the tests shouldn't need an installed Python targets, options = process_options(flag_list, require_targets=False) if targets: # TODO: support specifying targets via the flags pragma diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index 337b591fb0dd..89b8e4b1964e 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -60,9 +60,9 @@ def test_executable_inference(self) -> None: assert str(e.value) == 'Python version (2, 10) did not match executable {}, got' \ ' version {}.'.format(sys.executable, str(sys.version_info[:2])) - # test that --no-infer-executable will disable executable inference + # test that --no-site-packages will disable executable inference matching_version = base + ['--python-version={}'.format(sys_ver_str), - '--no-infer-executable'] + '--no-site-packages'] _, options = process_options(matching_version) assert options.python_version == sys.version_info[:2] assert options.python_executable is None diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py index 7a3016991cce..88ad272fab0d 100644 --- a/mypy/test/testcmdline.py +++ b/mypy/test/testcmdline.py @@ -47,7 +47,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None: file.write('{}\n'.format(s)) args = parse_args(testcase.input[0]) args.append('--show-traceback') - args.append('--no-infer-executable') + args.append('--no-site-packages') # Type check the program. fixed = [python3_path, os.path.join(testcase.old_cwd, 'scripts', 'mypy')] diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index 871753b833de..7b914322c7ce 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -49,7 +49,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None: version. """ assert testcase.old_cwd is not None, "test was not properly set up" - mypy_cmdline = ['--show-traceback', '--no-infer-executable'] + mypy_cmdline = ['--show-traceback', '--no-site-packages'] py2 = testcase.name.lower().endswith('python2') if py2: mypy_cmdline.append('--py2') diff --git a/runtests.py b/runtests.py index 0714cf88cabf..8ef9f5e69746 100755 --- a/runtests.py +++ b/runtests.py @@ -73,7 +73,7 @@ def add_mypy_cmd(self, name: str, mypy_args: List[str], cwd: Optional[str] = Non return args = [sys.executable, self.mypy] + mypy_args args.append('--show-traceback') - args.append('--no-infer-executable') + args.append('--no-site-packages') self.waiter.add(LazySubprocess(full_name, args, cwd=cwd, env=self.env)) def add_mypy(self, name: str, *args: str, cwd: Optional[str] = None) -> None: From 446735645cb95f2e8e2bd96a0a1594fd133718c4 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 7 Mar 2018 02:42:47 -0800 Subject: [PATCH 08/47] Add PEP 561 docs --- docs/source/index.rst | 1 + docs/source/installed_packages.rst | 114 +++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 docs/source/installed_packages.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 90cc74941da8..582c1c4ee1b8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -30,6 +30,7 @@ Mypy is a static type checker for Python. command_line config_file python36 + installed_packages faq cheat_sheet cheat_sheet_py3 diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst new file mode 100644 index 000000000000..01384f508e18 --- /dev/null +++ b/docs/source/installed_packages.rst @@ -0,0 +1,114 @@ +.. _installed-packages: + +Using Installed Packages +======================== + +`PEP 561 `_ specifies how to mark +a package as supporting type checking. Below is a summary of how to create +PEP 561 compatible packages and have mypy use them in type checking. + +Making PEP 561 compatible packages +********************************** + +Packages that must be imported at runtime and supply type information should +put a ``py.typed`` in their package directory. For example, with a directory +structure as follows: + +.. code-block:: text + + setup.py + package_a/ + __init__.py + lib.py + py.typed + +the setup.py might look like: + +.. code-block:: python + + from distutils.core import setup + + setup( + name="SuperPackageA", + author="Me", + version="0.1", + package_data={"package_a": ["py.typed"]}, + packages=["package_a"] + ) + +Some packages have a mix of stub files and runtime files. These packages also require +a ``py.typed`` file. An example can be seen below: + +.. code-block:: text + + setup.py + package_b/ + __init__.py + lib.py + lib.pyi + py.typed + +the setup.py might look like: + +.. code-block:: python + + from distutils.core import setup + + setup( + name="SuperPackageB", + author="Me", + version="0.1", + package_data={"package_b": ["py.typed", "lib.pyi"]}, + packages=["package_b"] + ) + +In this example, both ``lib.py`` and ``lib.pyi`` exist. At runtime, ``lib.py`` +will be used, however mypy will use ``lib.pyi``. + +If the package is stub-only (not imported at runtime), the package should have +a prefix of the runtime package name and a suffix of ``-stubs``. +A ``py.typed`` file is not needed for stub-only packages. For example, if we +had stubs for ``package_c``, we might do the following: + +.. code-block:: text + + setup.py + package_c-stubs/ + __init__.pyi + lib.pyi + +the setup.py might look like: + +.. code-block:: python + + from distutils.core import setup + + setup( + name="SuperPackageC", + author="Me", + version="0.1", + package_data={"package_c-stubs": ["__init__.pyi", "lib.pyi"]}, + packages=["package_c-stubs"] + ) + +Using PEP 561 compatible packages with mypy +******************************************* + +Generally, you do not need to do anything to use installed packages for the +Python executable used to run mypy. They should be automatically picked up by +mypy and used for type checking. + +By default, mypy searches for packages installed for the Python executable +running mypy. It is highly unlikely you want this situation if you have +installed typed packages in another Python's package directory. + +Generally, you can use the ``--python-version`` flag and mypy will try to find +the correct package directory. If that fails, you can use the +``--python-executable`` flag to point to the exact executable, and mypy will +find packages installed for that Python executable. + +Note that mypy does not support some more advanced import features, such as zip +imports, namespace packages, and custom import hooks. + +If you do not want to use typed packages, use the ``--no-site-packages`` flag +to disable searching. \ No newline at end of file From 943e8e3ceae4709e8fc5379ccb15c9536d39f3ed Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 7 Mar 2018 02:49:47 -0800 Subject: [PATCH 09/47] Add PEP 561 tests --- mypy/test/config.py | 1 + mypy/test/testpep561.py | 115 ++++++++++++++++++++++++++++++++++++++++ runtests.py | 1 + 3 files changed, 117 insertions(+) create mode 100644 mypy/test/testpep561.py diff --git a/mypy/test/config.py b/mypy/test/config.py index 5dbe791e593a..5e6792fb0d57 100644 --- a/mypy/test/config.py +++ b/mypy/test/config.py @@ -5,6 +5,7 @@ # Location of test data files such as test case descriptions. test_data_prefix = os.path.join(PREFIX, 'test-data', 'unit') +package_path = os.path.join(PREFIX, 'test-data', 'packages') assert os.path.isdir(test_data_prefix), \ 'Test data prefix ({}) not set correctly'.format(test_data_prefix) diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py new file mode 100644 index 000000000000..5ab2bbf3cb3b --- /dev/null +++ b/mypy/test/testpep561.py @@ -0,0 +1,115 @@ +from contextlib import contextmanager +import os +import shutil +import sys +from typing import Generator, List +from unittest import TestCase, main + +import mypy.api +from mypy.build import get_site_packages_dirs +from mypy.test.config import package_path +from mypy.test.helpers import run_command +from mypy.util import try_find_python2_interpreter + +SIMPLE_PROGRAM = """ +from typedpkg.sample import ex +a = ex(['']) +reveal_type(a) +""" + + +class TestPEP561(TestCase): + @contextmanager + def install_package(self, pkg: str, + python_executable: str = sys.executable) -> Generator[None, None, None]: + """Context manager to temporarily install a package from test-data/packages/pkg/""" + working_dir = os.path.join(package_path, pkg) + install_cmd = [python_executable, '-m', 'pip', 'install', '.'] + # if we aren't in a virtualenv, install in the + # user package directory so we don't need sudo + if not hasattr(sys, 'real_prefix') or python_executable != sys.executable: + install_cmd.append('--user') + returncode, lines = run_command(install_cmd, cwd=working_dir) + if returncode != 0: + self.fail('\n'.join(lines)) + try: + yield + finally: + run_command([python_executable, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) + + def test_get_pkg_dirs(self) -> None: + """Check that get_package_dirs works.""" + dirs = get_site_packages_dirs(sys.executable) + assert dirs + + @staticmethod + def check_mypy_run(cmd_line: List[str], + expected_out: str, + expected_err: str = '', + expected_returncode: int = 1) -> None: + """Helper to run mypy and check the output.""" + out, err, returncode = mypy.api.run(cmd_line) + assert out == expected_out, err + assert err == expected_err, out + assert returncode == expected_returncode, returncode + + def test_typed_pkg(self) -> None: + """Tests type checking based on installed packages. + + This test CANNOT be split up, concurrency means that simultaneously + installing/uninstalling will break tests""" + test_file = 'simple.py' + if not os.path.isdir('test-packages-data'): + os.mkdir('test-packages-data') + old_cwd = os.getcwd() + os.chdir('test-packages-data') + with open(test_file, 'w') as f: + f.write(SIMPLE_PROGRAM) + try: + with self.install_package('typedpkg-stubs'): + self.check_mypy_run( + [test_file], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) + + # The Python 2 tests are intentionally placed after a Python 3 test to check + # the package_dir_cache is behaving correctly. + python2 = try_find_python2_interpreter() + if python2: + with self.install_package('typedpkg-stubs', python2): + self.check_mypy_run( + ['--python-executable={}'.format(python2), test_file], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) + with self.install_package('typedpkg', python2): + self.check_mypy_run( + ['--python-executable={}'.format(python2), 'simple.py'], + "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + ) + + with self.install_package('typedpkg', python2): + with self.install_package('typedpkg-stubs', python2): + self.check_mypy_run( + ['--python-executable={}'.format(python2), test_file], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) + + with self.install_package('typedpkg'): + self.check_mypy_run( + [test_file], + "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + ) + + with self.install_package('typedpkg'): + with self.install_package('typedpkg-stubs'): + self.check_mypy_run( + [test_file], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) + finally: + os.chdir(old_cwd) + shutil.rmtree('test-packages-data') + + +if __name__ == '__main__': + main() diff --git a/runtests.py b/runtests.py index 8ef9f5e69746..213ffc90e978 100755 --- a/runtests.py +++ b/runtests.py @@ -230,6 +230,7 @@ def test_path(*names: str): ) SLOW_FILES = test_path( + 'testpep561', 'testpythoneval', 'testcmdline', 'teststubgen', From f28b9014454e7f0bef8aef5f17d391f3ae46c51b Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 7 Mar 2018 03:30:00 -0800 Subject: [PATCH 10/47] Add PEP 561 test packages --- test-data/packages/typedpkg-stubs/setup.py | 13 +++++++++++++ .../typedpkg-stubs/typedpkg-stubs/__init__.pyi | 0 .../typedpkg-stubs/typedpkg-stubs/sample.pyi | 2 ++ test-data/packages/typedpkg/setup.py | 14 ++++++++++++++ test-data/packages/typedpkg/typedpkg/__init__.py | 0 test-data/packages/typedpkg/typedpkg/py.typed | 0 test-data/packages/typedpkg/typedpkg/sample.py | 7 +++++++ 7 files changed, 36 insertions(+) create mode 100644 test-data/packages/typedpkg-stubs/setup.py create mode 100644 test-data/packages/typedpkg-stubs/typedpkg-stubs/__init__.pyi create mode 100644 test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi create mode 100644 test-data/packages/typedpkg/setup.py create mode 100644 test-data/packages/typedpkg/typedpkg/__init__.py create mode 100644 test-data/packages/typedpkg/typedpkg/py.typed create mode 100644 test-data/packages/typedpkg/typedpkg/sample.py diff --git a/test-data/packages/typedpkg-stubs/setup.py b/test-data/packages/typedpkg-stubs/setup.py new file mode 100644 index 000000000000..b90e3a011f23 --- /dev/null +++ b/test-data/packages/typedpkg-stubs/setup.py @@ -0,0 +1,13 @@ +""" +This setup file installs packages to test mypy's PEP 561 implementation +""" + +from distutils.core import setup + +setup( + name='typedpkg-stubs', + author="The mypy team", + version='0.1', + package_data={'typedpkg-stubs': ['sample.pyi', '__init__.pyi']}, + packages=['typedpkg-stubs'], +) diff --git a/test-data/packages/typedpkg-stubs/typedpkg-stubs/__init__.pyi b/test-data/packages/typedpkg-stubs/typedpkg-stubs/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi b/test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi new file mode 100644 index 000000000000..355deefd6a2d --- /dev/null +++ b/test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi @@ -0,0 +1,2 @@ +from typing import Iterable, List +def ex(a: Iterable[str]) -> List[str]: ... diff --git a/test-data/packages/typedpkg/setup.py b/test-data/packages/typedpkg/setup.py new file mode 100644 index 000000000000..6da37d2f6629 --- /dev/null +++ b/test-data/packages/typedpkg/setup.py @@ -0,0 +1,14 @@ +""" +This setup file installs packages to test mypy's PEP 561 implementation +""" + +from distutils.core import setup + +setup( + name='typedpkg', + author="The mypy team", + version='0.1', + package_data={'typedpkg': ['py.typed']}, + packages=['typedpkg'], + include_package_data=True, +) diff --git a/test-data/packages/typedpkg/typedpkg/__init__.py b/test-data/packages/typedpkg/typedpkg/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg/typedpkg/py.typed b/test-data/packages/typedpkg/typedpkg/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg/typedpkg/sample.py b/test-data/packages/typedpkg/typedpkg/sample.py new file mode 100644 index 000000000000..59f6aec1548e --- /dev/null +++ b/test-data/packages/typedpkg/typedpkg/sample.py @@ -0,0 +1,7 @@ +from typing import Iterable, Tuple + + +def ex(a): + # type: (Iterable[str]) -> Tuple[str, ...] + """Example typed package. This intentionally has an error.""" + return tuple(a) From 3a6f8fb5f80d54a27a900dcaccaa694b2441ef42 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 13 Mar 2018 22:23:30 -0700 Subject: [PATCH 11/47] Add PEP 561 packages searching implementation --- mypy/build.py | 90 +++++++++++++++++++++++++++++++++++------ mypy/main.py | 3 +- mypy/stubgen.py | 3 +- mypy/test/testcheck.py | 4 +- mypy/test/testdmypy.py | 3 +- mypy/test/testpep561.py | 4 +- 6 files changed, 89 insertions(+), 18 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index acf4041ad2fe..48303a92828f 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -10,10 +10,12 @@ """ # TODO: More consistent terminology, e.g. path/fnam, module/id, state/file +import ast import binascii import collections import contextlib from distutils.sysconfig import get_python_lib +import functools import gc import hashlib import json @@ -21,6 +23,7 @@ import re import site import stat +import subprocess import sys import time from os.path import dirname, basename @@ -708,7 +711,8 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: def is_module(self, id: str) -> bool: """Is there a file in the file system corresponding to module id?""" - return self.find_module_cache.find_module(id, self.lib_path) is not None + return self.find_module_cache.find_module(id, self.lib_path, + self.options.python_executable) is not None def parse_file(self, id: str, path: str, source: str, ignore_errors: bool) -> MypyFile: """Parse the source of a file with the given name. @@ -821,6 +825,18 @@ def remove_cwd_prefix_from_path(fscache: FileSystemCache, p: str) -> str: return p +USER_SITE_PACKAGES = \ + 'from __future__ import print_function; import site; print(repr(site.getsitepackages()' \ + ' + [site.getusersitepackages()]))' +VIRTUALENV_SITE_PACKAGES = \ + 'from distutils.sysconfig import get_python_lib; print(repr([get_python_lib()]))' + + +def call_python(python_executable: str, command: str) -> str: + return subprocess.check_output([python_executable, '-c', command], + stderr=subprocess.PIPE).decode() + + class FindModuleCache: """Module finder with integrated cache. @@ -835,7 +851,7 @@ class FindModuleCache: def __init__(self, fscache: Optional[FileSystemMetaCache] = None) -> None: self.fscache = fscache or FileSystemMetaCache() # Cache find_module: (id, lib_path) -> result. - self.results = {} # type: Dict[Tuple[str, Tuple[str, ...]], Optional[str]] + self.results = {} # type: Dict[Tuple[str, Optional[str], Tuple[str, ...]], Optional[str]] # Cache some repeated work within distinct find_module calls: finding which # elements of lib_path have even the subdirectory they'd need for the module @@ -847,7 +863,33 @@ def clear(self) -> None: self.results.clear() self.dirs.clear() - def _find_module(self, id: str, lib_path: Tuple[str, ...]) -> Optional[str]: + @functools.lru_cache(maxsize=None) + def _get_site_packages_dirs(self, python_executable: Optional[str]) -> List[str]: + """Find package directories for given python.""" + if python_executable is None: + return [] + if python_executable == sys.executable: + # Use running Python's package dirs + if hasattr(site, 'getusersitepackages') and hasattr(site, 'getsitepackages'): + user_dir = site.getusersitepackages() + return site.getsitepackages() + [user_dir] + # If site doesn't have get(user)sitepackages, we are running in a + # virtualenv, and should fall back to get_python_lib + return [get_python_lib()] + else: + # Use subprocess to get the package directory of given Python + # executable + try: + output = call_python(python_executable, USER_SITE_PACKAGES) + except subprocess.CalledProcessError: + # if no paths are found (raising a CalledProcessError), we fall back on sysconfig, + # the python executable is likely in a virtual environment, thus lacking + # needed site methods + output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) + return ast.literal_eval(output) + + def _find_module(self, id: str, lib_path: Tuple[str, ...], + python_executable: Optional[str]) -> Optional[str]: fscache = self.fscache # If we're looking for a module like 'foo.bar.baz', it's likely that most of the @@ -856,6 +898,7 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...]) -> Optional[str]: # that will require the same subdirectory. components = id.split('.') dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' + # TODO (ethanhs): refactor each path search to its own method with lru_cache if (dir_chain, lib_path) not in self.dirs: dirs = [] for pathitem in lib_path: @@ -864,7 +907,22 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...]) -> Optional[str]: if fscache.isdir(dir): dirs.append(dir) self.dirs[dir_chain, lib_path] = dirs - candidate_base_dirs = self.dirs[dir_chain, lib_path] + + third_party_dirs = [] + # Third-party stub/typed packages + for pkg_dir in self._get_site_packages_dirs(python_executable): + stub_name = components[0] + '-stubs' + typed_file = os.path.join(pkg_dir, components[0], 'py.typed') + stub_dir = os.path.join(pkg_dir, stub_name) + if os.path.isdir(stub_dir): + stub_components = [stub_name] + components[1:] + path = os.path.join(pkg_dir, *stub_components[:-1]) + if os.path.isdir(path): + third_party_dirs.append(path) + elif os.path.isfile(typed_file): + path = os.path.join(pkg_dir, dir_chain) + third_party_dirs.append(path) + candidate_base_dirs = self.dirs[dir_chain, lib_path] + third_party_dirs # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now # contains just the subdirectories 'foo/bar' that actually exist under the @@ -877,8 +935,11 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...]) -> Optional[str]: # Prefer package over module, i.e. baz/__init__.py* over baz.py*. for extension in PYTHON_EXTENSIONS: path = base_path + sepinit + extension + path_stubs = base_path + '-stubs' + sepinit + extension if fscache.isfile_case(path) and verify_module(fscache, id, path): return path + elif fscache.isfile_case(path_stubs) and verify_module(fscache, id, path_stubs): + return path_stubs # No package, look for module. for extension in PYTHON_EXTENSIONS: path = base_path + extension @@ -886,17 +947,19 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...]) -> Optional[str]: return path return None - def find_module(self, id: str, lib_path_arg: Iterable[str]) -> Optional[str]: + def find_module(self, id: str, lib_path_arg: Iterable[str], + python_executable: Optional[str]) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) - key = (id, lib_path) + key = (id, python_executable, lib_path) if key not in self.results: - self.results[key] = self._find_module(id, lib_path) + self.results[key] = self._find_module(id, lib_path, python_executable) return self.results[key] - def find_modules_recursive(self, module: str, lib_path: List[str]) -> List[BuildSource]: - module_path = self.find_module(module, lib_path) + def find_modules_recursive(self, module: str, lib_path: List[str], + python_executable: Optional[str]) -> List[BuildSource]: + module_path = self.find_module(module, lib_path, python_executable) if not module_path: return [] result = [BuildSource(module_path, module, None)] @@ -916,13 +979,15 @@ def find_modules_recursive(self, module: str, lib_path: List[str]) -> List[Build (os.path.isfile(os.path.join(abs_path, '__init__.py')) or os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): hits.add(item) - result += self.find_modules_recursive(module + '.' + item, lib_path) + result += self.find_modules_recursive(module + '.' + item, lib_path, + python_executable) elif item != '__init__.py' and item != '__init__.pyi' and \ item.endswith(('.py', '.pyi')): mod = item.split('.')[0] if mod not in hits: hits.add(mod) - result += self.find_modules_recursive(module + '.' + mod, lib_path) + result += self.find_modules_recursive(module + '.' + mod, lib_path, + python_executable) return result @@ -1540,7 +1605,8 @@ def __init__(self, # difference and just assume 'builtins' everywhere, # which simplifies code. file_id = '__builtin__' - path = manager.find_module_cache.find_module(file_id, manager.lib_path) + path = manager.find_module_cache.find_module(file_id, manager.lib_path, + manager.options.python_executable) if path: # For non-stubs, look at options.follow_imports: # - normal (default) -> fully analyze diff --git a/mypy/main.py b/mypy/main.py index 2d22999e136b..a98f0c03323a 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -618,7 +618,8 @@ def add_invertible_flag(flag: str, options.build_type = BuildType.MODULE lib_path = [os.getcwd()] + build.mypy_path() # TODO: use the same cache as the BuildManager will - targets = build.FindModuleCache().find_modules_recursive(special_opts.package, lib_path) + targets = build.FindModuleCache().find_modules_recursive(special_opts.package, lib_path, + options.python_executable) if not targets: fail("Can't find package '{}'".format(special_opts.package)) return targets, options diff --git a/mypy/stubgen.py b/mypy/stubgen.py index bb9112d5dc67..5179511febfe 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -156,7 +156,8 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int], module_all = getattr(mod, '__all__', None) else: # Find module by going through search path. - module_path = mypy.build.FindModuleCache().find_module(module, ['.'] + search_path) + module_path = mypy.build.FindModuleCache().find_module(module, ['.'] + search_path, + interpreter) if not module_path: raise SystemExit( "Can't find module '{}' (consider using --search-path)".format(module)) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index d406ff6ade44..fb97167e2501 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -3,6 +3,7 @@ import os import re import shutil +import sys from typing import Dict, List, Optional, Set, Tuple @@ -291,7 +292,8 @@ def parse_module(self, module_names = m.group(1) out = [] for module_name in module_names.split(' '): - path = build.FindModuleCache().find_module(module_name, [test_temp_dir]) + path = build.FindModuleCache().find_module(module_name, [test_temp_dir], + sys.executable) assert path is not None, "Can't find ad hoc case file" with open(path) as f: program_text = f.read() diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py index e5bfdf231bc3..ea55b611d0fd 100644 --- a/mypy/test/testdmypy.py +++ b/mypy/test/testdmypy.py @@ -260,7 +260,8 @@ def parse_module(self, module_names = m.group(1) out = [] # type: List[Tuple[str, str, Optional[str]]] for module_name in module_names.split(' '): - path = build.FindModuleCache().find_module(module_name, [test_temp_dir]) + path = build.FindModuleCache().find_module(module_name, [test_temp_dir], + sys.executable) if path is None and module_name.startswith(NON_EXISTENT_PREFIX): # This is a special name for a file that we don't want to exist. assert '.' not in module_name # TODO: Packages not supported here diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index 5ab2bbf3cb3b..e20e1f94ff79 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -6,7 +6,7 @@ from unittest import TestCase, main import mypy.api -from mypy.build import get_site_packages_dirs +from mypy.build import FindModuleCache from mypy.test.config import package_path from mypy.test.helpers import run_command from mypy.util import try_find_python2_interpreter @@ -39,7 +39,7 @@ def install_package(self, pkg: str, def test_get_pkg_dirs(self) -> None: """Check that get_package_dirs works.""" - dirs = get_site_packages_dirs(sys.executable) + dirs = FindModuleCache()._get_site_packages_dirs(sys.executable) assert dirs @staticmethod From d3106e345a37ab8b8f198d864c529d415e0f9a4a Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 13 Mar 2018 22:30:34 -0700 Subject: [PATCH 12/47] Make Python inference inplace and clean up test --- mypy/main.py | 6 ++---- mypy/test/testargs.py | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index a98f0c03323a..03de9fa1a6d8 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -255,8 +255,7 @@ def _python_executable_from_version(python_version: Tuple[int, int]) -> str: def infer_python_version_and_executable(options: Options, - special_opts: argparse.Namespace - ) -> Options: + special_opts: argparse.Namespace) -> None: # Infer Python version and/or executable if one is not given if special_opts.python_executable is not None and special_opts.python_version is not None: py_exe_ver = _python_version_from_executable(special_opts.python_executable) @@ -278,7 +277,6 @@ def infer_python_version_and_executable(options: Options, options.python_version = _python_version_from_executable( special_opts.python_executable) options.python_executable = special_opts.python_executable - return options def process_options(args: List[str], @@ -565,7 +563,7 @@ def add_invertible_flag(flag: str, "is now mypy's default and only parser.") try: - options = infer_python_version_and_executable(options, special_opts) + infer_python_version_and_executable(options, special_opts) except PythonExecutableInferenceError as e: parser.error(str(e)) diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index 89b8e4b1964e..bde251dd95a0 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -25,7 +25,7 @@ def test_coherence(self) -> None: def test_executable_inference(self) -> None: """Test the --python-executable flag with --python-version""" - sys_ver_str = '.'.join(map(str, sys.version_info[:2])) + sys_ver_str = '{ver.major}.{ver.minor}'.format(ver=sys.version_info[:2]) base = ['file.py'] # dummy file @@ -56,9 +56,9 @@ def test_executable_inference(self) -> None: special_opts.python_version = (2, 10) # obviously wrong special_opts.no_executable = None with pytest.raises(PythonExecutableInferenceError) as e: - options = infer_python_version_and_executable(options, special_opts) + infer_python_version_and_executable(options, special_opts) assert str(e.value) == 'Python version (2, 10) did not match executable {}, got' \ - ' version {}.'.format(sys.executable, str(sys.version_info[:2])) + ' version {}.'.format(sys.executable, sys.version_info[:2]) # test that --no-site-packages will disable executable inference matching_version = base + ['--python-version={}'.format(sys_ver_str), From 3e94ac33bdadadf07142f63bba2f366eeaca1d6a Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 13 Mar 2018 22:36:37 -0700 Subject: [PATCH 13/47] Fix testargs --- mypy/test/testargs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index bde251dd95a0..ecfe4143d7b4 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -25,7 +25,7 @@ def test_coherence(self) -> None: def test_executable_inference(self) -> None: """Test the --python-executable flag with --python-version""" - sys_ver_str = '{ver.major}.{ver.minor}'.format(ver=sys.version_info[:2]) + sys_ver_str = '{ver.major}.{ver.minor}'.format(ver=sys.version_info) base = ['file.py'] # dummy file From 3b969deafb8d8edb7df1226d57e4535ae4ff848e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 13 Mar 2018 23:58:27 -0700 Subject: [PATCH 14/47] Reorder FindModuleCache.find_module key --- mypy/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 48303a92828f..bd041dd73eb2 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -851,7 +851,7 @@ class FindModuleCache: def __init__(self, fscache: Optional[FileSystemMetaCache] = None) -> None: self.fscache = fscache or FileSystemMetaCache() # Cache find_module: (id, lib_path) -> result. - self.results = {} # type: Dict[Tuple[str, Optional[str], Tuple[str, ...]], Optional[str]] + self.results = {} # type: Dict[Tuple[str, Tuple[str, ...], Optional[str]], Optional[str]] # Cache some repeated work within distinct find_module calls: finding which # elements of lib_path have even the subdirectory they'd need for the module @@ -952,7 +952,7 @@ def find_module(self, id: str, lib_path_arg: Iterable[str], """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) - key = (id, python_executable, lib_path) + key = (id, lib_path, python_executable) if key not in self.results: self.results[key] = self._find_module(id, lib_path, python_executable) return self.results[key] From 89492d2ba1e39899ff8437bb7f32646ab98aeb2a Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Mar 2018 15:50:02 -0700 Subject: [PATCH 15/47] Trigger new CI run --- mypy/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index bd041dd73eb2..2d34b09ca4f5 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -884,7 +884,7 @@ def _get_site_packages_dirs(self, python_executable: Optional[str]) -> List[str] except subprocess.CalledProcessError: # if no paths are found (raising a CalledProcessError), we fall back on sysconfig, # the python executable is likely in a virtual environment, thus lacking - # needed site methods + # the needed site methods output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) return ast.literal_eval(output) From 768efe05f186be4c6cf850dbb6c5f2089962b547 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Mar 2018 20:57:26 -0700 Subject: [PATCH 16/47] Try reducing parrallelism pytest on travis seems to be crashing, so reduce to running single threaded until the problem is solved. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ff4077c0dda1..7c9448c84301 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,6 @@ install: - pip install . script: - - python runtests.py -j12 -x lint -x package + - python runtests.py -j1 -x lint -x package - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then flake8; fi - if [[ $TRAVIS_PYTHON_VERSION == '3.5.1' ]]; then python runtests.py package; fi From e43efeee100b9c47e8f1028048887da16b58ba87 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 25 Mar 2018 01:12:14 -0700 Subject: [PATCH 17/47] Try reducing size of fscache --- mypy/fscache.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/mypy/fscache.py b/mypy/fscache.py index 75600dba2951..7646bcf017d2 100644 --- a/mypy/fscache.py +++ b/mypy/fscache.py @@ -30,21 +30,34 @@ import os import stat -from typing import Tuple, Dict, List, Optional +from typing import Dict, List, Optional, Tuple, TypeVar from mypy.util import read_with_python_encoding +T = TypeVar('T') + + +class CacheDict(Dict[str, T]): + def __init__(self, max_size: int = 100) -> None: + self.max_size = max_size + + def __set_item__(self, key: str, item: T) -> None: + if len(self) > self.max_size: + self.clear() + dict.__set_item__(self, key, item) # type: ignore + + class FileSystemMetaCache: def __init__(self) -> None: self.flush() def flush(self) -> None: """Start another transaction and empty all caches.""" - self.stat_cache = {} # type: Dict[str, os.stat_result] - self.stat_error_cache = {} # type: Dict[str, Exception] - self.listdir_cache = {} # type: Dict[str, List[str]] - self.listdir_error_cache = {} # type: Dict[str, Exception] - self.isfile_case_cache = {} # type: Dict[str, bool] + self.stat_cache = CacheDict() # type: CacheDict[os.stat_result] + self.stat_error_cache = CacheDict() # type: CacheDict[Exception] + self.listdir_cache = CacheDict() # type: CacheDict[List[str]] + self.listdir_error_cache = CacheDict() # type: CacheDict[Exception] + self.isfile_case_cache = CacheDict() # type: CacheDict[bool] def stat(self, path: str) -> os.stat_result: if path in self.stat_cache: From 090ac93565782e864f94b17220b2204b5aecea3a Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 25 Mar 2018 17:03:57 -0700 Subject: [PATCH 18/47] Revert travis back to using 12 concurrent processes --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7c9448c84301..ff4077c0dda1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,6 @@ install: - pip install . script: - - python runtests.py -j1 -x lint -x package + - python runtests.py -j12 -x lint -x package - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then flake8; fi - if [[ $TRAVIS_PYTHON_VERSION == '3.5.1' ]]; then python runtests.py package; fi From a10541d7077da21865a6dd1d420c2d1ec8176cbb Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 25 Mar 2018 17:46:57 -0700 Subject: [PATCH 19/47] Try to remove all caching --- mypy/build.py | 48 +++++++++++++++++++------------------------ mypy/fscache.py | 54 ++++++++++--------------------------------------- 2 files changed, 32 insertions(+), 70 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 862bd19e57fc..77cc8f282f45 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -838,18 +838,10 @@ class FindModuleCache: def __init__(self, fscache: Optional[FileSystemMetaCache] = None) -> None: self.fscache = fscache or FileSystemMetaCache() - # Cache find_module: (id, lib_path) -> result. - self.results = {} # type: Dict[Tuple[str, Tuple[str, ...], Optional[str]], Optional[str]] - - # Cache some repeated work within distinct find_module calls: finding which - # elements of lib_path have even the subdirectory they'd need for the module - # to exist. This is shared among different module ids when they differ only - # in the last component. - self.dirs = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]] def clear(self) -> None: - self.results.clear() - self.dirs.clear() + self._find_module.cache_clear() + self._find_lib_path_dirs.cache_clear() @functools.lru_cache(maxsize=None) def _get_site_packages_dirs(self, python_executable: Optional[str]) -> List[str]: @@ -876,6 +868,22 @@ def _get_site_packages_dirs(self, python_executable: Optional[str]) -> List[str] output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) return ast.literal_eval(output) + @functools.lru_cache(maxsize=0) + def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...], + python_executable: str) -> List[str]: + # Cache some repeated work within distinct find_module calls: finding which + # elements of lib_path have even the subdirectory they'd need for the module + # to exist. This is shared among different module ids when they differ only + # in the last component. + dirs = [] + for pathitem in lib_path: + # e.g., '/usr/lib/python3.4/foo/bar' + dir = os.path.normpath(os.path.join(pathitem, dir_chain)) + if self.fscache.isdir(dir): + dirs.append(dir) + return dirs + + @functools.lru_cache(maxsize=0) def _find_module(self, id: str, lib_path: Tuple[str, ...], python_executable: Optional[str]) -> Optional[str]: fscache = self.fscache @@ -887,14 +895,6 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...], components = id.split('.') dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' # TODO (ethanhs): refactor each path search to its own method with lru_cache - if (dir_chain, lib_path) not in self.dirs: - dirs = [] - for pathitem in lib_path: - # e.g., '/usr/lib/python3.4/foo/bar' - dir = os.path.normpath(os.path.join(pathitem, dir_chain)) - if fscache.isdir(dir): - dirs.append(dir) - self.dirs[dir_chain, lib_path] = dirs third_party_dirs = [] # Third-party stub/typed packages @@ -910,7 +910,7 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...], elif os.path.isfile(typed_file): path = os.path.join(pkg_dir, dir_chain) third_party_dirs.append(path) - candidate_base_dirs = self.dirs[dir_chain, lib_path] + third_party_dirs + candidate_base_dirs = self._find_lib_path_dirs(dir_chain, lib_path, python_executable) + third_party_dirs # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now # contains just the subdirectories 'foo/bar' that actually exist under the @@ -939,12 +939,8 @@ def find_module(self, id: str, lib_path_arg: Iterable[str], python_executable: Optional[str]) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) - - key = (id, lib_path, python_executable) - if key not in self.results: - self.results[key] = self._find_module(id, lib_path, python_executable) - return self.results[key] - + return self._find_module(id, lib_path, python_executable) + def find_modules_recursive(self, module: str, lib_path: List[str], python_executable: Optional[str]) -> List[BuildSource]: module_path = self.find_module(module, lib_path, python_executable) @@ -2103,8 +2099,6 @@ def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph: stubs_found=sum(g.path is not None and g.path.endswith('.pyi') for g in graph.values()), graph_load_time=(t1 - t0), - fm_cache_size=len(manager.find_module_cache.results), - fm_dir_cache_size=len(manager.find_module_cache.dirs), ) if not graph: print("Nothing to do?!") diff --git a/mypy/fscache.py b/mypy/fscache.py index 7646bcf017d2..9c5cc28ca2bf 100644 --- a/mypy/fscache.py +++ b/mypy/fscache.py @@ -28,62 +28,32 @@ advantage of the benefits. """ +import functools import os import stat from typing import Dict, List, Optional, Tuple, TypeVar from mypy.util import read_with_python_encoding -T = TypeVar('T') - - -class CacheDict(Dict[str, T]): - def __init__(self, max_size: int = 100) -> None: - self.max_size = max_size - - def __set_item__(self, key: str, item: T) -> None: - if len(self) > self.max_size: - self.clear() - dict.__set_item__(self, key, item) # type: ignore - - class FileSystemMetaCache: def __init__(self) -> None: self.flush() def flush(self) -> None: """Start another transaction and empty all caches.""" - self.stat_cache = CacheDict() # type: CacheDict[os.stat_result] - self.stat_error_cache = CacheDict() # type: CacheDict[Exception] - self.listdir_cache = CacheDict() # type: CacheDict[List[str]] - self.listdir_error_cache = CacheDict() # type: CacheDict[Exception] - self.isfile_case_cache = CacheDict() # type: CacheDict[bool] + self.stat_cache = {} # type: Dict[str, os.stat_result] + self.stat_error_cache = {} # type: Dict[str, Exception] + self.listdir_cache = {} # type: Dict[str, List[str]] + self.listdir_error_cache = {} # type: Dict[str, Exception] + self.isfile_case_cache = {} # type: Dict[str, bool] + @functools.lru_cache(maxsize=0) def stat(self, path: str) -> os.stat_result: - if path in self.stat_cache: - return self.stat_cache[path] - if path in self.stat_error_cache: - raise self.stat_error_cache[path] - try: - st = os.stat(path) - except Exception as err: - self.stat_error_cache[path] = err - raise - self.stat_cache[path] = st - return st + return os.stat(path) + @functools.lru_cache(maxsize=0) def listdir(self, path: str) -> List[str]: - if path in self.listdir_cache: - return self.listdir_cache[path] - if path in self.listdir_error_cache: - raise self.listdir_error_cache[path] - try: - results = os.listdir(path) - except Exception as err: - self.listdir_error_cache[path] = err - raise err - self.listdir_cache[path] = results - return results + return os.listdir(path) def isfile(self, path: str) -> bool: try: @@ -92,6 +62,7 @@ def isfile(self, path: str) -> bool: return False return stat.S_ISREG(st.st_mode) + @functools.lru_cache(maxsize=0) def isfile_case(self, path: str) -> bool: """Return whether path exists and is a file. @@ -101,8 +72,6 @@ def isfile_case(self, path: str) -> bool: TODO: We should maybe check the case for some directory components also, to avoid permitting wrongly-cased *packages*. """ - if path in self.isfile_case_cache: - return self.isfile_case_cache[path] head, tail = os.path.split(path) if not tail: res = False @@ -112,7 +81,6 @@ def isfile_case(self, path: str) -> bool: res = tail in names and self.isfile(path) except OSError: res = False - self.isfile_case_cache[path] = res return res def isdir(self, path: str) -> bool: From bfe388d289b5927c5e22d9b0ec13db297fbb51f0 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 25 Mar 2018 17:47:15 -0700 Subject: [PATCH 20/47] Update typeshed --- typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typeshed b/typeshed index 9b6df1d6bcb3..afeecb478b69 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 9b6df1d6bcb367e6dfc8fc48737ff481001d8b97 +Subproject commit afeecb478b69f3e73e757d0868f158099acf0dce From 7d1578115f4b92cf5a8c1894444787eded990e18 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 25 Mar 2018 18:05:31 -0700 Subject: [PATCH 21/47] Clear caches correctly --- mypy/fscache.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mypy/fscache.py b/mypy/fscache.py index 9c5cc28ca2bf..e35079d1a38d 100644 --- a/mypy/fscache.py +++ b/mypy/fscache.py @@ -41,11 +41,9 @@ def __init__(self) -> None: def flush(self) -> None: """Start another transaction and empty all caches.""" - self.stat_cache = {} # type: Dict[str, os.stat_result] - self.stat_error_cache = {} # type: Dict[str, Exception] - self.listdir_cache = {} # type: Dict[str, List[str]] - self.listdir_error_cache = {} # type: Dict[str, Exception] - self.isfile_case_cache = {} # type: Dict[str, bool] + self.stat.cache_clear() + self.listdir.cache_clear() + self.isfile_case.cache_clear() @functools.lru_cache(maxsize=0) def stat(self, path: str) -> os.stat_result: From 3e35a575e0b7ef67d534510e4486898a04053ce3 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 25 Mar 2018 18:06:01 -0700 Subject: [PATCH 22/47] Increase sizes of find_module caches --- mypy/build.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 77cc8f282f45..b0ea5fb1448e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -868,7 +868,7 @@ def _get_site_packages_dirs(self, python_executable: Optional[str]) -> List[str] output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) return ast.literal_eval(output) - @functools.lru_cache(maxsize=0) + @functools.lru_cache(maxsize=10) def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...], python_executable: str) -> List[str]: # Cache some repeated work within distinct find_module calls: finding which @@ -883,7 +883,7 @@ def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...], dirs.append(dir) return dirs - @functools.lru_cache(maxsize=0) + @functools.lru_cache(maxsize=None) def _find_module(self, id: str, lib_path: Tuple[str, ...], python_executable: Optional[str]) -> Optional[str]: fscache = self.fscache @@ -910,7 +910,8 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...], elif os.path.isfile(typed_file): path = os.path.join(pkg_dir, dir_chain) third_party_dirs.append(path) - candidate_base_dirs = self._find_lib_path_dirs(dir_chain, lib_path, python_executable) + third_party_dirs + candidate_base_dirs = self._find_lib_path_dirs(dir_chain, lib_path, + python_executable) + third_party_dirs # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now # contains just the subdirectories 'foo/bar' that actually exist under the @@ -940,7 +941,7 @@ def find_module(self, id: str, lib_path_arg: Iterable[str], """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) return self._find_module(id, lib_path, python_executable) - + def find_modules_recursive(self, module: str, lib_path: List[str], python_executable: Optional[str]) -> List[BuildSource]: module_path = self.find_module(module, lib_path, python_executable) From 4d53df5353dc459c0d18ab0db34d8073b574b91c Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 26 Mar 2018 23:34:04 -0700 Subject: [PATCH 23/47] This should fail --- mypy/build.py | 2 +- mypy/fscache.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index b0ea5fb1448e..76bafbabc2e1 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -868,7 +868,7 @@ def _get_site_packages_dirs(self, python_executable: Optional[str]) -> List[str] output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) return ast.literal_eval(output) - @functools.lru_cache(maxsize=10) + @functools.lru_cache(maxsize=None) def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...], python_executable: str) -> List[str]: # Cache some repeated work within distinct find_module calls: finding which diff --git a/mypy/fscache.py b/mypy/fscache.py index e35079d1a38d..c16f53d2f94a 100644 --- a/mypy/fscache.py +++ b/mypy/fscache.py @@ -43,13 +43,12 @@ def flush(self) -> None: """Start another transaction and empty all caches.""" self.stat.cache_clear() self.listdir.cache_clear() - self.isfile_case.cache_clear() - @functools.lru_cache(maxsize=0) + @functools.lru_cache(maxsize=None) def stat(self, path: str) -> os.stat_result: return os.stat(path) - @functools.lru_cache(maxsize=0) + @functools.lru_cache(maxsize=None) def listdir(self, path: str) -> List[str]: return os.listdir(path) @@ -60,7 +59,7 @@ def isfile(self, path: str) -> bool: return False return stat.S_ISREG(st.st_mode) - @functools.lru_cache(maxsize=0) + @functools.lru_cache(maxsize=None) def isfile_case(self, path: str) -> bool: """Return whether path exists and is a file. From 8d25d5decd3567f3ecaf1afdb090c9d791b297c9 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 26 Mar 2018 23:58:26 -0700 Subject: [PATCH 24/47] Remove isfile_case cache This cache almost always misses. --- mypy/fscache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mypy/fscache.py b/mypy/fscache.py index c16f53d2f94a..d561e2839f90 100644 --- a/mypy/fscache.py +++ b/mypy/fscache.py @@ -59,7 +59,6 @@ def isfile(self, path: str) -> bool: return False return stat.S_ISREG(st.st_mode) - @functools.lru_cache(maxsize=None) def isfile_case(self, path: str) -> bool: """Return whether path exists and is a file. From c4410e0243c67bb4921ec46dc9b4333d7445dfba Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 27 Mar 2018 00:32:41 -0700 Subject: [PATCH 25/47] Move _get_site_packages_dirs --- mypy/build.py | 53 +++++++++++++++++++++-------------------- mypy/test/testpep561.py | 4 ++-- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 76bafbabc2e1..1497566c4154 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -825,6 +825,32 @@ def call_python(python_executable: str, command: str) -> str: stderr=subprocess.PIPE).decode() +@functools.lru_cache(maxsize=None) +def _get_site_packages_dirs(python_executable: Optional[str]) -> List[str]: + """Find package directories for given python.""" + if python_executable is None: + return [] + if python_executable == sys.executable: + # Use running Python's package dirs + if hasattr(site, 'getusersitepackages') and hasattr(site, 'getsitepackages'): + user_dir = site.getusersitepackages() + return site.getsitepackages() + [user_dir] + # If site doesn't have get(user)sitepackages, we are running in a + # virtualenv, and should fall back to get_python_lib + return [get_python_lib()] + else: + # Use subprocess to get the package directory of given Python + # executable + try: + output = call_python(python_executable, USER_SITE_PACKAGES) + except subprocess.CalledProcessError: + # if no paths are found (raising a CalledProcessError), we fall back on sysconfig, + # the python executable is likely in a virtual environment, thus lacking + # the needed site methods + output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) + return ast.literal_eval(output) + + class FindModuleCache: """Module finder with integrated cache. @@ -843,31 +869,6 @@ def clear(self) -> None: self._find_module.cache_clear() self._find_lib_path_dirs.cache_clear() - @functools.lru_cache(maxsize=None) - def _get_site_packages_dirs(self, python_executable: Optional[str]) -> List[str]: - """Find package directories for given python.""" - if python_executable is None: - return [] - if python_executable == sys.executable: - # Use running Python's package dirs - if hasattr(site, 'getusersitepackages') and hasattr(site, 'getsitepackages'): - user_dir = site.getusersitepackages() - return site.getsitepackages() + [user_dir] - # If site doesn't have get(user)sitepackages, we are running in a - # virtualenv, and should fall back to get_python_lib - return [get_python_lib()] - else: - # Use subprocess to get the package directory of given Python - # executable - try: - output = call_python(python_executable, USER_SITE_PACKAGES) - except subprocess.CalledProcessError: - # if no paths are found (raising a CalledProcessError), we fall back on sysconfig, - # the python executable is likely in a virtual environment, thus lacking - # the needed site methods - output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) - return ast.literal_eval(output) - @functools.lru_cache(maxsize=None) def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...], python_executable: str) -> List[str]: @@ -898,7 +899,7 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...], third_party_dirs = [] # Third-party stub/typed packages - for pkg_dir in self._get_site_packages_dirs(python_executable): + for pkg_dir in _get_site_packages_dirs(python_executable): stub_name = components[0] + '-stubs' typed_file = os.path.join(pkg_dir, components[0], 'py.typed') stub_dir = os.path.join(pkg_dir, stub_name) diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index e20e1f94ff79..87c5d96cd538 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -6,7 +6,7 @@ from unittest import TestCase, main import mypy.api -from mypy.build import FindModuleCache +from mypy.build import FindModuleCache, _get_site_packages_dirs from mypy.test.config import package_path from mypy.test.helpers import run_command from mypy.util import try_find_python2_interpreter @@ -39,7 +39,7 @@ def install_package(self, pkg: str, def test_get_pkg_dirs(self) -> None: """Check that get_package_dirs works.""" - dirs = FindModuleCache()._get_site_packages_dirs(sys.executable) + dirs = _get_site_packages_dirs(sys.executable) assert dirs @staticmethod From 73fefc49ad1fad011286e2cc63a6548c4a981e7d Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 27 Mar 2018 20:18:33 -0700 Subject: [PATCH 26/47] Make lru_caches per-instance --- mypy/build.py | 21 ++++++++------------- mypy/fscache.py | 10 +++++----- mypy/main.py | 2 +- mypy/test/testcheck.py | 2 +- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 1497566c4154..10d8feb74aee 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -864,12 +864,13 @@ class FindModuleCache: def __init__(self, fscache: Optional[FileSystemMetaCache] = None) -> None: self.fscache = fscache or FileSystemMetaCache() + self.find_lib_path_dirs = functools.lru_cache(maxsize=None)(self._find_lib_path_dirs) + self.find_module = functools.lru_cache(maxsize=None)(self._find_module) def clear(self) -> None: - self._find_module.cache_clear() - self._find_lib_path_dirs.cache_clear() + self.find_module.cache_clear() + self.find_lib_path_dirs.cache_clear() - @functools.lru_cache(maxsize=None) def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...], python_executable: str) -> List[str]: # Cache some repeated work within distinct find_module calls: finding which @@ -884,9 +885,9 @@ def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...], dirs.append(dir) return dirs - @functools.lru_cache(maxsize=None) def _find_module(self, id: str, lib_path: Tuple[str, ...], python_executable: Optional[str]) -> Optional[str]: + """Return the path of the module source file, or None if not found.""" fscache = self.fscache # If we're looking for a module like 'foo.bar.baz', it's likely that most of the @@ -911,8 +912,8 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...], elif os.path.isfile(typed_file): path = os.path.join(pkg_dir, dir_chain) third_party_dirs.append(path) - candidate_base_dirs = self._find_lib_path_dirs(dir_chain, lib_path, - python_executable) + third_party_dirs + candidate_base_dirs = self.find_lib_path_dirs(dir_chain, lib_path, + python_executable) + third_party_dirs # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now # contains just the subdirectories 'foo/bar' that actually exist under the @@ -937,13 +938,7 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...], return path return None - def find_module(self, id: str, lib_path_arg: Iterable[str], - python_executable: Optional[str]) -> Optional[str]: - """Return the path of the module source file, or None if not found.""" - lib_path = tuple(lib_path_arg) - return self._find_module(id, lib_path, python_executable) - - def find_modules_recursive(self, module: str, lib_path: List[str], + def find_modules_recursive(self, module: str, lib_path: Tuple[str, ...], python_executable: Optional[str]) -> List[BuildSource]: module_path = self.find_module(module, lib_path, python_executable) if not module_path: diff --git a/mypy/fscache.py b/mypy/fscache.py index d561e2839f90..95b60b0cb66b 100644 --- a/mypy/fscache.py +++ b/mypy/fscache.py @@ -37,19 +37,18 @@ class FileSystemMetaCache: def __init__(self) -> None: - self.flush() + self.stat = functools.lru_cache(maxsize=None)(self._stat) + self.listdir = functools.lru_cache(maxsize=None)(self._listdir) def flush(self) -> None: """Start another transaction and empty all caches.""" self.stat.cache_clear() self.listdir.cache_clear() - @functools.lru_cache(maxsize=None) - def stat(self, path: str) -> os.stat_result: + def _stat(self, path: str) -> os.stat_result: return os.stat(path) - @functools.lru_cache(maxsize=None) - def listdir(self, path: str) -> List[str]: + def _listdir(self, path: str) -> List[str]: return os.listdir(path) def isfile(self, path: str) -> bool: @@ -96,6 +95,7 @@ def exists(self, path: str) -> bool: class FileSystemCache(FileSystemMetaCache): def __init__(self, pyversion: Tuple[int, int]) -> None: + super().__init__() self.pyversion = pyversion self.flush() diff --git a/mypy/main.py b/mypy/main.py index abf09fe7fe67..260f768939fd 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -614,7 +614,7 @@ def add_invertible_flag(flag: str, for p in special_opts.packages: if os.sep in p or os.altsep and os.altsep in p: fail("Package name '{}' cannot have a slash in it.".format(p)) - p_targets = cache.find_modules_recursive(p, lib_path, options.python_executable) + p_targets = cache.find_modules_recursive(p, tuple(lib_path), options.python_executable) if not p_targets: fail("Can't find package '{}'".format(p)) targets.extend(p_targets) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 850dd72efef3..bc3feab1a90a 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -277,7 +277,7 @@ def parse_module(self, module_names = m.group(1) out = [] for module_name in module_names.split(' '): - path = build.FindModuleCache().find_module(module_name, [test_temp_dir], + path = build.FindModuleCache().find_module(module_name, (test_temp_dir,), sys.executable) assert path is not None, "Can't find ad hoc case file" with open(path) as f: From ea8541a3a2bcff5e9f20fcae066d7c212f429276 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 28 Mar 2018 13:30:36 -0700 Subject: [PATCH 27/47] Add exception cache back --- mypy/fscache.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/mypy/fscache.py b/mypy/fscache.py index 95b60b0cb66b..9075b41fbe77 100644 --- a/mypy/fscache.py +++ b/mypy/fscache.py @@ -39,17 +39,34 @@ class FileSystemMetaCache: def __init__(self) -> None: self.stat = functools.lru_cache(maxsize=None)(self._stat) self.listdir = functools.lru_cache(maxsize=None)(self._listdir) + # lru_cache doesn't handle exceptions, so we need special caches for them. + self.stat_error_cache = {} # type: Dict[str, Exception] + self.listdir_error_cache = {} # type: Dict[str, Exception] def flush(self) -> None: """Start another transaction and empty all caches.""" self.stat.cache_clear() self.listdir.cache_clear() + self.stat_error_cache.clear() + self.listdir_error_cache.clear() def _stat(self, path: str) -> os.stat_result: - return os.stat(path) + if path in self.stat_error_cache: + raise self.stat_error_cache[path] + try: + return os.stat(path) + except Exception as err: + self.stat_error_cache[path] = err + raise def _listdir(self, path: str) -> List[str]: - return os.listdir(path) + if path in self.listdir_error_cache: + raise self.listdir_error_cache[path] + try: + return os.listdir(path) + except Exception as err: + self.listdir_error_cache[path] = err + raise err def isfile(self, path: str) -> bool: try: From 5142add2311993ff0ef7220f7bab28e20f8d09a5 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 29 Mar 2018 12:00:40 -0700 Subject: [PATCH 28/47] Fold logic into subprocess call for simplicity --- mypy/build.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 948e08a2a846..cbd2fb60fa5c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -813,11 +813,14 @@ def remove_cwd_prefix_from_path(fscache: FileSystemCache, p: str) -> str: return p -USER_SITE_PACKAGES = \ - 'from __future__ import print_function; import site; print(repr(site.getsitepackages()' \ - ' + [site.getusersitepackages()]))' -VIRTUALENV_SITE_PACKAGES = \ - 'from distutils.sysconfig import get_python_lib; print(repr([get_python_lib()]))' +SITE_PACKAGES = \ + '''from __future__ import print_function +from distutils.sysconfig import get_python_lib +import site +try: + print(repr(site.getsitepackages() + [site.getusersitepackages()])) +except: + print(repr([get_python_lib()]))''' def call_python(python_executable: str, command: str) -> str: @@ -841,13 +844,7 @@ def _get_site_packages_dirs(python_executable: Optional[str]) -> List[str]: else: # Use subprocess to get the package directory of given Python # executable - try: - output = call_python(python_executable, USER_SITE_PACKAGES) - except subprocess.CalledProcessError: - # if no paths are found (raising a CalledProcessError), we fall back on sysconfig, - # the python executable is likely in a virtual environment, thus lacking - # the needed site methods - output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) + output = call_python(python_executable, SITE_PACKAGES) return ast.literal_eval(output) From 432132f1a18b8e43d9fc3992424eb5ca634fde08 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 29 Mar 2018 12:22:14 -0700 Subject: [PATCH 29/47] Clarify docs and re-order --- docs/source/installed_packages.rst | 64 ++++++++++++++++-------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index 01384f508e18..076ed6a328f4 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -7,12 +7,40 @@ Using Installed Packages a package as supporting type checking. Below is a summary of how to create PEP 561 compatible packages and have mypy use them in type checking. +Using PEP 561 compatible packages with mypy +******************************************* + +Generally, you do not need to do anything to use installed packages for the +Python executable used to run mypy. They should be automatically picked up by +mypy and used for type checking. + +By default, mypy searches for packages installed for the Python executable +running mypy. It is highly unlikely you want this situation if you have +installed typed packages in another Python's package directory. + +Generally, you can use the ``--python-version`` flag and mypy will try to find +the correct package directory. If that fails, you can use the +``--python-executable`` flag to point to the exact executable, and mypy will +find packages installed for that Python executable. + +Note that mypy does not support some more advanced import features, such as zip +imports, namespace packages, and custom import hooks. + +If you do not want to use typed packages, use the ``--no-site-packages`` flag +to disable searching. + Making PEP 561 compatible packages ********************************** -Packages that must be imported at runtime and supply type information should -put a ``py.typed`` in their package directory. For example, with a directory -structure as follows: +PEP 561 notes three main ways to distribute type information. The first is a +package that has only inline type annotations in the code itself. The second is +a package that ships stub files with type information alongside the runtime +code. The third method, also known as a "stub only package" is a package that +ships type information for a package seperately as stub files. + +Packages that must be used at runtime and supply type information via type +comments or annotations in the code should put a ``py.typed`` in their package +directory. For example, with a directory structure as follows: .. code-block:: text @@ -36,8 +64,8 @@ the setup.py might look like: packages=["package_a"] ) -Some packages have a mix of stub files and runtime files. These packages also require -a ``py.typed`` file. An example can be seen below: +Some packages have a mix of stub files and runtime files. These packages also +require a ``py.typed`` file. An example can be seen below: .. code-block:: text @@ -62,8 +90,8 @@ the setup.py might look like: packages=["package_b"] ) -In this example, both ``lib.py`` and ``lib.pyi`` exist. At runtime, ``lib.py`` -will be used, however mypy will use ``lib.pyi``. +In this example, both ``lib.py`` and ``lib.pyi`` exist. At runtime, the Python +interpeter will use ``lib.py``, but mypy will use ``lib.pyi`` instead. If the package is stub-only (not imported at runtime), the package should have a prefix of the runtime package name and a suffix of ``-stubs``. @@ -90,25 +118,3 @@ the setup.py might look like: package_data={"package_c-stubs": ["__init__.pyi", "lib.pyi"]}, packages=["package_c-stubs"] ) - -Using PEP 561 compatible packages with mypy -******************************************* - -Generally, you do not need to do anything to use installed packages for the -Python executable used to run mypy. They should be automatically picked up by -mypy and used for type checking. - -By default, mypy searches for packages installed for the Python executable -running mypy. It is highly unlikely you want this situation if you have -installed typed packages in another Python's package directory. - -Generally, you can use the ``--python-version`` flag and mypy will try to find -the correct package directory. If that fails, you can use the -``--python-executable`` flag to point to the exact executable, and mypy will -find packages installed for that Python executable. - -Note that mypy does not support some more advanced import features, such as zip -imports, namespace packages, and custom import hooks. - -If you do not want to use typed packages, use the ``--no-site-packages`` flag -to disable searching. \ No newline at end of file From 3fd86ee45d20b6df47e248975071f67951f086b6 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 29 Mar 2018 14:50:58 -0700 Subject: [PATCH 30/47] Make subprocess call identical to runtime code --- mypy/build.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index cbd2fb60fa5c..8bc90bfca2b8 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -813,14 +813,16 @@ def remove_cwd_prefix_from_path(fscache: FileSystemCache, p: str) -> str: return p -SITE_PACKAGES = \ - '''from __future__ import print_function +SITE_PACKAGES = ''' +from __future__ import print_function from distutils.sysconfig import get_python_lib import site -try: - print(repr(site.getsitepackages() + [site.getusersitepackages()])) -except: - print(repr([get_python_lib()]))''' +if hasattr(site, 'getusersitepackages') and hasattr(site, 'getsitepackages'): + user_dir = site.getusersitepackages() + print(repr(site.getsitepackages() + [user_dir])) +else: + print(repr([get_python_lib()])) +''' def call_python(python_executable: str, command: str) -> str: From b0a302c9662f445c2eb577a1b64e4aeb2d233515 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 29 Mar 2018 21:05:02 -0700 Subject: [PATCH 31/47] Add introductory text to installed_packages --- docs/source/installed_packages.rst | 33 ++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index 076ed6a328f4..a3d64b19aa82 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -3,9 +3,32 @@ Using Installed Packages ======================== +One common pattern of modularizing Python code is through packages, which are +collections of modules (``*.py`` files). Packages usually have ``__init__.py`` +files in them. + +Packages can be uploaded to PyPi and installed via ``pip`` as part of +distributions. Python installations have special directories to place packages +installed through tools such as ``pip``. + `PEP 561 `_ specifies how to mark -a package as supporting type checking. Below is a summary of how to create -PEP 561 compatible packages and have mypy use them in type checking. +an installed package as supporting type checking. Supporting type checking +means that the package can be used as a source of type information for tools +like mypy. + +There are three main kinds of typed packages. The first is a +package that has only inline type annotations in the code itself. The second is +a package that ships stub files with type information alongside the runtime +code. The third method, also known as a "stub only package" is a package that +ships type information for a package seperately as stub files. + +These packages differ from the stubs in typeshed as they are installable +through, for example ``pip``, instead of tied to a mypy release. In addition, +they allow for the distribution of type information seperate from the regular +package itself. + +Below is a summary of how to make sure mypy is finding installed typed packages +and how to create PEP 561 compatible packages of your own. Using PEP 561 compatible packages with mypy ******************************************* @@ -32,12 +55,6 @@ to disable searching. Making PEP 561 compatible packages ********************************** -PEP 561 notes three main ways to distribute type information. The first is a -package that has only inline type annotations in the code itself. The second is -a package that ships stub files with type information alongside the runtime -code. The third method, also known as a "stub only package" is a package that -ships type information for a package seperately as stub files. - Packages that must be used at runtime and supply type information via type comments or annotations in the code should put a ``py.typed`` in their package directory. For example, with a directory structure as follows: From 7cd5ebc353eb9714ceaf56fe476751ddb6dd1e83 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 29 Mar 2018 21:48:25 -0700 Subject: [PATCH 32/47] Move logic into new file, reduce duplication --- mypy/build.py | 29 ++++------------------------- mypy/sitepkgs.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 25 deletions(-) create mode 100644 mypy/sitepkgs.py diff --git a/mypy/build.py b/mypy/build.py index 8bc90bfca2b8..0b7af48c17b7 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -36,6 +36,7 @@ if MYPY: from typing import Deque +from . import sitepkgs from mypy.nodes import (MODULE_REF, MypyFile, Node, ImportBase, Import, ImportFrom, ImportAll) from mypy.semanal_pass1 import SemanticAnalyzerPass1 from mypy.semanal import SemanticAnalyzerPass2, apply_semantic_analyzer_patches @@ -813,23 +814,6 @@ def remove_cwd_prefix_from_path(fscache: FileSystemCache, p: str) -> str: return p -SITE_PACKAGES = ''' -from __future__ import print_function -from distutils.sysconfig import get_python_lib -import site -if hasattr(site, 'getusersitepackages') and hasattr(site, 'getsitepackages'): - user_dir = site.getusersitepackages() - print(repr(site.getsitepackages() + [user_dir])) -else: - print(repr([get_python_lib()])) -''' - - -def call_python(python_executable: str, command: str) -> str: - return subprocess.check_output([python_executable, '-c', command], - stderr=subprocess.PIPE).decode() - - @functools.lru_cache(maxsize=None) def _get_site_packages_dirs(python_executable: Optional[str]) -> List[str]: """Find package directories for given python.""" @@ -837,17 +821,12 @@ def _get_site_packages_dirs(python_executable: Optional[str]) -> List[str]: return [] if python_executable == sys.executable: # Use running Python's package dirs - if hasattr(site, 'getusersitepackages') and hasattr(site, 'getsitepackages'): - user_dir = site.getusersitepackages() - return site.getsitepackages() + [user_dir] - # If site doesn't have get(user)sitepackages, we are running in a - # virtualenv, and should fall back to get_python_lib - return [get_python_lib()] + return sitepkgs.getsitepackages() else: # Use subprocess to get the package directory of given Python # executable - output = call_python(python_executable, SITE_PACKAGES) - return ast.literal_eval(output) + return ast.literal_eval(subprocess.check_output([python_executable, sitepkgs.__file__], + stderr=subprocess.PIPE).decode()) class FindModuleCache: diff --git a/mypy/sitepkgs.py b/mypy/sitepkgs.py new file mode 100644 index 000000000000..e90a03f8b31f --- /dev/null +++ b/mypy/sitepkgs.py @@ -0,0 +1,20 @@ +from __future__ import print_function +# NOTE: This file must remain compatible with Python 2 + + +from distutils.sysconfig import get_python_lib +import site +from typing import List + + +def getsitepackages(): + # type: () -> List[str] + if hasattr(site, 'getusersitepackages') and hasattr(site, 'getsitepackages'): + user_dir = site.getusersitepackages() + return site.getsitepackages() + [user_dir] + else: + return [get_python_lib()] + + +if __name__ == '__main__': + print(repr(getsitepackages())) From 7a593b2e467e6696b26e74056c0f60ab5a195e7e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 29 Mar 2018 22:26:15 -0700 Subject: [PATCH 33/47] Just in case, don't actually import typing --- mypy/sitepkgs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypy/sitepkgs.py b/mypy/sitepkgs.py index e90a03f8b31f..76721af56f8d 100644 --- a/mypy/sitepkgs.py +++ b/mypy/sitepkgs.py @@ -4,7 +4,9 @@ from distutils.sysconfig import get_python_lib import site -from typing import List +MYPY = False +if MYPY: + from typing import List def getsitepackages(): From ddfa26a38762f421b7cd1bfb5a5b6afa2028cac6 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 4 Apr 2018 19:43:26 -0700 Subject: [PATCH 34/47] Make docs clearer --- docs/source/installed_packages.rst | 48 +++++++++++------------------- 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index a3d64b19aa82..a4e8c9e6abb6 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -3,39 +3,17 @@ Using Installed Packages ======================== -One common pattern of modularizing Python code is through packages, which are -collections of modules (``*.py`` files). Packages usually have ``__init__.py`` -files in them. - -Packages can be uploaded to PyPi and installed via ``pip`` as part of -distributions. Python installations have special directories to place packages -installed through tools such as ``pip``. - `PEP 561 `_ specifies how to mark -an installed package as supporting type checking. Supporting type checking -means that the package can be used as a source of type information for tools -like mypy. - -There are three main kinds of typed packages. The first is a -package that has only inline type annotations in the code itself. The second is -a package that ships stub files with type information alongside the runtime -code. The third method, also known as a "stub only package" is a package that -ships type information for a package seperately as stub files. - -These packages differ from the stubs in typeshed as they are installable -through, for example ``pip``, instead of tied to a mypy release. In addition, -they allow for the distribution of type information seperate from the regular -package itself. - -Below is a summary of how to make sure mypy is finding installed typed packages -and how to create PEP 561 compatible packages of your own. +a package as supporting type checking. Below is a summary of how to create +PEP 561 compatible packages and have mypy use them in type checking. Using PEP 561 compatible packages with mypy ******************************************* -Generally, you do not need to do anything to use installed packages for the -Python executable used to run mypy. They should be automatically picked up by -mypy and used for type checking. +Generally, you do not need to do anything to use installed packages that +support typing for the Python executable used to run mypy. Note that most +packages do not support typing. Packages that do support typing should be +automatically picked up by mypy and used for type checking. By default, mypy searches for packages installed for the Python executable running mypy. It is highly unlikely you want this situation if you have @@ -55,9 +33,17 @@ to disable searching. Making PEP 561 compatible packages ********************************** -Packages that must be used at runtime and supply type information via type -comments or annotations in the code should put a ``py.typed`` in their package -directory. For example, with a directory structure as follows: +PEP 561 notes three main ways to distribute type information. The first is a +package that has only inline type annotations in the code itself. The second is +a package that ships stub files with type information alongside the runtime +code. The third method, also known as a "stub only package" is a package that +ships type information for a package separately as stub files. + +If you would like to publish a library package to a package repository (e.g. +PyPI) for either internal or external use in type checking, packages that +supply type information via type comments or annotations in the code should put +a ``py.typed`` in their package directory. For example, with a directory +structure as follows: .. code-block:: text From a5e636c02d868050afdbe6987aea6d9013799e1a Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 4 Apr 2018 19:49:54 -0700 Subject: [PATCH 35/47] Code cleanup, add details of _get_site_packages_dirs --- mypy/build.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 0b7af48c17b7..841bf44aab1c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -36,7 +36,7 @@ if MYPY: from typing import Deque -from . import sitepkgs +from mypy import sitepkgs from mypy.nodes import (MODULE_REF, MypyFile, Node, ImportBase, Import, ImportFrom, ImportAll) from mypy.semanal_pass1 import SemanticAnalyzerPass1 from mypy.semanal import SemanticAnalyzerPass2, apply_semantic_analyzer_patches @@ -816,7 +816,10 @@ def remove_cwd_prefix_from_path(fscache: FileSystemCache, p: str) -> str: @functools.lru_cache(maxsize=None) def _get_site_packages_dirs(python_executable: Optional[str]) -> List[str]: - """Find package directories for given python.""" + """Find package directories for given python. + + This runs a subprocess call, which generates a list of the site package directories. + To avoid repeatedly calling a subprocess (which can be slow!) we lru_cache the results.""" if python_executable is None: return [] if python_executable == sys.executable: From 262769cfbc02e7dc3ae315e8dfa6ca5c35aaf879 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 4 Apr 2018 19:57:53 -0700 Subject: [PATCH 36/47] Remove unneeded argument --- mypy/build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 841bf44aab1c..3005ee1adb0b 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -852,8 +852,7 @@ def clear(self) -> None: self.find_module.cache_clear() self.find_lib_path_dirs.cache_clear() - def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...], - python_executable: str) -> List[str]: + def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...]) -> List[str]: # Cache some repeated work within distinct find_module calls: finding which # elements of lib_path have even the subdirectory they'd need for the module # to exist. This is shared among different module ids when they differ only From 12920c919e2d98f051cf5e46ebb7bc6e257720d7 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 4 Apr 2018 20:04:25 -0700 Subject: [PATCH 37/47] Refactor python_executable_prefix --- mypy/main.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 260f768939fd..53cd53b29b54 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -219,11 +219,14 @@ class PythonExecutableInferenceError(Exception): """Represents a failure to infer the version or executable while searching.""" -if sys.platform == 'win32': - def python_executable_prefix(v: str) -> List[str]: +def python_executable_prefix(v: str) -> List[str]: + if sys.platform == 'win32': + # on Windows, all Python executables are named `python`. To handle this, there + # is the `py` launcher, which can be passed a version e.g. `py -3.5`, and it will + # execute an installed Python 3.5 interpreter. See also: + # https://docs.python.org/3/using/windows.html#python-launcher-for-windows return ['py', '-{}'.format(v)] -else: - def python_executable_prefix(v: str) -> List[str]: + else: return ['python{}'.format(v)] @@ -242,7 +245,6 @@ def _python_executable_from_version(python_version: Tuple[int, int]) -> str: if sys.version_info[:2] == python_version: return sys.executable str_ver = '.'.join(map(str, python_version)) - print(str_ver) try: sys_exe = subprocess.check_output(python_executable_prefix(str_ver) + ['-c', 'import sys; print(sys.executable)'], From 41772572efc7cec73504a9027b8964e3a45484f2 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 4 Apr 2018 20:22:48 -0700 Subject: [PATCH 38/47] Clarify python-executable flag --- mypy/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 53cd53b29b54..9da2699bf07f 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -333,14 +333,14 @@ def add_invertible_flag(flag: str, parser.add_argument('--python-version', type=parse_version, metavar='x.y', help='use Python x.y', dest='special-opts:python_version') parser.add_argument('--python-executable', action='store', metavar='EXECUTABLE', - help="Python executable which will be used in typechecking.", - dest='special-opts:python_executable') + help="Python executable used for finding PEP 561 compliant installed" + " packages and stubs", dest='special-opts:python_executable') parser.add_argument('--no-site-packages', action='store_true', dest='special-opts:no_executable', - help="Do not search for installed PEP 561 compliant packages.") + help="Do not search for installed PEP 561 compliant packages") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " - "(defaults to sys.platform).") + "(defaults to sys.platform)") parser.add_argument('-2', '--py2', dest='python_version', action='store_const', const=defaults.PYTHON2_VERSION, help="use Python 2 mode") parser.add_argument('--ignore-missing-imports', action='store_true', From 8948d6aa6e0ab8b6f6f0e28ce6c9a197ccb3b250 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 4 Apr 2018 20:34:17 -0700 Subject: [PATCH 39/47] Remove unused argument --- mypy/build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index c4a7771dd2cb..c572b95a1e6b 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -872,8 +872,7 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...], elif os.path.isfile(typed_file): path = os.path.join(pkg_dir, dir_chain) third_party_dirs.append(path) - candidate_base_dirs = self.find_lib_path_dirs(dir_chain, lib_path, - python_executable) + third_party_dirs + candidate_base_dirs = self.find_lib_path_dirs(dir_chain, lib_path) + third_party_dirs # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now # contains just the subdirectories 'foo/bar' that actually exist under the From 6916b0907a53627cf18b730ae8ac12a99d2474d8 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 4 Apr 2018 20:44:37 -0700 Subject: [PATCH 40/47] Change python_version -> python_executable --- mypy/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index c572b95a1e6b..611c66db586d 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -2038,8 +2038,8 @@ def find_module_and_diagnose(manager: BuildManager, # difference and just assume 'builtins' everywhere, # which simplifies code. file_id = '__builtin__' - path = manager.find_module_cache.find_module(file_id, manager.lib_path, - manager.options.python_version) + path = manager.find_module_cache.find_module(file_id, manager.lib_path, + manager.options.python_executable) if path: # For non-stubs, look at options.follow_imports: # - normal (default) -> fully analyze From 87cbdb22be8fb686c24b06d82d9302e51d4dfb07 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 6 Apr 2018 15:12:03 -0700 Subject: [PATCH 41/47] Add information about inference and add TODO --- mypy/main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mypy/main.py b/mypy/main.py index 5cfea31ba2e2..5d235f96889a 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -253,7 +253,15 @@ def _python_executable_from_version(python_version: Tuple[int, int]) -> str: def infer_python_version_and_executable(options: Options, special_opts: argparse.Namespace) -> None: + """Infer the Python version or executable from each other. Check they are consistent. + + This function mutates options based on special_opts to infer the correct Python version and + executable to use. + """ # Infer Python version and/or executable if one is not given + + # TODO: (ethanhs) Look at folding these checks and the site packages subprocess calls into + # one subprocess call for speed. if special_opts.python_executable is not None and special_opts.python_version is not None: py_exe_ver = _python_version_from_executable(special_opts.python_executable) if py_exe_ver != special_opts.python_version: From e44b1f5c9353f0d969a17d2717e384451c3a1532 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 6 Apr 2018 15:12:24 -0700 Subject: [PATCH 42/47] Refactor testpep561 --- mypy/test/testpep561.py | 43 ++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index 87c5d96cd538..5d2e3a5963e3 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -2,7 +2,7 @@ import os import shutil import sys -from typing import Generator, List +from typing import Iterator, List from unittest import TestCase, main import mypy.api @@ -18,15 +18,28 @@ """ +def check_mypy_run(cmd_line: List[str], + expected_out: str, + expected_err: str = '', + expected_returncode: int = 1) -> None: + """Helper to run mypy and check the output.""" + out, err, returncode = mypy.api.run(cmd_line) + assert out == expected_out, err + assert err == expected_err, out + assert returncode == expected_returncode, returncode + + class TestPEP561(TestCase): @contextmanager def install_package(self, pkg: str, - python_executable: str = sys.executable) -> Generator[None, None, None]: + python_executable: str = sys.executable) -> Iterator[None]: """Context manager to temporarily install a package from test-data/packages/pkg/""" working_dir = os.path.join(package_path, pkg) install_cmd = [python_executable, '-m', 'pip', 'install', '.'] # if we aren't in a virtualenv, install in the # user package directory so we don't need sudo + # In a virtualenv, real_prefix is patched onto + # sys if not hasattr(sys, 'real_prefix') or python_executable != sys.executable: install_cmd.append('--user') returncode, lines = run_command(install_cmd, cwd=working_dir) @@ -42,22 +55,12 @@ def test_get_pkg_dirs(self) -> None: dirs = _get_site_packages_dirs(sys.executable) assert dirs - @staticmethod - def check_mypy_run(cmd_line: List[str], - expected_out: str, - expected_err: str = '', - expected_returncode: int = 1) -> None: - """Helper to run mypy and check the output.""" - out, err, returncode = mypy.api.run(cmd_line) - assert out == expected_out, err - assert err == expected_err, out - assert returncode == expected_returncode, returncode - def test_typed_pkg(self) -> None: """Tests type checking based on installed packages. This test CANNOT be split up, concurrency means that simultaneously - installing/uninstalling will break tests""" + installing/uninstalling will break tests. + """ test_file = 'simple.py' if not os.path.isdir('test-packages-data'): os.mkdir('test-packages-data') @@ -67,7 +70,7 @@ def test_typed_pkg(self) -> None: f.write(SIMPLE_PROGRAM) try: with self.install_package('typedpkg-stubs'): - self.check_mypy_run( + check_mypy_run( [test_file], "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) @@ -77,32 +80,32 @@ def test_typed_pkg(self) -> None: python2 = try_find_python2_interpreter() if python2: with self.install_package('typedpkg-stubs', python2): - self.check_mypy_run( + check_mypy_run( ['--python-executable={}'.format(python2), test_file], "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) with self.install_package('typedpkg', python2): - self.check_mypy_run( + check_mypy_run( ['--python-executable={}'.format(python2), 'simple.py'], "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" ) with self.install_package('typedpkg', python2): with self.install_package('typedpkg-stubs', python2): - self.check_mypy_run( + check_mypy_run( ['--python-executable={}'.format(python2), test_file], "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) with self.install_package('typedpkg'): - self.check_mypy_run( + check_mypy_run( [test_file], "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" ) with self.install_package('typedpkg'): with self.install_package('typedpkg-stubs'): - self.check_mypy_run( + check_mypy_run( [test_file], "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) From 88279fa3e85cb2b2f84e2d35d72cb9cee9500665 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 6 Apr 2018 15:12:51 -0700 Subject: [PATCH 43/47] Fix outdated docstring --- test-data/packages/typedpkg/typedpkg/sample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/packages/typedpkg/typedpkg/sample.py b/test-data/packages/typedpkg/typedpkg/sample.py index 59f6aec1548e..6a5f88a0f372 100644 --- a/test-data/packages/typedpkg/typedpkg/sample.py +++ b/test-data/packages/typedpkg/typedpkg/sample.py @@ -3,5 +3,5 @@ def ex(a): # type: (Iterable[str]) -> Tuple[str, ...] - """Example typed package. This intentionally has an error.""" + """Example typed package.""" return tuple(a) From e590e665825d12edc39b5619a0932490425f3dbc Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 6 Apr 2018 15:20:11 -0700 Subject: [PATCH 44/47] Add docstring describing sitepkgs.py --- mypy/sitepkgs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mypy/sitepkgs.py b/mypy/sitepkgs.py index 76721af56f8d..8a36bffb98c9 100644 --- a/mypy/sitepkgs.py +++ b/mypy/sitepkgs.py @@ -1,5 +1,11 @@ from __future__ import print_function -# NOTE: This file must remain compatible with Python 2 +"""This file is used to find the site packages of a Python executable, which may be Python 2. + +This file MUST remain compatible with Python 2. Since we cannot make any assumptions about the +Python being executed, this module should not use *any* dependencies outside of the standard +library found in Python 2. This file is run each mypy run, so it should be kept as fast as +possible. +""" from distutils.sysconfig import get_python_lib From df13f2a005ea1c12f3ad9f5de0505c4873e80c7e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 6 Apr 2018 15:22:25 -0700 Subject: [PATCH 45/47] Add note about python_executable option --- mypy/options.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/options.py b/mypy/options.py index f6aef420e8d5..33728fff62cf 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -56,6 +56,8 @@ def __init__(self) -> None: # -- build options -- self.build_type = BuildType.STANDARD self.python_version = sys.version_info[:2] # type: Tuple[int, int] + # The executable used to search for PEP 561 packages. If this is None, + # then mypy does not search for PEP 561 packages. self.python_executable = sys.executable # type: Optional[str] self.platform = sys.platform self.custom_typing_module = None # type: Optional[str] From 6525f13efa3f1196fd89158ba05b9df748b334d0 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 6 Apr 2018 15:30:55 -0700 Subject: [PATCH 46/47] Use fscache in a few more places --- mypy/build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 611c66db586d..138476e17a9c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -864,12 +864,12 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...], stub_name = components[0] + '-stubs' typed_file = os.path.join(pkg_dir, components[0], 'py.typed') stub_dir = os.path.join(pkg_dir, stub_name) - if os.path.isdir(stub_dir): + if fscache.isdir(stub_dir): stub_components = [stub_name] + components[1:] path = os.path.join(pkg_dir, *stub_components[:-1]) - if os.path.isdir(path): + if fscache.isdir(path): third_party_dirs.append(path) - elif os.path.isfile(typed_file): + elif fscache.isfile(typed_file): path = os.path.join(pkg_dir, dir_chain) third_party_dirs.append(path) candidate_base_dirs = self.find_lib_path_dirs(dir_chain, lib_path) + third_party_dirs From bc0141e07491fc3e3bbd33d0b0d717c9cacf0444 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 6 Apr 2018 15:34:41 -0700 Subject: [PATCH 47/47] Add back some cache info in dispatch --- mypy/build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypy/build.py b/mypy/build.py index 138476e17a9c..cdd955defadc 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -2163,10 +2163,14 @@ def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph: graph = load_graph(sources, manager) t1 = time.time() + fm_cache_size = manager.find_module_cache.find_module.cache_info().currsize + fm_dir_cache_size = manager.find_module_cache.find_lib_path_dirs.cache_info().currsize manager.add_stats(graph_size=len(graph), stubs_found=sum(g.path is not None and g.path.endswith('.pyi') for g in graph.values()), graph_load_time=(t1 - t0), + fm_cache_size=fm_cache_size, + fm_dir_cache_size=fm_dir_cache_size, ) if not graph: print("Nothing to do?!")