From fad1364006eb66dd6bd1d07f5319ee21e9277a88 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Thu, 20 Jul 2023 19:18:44 +0100 Subject: [PATCH 1/3] Stubtest: error if typeshed is missing modules from the stdlib --- mypy/stubtest.py | 49 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index f06faa962b07..1ddf389b5006 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1635,7 +1635,7 @@ def get_stub(module: str) -> nodes.MypyFile | None: def get_typeshed_stdlib_modules( custom_typeshed_dir: str | None, version_info: tuple[int, int] | None = None -) -> list[str]: +) -> set[str]: """Returns a list of stdlib modules in typeshed (for current Python version).""" stdlib_py_versions = mypy.modulefinder.load_stdlib_py_versions(custom_typeshed_dir) if version_info is None: @@ -1657,14 +1657,44 @@ def exists_in_version(module: str) -> bool: typeshed_dir = Path(mypy.build.default_data_dir()) / "typeshed" stdlib_dir = typeshed_dir / "stdlib" - modules = [] + modules: set[str] = set() for path in stdlib_dir.rglob("*.pyi"): if path.stem == "__init__": path = path.parent module = ".".join(path.relative_to(stdlib_dir).parts[:-1] + (path.stem,)) if exists_in_version(module): - modules.append(module) - return sorted(modules) + modules.add(module) + return modules + + +def get_importable_stdlib_modules() -> set[str] | None: + """If possible, return all importable stdlib modules at runtime. + + This isn't so easy on Python <3.10; just return `None` on older versions to signal failure. + """ + if sys.version_info < (3, 10): + return None + modules: set[str] = set() + for module in sys.stdlib_module_names: + if module in ANNOYING_STDLIB_MODULES: + continue + try: + runtime = silent_import_module(module) + except ImportError: + continue + else: + modules.add(module) + try: + # some stdlib modules (e.g. `nt`) don't have __path__ set... + runtime_path = runtime.__path__ + runtime_name = runtime.__name__ + except AttributeError: + continue + else: + modules.update( + m.name for m in pkgutil.walk_packages(runtime_path, runtime_name + ".") + ) + return modules def get_allowlist_entries(allowlist_file: str) -> Iterator[str]: @@ -1695,6 +1725,10 @@ class _Arguments: version: str +# typeshed added a stub for __main__, but that causes stubtest to check itself +ANNOYING_STDLIB_MODULES: typing_extensions.Final = frozenset({"antigravity", "this", "__main__"}) + + def test_stubs(args: _Arguments, use_builtins_fixtures: bool = False) -> int: """This is stubtest! It's time to test the stubs!""" # Load the allowlist. This is a series of strings corresponding to Error.object_desc @@ -1717,10 +1751,9 @@ def test_stubs(args: _Arguments, use_builtins_fixtures: bool = False) -> int: "cannot pass both --check-typeshed and a list of modules", ) return 1 - modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir) - # typeshed added a stub for __main__, but that causes stubtest to check itself - annoying_modules = {"antigravity", "this", "__main__"} - modules = [m for m in modules if m not in annoying_modules] + typeshed_modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir) + runtime_modules = get_importable_stdlib_modules() or set() + modules = sorted((typeshed_modules | runtime_modules) - ANNOYING_STDLIB_MODULES) if not modules: print(_style("error:", color="red", bold=True), "no modules to check") From 820698d0c05af9a95581d44ec4cb165eb07c5546 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Wed, 23 Aug 2023 18:52:54 +0100 Subject: [PATCH 2/3] Improvements --- mypy/stubtest.py | 73 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index e9c2d35cae97..f664b17ce4af 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -11,6 +11,7 @@ import copy import enum import importlib +import importlib.machinery import inspect import os import pkgutil @@ -25,7 +26,7 @@ from contextlib import redirect_stderr, redirect_stdout from functools import singledispatch from pathlib import Path -from typing import Any, Generic, Iterator, TypeVar, Union +from typing import AbstractSet, Any, Generic, Iterator, TypeVar, Union from typing_extensions import get_origin, is_typeddict import mypy.build @@ -1671,34 +1672,64 @@ def exists_in_version(module: str) -> bool: return modules -def get_importable_stdlib_modules() -> set[str] | None: - """If possible, return all importable stdlib modules at runtime. +def get_importable_stdlib_modules() -> set[str]: + """Return all importable stdlib modules at runtime.""" + all_stdlib_modules: AbstractSet[str] + if sys.version_info >= (3, 10): + all_stdlib_modules = sys.stdlib_module_names + else: + python_exe_dir = Path(sys.executable).parent + all_stdlib_modules = { + m.name + for m in pkgutil.iter_modules() + if ( + isinstance(m.module_finder, importlib.machinery.FileFinder) + and python_exe_dir in Path(m.module_finder.path).parents + ) + } + all_stdlib_modules.update(sys.builtin_module_names) - This isn't so easy on Python <3.10; just return `None` on older versions to signal failure. - """ - if sys.version_info < (3, 10): - return None - modules: set[str] = set() - for module in sys.stdlib_module_names: - if module in ANNOYING_STDLIB_MODULES: + importable_stdlib_modules: set[str] = set() + for module_name in all_stdlib_modules: + if module_name in ANNOYING_STDLIB_MODULES: continue + try: - runtime = silent_import_module(module) + runtime = silent_import_module(module_name) except ImportError: continue else: - modules.add(module) + importable_stdlib_modules.add(module_name) + + try: + # some stdlib modules (e.g. `nt`) don't have __path__ set... + runtime_path = runtime.__path__ + runtime_name = runtime.__name__ + except AttributeError: + continue + + for submodule in pkgutil.walk_packages(runtime_path, runtime_name + "."): + submodule_name = submodule.name + + # There are many annoying *.__main__ stdlib modules, + # and including stubs for them isn't really that useful anyway: + # tkinter.__main__ opens a tkinter windows; unittest.__main__ raises SystemExit; etc. + # + # The idlelib.* submodules are similarly annoying in opening random tkinter windows, + # and we're unlikely to ever add stubs for idlelib in typeshed + # (see discussion in https://github.com/python/typeshed/pull/9193) + if submodule_name.endswith(".__main__") or submodule_name.startswith("idlelib."): + continue + try: - # some stdlib modules (e.g. `nt`) don't have __path__ set... - runtime_path = runtime.__path__ - runtime_name = runtime.__name__ - except AttributeError: + silent_import_module(submodule_name) + # importing multiprocessing.popen_forkserver on Windows raises AttributeError... + except Exception: continue else: - modules.update( - m.name for m in pkgutil.walk_packages(runtime_path, runtime_name + ".") - ) - return modules + importable_stdlib_modules.add(submodule_name) + + return importable_stdlib_modules def get_allowlist_entries(allowlist_file: str) -> Iterator[str]: @@ -1756,7 +1787,7 @@ def test_stubs(args: _Arguments, use_builtins_fixtures: bool = False) -> int: ) return 1 typeshed_modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir) - runtime_modules = get_importable_stdlib_modules() or set() + runtime_modules = get_importable_stdlib_modules() modules = sorted((typeshed_modules | runtime_modules) - ANNOYING_STDLIB_MODULES) if not modules: From 35be8475d614e44a7f245e699471ecf2dcb4c518 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Wed, 23 Aug 2023 19:26:07 +0100 Subject: [PATCH 3/3] Fix --- mypy/stubtest.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index f664b17ce4af..d8a613034b3a 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1678,16 +1678,17 @@ def get_importable_stdlib_modules() -> set[str]: if sys.version_info >= (3, 10): all_stdlib_modules = sys.stdlib_module_names else: + all_stdlib_modules = set(sys.builtin_module_names) python_exe_dir = Path(sys.executable).parent - all_stdlib_modules = { - m.name - for m in pkgutil.iter_modules() - if ( - isinstance(m.module_finder, importlib.machinery.FileFinder) - and python_exe_dir in Path(m.module_finder.path).parents - ) - } - all_stdlib_modules.update(sys.builtin_module_names) + for m in pkgutil.iter_modules(): + finder = m.module_finder + if isinstance(finder, importlib.machinery.FileFinder): + finder_path = Path(finder.path) + if ( + python_exe_dir in finder_path.parents + and "site-packages" not in finder_path.parts + ): + all_stdlib_modules.add(m.name) importable_stdlib_modules: set[str] = set() for module_name in all_stdlib_modules: