Add infrastructure allowing for test cases for third-party stubs

Author: AlexWaygood

.github/workflows/tests.yml

@@ -9,6 +9,7 @@ on:
   pull_request:
     paths-ignore:
       - '**/*.md'
+      - 'scripts/**'
 
 permissions:
   contents: read
@@ -65,7 +66,7 @@ jobs:
       - run: ./tests/pytype_test.py --print-stderr
 
   mypy:
-    name: Test the stubs with mypy
+    name: Run mypy against the stubs
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -80,6 +81,17 @@ jobs:
       - run: pip install -r requirements-tests.txt
       - run: ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }}
 
+  regression-tests:
+    name: Run mypy on the test cases
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - run: pip install -r requirements-tests.txt
+      - run: python ./tests/regr_test.py --all
+
   pyright:
     name: Test the stubs with pyright
     runs-on: ubuntu-latest

stubs/requests/@tests/test_cases/check_post.py

@@ -0,0 +1,13 @@
+# pyright: reportUnnecessaryTypeIgnoreComment=true
+
+import requests
+
+# Regression test for #7988 (multiple files should be allowed for the "files" argument)
+# This snippet comes from the requests documentation (https://requests.readthedocs.io/en/latest/user/advanced/#post-multiple-multipart-encoded-files),
+# so should pass a type checker without error
+url = "https://httpbin.org/post"
+multiple_files = [
+    ("images", ("foo.png", open("foo.png", "rb"), "image/png")),
+    ("images", ("bar.png", open("bar.png", "rb"), "image/png")),
+]
+r = requests.post(url, files=multiple_files)

test_cases/README.md

@@ -1,14 +1,22 @@
 ## Regression tests for typeshed
 
 This directory contains code samples that act as a regression test for the
-standard library stubs found elsewhere in the typeshed repo.
+typeshed's stdlib stubs.
 
 **This directory should *only* contain test cases for functions and classes which
 are known to have caused problems in the past, where the stubs are difficult to
 get right.** 100% test coverage for typeshed is neither necessary nor
 desirable, as it would lead to code duplication. Moreover, typeshed has
 multiple other mechanisms for spotting errors in the stubs.
 
+### Where are the third-party test cases?
+
+Not all third-party stubs packages in typeshed have test cases, and not all of
+them need test cases. For those that do have test cases, however, the samples
+can be found in `@tests/test_cases` subdirectories for each stubs package. For
+example, the test cases for `requests` can be found in the
+`stubs/requests/@tests/test_cases` directory.
+
 ### The purpose of these tests
 
 Different test cases in this directory serve different purposes. For some stubs in

tests/README.md

@@ -5,6 +5,8 @@ tests the stubs with [mypy](https://github.com/python/mypy/)
 [pytype](https://github.com/google/pytype/).
 - `tests/pyright_test.py` tests the stubs with
 [pyright](https://github.com/microsoft/pyright).
+- `tests/regr_test.py` runs mypy against the test cases for typeshed's
+stubs, guarding against accidental regressions.
 - `tests/check_consistent.py` checks certain files in typeshed remain
 consistent with each other.
 - `tests/stubtest_stdlib.py` checks standard library stubs against the
@@ -24,17 +26,15 @@ Run using:
 (.venv3)$ python3 tests/mypy_test.py
 ```
 
-The test has three parts. Each part uses mypy with slightly different configuration options:
-- Running mypy on the stdlib stubs
-- Running mypy on the third-party stubs
-- Running mypy `--strict` on the regression tests in the `test_cases` directory.
+The test has two parts: running mypy on the stdlib stubs,
+and running mypy on the third-party stubs.
 
-When running mypy on the stubs, this test is shallow — it verifies that all stubs can be
+This test is shallow — it verifies that all stubs can be
 imported but doesn't check whether stubs match their implementation
 (in the Python standard library or a third-party package).
 
 Run `python tests/mypy_test.py --help` for information on the various configuration options
-for this test script.
+for this script.
 
 ## pytype\_test.py
 
@@ -64,6 +64,13 @@ checks that would typically fail on incomplete stubs (such as `Unknown` checks).
 In typeshed's CI, pyright is run with these configuration settings on a subset of
 the stubs in typeshed (including the standard library).
 
+## regr\_test.py
+
+This test runs mypy against the test cases for typeshed's stdlib and third-party
+stubs. See the README in the `test_cases` directory for more information about what
+these test cases are for and how they work. Run `python tests/regr_test.py --help`
+for information on the various configuration options.
+
 ## check\_consistent.py
 
 Run using:

tests/check_consistent.py

@@ -15,6 +15,7 @@
 from packaging.requirements import Requirement
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version
+from utils import get_all_testcase_directories
 
 metadata_keys = {"version", "requires", "extra_description", "obsolete_since", "no_longer_updated", "tool"}
 tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub"}}
@@ -58,10 +59,21 @@ def check_stubs() -> None:
 
 
 def check_test_cases() -> None:
-    assert_consistent_filetypes(Path("test_cases"), kind=".py", allowed={"README.md"})
-    bad_test_case_filename = 'Files in the `test_cases` directory must have names starting with "check_"; got "{}"'
-    for file in Path("test_cases").rglob("*.py"):
-        assert file.stem.startswith("check_"), bad_test_case_filename.format(file)
+    for package_name, testcase_dir in get_all_testcase_directories():
+        assert_consistent_filetypes(testcase_dir, kind=".py", allowed={"README.md"})
+        bad_test_case_filename = 'Files in a `test_cases` directory must have names starting with "check_"; got "{}"'
+        for file in testcase_dir.rglob("*.py"):
+            assert file.stem.startswith("check_"), bad_test_case_filename.format(file)
+            if package_name != "stdlib":
+                with open(file) as f:
+                    lines = [line.strip() for line in f]
+                pyright_setting_not_enabled_msg = (
+                    f'Third-party test-case file "{file}" must have '
+                    f'"# pyright: reportUnnecessaryTypeIgnoreComment=true" '
+                    f"at the top of the file"
+                )
+                has_pyright_setting_enabled = any(line == "# pyright: reportUnnecessaryTypeIgnoreComment=true" for line in lines)
+                assert has_pyright_setting_enabled, pyright_setting_not_enabled_msg
 
 
 def check_no_symlinks() -> None:

tests/colors.py

@@ -1,18 +1,13 @@
 #!/usr/bin/env python3
-"""Run mypy on various typeshed directories, with varying command-line arguments.
+"""Run mypy on typeshed's stdlib and third-party stubs."""
 
-Depends on mypy being installed.
-"""
 from __future__ import annotations
 
 import argparse
 import os
 import re
-import shutil
-import subprocess
 import sys
 import tempfile
-from collections.abc import Iterable
 from contextlib import redirect_stderr, redirect_stdout
 from dataclasses import dataclass
 from io import StringIO
@@ -26,11 +21,11 @@
 from typing_extensions import Annotated, TypeAlias
 
 import tomli
-from colors import colored, print_error, print_success_msg
+from utils import colored, print_error, print_success_msg, read_dependencies
 
 SUPPORTED_VERSIONS = [(3, 11), (3, 10), (3, 9), (3, 8), (3, 7)]
 SUPPORTED_PLATFORMS = ("linux", "win32", "darwin")
-TYPESHED_DIRECTORIES = frozenset({"stdlib", "stubs", "test_cases"})
+TYPESHED_DIRECTORIES = frozenset({"stdlib", "stubs"})
 
 ReturnCode: TypeAlias = int
 MajorVersion: TypeAlias = int
@@ -58,7 +53,9 @@ class CommandLineArgs(argparse.Namespace):
     filter: list[str]
 
 
-parser = argparse.ArgumentParser(description="Test runner for typeshed. Patterns are unanchored regexps on the full path.")
+parser = argparse.ArgumentParser(
+    description="Typecheck typeshed's stubs with mypy. Patterns are unanchored regexps on the full path."
+)
 parser.add_argument("-v", "--verbose", action="count", default=0, help="More output")
 parser.add_argument("-x", "--exclude", type=str, nargs="*", help="Exclude pattern")
 parser.add_argument(
@@ -239,20 +236,8 @@ def run_mypy(args: TestConfig, configurations: list[MypyDistConf], files: list[P
         return exit_code
 
 
-def run_mypy_as_subprocess(directory: StrPath, flags: Iterable[str]) -> ReturnCode:
-    result = subprocess.run([sys.executable, "-m", "mypy", directory, *flags], capture_output=True)
-    stdout, stderr = result.stdout, result.stderr
-    if stderr:
-        print_error(stderr.decode())
-    if stdout:
-        print_error(stdout.decode())
-    return result.returncode
-
-
-def get_mypy_flags(
-    args: TestConfig, temp_name: str | None, *, strict: bool = False, enforce_error_codes: bool = True
-) -> list[str]:
-    flags = [
+def get_mypy_flags(args: TestConfig, temp_name: str) -> list[str]:
+    return [
         "--python-version",
         f"{args.major}.{args.minor}",
         "--show-traceback",
@@ -264,29 +249,15 @@ def get_mypy_flags(
         "--no-site-packages",
         "--custom-typeshed-dir",
         str(Path(__file__).parent.parent),
+        "--no-implicit-optional",
+        "--disallow-untyped-decorators",
+        "--disallow-any-generics",
+        "--strict-equality",
+        "--enable-error-code",
+        "ignore-without-code",
+        "--config-file",
+        temp_name,
     ]
-    if strict:
-        flags.append("--strict")
-    else:
-        flags.extend(["--no-implicit-optional", "--disallow-untyped-decorators", "--disallow-any-generics", "--strict-equality"])
-    if temp_name is not None:
-        flags.extend(["--config-file", temp_name])
-    if enforce_error_codes:
-        flags.extend(["--enable-error-code", "ignore-without-code"])
-    return flags
-
-
-def read_dependencies(distribution: str) -> list[str]:
-    with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
-        data = tomli.load(f)
-    requires = data.get("requires", [])
-    assert isinstance(requires, list)
-    dependencies = []
-    for dependency in requires:
-        assert isinstance(dependency, str)
-        assert dependency.startswith("types-")
-        dependencies.append(dependency[6:].split("<")[0])
-    return dependencies
 
 
 def add_third_party_files(
@@ -382,23 +353,6 @@ def test_third_party_stubs(code: int, args: TestConfig) -> TestResults:
     return TestResults(code, files_checked)
 
 
-def test_the_test_cases(code: int, args: TestConfig) -> TestResults:
-    test_case_files = list(map(str, Path("test_cases").rglob("*.py")))
-    num_test_case_files = len(test_case_files)
-    flags = get_mypy_flags(args, None, strict=True, enforce_error_codes=False)
-    print(f"Running mypy on the test_cases directory ({num_test_case_files} files)...")
-    print("Running mypy " + " ".join(flags))
-    # --warn-unused-ignores doesn't work for files inside typeshed.
-    # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory.
-    with tempfile.TemporaryDirectory() as td:
-        shutil.copytree(Path("test_cases"), Path(td) / "test_cases")
-        this_code = run_mypy_as_subprocess(td, flags)
-    if not this_code:
-        print_success_msg()
-    code = max(code, this_code)
-    return TestResults(code, num_test_case_files)
-
-
 def test_typeshed(code: int, args: TestConfig) -> TestResults:
     print(f"*** Testing Python {args.major}.{args.minor} on {args.platform}")
     files_checked_this_version = 0
@@ -412,11 +366,6 @@ def test_typeshed(code: int, args: TestConfig) -> TestResults:
         files_checked_this_version += third_party_files_checked
         print()
 
-    if "test_cases" in args.directories:
-        code, test_case_files_checked = test_the_test_cases(code, args)
-        files_checked_this_version += test_case_files_checked
-        print()
-
     return TestResults(code, files_checked_this_version)