Further enhance check_consistent.py (#8604)

Author: AlexWaygood

requirements-tests.txt

@@ -14,4 +14,6 @@ tomlkit==0.11.4
 # must match .pre-commit-config.yaml
 pycln==2.1.1
 packaging==21.3
+pyyaml==6.0
 termcolor
+types-pyyaml

tests/check_consistent.py

@@ -11,15 +11,18 @@
 from pathlib import Path
 
 import tomli
+import yaml
 from packaging.requirements import Requirement
+from packaging.specifiers import SpecifierSet
 from packaging.version import Version
 
 metadata_keys = {"version", "requires", "extra_description", "obsolete_since", "no_longer_updated", "tool"}
 tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub"}}
+extension_descriptions = {".pyi": "stub", ".py": ".py"}
 
 
-def assert_stubs_only(directory: Path, allowed: set[str]) -> None:
-    """Check that given directory contains only valid stub files."""
+def assert_consistent_filetypes(directory: Path, *, kind: str, allowed: set[str]) -> None:
+    """Check that given directory contains only valid Python files of a certain kind."""
     allowed_paths = {Path(f) for f in allowed}
     contents = list(directory.iterdir())
     while contents:
@@ -28,15 +31,16 @@ def assert_stubs_only(directory: Path, allowed: set[str]) -> None:
             # Note if a subdirectory is allowed, we will not check its contents
             continue
         if entry.is_file():
-            assert entry.stem.isidentifier(), f"Files must be valid modules, got: {entry}"
-            assert entry.suffix == ".pyi", f"Only stub files allowed, got: {entry}"
+            assert entry.stem.isidentifier(), f'Files must be valid modules, got: "{entry}"'
+            bad_filetype = f'Only {extension_descriptions[kind]!r} files allowed in the "{directory}" directory; got: {entry}'
+            assert entry.suffix == kind, bad_filetype
         else:
             assert entry.name.isidentifier(), f"Directories must be valid packages, got: {entry}"
             contents.extend(entry.iterdir())
 
 
 def check_stdlib() -> None:
-    assert_stubs_only(Path("stdlib"), allowed={"_typeshed/README.md", "VERSIONS"})
+    assert_consistent_filetypes(Path("stdlib"), kind=".pyi", allowed={"_typeshed/README.md", "VERSIONS"})
 
 
 def check_stubs() -> None:
@@ -46,14 +50,21 @@ def check_stubs() -> None:
         valid_dist_name = "^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$"  # courtesy of PEP 426
         assert re.fullmatch(
             valid_dist_name, dist.name, re.IGNORECASE
-        ), f"Directory name must have valid distribution name: {dist}"
+        ), f"Directory name must be a valid distribution name: {dist}"
         assert not dist.name.startswith("types-"), f"Directory name not allowed to start with 'types-': {dist}"
 
         allowed = {"METADATA.toml", "README", "README.md", "README.rst", "@tests"}
-        assert_stubs_only(dist, allowed)
+        assert_consistent_filetypes(dist, kind=".pyi", allowed=allowed)
 
 
-def check_same_files() -> None:
+def check_test_cases() -> None:
+    assert_consistent_filetypes(Path("test_cases"), kind=".py", allowed={"README.md"})
+    bad_test_case_filename = 'Files in the `test_cases` directory must have names starting with "test_"; got "{}"'
+    for file in Path("test_cases").rglob("*.py"):
+        assert file.stem.startswith("test_"), bad_test_case_filename.format(file)
+
+
+def check_no_symlinks() -> None:
     files = [os.path.join(root, file) for root, _, files in os.walk(".") for file in files]
     no_symlink = "You cannot use symlinks in typeshed, please copy {} to its link."
     for file in files:
@@ -65,12 +76,16 @@ def check_same_files() -> None:
 _VERSIONS_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): [23]\.\d{1,2}-(?:[23]\.\d{1,2})?$")
 
 
+def strip_comments(text: str) -> str:
+    return text.split("#")[0].strip()
+
+
 def check_versions() -> None:
     versions = set()
     with open("stdlib/VERSIONS") as f:
         data = f.read().splitlines()
     for line in data:
-        line = line.split("#")[0].strip()
+        line = strip_comments(line)
         if line == "":
             continue
         m = _VERSIONS_RE.match(line)
@@ -126,10 +141,48 @@ def check_metadata() -> None:
                 assert key in tk, f"Unrecognised {tool} key {key} for {distribution}"
 
 
+def get_txt_requirements() -> dict[str, SpecifierSet]:
+    with open("requirements-tests.txt") as requirements_file:
+        stripped_lines = map(strip_comments, requirements_file)
+        requirements = map(Requirement, filter(None, stripped_lines))
+        return {requirement.name: requirement.specifier for requirement in requirements}
+
+
+def get_precommit_requirements() -> dict[str, SpecifierSet]:
+    with open(".pre-commit-config.yaml") as precommit_file:
+        precommit = precommit_file.read()
+    yam = yaml.load(precommit, Loader=yaml.Loader)
+    precommit_requirements = {}
+    for repo in yam["repos"]:
+        hook = repo["hooks"][0]
+        package_name, package_rev = hook["id"], repo["rev"]
+        package_specifier = SpecifierSet(f"=={package_rev.removeprefix('v')}")
+        precommit_requirements[package_name] = package_specifier
+        for additional_req in hook.get("additional_dependencies", []):
+            req = Requirement(additional_req)
+            precommit_requirements[req.name] = req.specifier
+    return precommit_requirements
+
+
+def check_requirements() -> None:
+    requirements_txt_requirements = get_txt_requirements()
+    precommit_requirements = get_precommit_requirements()
+    no_txt_entry_msg = "All pre-commit requirements must also be listed in `requirements-tests.txt` (missing {requirement!r})"
+    for requirement, specifier in precommit_requirements.items():
+        assert requirement in requirements_txt_requirements, no_txt_entry_msg.format(requirement)
+        specifier_mismatch = (
+            f'Specifier "{specifier}" for {requirement!r} in `.pre-commit-config.yaml` '
+            f'does not match specifier "{requirements_txt_requirements[requirement]}" in `requirements-tests.txt`'
+        )
+        assert specifier == requirements_txt_requirements[requirement], specifier_mismatch
+
+
 if __name__ == "__main__":
     assert sys.version_info >= (3, 9), "Python 3.9+ is required to run this test"
     check_stdlib()
     check_versions()
     check_stubs()
     check_metadata()
-    check_same_files()
+    check_no_symlinks()
+    check_test_cases()
+    check_requirements()