From 9ea524bd8863cca19accf1e3e5ae446054e7f981 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 13:40:48 +0100 Subject: [PATCH 01/16] Turn TextIOWrapper.__init__(buffer) into a protocol --- stdlib/io.pyi | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/stdlib/io.pyi b/stdlib/io.pyi index d949971048b0..68027e291fa7 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -6,7 +6,7 @@ from _typeshed import FileDescriptorOrPath, ReadableBuffer, WriteableBuffer from collections.abc import Callable, Iterable, Iterator from os import _Opener from types import TracebackType -from typing import IO, Any, BinaryIO, Literal, TextIO, TypeVar, overload +from typing import IO, Any, BinaryIO, Literal, Protocol, TextIO, TypeVar, overload, type_check_only from typing_extensions import Self __all__ = [ @@ -146,10 +146,29 @@ class TextIOBase(IOBase): def readlines(self, __hint: int = -1) -> list[str]: ... # type: ignore[override] def read(self, __size: int | None = ...) -> str: ... +@type_check_only +class _WrappedBuffer(Protocol): + name: str + closed: bool + def read(self, size: int = ...) -> bytes: ... + # Optional: def read1(size: int, /) -> bytes: ... + def write(self, b: bytes, /) -> object: ... + def flush(self) -> object: ... + def close(self) -> object: ... + def seekable() -> bool: ... + def readable() -> bool: ... + def writable() -> bool: ... + def truncate(size: int, /) -> int: ... + def fileno() -> int: ... + def isatty() -> int: ... + # Optional: Only needs to be present if seekable() returns True. + # def seek(self, offset: Literal[0], whence: Literal[2]) -> int: ... + # def tell(self) -> int: ... + class TextIOWrapper(TextIOBase, TextIO): # type: ignore[misc] # incompatible definitions of write in the base classes def __init__( self, - buffer: IO[bytes], + buffer: _WrappedBuffer, encoding: str | None = ..., errors: str | None = ..., newline: str | None = ..., @@ -180,7 +199,10 @@ class TextIOWrapper(TextIOBase, TextIO): # type: ignore[misc] # incompatible d def writelines(self, __lines: Iterable[str]) -> None: ... # type: ignore[override] def readline(self, __size: int = -1) -> str: ... # type: ignore[override] def readlines(self, __hint: int = -1) -> list[str]: ... # type: ignore[override] - def seek(self, __cookie: int, __whence: int = 0) -> int: ... # stubtest needs this + @overload + def seek(self, __cookie: int, __whence: Literal[0] = 0) -> int: ... + @overload + def seek(self, __cookie: Literal[0], __whence: Literal[1, 2]) -> int: ... class StringIO(TextIOWrapper): def __init__(self, initial_value: str | None = ..., newline: str | None = ...) -> None: ... From b2b7077a692c9528669c277ceea456eaee39ccc0 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 13:44:55 +0100 Subject: [PATCH 02/16] Add missing self arguments --- stdlib/io.pyi | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stdlib/io.pyi b/stdlib/io.pyi index 68027e291fa7..7fda5e96dce9 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -155,12 +155,12 @@ class _WrappedBuffer(Protocol): def write(self, b: bytes, /) -> object: ... def flush(self) -> object: ... def close(self) -> object: ... - def seekable() -> bool: ... - def readable() -> bool: ... - def writable() -> bool: ... - def truncate(size: int, /) -> int: ... - def fileno() -> int: ... - def isatty() -> int: ... + def seekable(self) -> bool: ... + def readable(self) -> bool: ... + def writable(self) -> bool: ... + def truncate(self, size: int, /) -> int: ... + def fileno(self) -> int: ... + def isatty(self) -> int: ... # Optional: Only needs to be present if seekable() returns True. # def seek(self, offset: Literal[0], whence: Literal[2]) -> int: ... # def tell(self) -> int: ... From 789b96540d47d5e2e876e080b541d646c7b74acc Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 13:45:30 +0100 Subject: [PATCH 03/16] Add a todo item --- stdlib/io.pyi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stdlib/io.pyi b/stdlib/io.pyi index 7fda5e96dce9..f4dfafe08983 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -165,6 +165,8 @@ class _WrappedBuffer(Protocol): # def seek(self, offset: Literal[0], whence: Literal[2]) -> int: ... # def tell(self) -> int: ... +# TODO: Should be generic over the buffer type, but needs to wait for +# TypeVar defaults. class TextIOWrapper(TextIOBase, TextIO): # type: ignore[misc] # incompatible definitions of write in the base classes def __init__( self, From fdb4aa7ea0b2c793594122e6ef2fca175717c957 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 13:48:19 +0100 Subject: [PATCH 04/16] seek() is less powerful than usual --- stdlib/io.pyi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stdlib/io.pyi b/stdlib/io.pyi index f4dfafe08983..b3673ec217b3 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -201,7 +201,9 @@ class TextIOWrapper(TextIOBase, TextIO): # type: ignore[misc] # incompatible d def writelines(self, __lines: Iterable[str]) -> None: ... # type: ignore[override] def readline(self, __size: int = -1) -> str: ... # type: ignore[override] def readlines(self, __hint: int = -1) -> list[str]: ... # type: ignore[override] - @overload + # TextIOWrapper's version of seek only supports a limited subset of + # operations. + @overload # type: ignore[override] def seek(self, __cookie: int, __whence: Literal[0] = 0) -> int: ... @overload def seek(self, __cookie: Literal[0], __whence: Literal[1, 2]) -> int: ... From 76509762f3ef556b4b34090a88652b8276f061cc Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 13:56:36 +0100 Subject: [PATCH 05/16] Add a test, use properties --- stdlib/io.pyi | 6 ++++-- test_cases/stdlib/check_io.py | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 test_cases/stdlib/check_io.py diff --git a/stdlib/io.pyi b/stdlib/io.pyi index b3673ec217b3..edae5f57521a 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -148,8 +148,10 @@ class TextIOBase(IOBase): @type_check_only class _WrappedBuffer(Protocol): - name: str - closed: bool + @property + def name(self) -> str: ... + @property + def closed(self) -> bool: ... def read(self, size: int = ...) -> bytes: ... # Optional: def read1(size: int, /) -> bytes: ... def write(self, b: bytes, /) -> object: ... diff --git a/test_cases/stdlib/check_io.py b/test_cases/stdlib/check_io.py new file mode 100644 index 000000000000..cda3e808b41b --- /dev/null +++ b/test_cases/stdlib/check_io.py @@ -0,0 +1,4 @@ +from gzip import GzipFile +from io import TextIOWrapper + +TextIOWrapper(GzipFile("")) From 6768aa3326c01ed01a414c6cc9f88c0dd0a54d5e Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 13:58:47 +0100 Subject: [PATCH 06/16] Use more standard seek definition --- stdlib/io.pyi | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/stdlib/io.pyi b/stdlib/io.pyi index edae5f57521a..275533f21919 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -205,10 +205,7 @@ class TextIOWrapper(TextIOBase, TextIO): # type: ignore[misc] # incompatible d def readlines(self, __hint: int = -1) -> list[str]: ... # type: ignore[override] # TextIOWrapper's version of seek only supports a limited subset of # operations. - @overload # type: ignore[override] - def seek(self, __cookie: int, __whence: Literal[0] = 0) -> int: ... - @overload - def seek(self, __cookie: Literal[0], __whence: Literal[1, 2]) -> int: ... + def seek(self, __cookie: int, __whence: Literal[0, 1, 2] = 0) -> int: ... class StringIO(TextIOWrapper): def __init__(self, initial_value: str | None = ..., newline: str | None = ...) -> None: ... From fb3ab4e32346e6c89fae571064bb882100941344 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 14:00:55 +0100 Subject: [PATCH 07/16] Use int for whence for now --- stdlib/io.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/io.pyi b/stdlib/io.pyi index 275533f21919..306d73ccbf0c 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -205,7 +205,7 @@ class TextIOWrapper(TextIOBase, TextIO): # type: ignore[misc] # incompatible d def readlines(self, __hint: int = -1) -> list[str]: ... # type: ignore[override] # TextIOWrapper's version of seek only supports a limited subset of # operations. - def seek(self, __cookie: int, __whence: Literal[0, 1, 2] = 0) -> int: ... + def seek(self, __cookie: int, __whence: int = 0) -> int: ... class StringIO(TextIOWrapper): def __init__(self, initial_value: str | None = ..., newline: str | None = ...) -> None: ... From 4325219607cf4187792b12b0abe290950bcb2c2e Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 14:08:09 +0100 Subject: [PATCH 08/16] Add test case __init__.py files to suppress mypy error --- test_cases/stdlib/asyncio/__init__.py | 0 test_cases/stdlib/builtins/__init__.py | 0 test_cases/stdlib/collections/__init__.py | 0 test_cases/stdlib/itertools/__init__.py | 0 test_cases/stdlib/typing/__init__.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 test_cases/stdlib/asyncio/__init__.py create mode 100644 test_cases/stdlib/builtins/__init__.py create mode 100644 test_cases/stdlib/collections/__init__.py create mode 100644 test_cases/stdlib/itertools/__init__.py create mode 100644 test_cases/stdlib/typing/__init__.py diff --git a/test_cases/stdlib/asyncio/__init__.py b/test_cases/stdlib/asyncio/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test_cases/stdlib/builtins/__init__.py b/test_cases/stdlib/builtins/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test_cases/stdlib/collections/__init__.py b/test_cases/stdlib/collections/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test_cases/stdlib/itertools/__init__.py b/test_cases/stdlib/itertools/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test_cases/stdlib/typing/__init__.py b/test_cases/stdlib/typing/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 From 45478e2129915bff5d4b7501e5146f1646b73192 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 14:11:57 +0100 Subject: [PATCH 09/16] Allow __init__.py files in test case directories --- tests/check_consistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 1fc40db576c9..72eec2a764b8 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -82,7 +82,7 @@ def check_test_cases() -> None: assert_consistent_filetypes(testcase_dir, kind=".py", allowed={"README.md"}, allow_nonidentifier_filenames=True) bad_test_case_filename = 'Files in a `test_cases` directory must have names starting with "check_"; got "{}"' for file in testcase_dir.rglob("*.py"): - assert file.stem.startswith("check_"), bad_test_case_filename.format(file) + assert file.name == "__init__.py" or file.stem.startswith("check_"), bad_test_case_filename.format(file) def check_no_symlinks() -> None: From ffbcde3142bf9f8975fb136967fc1974eb2d0c08 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 14:28:20 +0100 Subject: [PATCH 10/16] Work around regression test problems in a different way --- test_cases/stdlib/asyncio/__init__.py | 0 test_cases/stdlib/builtins/__init__.py | 0 test_cases/stdlib/collections/__init__.py | 0 test_cases/stdlib/itertools/__init__.py | 0 test_cases/stdlib/typing/__init__.py | 0 test_cases/stdlib/typing/{check_io.py => check_typing_io.py} | 0 tests/check_consistent.py | 2 +- 7 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 test_cases/stdlib/asyncio/__init__.py delete mode 100644 test_cases/stdlib/builtins/__init__.py delete mode 100644 test_cases/stdlib/collections/__init__.py delete mode 100644 test_cases/stdlib/itertools/__init__.py delete mode 100644 test_cases/stdlib/typing/__init__.py rename test_cases/stdlib/typing/{check_io.py => check_typing_io.py} (100%) diff --git a/test_cases/stdlib/asyncio/__init__.py b/test_cases/stdlib/asyncio/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/test_cases/stdlib/builtins/__init__.py b/test_cases/stdlib/builtins/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/test_cases/stdlib/collections/__init__.py b/test_cases/stdlib/collections/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/test_cases/stdlib/itertools/__init__.py b/test_cases/stdlib/itertools/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/test_cases/stdlib/typing/__init__.py b/test_cases/stdlib/typing/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/test_cases/stdlib/typing/check_io.py b/test_cases/stdlib/typing/check_typing_io.py similarity index 100% rename from test_cases/stdlib/typing/check_io.py rename to test_cases/stdlib/typing/check_typing_io.py diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 72eec2a764b8..1fc40db576c9 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -82,7 +82,7 @@ def check_test_cases() -> None: assert_consistent_filetypes(testcase_dir, kind=".py", allowed={"README.md"}, allow_nonidentifier_filenames=True) bad_test_case_filename = 'Files in a `test_cases` directory must have names starting with "check_"; got "{}"' for file in testcase_dir.rglob("*.py"): - assert file.name == "__init__.py" or file.stem.startswith("check_"), bad_test_case_filename.format(file) + assert file.stem.startswith("check_"), bad_test_case_filename.format(file) def check_no_symlinks() -> None: From ad8461f2b51f95efe5a3a2c125d707bb89a18e6c Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 14:31:27 +0100 Subject: [PATCH 11/16] Make FileIO compatible --- stdlib/io.pyi | 5 ++++- test_cases/stdlib/check_io.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/stdlib/io.pyi b/stdlib/io.pyi index 306d73ccbf0c..b89e53e600e8 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -94,7 +94,10 @@ class BufferedIOBase(IOBase): class FileIO(RawIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of writelines in the base classes mode: str - name: FileDescriptorOrPath + # The type of "name" equals the argument passed in to the constructor, + # but that can make FileIO incompatible with other I/O types that assume + # "name" is a str. In the future, making FileIO generic might help. + name: Any def __init__( self, file: FileDescriptorOrPath, mode: str = ..., closefd: bool = ..., opener: _Opener | None = ... ) -> None: ... diff --git a/test_cases/stdlib/check_io.py b/test_cases/stdlib/check_io.py index cda3e808b41b..abf84dd5a103 100644 --- a/test_cases/stdlib/check_io.py +++ b/test_cases/stdlib/check_io.py @@ -1,4 +1,6 @@ from gzip import GzipFile -from io import TextIOWrapper +from io import FileIO, TextIOWrapper +TextIOWrapper(FileIO("")) +TextIOWrapper(FileIO(13)) TextIOWrapper(GzipFile("")) From 2580cb791c2c1984559cf66894e919969870a239 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 15:10:09 +0100 Subject: [PATCH 12/16] Apply suggestions from code review Co-authored-by: Alex Waygood --- stdlib/io.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/io.pyi b/stdlib/io.pyi index b89e53e600e8..b32e5958f997 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -155,8 +155,8 @@ class _WrappedBuffer(Protocol): def name(self) -> str: ... @property def closed(self) -> bool: ... - def read(self, size: int = ...) -> bytes: ... - # Optional: def read1(size: int, /) -> bytes: ... + def read(self, size: int = ..., /) -> bytes: ... + # Optional: def read1(self, size: int, /) -> bytes: ... def write(self, b: bytes, /) -> object: ... def flush(self) -> object: ... def close(self) -> object: ... From 36226c97c7664e2b3a9297277f7cccce569900b8 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 15:49:07 +0100 Subject: [PATCH 13/16] name -> Any --- stdlib/io.pyi | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stdlib/io.pyi b/stdlib/io.pyi index b32e5958f997..cd32310c4f88 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -151,8 +151,11 @@ class TextIOBase(IOBase): @type_check_only class _WrappedBuffer(Protocol): + # "name" is wrapped by TextIOWrapper. Its type is inconsistent between + # the various I/O types, see the comments on TextIOWrapper.name and + # TextIO.name. @property - def name(self) -> str: ... + def name(self) -> Any: ... @property def closed(self) -> bool: ... def read(self, size: int = ..., /) -> bytes: ... From de9c18d95ab8c7274b4c498e8513a850430cdad5 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 16:06:23 +0100 Subject: [PATCH 14/16] Add a comment to the buffer attribute --- stdlib/io.pyi | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stdlib/io.pyi b/stdlib/io.pyi index cd32310c4f88..0f749514bbb0 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -185,6 +185,9 @@ class TextIOWrapper(TextIOBase, TextIO): # type: ignore[misc] # incompatible d line_buffering: bool = ..., write_through: bool = ..., ) -> None: ... + # Equals the "buffer" argument passed in to the constructor. + # TODO: Make TextIOWrapper generic over the buffer argument once + # PEP 696 get accepted. @property def buffer(self) -> BinaryIO: ... @property From 4c12b2db6ac271ebdf64bf07263f7bdf854c5fc2 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 16:35:46 +0100 Subject: [PATCH 15/16] Use ReadableBuffer --- stdlib/io.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/io.pyi b/stdlib/io.pyi index 0f749514bbb0..8cf1e3a69da9 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -158,8 +158,8 @@ class _WrappedBuffer(Protocol): def name(self) -> Any: ... @property def closed(self) -> bool: ... - def read(self, size: int = ..., /) -> bytes: ... - # Optional: def read1(self, size: int, /) -> bytes: ... + def read(self, size: int = ..., /) -> ReadableBuffer: ... + # Optional: def read1(self, size: int, /) -> ReadableBuffer: ... def write(self, b: bytes, /) -> object: ... def flush(self) -> object: ... def close(self) -> object: ... From 7c9d035616f5464c2b0a950fbebd5ba4cc78ccc4 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 14 Feb 2024 17:48:26 +0100 Subject: [PATCH 16/16] Add detach() --- stdlib/io.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/io.pyi b/stdlib/io.pyi index 8cf1e3a69da9..659b216c43dc 100644 --- a/stdlib/io.pyi +++ b/stdlib/io.pyi @@ -186,8 +186,6 @@ class TextIOWrapper(TextIOBase, TextIO): # type: ignore[misc] # incompatible d write_through: bool = ..., ) -> None: ... # Equals the "buffer" argument passed in to the constructor. - # TODO: Make TextIOWrapper generic over the buffer argument once - # PEP 696 get accepted. @property def buffer(self) -> BinaryIO: ... @property @@ -212,6 +210,8 @@ class TextIOWrapper(TextIOBase, TextIO): # type: ignore[misc] # incompatible d def writelines(self, __lines: Iterable[str]) -> None: ... # type: ignore[override] def readline(self, __size: int = -1) -> str: ... # type: ignore[override] def readlines(self, __hint: int = -1) -> list[str]: ... # type: ignore[override] + # Equals the "buffer" argument passed in to the constructor. + def detach(self) -> BinaryIO: ... # TextIOWrapper's version of seek only supports a limited subset of # operations. def seek(self, __cookie: int, __whence: int = 0) -> int: ...