open: introduce concrete return types (#4146)

* make io classes inherit from typing IO classes

This makes these classes usable if type annotations are given as "IO"
or "TextIO". In the future, we'll then be able to move open() to
return a concrete class instead (#3951).

* open: introduce concrete return types

Fixes #3951.

We use the values of the "mode" and "buffering" arguments to figure out
the concrete type open() will return at runtime. (Compare the CPython
code in https://github.com/python/cpython/blob/master/Modules/_io/_iomodule.c#L231.)

Author: JelleZijlstra

stdlib/2/__builtin__.pyi

@@ -5,13 +5,16 @@ from typing import (
     TypeVar, Iterator, Iterable, NoReturn, overload, Container,
     Sequence, MutableSequence, Mapping, MutableMapping, Tuple, List, Any, Dict, Callable, Generic,
     Set, AbstractSet, FrozenSet, MutableSet, Sized, Reversible, SupportsInt, SupportsFloat, SupportsAbs,
-    SupportsComplex, IO, BinaryIO, TextIO, Union,
+    SupportsComplex, IO, BinaryIO, Union,
     ItemsView, KeysView, ValuesView, ByteString, Optional, AnyStr, Type, Text,
     Protocol,
 )
-from abc import abstractmethod, ABCMeta
+from abc import ABCMeta
 from ast import mod, AST
-from io import _OpenBinaryMode, _OpenTextMode
+from io import (
+    _OpenBinaryMode, _OpenTextMode, _OpenBinaryModeUpdating, _OpenBinaryModeWriting, _OpenBinaryModeReading,
+    TextIOWrapper, FileIO, BufferedRandom, BufferedReader, BufferedWriter
+)
 from types import TracebackType, CodeType
 import sys
 
@@ -1364,7 +1367,9 @@ if sys.version_info >= (3,):
         _OpenFile = Union[str, bytes, int, _PathLike[Any]]
     else:
         _OpenFile = Union[str, bytes, int]
+    _Opener = Callable[[str, int], int]
 
+    # Text mode: always returns a TextIOWrapper
     @overload
     def open(
         file: _OpenFile,
@@ -1374,19 +1379,71 @@ if sys.version_info >= (3,):
         errors: Optional[str] = ...,
         newline: Optional[str] = ...,
         closefd: bool = ...,
-        opener: Optional[Callable[[str, int], int]] = ...,
-    ) -> TextIO: ...
+        opener: Optional[_Opener] = ...,
+    ) -> TextIOWrapper: ...
+
+    # Unbuffered binary mode: returns a FileIO
     @overload
     def open(
         file: _OpenFile,
         mode: _OpenBinaryMode,
-        buffering: int = ...,
+        buffering: Literal[0],
+        encoding: None = ...,
+        errors: None = ...,
+        newline: None = ...,
+        closefd: bool = ...,
+        opener: Optional[_Opener] = ...,
+    ) -> FileIO: ...
+
+    # Buffering is on: return BufferedRandom, BufferedReader, or BufferedWriter
+    @overload
+    def open(
+        file: _OpenFile,
+        mode: _OpenBinaryModeUpdating,
+        buffering: Literal[-1, 1] = ...,
+        encoding: None = ...,
+        errors: None = ...,
+        newline: None = ...,
+        closefd: bool = ...,
+        opener: Optional[_Opener] = ...,
+    ) -> BufferedRandom: ...
+    @overload
+    def open(
+        file: _OpenFile,
+        mode: _OpenBinaryModeWriting,
+        buffering: Literal[-1, 1] = ...,
         encoding: None = ...,
         errors: None = ...,
         newline: None = ...,
         closefd: bool = ...,
-        opener: Optional[Callable[[str, int], int]] = ...,
+        opener: Optional[_Opener] = ...,
+    ) -> BufferedWriter: ...
+    @overload
+    def open(
+        file: _OpenFile,
+        mode: _OpenBinaryModeReading,
+        buffering: Literal[-1, 1] = ...,
+        encoding: None = ...,
+        errors: None = ...,
+        newline: None = ...,
+        closefd: bool = ...,
+        opener: Optional[_Opener] = ...,
+    ) -> BufferedReader: ...
+
+    # Buffering cannot be determined: fall back to BinaryIO
+    @overload
+    def open(
+        file: _OpenFile,
+        mode: _OpenBinaryMode,
+        buffering: int,
+        encoding: None = ...,
+        errors: None = ...,
+        newline: None = ...,
+        closefd: bool = ...,
+        opener: Optional[_Opener] = ...,
     ) -> BinaryIO: ...
+
+    # Fallback if mode is not specified
     @overload
     def open(
         file: _OpenFile,
@@ -1396,7 +1453,7 @@ if sys.version_info >= (3,):
         errors: Optional[str] = ...,
         newline: Optional[str] = ...,
         closefd: bool = ...,
-        opener: Optional[Callable[[str, int], int]] = ...,
+        opener: Optional[_Opener] = ...,
     ) -> IO[Any]: ...
 
 else:

stdlib/2/io.pyi

@@ -28,12 +28,20 @@ _OpenTextMode = Literal[
     'a', 'a+', '+a', 'at', 'ta', 'at+', 'a+t', '+at', 'ta+', 't+a', '+ta',
     'U', 'rU', 'Ur', 'rtU', 'rUt', 'Urt', 'trU', 'tUr', 'Utr',
 ]
-_OpenBinaryMode = Literal[
-    'rb', 'br', 'rb+', 'r+b', '+rb', 'br+', 'b+r', '+br',
-    'wb', 'bw', 'wb+', 'w+b', '+wb', 'bw+', 'b+w', '+bw',
-    'ab', 'ba', 'ab+', 'a+b', '+ab', 'ba+', 'b+a', '+ba',
+_OpenBinaryModeUpdating = Literal[
+    'rb+', 'r+b', '+rb', 'br+', 'b+r', '+br',
+    'wb+', 'w+b', '+wb', 'bw+', 'b+w', '+bw',
+    'ab+', 'a+b', '+ab', 'ba+', 'b+a', '+ba',
+]
+_OpenBinaryModeWriting = Literal[
+    'wb', 'bw',
+    'ab', 'ba',
+]
+_OpenBinaryModeReading = Literal[
+    'rb', 'br',
     'rbU', 'rUb', 'Urb', 'brU', 'bUr', 'Ubr',
 ]
+_OpenBinaryMode = Union[_OpenBinaryModeUpdating, _OpenBinaryModeReading, _OpenBinaryModeWriting]
 
 def _OpenWrapper(file: Union[str, unicode, int],
                  mode: unicode = ..., buffering: int = ..., encoding: unicode = ...,

stdlib/2and3/builtins.pyi

@@ -5,13 +5,16 @@ from typing import (
     TypeVar, Iterator, Iterable, NoReturn, overload, Container,
     Sequence, MutableSequence, Mapping, MutableMapping, Tuple, List, Any, Dict, Callable, Generic,
     Set, AbstractSet, FrozenSet, MutableSet, Sized, Reversible, SupportsInt, SupportsFloat, SupportsAbs,
-    SupportsComplex, IO, BinaryIO, TextIO, Union,
+    SupportsComplex, IO, BinaryIO, Union,
     ItemsView, KeysView, ValuesView, ByteString, Optional, AnyStr, Type, Text,
     Protocol,
 )
-from abc import abstractmethod, ABCMeta
+from abc import ABCMeta
 from ast import mod, AST
-from io import _OpenBinaryMode, _OpenTextMode
+from io import (
+    _OpenBinaryMode, _OpenTextMode, _OpenBinaryModeUpdating, _OpenBinaryModeWriting, _OpenBinaryModeReading,
+    TextIOWrapper, FileIO, BufferedRandom, BufferedReader, BufferedWriter
+)
 from types import TracebackType, CodeType
 import sys
 
@@ -1364,7 +1367,9 @@ if sys.version_info >= (3,):
         _OpenFile = Union[str, bytes, int, _PathLike[Any]]
     else:
         _OpenFile = Union[str, bytes, int]
+    _Opener = Callable[[str, int], int]
 
+    # Text mode: always returns a TextIOWrapper
     @overload
     def open(
         file: _OpenFile,
@@ -1374,19 +1379,71 @@ if sys.version_info >= (3,):
         errors: Optional[str] = ...,
         newline: Optional[str] = ...,
         closefd: bool = ...,
-        opener: Optional[Callable[[str, int], int]] = ...,
-    ) -> TextIO: ...
+        opener: Optional[_Opener] = ...,
+    ) -> TextIOWrapper: ...
+
+    # Unbuffered binary mode: returns a FileIO
     @overload
     def open(
         file: _OpenFile,
         mode: _OpenBinaryMode,
-        buffering: int = ...,
+        buffering: Literal[0],
+        encoding: None = ...,
+        errors: None = ...,
+        newline: None = ...,
+        closefd: bool = ...,
+        opener: Optional[_Opener] = ...,
+    ) -> FileIO: ...
+
+    # Buffering is on: return BufferedRandom, BufferedReader, or BufferedWriter
+    @overload
+    def open(
+        file: _OpenFile,
+        mode: _OpenBinaryModeUpdating,
+        buffering: Literal[-1, 1] = ...,
+        encoding: None = ...,
+        errors: None = ...,
+        newline: None = ...,
+        closefd: bool = ...,
+        opener: Optional[_Opener] = ...,
+    ) -> BufferedRandom: ...
+    @overload
+    def open(
+        file: _OpenFile,
+        mode: _OpenBinaryModeWriting,
+        buffering: Literal[-1, 1] = ...,
         encoding: None = ...,
         errors: None = ...,
         newline: None = ...,
         closefd: bool = ...,
-        opener: Optional[Callable[[str, int], int]] = ...,
+        opener: Optional[_Opener] = ...,
+    ) -> BufferedWriter: ...
+    @overload
+    def open(
+        file: _OpenFile,
+        mode: _OpenBinaryModeReading,
+        buffering: Literal[-1, 1] = ...,
+        encoding: None = ...,
+        errors: None = ...,
+        newline: None = ...,
+        closefd: bool = ...,
+        opener: Optional[_Opener] = ...,
+    ) -> BufferedReader: ...
+
+    # Buffering cannot be determined: fall back to BinaryIO
+    @overload
+    def open(
+        file: _OpenFile,
+        mode: _OpenBinaryMode,
+        buffering: int,
+        encoding: None = ...,
+        errors: None = ...,
+        newline: None = ...,
+        closefd: bool = ...,
+        opener: Optional[_Opener] = ...,
     ) -> BinaryIO: ...
+
+    # Fallback if mode is not specified
     @overload
     def open(
         file: _OpenFile,
@@ -1396,7 +1453,7 @@ if sys.version_info >= (3,):
         errors: Optional[str] = ...,
         newline: Optional[str] = ...,
         closefd: bool = ...,
-        opener: Optional[Callable[[str, int], int]] = ...,
+        opener: Optional[_Opener] = ...,
     ) -> IO[Any]: ...
 
 else:

stdlib/3/io.pyi

@@ -29,13 +29,22 @@ _OpenTextMode = Literal[
     'x', 'x+', '+x', 'xt', 'tx', 'xt+', 'x+t', '+xt', 'tx+', 't+x', '+tx',
     'U', 'rU', 'Ur', 'rtU', 'rUt', 'Urt', 'trU', 'tUr', 'Utr',
 ]
-_OpenBinaryMode = Literal[
-    'rb', 'br', 'rb+', 'r+b', '+rb', 'br+', 'b+r', '+br',
-    'wb', 'bw', 'wb+', 'w+b', '+wb', 'bw+', 'b+w', '+bw',
-    'ab', 'ba', 'ab+', 'a+b', '+ab', 'ba+', 'b+a', '+ba',
-    'xb', 'bx', 'xb+', 'x+b', '+xb', 'bx+', 'b+x', '+bx',
+_OpenBinaryModeUpdating = Literal[
+    'rb+', 'r+b', '+rb', 'br+', 'b+r', '+br',
+    'wb+', 'w+b', '+wb', 'bw+', 'b+w', '+bw',
+    'ab+', 'a+b', '+ab', 'ba+', 'b+a', '+ba',
+    'xb+', 'x+b', '+xb', 'bx+', 'b+x', '+bx',
+]
+_OpenBinaryModeWriting = Literal[
+    'wb', 'bw',
+    'ab', 'ba',
+    'xb', 'bx',
+]
+_OpenBinaryModeReading = Literal[
+    'rb', 'br',
     'rbU', 'rUb', 'Urb', 'brU', 'bUr', 'Ubr',
 ]
+_OpenBinaryMode = Union[_OpenBinaryModeUpdating, _OpenBinaryModeReading, _OpenBinaryModeWriting]
 
 open = builtins.open