Fix crash with forward reference in TypedDict (#3560)

Move join to happen after semantic analysis to avoid crash due to
incomplete MRO in TypeInfo. The implementation uses a new semantic
analysis 'patch' phase.

Fixes #3319. Fixes #2489. Fixes #3316.

Author: JukkaL

mypy/build.py

@@ -22,7 +22,7 @@
 from os.path import dirname, basename
 
 from typing import (AbstractSet, Dict, Iterable, Iterator, List,
-                    NamedTuple, Optional, Set, Tuple, Union)
+                    NamedTuple, Optional, Set, Tuple, Union, Callable)
 # Can't use TYPE_CHECKING because it's not in the Python 3.5.1 stdlib
 MYPY = False
 if MYPY:
@@ -1651,15 +1651,21 @@ def parse_file(self) -> None:
         self.check_blockers()
 
     def semantic_analysis(self) -> None:
+        patches = []  # type: List[Callable[[], None]]
         with self.wrap_context():
-            self.manager.semantic_analyzer.visit_file(self.tree, self.xpath, self.options)
+            self.manager.semantic_analyzer.visit_file(self.tree, self.xpath, self.options, patches)
+        self.patches = patches
 
     def semantic_analysis_pass_three(self) -> None:
         with self.wrap_context():
             self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath, self.options)
             if self.options.dump_type_stats:
                 dump_type_stats(self.tree, self.xpath)
 
+    def semantic_analysis_apply_patches(self) -> None:
+        for patch_func in self.patches:
+            patch_func()
+
     def type_check_first_pass(self) -> None:
         manager = self.manager
         if self.options.semantic_analysis_only:
@@ -2095,6 +2101,8 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No
         graph[id].semantic_analysis_pass_three()
     for id in fresh:
         graph[id].calculate_mros()
+    for id in stale:
+        graph[id].semantic_analysis_apply_patches()
     for id in stale:
         graph[id].type_check_first_pass()
     more = True

mypy/semanal.py

@@ -261,13 +261,20 @@ def __init__(self,
         self.all_exports = set()  # type: Set[str]
         self.plugin = plugin
 
-    def visit_file(self, file_node: MypyFile, fnam: str, options: Options) -> None:
+    def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
+                   patches: List[Callable[[], None]]) -> None:
+        """Run semantic analysis phase 2 over a file.
+
+        Add callbacks by mutating the patches list argument. They will be called
+        after all semantic analysis phases but before type checking.
+        """
         self.options = options
         self.errors.set_file(fnam, file_node.fullname())
         self.cur_mod_node = file_node
         self.cur_mod_id = file_node.fullname()
         self.is_stub_file = fnam.lower().endswith('.pyi')
         self.globals = file_node.names
+        self.patches = patches
 
         if 'builtins' in self.modules:
             self.globals['__builtins__'] = SymbolTableNode(
@@ -294,6 +301,7 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options) -> None:
                     g.module_public = False
 
         del self.options
+        del self.patches
 
     def refresh_partial(self, node: Union[MypyFile, FuncItem]) -> None:
         """Refresh a stale target in fine-grained incremental mode."""
@@ -2373,11 +2381,19 @@ def fail_typeddict_arg(self, message: str,
 
     def build_typeddict_typeinfo(self, name: str, items: List[str],
                                  types: List[Type]) -> TypeInfo:
-        mapping_value_type = join.join_type_list(types)
         fallback = (self.named_type_or_none('typing.Mapping',
-                                            [self.str_type(), mapping_value_type])
+                                            [self.str_type(), self.object_type()])
                     or self.object_type())
 
+        def patch() -> None:
+            # Calculate the correct value type for the fallback Mapping.
+            fallback.args[1] = join.join_type_list(types)
+
+        # We can't calculate the complete fallback type until after semantic
+        # analysis, since otherwise MROs might be incomplete. Postpone a callback
+        # function that patches the fallback.
+        self.patches.append(patch)
+
         info = self.basic_new_typeinfo(name, fallback)
         info.typeddict_type = TypedDictType(OrderedDict(zip(items, types)), fallback)
 

mypy/server/update.py

@@ -147,6 +147,7 @@ def build_incremental_step(manager: BuildManager,
     # TODO: state.fix_suppressed_dependencies()?
     state.semantic_analysis()
     state.semantic_analysis_pass_three()
+    # TODO: state.semantic_analysis_apply_patches()
     state.type_check_first_pass()
     # TODO: state.type_check_second_pass()?
     state.finish_passes()

test-data/unit/check-typeddict.test

@@ -813,3 +813,43 @@ p = TaggedPoint(type='2d', x=42, y=1337)
 p.get('x', 1 + 'y')     # E: Unsupported operand types for + ("int" and "str")
 [builtins fixtures/dict.pyi]
 [typing fixtures/typing-full.pyi]
+
+
+-- Special cases
+
+[case testForwardReferenceInTypedDict]
+from typing import Mapping
+from mypy_extensions import TypedDict
+X = TypedDict('X', {'b': 'B', 'c': 'C'})
+class B: pass
+class C(B): pass
+x: X
+reveal_type(x) # E: Revealed type is 'TypedDict(b=__main__.B, c=__main__.C, _fallback=__main__.X)'
+m1: Mapping[str, B] = x
+m2: Mapping[str, C] = x # E: Incompatible types in assignment (expression has type "X", variable has type Mapping[str, C])
+[builtins fixtures/dict.pyi]
+
+[case testForwardReferenceInClassTypedDict]
+from typing import Mapping
+from mypy_extensions import TypedDict
+class X(TypedDict):
+    b: 'B'
+    c: 'C'
+class B: pass
+class C(B): pass
+x: X
+reveal_type(x) # E: Revealed type is 'TypedDict(b=__main__.B, c=__main__.C, _fallback=__main__.X)'
+m1: Mapping[str, B] = x
+m2: Mapping[str, C] = x # E: Incompatible types in assignment (expression has type "X", variable has type Mapping[str, C])
+[builtins fixtures/dict.pyi]
+
+[case testForwardReferenceToTypedDictInTypedDict]
+from typing import Mapping
+from mypy_extensions import TypedDict
+# Forward references don't quite work yet
+X = TypedDict('X', {'a': 'A'}) # E: Invalid type "__main__.A"
+A = TypedDict('A', {'b': int})
+x: X
+reveal_type(x) # E: Revealed type is 'TypedDict(a=TypedDict(b=builtins.int, _fallback=__main__.A), _fallback=__main__.X)'
+reveal_type(x['a']['b']) # E: Revealed type is 'builtins.int'
+[builtins fixtures/dict.pyi]