From d7a8c04c0349275e7c402d135d347c50fbaba2a5 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Tue, 10 Jan 2023 14:37:15 +0100 Subject: [PATCH 1/5] Fix checking multiple assignments based on tuple unpacking involving partially initialised variables (Fixes #12915). This proposal is an alternative to #14423. Similar to #14423, the main idea is to convert unions of tuples to tuples of (simplified) unions during multi-assignment checks. In addition, it extends this idea to other iterable types, which allows removing the `undefined_rvalue` logic and the `no_partial_types` logic. Hence, the problem reported in #12915 with partially initialised variables should be fixed for unions that combine, for example, tuples and lists, as well. Besides the new test case also provided by #14423 (`testDefinePartiallyInitialisedVariableDuringTupleUnpacking`), this commit also adds the test cases `testUnionUnpackingIncludingListPackingSameItemTypes`, `testUnionUnpackingIncludingListPackingDifferentItemTypes`, and `testUnionUnpackingIncludingListPackingForVariousItemTypes`. --- mypy/checker.py | 188 ++++++++++---------- test-data/unit/check-inference-context.test | 8 +- test-data/unit/check-inference.test | 48 +++++ test-data/unit/check-tuples.test | 3 +- test-data/unit/check-unions.test | 153 ++++++++++++++-- test-data/unit/check-varargs.test | 32 ++-- 6 files changed, 301 insertions(+), 131 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 065758cd2be9..088c9995a9e2 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -190,6 +190,7 @@ NoneType, Overloaded, PartialType, + PlaceholderType, ProperType, StarType, TupleType, @@ -338,8 +339,6 @@ class TypeChecker(NodeVisitor[None], CheckerPluginInterface): # Used for collecting inferred attribute types so that they can be checked # for consistency. inferred_attribute_types: dict[Var, Type] | None = None - # Don't infer partial None types if we are processing assignment from Union - no_partial_types: bool = False # The set of all dependencies (suppressed or not) that this module accesses, either # directly or indirectly. @@ -3375,7 +3374,6 @@ def check_multi_assignment( context: Context, infer_lvalue_type: bool = True, rv_type: Type | None = None, - undefined_rvalue: bool = False, ) -> None: """Check the assignment of one rvalue to a number of lvalues.""" @@ -3386,12 +3384,6 @@ def check_multi_assignment( if isinstance(rvalue_type, TypeVarLikeType): rvalue_type = get_proper_type(rvalue_type.upper_bound) - if isinstance(rvalue_type, UnionType): - # If this is an Optional type in non-strict Optional code, unwrap it. - relevant_items = rvalue_type.relevant_items() - if len(relevant_items) == 1: - rvalue_type = get_proper_type(relevant_items[0]) - if isinstance(rvalue_type, AnyType): for lv in lvalues: if isinstance(lv, StarExpr): @@ -3402,7 +3394,7 @@ def check_multi_assignment( self.check_assignment(lv, temp_node, infer_lvalue_type) elif isinstance(rvalue_type, TupleType): self.check_multi_assignment_from_tuple( - lvalues, rvalue, rvalue_type, context, undefined_rvalue, infer_lvalue_type + lvalues, rvalue, rvalue_type, context, infer_lvalue_type ) elif isinstance(rvalue_type, UnionType): self.check_multi_assignment_from_union( @@ -3430,58 +3422,86 @@ def check_multi_assignment_from_union( x, y = t reveal_type(x) # Union[int, str] - The idea in this case is to process the assignment for every item of the union. - Important note: the types are collected in two places, 'union_types' contains - inferred types for first assignments, 'assignments' contains the narrowed types - for binder. + The idea is to convert unions of tuples or other iterables to tuples of (simplified) + unions and then simply apply `check_multi_assignment_from_tuple`. """ - self.no_partial_types = True - transposed: tuple[list[Type], ...] = tuple([] for _ in self.flatten_lvalues(lvalues)) - # Notify binder that we want to defer bindings and instead collect types. - with self.binder.accumulate_type_assignments() as assignments: - for item in rvalue_type.items: - # Type check the assignment separately for each union item and collect - # the inferred lvalue types for each union item. - self.check_multi_assignment( - lvalues, - rvalue, - context, - infer_lvalue_type=infer_lvalue_type, - rv_type=item, - undefined_rvalue=True, - ) - for t, lv in zip(transposed, self.flatten_lvalues(lvalues)): - # We can access _type_maps directly since temporary type maps are - # only created within expressions. - t.append(self._type_maps[0].pop(lv, AnyType(TypeOfAny.special_form))) - union_types = tuple(make_simplified_union(col) for col in transposed) - for expr, items in assignments.items(): - # Bind a union of types collected in 'assignments' to every expression. - if isinstance(expr, StarExpr): - expr = expr.expr - - # TODO: See todo in binder.py, ConditionalTypeBinder.assign_type - # It's unclear why the 'declared_type' param is sometimes 'None' - clean_items: list[tuple[Type, Type]] = [] - for type, declared_type in items: - assert declared_type is not None - clean_items.append((type, declared_type)) - - types, declared_types = zip(*clean_items) - self.binder.assign_type( - expr, - make_simplified_union(list(types)), - make_simplified_union(list(declared_types)), - False, + # if `rvalue_type` is Optional type in non-strict Optional code, unwap it: + relevant_items = rvalue_type.relevant_items() + if len(relevant_items) == 1: + self.check_multi_assignment( + lvalues, rvalue, context, infer_lvalue_type, relevant_items[0] ) - for union, lv in zip(union_types, self.flatten_lvalues(lvalues)): - # Properly store the inferred types. - _1, _2, inferred = self.check_lvalue(lv) - if inferred: - self.set_inferred_type(inferred, lv, union) + return + + # union to tuple conversion: + star_idx = next((i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), None) + + def handle_star_index(orig_types: list[Type]) -> list[Type]: + if star_idx is not None: + orig_types[star_idx] = self.named_generic_type( + "builtins.list", [orig_types[star_idx]] + ) + return orig_types + + nmb_subitems = len(lvalues) + items: list[list[Type]] = [] + for idx, item in enumerate(rvalue_type.items): + item = get_proper_type(item) + if isinstance(item, TupleType): + delta = len(item.items) - nmb_subitems + if star_idx is None: + if delta != 0: # a, b = x, y, z or a, b, c = x, y + self.msg.wrong_number_values_to_unpack( + len(item.items), nmb_subitems, context + ) + return + items.append(item.items.copy()) # a, b = x, y + else: + if delta < -1: # a, b, c, *d = x, y + self.msg.wrong_number_values_to_unpack( + len(item.items), nmb_subitems - 1, context + ) + return + if delta == -1: # a, b, *c = x, y + items.append(item.items.copy()) + # to be removed after transposing: + items[-1].insert(star_idx, PlaceholderType("temp", [], -1)) + elif delta == 0: # a, b, *c = x, y, z + items.append(handle_star_index(item.items.copy())) + else: # a, *b = x, y, z + union = make_simplified_union(item.items[star_idx : star_idx + delta + 1]) + subitems = ( + item.items[:star_idx] + [union] + item.items[star_idx + delta + 1 :] + ) + items.append(handle_star_index(subitems)) else: - self.store_type(lv, union) - self.no_partial_types = False + if isinstance(item, AnyType): + items.append(handle_star_index(nmb_subitems * [cast(Type, item)])) + elif isinstance(item, Instance) and (item.type.fullname == "builtins.str"): + self.msg.unpacking_strings_disallowed(context) + return + elif isinstance(item, Instance) and self.type_is_iterable(item): + items.append(handle_star_index(nmb_subitems * [self.iterable_item_type(item)])) + else: + self.msg.type_not_iterable(item, context) + return + items_transposed = zip(*items) + items_cleared = [] + for subitems_ in items_transposed: + subitems = [] + for item in subitems_: + item = get_proper_type(item) + if not isinstance(item, PlaceholderType): + subitems.append(item) + items_cleared.append(subitems) + tupletype = TupleType( + [make_simplified_union(subitems) for subitems in items_cleared], + fallback=self.named_type("builtins.tuple"), + ) + self.check_multi_assignment_from_tuple( + lvalues, rvalue, tupletype, context, infer_lvalue_type, False + ) + return def flatten_lvalues(self, lvalues: list[Expression]) -> list[Expression]: res: list[Expression] = [] @@ -3500,8 +3520,8 @@ def check_multi_assignment_from_tuple( rvalue: Expression, rvalue_type: TupleType, context: Context, - undefined_rvalue: bool, infer_lvalue_type: bool = True, + convert_star_rvalue_type: bool = True, ) -> None: if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context): star_index = next( @@ -3512,34 +3532,6 @@ def check_multi_assignment_from_tuple( star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None right_lvs = lvalues[star_index + 1 :] - if not undefined_rvalue: - # Infer rvalue again, now in the correct type context. - lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type) - reinferred_rvalue_type = get_proper_type( - self.expr_checker.accept(rvalue, lvalue_type) - ) - - if isinstance(reinferred_rvalue_type, UnionType): - # If this is an Optional type in non-strict Optional code, unwrap it. - relevant_items = reinferred_rvalue_type.relevant_items() - if len(relevant_items) == 1: - reinferred_rvalue_type = get_proper_type(relevant_items[0]) - if isinstance(reinferred_rvalue_type, UnionType): - self.check_multi_assignment_from_union( - lvalues, rvalue, reinferred_rvalue_type, context, infer_lvalue_type - ) - return - if isinstance(reinferred_rvalue_type, AnyType): - # We can get Any if the current node is - # deferred. Doing more inference in deferred nodes - # is hard, so give up for now. We can also get - # here if reinferring types above changes the - # inferred return type for an overloaded function - # to be ambiguous. - return - assert isinstance(reinferred_rvalue_type, TupleType) - rvalue_type = reinferred_rvalue_type - left_rv_types, star_rv_types, right_rv_types = self.split_around_star( rvalue_type.items, star_index, len(lvalues) ) @@ -3547,11 +3539,16 @@ def check_multi_assignment_from_tuple( for lv, rv_type in zip(left_lvs, left_rv_types): self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) if star_lv: - list_expr = ListExpr( - [self.temp_node(rv_type, context) for rv_type in star_rv_types] - ) - list_expr.set_line(context) - self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) + if convert_star_rvalue_type: + list_expr = ListExpr( + [self.temp_node(rv_type, context) for rv_type in star_rv_types] + ) + list_expr.set_line(context) + self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) + else: + self.check_assignment( + star_lv.expr, self.temp_node(star_rv_types[0], context), infer_lvalue_type + ) for lv, rv_type in zip(right_lvs, right_rv_types): self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) @@ -3702,10 +3699,7 @@ def infer_variable_type( """Infer the type of initialized variables from initializer type.""" if isinstance(init_type, DeletedType): self.msg.deleted_as_rvalue(init_type, context) - elif ( - not is_valid_inferred_type(init_type, is_lvalue_final=name.is_final) - and not self.no_partial_types - ): + elif not is_valid_inferred_type(init_type, is_lvalue_final=name.is_final): # We cannot use the type of the initialization expression for full type # inference (it's not specific enough), but we might be able to give # partial type which will be made more specific later. A partial type diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index f80f93eb2615..266de52f0583 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -168,9 +168,8 @@ if int(): ab, ao = f(b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") if int(): ao, ab = f(b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") - if int(): - ao, ao = f(b) + ao, ao = f(b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") if int(): ab, ab = f(b) if int(): @@ -199,11 +198,10 @@ if int(): ao, ab, ab, ab = h(b, b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") if int(): ab, ab, ao, ab = h(b, b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") - if int(): - ao, ab, ab = f(b, b) + ao, ab, ab = f(b, b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") if int(): - ab, ab, ao = g(b, b) + ab, ab, ao = g(b, b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") if int(): ab, ab, ab, ab = h(b, b) diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 45a833e5210c..8e997d6bf9eb 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -1919,6 +1919,54 @@ class C: a = 42 [out] +[case testDefinePartiallyInitialisedVariableDuringTupleUnpacking] +# flags: --strict-optional +from typing import Tuple, Union + +t1: Union[Tuple[None], Tuple[str]] +x1 = None +x1, = t1 +reveal_type(x1) # N: Revealed type is "Union[None, builtins.str]" + +t2: Union[Tuple[str], Tuple[None]] +x2 = None +x2, = t2 +reveal_type(x2) # N: Revealed type is "Union[builtins.str, None]" + +t3: Union[Tuple[int], Tuple[str]] +x3 = None +x3, = t3 +reveal_type(x3) # N: Revealed type is "Union[builtins.int, builtins.str]" + +def f() -> Union[ + Tuple[None, None, None, int, int, int, int, int, int], + Tuple[None, None, None, int, int, int, str, str, str] + ]: ... +a1 = None +b1 = None +c1 = None +a2: object +b2: object +c2: object +a1, a2, a3, b1, b2, b3, c1, c2, c3 = f() +reveal_type(a1) # N: Revealed type is "None" +reveal_type(a2) # N: Revealed type is "None" +reveal_type(a3) # N: Revealed type is "None" +reveal_type(b1) # N: Revealed type is "builtins.int" +reveal_type(b2) # N: Revealed type is "builtins.int" +reveal_type(b3) # N: Revealed type is "builtins.int" +reveal_type(c1) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(c2) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(c3) # N: Revealed type is "Union[builtins.int, builtins.str]" + +tt: Tuple[Union[Tuple[None], Tuple[str], Tuple[int]]] +z = None +z, = tt[0] +reveal_type(z) # N: Revealed type is "Union[None, builtins.str, builtins.int]" + +[builtins fixtures/tuple.pyi] + + -- More partial type errors -- ------------------------ diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index cdb27d10fe0c..155fd270ed4b 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1054,7 +1054,8 @@ def g(x: T) -> Tuple[T, T]: return (x, x) z = 1 -x, y = g(z) # E: Argument 1 to "g" has incompatible type "int"; expected "Tuple[B1, B2]" +x, y = g(z) # E: Incompatible types in assignment (expression has type "int", variable has type "Tuple[A, ...]") \ + # E: Incompatible types in assignment (expression has type "int", variable has type "Tuple[Union[B1, C], Union[B2, C]]") [builtins fixtures/tuple.pyi] [out] diff --git a/test-data/unit/check-unions.test b/test-data/unit/check-unions.test index 4c4fbc32ec3f..a6cfb6b12511 100644 --- a/test-data/unit/check-unions.test +++ b/test-data/unit/check-unions.test @@ -619,7 +619,7 @@ from typing import Union, Tuple a: Union[Tuple[int, int], Tuple[int, float]] a1: object a2: int -(a1, a2) = a # E: Incompatible types in assignment (expression has type "float", variable has type "int") +(a1, a2) = a # E: Incompatible types in assignment (expression has type "Union[int, float]", variable has type "int") b: Union[Tuple[float, int], Tuple[int, int]] b1: object @@ -721,10 +721,11 @@ reveal_type(d) # N: Revealed type is "builtins.list[builtins.int]" from typing import Union bad: Union[int, str] -x, y = bad # E: "int" object is not iterable \ - # E: Unpacking a string is disallowed -reveal_type(x) # N: Revealed type is "Any" -reveal_type(y) # N: Revealed type is "Any" +x, y = bad # E: "int" object is not iterable +reveal_type(x) # E: Cannot determine type of "x" \ + # N: Revealed type is "Any" +reveal_type(y) # E: Cannot determine type of "y" \ + # N: Revealed type is "Any" [out] [case testStringDisallowedUnpacking] @@ -746,8 +747,10 @@ from typing import Union, Tuple bad: Union[Tuple[int, int, int], Tuple[str, str, str]] x, y = bad # E: Too many values to unpack (2 expected, 3 provided) -reveal_type(x) # N: Revealed type is "Any" -reveal_type(y) # N: Revealed type is "Any" +reveal_type(x) # E: Cannot determine type of "x" \ + # N: Revealed type is "Any" +reveal_type(y) # E: Cannot determine type of "y" \ + # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] [out] @@ -756,10 +759,14 @@ from typing import Union, Tuple bad: Union[Tuple[int, int, int], Tuple[str, str, str]] x, y, z, w = bad # E: Need more than 3 values to unpack (4 expected) -reveal_type(x) # N: Revealed type is "Any" -reveal_type(y) # N: Revealed type is "Any" -reveal_type(z) # N: Revealed type is "Any" -reveal_type(w) # N: Revealed type is "Any" +reveal_type(x) # E: Cannot determine type of "x" \ + # N: Revealed type is "Any" +reveal_type(y) # E: Cannot determine type of "y" \ + # N: Revealed type is "Any" +reveal_type(z) # E: Cannot determine type of "z" \ + # N: Revealed type is "Any" +reveal_type(w) # E: Cannot determine type of "w" \ + # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] [out] @@ -820,6 +827,125 @@ reveal_type(lst) # N: Revealed type is "Union[builtins.list[builtins.int], built [builtins fixtures/list.pyi] [out] +[case testUnionUnpackingIncludingListPackingSameItemTypes] +from typing import Tuple, Union +t: Union[Tuple[int, int], Tuple[float, float, float], Tuple[str, str, str, str]] + +a1, = t # E: Too many values to unpack (1 expected, 2 provided) +reveal_type(a1) # E: Cannot determine type of "a1" \ + # N: Revealed type is "Any" +*b2, = t +reveal_type(b2) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.float], builtins.list[builtins.str]]" +a1, b2 = t # E: Too many values to unpack (2 expected, 3 provided) +*c2, d2 = t +reveal_type(c2) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.float], builtins.list[builtins.str]]" +reveal_type(d2) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.str]" +e2, *h2 = t +reveal_type(e2) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.str]" +reveal_type(h2) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.float], builtins.list[builtins.str]]" + +a3, b3, c3 = t # E: Need more than 2 values to unpack (3 expected) + +*e3, f3, g3 = t +reveal_type(e3) # N: Revealed type is "Union[builtins.list[builtins.float], builtins.list[builtins.str]]" +reveal_type(f3) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.str]" +reveal_type(g3) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.str]" + +h3, *i3, j3 = t +reveal_type(h3) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.str]" +reveal_type(i3) # N: Revealed type is "Union[builtins.list[builtins.float], builtins.list[builtins.str]]" +reveal_type(g3) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.str]" + +k3, l3, *m3 = t +reveal_type(k3) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.str]" +reveal_type(l3) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.str]" +reveal_type(m3) # N: Revealed type is "Union[builtins.list[builtins.float], builtins.list[builtins.str]]" + +a4, b4, c4, d4 = t # E: Need more than 2 values to unpack (4 expected) + +*e4, f4, g4, h4 = t # E: Need more than 2 values to unpack (3 expected) + +[builtins fixtures/tuple.pyi] + +[case testUnionUnpackingIncludingListPackingDifferentItemTypes] +from typing import Tuple, Union + +class A1: ... +class B1: ... +class C1: ... +class D1: ... +class A2: ... +class B2: ... +class C2: ... +class D2: ... + +t: Union[Tuple[A1, B1, C1, D1], Tuple[A2, B2, C2, D2]] + +a1, b1, c1, d1 = t +reveal_type(a1) # N: Revealed type is "Union[__main__.A1, __main__.A2]" +reveal_type(b1) # N: Revealed type is "Union[__main__.B1, __main__.B2]" +reveal_type(c1) # N: Revealed type is "Union[__main__.C1, __main__.C2]" +reveal_type(d1) # N: Revealed type is "Union[__main__.D1, __main__.D2]" + +a2, *bc2, d2 = t +reveal_type(a2) # N: Revealed type is "Union[__main__.A1, __main__.A2]" +reveal_type(bc2) # N: Revealed type is "Union[builtins.list[Union[__main__.B1, __main__.C1]], builtins.list[Union[__main__.B2, __main__.C2]]]" +reveal_type(d2) # N: Revealed type is "Union[__main__.D1, __main__.D2]" + +*abc3, d3 = t +reveal_type(abc3) # N: Revealed type is "Union[builtins.list[Union[__main__.A1, __main__.B1, __main__.C1]], builtins.list[Union[__main__.A2, __main__.B2, __main__.C2]]]" +reveal_type(d3) # N: Revealed type is "Union[__main__.D1, __main__.D2]" + +a4, *bcd4 = t +reveal_type(a4) # N: Revealed type is "Union[__main__.A1, __main__.A2]" +reveal_type(bcd4) # N: Revealed type is "Union[builtins.list[Union[__main__.B1, __main__.C1, __main__.D1]], builtins.list[Union[__main__.B2, __main__.C2, __main__.D2]]]" + +*abcd5, = t +reveal_type(abcd5) # N: Revealed type is "Union[builtins.list[Union[__main__.A1, __main__.B1, __main__.C1, __main__.D1]], builtins.list[Union[__main__.A2, __main__.B2, __main__.C2, __main__.D2]]]" + +[builtins fixtures/tuple.pyi] + +[case testUnionUnpackingIncludingListPackingForVariousItemTypes] +from typing import Any, List, Tuple, Union + +class A: ... +class B: ... +class C: ... +class D: ... + +t1: Union[Tuple[A, B]] +a1, b1, *c1 = t1 # E: Need type annotation for "c1" (hint: "c1: List[] = ...") +reveal_type(a1) # N: Revealed type is "__main__.A" +reveal_type(b1) # N: Revealed type is "__main__.B" +reveal_type(c1) # N: Revealed type is "builtins.list[Any]" + +t2: Union[Tuple[A, B], List[D]] +a2, b2, *c2 = t2 +reveal_type(a2) # N: Revealed type is "Union[__main__.A, __main__.D]" +reveal_type(b2) # N: Revealed type is "Union[__main__.B, __main__.D]" +reveal_type(c2) # N: Revealed type is "builtins.list[__main__.D]" + +t3: Union[Tuple[A, B, C], List[D]] +a3, b3, *c3 = t3 +reveal_type(a3) # N: Revealed type is "Union[__main__.A, __main__.D]" +reveal_type(b3) # N: Revealed type is "Union[__main__.B, __main__.D]" +reveal_type(c3) # N: Revealed type is "Union[builtins.list[__main__.C], builtins.list[__main__.D]]" + +t4: Union[Tuple[A, B, C], List[D], Any] +a4, b4, *c4 = t4 +reveal_type(a4) # N: Revealed type is "Union[__main__.A, __main__.D, Any]" +reveal_type(b4) # N: Revealed type is "Union[__main__.B, __main__.D, Any]" +reveal_type(c4) # N: Revealed type is "Union[builtins.list[__main__.C], builtins.list[__main__.D], builtins.list[Any]]" + +t5: Union[Tuple[A, B, C], str] +a5, b5, *c5 = t5 # E: Unpacking a string is disallowed + +t6: Union[Tuple[A, B, C], D] +a6, b6, *c6 = t6 # E: "D" object is not iterable + +[builtins fixtures/tuple.pyi] + + [case testUnionUnpackingInForTuple] from typing import Union, Tuple, NamedTuple class NTInt(NamedTuple): @@ -977,8 +1103,9 @@ from typing import Dict, Tuple, List, Any a: Any d: Dict[str, Tuple[List[Tuple[str, str]], str]] -x, _ = d.get(a, ([], [])) -reveal_type(x) # N: Revealed type is "Union[builtins.list[Tuple[builtins.str, builtins.str]], builtins.list[]]" +x, _ = d.get(a, ([], [])) # E: Need type annotation for "x" \ + # E: Need type annotation for "_" +reveal_type(x) # N: Revealed type is "Union[builtins.list[Tuple[builtins.str, builtins.str]], builtins.list[Any]]" for y in x: pass [builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test index 00ac7df320d2..52c8af004d54 100644 --- a/test-data/unit/check-varargs.test +++ b/test-data/unit/check-varargs.test @@ -531,19 +531,19 @@ T = TypeVar('T') a, b, aa = None, None, None # type: (A, B, List[A]) if int(): - a, b = f(*aa) # E: Argument 1 to "f" has incompatible type "*List[A]"; expected "B" + a, b = f(*aa) # E: Incompatible types in assignment (expression has type "A", variable has type "B") if int(): - b, b = f(*aa) # E: Argument 1 to "f" has incompatible type "*List[A]"; expected "B" + b, b = f(*aa) # E: Incompatible types in assignment (expression has type "A", variable has type "B") if int(): - a, a = f(b, *aa) # E: Argument 1 to "f" has incompatible type "B"; expected "A" + a, a = f(b, *aa) # E: Incompatible types in assignment (expression has type "B", variable has type "A") if int(): - b, b = f(b, *aa) # E: Argument 2 to "f" has incompatible type "*List[A]"; expected "B" + b, b = f(b, *aa) # E: Incompatible types in assignment (expression has type "A", variable has type "B") if int(): - b, b = f(b, b, *aa) # E: Argument 3 to "f" has incompatible type "*List[A]"; expected "B" + b, b = f(b, b, *aa) # E: Incompatible types in assignment (expression has type "object", variable has type "B") if int(): - a, b = f(a, *a) # E: List or tuple expected as variadic arguments + a, b = f(a, *a) # E: List or tuple expected as variadic arguments if int(): - a, b = f(*a) # E: List or tuple expected as variadic arguments + a, b = f(*a) # E: List or tuple expected as variadic arguments if int(): a, a = f(*aa) @@ -566,13 +566,13 @@ T = TypeVar('T') a, b = None, None # type: (A, B) if int(): - a, a = f(*(a, b)) # E: Argument 1 to "f" has incompatible type "*Tuple[A, B]"; expected "A" + a, a = f(*(a, b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") if int(): - b, b = f(a, *(b,)) # E: Argument 1 to "f" has incompatible type "A"; expected "B" + b, b = f(a, *(b,)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") if int(): - a, a = f(*(a, b)) # E: Argument 1 to "f" has incompatible type "*Tuple[A, B]"; expected "A" + a, a = f(*(a, b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") if int(): - b, b = f(a, *(b,)) # E: Argument 1 to "f" has incompatible type "A"; expected "B" + b, b = f(a, *(b,)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") if int(): a, b = f(*(a, b, b)) # E: Too many arguments for "f" if int(): @@ -606,16 +606,18 @@ if int(): aa, a = G().f(*[a]) # E: Incompatible types in assignment (expression has type "List[]", variable has type "A") if int(): ab, aa = G().f(*[a]) \ + # E: Incompatible types in assignment (expression has type "List[A]", variable has type "List[B]") \ # E: Incompatible types in assignment (expression has type "List[]", variable has type "List[A]") \ # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ - # N: Consider using "Sequence" instead, which is covariant \ - # E: Argument 1 to "f" of "G" has incompatible type "*List[A]"; expected "B" + # N: Consider using "Sequence" instead, which is covariant if int(): ao, ao = G().f(*[a]) \ - # E: Incompatible types in assignment (expression has type "List[]", variable has type "List[object]") \ + # E: Incompatible types in assignment (expression has type "List[A]", variable has type "List[object]") \ # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ - # N: Consider using "Sequence" instead, which is covariant + # N: Consider using "Sequence" instead, which is covariant \ + # E: Incompatible types in assignment (expression has type "List[]", variable has type "List[object]") + if int(): aa, aa = G().f(*[a]) \ # E: Incompatible types in assignment (expression has type "List[]", variable has type "List[A]") \ From 9446ef40a34b4118fc2c375377654b4e812fd541 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Thu, 12 Jan 2023 20:53:28 +0100 Subject: [PATCH 2/5] check Tuple[int, ...] --- test-data/unit/check-unions.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/check-unions.test b/test-data/unit/check-unions.test index a6cfb6b12511..2c92c7b4f958 100644 --- a/test-data/unit/check-unions.test +++ b/test-data/unit/check-unions.test @@ -931,7 +931,7 @@ reveal_type(a3) # N: Revealed type is "Union[__main__.A, __main__.D]" reveal_type(b3) # N: Revealed type is "Union[__main__.B, __main__.D]" reveal_type(c3) # N: Revealed type is "Union[builtins.list[__main__.C], builtins.list[__main__.D]]" -t4: Union[Tuple[A, B, C], List[D], Any] +t4: Union[Tuple[A, B, C], Tuple[D, ...], Any] a4, b4, *c4 = t4 reveal_type(a4) # N: Revealed type is "Union[__main__.A, __main__.D, Any]" reveal_type(b4) # N: Revealed type is "Union[__main__.B, __main__.D, Any]" From 965f7d929f3b4bceb5e23a696a095c63efa8cb3b Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Sat, 14 Jan 2023 19:53:20 +0100 Subject: [PATCH 3/5] no early quitting and no tuple construction --- mypy/checker.py | 97 ++++++++++++++------------------ test-data/unit/check-unions.test | 40 ++++++++++--- 2 files changed, 74 insertions(+), 63 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 088c9995a9e2..e3247c14afae 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3416,14 +3416,18 @@ def check_multi_assignment_from_union( infer_lvalue_type: bool, ) -> None: """Check assignment to multiple lvalue targets when rvalue type is a Union[...]. + For example: - t: Union[Tuple[int, int], Tuple[str, str]] + t: Union[Tuple[int, int], Tuple[str, float]] x, y = t reveal_type(x) # Union[int, str] - The idea is to convert unions of tuples or other iterables to tuples of (simplified) - unions and then simply apply `check_multi_assignment_from_tuple`. + The idea is to check each single assignment by constructing a union of the + relevant rvalue types: + + x = Union[int, str] + y = Union[int, float] """ # if `rvalue_type` is Optional type in non-strict Optional code, unwap it: relevant_items = rvalue_type.relevant_items() @@ -3433,7 +3437,7 @@ def check_multi_assignment_from_union( ) return - # union to tuple conversion: + # cases like a, *b, c require special care star_idx = next((i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), None) def handle_star_index(orig_types: list[Type]) -> list[Type]: @@ -3443,6 +3447,7 @@ def handle_star_index(orig_types: list[Type]) -> list[Type]: ) return orig_types + # collect the relevant rvalue types nmb_subitems = len(lvalues) items: list[list[Type]] = [] for idx, item in enumerate(rvalue_type.items): @@ -3450,58 +3455,48 @@ def handle_star_index(orig_types: list[Type]) -> list[Type]: if isinstance(item, TupleType): delta = len(item.items) - nmb_subitems if star_idx is None: - if delta != 0: # a, b = x, y, z or a, b, c = x, y + if delta == 0: # a, b = x, y + items.append(item.items.copy()) + else: # a, b = x, y, z or a, b, c = x, y self.msg.wrong_number_values_to_unpack( len(item.items), nmb_subitems, context ) - return - items.append(item.items.copy()) # a, b = x, y - else: - if delta < -1: # a, b, c, *d = x, y - self.msg.wrong_number_values_to_unpack( - len(item.items), nmb_subitems - 1, context - ) - return - if delta == -1: # a, b, *c = x, y - items.append(item.items.copy()) - # to be removed after transposing: - items[-1].insert(star_idx, PlaceholderType("temp", [], -1)) - elif delta == 0: # a, b, *c = x, y, z - items.append(handle_star_index(item.items.copy())) - else: # a, *b = x, y, z - union = make_simplified_union(item.items[star_idx : star_idx + delta + 1]) - subitems = ( - item.items[:star_idx] + [union] + item.items[star_idx + delta + 1 :] - ) - items.append(handle_star_index(subitems)) - else: - if isinstance(item, AnyType): - items.append(handle_star_index(nmb_subitems * [cast(Type, item)])) - elif isinstance(item, Instance) and (item.type.fullname == "builtins.str"): + elif delta < -1: # a, b, c, *d = x, y + self.msg.wrong_number_values_to_unpack( + len(item.items), nmb_subitems - 1, context + ) + elif delta == -1: # a, b, *c = x, y + items.append(item.items.copy()) + # to be removed after transposing: + items[-1].insert(star_idx, PlaceholderType("temp", [], -1)) + elif delta == 0: # a, b, *c = x, y, z + items.append(handle_star_index(item.items.copy())) + else: # a, *b = x, y, z + union = make_simplified_union(item.items[star_idx : star_idx + delta + 1]) + subitems = item.items[:star_idx] + [union] + item.items[star_idx + delta + 1 :] + items.append(handle_star_index(subitems)) + elif isinstance(item, AnyType): + items.append(handle_star_index(nmb_subitems * [cast(Type, item)])) + elif isinstance(item, Instance): + if item.type.fullname == "builtins.str": self.msg.unpacking_strings_disallowed(context) - return - elif isinstance(item, Instance) and self.type_is_iterable(item): + elif self.type_is_iterable(item): items.append(handle_star_index(nmb_subitems * [self.iterable_item_type(item)])) else: self.msg.type_not_iterable(item, context) - return + + # construct the unions and perform the single assignment checks items_transposed = zip(*items) - items_cleared = [] - for subitems_ in items_transposed: + for lvalue, subitems_ in zip(lvalues, items_transposed): subitems = [] for item in subitems_: item = get_proper_type(item) if not isinstance(item, PlaceholderType): subitems.append(item) - items_cleared.append(subitems) - tupletype = TupleType( - [make_simplified_union(subitems) for subitems in items_cleared], - fallback=self.named_type("builtins.tuple"), - ) - self.check_multi_assignment_from_tuple( - lvalues, rvalue, tupletype, context, infer_lvalue_type, False - ) - return + uniontype = make_simplified_union(subitems) + if isinstance(lvalue, StarExpr): + lvalue = lvalue.expr + self.check_assignment(lvalue, self.temp_node(uniontype, context), infer_lvalue_type) def flatten_lvalues(self, lvalues: list[Expression]) -> list[Expression]: res: list[Expression] = [] @@ -3521,7 +3516,6 @@ def check_multi_assignment_from_tuple( rvalue_type: TupleType, context: Context, infer_lvalue_type: bool = True, - convert_star_rvalue_type: bool = True, ) -> None: if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context): star_index = next( @@ -3539,16 +3533,11 @@ def check_multi_assignment_from_tuple( for lv, rv_type in zip(left_lvs, left_rv_types): self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) if star_lv: - if convert_star_rvalue_type: - list_expr = ListExpr( - [self.temp_node(rv_type, context) for rv_type in star_rv_types] - ) - list_expr.set_line(context) - self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) - else: - self.check_assignment( - star_lv.expr, self.temp_node(star_rv_types[0], context), infer_lvalue_type - ) + list_expr = ListExpr( + [self.temp_node(rv_type, context) for rv_type in star_rv_types] + ) + list_expr.set_line(context) + self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) for lv, rv_type in zip(right_lvs, right_rv_types): self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) diff --git a/test-data/unit/check-unions.test b/test-data/unit/check-unions.test index 2c92c7b4f958..416a0916089d 100644 --- a/test-data/unit/check-unions.test +++ b/test-data/unit/check-unions.test @@ -721,7 +721,8 @@ reveal_type(d) # N: Revealed type is "builtins.list[builtins.int]" from typing import Union bad: Union[int, str] -x, y = bad # E: "int" object is not iterable +x, y = bad # E: "int" object is not iterable \ + # E: Unpacking a string is disallowed reveal_type(x) # E: Cannot determine type of "x" \ # N: Revealed type is "Any" reveal_type(y) # E: Cannot determine type of "y" \ @@ -831,20 +832,32 @@ reveal_type(lst) # N: Revealed type is "Union[builtins.list[builtins.int], built from typing import Tuple, Union t: Union[Tuple[int, int], Tuple[float, float, float], Tuple[str, str, str, str]] -a1, = t # E: Too many values to unpack (1 expected, 2 provided) +a1, = t # E: Too many values to unpack (1 expected, 2 provided) \ + # E: Too many values to unpack (1 expected, 3 provided) \ + # E: Too many values to unpack (1 expected, 4 provided) reveal_type(a1) # E: Cannot determine type of "a1" \ # N: Revealed type is "Any" -*b2, = t -reveal_type(b2) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.float], builtins.list[builtins.str]]" -a1, b2 = t # E: Too many values to unpack (2 expected, 3 provided) +*b1, = t +reveal_type(b1) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.float], builtins.list[builtins.str]]" + +a2, b2 = t # E: Too many values to unpack (2 expected, 3 provided) \ + # E: Too many values to unpack (2 expected, 4 provided) +reveal_type(a2) # N: Revealed type is "builtins.int" +reveal_type(b2) # N: Revealed type is "builtins.int" + *c2, d2 = t reveal_type(c2) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.float], builtins.list[builtins.str]]" reveal_type(d2) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.str]" + e2, *h2 = t reveal_type(e2) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.str]" reveal_type(h2) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.float], builtins.list[builtins.str]]" -a3, b3, c3 = t # E: Need more than 2 values to unpack (3 expected) +a3, b3, c3 = t # E: Need more than 2 values to unpack (3 expected) \ + # E: Too many values to unpack (3 expected, 4 provided) +reveal_type(a3) # N: Revealed type is "builtins.float" +reveal_type(b3) # N: Revealed type is "builtins.float" +reveal_type(c3) # N: Revealed type is "builtins.float" *e3, f3, g3 = t reveal_type(e3) # N: Revealed type is "Union[builtins.list[builtins.float], builtins.list[builtins.str]]" @@ -861,9 +874,18 @@ reveal_type(k3) # N: Revealed type is "Union[builtins.int, builtins.float, buil reveal_type(l3) # N: Revealed type is "Union[builtins.int, builtins.float, builtins.str]" reveal_type(m3) # N: Revealed type is "Union[builtins.list[builtins.float], builtins.list[builtins.str]]" -a4, b4, c4, d4 = t # E: Need more than 2 values to unpack (4 expected) - -*e4, f4, g4, h4 = t # E: Need more than 2 values to unpack (3 expected) +a4, b4, c4, d4 = t # E: Need more than 2 values to unpack (4 expected) \ + # E: Need more than 3 values to unpack (4 expected) +reveal_type(a4) # N: Revealed type is "builtins.str" +reveal_type(b4) # N: Revealed type is "builtins.str" +reveal_type(c4) # N: Revealed type is "builtins.str" +reveal_type(d4) # N: Revealed type is "builtins.str" + +*e4, f4, g4, h4 = t # E: Need more than 2 values to unpack (3 expected) +reveal_type(e4) # N: Revealed type is "builtins.list[builtins.str]" +reveal_type(f4) # N: Revealed type is "Union[builtins.float, builtins.str]" +reveal_type(g4) # N: Revealed type is "Union[builtins.float, builtins.str]" +reveal_type(h4) # N: Revealed type is "Union[builtins.float, builtins.str]" [builtins fixtures/tuple.pyi] From f6a2b33d91578c762662262ccd78818b9b9bcf72 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Sun, 15 Jan 2023 21:40:33 +0100 Subject: [PATCH 4/5] Handle other not iterable types like None and Literal. --- mypy/checker.py | 2 ++ test-data/unit/check-unions.test | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index e3247c14afae..7bc3535f28f7 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3484,6 +3484,8 @@ def handle_star_index(orig_types: list[Type]) -> list[Type]: items.append(handle_star_index(nmb_subitems * [self.iterable_item_type(item)])) else: self.msg.type_not_iterable(item, context) + else: + self.msg.type_not_iterable(item, context) # construct the unions and perform the single assignment checks items_transposed = zip(*items) diff --git a/test-data/unit/check-unions.test b/test-data/unit/check-unions.test index 416a0916089d..c2f0ec345d3d 100644 --- a/test-data/unit/check-unions.test +++ b/test-data/unit/check-unions.test @@ -929,6 +929,7 @@ reveal_type(abcd5) # N: Revealed type is "Union[builtins.list[Union[__main__.A1 [case testUnionUnpackingIncludingListPackingForVariousItemTypes] from typing import Any, List, Tuple, Union +from typing_extensions import Literal class A: ... class B: ... @@ -961,9 +962,17 @@ reveal_type(c4) # N: Revealed type is "Union[builtins.list[__main__.C], builti t5: Union[Tuple[A, B, C], str] a5, b5, *c5 = t5 # E: Unpacking a string is disallowed - -t6: Union[Tuple[A, B, C], D] -a6, b6, *c6 = t6 # E: "D" object is not iterable +reveal_type(a5) # N: Revealed type is "__main__.A" +reveal_type(b5) # N: Revealed type is "__main__.B" +reveal_type(c5) # N: Revealed type is "builtins.list[__main__.C]" + +t6: Union[Tuple[A, B, C], D, Literal["abc"], None] +a6, b6, *c6 = t6 # E: "D" object is not iterable \ + # E: "Literal['abc']" object is not iterable \ + # E: "None" object is not iterable +reveal_type(a6) # N: Revealed type is "__main__.A" +reveal_type(b6) # N: Revealed type is "__main__.B" +reveal_type(c6) # N: Revealed type is "builtins.list[__main__.C]" [builtins fixtures/tuple.pyi] From 18b5e567b3ce772b901535a7d2dca09d7da6aa14 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Mon, 16 Jan 2023 06:18:10 +0100 Subject: [PATCH 5/5] Remove method `lvalue_type_for_inference` (not used anymore). --- mypy/checker.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 7bc3535f28f7..658eae57b7d1 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3543,44 +3543,6 @@ def check_multi_assignment_from_tuple( for lv, rv_type in zip(right_lvs, right_rv_types): self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) - def lvalue_type_for_inference(self, lvalues: list[Lvalue], rvalue_type: TupleType) -> Type: - star_index = next( - (i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), len(lvalues) - ) - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1 :] - left_rv_types, star_rv_types, right_rv_types = self.split_around_star( - rvalue_type.items, star_index, len(lvalues) - ) - - type_parameters: list[Type] = [] - - def append_types_for_inference(lvs: list[Expression], rv_types: list[Type]) -> None: - for lv, rv_type in zip(lvs, rv_types): - sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv) - if sub_lvalue_type and not isinstance(sub_lvalue_type, PartialType): - type_parameters.append(sub_lvalue_type) - else: # index lvalue - # TODO Figure out more precise type context, probably - # based on the type signature of the _set method. - type_parameters.append(rv_type) - - append_types_for_inference(left_lvs, left_rv_types) - - if star_lv: - sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr) - if sub_lvalue_type and not isinstance(sub_lvalue_type, PartialType): - type_parameters.extend([sub_lvalue_type] * len(star_rv_types)) - else: # index lvalue - # TODO Figure out more precise type context, probably - # based on the type signature of the _set method. - type_parameters.extend(star_rv_types) - - append_types_for_inference(right_lvs, right_rv_types) - - return TupleType(type_parameters, self.named_type("builtins.tuple")) - def split_around_star( self, items: list[T], star_index: int, length: int ) -> tuple[list[T], list[T], list[T]]: