From ccfc4adbbb795ffe84c64ede92ea941d33e6f8c7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 14 Jun 2017 10:36:07 +0100 Subject: [PATCH 01/17] Basic support for TypedDicts with missing keys (total=False) Only the functional syntax is supported. --- mypy/checkexpr.py | 9 ++- mypy/join.py | 2 +- mypy/meet.py | 2 +- mypy/messages.py | 7 ++ mypy/semanal.py | 72 ++++++++++++------- mypy/typeanal.py | 2 +- mypy/types.py | 26 +++++-- test-data/unit/check-typeddict.test | 80 +++++++++++++++++++++ test-data/unit/lib-stub/mypy_extensions.pyi | 2 +- test-data/unit/semanal-typeddict.test | 40 ----------- 10 files changed, 164 insertions(+), 78 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index c78919791ffa..e5b3bda42919 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -292,8 +292,9 @@ def check_typeddict_call_with_dict(self, callee: TypedDictType, def check_typeddict_call_with_kwargs(self, callee: TypedDictType, kwargs: 'OrderedDict[str, Expression]', context: Context) -> Type: - if callee.items.keys() != kwargs.keys(): - callee_item_names = callee.items.keys() + if not (callee.required_keys <= kwargs.keys() <= callee.items.keys()): + callee_item_names = [key for key in callee.items.keys() + if key in callee.required_keys or key in kwargs.keys()] kwargs_item_names = kwargs.keys() self.msg.typeddict_instantiated_with_unexpected_items( @@ -316,7 +317,7 @@ def check_typeddict_call_with_kwargs(self, callee: TypedDictType, mapping_value_type = join.join_type_list(list(items.values())) fallback = self.chk.named_generic_type('typing.Mapping', [self.chk.str_type(), mapping_value_type]) - return TypedDictType(items, fallback) + return TypedDictType(items, set(callee.required_keys), fallback) # Types and methods that can be used to infer partial types. item_args = {'builtins.list': ['append'], @@ -1656,6 +1657,8 @@ def visit_typeddict_index_expr(self, td_type: TypedDictType, index: Expression) if item_type is None: self.msg.typeddict_item_name_not_found(td_type, item_name, index) return AnyType() + if item_name not in td_type.required_keys: + self.msg.typeddict_item_may_be_undefined(item_name, index) return item_type def visit_enum_index_expr(self, enum_type: TypeInfo, index: Expression, diff --git a/mypy/join.py b/mypy/join.py index aaaa99fa3798..c059561d026b 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -232,7 +232,7 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: ]) mapping_value_type = join_type_list(list(items.values())) fallback = self.s.create_anonymous_fallback(value_type=mapping_value_type) - return TypedDictType(items, fallback) + return TypedDictType(items, set(items.keys()), fallback) # XXX required elif isinstance(self.s, Instance): return join_instances(self.s, t.fallback) else: diff --git a/mypy/meet.py b/mypy/meet.py index 62940b08d62b..6771099f91f9 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -266,7 +266,7 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: items = OrderedDict(item_list) mapping_value_type = join_type_list(list(items.values())) fallback = self.s.create_anonymous_fallback(value_type=mapping_value_type) - return TypedDictType(items, fallback) + return TypedDictType(items, set(items.keys()), fallback) # XXX required else: return self.default(self.s) diff --git a/mypy/messages.py b/mypy/messages.py index 223118c40fff..b830aec2aafe 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -901,6 +901,13 @@ def typeddict_item_name_not_found(self, self.fail('\'{}\' is not a valid TypedDict key; expected one of {}'.format( item_name, format_item_name_list(typ.items.keys())), context) + def typeddict_item_may_be_undefined(self, + item_name: str, + context: Context, + ) -> None: + self.fail("TypedDict key '{}' may be undefined".format(item_name), context) + self.note("Consider using get() instead", context) + def type_arguments_not_allowed(self, context: Context) -> None: self.fail('Parameterized generics cannot be used with class or instance checks', context) diff --git a/mypy/semanal.py b/mypy/semanal.py index 523edc8563e0..6cd37c22be10 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2302,46 +2302,64 @@ def check_typeddict(self, node: Expression, var_name: str = None) -> Optional[Ty fullname = callee.fullname if fullname != 'mypy_extensions.TypedDict': return None - items, types, ok = self.parse_typeddict_args(call, fullname) + items, types, total, ok = self.parse_typeddict_args(call, fullname) if not ok: # Error. Construct dummy return value. - return self.build_typeddict_typeinfo('TypedDict', [], []) - name = cast(StrExpr, call.args[0]).value - if name != var_name or self.is_func_scope(): - # Give it a unique name derived from the line number. - name += '@' + str(call.line) - info = self.build_typeddict_typeinfo(name, items, types) - # Store it as a global just in case it would remain anonymous. - # (Or in the nearest class if there is one.) - stnode = SymbolTableNode(GDEF, info, self.cur_mod_id) - if self.type: - self.type.names[name] = stnode + info = self.build_typeddict_typeinfo('TypedDict', [], []) else: - self.globals[name] = stnode + name = cast(StrExpr, call.args[0]).value + if name != var_name or self.is_func_scope(): + # Give it a unique name derived from the line number. + name += '@' + str(call.line) + info = self.build_typeddict_typeinfo(name, items, types, total) + # Store it as a global just in case it would remain anonymous. + # (Or in the nearest class if there is one.) + stnode = SymbolTableNode(GDEF, info, self.cur_mod_id) + if self.type: + self.type.names[name] = stnode + else: + self.globals[name] = stnode call.analyzed = TypedDictExpr(info) call.analyzed.set_line(call.line, call.column) return info def parse_typeddict_args(self, call: CallExpr, - fullname: str) -> Tuple[List[str], List[Type], bool]: + fullname: str) -> Tuple[List[str], List[Type], bool, bool]: # TODO: Share code with check_argument_count in checkexpr.py? args = call.args if len(args) < 2: return self.fail_typeddict_arg("Too few arguments for TypedDict()", call) - if len(args) > 2: + if len(args) > 3: return self.fail_typeddict_arg("Too many arguments for TypedDict()", call) # TODO: Support keyword arguments - if call.arg_kinds != [ARG_POS, ARG_POS]: + if call.arg_kinds not in ([ARG_POS, ARG_POS], [ARG_POS, ARG_POS, ARG_NAMED]): return self.fail_typeddict_arg("Unexpected arguments to TypedDict()", call) + if len(args) == 3 and call.arg_names[2] != 'total': + return self.fail_typeddict_arg( + 'Unexpected keyword argument "{}" for "TypedDict"'.format(call.arg_names[2]), call) if not isinstance(args[0], (StrExpr, BytesExpr, UnicodeExpr)): return self.fail_typeddict_arg( "TypedDict() expects a string literal as the first argument", call) if not isinstance(args[1], DictExpr): return self.fail_typeddict_arg( "TypedDict() expects a dictionary literal as the second argument", call) + total = True + if len(args) == 3: + total = self.parse_bool(call.args[2]) + if total is None: + return self.fail_typeddict_arg( + 'TypedDict() "total" argument must be True or False', call) dictexpr = args[1] items, types, ok = self.parse_typeddict_fields_with_types(dictexpr.items, call) - return items, types, ok + return items, types, total, ok + + def parse_bool(self, expr: Expression) -> Optional[bool]: + if isinstance(expr, NameExpr): + if expr.fullname == 'builtins.True': + return True + if expr.fullname == 'builtins.False': + return False + return None def parse_typeddict_fields_with_types(self, dict_items: List[Tuple[Expression, Expression]], context: Context) -> Tuple[List[str], List[Type], bool]: @@ -2351,29 +2369,35 @@ def parse_typeddict_fields_with_types(self, dict_items: List[Tuple[Expression, E if isinstance(field_name_expr, (StrExpr, BytesExpr, UnicodeExpr)): items.append(field_name_expr.value) else: - return self.fail_typeddict_arg("Invalid TypedDict() field name", field_name_expr) + self.fail_typeddict_arg("Invalid TypedDict() field name", field_name_expr) + return [], [], False try: type = expr_to_unanalyzed_type(field_type_expr) except TypeTranslationError: - return self.fail_typeddict_arg('Invalid field type', field_type_expr) + self.fail_typeddict_arg('Invalid field type', field_type_expr) + return [], [], False types.append(self.anal_type(type)) return items, types, True def fail_typeddict_arg(self, message: str, - context: Context) -> Tuple[List[str], List[Type], bool]: + context: Context) -> Tuple[List[str], List[Type], bool, bool]: self.fail(message, context) - return [], [], False + return [], [], True, False def build_typeddict_typeinfo(self, name: str, items: List[str], - types: List[Type]) -> TypeInfo: + types: List[Type], total: bool = True) -> TypeInfo: mapping_value_type = join.join_type_list(types) fallback = (self.named_type_or_none('typing.Mapping', [self.str_type(), mapping_value_type]) or self.object_type()) info = self.basic_new_typeinfo(name, fallback) - info.typeddict_type = TypedDictType(OrderedDict(zip(items, types)), fallback) - + if total: + required_keys = set(items) + else: + required_keys = set() + info.typeddict_type = TypedDictType(OrderedDict(zip(items, types)), required_keys, + fallback) return info def check_classvar(self, s: AssignmentStmt) -> None: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index d36ecb98de6a..6ce1f99ec335 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -368,7 +368,7 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: (item_name, self.anal_type(item_type)) for (item_name, item_type) in t.items.items() ]) - return TypedDictType(items, t.fallback) + return TypedDictType(items, set(t.required_keys), t.fallback) def visit_star_type(self, t: StarType) -> Type: return StarType(self.anal_type(t.type), t.line) diff --git a/mypy/types.py b/mypy/types.py index d8598554aeb4..e9fc03d33f94 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -921,12 +921,14 @@ class TypedDictType(Type): whose TypeInfo has a typeddict_type that is anonymous. """ - items = None # type: OrderedDict[str, Type] # (item_name, item_type) + items = None # type: OrderedDict[str, Type] # item_name -> item_type + required_keys = None # type: Set[str] fallback = None # type: Instance - def __init__(self, items: 'OrderedDict[str, Type]', fallback: Instance, - line: int = -1, column: int = -1) -> None: + def __init__(self, items: 'OrderedDict[str, Type]', required_keys: Set[str], + fallback: Instance, line: int = -1, column: int = -1) -> None: self.items = items + self.required_keys = required_keys self.fallback = fallback self.can_be_true = len(self.items) > 0 self.can_be_false = len(self.items) == 0 @@ -938,6 +940,7 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: def serialize(self) -> JsonDict: return {'.class': 'TypedDictType', 'items': [[n, t.serialize()] for (n, t) in self.items.items()], + 'required_keys': sorted(self.required_keys), 'fallback': self.fallback.serialize(), } @@ -946,6 +949,7 @@ def deserialize(cls, data: JsonDict) -> 'TypedDictType': assert data['.class'] == 'TypedDictType' return TypedDictType(OrderedDict([(n, deserialize_type(t)) for (n, t) in data['items']]), + set(data['required_keys']), Instance.deserialize(data['fallback'])) def as_anonymous(self) -> 'TypedDictType': @@ -955,14 +959,15 @@ def as_anonymous(self) -> 'TypedDictType': return self.fallback.type.typeddict_type.as_anonymous() def copy_modified(self, *, fallback: Instance = None, - item_types: List[Type] = None) -> 'TypedDictType': + item_types: List[Type] = None, + required_keys: Set[str] = None) -> 'TypedDictType': if fallback is None: fallback = self.fallback if item_types is None: items = self.items else: items = OrderedDict(zip(self.items, item_types)) - return TypedDictType(items, fallback, self.line, self.column) + return TypedDictType(items, self.required_keys, fallback, self.line, self.column) def create_anonymous_fallback(self, *, value_type: Type) -> Instance: anonymous = self.as_anonymous() @@ -1371,6 +1376,7 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: for (item_name, item_type) in t.items.items() ]) return TypedDictType(items, + t.required_keys, # TODO: This appears to be unsafe. cast(Any, t.fallback.accept(self)), t.line, t.column) @@ -1516,11 +1522,17 @@ def visit_tuple_type(self, t: TupleType) -> str: def visit_typeddict_type(self, t: TypedDictType) -> str: s = self.keywords_str(t.items.items()) + if t.required_keys == set(t.items): + keys_str = '' + elif t.required_keys == set(): + keys_str = ', _total=False' + else: + keys_str = ', _required_keys=[{}]'.format(', '.join(sorted(t.required_keys))) if t.fallback and t.fallback.type: if s == '': - return 'TypedDict(_fallback={})'.format(t.fallback.accept(self)) + return 'TypedDict(_fallback={}{})'.format(t.fallback.accept(self), keys_str) else: - return 'TypedDict({}, _fallback={})'.format(s, t.fallback.accept(self)) + return 'TypedDict({}, _fallback={}{})'.format(s, t.fallback.accept(self), keys_str) return 'TypedDict({})'.format(s) def visit_star_type(self, t: StarType) -> str: diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index c16de82b67e5..e39d85957c1d 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -813,3 +813,83 @@ p = TaggedPoint(type='2d', x=42, y=1337) p.get('x', 1 + 'y') # E: Unsupported operand types for + ("int" and "str") [builtins fixtures/dict.pyi] [typing fixtures/typing-full.pyi] + + +-- Totality (the "total" keyword argument) + +[case testTypedDictWithTotalTrue] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int}, total=True) +d: D +reveal_type(d) # E: Revealed type is 'TypedDict(x=builtins.int, _fallback=__main__.D)' +[builtins fixtures/dict.pyi] + +[case testTypedDictWithInvalidTotalArgument] +from mypy_extensions import TypedDict +A = TypedDict('A', {'x': int}, total=0) # E: TypedDict() "total" argument must be True or False +B = TypedDict('B', {'x': int}, total=bool) # E: TypedDict() "total" argument must be True or False +C = TypedDict('C', {'x': int}, x=False) # E: Unexpected keyword argument "x" for "TypedDict" +D = TypedDict('D', {'x': int}, False) # E: Unexpected arguments to TypedDict() +[builtins fixtures/dict.pyi] + +[case testTypedDictWithTotalFalse] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int}, total=False) +d: D +reveal_type(d) # E: Revealed type is 'TypedDict(x=builtins.int, _fallback=__main__.D, _total=False)' +[builtins fixtures/dict.pyi] + +[case testTypedDictIndexingWithNonRequiredKey] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int, 'y': str}, total=False) +d: D +v = d['x'] # E: TypedDict key 'x' may be undefined \ + # N: Consider using get() instead +reveal_type(v) # E: Revealed type is 'builtins.int' +w = d['y'] # E: TypedDict key 'y' may be undefined \ + # N: Consider using get() instead +reveal_type(w) # E: Revealed type is 'builtins.str' +reveal_type(d.get('x')) # E: Revealed type is 'builtins.int' +reveal_type(d.get('y')) # E: Revealed type is 'builtins.str' +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + + +-- Create Type (Errors) + +[case testCannotCreateTypedDictTypeWithTooFewArguments] +from mypy_extensions import TypedDict +Point = TypedDict('Point') # E: Too few arguments for TypedDict() +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithTooManyArguments] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': int, 'y': int}, dict) # E: Unexpected arguments to TypedDict() +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithInvalidName] +from mypy_extensions import TypedDict +Point = TypedDict(dict, {'x': int, 'y': int}) # E: TypedDict() expects a string literal as the first argument +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithInvalidItems] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x'}) # E: TypedDict() expects a dictionary literal as the second argument +[builtins fixtures/dict.pyi] + +-- NOTE: The following code works at runtime but is not yet supported by mypy. +-- Keyword arguments may potentially be supported in the future. +[case testCannotCreateTypedDictTypeWithNonpositionalArgs] +from mypy_extensions import TypedDict +Point = TypedDict(typename='Point', fields={'x': int, 'y': int}) # E: Unexpected arguments to TypedDict() +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithInvalidItemName] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {int: int, int: int}) # E: Invalid TypedDict() field name +[builtins fixtures/dict.pyi] + +[case testCannotCreateTypedDictTypeWithInvalidItemType] +from mypy_extensions import TypedDict +Point = TypedDict('Point', {'x': 1, 'y': 1}) # E: Invalid field type +[builtins fixtures/dict.pyi] diff --git a/test-data/unit/lib-stub/mypy_extensions.pyi b/test-data/unit/lib-stub/mypy_extensions.pyi index fa540b99f4cd..a604c9684eeb 100644 --- a/test-data/unit/lib-stub/mypy_extensions.pyi +++ b/test-data/unit/lib-stub/mypy_extensions.pyi @@ -16,6 +16,6 @@ def VarArg(type: _T = ...) -> _T: ... def KwArg(type: _T = ...) -> _T: ... -def TypedDict(typename: str, fields: Dict[str, Type[_T]]) -> Type[dict]: ... +def TypedDict(typename: str, fields: Dict[str, Type[_T]], *, total: Any = ...) -> Type[dict]: ... class NoReturn: pass diff --git a/test-data/unit/semanal-typeddict.test b/test-data/unit/semanal-typeddict.test index ab6c428752d6..9c1454e49a06 100644 --- a/test-data/unit/semanal-typeddict.test +++ b/test-data/unit/semanal-typeddict.test @@ -34,43 +34,3 @@ MypyFile:1( AssignmentStmt:2( NameExpr(Point* [__main__.Point]) TypedDictExpr:2(Point))) - - --- Create Type (Errors) - -[case testCannotCreateTypedDictTypeWithTooFewArguments] -from mypy_extensions import TypedDict -Point = TypedDict('Point') # E: Too few arguments for TypedDict() -[builtins fixtures/dict.pyi] - -[case testCannotCreateTypedDictTypeWithTooManyArguments] -from mypy_extensions import TypedDict -Point = TypedDict('Point', {'x': int, 'y': int}, dict) # E: Too many arguments for TypedDict() -[builtins fixtures/dict.pyi] - -[case testCannotCreateTypedDictTypeWithInvalidName] -from mypy_extensions import TypedDict -Point = TypedDict(dict, {'x': int, 'y': int}) # E: TypedDict() expects a string literal as the first argument -[builtins fixtures/dict.pyi] - -[case testCannotCreateTypedDictTypeWithInvalidItems] -from mypy_extensions import TypedDict -Point = TypedDict('Point', {'x'}) # E: TypedDict() expects a dictionary literal as the second argument -[builtins fixtures/dict.pyi] - --- NOTE: The following code works at runtime but is not yet supported by mypy. --- Keyword arguments may potentially be supported in the future. -[case testCannotCreateTypedDictTypeWithNonpositionalArgs] -from mypy_extensions import TypedDict -Point = TypedDict(typename='Point', fields={'x': int, 'y': int}) # E: Unexpected arguments to TypedDict() -[builtins fixtures/dict.pyi] - -[case testCannotCreateTypedDictTypeWithInvalidItemName] -from mypy_extensions import TypedDict -Point = TypedDict('Point', {int: int, int: int}) # E: Invalid TypedDict() field name -[builtins fixtures/dict.pyi] - -[case testCannotCreateTypedDictTypeWithInvalidItemType] -from mypy_extensions import TypedDict -Point = TypedDict('Point', {'x': 1, 'y': 1}) # E: Invalid field type -[builtins fixtures/dict.pyi] From 44f53a9f76fbcbc1c3dbac318570545df0f2f934 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 15 Jun 2017 14:15:12 +0100 Subject: [PATCH 02/17] Support get(key, {}) and fix construction of partial typed dict --- mypy/checkexpr.py | 15 +++++++------ mypy/plugin.py | 26 +++++++++++++++++----- mypy/types.py | 4 +++- test-data/unit/check-typeddict.test | 34 ++++++++++++++++++++++++----- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index e5b3bda42919..934a67ec3036 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -305,13 +305,14 @@ def check_typeddict_call_with_kwargs(self, callee: TypedDictType, items = OrderedDict() # type: OrderedDict[str, Type] for (item_name, item_expected_type) in callee.items.items(): - item_value = kwargs[item_name] - - self.chk.check_simple_assignment( - lvalue_type=item_expected_type, rvalue=item_value, context=item_value, - msg=messages.INCOMPATIBLE_TYPES, - lvalue_name='TypedDict item "{}"'.format(item_name), - rvalue_name='expression') + if item_name in kwargs: + item_value = kwargs[item_name] + + self.chk.check_simple_assignment( + lvalue_type=item_expected_type, rvalue=item_value, context=item_value, + msg=messages.INCOMPATIBLE_TYPES, + lvalue_name='TypedDict item "{}"'.format(item_name), + rvalue_name='expression') items[item_name] = item_expected_type mapping_value_type = join.join_type_list(list(items.values())) diff --git a/mypy/plugin.py b/mypy/plugin.py index 7acd4d0b29a5..917a082a844f 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -1,6 +1,7 @@ +from collections import OrderedDict from typing import Callable, List, Tuple, Optional, NamedTuple, TypeVar -from mypy.nodes import Expression, StrExpr, IntExpr, UnaryExpr, Context +from mypy.nodes import Expression, StrExpr, IntExpr, UnaryExpr, Context, DictExpr from mypy.types import ( Type, Instance, CallableType, TypedDictType, UnionType, NoneTyp, FunctionLike, TypeVarType, AnyType @@ -205,17 +206,26 @@ def typed_dict_get_signature_callback( and len(args[0]) == 1 and isinstance(args[0][0], StrExpr) and len(signature.arg_types) == 2 - and len(signature.variables) == 1): + and len(signature.variables) == 1 + and len(args[1]) == 1): key = args[0][0].value value_type = object_type.items.get(key) + ret_type = signature.ret_type if value_type: + default_arg = args[1][0] + if (isinstance(value_type, TypedDictType) + and isinstance(default_arg, DictExpr) + and len(default_arg.items) == 0): + # Caller has empty dict {} as default for typed dict. + value_type = value_type.copy_modified(required_keys=set()) # Tweak the signature to include the value type as context. It's # only needed for type inference since there's a union with a type # variable that accepts everything. tv = TypeVarType(signature.variables[0]) return signature.copy_modified( arg_types=[signature.arg_types[0], - UnionType.make_simplified_union([value_type, tv])]) + UnionType.make_simplified_union([value_type, tv])], + ret_type=ret_type) return signature @@ -235,8 +245,14 @@ def typed_dict_get_callback( if value_type: if len(arg_types) == 1: return UnionType.make_simplified_union([value_type, NoneTyp()]) - elif len(arg_types) == 2 and len(arg_types[1]) == 1: - return UnionType.make_simplified_union([value_type, arg_types[1][0]]) + elif len(arg_types) == 2 and len(arg_types[1]) == 1 and len(args[1]) == 1: + default_arg = args[1][0] + if (isinstance(default_arg, DictExpr) and len(default_arg.items) == 0 + and isinstance(value_type, TypedDictType)): + # Special case '{}' as the default for a typed dict type. + return value_type.copy_modified(required_keys=set()) + else: + return UnionType.make_simplified_union([value_type, arg_types[1][0]]) else: context.msg.typeddict_item_name_not_found(object_type, key, context.context) return AnyType() diff --git a/mypy/types.py b/mypy/types.py index e9fc03d33f94..3fdb29933798 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -967,7 +967,9 @@ def copy_modified(self, *, fallback: Instance = None, items = self.items else: items = OrderedDict(zip(self.items, item_types)) - return TypedDictType(items, self.required_keys, fallback, self.line, self.column) + if required_keys is None: + required_keys = self.required_keys + return TypedDictType(items, required_keys, fallback, self.line, self.column) def create_anonymous_fallback(self, *, value_type: Type) -> Instance: anonymous = self.as_anonymous() diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index e39d85957c1d..f2352ee89747 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -802,7 +802,6 @@ D = TypedDict('D', {'x': int, 'y': str}) E = TypedDict('E', {'d': D}) p = E(d=D(x=0, y='')) reveal_type(p.get('d', {'x': 1, 'y': ''})) # E: Revealed type is 'TypedDict(x=builtins.int, y=builtins.str, _fallback=__main__.D)' -p.get('d', {}) # E: Expected TypedDict keys ('x', 'y') but found no keys [builtins fixtures/dict.pyi] [typing fixtures/typing-full.pyi] @@ -814,14 +813,31 @@ p.get('x', 1 + 'y') # E: Unsupported operand types for + ("int" and "str") [builtins fixtures/dict.pyi] [typing fixtures/typing-full.pyi] +[case testTypedDictChainedGetWithEmptyDictDefault] +# flags: --strict-optional +from mypy_extensions import TypedDict +C = TypedDict('C', {'a': int}) +D = TypedDict('D', {'x': C, 'y': str}) +d: D +reveal_type(d.get('x', {})) \ + # E: Revealed type is 'TypedDict(a=builtins.int, _fallback=__main__.C, _total=False)' +reveal_type(d.get('x', None)) \ + # E: Revealed type is 'Union[TypedDict(a=builtins.int, _fallback=__main__.C), builtins.None]' +reveal_type(d.get('x', {}).get('a')) # E: Revealed type is 'Union[builtins.int, builtins.None]' +d.get('x', {})['a'] # E: TypedDict key 'a' may be undefined \ + # N: Consider using get() instead +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + -- Totality (the "total" keyword argument) [case testTypedDictWithTotalTrue] from mypy_extensions import TypedDict -D = TypedDict('D', {'x': int}, total=True) +D = TypedDict('D', {'x': int, 'y': str}, total=True) d: D -reveal_type(d) # E: Revealed type is 'TypedDict(x=builtins.int, _fallback=__main__.D)' +reveal_type(d) \ + # E: Revealed type is 'TypedDict(x=builtins.int, y=builtins.str, _fallback=__main__.D)' [builtins fixtures/dict.pyi] [case testTypedDictWithInvalidTotalArgument] @@ -834,9 +850,15 @@ D = TypedDict('D', {'x': int}, False) # E: Unexpected arguments to TypedDict() [case testTypedDictWithTotalFalse] from mypy_extensions import TypedDict -D = TypedDict('D', {'x': int}, total=False) -d: D -reveal_type(d) # E: Revealed type is 'TypedDict(x=builtins.int, _fallback=__main__.D, _total=False)' +D = TypedDict('D', {'x': int, 'y': str}, total=False) +def f(d: D) -> None: + reveal_type(d) # E: Revealed type is 'TypedDict(x=builtins.int, y=builtins.str, _fallback=__main__.D, _total=False)' +f({}) +f({'x': 1}) +f({'y': ''}) +f({'x': 1, 'y': ''}) +f({'x': 1, 'z': ''}) # E: Expected TypedDict key 'x' but found keys ('x', 'z') +f({'x': ''}) # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") [builtins fixtures/dict.pyi] [case testTypedDictIndexingWithNonRequiredKey] From 6bb872e2bb814ee04ac7dda5c2222b98579ff7f9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 15 Jun 2017 15:19:46 +0100 Subject: [PATCH 03/17] Fix subtyping of non-total typed dicts --- mypy/subtypes.py | 7 ++++++- test-data/unit/check-typeddict.test | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index b03843fba9a4..836c3ffa9fd0 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -228,9 +228,14 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: elif isinstance(right, TypedDictType): if not left.names_are_wider_than(right): return False - for (_, l, r) in left.zip(right): + for name, l, r in left.zip(right): if not is_equivalent(l, r, self.check_type_parameter): return False + # Non-required key is not compatible with a required key since indexing + # may fail. Required key is not compatible with a non-required key + # since the prior doesn't support 'del' but the latter supports it. + if (name in left.required_keys) != (name in right.required_keys): + return False # (NOTE: Fallbacks don't matter.) return True else: diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index f2352ee89747..3ef6f4bb3ee3 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -876,6 +876,25 @@ reveal_type(d.get('y')) # E: Revealed type is 'builtins.str' [builtins fixtures/dict.pyi] [typing fixtures/typing-full.pyi] +[case testTypedDictSubtypingWithTotalFalse] +from mypy_extensions import TypedDict +A = TypedDict('A', {'x': int}) +B = TypedDict('B', {'x': int}, total=False) +C = TypedDict('C', {'x': int, 'y': str}, total=False) +def fa(a: A) -> None: pass +def fb(b: B) -> None: pass +def fc(c: C) -> None: pass +a: A +b: B +c: C +fb(b) +fc(c) +fb(c) +fb(a) # E: Argument 1 to "fb" has incompatible type "A"; expected "B" +fa(b) # E: Argument 1 to "fa" has incompatible type "B"; expected "A" +fc(b) # E: Argument 1 to "fc" has incompatible type "B"; expected "C" +[builtins fixtures/dict.pyi] + -- Create Type (Errors) From 4df39fc711c2e2ad3f7bc2ebd19dfbdc40bf7695 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 15 Jun 2017 15:39:10 +0100 Subject: [PATCH 04/17] Fix join with non-total typed dict --- mypy/join.py | 6 ++++-- test-data/unit/check-typeddict.test | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index c059561d026b..4d70fdb874b4 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -228,11 +228,13 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: items = OrderedDict([ (item_name, s_item_type) for (item_name, s_item_type, t_item_type) in self.s.zip(t) - if is_equivalent(s_item_type, t_item_type) + if (is_equivalent(s_item_type, t_item_type) and + (item_name in t.required_keys) == (item_name in self.s.required_keys)) ]) mapping_value_type = join_type_list(list(items.values())) fallback = self.s.create_anonymous_fallback(value_type=mapping_value_type) - return TypedDictType(items, set(items.keys()), fallback) # XXX required + required_keys = set(items.keys()) & t.required_keys & self.s.required_keys + return TypedDictType(items, required_keys, fallback) elif isinstance(self.s, Instance): return join_instances(self.s, t.fallback) else: diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 3ef6f4bb3ee3..637b74489c82 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -895,6 +895,29 @@ fa(b) # E: Argument 1 to "fa" has incompatible type "B"; expected "A" fc(b) # E: Argument 1 to "fc" has incompatible type "B"; expected "C" [builtins fixtures/dict.pyi] +[case testTypedDictJoinWithTotalFalse] +from typing import TypeVar +from mypy_extensions import TypedDict +A = TypedDict('A', {'x': int}) +B = TypedDict('B', {'x': int}, total=False) +C = TypedDict('C', {'x': int, 'y': str}, total=False) +T = TypeVar('T') +def j(x: T, y: T) -> T: return x +a: A +b: B +c: C +reveal_type(j(a, b)) \ + # E: Revealed type is 'TypedDict(_fallback=typing.Mapping[builtins.str, ])' +reveal_type(j(b, b)) \ + # E: Revealed type is 'TypedDict(x=builtins.int, _fallback=typing.Mapping[builtins.str, builtins.int], _total=False)' +reveal_type(j(c, c)) \ + # E: Revealed type is 'TypedDict(x=builtins.int, y=builtins.str, _fallback=typing.Mapping[builtins.str, builtins.object], _total=False)' +reveal_type(j(b, c)) \ + # E: Revealed type is 'TypedDict(x=builtins.int, _fallback=typing.Mapping[builtins.str, builtins.int], _total=False)' +reveal_type(j(c, b)) \ + # E: Revealed type is 'TypedDict(x=builtins.int, _fallback=typing.Mapping[builtins.str, builtins.int], _total=False)' +[builtins fixtures/dict.pyi] + -- Create Type (Errors) From 39f0d38a7860da8d940f63b23ce2240a3de42ee3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 15 Jun 2017 15:50:20 +0100 Subject: [PATCH 05/17] Fix meet with non-total typed dicts --- mypy/meet.py | 8 ++++--- test-data/unit/check-typeddict.test | 34 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/mypy/meet.py b/mypy/meet.py index 6771099f91f9..74cf29c11bf0 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -252,8 +252,9 @@ def visit_tuple_type(self, t: TupleType) -> Type: def visit_typeddict_type(self, t: TypedDictType) -> Type: if isinstance(self.s, TypedDictType): - for (_, l, r) in self.s.zip(t): - if not is_equivalent(l, r): + for (name, l, r) in self.s.zip(t): + if (not is_equivalent(l, r) or + (name in t.required_keys) != (name in self.s.required_keys)): return self.default(self.s) item_list = [] # type: List[Tuple[str, Type]] for (item_name, s_item_type, t_item_type) in self.s.zipall(t): @@ -266,7 +267,8 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: items = OrderedDict(item_list) mapping_value_type = join_type_list(list(items.values())) fallback = self.s.create_anonymous_fallback(value_type=mapping_value_type) - return TypedDictType(items, set(items.keys()), fallback) # XXX required + required_keys = set(items.keys()) & (t.required_keys | self.s.required_keys) + return TypedDictType(items, required_keys, fallback) else: return self.default(self.s) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 637b74489c82..228dacf0b531 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -520,6 +520,40 @@ def g(x: X, y: I) -> None: pass reveal_type(f(g)) # E: Revealed type is '' [builtins fixtures/dict.pyi] +[case testMeetOfTypedDictsWithNonTotal] +from mypy_extensions import TypedDict +from typing import TypeVar, Callable +XY = TypedDict('XY', {'x': int, 'y': int}, total=False) +YZ = TypedDict('YZ', {'y': int, 'z': int}, total=False) +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: XY, y: YZ) -> None: pass +reveal_type(f(g)) # E: Revealed type is 'TypedDict(x=builtins.int, y=builtins.int, z=builtins.int, _fallback=typing.Mapping[builtins.str, builtins.int], _total=False)' +[builtins fixtures/dict.pyi] + +[case testMeetOfTypedDictsWithNonTotalAndTotal] +from mypy_extensions import TypedDict +from typing import TypeVar, Callable +XY = TypedDict('XY', {'x': int}, total=False) +YZ = TypedDict('YZ', {'y': int, 'z': int}) +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: XY, y: YZ) -> None: pass +reveal_type(f(g)) # E: Revealed type is 'TypedDict(x=builtins.int, y=builtins.int, z=builtins.int, _fallback=typing.Mapping[builtins.str, builtins.int], _required_keys=[y, z])' +[builtins fixtures/dict.pyi] + +[case testMeetOfTypedDictsWithIncompatibleNonTotalAndTotal] +# flags: --strict-optional +from mypy_extensions import TypedDict +from typing import TypeVar, Callable +XY = TypedDict('XY', {'x': int, 'y': int}, total=False) +YZ = TypedDict('YZ', {'y': int, 'z': int}) +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: XY, y: YZ) -> None: pass +reveal_type(f(g)) # E: Revealed type is '' +[builtins fixtures/dict.pyi] + -- Constraint Solver From 7e042b837e685002239629aeecdc187f310ddab9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 15 Jun 2017 16:06:54 +0100 Subject: [PATCH 06/17] Add serialization test case --- test-data/unit/check-serialize.test | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test-data/unit/check-serialize.test b/test-data/unit/check-serialize.test index 2504fe6aa1c5..55ed8030455c 100644 --- a/test-data/unit/check-serialize.test +++ b/test-data/unit/check-serialize.test @@ -1026,6 +1026,19 @@ main:2: error: Revealed type is 'TypedDict(x=builtins.int, _fallback=typing.Mapp main:3: error: Revealed type is 'TypedDict(x=builtins.int, _fallback=ntcrash.C.A@4)' main:4: error: Revealed type is 'def () -> ntcrash.C.A@4' +[case testSerializeNonTotalTypedDict] +from m import d +reveal_type(d) +[file m.py] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int, 'y': str}, total=False) +d: D +[builtins fixtures/dict.pyi] +[out1] +main:2: error: Revealed type is 'TypedDict(x=builtins.int, y=builtins.str, _fallback=m.D, _total=False)' +[out2] +main:2: error: Revealed type is 'TypedDict(x=builtins.int, y=builtins.str, _fallback=m.D, _total=False)' + -- -- Modules -- From 981023a012fa7da34ef2daaa5f46357fba54db48 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 16 Jun 2017 13:14:28 +0100 Subject: [PATCH 07/17] Support TypedDict total keyword argument with class syntax --- mypy/fastparse.py | 5 ++- mypy/nodes.py | 6 ++- mypy/semanal.py | 67 ++++++++++++++++++----------- test-data/unit/check-typeddict.test | 32 ++++++++++++++ 4 files changed, 82 insertions(+), 28 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 10ad642dcdf0..58835c6de810 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -467,12 +467,15 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: metaclass = stringify_name(metaclass_arg.value) if metaclass is None: metaclass = '' # To be reported later + keywords = [(kw.arg, self.visit(kw.value)) + for kw in n.keywords] cdef = ClassDef(n.name, self.as_block(n.body, n.lineno), None, self.translate_expr_list(n.bases), - metaclass=metaclass) + metaclass=metaclass, + keywords=keywords) cdef.decorators = self.translate_expr_list(n.decorator_list) self.class_nesting -= 1 return cdef diff --git a/mypy/nodes.py b/mypy/nodes.py index cfd74b69f577..92f7aab53ba2 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2,6 +2,7 @@ import os from abc import abstractmethod +from collections import OrderedDict from typing import ( Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional, Callable, @@ -731,6 +732,7 @@ class ClassDef(Statement): info = None # type: TypeInfo # Related TypeInfo metaclass = '' # type: Optional[str] decorators = None # type: List[Expression] + keywords = None # type: OrderedDict[str, Expression] analyzed = None # type: Optional[Expression] has_incompatible_baseclass = False @@ -739,13 +741,15 @@ def __init__(self, defs: 'Block', type_vars: List['mypy.types.TypeVarDef'] = None, base_type_exprs: List[Expression] = None, - metaclass: str = None) -> None: + metaclass: str = None, + keywords: List[Tuple[str, Expression]] = None) -> None: self.name = name self.defs = defs self.type_vars = type_vars or [] self.base_type_exprs = base_type_exprs or [] self.metaclass = metaclass self.decorators = [] + self.keywords = OrderedDict(keywords or []) def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_class_def(self) diff --git a/mypy/semanal.py b/mypy/semanal.py index 6cd37c22be10..cf0ffc5a3fb4 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -654,6 +654,7 @@ def visit_class_def(self, defn: ClassDef) -> None: def analyze_class_body(self, defn: ClassDef) -> Iterator[bool]: with self.tvar_scope_frame(self.tvar_scope.class_frame()): self.clean_up_bases_and_infer_type_variables(defn) + self.analyze_class_keywords(defn) if self.analyze_typeddict_classdef(defn): yield False return @@ -703,6 +704,10 @@ def analyze_class_body(self, defn: ClassDef) -> Iterator[bool]: self.leave_class() + def analyze_class_keywords(self, defn: ClassDef) -> None: + for value in defn.keywords.values(): + value.accept(self) + def enter_class(self, info: TypeInfo) -> None: # Remember previous active class self.type_stack.append(self.type) @@ -1201,8 +1206,8 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> bool: isinstance(defn.base_type_exprs[0], RefExpr) and defn.base_type_exprs[0].fullname == 'mypy_extensions.TypedDict'): # Building a new TypedDict - fields, types = self.check_typeddict_classdef(defn) - info = self.build_typeddict_typeinfo(defn.name, fields, types) + fields, types, required_keys = self.check_typeddict_classdef(defn) + info = self.build_typeddict_typeinfo(defn.name, fields, types, required_keys) node.node = info defn.analyzed = TypedDictExpr(info) return True @@ -1212,38 +1217,43 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> bool: not self.is_typeddict(expr) for expr in defn.base_type_exprs): self.fail("All bases of a new TypedDict must be TypedDict types", defn) typeddict_bases = list(filter(self.is_typeddict, defn.base_type_exprs)) - newfields = [] # type: List[str] - newtypes = [] # type: List[Type] - tpdict = None # type: OrderedDict[str, Type] + keys = [] # type: List[str] + types = [] + required_keys = set() for base in typeddict_bases: assert isinstance(base, RefExpr) assert isinstance(base.node, TypeInfo) assert isinstance(base.node.typeddict_type, TypedDictType) - tpdict = base.node.typeddict_type.items - newdict = tpdict.copy() - for key in tpdict: - if key in newfields: + base_typed_dict = base.node.typeddict_type + base_items = base_typed_dict.items + valid_items = base_items.copy() + for key in base_items: + if key in keys: self.fail('Cannot overwrite TypedDict field "{}" while merging' .format(key), defn) - newdict.pop(key) - newfields.extend(newdict.keys()) - newtypes.extend(newdict.values()) - fields, types = self.check_typeddict_classdef(defn, newfields) - newfields.extend(fields) - newtypes.extend(types) - info = self.build_typeddict_typeinfo(defn.name, newfields, newtypes) + valid_items.pop(key) + keys.extend(valid_items.keys()) + types.extend(valid_items.values()) + required_keys.update(base_typed_dict.required_keys) + new_keys, new_types, new_required_keys = self.check_typeddict_classdef(defn, keys) + keys.extend(new_keys) + types.extend(new_types) + required_keys.update(new_required_keys) + info = self.build_typeddict_typeinfo(defn.name, keys, types, required_keys) node.node = info defn.analyzed = TypedDictExpr(info) return True return False def check_typeddict_classdef(self, defn: ClassDef, - oldfields: List[str] = None) -> Tuple[List[str], List[Type]]: + oldfields: List[str] = None) -> Tuple[List[str], + List[Type], + Set[str]]: TPDICT_CLASS_ERROR = ('Invalid statement in TypedDict definition; ' 'expected "field_name: field_type"') if self.options.python_version < (3, 6): self.fail('TypedDict class syntax is only supported in Python 3.6', defn) - return [], [] + return [], [], set() fields = [] # type: List[str] types = [] # type: List[Type] for stmt in defn.defs.body: @@ -1274,7 +1284,14 @@ def check_typeddict_classdef(self, defn: ClassDef, elif not isinstance(stmt.rvalue, TempNode): # x: int assigns rvalue to TempNode(AnyType()) self.fail('Right hand side values are not supported in TypedDict', stmt) - return fields, types + total = True + if 'total' in defn.keywords: + total = self.parse_bool(defn.keywords['total']) + if total is None: + self.fail('Value of "total" must be True or False', defn) + total = True + required_keys = set(fields) if total else set() + return fields, types, required_keys def visit_import(self, i: Import) -> None: for id, as_id in i.ids: @@ -2305,13 +2322,14 @@ def check_typeddict(self, node: Expression, var_name: str = None) -> Optional[Ty items, types, total, ok = self.parse_typeddict_args(call, fullname) if not ok: # Error. Construct dummy return value. - info = self.build_typeddict_typeinfo('TypedDict', [], []) + info = self.build_typeddict_typeinfo('TypedDict', [], [], set()) else: name = cast(StrExpr, call.args[0]).value if name != var_name or self.is_func_scope(): # Give it a unique name derived from the line number. name += '@' + str(call.line) - info = self.build_typeddict_typeinfo(name, items, types, total) + required_keys = set(items) if total else set() + info = self.build_typeddict_typeinfo(name, items, types, required_keys) # Store it as a global just in case it would remain anonymous. # (Or in the nearest class if there is one.) stnode = SymbolTableNode(GDEF, info, self.cur_mod_id) @@ -2385,17 +2403,14 @@ def fail_typeddict_arg(self, message: str, return [], [], True, False def build_typeddict_typeinfo(self, name: str, items: List[str], - types: List[Type], total: bool = True) -> TypeInfo: + types: List[Type], + required_keys: Set[str]) -> TypeInfo: mapping_value_type = join.join_type_list(types) fallback = (self.named_type_or_none('typing.Mapping', [self.str_type(), mapping_value_type]) or self.object_type()) info = self.basic_new_typeinfo(name, fallback) - if total: - required_keys = set(items) - else: - required_keys = set() info.typeddict_type = TypedDictType(OrderedDict(zip(items, types)), required_keys, fallback) return info diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 228dacf0b531..863492b8acb9 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -952,6 +952,38 @@ reveal_type(j(c, b)) \ # E: Revealed type is 'TypedDict(x=builtins.int, _fallback=typing.Mapping[builtins.str, builtins.int], _total=False)' [builtins fixtures/dict.pyi] +[case testTypedDictClassWithTotalArgument] +from mypy_extensions import TypedDict +class D(TypedDict, total=False): + x: int + y: str +d: D +reveal_type(d) # E: Revealed type is 'TypedDict(x=builtins.int, y=builtins.str, _fallback=__main__.D, _total=False)' +[builtins fixtures/dict.pyi] + +[case testTypedDictClassWithInvalidTotalArgument] +from mypy_extensions import TypedDict +class D(TypedDict, total=1): # E: Value of "total" must be True or False + x: int +class E(TypedDict, total=bool): # E: Value of "total" must be True or False + x: int +class F(TypedDict, total=xyz): # E: Value of "total" must be True or False \ + # E: Name 'xyz' is not defined + x: int +[builtins fixtures/dict.pyi] + +[case testTypedDictClassInheritanceWithTotalArgument] +from mypy_extensions import TypedDict +class A(TypedDict): + x: int +class B(TypedDict, A, total=False): + y: int +class C(TypedDict, B, total=True): + z: str +c: C +reveal_type(c) # E: Revealed type is 'TypedDict(x=builtins.int, y=builtins.int, z=builtins.str, _fallback=__main__.C, _required_keys=[x, z])' +[builtins fixtures/dict.pyi] + -- Create Type (Errors) From 6223d2a06a6a74b03d38b24d11d696b65c81a782 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 16 Jun 2017 13:46:14 +0100 Subject: [PATCH 08/17] Attempt to fix Python 3.3 --- mypy/checkexpr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 934a67ec3036..e75712dc4ddf 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -292,7 +292,7 @@ def check_typeddict_call_with_dict(self, callee: TypedDictType, def check_typeddict_call_with_kwargs(self, callee: TypedDictType, kwargs: 'OrderedDict[str, Expression]', context: Context) -> Type: - if not (callee.required_keys <= kwargs.keys() <= callee.items.keys()): + if not (callee.required_keys <= set(kwargs.keys()) <= set(callee.items.keys())): callee_item_names = [key for key in callee.items.keys() if key in callee.required_keys or key in kwargs.keys()] kwargs_item_names = kwargs.keys() From 5ecafb2549b7c77f5efacbad7271af51fdc3b723 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 16 Jun 2017 18:36:23 +0100 Subject: [PATCH 09/17] Add minimal runtime `total` support to mypy_extensions There is no support for introspection of `total` yet. --- extensions/mypy_extensions.py | 3 ++- mypy/test/testextensions.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/extensions/mypy_extensions.py b/extensions/mypy_extensions.py index 82eea32a31d8..45e66533182c 100644 --- a/extensions/mypy_extensions.py +++ b/extensions/mypy_extensions.py @@ -30,6 +30,7 @@ def _dict_new(cls, *args, **kwargs): def _typeddict_new(cls, _typename, _fields=None, **kwargs): + total = kwargs.pop('total', True) if _fields is None: _fields = kwargs elif kwargs: @@ -39,7 +40,7 @@ def _typeddict_new(cls, _typename, _fields=None, **kwargs): class _TypedDictMeta(type): - def __new__(cls, name, bases, ns): + def __new__(cls, name, bases, ns, total=True): # Create new typed dict class object. # This method is called directly when TypedDict is subclassed, # or via _typeddict_new when TypedDict is instantiated. This way diff --git a/mypy/test/testextensions.py b/mypy/test/testextensions.py index af3916f98e19..aeb596f59c59 100644 --- a/mypy/test/testextensions.py +++ b/mypy/test/testextensions.py @@ -120,6 +120,16 @@ def test_optional(self): self.assertEqual(typing.Optional[EmpD], typing.Union[None, EmpD]) self.assertNotEqual(typing.List[EmpD], typing.Tuple[EmpD]) + def test_total(self): + D = TypedDict('D', {'x': int}, total=False) + assert D() == {} + assert D(x=1) == {'x': 1} + + class E(TypedDict, total=False): + x: int + assert E() == {} + assert E(x=1) == {'x': 1} + if __name__ == '__main__': main() From 1c068e07906b8aa46fff4ebda218beee8c4a0a46 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jun 2017 16:22:20 +0100 Subject: [PATCH 10/17] Fix tests on pre-3.6 Python and improve introspection Make TypedDict `total` introspectable. --- extensions/mypy_extensions.py | 5 ++++- mypy/test/testextensions.py | 22 +++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/extensions/mypy_extensions.py b/extensions/mypy_extensions.py index 45e66533182c..c711e0023a0f 100644 --- a/extensions/mypy_extensions.py +++ b/extensions/mypy_extensions.py @@ -36,7 +36,8 @@ def _typeddict_new(cls, _typename, _fields=None, **kwargs): elif kwargs: raise TypeError("TypedDict takes either a dict or keyword arguments," " but not both") - return _TypedDictMeta(_typename, (), {'__annotations__': dict(_fields)}) + return _TypedDictMeta(_typename, (), {'__annotations__': dict(_fields), + '__total__': total}) class _TypedDictMeta(type): @@ -60,6 +61,8 @@ def __new__(cls, name, bases, ns, total=True): for base in bases: anns.update(base.__dict__.get('__annotations__', {})) tp_dict.__annotations__ = anns + if not hasattr(tp_dict, '__total__'): + tp_dict.__total__ = total return tp_dict __instancecheck__ = __subclasscheck__ = _check_fails diff --git a/mypy/test/testextensions.py b/mypy/test/testextensions.py index aeb596f59c59..30d766d2e114 100644 --- a/mypy/test/testextensions.py +++ b/mypy/test/testextensions.py @@ -37,6 +37,10 @@ class Point2D(TypedDict): y: int class LabelPoint2D(Point2D, Label): ... + +class Options(TypedDict, total=False): + log_level: int + log_path: str """ if PY36: @@ -58,6 +62,7 @@ def test_basics_iterable_syntax(self): self.assertEqual(Emp.__module__, 'mypy.test.testextensions') self.assertEqual(Emp.__bases__, (dict,)) self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) + self.assertEqual(Emp.__total__, True) def test_basics_keywords_syntax(self): Emp = TypedDict('Emp', name=str, id=int) @@ -72,6 +77,7 @@ def test_basics_keywords_syntax(self): self.assertEqual(Emp.__module__, 'mypy.test.testextensions') self.assertEqual(Emp.__bases__, (dict,)) self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) + self.assertEqual(Emp.__total__, True) def test_typeddict_errors(self): Emp = TypedDict('Emp', {'name': str, 'id': int}) @@ -94,6 +100,7 @@ def test_typeddict_errors(self): def test_py36_class_syntax_usage(self): self.assertEqual(LabelPoint2D.__annotations__, {'x': int, 'y': int, 'label': str}) # noqa self.assertEqual(LabelPoint2D.__bases__, (dict,)) # noqa + self.assertEqual(LabelPoint2D.__total__, True) self.assertNotIsSubclass(LabelPoint2D, typing.Sequence) # noqa not_origin = Point2D(x=0, y=1) # noqa self.assertEqual(not_origin['x'], 0) @@ -122,13 +129,14 @@ def test_optional(self): def test_total(self): D = TypedDict('D', {'x': int}, total=False) - assert D() == {} - assert D(x=1) == {'x': 1} - - class E(TypedDict, total=False): - x: int - assert E() == {} - assert E(x=1) == {'x': 1} + self.assertEqual(D(), {}) + self.assertEqual(D(x=1), {'x': 1}) + self.assertEqual(D.__total__, False) + + if PY36: + self.assertEqual(Options(), {}) + self.assertEqual(Options(log_level=2), {'log_level': 2}) + self.assertEqual(Options.__total__, False) if __name__ == '__main__': From 4429ce14c51296390b55e5c72770ad3fade016ca Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jun 2017 17:28:23 +0100 Subject: [PATCH 11/17] Fix lint --- mypy/test/testextensions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy/test/testextensions.py b/mypy/test/testextensions.py index 30d766d2e114..2203cf814f00 100644 --- a/mypy/test/testextensions.py +++ b/mypy/test/testextensions.py @@ -100,7 +100,7 @@ def test_typeddict_errors(self): def test_py36_class_syntax_usage(self): self.assertEqual(LabelPoint2D.__annotations__, {'x': int, 'y': int, 'label': str}) # noqa self.assertEqual(LabelPoint2D.__bases__, (dict,)) # noqa - self.assertEqual(LabelPoint2D.__total__, True) + self.assertEqual(LabelPoint2D.__total__, True) # noqa self.assertNotIsSubclass(LabelPoint2D, typing.Sequence) # noqa not_origin = Point2D(x=0, y=1) # noqa self.assertEqual(not_origin['x'], 0) @@ -134,9 +134,9 @@ def test_total(self): self.assertEqual(D.__total__, False) if PY36: - self.assertEqual(Options(), {}) - self.assertEqual(Options(log_level=2), {'log_level': 2}) - self.assertEqual(Options.__total__, False) + self.assertEqual(Options(), {}) # noqa + self.assertEqual(Options(log_level=2), {'log_level': 2}) # noqa + self.assertEqual(Options.__total__, False) # noqa if __name__ == '__main__': From 5359a98009a57a10430d502d2c5c3bfb88042b0d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jun 2017 15:46:08 +0100 Subject: [PATCH 12/17] Fix problems caused by merge --- mypy/plugin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/plugin.py b/mypy/plugin.py index 28b849a158fa..7349915c92a0 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -268,8 +268,9 @@ def typed_dict_get_signature_callback(ctx: MethodSigContext) -> CallableType: and len(ctx.args[1]) == 1): key = ctx.args[0][0].value value_type = ctx.type.items.get(key) + ret_type = signature.ret_type if value_type: - default_arg = args[1][0] + default_arg = ctx.args[1][0] if (isinstance(value_type, TypedDictType) and isinstance(default_arg, DictExpr) and len(default_arg.items) == 0): From a09c5b9c2aea4220e0af4b06e863e2965ae1a809 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jun 2017 16:02:19 +0100 Subject: [PATCH 13/17] Allow td['key'] even if td is not total --- mypy/checkexpr.py | 2 -- mypy/messages.py | 7 ------- test-data/unit/check-typeddict.test | 11 +++-------- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index f38aeaced219..85ba9b865b2c 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1668,8 +1668,6 @@ def visit_typeddict_index_expr(self, td_type: TypedDictType, index: Expression) if item_type is None: self.msg.typeddict_item_name_not_found(td_type, item_name, index) return AnyType() - if item_name not in td_type.required_keys: - self.msg.typeddict_item_may_be_undefined(item_name, index) return item_type def visit_enum_index_expr(self, enum_type: TypeInfo, index: Expression, diff --git a/mypy/messages.py b/mypy/messages.py index 59af24e474be..067cbd37f744 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -901,13 +901,6 @@ def typeddict_item_name_not_found(self, self.fail('\'{}\' is not a valid TypedDict key; expected one of {}'.format( item_name, format_item_name_list(typ.items.keys())), context) - def typeddict_item_may_be_undefined(self, - item_name: str, - context: Context, - ) -> None: - self.fail("TypedDict key '{}' may be undefined".format(item_name), context) - self.note("Consider using get() instead", context) - def type_arguments_not_allowed(self, context: Context) -> None: self.fail('Parameterized generics cannot be used with class or instance checks', context) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 863492b8acb9..4d55c4447710 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -858,8 +858,7 @@ reveal_type(d.get('x', {})) \ reveal_type(d.get('x', None)) \ # E: Revealed type is 'Union[TypedDict(a=builtins.int, _fallback=__main__.C), builtins.None]' reveal_type(d.get('x', {}).get('a')) # E: Revealed type is 'Union[builtins.int, builtins.None]' -d.get('x', {})['a'] # E: TypedDict key 'a' may be undefined \ - # N: Consider using get() instead +reveal_type(d.get('x', {})['a']) # E: Revealed type is 'builtins.int' [builtins fixtures/dict.pyi] [typing fixtures/typing-full.pyi] @@ -899,12 +898,8 @@ f({'x': ''}) # E: Incompatible types (expression has type "str", TypedDict item from mypy_extensions import TypedDict D = TypedDict('D', {'x': int, 'y': str}, total=False) d: D -v = d['x'] # E: TypedDict key 'x' may be undefined \ - # N: Consider using get() instead -reveal_type(v) # E: Revealed type is 'builtins.int' -w = d['y'] # E: TypedDict key 'y' may be undefined \ - # N: Consider using get() instead -reveal_type(w) # E: Revealed type is 'builtins.str' +reveal_type(d['x']) # E: Revealed type is 'builtins.int' +reveal_type(d['y']) # E: Revealed type is 'builtins.str' reveal_type(d.get('x')) # E: Revealed type is 'builtins.int' reveal_type(d.get('y')) # E: Revealed type is 'builtins.str' [builtins fixtures/dict.pyi] From 059bc21699c3801e9827f9a04a5c4630d31b703e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jun 2017 16:04:09 +0100 Subject: [PATCH 14/17] Fix lint --- mypy/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/plugin.py b/mypy/plugin.py index 7349915c92a0..37e516bc5030 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -299,7 +299,7 @@ def typed_dict_get_callback(ctx: MethodContext) -> Type: if len(ctx.arg_types) == 1: return UnionType.make_simplified_union([value_type, NoneTyp()]) elif (len(ctx.arg_types) == 2 and len(ctx.arg_types[1]) == 1 - and len(ctx.args[1]) == 1): + and len(ctx.args[1]) == 1): default_arg = ctx.args[1][0] if (isinstance(default_arg, DictExpr) and len(default_arg.items) == 0 and isinstance(value_type, TypedDictType)): From b47857e32cf1d9596214e299923886afe0f5f8c3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jun 2017 16:09:22 +0100 Subject: [PATCH 15/17] Add test case --- test-data/unit/check-typeddict.test | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 4d55c4447710..ad4efe97db12 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -894,6 +894,18 @@ f({'x': 1, 'z': ''}) # E: Expected TypedDict key 'x' but found keys ('x', 'z') f({'x': ''}) # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") [builtins fixtures/dict.pyi] +[case testTypedDictConstructorWithTotalFalse] +from mypy_extensions import TypedDict +D = TypedDict('D', {'x': int, 'y': str}, total=False) +def f(d: D) -> None: pass +reveal_type(D()) # E: Revealed type is 'TypedDict(x=builtins.int, y=builtins.str, _fallback=typing.Mapping[builtins.str, builtins.object], _total=False)' +reveal_type(D(x=1)) # E: Revealed type is 'TypedDict(x=builtins.int, y=builtins.str, _fallback=typing.Mapping[builtins.str, builtins.object], _total=False)' +f(D(y='')) +f(D(x=1, y='')) +f(D(x=1, z='')) # E: Expected TypedDict key 'x' but found keys ('x', 'z') +f(D(x='')) # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") +[builtins fixtures/dict.pyi] + [case testTypedDictIndexingWithNonRequiredKey] from mypy_extensions import TypedDict D = TypedDict('D', {'x': int, 'y': str}, total=False) From 31b6696ad0bb2254b7fc4b927a7a915922d517eb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jun 2017 13:05:07 +0100 Subject: [PATCH 16/17] Address review feedback --- mypy/checkexpr.py | 12 +++++------- mypy/join.py | 2 ++ mypy/meet.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 46acaba431c3..00c95cb2e6d4 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -293,13 +293,12 @@ def check_typeddict_call_with_kwargs(self, callee: TypedDictType, kwargs: 'OrderedDict[str, Expression]', context: Context) -> Type: if not (callee.required_keys <= set(kwargs.keys()) <= set(callee.items.keys())): - callee_item_names = [key for key in callee.items.keys() - if key in callee.required_keys or key in kwargs.keys()] - kwargs_item_names = kwargs.keys() - + expected_item_names = [key for key in callee.items.keys() + if key in callee.required_keys or key in kwargs.keys()] + actual_item_names = kwargs.keys() self.msg.typeddict_instantiated_with_unexpected_items( - expected_item_names=list(callee_item_names), - actual_item_names=list(kwargs_item_names), + expected_item_names=list(expected_item_names), + actual_item_names=list(actual_item_names), context=context) return AnyType() @@ -307,7 +306,6 @@ def check_typeddict_call_with_kwargs(self, callee: TypedDictType, for (item_name, item_expected_type) in callee.items.items(): if item_name in kwargs: item_value = kwargs[item_name] - self.chk.check_simple_assignment( lvalue_type=item_expected_type, rvalue=item_value, context=item_value, msg=messages.INCOMPATIBLE_TYPES, diff --git a/mypy/join.py b/mypy/join.py index 4d70fdb874b4..0ae8c3ab4058 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -233,6 +233,8 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: ]) mapping_value_type = join_type_list(list(items.values())) fallback = self.s.create_anonymous_fallback(value_type=mapping_value_type) + # We need to filter by items.keys() since some required keys present in both t and + # self.s might be missing from the join if the types are incompatible. required_keys = set(items.keys()) & t.required_keys & self.s.required_keys return TypedDictType(items, required_keys, fallback) elif isinstance(self.s, Instance): diff --git a/mypy/meet.py b/mypy/meet.py index 74cf29c11bf0..f0dcd8b56e34 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -267,7 +267,7 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: items = OrderedDict(item_list) mapping_value_type = join_type_list(list(items.values())) fallback = self.s.create_anonymous_fallback(value_type=mapping_value_type) - required_keys = set(items.keys()) & (t.required_keys | self.s.required_keys) + required_keys = t.required_keys | self.s.required_keys return TypedDictType(items, required_keys, fallback) else: return self.default(self.s) From d32fd68891159875488e349a336fefe1b327deb9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jun 2017 13:36:56 +0100 Subject: [PATCH 17/17] Update comment --- mypy/subtypes.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 836c3ffa9fd0..ebd6a1d13d3c 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -231,9 +231,15 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: for name, l, r in left.zip(right): if not is_equivalent(l, r, self.check_type_parameter): return False - # Non-required key is not compatible with a required key since indexing - # may fail. Required key is not compatible with a non-required key - # since the prior doesn't support 'del' but the latter supports it. + # Non-required key is not compatible with a required key since + # indexing may fail unexpectedly if a required key is missing. + # Required key is not compatible with a non-required key since + # the prior doesn't support 'del' but the latter should support + # it. + # + # NOTE: 'del' support is currently not implemented (#3550). We + # don't want to have to change subtyping after 'del' support + # lands so here we are anticipating that change. if (name in left.required_keys) != (name in right.required_keys): return False # (NOTE: Fallbacks don't matter.)