From 77daf906d05bbabc92b337fff3724e51afc27b05 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sat, 4 Jan 2020 13:36:11 +0300 Subject: [PATCH 01/76] refactor, fix method copying --- mypy_django_plugin/django/context.py | 8 +- mypy_django_plugin/lib/chk_helpers.py | 112 ++++++ mypy_django_plugin/lib/helpers.py | 246 ++----------- mypy_django_plugin/lib/sem_helpers.py | 117 +++++++ mypy_django_plugin/main.py | 5 +- mypy_django_plugin/transformers/fields.py | 28 +- mypy_django_plugin/transformers/forms.py | 6 +- .../transformers/init_create.py | 14 +- mypy_django_plugin/transformers/managers.py | 180 +++++++--- mypy_django_plugin/transformers/meta.py | 8 +- mypy_django_plugin/transformers/models.py | 326 ++++++++++++------ .../transformers/orm_lookups.py | 10 +- mypy_django_plugin/transformers/querysets.py | 20 +- mypy_django_plugin/transformers/request.py | 9 +- mypy_django_plugin/transformers/settings.py | 6 +- .../managers/querysets/test_from_queryset.yml | 56 ++- .../typecheck/managers/test_managers.yml | 8 +- 17 files changed, 714 insertions(+), 445 deletions(-) create mode 100644 mypy_django_plugin/lib/chk_helpers.py create mode 100644 mypy_django_plugin/lib/sem_helpers.py diff --git a/mypy_django_plugin/django/context.py b/mypy_django_plugin/django/context.py index 36ac0f869..eeee4512e 100644 --- a/mypy_django_plugin/django/context.py +++ b/mypy_django_plugin/django/context.py @@ -21,7 +21,7 @@ from mypy.types import Type as MypyType from mypy.types import TypeOfAny, UnionType -from mypy_django_plugin.lib import fullnames, helpers +from mypy_django_plugin.lib import fullnames, helpers, chk_helpers try: from django.contrib.postgres.fields import ArrayField @@ -356,11 +356,11 @@ def resolve_lookup_expected_type(self, ctx: MethodContext, model_cls: Type[Model return AnyType(TypeOfAny.explicit) if lookup_cls is None or isinstance(lookup_cls, Exact): - return self.get_field_lookup_exact_type(helpers.get_typechecker_api(ctx), field) + return self.get_field_lookup_exact_type(chk_helpers.get_typechecker_api(ctx), field) assert lookup_cls is not None - lookup_info = helpers.lookup_class_typeinfo(helpers.get_typechecker_api(ctx), lookup_cls) + lookup_info = helpers.lookup_class_typeinfo(chk_helpers.get_typechecker_api(ctx), lookup_cls) if lookup_info is None: return AnyType(TypeOfAny.explicit) @@ -370,7 +370,7 @@ def resolve_lookup_expected_type(self, ctx: MethodContext, model_cls: Type[Model # if it's Field, consider lookup_type a __get__ of current field if (isinstance(lookup_type, Instance) and lookup_type.type.fullname == fullnames.FIELD_FULLNAME): - field_info = helpers.lookup_class_typeinfo(helpers.get_typechecker_api(ctx), field.__class__) + field_info = helpers.lookup_class_typeinfo(chk_helpers.get_typechecker_api(ctx), field.__class__) if field_info is None: return AnyType(TypeOfAny.explicit) lookup_type = helpers.get_private_descriptor_type(field_info, '_pyi_private_get_type', diff --git a/mypy_django_plugin/lib/chk_helpers.py b/mypy_django_plugin/lib/chk_helpers.py new file mode 100644 index 000000000..22c12421d --- /dev/null +++ b/mypy_django_plugin/lib/chk_helpers.py @@ -0,0 +1,112 @@ +from typing import OrderedDict, List, Optional, Dict, Set, Union + +from mypy import checker +from mypy.checker import TypeChecker +from mypy.nodes import MypyFile, TypeInfo, Var, MDEF, SymbolTableNode, GDEF, Expression +from mypy.plugin import CheckerPluginInterface, FunctionContext, MethodContext, AttributeContext +from mypy.types import Type as MypyType, Instance, TupleType, TypeOfAny, AnyType, TypedDictType + +from mypy_django_plugin.lib import helpers + + +def add_new_class_for_current_module(current_module: MypyFile, + name: str, + bases: List[Instance], + fields: Optional[Dict[str, MypyType]] = None + ) -> TypeInfo: + new_class_unique_name = checker.gen_unique_name(name, current_module.names) + new_typeinfo = helpers.new_typeinfo(new_class_unique_name, + bases=bases, + module_name=current_module.fullname) + # new_typeinfo = helpers.make_new_typeinfo_in_current_module(new_class_unique_name, + # bases=bases, + # current_module_fullname=current_module.fullname) + # add fields + if fields: + for field_name, field_type in fields.items(): + var = Var(field_name, type=field_type) + var.info = new_typeinfo + var._fullname = new_typeinfo.fullname + '.' + field_name + new_typeinfo.names[field_name] = SymbolTableNode(MDEF, var, plugin_generated=True) + + current_module.names[new_class_unique_name] = SymbolTableNode(GDEF, new_typeinfo, plugin_generated=True) + current_module.defs.append(new_typeinfo.defn) + return new_typeinfo + + +def make_oneoff_named_tuple(api: TypeChecker, name: str, fields: 'Dict[str, MypyType]') -> TupleType: + current_module = helpers.get_current_module(api) + namedtuple_info = add_new_class_for_current_module(current_module, name, + bases=[api.named_generic_type('typing.NamedTuple', [])], + fields=fields) + return TupleType(list(fields.values()), fallback=Instance(namedtuple_info, [])) + + +def make_tuple(api: 'TypeChecker', fields: List[MypyType]) -> TupleType: + # fallback for tuples is any builtins.tuple instance + fallback = api.named_generic_type('builtins.tuple', + [AnyType(TypeOfAny.special_form)]) + return TupleType(fields, fallback=fallback) + + +def make_oneoff_typeddict(api: CheckerPluginInterface, fields: 'OrderedDict[str, MypyType]', + required_keys: Set[str]) -> TypedDictType: + object_type = api.named_generic_type('mypy_extensions._TypedDict', []) + typed_dict_type = TypedDictType(fields, required_keys=required_keys, fallback=object_type) + return typed_dict_type + + +def get_typechecker_api(ctx: Union[AttributeContext, MethodContext, FunctionContext]) -> TypeChecker: + if not isinstance(ctx.api, TypeChecker): + raise ValueError('Not a TypeChecker') + return ctx.api + + +def check_types_compatible(ctx: Union[FunctionContext, MethodContext], + *, expected_type: MypyType, actual_type: MypyType, error_message: str) -> None: + api = get_typechecker_api(ctx) + api.check_subtype(actual_type, expected_type, + ctx.context, error_message, + 'got', 'expected') + + +def get_call_argument_by_name(ctx: Union[FunctionContext, MethodContext], name: str) -> Optional[Expression]: + """ + Return the expression for the specific argument. + This helper should only be used with non-star arguments. + """ + if name not in ctx.callee_arg_names: + return None + idx = ctx.callee_arg_names.index(name) + args = ctx.args[idx] + if len(args) != 1: + # Either an error or no value passed. + return None + return args[0] + + +def get_call_argument_type_by_name(ctx: Union[FunctionContext, MethodContext], name: str) -> Optional[MypyType]: + """Return the type for the specific argument. + + This helper should only be used with non-star arguments. + """ + if name not in ctx.callee_arg_names: + return None + idx = ctx.callee_arg_names.index(name) + arg_types = ctx.arg_types[idx] + if len(arg_types) != 1: + # Either an error or no value passed. + return None + return arg_types[0] + + +def add_new_sym_for_info(info: TypeInfo, *, name: str, sym_type: MypyType) -> None: + # type=: type of the variable itself + var = Var(name=name, type=sym_type) + # var.info: type of the object variable is bound to + var.info = info + var._fullname = info.fullname + '.' + name + var.is_initialized_in_class = True + var.is_inferred = True + info.names[name] = SymbolTableNode(MDEF, var, + plugin_generated=True) diff --git a/mypy_django_plugin/lib/helpers.py b/mypy_django_plugin/lib/helpers.py index c99f2f82c..5b7991a3f 100644 --- a/mypy_django_plugin/lib/helpers.py +++ b/mypy_django_plugin/lib/helpers.py @@ -1,41 +1,33 @@ -from collections import OrderedDict from typing import ( - TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union, + TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Union, ) -from django.db.models.fields import Field from django.db.models.fields.related import RelatedField from django.db.models.fields.reverse_related import ForeignObjectRel -from mypy import checker from mypy.checker import TypeChecker from mypy.mro import calculate_mro from mypy.nodes import ( - GDEF, MDEF, Argument, Block, ClassDef, Expression, FuncDef, MemberExpr, MypyFile, NameExpr, PlaceholderNode, - StrExpr, SymbolNode, SymbolTable, SymbolTableNode, TypeInfo, Var, -) -from mypy.plugin import ( - AttributeContext, CheckerPluginInterface, ClassDefContext, DynamicClassDefContext, FunctionContext, MethodContext, + Block, ClassDef, Expression, MemberExpr, MypyFile, NameExpr, StrExpr, SymbolNode, + SymbolTable, SymbolTableNode, TypeInfo, Var, ) -from mypy.plugins.common import add_method from mypy.semanal import SemanticAnalyzer -from mypy.types import AnyType, CallableType, Instance, NoneTyp, TupleType +from mypy.types import AnyType, Instance, NoneTyp from mypy.types import Type as MypyType -from mypy.types import TypedDictType, TypeOfAny, UnionType +from mypy.types import TypeOfAny, UnionType +from django.db.models.fields import Field from mypy_django_plugin.lib import fullnames if TYPE_CHECKING: from mypy_django_plugin.django.context import DjangoContext +AnyPluginAPI = Union[TypeChecker, SemanticAnalyzer] + def get_django_metadata(model_info: TypeInfo) -> Dict[str, Any]: return model_info.metadata.setdefault('django', {}) -class IncompleteDefnException(Exception): - pass - - def lookup_fully_qualified_sym(fullname: str, all_modules: Dict[str, MypyFile]) -> Optional[SymbolTableNode]: if '.' not in fullname: return None @@ -57,14 +49,14 @@ def lookup_fully_qualified_generic(name: str, all_modules: Dict[str, MypyFile]) return sym.node -def lookup_fully_qualified_typeinfo(api: Union[TypeChecker, SemanticAnalyzer], fullname: str) -> Optional[TypeInfo]: +def lookup_fully_qualified_typeinfo(api: AnyPluginAPI, fullname: str) -> Optional[TypeInfo]: node = lookup_fully_qualified_generic(fullname, api.modules) if not isinstance(node, TypeInfo): return None return node -def lookup_class_typeinfo(api: TypeChecker, klass: type) -> Optional[TypeInfo]: +def lookup_class_typeinfo(api: AnyPluginAPI, klass: type) -> Optional[TypeInfo]: fullname = get_class_fullname(klass) field_info = lookup_fully_qualified_typeinfo(api, fullname) return field_info @@ -79,36 +71,6 @@ def get_class_fullname(klass: type) -> str: return klass.__module__ + '.' + klass.__qualname__ -def get_call_argument_by_name(ctx: Union[FunctionContext, MethodContext], name: str) -> Optional[Expression]: - """ - Return the expression for the specific argument. - This helper should only be used with non-star arguments. - """ - if name not in ctx.callee_arg_names: - return None - idx = ctx.callee_arg_names.index(name) - args = ctx.args[idx] - if len(args) != 1: - # Either an error or no value passed. - return None - return args[0] - - -def get_call_argument_type_by_name(ctx: Union[FunctionContext, MethodContext], name: str) -> Optional[MypyType]: - """Return the type for the specific argument. - - This helper should only be used with non-star arguments. - """ - if name not in ctx.callee_arg_names: - return None - idx = ctx.callee_arg_names.index(name) - arg_types = ctx.arg_types[idx] - if len(arg_types) != 1: - # Either an error or no value passed. - return None - return arg_types[0] - - def make_optional(typ: MypyType) -> MypyType: return UnionType.make_union([typ, NoneTyp()]) @@ -153,7 +115,7 @@ def get_private_descriptor_type(type_info: TypeInfo, private_field_name: str, is return AnyType(TypeOfAny.explicit) -def get_field_lookup_exact_type(api: TypeChecker, field: Field) -> MypyType: +def get_field_lookup_exact_type(api: AnyPluginAPI, field: Field) -> MypyType: if isinstance(field, (RelatedField, ForeignObjectRel)): lookup_type_class = field.related_model rel_model_info = lookup_class_typeinfo(api, lookup_type_class) @@ -168,44 +130,10 @@ def get_field_lookup_exact_type(api: TypeChecker, field: Field) -> MypyType: is_nullable=field.null) -def get_nested_meta_node_for_current_class(info: TypeInfo) -> Optional[TypeInfo]: - metaclass_sym = info.names.get('Meta') - if metaclass_sym is not None and isinstance(metaclass_sym.node, TypeInfo): - return metaclass_sym.node - return None - - -def add_new_class_for_module(module: MypyFile, - name: str, - bases: List[Instance], - fields: Optional[Dict[str, MypyType]] = None - ) -> TypeInfo: - new_class_unique_name = checker.gen_unique_name(name, module.names) - - # make new class expression - classdef = ClassDef(new_class_unique_name, Block([])) - classdef.fullname = module.fullname + '.' + new_class_unique_name - - # make new TypeInfo - new_typeinfo = TypeInfo(SymbolTable(), classdef, module.fullname) - new_typeinfo.bases = bases - calculate_mro(new_typeinfo) - new_typeinfo.calculate_metaclass_type() - - # add fields - if fields: - for field_name, field_type in fields.items(): - var = Var(field_name, type=field_type) - var.info = new_typeinfo - var._fullname = new_typeinfo.fullname + '.' + field_name - new_typeinfo.names[field_name] = SymbolTableNode(MDEF, var, plugin_generated=True) - - classdef.info = new_typeinfo - module.names[new_class_unique_name] = SymbolTableNode(GDEF, new_typeinfo, plugin_generated=True) - return new_typeinfo +def get_current_module(api: AnyPluginAPI) -> MypyFile: + if isinstance(api, SemanticAnalyzer): + return api.cur_mod_node - -def get_current_module(api: TypeChecker) -> MypyFile: current_module = None for item in reversed(api.scope.stack): if isinstance(item, MypyFile): @@ -215,21 +143,6 @@ def get_current_module(api: TypeChecker) -> MypyFile: return current_module -def make_oneoff_named_tuple(api: TypeChecker, name: str, fields: 'OrderedDict[str, MypyType]') -> TupleType: - current_module = get_current_module(api) - namedtuple_info = add_new_class_for_module(current_module, name, - bases=[api.named_generic_type('typing.NamedTuple', [])], - fields=fields) - return TupleType(list(fields.values()), fallback=Instance(namedtuple_info, [])) - - -def make_tuple(api: 'TypeChecker', fields: List[MypyType]) -> TupleType: - # fallback for tuples is any builtins.tuple instance - fallback = api.named_generic_type('builtins.tuple', - [AnyType(TypeOfAny.special_form)]) - return TupleType(fields, fallback=fallback) - - def convert_any_to_type(typ: MypyType, referred_to_type: MypyType) -> MypyType: if isinstance(typ, UnionType): converted_items = [] @@ -252,13 +165,6 @@ def convert_any_to_type(typ: MypyType, referred_to_type: MypyType) -> MypyType: return typ -def make_typeddict(api: CheckerPluginInterface, fields: 'OrderedDict[str, MypyType]', - required_keys: Set[str]) -> TypedDictType: - object_type = api.named_generic_type('mypy_extensions._TypedDict', []) - typed_dict_type = TypedDictType(fields, required_keys=required_keys, fallback=object_type) - return typed_dict_type - - def resolve_string_attribute_value(attr_expr: Expression, django_context: 'DjangoContext') -> Optional[str]: if isinstance(attr_expr, StrExpr): return attr_expr.value @@ -272,113 +178,25 @@ def resolve_string_attribute_value(attr_expr: Expression, django_context: 'Djang return None -def get_semanal_api(ctx: Union[ClassDefContext, DynamicClassDefContext]) -> SemanticAnalyzer: - if not isinstance(ctx.api, SemanticAnalyzer): - raise ValueError('Not a SemanticAnalyzer') - return ctx.api - - -def get_typechecker_api(ctx: Union[AttributeContext, MethodContext, FunctionContext]) -> TypeChecker: - if not isinstance(ctx.api, TypeChecker): - raise ValueError('Not a TypeChecker') - return ctx.api - - -def is_model_subclass_info(info: TypeInfo, django_context: 'DjangoContext') -> bool: +def is_subclass_of_model(info: TypeInfo, django_context: 'DjangoContext') -> bool: return (info.fullname in django_context.all_registered_model_class_fullnames or info.has_base(fullnames.MODEL_CLASS_FULLNAME)) -def check_types_compatible(ctx: Union[FunctionContext, MethodContext], - *, expected_type: MypyType, actual_type: MypyType, error_message: str) -> None: - api = get_typechecker_api(ctx) - api.check_subtype(actual_type, expected_type, - ctx.context, error_message, - 'got', 'expected') - - -def add_new_sym_for_info(info: TypeInfo, *, name: str, sym_type: MypyType) -> None: - # type=: type of the variable itself - var = Var(name=name, type=sym_type) - # var.info: type of the object variable is bound to - var.info = info - var._fullname = info.fullname + '.' + name - var.is_initialized_in_class = True - var.is_inferred = True - info.names[name] = SymbolTableNode(MDEF, var, - plugin_generated=True) - - -def build_unannotated_method_args(method_node: FuncDef) -> Tuple[List[Argument], MypyType]: - prepared_arguments = [] - try: - arguments = method_node.arguments[1:] - except AttributeError: - arguments = [] - for argument in arguments: - argument.type_annotation = AnyType(TypeOfAny.unannotated) - prepared_arguments.append(argument) - return_type = AnyType(TypeOfAny.unannotated) - return prepared_arguments, return_type - - -def copy_method_to_another_class(ctx: ClassDefContext, self_type: Instance, - new_method_name: str, method_node: FuncDef) -> None: - semanal_api = get_semanal_api(ctx) - if method_node.type is None: - if not semanal_api.final_iteration: - semanal_api.defer() - return - - arguments, return_type = build_unannotated_method_args(method_node) - add_method(ctx, - new_method_name, - args=arguments, - return_type=return_type, - self_type=self_type) - return - - method_type = method_node.type - if not isinstance(method_type, CallableType): - if not semanal_api.final_iteration: - semanal_api.defer() - return - - arguments = [] - bound_return_type = semanal_api.anal_type(method_type.ret_type, - allow_placeholder=True) - - assert bound_return_type is not None - - if isinstance(bound_return_type, PlaceholderNode): - return - - for arg_name, arg_type, original_argument in zip(method_type.arg_names[1:], - method_type.arg_types[1:], - method_node.arguments[1:]): - bound_arg_type = semanal_api.anal_type(arg_type, allow_placeholder=True) - if bound_arg_type is None and not semanal_api.final_iteration: - semanal_api.defer() - return - - assert bound_arg_type is not None - - if isinstance(bound_arg_type, PlaceholderNode): - return - - var = Var(name=original_argument.variable.name, - type=arg_type) - var.line = original_argument.variable.line - var.column = original_argument.variable.column - argument = Argument(variable=var, - type_annotation=bound_arg_type, - initializer=original_argument.initializer, - kind=original_argument.kind) - argument.set_line(original_argument) - arguments.append(argument) - - add_method(ctx, - new_method_name, - args=arguments, - return_type=bound_return_type, - self_type=self_type) +def new_typeinfo(name: str, + *, + bases: List[Instance], + module_name: str) -> TypeInfo: + """ + Construct new TypeInfo instance. Cannot be used for nested classes. + """ + class_def = ClassDef(name, Block([])) + class_def.fullname = module_name + '.' + name + + info = TypeInfo(SymbolTable(), class_def, module_name) + info.bases = bases + calculate_mro(info) + info.metaclass_type = info.calculate_metaclass_type() + + class_def.info = info + return info diff --git a/mypy_django_plugin/lib/sem_helpers.py b/mypy_django_plugin/lib/sem_helpers.py new file mode 100644 index 000000000..30d67bb2d --- /dev/null +++ b/mypy_django_plugin/lib/sem_helpers.py @@ -0,0 +1,117 @@ +from typing import Union, Tuple, List, Optional, NamedTuple, cast + +from mypy.nodes import Argument, FuncDef, Var, TypeInfo +from mypy.plugin import DynamicClassDefContext, ClassDefContext +from mypy.plugins.common import add_method +from mypy.semanal import SemanticAnalyzer +from mypy.types import Instance, CallableType, AnyType, TypeOfAny, PlaceholderType +from mypy.types import Type as MypyType + + +class IncompleteDefnException(Exception): + def __init__(self, error_message: str = '') -> None: + super().__init__(error_message) + + +class BoundNameNotFound(IncompleteDefnException): + def __init__(self, fullname: str) -> None: + super().__init__(f'No {fullname!r} found') + + +def get_semanal_api(ctx: Union[ClassDefContext, DynamicClassDefContext]) -> SemanticAnalyzer: + return cast(SemanticAnalyzer, ctx.api) + + +def get_nested_meta_node_for_current_class(info: TypeInfo) -> Optional[TypeInfo]: + metaclass_sym = info.names.get('Meta') + if metaclass_sym is not None and isinstance(metaclass_sym.node, TypeInfo): + return metaclass_sym.node + return None + + +def prepare_unannotated_method_signature(method_node: FuncDef) -> Tuple[List[Argument], MypyType]: + prepared_arguments = [] + for argument in method_node.arguments[1:]: + argument.type_annotation = AnyType(TypeOfAny.unannotated) + prepared_arguments.append(argument) + return_type = AnyType(TypeOfAny.unannotated) + return prepared_arguments, return_type + + +class SignatureTuple(NamedTuple): + arguments: Optional[List[Argument]] + return_type: Optional[MypyType] + cannot_be_bound: bool + + +def analyze_callable_signature(api: SemanticAnalyzer, method_node: FuncDef) -> SignatureTuple: + method_type = method_node.type + assert isinstance(method_type, CallableType) + + arguments = [] + unbound = False + for arg_name, arg_type, original_argument in zip(method_type.arg_names[1:], + method_type.arg_types[1:], + method_node.arguments[1:]): + arg_type = api.anal_type(arg_type, allow_placeholder=True) + if isinstance(arg_type, PlaceholderType): + unbound = True + + var = Var(name=original_argument.variable.name, + type=arg_type) + var.set_line(original_argument.variable) + + if isinstance(arg_type, PlaceholderType): + unbound = True + argument = Argument(variable=var, + type_annotation=arg_type, + initializer=original_argument.initializer, + kind=original_argument.kind) + argument.set_line(original_argument) + arguments.append(argument) + + ret_type = api.anal_type(method_type.ret_type, allow_placeholder=True) + if isinstance(ret_type, PlaceholderType): + unbound = True + return SignatureTuple(arguments, ret_type, unbound) + + +def copy_method_or_incomplete_defn_exception(ctx: ClassDefContext, + self_type: Instance, + new_method_name: str, + method_node: FuncDef) -> None: + semanal_api = get_semanal_api(ctx) + + if method_node.type is None: + if not semanal_api.final_iteration: + raise IncompleteDefnException(f'Unannotated method {method_node.fullname!r}') + + arguments, return_type = prepare_unannotated_method_signature(method_node) + add_method(ctx, + new_method_name, + args=arguments, + return_type=return_type, + self_type=self_type) + return + + assert isinstance(method_node.type, CallableType) + + # copy global SymbolTableNode objects from original class to the current node, if not present + original_module = semanal_api.modules[method_node.info.module_name] + for name, sym in original_module.names.items(): + if (not sym.plugin_generated + and name not in semanal_api.cur_mod_node.names): + semanal_api.add_imported_symbol(name, sym, context=semanal_api.cur_mod_node) + + arguments, return_type, unbound = analyze_callable_signature(semanal_api, method_node) + assert len(arguments) + 1 == len(method_node.arguments) + if unbound: + raise IncompleteDefnException(f'Signature of method {method_node.fullname!r} is not ready') + + if new_method_name in ctx.cls.info.names: + del ctx.cls.info.names[new_method_name] + add_method(ctx, + new_method_name, + args=arguments, + return_type=return_type, + self_type=self_type) diff --git a/mypy_django_plugin/main.py b/mypy_django_plugin/main.py index 3f64ba417..2dfb3b9d2 100644 --- a/mypy_django_plugin/main.py +++ b/mypy_django_plugin/main.py @@ -8,6 +8,7 @@ from mypy.plugin import ( AttributeContext, ClassDefContext, DynamicClassDefContext, FunctionContext, MethodContext, Plugin, ) +from mypy.semanal import dummy_context from mypy.types import Type as MypyType import mypy_django_plugin.transformers.orm_lookups @@ -29,7 +30,7 @@ def transform_model_class(ctx: ClassDefContext, if sym is not None and isinstance(sym.node, TypeInfo): helpers.get_django_metadata(sym.node)['model_bases'][ctx.cls.fullname] = 1 else: - if not ctx.api.final_iteration: + if not ctx.api.final_iteration and not ctx.api.deferred: ctx.api.defer() return @@ -194,7 +195,7 @@ def get_function_hook(self, fullname: str if info.has_base(fullnames.FIELD_FULLNAME): return partial(fields.transform_into_proper_return_type, django_context=self.django_context) - if helpers.is_model_subclass_info(info, self.django_context): + if helpers.is_subclass_of_model(info, self.django_context): return partial(init_create.redefine_and_typecheck_model_init, django_context=self.django_context) return None diff --git a/mypy_django_plugin/transformers/fields.py b/mypy_django_plugin/transformers/fields.py index b88fdbf59..c711f91cb 100644 --- a/mypy_django_plugin/transformers/fields.py +++ b/mypy_django_plugin/transformers/fields.py @@ -9,13 +9,13 @@ from mypy.types import TypeOfAny from mypy_django_plugin.django.context import DjangoContext -from mypy_django_plugin.lib import fullnames, helpers +from mypy_django_plugin.lib import fullnames, helpers, chk_helpers def _get_current_field_from_assignment(ctx: FunctionContext, django_context: DjangoContext) -> Optional[Field]: - outer_model_info = helpers.get_typechecker_api(ctx).scope.active_class() + outer_model_info = chk_helpers.get_typechecker_api(ctx).scope.active_class() if (outer_model_info is None - or not helpers.is_model_subclass_info(outer_model_info, django_context)): + or not helpers.is_subclass_of_model(outer_model_info, django_context)): return None field_name = None @@ -66,21 +66,21 @@ def fill_descriptor_types_for_related_field(ctx: FunctionContext, django_context # __get__/__set__ of ForeignKey of derived model for model_cls in django_context.all_registered_model_classes: if issubclass(model_cls, current_model_cls) and not model_cls._meta.abstract: - derived_model_info = helpers.lookup_class_typeinfo(helpers.get_typechecker_api(ctx), model_cls) + derived_model_info = helpers.lookup_class_typeinfo(chk_helpers.get_typechecker_api(ctx), model_cls) if derived_model_info is not None: fk_ref_type = Instance(derived_model_info, []) derived_fk_type = reparametrize_related_field_type(default_related_field_type, set_type=fk_ref_type, get_type=fk_ref_type) - helpers.add_new_sym_for_info(derived_model_info, - name=current_field.name, - sym_type=derived_fk_type) + chk_helpers.add_new_sym_for_info(derived_model_info, + name=current_field.name, + sym_type=derived_fk_type) related_model = related_model_cls related_model_to_set = related_model_cls if related_model_to_set._meta.proxy_for_model is not None: related_model_to_set = related_model_to_set._meta.proxy_for_model - typechecker_api = helpers.get_typechecker_api(ctx) + typechecker_api = chk_helpers.get_typechecker_api(ctx) related_model_info = helpers.lookup_class_typeinfo(typechecker_api, related_model) if related_model_info is None: @@ -114,7 +114,7 @@ def set_descriptor_types_for_field(ctx: FunctionContext) -> Instance: default_return_type = cast(Instance, ctx.default_return_type) is_nullable = False - null_expr = helpers.get_call_argument_by_name(ctx, 'null') + null_expr = chk_helpers.get_call_argument_by_name(ctx, 'null') if null_expr is not None: is_nullable = helpers.parse_bool(null_expr) or False @@ -122,10 +122,10 @@ def set_descriptor_types_for_field(ctx: FunctionContext) -> Instance: return helpers.reparametrize_instance(default_return_type, [set_type, get_type]) -def determine_type_of_array_field(ctx: FunctionContext, django_context: DjangoContext) -> MypyType: +def determine_type_of_array_field(ctx: FunctionContext) -> MypyType: default_return_type = set_descriptor_types_for_field(ctx) - base_field_arg_type = helpers.get_call_argument_type_by_name(ctx, 'base_field') + base_field_arg_type = chk_helpers.get_call_argument_type_by_name(ctx, 'base_field') if not base_field_arg_type or not isinstance(base_field_arg_type, Instance): return default_return_type @@ -141,9 +141,9 @@ def transform_into_proper_return_type(ctx: FunctionContext, django_context: Djan default_return_type = ctx.default_return_type assert isinstance(default_return_type, Instance) - outer_model_info = helpers.get_typechecker_api(ctx).scope.active_class() + outer_model_info = chk_helpers.get_typechecker_api(ctx).scope.active_class() if (outer_model_info is None - or not helpers.is_model_subclass_info(outer_model_info, django_context)): + or not helpers.is_subclass_of_model(outer_model_info, django_context)): return ctx.default_return_type assert isinstance(outer_model_info, TypeInfo) @@ -152,6 +152,6 @@ def transform_into_proper_return_type(ctx: FunctionContext, django_context: Djan return fill_descriptor_types_for_related_field(ctx, django_context) if default_return_type.type.has_base(fullnames.ARRAY_FIELD_FULLNAME): - return determine_type_of_array_field(ctx, django_context) + return determine_type_of_array_field(ctx) return set_descriptor_types_for_field(ctx) diff --git a/mypy_django_plugin/transformers/forms.py b/mypy_django_plugin/transformers/forms.py index 7bd0e1116..26651f80d 100644 --- a/mypy_django_plugin/transformers/forms.py +++ b/mypy_django_plugin/transformers/forms.py @@ -5,11 +5,11 @@ from mypy.types import Type as MypyType from mypy.types import TypeType -from mypy_django_plugin.lib import helpers +from mypy_django_plugin.lib import sem_helpers, chk_helpers def make_meta_nested_class_inherit_from_any(ctx: ClassDefContext) -> None: - meta_node = helpers.get_nested_meta_node_for_current_class(ctx.cls.info) + meta_node = sem_helpers.get_nested_meta_node_for_current_class(ctx.cls.info) if meta_node is None: if not ctx.api.final_iteration: ctx.api.defer() @@ -28,7 +28,7 @@ def extract_proper_type_for_get_form(ctx: MethodContext) -> MypyType: object_type = ctx.type assert isinstance(object_type, Instance) - form_class_type = helpers.get_call_argument_type_by_name(ctx, 'form_class') + form_class_type = chk_helpers.get_call_argument_type_by_name(ctx, 'form_class') if form_class_type is None or isinstance(form_class_type, NoneTyp): form_class_type = get_specified_form_class(object_type) diff --git a/mypy_django_plugin/transformers/init_create.py b/mypy_django_plugin/transformers/init_create.py index fe0b19ee2..549f826ba 100644 --- a/mypy_django_plugin/transformers/init_create.py +++ b/mypy_django_plugin/transformers/init_create.py @@ -6,7 +6,7 @@ from mypy.types import Type as MypyType from mypy_django_plugin.django.context import DjangoContext -from mypy_django_plugin.lib import helpers +from mypy_django_plugin.lib import chk_helpers def get_actual_types(ctx: Union[MethodContext, FunctionContext], @@ -32,7 +32,7 @@ def get_actual_types(ctx: Union[MethodContext, FunctionContext], def typecheck_model_method(ctx: Union[FunctionContext, MethodContext], django_context: DjangoContext, model_cls: Type[Model], method: str) -> MypyType: - typechecker_api = helpers.get_typechecker_api(ctx) + typechecker_api = chk_helpers.get_typechecker_api(ctx) expected_types = django_context.get_expected_types(typechecker_api, model_cls, method=method) expected_keys = [key for key in expected_types.keys() if key != 'pk'] @@ -42,11 +42,11 @@ def typecheck_model_method(ctx: Union[FunctionContext, MethodContext], django_co model_cls.__name__), ctx.context) continue - helpers.check_types_compatible(ctx, - expected_type=expected_types[actual_name], - actual_type=actual_type, - error_message='Incompatible type for "{}" of "{}"'.format(actual_name, - model_cls.__name__)) + error_message = 'Incompatible type for "{}" of "{}"'.format(actual_name, model_cls.__name__) + chk_helpers.check_types_compatible(ctx, + expected_type=expected_types[actual_name], + actual_type=actual_type, + error_message=error_message) return ctx.default_return_type diff --git a/mypy_django_plugin/transformers/managers.py b/mypy_django_plugin/transformers/managers.py index 88201b439..c30427323 100644 --- a/mypy_django_plugin/transformers/managers.py +++ b/mypy_django_plugin/transformers/managers.py @@ -1,77 +1,149 @@ +from typing import Iterator, Tuple, Optional + from mypy.nodes import ( - GDEF, FuncDef, MemberExpr, NameExpr, RefExpr, StrExpr, SymbolTableNode, TypeInfo, + FuncDef, MemberExpr, NameExpr, RefExpr, StrExpr, TypeInfo, + PlaceholderNode, SymbolTableNode, GDEF ) from mypy.plugin import ClassDefContext, DynamicClassDefContext from mypy.types import AnyType, Instance, TypeOfAny +from mypy.typevars import fill_typevars -from mypy_django_plugin.lib import fullnames, helpers +from mypy_django_plugin.lib import fullnames, sem_helpers, helpers -def create_new_manager_class_from_from_queryset_method(ctx: DynamicClassDefContext) -> None: - semanal_api = helpers.get_semanal_api(ctx) +def iter_all_custom_queryset_methods(derived_queryset_info: TypeInfo) -> Iterator[Tuple[str, FuncDef]]: + for base_queryset_info in derived_queryset_info.mro: + if base_queryset_info.fullname == fullnames.QUERYSET_CLASS_FULLNAME: + break + for name, sym in base_queryset_info.names.items(): + if isinstance(sym.node, FuncDef): + yield name, sym.node + +def resolve_callee_manager_info_or_exception(ctx: DynamicClassDefContext) -> Optional[TypeInfo]: callee = ctx.call.callee assert isinstance(callee, MemberExpr) assert isinstance(callee.expr, RefExpr) - base_manager_info = callee.expr.node - if base_manager_info is None: - if not semanal_api.final_iteration: - semanal_api.defer() - return - - assert isinstance(base_manager_info, TypeInfo) - new_manager_info = semanal_api.basic_new_typeinfo(ctx.name, - basetype_or_fallback=Instance(base_manager_info, - [AnyType(TypeOfAny.unannotated)])) - new_manager_info.line = ctx.call.line - new_manager_info.defn.line = ctx.call.line - new_manager_info.metaclass_type = new_manager_info.calculate_metaclass_type() - - current_module = semanal_api.cur_mod_node - current_module.names[ctx.name] = SymbolTableNode(GDEF, new_manager_info, - plugin_generated=True) - passed_queryset = ctx.call.args[0] - assert isinstance(passed_queryset, NameExpr) - - derived_queryset_fullname = passed_queryset.fullname - assert derived_queryset_fullname is not None - - sym = semanal_api.lookup_fully_qualified_or_none(derived_queryset_fullname) - assert sym is not None - if sym.node is None: - if not semanal_api.final_iteration: - semanal_api.defer() - else: - # inherit from Any to prevent false-positives, if queryset class cannot be resolved - new_manager_info.fallback_to_any = True - return + callee_manager_info = callee.expr.node + if (callee_manager_info is None + or isinstance(callee_manager_info, PlaceholderNode)): + raise sem_helpers.IncompleteDefnException(f'Definition of base manager {callee_manager_info.fullname} ' + f'is incomplete.') + + assert isinstance(callee_manager_info, TypeInfo) + return callee_manager_info + + +def resolve_passed_queryset_info_or_exception(ctx: DynamicClassDefContext) -> Optional[TypeInfo]: + api = sem_helpers.get_semanal_api(ctx) + + passed_queryset_name_expr = ctx.call.args[0] + assert isinstance(passed_queryset_name_expr, NameExpr) + + sym = api.lookup_qualified(passed_queryset_name_expr.name, ctx=ctx.call) + if (sym is None + or sym.node is None + or isinstance(sym.node, PlaceholderNode)): + raise sem_helpers.BoundNameNotFound(passed_queryset_name_expr.fullname) + + assert isinstance(sym.node, TypeInfo) + return sym.node + + +def resolve_django_manager_info_or_exception(ctx: DynamicClassDefContext) -> Optional[TypeInfo]: + api = sem_helpers.get_semanal_api(ctx) + + sym = api.lookup_fully_qualified_or_none(fullnames.MANAGER_CLASS_FULLNAME) + if (sym is None + or sym.node is None + or isinstance(sym.node, PlaceholderNode)): + raise sem_helpers.BoundNameNotFound(fullnames.MANAGER_CLASS_FULLNAME) + + assert isinstance(sym.node, TypeInfo) + return sym.node - derived_queryset_info = sym.node - assert isinstance(derived_queryset_info, TypeInfo) +def new_manager_typeinfo(ctx: DynamicClassDefContext, callee_manager_info: TypeInfo) -> TypeInfo: + callee_manager_type = Instance(callee_manager_info, [AnyType(TypeOfAny.unannotated)]) + api = sem_helpers.get_semanal_api(ctx) + + new_manager_class_name = ctx.name + new_manager_info = helpers.new_typeinfo(new_manager_class_name, + bases=[callee_manager_type], module_name=api.cur_mod_id) + new_manager_info.set_line(ctx.call) + return new_manager_info + + +def record_new_manager_info_fullname_into_metadata(ctx: DynamicClassDefContext, + new_manager_fullname: str, + callee_manager_info: TypeInfo, + queryset_info: TypeInfo, + django_manager_info: TypeInfo) -> None: if len(ctx.call.args) > 1: expr = ctx.call.args[1] assert isinstance(expr, StrExpr) custom_manager_generated_name = expr.value else: - custom_manager_generated_name = base_manager_info.name + 'From' + derived_queryset_info.name + custom_manager_generated_name = callee_manager_info.name + 'From' + queryset_info.name + + custom_manager_generated_fullname = 'django.db.models.manager' + '.' + custom_manager_generated_name + + metadata = django_manager_info.metadata.setdefault('from_queryset_managers', {}) + metadata[custom_manager_generated_fullname] = new_manager_fullname + + +def create_new_manager_class_from_from_queryset_method(ctx: DynamicClassDefContext) -> None: + semanal_api = sem_helpers.get_semanal_api(ctx) + try: + callee_manager_info = resolve_callee_manager_info_or_exception(ctx) + queryset_info = resolve_passed_queryset_info_or_exception(ctx) + django_manager_info = resolve_django_manager_info_or_exception(ctx) + except sem_helpers.IncompleteDefnException: + if not semanal_api.final_iteration: + semanal_api.defer() + return + else: + raise - custom_manager_generated_fullname = '.'.join(['django.db.models.manager', custom_manager_generated_name]) - if 'from_queryset_managers' not in base_manager_info.metadata: - base_manager_info.metadata['from_queryset_managers'] = {} - base_manager_info.metadata['from_queryset_managers'][custom_manager_generated_fullname] = new_manager_info.fullname + new_manager_info = new_manager_typeinfo(ctx, callee_manager_info) + record_new_manager_info_fullname_into_metadata(ctx, + new_manager_info.fullname, + callee_manager_info, + queryset_info, + django_manager_info) class_def_context = ClassDefContext(cls=new_manager_info.defn, reason=ctx.call, api=semanal_api) - self_type = Instance(new_manager_info, []) - # we need to copy all methods in MRO before django.db.models.query.QuerySet - for class_mro_info in derived_queryset_info.mro: - if class_mro_info.fullname == fullnames.QUERYSET_CLASS_FULLNAME: - break - for name, sym in class_mro_info.names.items(): - if isinstance(sym.node, FuncDef): - helpers.copy_method_to_another_class(class_def_context, - self_type, - new_method_name=name, - method_node=sym.node) + self_type = fill_typevars(new_manager_info) + # self_type = Instance(new_manager_info, []) + + try: + for name, method_node in iter_all_custom_queryset_methods(queryset_info): + sem_helpers.copy_method_or_incomplete_defn_exception(class_def_context, + self_type, + new_method_name=name, + method_node=method_node) + except sem_helpers.IncompleteDefnException: + if not semanal_api.final_iteration: + semanal_api.defer() + return + else: + raise + + new_manager_sym = SymbolTableNode(GDEF, new_manager_info, plugin_generated=True) + + # context=None - forcibly replace old node + added = semanal_api.add_symbol_table_node(ctx.name, new_manager_sym, context=None) + if added: + # replace all references to the old manager Var everywhere + for _, module in semanal_api.modules.items(): + if module.fullname != semanal_api.cur_mod_id: + for sym_name, sym in module.names.items(): + if sym.fullname == new_manager_info.fullname: + module.names[sym_name] = new_manager_sym.copy() + + # we need another iteration to process methods + if (not added + and not semanal_api.final_iteration): + semanal_api.defer() diff --git a/mypy_django_plugin/transformers/meta.py b/mypy_django_plugin/transformers/meta.py index 64e6e12fe..cc69a7d44 100644 --- a/mypy_django_plugin/transformers/meta.py +++ b/mypy_django_plugin/transformers/meta.py @@ -5,12 +5,12 @@ from mypy.types import TypeOfAny from mypy_django_plugin.django.context import DjangoContext -from mypy_django_plugin.lib import helpers +from mypy_django_plugin.lib import helpers, chk_helpers def _get_field_instance(ctx: MethodContext, field_fullname: str) -> MypyType: - field_info = helpers.lookup_fully_qualified_typeinfo(helpers.get_typechecker_api(ctx), - field_fullname) + api = chk_helpers.get_typechecker_api(ctx) + field_info = helpers.lookup_fully_qualified_typeinfo(api, field_fullname) if field_info is None: return AnyType(TypeOfAny.unannotated) return Instance(field_info, [AnyType(TypeOfAny.explicit), AnyType(TypeOfAny.explicit)]) @@ -32,7 +32,7 @@ def return_proper_field_type_from_get_field(ctx: MethodContext, django_context: if model_cls is None: return ctx.default_return_type - field_name_expr = helpers.get_call_argument_by_name(ctx, 'field_name') + field_name_expr = chk_helpers.get_call_argument_by_name(ctx, 'field_name') if field_name_expr is None: return ctx.default_return_type diff --git a/mypy_django_plugin/transformers/models.py b/mypy_django_plugin/transformers/models.py index f0c436ca0..ee7398af0 100644 --- a/mypy_django_plugin/transformers/models.py +++ b/mypy_django_plugin/transformers/models.py @@ -1,21 +1,21 @@ -from typing import Dict, List, Optional, Type, cast +from typing import List, Optional, Type, cast from django.db.models.base import Model -from django.db.models.fields import DateField, DateTimeField -from django.db.models.fields.related import ForeignKey +from django.db.models.fields.related import ForeignKey, OneToOneField from django.db.models.fields.reverse_related import ( ManyToManyRel, ManyToOneRel, OneToOneRel, ) -from mypy.nodes import ARG_STAR2, Argument, Context, FuncDef, TypeInfo, Var +from mypy.nodes import ARG_STAR2, Argument, FuncDef, TypeInfo, Var, SymbolTableNode, MDEF, GDEF from mypy.plugin import ClassDefContext from mypy.plugins import common -from mypy.semanal import SemanticAnalyzer +from mypy.semanal import SemanticAnalyzer, dummy_context from mypy.types import AnyType, Instance from mypy.types import Type as MypyType from mypy.types import TypeOfAny +from django.db.models.fields import DateField, DateTimeField from mypy_django_plugin.django.context import DjangoContext -from mypy_django_plugin.lib import fullnames, helpers +from mypy_django_plugin.lib import fullnames, helpers, sem_helpers from mypy_django_plugin.transformers import fields from mypy_django_plugin.transformers.fields import get_field_descriptor_types @@ -35,7 +35,7 @@ def lookup_typeinfo(self, fullname: str) -> Optional[TypeInfo]: def lookup_typeinfo_or_incomplete_defn_error(self, fullname: str) -> TypeInfo: info = self.lookup_typeinfo(fullname) if info is None: - raise helpers.IncompleteDefnException(f'No {fullname!r} found') + raise sem_helpers.IncompleteDefnException(f'No {fullname!r} found') return info def lookup_class_typeinfo_or_incomplete_defn_error(self, klass: type) -> TypeInfo: @@ -43,26 +43,74 @@ def lookup_class_typeinfo_or_incomplete_defn_error(self, klass: type) -> TypeInf field_info = self.lookup_typeinfo_or_incomplete_defn_error(fullname) return field_info - def create_new_var(self, name: str, typ: MypyType) -> Var: - # type=: type of the variable itself - var = Var(name=name, type=typ) - # var.info: type of the object variable is bound to + def model_class_has_attribute_defined(self, name: str, traverse_mro: bool = True) -> bool: + if not traverse_mro: + sym = self.model_classdef.info.names.get(name) + else: + sym = self.model_classdef.info.get(name) + return sym is not None + + def resolve_manager_fullname(self, manager_fullname: str) -> str: + base_manager_info = self.lookup_typeinfo(fullnames.MANAGER_CLASS_FULLNAME) + if (base_manager_info is None + or 'from_queryset_managers' not in base_manager_info.metadata): + return manager_fullname + + metadata = base_manager_info.metadata['from_queryset_managers'] + return metadata.get(manager_fullname, manager_fullname) + + def add_new_node_to_model_class(self, name: str, typ: MypyType, + force_replace_existing: bool = False) -> None: + if not force_replace_existing and name in self.model_classdef.info.names: + raise ValueError(f'Member {name!r} already defined at model {self.model_classdef.info.fullname!r}.') + + var = Var(name, type=typ) + # TypeInfo of the object variable is bound to var.info = self.model_classdef.info - var._fullname = self.model_classdef.info.fullname + '.' + name + var._fullname = self.api.qualified_name(name) var.is_initialized_in_class = True - var.is_inferred = True - return var - - def add_new_node_to_model_class(self, name: str, typ: MypyType) -> None: - helpers.add_new_sym_for_info(self.model_classdef.info, - name=name, - sym_type=typ) - def add_new_class_for_current_module(self, name: str, bases: List[Instance]) -> TypeInfo: - current_module = self.api.modules[self.model_classdef.info.module_name] - new_class_info = helpers.add_new_class_for_module(current_module, - name=name, bases=bases) - return new_class_info + sym = SymbolTableNode(MDEF, var, plugin_generated=True) + context = dummy_context() + if force_replace_existing: + context = None + self.api.add_symbol_table_node(name, sym, context=context) + + def add_new_class_for_current_module(self, name: str, bases: List[Instance], + force_replace_existing: bool = False) -> Optional[TypeInfo]: + current_module = self.api.cur_mod_node + if not force_replace_existing and name in current_module: + raise ValueError(f'Class {name!r} already defined for module {current_module.fullname!r}') + + new_typeinfo = helpers.new_typeinfo(name, + bases=bases, + module_name=current_module.fullname) + # sym = SymbolTableNode(GDEF, new_typeinfo, + # plugin_generated=True) + # context = dummy_context() + # if force_replace_existing: + # context = None + + if name in current_module.names: + del current_module.names[name] + current_module.names[name] = SymbolTableNode(GDEF, new_typeinfo, plugin_generated=True) + # current_module.defs.append(new_typeinfo.defn) + # self.api.cur_mod_node. + # self.api.leave_class() + # added = self.api.add_symbol_table_node(name, sym, context=context) + # self.api.enter_class(self.model_classdef.info) + # + # self.api.cur_mod_node.defs.append(new_typeinfo.defn) + + # if not added and force_replace_existing: + # return None + return new_typeinfo + + # current_module = self.api.modules[self.model_classdef.info.module_name] + # context = + # new_class_info = helpers.add_new_class_for_module(current_module, + # name=name, bases=bases) + # return new_class_info def run(self) -> None: model_cls = self.django_context.get_model_class_by_fullname(self.model_classdef.fullname) @@ -88,58 +136,90 @@ class Meta(Any): """ def run(self) -> None: - meta_node = helpers.get_nested_meta_node_for_current_class(self.model_classdef.info) + meta_node = sem_helpers.get_nested_meta_node_for_current_class(self.model_classdef.info) if meta_node is None: return None meta_node.fallback_to_any = True class AddDefaultPrimaryKey(ModelClassInitializer): + """ + Adds default primary key to models which does not define their own. + ``` + class User(models.Model): + name = models.TextField() + ``` + """ + def run_with_model_cls(self, model_cls: Type[Model]) -> None: auto_field = model_cls._meta.auto_field - if auto_field and not self.model_classdef.info.has_readable_member(auto_field.attname): - # autogenerated field - auto_field_fullname = helpers.get_class_fullname(auto_field.__class__) - auto_field_info = self.lookup_typeinfo_or_incomplete_defn_error(auto_field_fullname) + if auto_field is None: + return - set_type, get_type = fields.get_field_descriptor_types(auto_field_info, is_nullable=False) - self.add_new_node_to_model_class(auto_field.attname, Instance(auto_field_info, - [set_type, get_type])) + primary_key_attrname = auto_field.attname + if self.model_class_has_attribute_defined(primary_key_attrname): + return + + auto_field_class_fullname = helpers.get_class_fullname(auto_field.__class__) + auto_field_info = self.lookup_typeinfo_or_incomplete_defn_error(auto_field_class_fullname) + + set_type, get_type = fields.get_field_descriptor_types(auto_field_info, is_nullable=False) + self.add_new_node_to_model_class(primary_key_attrname, Instance(auto_field_info, + [set_type, get_type])) class AddRelatedModelsId(ModelClassInitializer): + """ + Adds `FIELDNAME_id` attributes to models. + ``` + class User(models.Model): + pass + class Blog(models.Model): + user = models.ForeignKey(User) + ``` + + `user_id` will be added to `Blog`. + """ + def run_with_model_cls(self, model_cls: Type[Model]) -> None: for field in model_cls._meta.get_fields(): - if isinstance(field, ForeignKey): - related_model_cls = self.django_context.get_field_related_model_cls(field) - if related_model_cls is None: - error_context: Context = self.ctx.cls - field_sym = self.ctx.cls.info.get(field.name) - if field_sym is not None and field_sym.node is not None: - error_context = field_sym.node - self.api.fail(f'Cannot find model {field.related_model!r} ' - f'referenced in field {field.name!r} ', - ctx=error_context) - self.add_new_node_to_model_class(field.attname, - AnyType(TypeOfAny.explicit)) - continue + if not isinstance(field, (OneToOneField, ForeignKey)): + continue + related_id_attr_name = field.attname + if self.model_class_has_attribute_defined(related_id_attr_name): + continue + # if self.get_model_class_attr(related_id_attr_name) is not None: + # continue - if related_model_cls._meta.abstract: - continue + related_model_cls = self.django_context.get_field_related_model_cls(field) + if related_model_cls is None: + error_context = self.ctx.cls + field_sym = self.ctx.cls.info.get(field.name) + if field_sym is not None and field_sym.node is not None: + error_context = field_sym.node + self.api.fail(f'Cannot find model {field.related_model!r} ' + f'referenced in field {field.name!r} ', + ctx=error_context) + self.add_new_node_to_model_class(related_id_attr_name, + AnyType(TypeOfAny.explicit)) + continue - rel_primary_key_field = self.django_context.get_primary_key_field(related_model_cls) - try: - field_info = self.lookup_class_typeinfo_or_incomplete_defn_error(rel_primary_key_field.__class__) - except helpers.IncompleteDefnException as exc: - if not self.api.final_iteration: - raise exc - else: - continue + if related_model_cls._meta.abstract: + continue - is_nullable = self.django_context.get_field_nullability(field, None) - set_type, get_type = get_field_descriptor_types(field_info, is_nullable) - self.add_new_node_to_model_class(field.attname, - Instance(field_info, [set_type, get_type])) + rel_primary_key_field = self.django_context.get_primary_key_field(related_model_cls) + try: + field_info = self.lookup_class_typeinfo_or_incomplete_defn_error(rel_primary_key_field.__class__) + except sem_helpers.IncompleteDefnException as exc: + if not self.api.final_iteration: + raise exc + else: + continue + + is_nullable = self.django_context.get_field_nullability(field, None) + set_type, get_type = get_field_descriptor_types(field_info, is_nullable) + self.add_new_node_to_model_class(related_id_attr_name, + Instance(field_info, [set_type, get_type])) class AddManagers(ModelClassInitializer): @@ -152,25 +232,15 @@ def has_any_parametrized_manager_as_base(self, info: TypeInfo) -> bool: def is_any_parametrized_manager(self, typ: Instance) -> bool: return typ.type.fullname in fullnames.MANAGER_CLASSES and isinstance(typ.args[0], AnyType) - def get_generated_manager_mappings(self, base_manager_fullname: str) -> Dict[str, str]: - base_manager_info = self.lookup_typeinfo(base_manager_fullname) - if (base_manager_info is None - or 'from_queryset_managers' not in base_manager_info.metadata): - return {} - return base_manager_info.metadata['from_queryset_managers'] - def create_new_model_parametrized_manager(self, name: str, base_manager_info: TypeInfo) -> Instance: bases = [] for original_base in base_manager_info.bases: if self.is_any_parametrized_manager(original_base): - if original_base.type is None: - raise helpers.IncompleteDefnException() - original_base = helpers.reparametrize_instance(original_base, [Instance(self.model_classdef.info, [])]) bases.append(original_base) - new_manager_info = self.add_new_class_for_current_module(name, bases) + new_manager_info = self.add_new_class_for_current_module(name, bases, force_replace_existing=True) # copy fields to a new manager new_cls_def_context = ClassDefContext(cls=new_manager_info.defn, reason=self.ctx.reason, @@ -178,12 +248,15 @@ def create_new_model_parametrized_manager(self, name: str, base_manager_info: Ty custom_manager_type = Instance(new_manager_info, [Instance(self.model_classdef.info, [])]) for name, sym in base_manager_info.names.items(): + if name in new_manager_info.names: + raise ValueError(f'Name {name!r} already exists on newly-created {new_manager_info.fullname!r} class.') + # replace self type with new class, if copying method if isinstance(sym.node, FuncDef): - helpers.copy_method_to_another_class(new_cls_def_context, - self_type=custom_manager_type, - new_method_name=name, - method_node=sym.node) + sem_helpers.copy_method_or_incomplete_defn_exception(new_cls_def_context, + self_type=custom_manager_type, + new_method_name=name, + method_node=sym.node) continue new_sym = sym.copy() @@ -192,32 +265,36 @@ def create_new_model_parametrized_manager(self, name: str, base_manager_info: Ty new_var.info = new_manager_info new_var._fullname = new_manager_info.fullname + '.' + name new_sym.node = new_var + new_manager_info.names[name] = new_sym return custom_manager_type def run_with_model_cls(self, model_cls: Type[Model]) -> None: for manager_name, manager in model_cls._meta.managers_map.items(): - manager_class_name = manager.__class__.__name__ - manager_fullname = helpers.get_class_fullname(manager.__class__) - try: - manager_info = self.lookup_typeinfo_or_incomplete_defn_error(manager_fullname) - except helpers.IncompleteDefnException as exc: - if not self.api.final_iteration: - raise exc - else: - base_manager_fullname = helpers.get_class_fullname(manager.__class__.__bases__[0]) - generated_managers = self.get_generated_manager_mappings(base_manager_fullname) - if manager_fullname not in generated_managers: - # not a generated manager, continue with the loop - continue - real_manager_fullname = generated_managers[manager_fullname] - manager_info = self.lookup_typeinfo(real_manager_fullname) # type: ignore - if manager_info is None: - continue - manager_class_name = real_manager_fullname.rsplit('.', maxsplit=1)[1] + if self.model_class_has_attribute_defined(manager_name, traverse_mro=False): + sym = self.model_classdef.info.names.get(manager_name) + assert sym is not None + + if (sym.type is not None + and isinstance(sym.type, Instance) + and sym.type.type.has_base(fullnames.BASE_MANAGER_CLASS_FULLNAME) + and not self.has_any_parametrized_manager_as_base(sym.type.type)): + # already defined and parametrized properly + continue + + if getattr(manager, '_built_with_as_manager', False): + # as_manager is not supported yet + if not self.model_class_has_attribute_defined(manager_name, traverse_mro=True): + self.add_new_node_to_model_class(manager_name, AnyType(TypeOfAny.explicit)) + continue + + manager_fullname = self.resolve_manager_fullname(helpers.get_class_fullname(manager.__class__)) + manager_info = self.lookup_typeinfo_or_incomplete_defn_error(manager_fullname) + manager_class_name = manager_fullname.rsplit('.', maxsplit=1)[1] if manager_name not in self.model_classdef.info.names: + # manager not yet defined, just add models.Manager[ModelName] manager_type = Instance(manager_info, [Instance(self.model_classdef.info, [])]) self.add_new_node_to_model_class(manager_name, manager_type) else: @@ -226,56 +303,67 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: continue custom_model_manager_name = manager.model.__name__ + '_' + manager_class_name - try: - custom_manager_type = self.create_new_model_parametrized_manager(custom_model_manager_name, - base_manager_info=manager_info) - except helpers.IncompleteDefnException: - continue + custom_manager_type = self.create_new_model_parametrized_manager(custom_model_manager_name, + base_manager_info=manager_info) - self.add_new_node_to_model_class(manager_name, custom_manager_type) + self.add_new_node_to_model_class(manager_name, custom_manager_type, + force_replace_existing=True) class AddDefaultManagerAttribute(ModelClassInitializer): def run_with_model_cls(self, model_cls: Type[Model]) -> None: - # add _default_manager - if '_default_manager' not in self.model_classdef.info.names: - default_manager_fullname = helpers.get_class_fullname(model_cls._meta.default_manager.__class__) - default_manager_info = self.lookup_typeinfo_or_incomplete_defn_error(default_manager_fullname) - default_manager = Instance(default_manager_info, [Instance(self.model_classdef.info, [])]) - self.add_new_node_to_model_class('_default_manager', default_manager) + if self.model_class_has_attribute_defined('_default_manager', traverse_mro=False): + return + if model_cls._meta.default_manager is None: + return + if getattr(model_cls._meta.default_manager, '_built_with_as_manager', False): + self.add_new_node_to_model_class('_default_manager', + AnyType(TypeOfAny.explicit)) + return + + default_manager_fullname = helpers.get_class_fullname(model_cls._meta.default_manager.__class__) + resolved_default_manager_fullname = self.resolve_manager_fullname(default_manager_fullname) + + default_manager_info = self.lookup_typeinfo_or_incomplete_defn_error(resolved_default_manager_fullname) + default_manager = Instance(default_manager_info, [Instance(self.model_classdef.info, [])]) + self.add_new_node_to_model_class('_default_manager', default_manager) class AddRelatedManagers(ModelClassInitializer): def run_with_model_cls(self, model_cls: Type[Model]) -> None: # add related managers for relation in self.django_context.get_model_relations(model_cls): - attname = relation.get_accessor_name() - if attname is None: + related_manager_attr_name = relation.get_accessor_name() + if related_manager_attr_name is None: # no reverse accessor continue + if self.model_class_has_attribute_defined(related_manager_attr_name, traverse_mro=False): + continue + related_model_cls = self.django_context.get_field_related_model_cls(relation) if related_model_cls is None: continue try: related_model_info = self.lookup_class_typeinfo_or_incomplete_defn_error(related_model_cls) - except helpers.IncompleteDefnException as exc: + except sem_helpers.IncompleteDefnException as exc: if not self.api.final_iteration: raise exc else: continue if isinstance(relation, OneToOneRel): - self.add_new_node_to_model_class(attname, Instance(related_model_info, [])) + self.add_new_node_to_model_class(related_manager_attr_name, Instance(related_model_info, [])) continue if isinstance(relation, (ManyToOneRel, ManyToManyRel)): try: - related_manager_info = self.lookup_typeinfo_or_incomplete_defn_error(fullnames.RELATED_MANAGER_CLASS) # noqa: E501 + related_manager_info = self.lookup_typeinfo_or_incomplete_defn_error( + fullnames.RELATED_MANAGER_CLASS) # noqa: E501 if 'objects' not in related_model_info.names: - raise helpers.IncompleteDefnException() - except helpers.IncompleteDefnException as exc: + raise sem_helpers.IncompleteDefnException() + except sem_helpers.IncompleteDefnException as exc: if not self.api.final_iteration: raise exc else: @@ -288,14 +376,20 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: if (default_manager_type is None or not isinstance(default_manager_type, Instance) or default_manager_type.type.fullname == fullnames.MANAGER_CLASS_FULLNAME): - self.add_new_node_to_model_class(attname, parametrized_related_manager_type) + self.add_new_node_to_model_class(related_manager_attr_name, parametrized_related_manager_type) continue name = related_model_cls.__name__ + '_' + 'RelatedManager' bases = [parametrized_related_manager_type, default_manager_type] - new_related_manager_info = self.add_new_class_for_current_module(name, bases) + new_related_manager_info = self.add_new_class_for_current_module(name, bases, + force_replace_existing=True) + if new_related_manager_info is None: + # wasn't added for some reason, defer + if not self.api.final_iteration: + self.api.defer() + continue - self.add_new_node_to_model_class(attname, Instance(new_related_manager_info, [])) + self.add_new_node_to_model_class(related_manager_attr_name, Instance(new_related_manager_info, [])) class AddExtraFieldMethods(ModelClassInitializer): @@ -355,6 +449,8 @@ def process_model_class(ctx: ClassDefContext, for initializer_cls in initializers: try: initializer_cls(ctx, django_context).run() - except helpers.IncompleteDefnException: + except sem_helpers.IncompleteDefnException as exc: if not ctx.api.final_iteration: ctx.api.defer() + continue + raise exc diff --git a/mypy_django_plugin/transformers/orm_lookups.py b/mypy_django_plugin/transformers/orm_lookups.py index 0aa516be0..5e5d2804a 100644 --- a/mypy_django_plugin/transformers/orm_lookups.py +++ b/mypy_django_plugin/transformers/orm_lookups.py @@ -4,7 +4,7 @@ from mypy.types import TypeOfAny from mypy_django_plugin.django.context import DjangoContext -from mypy_django_plugin.lib import fullnames, helpers +from mypy_django_plugin.lib import fullnames, helpers, chk_helpers def typecheck_queryset_filter(ctx: MethodContext, django_context: DjangoContext) -> MypyType: @@ -35,10 +35,10 @@ def typecheck_queryset_filter(ctx: MethodContext, django_context: DjangoContext) fullnames.QUERYSET_CLASS_FULLNAME))): return ctx.default_return_type - helpers.check_types_compatible(ctx, - expected_type=lookup_type, - actual_type=provided_type, - error_message=f'Incompatible type for lookup {lookup_kwarg!r}:') + chk_helpers.check_types_compatible(ctx, + expected_type=lookup_type, + actual_type=provided_type, + error_message=f'Incompatible type for lookup {lookup_kwarg!r}:') return ctx.default_return_type diff --git a/mypy_django_plugin/transformers/querysets.py b/mypy_django_plugin/transformers/querysets.py index c157bb4a0..0d27dfcab 100644 --- a/mypy_django_plugin/transformers/querysets.py +++ b/mypy_django_plugin/transformers/querysets.py @@ -14,7 +14,7 @@ from mypy_django_plugin.django.context import ( DjangoContext, LookupsAreUnsupported, ) -from mypy_django_plugin.lib import fullnames, helpers +from mypy_django_plugin.lib import fullnames, helpers, chk_helpers def _extract_model_type_from_queryset(queryset_type: Instance) -> Optional[Instance]: @@ -30,7 +30,7 @@ def determine_proper_manager_type(ctx: FunctionContext) -> MypyType: default_return_type = ctx.default_return_type assert isinstance(default_return_type, Instance) - outer_model_info = helpers.get_typechecker_api(ctx).scope.active_class() + outer_model_info = chk_helpers.get_typechecker_api(ctx).scope.active_class() if (outer_model_info is None or not outer_model_info.has_base(fullnames.MODEL_CLASS_FULLNAME)): return default_return_type @@ -55,7 +55,7 @@ def get_field_type_from_lookup(ctx: MethodContext, django_context: DjangoContext return AnyType(TypeOfAny.from_error) lookup_field = django_context.get_primary_key_field(related_model_cls) - field_get_type = django_context.get_field_get_type(helpers.get_typechecker_api(ctx), + field_get_type = django_context.get_field_get_type(chk_helpers.get_typechecker_api(ctx), lookup_field, method=method) return field_get_type @@ -66,7 +66,7 @@ def get_values_list_row_type(ctx: MethodContext, django_context: DjangoContext, if field_lookups is None: return AnyType(TypeOfAny.from_error) - typechecker_api = helpers.get_typechecker_api(ctx) + typechecker_api = chk_helpers.get_typechecker_api(ctx) if len(field_lookups) == 0: if flat: primary_key_field = django_context.get_primary_key_field(model_cls) @@ -80,7 +80,7 @@ def get_values_list_row_type(ctx: MethodContext, django_context: DjangoContext, column_type = django_context.get_field_get_type(typechecker_api, field, method='values_list') column_types[field.attname] = column_type - return helpers.make_oneoff_named_tuple(typechecker_api, 'Row', column_types) + return chk_helpers.make_oneoff_named_tuple(typechecker_api, 'Row', column_types) else: # flat=False, named=False, all fields field_lookups = [] @@ -103,9 +103,9 @@ def get_values_list_row_type(ctx: MethodContext, django_context: DjangoContext, assert len(column_types) == 1 row_type = next(iter(column_types.values())) elif named: - row_type = helpers.make_oneoff_named_tuple(typechecker_api, 'Row', column_types) + row_type = chk_helpers.make_oneoff_named_tuple(typechecker_api, 'Row', column_types) else: - row_type = helpers.make_tuple(typechecker_api, list(column_types.values())) + row_type = chk_helpers.make_tuple(typechecker_api, list(column_types.values())) return row_type @@ -123,13 +123,13 @@ def extract_proper_type_queryset_values_list(ctx: MethodContext, django_context: if model_cls is None: return ctx.default_return_type - flat_expr = helpers.get_call_argument_by_name(ctx, 'flat') + flat_expr = chk_helpers.get_call_argument_by_name(ctx, 'flat') if flat_expr is not None and isinstance(flat_expr, NameExpr): flat = helpers.parse_bool(flat_expr) else: flat = False - named_expr = helpers.get_call_argument_by_name(ctx, 'named') + named_expr = chk_helpers.get_call_argument_by_name(ctx, 'named') if named_expr is not None and isinstance(named_expr, NameExpr): named = helpers.parse_bool(named_expr) else: @@ -188,5 +188,5 @@ def extract_proper_type_queryset_values(ctx: MethodContext, django_context: Djan column_types[field_lookup] = field_lookup_type - row_type = helpers.make_typeddict(ctx.api, column_types, set(column_types.keys())) + row_type = chk_helpers.make_oneoff_typeddict(ctx.api, column_types, set(column_types.keys())) return helpers.reparametrize_instance(ctx.default_return_type, [model_type, row_type]) diff --git a/mypy_django_plugin/transformers/request.py b/mypy_django_plugin/transformers/request.py index 83899ce29..55cb6b15b 100644 --- a/mypy_django_plugin/transformers/request.py +++ b/mypy_django_plugin/transformers/request.py @@ -4,7 +4,7 @@ from mypy.types import UnionType from mypy_django_plugin.django.context import DjangoContext -from mypy_django_plugin.lib import helpers +from mypy_django_plugin.lib import helpers, chk_helpers def set_auth_user_model_as_type_for_request_user(ctx: AttributeContext, django_context: DjangoContext) -> MypyType: @@ -25,10 +25,9 @@ def set_auth_user_model_as_type_for_request_user(ctx: AttributeContext, django_c return ctx.default_attr_type auth_user_model = django_context.settings.AUTH_USER_MODEL - user_cls = django_context.apps_registry.get_model(auth_user_model) - user_info = helpers.lookup_class_typeinfo(helpers.get_typechecker_api(ctx), user_cls) - - if user_info is None: + model_cls = django_context.apps_registry.get_model(auth_user_model) + model_info = helpers.lookup_class_typeinfo(chk_helpers.get_typechecker_api(ctx), model_cls) + if model_info is None: return ctx.default_attr_type return UnionType([Instance(user_info, []), Instance(anonymous_user_info, [])]) diff --git a/mypy_django_plugin/transformers/settings.py b/mypy_django_plugin/transformers/settings.py index ba6490b4e..67022abcf 100644 --- a/mypy_django_plugin/transformers/settings.py +++ b/mypy_django_plugin/transformers/settings.py @@ -5,7 +5,7 @@ from mypy.types import TypeOfAny, TypeType from mypy_django_plugin.django.context import DjangoContext -from mypy_django_plugin.lib import helpers +from mypy_django_plugin.lib import helpers, chk_helpers def get_user_model_hook(ctx: FunctionContext, django_context: DjangoContext) -> MypyType: @@ -13,7 +13,7 @@ def get_user_model_hook(ctx: FunctionContext, django_context: DjangoContext) -> model_cls = django_context.apps_registry.get_model(auth_user_model) model_cls_fullname = helpers.get_class_fullname(model_cls) - model_info = helpers.lookup_fully_qualified_typeinfo(helpers.get_typechecker_api(ctx), + model_info = helpers.lookup_fully_qualified_typeinfo(chk_helpers.get_typechecker_api(ctx), model_cls_fullname) if model_info is None: return AnyType(TypeOfAny.unannotated) @@ -28,7 +28,7 @@ def get_type_of_settings_attribute(ctx: AttributeContext, django_context: Django ctx.api.fail(f"'Settings' object has no attribute {setting_name!r}", ctx.context) return ctx.default_attr_type - typechecker_api = helpers.get_typechecker_api(ctx) + typechecker_api = chk_helpers.get_typechecker_api(ctx) # first look for the setting in the project settings file, then global settings settings_module = typechecker_api.modules.get(django_context.django_settings_module) diff --git a/test-data/typecheck/managers/querysets/test_from_queryset.yml b/test-data/typecheck/managers/querysets/test_from_queryset.yml index e9f2ad4ff..96bdf9a3a 100644 --- a/test-data/typecheck/managers/querysets/test_from_queryset.yml +++ b/test-data/typecheck/managers/querysets/test_from_queryset.yml @@ -3,6 +3,7 @@ from myapp.models import MyModel reveal_type(MyModel().objects) # N: Revealed type is 'myapp.models.MyModel_NewManager[myapp.models.MyModel]' reveal_type(MyModel().objects.get()) # N: Revealed type is 'myapp.models.MyModel*' + reveal_type(MyModel().objects.queryset_method) # N: Revealed type is 'def () -> builtins.str' reveal_type(MyModel().objects.queryset_method()) # N: Revealed type is 'builtins.str' installed_apps: - myapp @@ -178,4 +179,57 @@ from django.db import models class BaseQuerySet(models.QuerySet): def base_queryset_method(self, param: Union[int, str]) -> NoReturn: - raise ValueError \ No newline at end of file + raise ValueError + + +- case: from_queryset_with_inherited_manager_and_fk_to_auth_contrib + disable_cache: true + main: | + from myapp.base_queryset import BaseQuerySet + reveal_type(BaseQuerySet().base_queryset_method) # N: Revealed type is 'def (param: builtins.dict[builtins.str, Union[builtins.int, builtins.str]]) -> Union[builtins.int, builtins.str]' + + from django.contrib.auth.models import Permission + reveal_type(Permission().another_models) # N: Revealed type is 'django.db.models.manager.RelatedManager[myapp.models.AnotherModelInProjectWithContribAuthM2M]' + + from myapp.managers import NewManager + reveal_type(NewManager()) # N: Revealed type is 'myapp.managers.NewManager' + reveal_type(NewManager().base_queryset_method) # N: Revealed type is 'def (param: builtins.dict[builtins.str, Union[builtins.int, builtins.str]]) -> Union[builtins.int, builtins.str]' + + from myapp.models import MyModel + reveal_type(MyModel().objects) # N: Revealed type is 'myapp.models.MyModel_NewManager[myapp.models.MyModel]' + reveal_type(MyModel().objects.get()) # N: Revealed type is 'myapp.models.MyModel*' + reveal_type(MyModel().objects.base_queryset_method) # N: Revealed type is 'def (param: builtins.dict[builtins.str, Union[builtins.int, builtins.str]]) -> Union[builtins.int, builtins.str]' + installed_apps: + - myapp + - django.contrib.auth + files: + - path: myapp/__init__.py + - path: myapp/models.py + content: | + from django.db import models + from myapp.managers import NewManager + from django.contrib.auth.models import Permission + + class MyModel(models.Model): + objects = NewManager() + + class AnotherModelInProjectWithContribAuthM2M(models.Model): + permissions = models.ForeignKey( + Permission, + on_delete=models.PROTECT, + related_name='another_models' + ) + - path: myapp/managers.py + content: | + from django.db import models + from myapp.base_queryset import BaseQuerySet + class ModelQuerySet(BaseQuerySet): + pass + NewManager = models.Manager.from_queryset(ModelQuerySet) + - path: myapp/base_queryset.py + content: | + from typing import Union, Dict + from django.db import models + class BaseQuerySet(models.QuerySet): + def base_queryset_method(self, param: Dict[str, Union[int, str]]) -> Union[int, str]: + return param["hello"] \ No newline at end of file diff --git a/test-data/typecheck/managers/test_managers.yml b/test-data/typecheck/managers/test_managers.yml index 6018938fa..1f40552b5 100644 --- a/test-data/typecheck/managers/test_managers.yml +++ b/test-data/typecheck/managers/test_managers.yml @@ -307,15 +307,15 @@ - case: custom_manager_returns_proper_model_types main: | from myapp.models import User - reveal_type(User.objects) # N: Revealed type is 'myapp.models.User_MyManager2[myapp.models.User]' - reveal_type(User.objects.select_related()) # N: Revealed type is 'myapp.models.User_MyManager2[myapp.models.User]' + reveal_type(User.objects) # N: Revealed type is 'myapp.models.User_MyManager[myapp.models.User]' + reveal_type(User.objects.select_related()) # N: Revealed type is 'myapp.models.User_MyManager[myapp.models.User]' reveal_type(User.objects.get()) # N: Revealed type is 'myapp.models.User*' reveal_type(User.objects.get_instance()) # N: Revealed type is 'builtins.int' reveal_type(User.objects.get_instance_untyped('hello')) # N: Revealed type is 'Any' from myapp.models import ChildUser - reveal_type(ChildUser.objects) # N: Revealed type is 'myapp.models.ChildUser_MyManager2[myapp.models.ChildUser]' - reveal_type(ChildUser.objects.select_related()) # N: Revealed type is 'myapp.models.ChildUser_MyManager2[myapp.models.ChildUser]' + reveal_type(ChildUser.objects) # N: Revealed type is 'myapp.models.ChildUser_MyManager[myapp.models.ChildUser]' + reveal_type(ChildUser.objects.select_related()) # N: Revealed type is 'myapp.models.ChildUser_MyManager[myapp.models.ChildUser]' reveal_type(ChildUser.objects.get()) # N: Revealed type is 'myapp.models.ChildUser*' reveal_type(ChildUser.objects.get_instance()) # N: Revealed type is 'builtins.int' reveal_type(ChildUser.objects.get_instance_untyped('hello')) # N: Revealed type is 'Any' From db925ac70a7e5cee96996ad6a2061553232b0070 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sat, 4 Jan 2020 16:52:13 +0300 Subject: [PATCH 02/76] QuerySet.as_manager() support --- mypy_django_plugin/main.py | 17 +- mypy_django_plugin/transformers/managers.py | 212 ++++++++++++++++-- .../managers/querysets/test_as_manager.yml | 42 ++++ 3 files changed, 246 insertions(+), 25 deletions(-) create mode 100644 test-data/typecheck/managers/querysets/test_as_manager.yml diff --git a/mypy_django_plugin/main.py b/mypy_django_plugin/main.py index 2dfb3b9d2..f9032307a 100644 --- a/mypy_django_plugin/main.py +++ b/mypy_django_plugin/main.py @@ -19,7 +19,7 @@ ) from mypy_django_plugin.transformers.managers import ( create_new_manager_class_from_from_queryset_method, -) + create_manager_class_from_as_manager_method, instantiate_anonymous_queryset_from_as_manager) from mypy_django_plugin.transformers.models import process_model_class @@ -138,6 +138,10 @@ def _new_dependency(self, module: str) -> Tuple[int, str, int]: return 10, module, -1 def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]: + # load QuerySet and Manager together (for as_manager) + if file.fullname == 'django.db.models.query': + return [self._new_dependency('django.db.models.manager')] + # for settings if file.fullname == 'django.conf' and self.django_context.django_settings_module: return [self._new_dependency(self.django_context.django_settings_module)] @@ -227,6 +231,11 @@ def get_method_hook(self, fullname: str if info and info.has_base(fullnames.OPTIONS_CLASS_FULLNAME): return partial(meta.return_proper_field_type_from_get_field, django_context=self.django_context) + if method_name == 'as_manager': + info = self._get_typeinfo_or_none(class_fullname) + if info and info.has_base(fullnames.QUERYSET_CLASS_FULLNAME): + return instantiate_anonymous_queryset_from_as_manager + manager_classes = self._get_current_manager_bases() if class_fullname in manager_classes and method_name == 'create': return partial(init_create.redefine_and_typecheck_model_create, django_context=self.django_context) @@ -267,6 +276,12 @@ def get_dynamic_class_hook(self, fullname: str info = self._get_typeinfo_or_none(class_name) if info and info.has_base(fullnames.BASE_MANAGER_CLASS_FULLNAME): return create_new_manager_class_from_from_queryset_method + if fullname.endswith('as_manager'): + class_name, _, _ = fullname.rpartition('.') + info = self._get_typeinfo_or_none(class_name) + if info and info.has_base(fullnames.QUERYSET_CLASS_FULLNAME): + return create_manager_class_from_as_manager_method + return None diff --git a/mypy_django_plugin/transformers/managers.py b/mypy_django_plugin/transformers/managers.py index c30427323..6d43e4a9a 100644 --- a/mypy_django_plugin/transformers/managers.py +++ b/mypy_django_plugin/transformers/managers.py @@ -1,14 +1,16 @@ -from typing import Iterator, Tuple, Optional +from typing import Iterator, Tuple, Optional, Any, Dict from mypy.nodes import ( FuncDef, MemberExpr, NameExpr, RefExpr, StrExpr, TypeInfo, - PlaceholderNode, SymbolTableNode, GDEF -) -from mypy.plugin import ClassDefContext, DynamicClassDefContext -from mypy.types import AnyType, Instance, TypeOfAny + PlaceholderNode, SymbolTableNode, GDEF, + CallExpr, Context, Decorator, OverloadedFuncDef, SymbolTable) +from mypy.plugin import ClassDefContext, DynamicClassDefContext, MethodContext +from mypy.semanal import SemanticAnalyzer, is_valid_replacement, is_same_symbol +from mypy.types import AnyType, Instance, TypeOfAny, CallableType +from mypy.types import Type as MypyType from mypy.typevars import fill_typevars -from mypy_django_plugin.lib import fullnames, sem_helpers, helpers +from mypy_django_plugin.lib import fullnames, sem_helpers, helpers, chk_helpers def iter_all_custom_queryset_methods(derived_queryset_info: TypeInfo) -> Iterator[Tuple[str, FuncDef]]: @@ -20,19 +22,23 @@ def iter_all_custom_queryset_methods(derived_queryset_info: TypeInfo) -> Iterato yield name, sym.node -def resolve_callee_manager_info_or_exception(ctx: DynamicClassDefContext) -> Optional[TypeInfo]: +def generate_from_queryset_name(base_manager_info: TypeInfo, queryset_info: TypeInfo) -> str: + return base_manager_info.name + 'From' + queryset_info.name + + +def resolve_callee_info_or_exception(ctx: DynamicClassDefContext) -> Optional[TypeInfo]: callee = ctx.call.callee assert isinstance(callee, MemberExpr) assert isinstance(callee.expr, RefExpr) - callee_manager_info = callee.expr.node - if (callee_manager_info is None - or isinstance(callee_manager_info, PlaceholderNode)): - raise sem_helpers.IncompleteDefnException(f'Definition of base manager {callee_manager_info.fullname} ' + callee_info = callee.expr.node + if (callee_info is None + or isinstance(callee_info, PlaceholderNode)): + raise sem_helpers.IncompleteDefnException(f'Definition of base manager {callee_info.fullname} ' f'is incomplete.') - assert isinstance(callee_manager_info, TypeInfo) - return callee_manager_info + assert isinstance(callee_info, TypeInfo) + return callee_info def resolve_passed_queryset_info_or_exception(ctx: DynamicClassDefContext) -> Optional[TypeInfo]: @@ -75,28 +81,39 @@ def new_manager_typeinfo(ctx: DynamicClassDefContext, callee_manager_info: TypeI return new_manager_info -def record_new_manager_info_fullname_into_metadata(ctx: DynamicClassDefContext, - new_manager_fullname: str, - callee_manager_info: TypeInfo, - queryset_info: TypeInfo, - django_manager_info: TypeInfo) -> None: - if len(ctx.call.args) > 1: - expr = ctx.call.args[1] +def get_generated_manager_fullname(call: CallExpr, base_manager_info: TypeInfo, queryset_info: TypeInfo) -> str: + if len(call.args) > 1: + # only for from_queryset() + expr = call.args[1] assert isinstance(expr, StrExpr) custom_manager_generated_name = expr.value else: - custom_manager_generated_name = callee_manager_info.name + 'From' + queryset_info.name + custom_manager_generated_name = base_manager_info.name + 'From' + queryset_info.name custom_manager_generated_fullname = 'django.db.models.manager' + '.' + custom_manager_generated_name + return custom_manager_generated_fullname + + +def get_generated_managers_metadata(django_manager_info: TypeInfo) -> Dict[str, Any]: + return django_manager_info.metadata.setdefault('from_queryset_managers', {}) + - metadata = django_manager_info.metadata.setdefault('from_queryset_managers', {}) +def record_new_manager_info_fullname_into_metadata(ctx: DynamicClassDefContext, + new_manager_fullname: str, + callee_manager_info: TypeInfo, + queryset_info: TypeInfo, + django_manager_info: TypeInfo) -> None: + custom_manager_generated_fullname = get_generated_manager_fullname(ctx.call, + base_manager_info=callee_manager_info, + queryset_info=queryset_info) + metadata = get_generated_managers_metadata(django_manager_info) metadata[custom_manager_generated_fullname] = new_manager_fullname def create_new_manager_class_from_from_queryset_method(ctx: DynamicClassDefContext) -> None: semanal_api = sem_helpers.get_semanal_api(ctx) try: - callee_manager_info = resolve_callee_manager_info_or_exception(ctx) + callee_manager_info = resolve_callee_info_or_exception(ctx) queryset_info = resolve_passed_queryset_info_or_exception(ctx) django_manager_info = resolve_django_manager_info_or_exception(ctx) except sem_helpers.IncompleteDefnException: @@ -116,7 +133,6 @@ def create_new_manager_class_from_from_queryset_method(ctx: DynamicClassDefConte class_def_context = ClassDefContext(cls=new_manager_info.defn, reason=ctx.call, api=semanal_api) self_type = fill_typevars(new_manager_info) - # self_type = Instance(new_manager_info, []) try: for name, method_node in iter_all_custom_queryset_methods(queryset_info): @@ -147,3 +163,151 @@ def create_new_manager_class_from_from_queryset_method(ctx: DynamicClassDefConte if (not added and not semanal_api.final_iteration): semanal_api.defer() + + +def add_symbol_table_node(api: SemanticAnalyzer, + name: str, + symbol: SymbolTableNode, + context: Optional[Context] = None, + symbol_table: Optional[SymbolTable] = None, + can_defer: bool = True, + escape_comprehensions: bool = False) -> bool: + """Add symbol table node to the currently active symbol table. + + Return True if we actually added the symbol, or False if we refused + to do so (because something is not ready or it was a no-op). + + Generate an error if there is an invalid redefinition. + + If context is None, unconditionally add node, since we can't report + an error. Note that this is used by plugins to forcibly replace nodes! + + TODO: Prevent plugins from replacing nodes, as it could cause problems? + + Args: + name: short name of symbol + symbol: Node to add + can_defer: if True, defer current target if adding a placeholder + context: error context (see above about None value) + """ + names = symbol_table or api.current_symbol_table(escape_comprehensions=escape_comprehensions) + existing = names.get(name) + if isinstance(symbol.node, PlaceholderNode) and can_defer: + api.defer(context) + if (existing is not None + and context is not None + and not is_valid_replacement(existing, symbol)): + # There is an existing node, so this may be a redefinition. + # If the new node points to the same node as the old one, + # or if both old and new nodes are placeholders, we don't + # need to do anything. + old = existing.node + new = symbol.node + if isinstance(new, PlaceholderNode): + # We don't know whether this is okay. Let's wait until the next iteration. + return False + if not is_same_symbol(old, new): + if isinstance(new, (FuncDef, Decorator, OverloadedFuncDef, TypeInfo)): + api.add_redefinition(names, name, symbol) + if not (isinstance(new, (FuncDef, Decorator)) + and api.set_original_def(old, new)): + api.name_already_defined(name, context, existing) + elif name not in api.missing_names and '*' not in api.missing_names: + names[name] = symbol + api.progress = True + return True + return False + + + +def create_manager_class_from_as_manager_method(ctx: DynamicClassDefContext) -> None: + semanal_api = sem_helpers.get_semanal_api(ctx) + try: + queryset_info = resolve_callee_info_or_exception(ctx) + django_manager_info = resolve_django_manager_info_or_exception(ctx) + except sem_helpers.IncompleteDefnException: + if not semanal_api.final_iteration: + semanal_api.defer() + return + else: + raise + + generic_param = AnyType(TypeOfAny.explicit) + generic_param_name = 'Any' + if (semanal_api.scope.classes + and semanal_api.scope.classes[-1].has_base(fullnames.MODEL_CLASS_FULLNAME)): + info = semanal_api.scope.classes[-1] # type: TypeInfo + generic_param = Instance(info, []) + generic_param_name = info.name + + new_manager_class_name = queryset_info.name + '_AsManager_' + generic_param_name + new_manager_info = helpers.new_typeinfo(new_manager_class_name, + bases=[Instance(django_manager_info, [generic_param])], + module_name=semanal_api.cur_mod_id) + new_manager_info.set_line(ctx.call) + + record_new_manager_info_fullname_into_metadata(ctx, + new_manager_info.fullname, + django_manager_info, + queryset_info, + django_manager_info) + + class_def_context = ClassDefContext(cls=new_manager_info.defn, + reason=ctx.call, api=semanal_api) + self_type = fill_typevars(new_manager_info) + + try: + for name, method_node in iter_all_custom_queryset_methods(queryset_info): + sem_helpers.copy_method_or_incomplete_defn_exception(class_def_context, + self_type, + new_method_name=name, + method_node=method_node) + except sem_helpers.IncompleteDefnException: + if not semanal_api.final_iteration: + semanal_api.defer() + return + else: + raise + + new_manager_sym = SymbolTableNode(GDEF, new_manager_info, plugin_generated=True) + + # context=None - forcibly replace old node + added = add_symbol_table_node(semanal_api, new_manager_class_name, new_manager_sym, + context=None, + symbol_table=semanal_api.globals) + if added: + # replace all references to the old manager Var everywhere + for _, module in semanal_api.modules.items(): + if module.fullname != semanal_api.cur_mod_id: + for sym_name, sym in module.names.items(): + if sym.fullname == new_manager_info.fullname: + module.names[sym_name] = new_manager_sym.copy() + + # we need another iteration to process methods + if (not added + and not semanal_api.final_iteration): + semanal_api.defer() + + +def instantiate_anonymous_queryset_from_as_manager(ctx: MethodContext) -> MypyType: + api = chk_helpers.get_typechecker_api(ctx) + django_manager_info = helpers.lookup_fully_qualified_typeinfo(api, fullnames.MANAGER_CLASS_FULLNAME) + assert django_manager_info is not None + + assert isinstance(ctx.type, CallableType) + assert isinstance(ctx.type.ret_type, Instance) + queryset_info = ctx.type.ret_type.type + + fullname = get_generated_manager_fullname(ctx.context, + base_manager_info=django_manager_info, + queryset_info=queryset_info) + metadata = get_generated_managers_metadata(django_manager_info) + if fullname not in metadata: + raise ValueError(f'{fullname!r} is not present in generated managers list') + + module_name, _, class_name = metadata[fullname].rpartition('.') + current_module = helpers.get_current_module(api) + assert module_name == current_module.fullname + + generated_manager_info = current_module.names[class_name].node + return fill_typevars(generated_manager_info) diff --git a/test-data/typecheck/managers/querysets/test_as_manager.yml b/test-data/typecheck/managers/querysets/test_as_manager.yml new file mode 100644 index 000000000..e08c02168 --- /dev/null +++ b/test-data/typecheck/managers/querysets/test_as_manager.yml @@ -0,0 +1,42 @@ +- case: anonymous_queryset_from_as_manager_inside_model + main: | + from myapp.models import MyModel + + reveal_type(MyModel.objects) # N: Revealed type is 'myapp.models.MyQuerySet_AsManager_MyModel' + reveal_type(MyModel.objects.get()) # N: Revealed type is 'myapp.models.MyModel*' + reveal_type(MyModel.objects.queryset_method) # N: Revealed type is 'def () -> builtins.int' + reveal_type(MyModel.objects.queryset_method()) # N: Revealed type is 'builtins.int' + installed_apps: + - myapp + files: + - path: myapp/__init__.py + - path: myapp/models.py + content: | + from django.db import models + class MyQuerySet(models.QuerySet): + def queryset_method(self) -> int: + pass + class MyModel(models.Model): + objects = MyQuerySet.as_manager() + + +- case: as_manager_outside_model_parametrized_with_any + main: | + from myapp.models import NotModel, outside_objects + reveal_type(NotModel.objects) # N: Revealed type is 'myapp.models.MyQuerySet_AsManager_Any' + reveal_type(NotModel.objects.get()) # N: Revealed type is 'Any' + reveal_type(outside_objects) # N: Revealed type is 'myapp.models.MyQuerySet_AsManager_Any' + reveal_type(outside_objects.get()) # N: Revealed type is 'Any' + installed_apps: + - myapp + files: + - path: myapp/__init__.py + - path: myapp/models.py + content: | + from django.db import models + class MyQuerySet(models.QuerySet): + def queryset_method(self) -> int: + pass + outside_objects = MyQuerySet.as_manager() + class NotModel: + objects = MyQuerySet.as_manager() From 74ccc4d26f2e25479d5e1da15694a8310750f85a Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sat, 4 Jan 2020 17:56:05 +0300 Subject: [PATCH 03/76] allow manager classes nested inside model classes --- mypy_django_plugin/lib/helpers.py | 27 +++++++++++---- mypy_django_plugin/transformers/managers.py | 2 +- scripts/enabled_test_modules.py | 3 ++ .../typecheck/managers/test_managers.yml | 34 ++++++++++++++----- 4 files changed, 51 insertions(+), 15 deletions(-) diff --git a/mypy_django_plugin/lib/helpers.py b/mypy_django_plugin/lib/helpers.py index 5b7991a3f..6740fe370 100644 --- a/mypy_django_plugin/lib/helpers.py +++ b/mypy_django_plugin/lib/helpers.py @@ -31,15 +31,30 @@ def get_django_metadata(model_info: TypeInfo) -> Dict[str, Any]: def lookup_fully_qualified_sym(fullname: str, all_modules: Dict[str, MypyFile]) -> Optional[SymbolTableNode]: if '.' not in fullname: return None - module, cls_name = fullname.rsplit('.', 1) - module_file = all_modules.get(module) + module_file = None + parts = fullname.split('.') + for i in range(len(parts), 0, -1): + possible_module_name = '.'.join(parts[:i]) + if possible_module_name in all_modules: + module_file = all_modules[possible_module_name] + break + if module_file is None: return None - sym = module_file.names.get(cls_name) - if sym is None: - return None - return sym + + cls_name = fullname.replace(module_file.fullname, '').lstrip('.') + sym_table = module_file.names + if '.' in cls_name: + parent_cls_name, _, cls_name = cls_name.rpartition('.') + # nested class + for parent_cls_name in parent_cls_name.split('.'): + sym = sym_table.get(parent_cls_name) + if sym is None: + return None + sym_table = sym.node.names + + return sym_table.get(cls_name) def lookup_fully_qualified_generic(name: str, all_modules: Dict[str, MypyFile]) -> Optional[SymbolNode]: diff --git a/mypy_django_plugin/transformers/managers.py b/mypy_django_plugin/transformers/managers.py index 6d43e4a9a..57de98c4f 100644 --- a/mypy_django_plugin/transformers/managers.py +++ b/mypy_django_plugin/transformers/managers.py @@ -310,4 +310,4 @@ def instantiate_anonymous_queryset_from_as_manager(ctx: MethodContext) -> MypyTy assert module_name == current_module.fullname generated_manager_info = current_module.names[class_name].node - return fill_typevars(generated_manager_info) + return Instance(generated_manager_info, []) diff --git a/scripts/enabled_test_modules.py b/scripts/enabled_test_modules.py index 49e519e66..4b1a49773 100644 --- a/scripts/enabled_test_modules.py +++ b/scripts/enabled_test_modules.py @@ -325,6 +325,9 @@ 'model_enums': [ "'bool' is not a valid base class", ], + 'multiple_database': [ + 'Unexpected attribute "extra_arg" for model "Book"', + ], 'null_queries': [ "Cannot resolve keyword 'foo' into field" ], diff --git a/test-data/typecheck/managers/test_managers.yml b/test-data/typecheck/managers/test_managers.yml index 1f40552b5..21ea5a840 100644 --- a/test-data/typecheck/managers/test_managers.yml +++ b/test-data/typecheck/managers/test_managers.yml @@ -336,7 +336,7 @@ class ChildUser(models.Model): objects = MyManager() -- case: custom_manager_annotate_method_before_type_declaration +- case: custom_manager_annotate_method_before_type_declaration main: | from myapp.models import ModelA, ModelB, ManagerA reveal_type(ModelA.objects) # N: Revealed type is 'myapp.models.ModelA_ManagerA1[myapp.models.ModelA]' @@ -344,16 +344,34 @@ installed_apps: - myapp files: - - path: myapp/__init__.py - - path: myapp/models.py - content: | - from django.db import models - class ManagerA(models.Manager): + - path: myapp/__init__.py + - path: myapp/models.py + content: | + from django.db import models + class ManagerA(models.Manager): def do_something(self, other_obj: "ModelB") -> str: return 'test' - class ModelA(models.Model): + class ModelA(models.Model): title = models.TextField() objects = ManagerA() - class ModelB(models.Model): + class ModelB(models.Model): movie = models.TextField() +- case: manager_defined_in_the_nested_class + main: | + from myapp.models import MyModel + reveal_type(MyModel.objects) # N: Revealed type is 'myapp.models.MyModel_MyManager[myapp.models.MyModel]' + reveal_type(MyModel.objects.get()) # N: Revealed type is 'myapp.models.MyModel*' + reveal_type(MyModel.objects.mymethod()) # N: Revealed type is 'builtins.int' + installed_apps: + - myapp + files: + - path: myapp/__init__.py + - path: myapp/models.py + content: | + from django.db import models + class MyModel(models.Model): + class MyManager(models.Manager): + def mymethod(self) -> int: + pass + objects = MyManager() From f58cbeabfc256751a07dfbc66c74bf517b78f970 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sat, 4 Jan 2020 19:09:50 +0300 Subject: [PATCH 04/76] lint fixes --- mypy_django_plugin/django/context.py | 2 +- mypy_django_plugin/lib/chk_helpers.py | 20 +++++--- mypy_django_plugin/lib/helpers.py | 9 ++-- mypy_django_plugin/lib/sem_helpers.py | 31 ++++++------ mypy_django_plugin/main.py | 8 ++-- mypy_django_plugin/transformers/fields.py | 2 +- mypy_django_plugin/transformers/forms.py | 2 +- mypy_django_plugin/transformers/managers.py | 48 ++++++++++--------- mypy_django_plugin/transformers/meta.py | 2 +- mypy_django_plugin/transformers/models.py | 45 ++++------------- .../transformers/orm_lookups.py | 2 +- mypy_django_plugin/transformers/querysets.py | 2 +- mypy_django_plugin/transformers/request.py | 2 +- mypy_django_plugin/transformers/settings.py | 2 +- 14 files changed, 82 insertions(+), 95 deletions(-) diff --git a/mypy_django_plugin/django/context.py b/mypy_django_plugin/django/context.py index eeee4512e..f9dd152b7 100644 --- a/mypy_django_plugin/django/context.py +++ b/mypy_django_plugin/django/context.py @@ -21,7 +21,7 @@ from mypy.types import Type as MypyType from mypy.types import TypeOfAny, UnionType -from mypy_django_plugin.lib import fullnames, helpers, chk_helpers +from mypy_django_plugin.lib import chk_helpers, fullnames, helpers try: from django.contrib.postgres.fields import ArrayField diff --git a/mypy_django_plugin/lib/chk_helpers.py b/mypy_django_plugin/lib/chk_helpers.py index 22c12421d..3f657ff81 100644 --- a/mypy_django_plugin/lib/chk_helpers.py +++ b/mypy_django_plugin/lib/chk_helpers.py @@ -1,10 +1,16 @@ -from typing import OrderedDict, List, Optional, Dict, Set, Union +from typing import Dict, List, Optional, Set, Union from mypy import checker from mypy.checker import TypeChecker -from mypy.nodes import MypyFile, TypeInfo, Var, MDEF, SymbolTableNode, GDEF, Expression -from mypy.plugin import CheckerPluginInterface, FunctionContext, MethodContext, AttributeContext -from mypy.types import Type as MypyType, Instance, TupleType, TypeOfAny, AnyType, TypedDictType +from mypy.nodes import ( + GDEF, MDEF, Expression, MypyFile, SymbolTableNode, TypeInfo, Var, +) +from mypy.plugin import ( + AttributeContext, CheckerPluginInterface, FunctionContext, MethodContext, +) +from mypy.types import AnyType, Instance, TupleType +from mypy.types import Type as MypyType +from mypy.types import TypedDictType, TypeOfAny from mypy_django_plugin.lib import helpers @@ -49,10 +55,12 @@ def make_tuple(api: 'TypeChecker', fields: List[MypyType]) -> TupleType: return TupleType(fields, fallback=fallback) -def make_oneoff_typeddict(api: CheckerPluginInterface, fields: 'OrderedDict[str, MypyType]', +def make_oneoff_typeddict(api: CheckerPluginInterface, fields: 'Dict[str, MypyType]', required_keys: Set[str]) -> TypedDictType: object_type = api.named_generic_type('mypy_extensions._TypedDict', []) - typed_dict_type = TypedDictType(fields, required_keys=required_keys, fallback=object_type) + typed_dict_type = TypedDictType(fields, # type: ignore + required_keys=required_keys, + fallback=object_type) return typed_dict_type diff --git a/mypy_django_plugin/lib/helpers.py b/mypy_django_plugin/lib/helpers.py index 6740fe370..f743e074f 100644 --- a/mypy_django_plugin/lib/helpers.py +++ b/mypy_django_plugin/lib/helpers.py @@ -2,20 +2,20 @@ TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Union, ) +from django.db.models.fields import Field from django.db.models.fields.related import RelatedField from django.db.models.fields.reverse_related import ForeignObjectRel from mypy.checker import TypeChecker from mypy.mro import calculate_mro from mypy.nodes import ( - Block, ClassDef, Expression, MemberExpr, MypyFile, NameExpr, StrExpr, SymbolNode, - SymbolTable, SymbolTableNode, TypeInfo, Var, + Block, ClassDef, Expression, MemberExpr, MypyFile, NameExpr, StrExpr, SymbolNode, SymbolTable, SymbolTableNode, + TypeInfo, Var, ) from mypy.semanal import SemanticAnalyzer from mypy.types import AnyType, Instance, NoneTyp from mypy.types import Type as MypyType from mypy.types import TypeOfAny, UnionType -from django.db.models.fields import Field from mypy_django_plugin.lib import fullnames if TYPE_CHECKING: @@ -50,7 +50,8 @@ def lookup_fully_qualified_sym(fullname: str, all_modules: Dict[str, MypyFile]) # nested class for parent_cls_name in parent_cls_name.split('.'): sym = sym_table.get(parent_cls_name) - if sym is None: + if (sym is None or sym.node is None + or not isinstance(sym.node, TypeInfo)): return None sym_table = sym.node.names diff --git a/mypy_django_plugin/lib/sem_helpers.py b/mypy_django_plugin/lib/sem_helpers.py index 30d67bb2d..33db42e03 100644 --- a/mypy_django_plugin/lib/sem_helpers.py +++ b/mypy_django_plugin/lib/sem_helpers.py @@ -1,11 +1,12 @@ -from typing import Union, Tuple, List, Optional, NamedTuple, cast +from typing import List, NamedTuple, Optional, Tuple, Union, cast -from mypy.nodes import Argument, FuncDef, Var, TypeInfo -from mypy.plugin import DynamicClassDefContext, ClassDefContext +from mypy.nodes import Argument, FuncDef, TypeInfo, Var +from mypy.plugin import ClassDefContext, DynamicClassDefContext from mypy.plugins.common import add_method from mypy.semanal import SemanticAnalyzer -from mypy.types import Instance, CallableType, AnyType, TypeOfAny, PlaceholderType +from mypy.types import AnyType, CallableType, Instance from mypy.types import Type as MypyType +from mypy.types import TypeOfAny class IncompleteDefnException(Exception): @@ -39,7 +40,7 @@ def prepare_unannotated_method_signature(method_node: FuncDef) -> Tuple[List[Arg class SignatureTuple(NamedTuple): - arguments: Optional[List[Argument]] + arguments: List[Argument] return_type: Optional[MypyType] cannot_be_bound: bool @@ -53,16 +54,14 @@ def analyze_callable_signature(api: SemanticAnalyzer, method_node: FuncDef) -> S for arg_name, arg_type, original_argument in zip(method_type.arg_names[1:], method_type.arg_types[1:], method_node.arguments[1:]): - arg_type = api.anal_type(arg_type, allow_placeholder=True) - if isinstance(arg_type, PlaceholderType): + analyzed_arg_type = api.anal_type(arg_type) + if analyzed_arg_type is None: unbound = True var = Var(name=original_argument.variable.name, - type=arg_type) + type=analyzed_arg_type) var.set_line(original_argument.variable) - if isinstance(arg_type, PlaceholderType): - unbound = True argument = Argument(variable=var, type_annotation=arg_type, initializer=original_argument.initializer, @@ -70,10 +69,10 @@ def analyze_callable_signature(api: SemanticAnalyzer, method_node: FuncDef) -> S argument.set_line(original_argument) arguments.append(argument) - ret_type = api.anal_type(method_type.ret_type, allow_placeholder=True) - if isinstance(ret_type, PlaceholderType): + analyzed_ret_type = api.anal_type(method_type.ret_type) + if analyzed_ret_type is None: unbound = True - return SignatureTuple(arguments, ret_type, unbound) + return SignatureTuple(arguments, analyzed_ret_type, unbound) def copy_method_or_incomplete_defn_exception(ctx: ClassDefContext, @@ -103,15 +102,17 @@ def copy_method_or_incomplete_defn_exception(ctx: ClassDefContext, and name not in semanal_api.cur_mod_node.names): semanal_api.add_imported_symbol(name, sym, context=semanal_api.cur_mod_node) - arguments, return_type, unbound = analyze_callable_signature(semanal_api, method_node) + arguments, analyzed_return_type, unbound = analyze_callable_signature(semanal_api, method_node) assert len(arguments) + 1 == len(method_node.arguments) if unbound: raise IncompleteDefnException(f'Signature of method {method_node.fullname!r} is not ready') + assert analyzed_return_type is not None + if new_method_name in ctx.cls.info.names: del ctx.cls.info.names[new_method_name] add_method(ctx, new_method_name, args=arguments, - return_type=return_type, + return_type=analyzed_return_type, self_type=self_type) diff --git a/mypy_django_plugin/main.py b/mypy_django_plugin/main.py index f9032307a..39a3f3e4e 100644 --- a/mypy_django_plugin/main.py +++ b/mypy_django_plugin/main.py @@ -8,7 +8,6 @@ from mypy.plugin import ( AttributeContext, ClassDefContext, DynamicClassDefContext, FunctionContext, MethodContext, Plugin, ) -from mypy.semanal import dummy_context from mypy.types import Type as MypyType import mypy_django_plugin.transformers.orm_lookups @@ -18,8 +17,9 @@ fields, forms, init_create, meta, querysets, request, settings, ) from mypy_django_plugin.transformers.managers import ( - create_new_manager_class_from_from_queryset_method, - create_manager_class_from_as_manager_method, instantiate_anonymous_queryset_from_as_manager) + create_manager_class_from_as_manager_method, create_new_manager_class_from_from_queryset_method, + instantiate_anonymous_queryset_from_as_manager, +) from mypy_django_plugin.transformers.models import process_model_class @@ -30,7 +30,7 @@ def transform_model_class(ctx: ClassDefContext, if sym is not None and isinstance(sym.node, TypeInfo): helpers.get_django_metadata(sym.node)['model_bases'][ctx.cls.fullname] = 1 else: - if not ctx.api.final_iteration and not ctx.api.deferred: + if not ctx.api.final_iteration: ctx.api.defer() return diff --git a/mypy_django_plugin/transformers/fields.py b/mypy_django_plugin/transformers/fields.py index c711f91cb..d2082b54e 100644 --- a/mypy_django_plugin/transformers/fields.py +++ b/mypy_django_plugin/transformers/fields.py @@ -9,7 +9,7 @@ from mypy.types import TypeOfAny from mypy_django_plugin.django.context import DjangoContext -from mypy_django_plugin.lib import fullnames, helpers, chk_helpers +from mypy_django_plugin.lib import chk_helpers, fullnames, helpers def _get_current_field_from_assignment(ctx: FunctionContext, django_context: DjangoContext) -> Optional[Field]: diff --git a/mypy_django_plugin/transformers/forms.py b/mypy_django_plugin/transformers/forms.py index 26651f80d..6f3741ff5 100644 --- a/mypy_django_plugin/transformers/forms.py +++ b/mypy_django_plugin/transformers/forms.py @@ -5,7 +5,7 @@ from mypy.types import Type as MypyType from mypy.types import TypeType -from mypy_django_plugin.lib import sem_helpers, chk_helpers +from mypy_django_plugin.lib import chk_helpers, sem_helpers def make_meta_nested_class_inherit_from_any(ctx: ClassDefContext) -> None: diff --git a/mypy_django_plugin/transformers/managers.py b/mypy_django_plugin/transformers/managers.py index 57de98c4f..747e273a9 100644 --- a/mypy_django_plugin/transformers/managers.py +++ b/mypy_django_plugin/transformers/managers.py @@ -1,16 +1,16 @@ -from typing import Iterator, Tuple, Optional, Any, Dict +from typing import Any, Dict, Iterator, Optional, Tuple from mypy.nodes import ( - FuncDef, MemberExpr, NameExpr, RefExpr, StrExpr, TypeInfo, - PlaceholderNode, SymbolTableNode, GDEF, - CallExpr, Context, Decorator, OverloadedFuncDef, SymbolTable) + GDEF, CallExpr, Context, Decorator, FuncDef, MemberExpr, NameExpr, OverloadedFuncDef, PlaceholderNode, RefExpr, + StrExpr, SymbolTable, SymbolTableNode, TypeInfo, +) from mypy.plugin import ClassDefContext, DynamicClassDefContext, MethodContext -from mypy.semanal import SemanticAnalyzer, is_valid_replacement, is_same_symbol -from mypy.types import AnyType, Instance, TypeOfAny, CallableType +from mypy.semanal import SemanticAnalyzer, is_same_symbol, is_valid_replacement +from mypy.types import AnyType, CallableType, Instance from mypy.types import Type as MypyType -from mypy.typevars import fill_typevars +from mypy.types import TypeOfAny -from mypy_django_plugin.lib import fullnames, sem_helpers, helpers, chk_helpers +from mypy_django_plugin.lib import chk_helpers, fullnames, helpers, sem_helpers def iter_all_custom_queryset_methods(derived_queryset_info: TypeInfo) -> Iterator[Tuple[str, FuncDef]]: @@ -26,7 +26,7 @@ def generate_from_queryset_name(base_manager_info: TypeInfo, queryset_info: Type return base_manager_info.name + 'From' + queryset_info.name -def resolve_callee_info_or_exception(ctx: DynamicClassDefContext) -> Optional[TypeInfo]: +def resolve_callee_info_or_exception(ctx: DynamicClassDefContext) -> TypeInfo: callee = ctx.call.callee assert isinstance(callee, MemberExpr) assert isinstance(callee.expr, RefExpr) @@ -34,14 +34,14 @@ def resolve_callee_info_or_exception(ctx: DynamicClassDefContext) -> Optional[Ty callee_info = callee.expr.node if (callee_info is None or isinstance(callee_info, PlaceholderNode)): - raise sem_helpers.IncompleteDefnException(f'Definition of base manager {callee_info.fullname} ' + raise sem_helpers.IncompleteDefnException(f'Definition of base manager {callee.fullname!r} ' f'is incomplete.') assert isinstance(callee_info, TypeInfo) return callee_info -def resolve_passed_queryset_info_or_exception(ctx: DynamicClassDefContext) -> Optional[TypeInfo]: +def resolve_passed_queryset_info_or_exception(ctx: DynamicClassDefContext) -> TypeInfo: api = sem_helpers.get_semanal_api(ctx) passed_queryset_name_expr = ctx.call.args[0] @@ -51,13 +51,14 @@ def resolve_passed_queryset_info_or_exception(ctx: DynamicClassDefContext) -> Op if (sym is None or sym.node is None or isinstance(sym.node, PlaceholderNode)): - raise sem_helpers.BoundNameNotFound(passed_queryset_name_expr.fullname) + bound_name = passed_queryset_name_expr.fullname or passed_queryset_name_expr.name + raise sem_helpers.BoundNameNotFound(bound_name) assert isinstance(sym.node, TypeInfo) return sym.node -def resolve_django_manager_info_or_exception(ctx: DynamicClassDefContext) -> Optional[TypeInfo]: +def resolve_django_manager_info_or_exception(ctx: DynamicClassDefContext) -> TypeInfo: api = sem_helpers.get_semanal_api(ctx) sym = api.lookup_fully_qualified_or_none(fullnames.MANAGER_CLASS_FULLNAME) @@ -132,7 +133,7 @@ def create_new_manager_class_from_from_queryset_method(ctx: DynamicClassDefConte class_def_context = ClassDefContext(cls=new_manager_info.defn, reason=ctx.call, api=semanal_api) - self_type = fill_typevars(new_manager_info) + self_type = Instance(new_manager_info, [AnyType(TypeOfAny.explicit)]) try: for name, method_node in iter_all_custom_queryset_methods(queryset_info): @@ -219,7 +220,6 @@ def add_symbol_table_node(api: SemanticAnalyzer, return False - def create_manager_class_from_as_manager_method(ctx: DynamicClassDefContext) -> None: semanal_api = sem_helpers.get_semanal_api(ctx) try: @@ -232,7 +232,7 @@ def create_manager_class_from_as_manager_method(ctx: DynamicClassDefContext) -> else: raise - generic_param = AnyType(TypeOfAny.explicit) + generic_param: MypyType = AnyType(TypeOfAny.explicit) generic_param_name = 'Any' if (semanal_api.scope.classes and semanal_api.scope.classes[-1].has_base(fullnames.MODEL_CLASS_FULLNAME)): @@ -254,7 +254,7 @@ def create_manager_class_from_as_manager_method(ctx: DynamicClassDefContext) -> class_def_context = ClassDefContext(cls=new_manager_info.defn, reason=ctx.call, api=semanal_api) - self_type = fill_typevars(new_manager_info) + self_type = Instance(new_manager_info, [AnyType(TypeOfAny.explicit)]) try: for name, method_node in iter_all_custom_queryset_methods(queryset_info): @@ -298,16 +298,18 @@ def instantiate_anonymous_queryset_from_as_manager(ctx: MethodContext) -> MypyTy assert isinstance(ctx.type.ret_type, Instance) queryset_info = ctx.type.ret_type.type - fullname = get_generated_manager_fullname(ctx.context, - base_manager_info=django_manager_info, - queryset_info=queryset_info) + gen_name = django_manager_info.name + 'From' + queryset_info.name + gen_fullname = 'django.db.models.manager' + '.' + gen_name + metadata = get_generated_managers_metadata(django_manager_info) - if fullname not in metadata: - raise ValueError(f'{fullname!r} is not present in generated managers list') + if gen_fullname not in metadata: + raise ValueError(f'{gen_fullname!r} is not present in generated managers list') - module_name, _, class_name = metadata[fullname].rpartition('.') + module_name, _, class_name = metadata[gen_fullname].rpartition('.') current_module = helpers.get_current_module(api) assert module_name == current_module.fullname generated_manager_info = current_module.names[class_name].node + assert isinstance(generated_manager_info, TypeInfo) + return Instance(generated_manager_info, []) diff --git a/mypy_django_plugin/transformers/meta.py b/mypy_django_plugin/transformers/meta.py index cc69a7d44..0af47176b 100644 --- a/mypy_django_plugin/transformers/meta.py +++ b/mypy_django_plugin/transformers/meta.py @@ -5,7 +5,7 @@ from mypy.types import TypeOfAny from mypy_django_plugin.django.context import DjangoContext -from mypy_django_plugin.lib import helpers, chk_helpers +from mypy_django_plugin.lib import chk_helpers, helpers def _get_field_instance(ctx: MethodContext, field_fullname: str) -> MypyType: diff --git a/mypy_django_plugin/transformers/models.py b/mypy_django_plugin/transformers/models.py index ee7398af0..6e22aae97 100644 --- a/mypy_django_plugin/transformers/models.py +++ b/mypy_django_plugin/transformers/models.py @@ -1,11 +1,14 @@ from typing import List, Optional, Type, cast from django.db.models.base import Model +from django.db.models.fields import DateField, DateTimeField from django.db.models.fields.related import ForeignKey, OneToOneField from django.db.models.fields.reverse_related import ( ManyToManyRel, ManyToOneRel, OneToOneRel, ) -from mypy.nodes import ARG_STAR2, Argument, FuncDef, TypeInfo, Var, SymbolTableNode, MDEF, GDEF +from mypy.nodes import ( + ARG_STAR2, GDEF, MDEF, Argument, Context, FuncDef, SymbolTableNode, TypeInfo, Var, +) from mypy.plugin import ClassDefContext from mypy.plugins import common from mypy.semanal import SemanticAnalyzer, dummy_context @@ -13,7 +16,6 @@ from mypy.types import Type as MypyType from mypy.types import TypeOfAny -from django.db.models.fields import DateField, DateTimeField from mypy_django_plugin.django.context import DjangoContext from mypy_django_plugin.lib import fullnames, helpers, sem_helpers from mypy_django_plugin.transformers import fields @@ -71,47 +73,25 @@ def add_new_node_to_model_class(self, name: str, typ: MypyType, var.is_initialized_in_class = True sym = SymbolTableNode(MDEF, var, plugin_generated=True) - context = dummy_context() + context: Optional[Context] = dummy_context() if force_replace_existing: context = None self.api.add_symbol_table_node(name, sym, context=context) def add_new_class_for_current_module(self, name: str, bases: List[Instance], - force_replace_existing: bool = False) -> Optional[TypeInfo]: + force_replace_existing: bool = False) -> TypeInfo: current_module = self.api.cur_mod_node - if not force_replace_existing and name in current_module: + if not force_replace_existing and name in current_module.names: raise ValueError(f'Class {name!r} already defined for module {current_module.fullname!r}') new_typeinfo = helpers.new_typeinfo(name, bases=bases, module_name=current_module.fullname) - # sym = SymbolTableNode(GDEF, new_typeinfo, - # plugin_generated=True) - # context = dummy_context() - # if force_replace_existing: - # context = None - if name in current_module.names: del current_module.names[name] current_module.names[name] = SymbolTableNode(GDEF, new_typeinfo, plugin_generated=True) - # current_module.defs.append(new_typeinfo.defn) - # self.api.cur_mod_node. - # self.api.leave_class() - # added = self.api.add_symbol_table_node(name, sym, context=context) - # self.api.enter_class(self.model_classdef.info) - # - # self.api.cur_mod_node.defs.append(new_typeinfo.defn) - - # if not added and force_replace_existing: - # return None return new_typeinfo - # current_module = self.api.modules[self.model_classdef.info.module_name] - # context = - # new_class_info = helpers.add_new_class_for_module(current_module, - # name=name, bases=bases) - # return new_class_info - def run(self) -> None: model_cls = self.django_context.get_model_class_by_fullname(self.model_classdef.fullname) if model_cls is None: @@ -193,7 +173,7 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: related_model_cls = self.django_context.get_field_related_model_cls(field) if related_model_cls is None: - error_context = self.ctx.cls + error_context: Context = self.ctx.cls field_sym = self.ctx.cls.info.get(field.name) if field_sym is not None and field_sym.node is not None: error_context = field_sym.node @@ -383,13 +363,8 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: bases = [parametrized_related_manager_type, default_manager_type] new_related_manager_info = self.add_new_class_for_current_module(name, bases, force_replace_existing=True) - if new_related_manager_info is None: - # wasn't added for some reason, defer - if not self.api.final_iteration: - self.api.defer() - continue - - self.add_new_node_to_model_class(related_manager_attr_name, Instance(new_related_manager_info, [])) + self.add_new_node_to_model_class(related_manager_attr_name, + Instance(new_related_manager_info, [])) class AddExtraFieldMethods(ModelClassInitializer): diff --git a/mypy_django_plugin/transformers/orm_lookups.py b/mypy_django_plugin/transformers/orm_lookups.py index 5e5d2804a..1dacdc874 100644 --- a/mypy_django_plugin/transformers/orm_lookups.py +++ b/mypy_django_plugin/transformers/orm_lookups.py @@ -4,7 +4,7 @@ from mypy.types import TypeOfAny from mypy_django_plugin.django.context import DjangoContext -from mypy_django_plugin.lib import fullnames, helpers, chk_helpers +from mypy_django_plugin.lib import chk_helpers, fullnames, helpers def typecheck_queryset_filter(ctx: MethodContext, django_context: DjangoContext) -> MypyType: diff --git a/mypy_django_plugin/transformers/querysets.py b/mypy_django_plugin/transformers/querysets.py index 0d27dfcab..1476e105e 100644 --- a/mypy_django_plugin/transformers/querysets.py +++ b/mypy_django_plugin/transformers/querysets.py @@ -14,7 +14,7 @@ from mypy_django_plugin.django.context import ( DjangoContext, LookupsAreUnsupported, ) -from mypy_django_plugin.lib import fullnames, helpers, chk_helpers +from mypy_django_plugin.lib import chk_helpers, fullnames, helpers def _extract_model_type_from_queryset(queryset_type: Instance) -> Optional[Instance]: diff --git a/mypy_django_plugin/transformers/request.py b/mypy_django_plugin/transformers/request.py index 55cb6b15b..76575ae74 100644 --- a/mypy_django_plugin/transformers/request.py +++ b/mypy_django_plugin/transformers/request.py @@ -4,7 +4,7 @@ from mypy.types import UnionType from mypy_django_plugin.django.context import DjangoContext -from mypy_django_plugin.lib import helpers, chk_helpers +from mypy_django_plugin.lib import chk_helpers, helpers def set_auth_user_model_as_type_for_request_user(ctx: AttributeContext, django_context: DjangoContext) -> MypyType: diff --git a/mypy_django_plugin/transformers/settings.py b/mypy_django_plugin/transformers/settings.py index 67022abcf..1b5972485 100644 --- a/mypy_django_plugin/transformers/settings.py +++ b/mypy_django_plugin/transformers/settings.py @@ -5,7 +5,7 @@ from mypy.types import TypeOfAny, TypeType from mypy_django_plugin.django.context import DjangoContext -from mypy_django_plugin.lib import helpers, chk_helpers +from mypy_django_plugin.lib import chk_helpers, helpers def get_user_model_hook(ctx: FunctionContext, django_context: DjangoContext) -> MypyType: From 4bf5ec940a6a4b0241a9b967b0d54727fefa32e7 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sun, 5 Jan 2020 08:18:43 +0300 Subject: [PATCH 05/76] fix tests --- mypy_django_plugin/lib/helpers.py | 42 ++++++++++----------- mypy_django_plugin/lib/sem_helpers.py | 16 ++++---- mypy_django_plugin/transformers/managers.py | 10 ++--- 3 files changed, 33 insertions(+), 35 deletions(-) diff --git a/mypy_django_plugin/lib/helpers.py b/mypy_django_plugin/lib/helpers.py index f743e074f..e4ca1acfe 100644 --- a/mypy_django_plugin/lib/helpers.py +++ b/mypy_django_plugin/lib/helpers.py @@ -1,5 +1,5 @@ from typing import ( - TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Union, + TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union, ) from django.db.models.fields import Field @@ -8,7 +8,7 @@ from mypy.checker import TypeChecker from mypy.mro import calculate_mro from mypy.nodes import ( - Block, ClassDef, Expression, MemberExpr, MypyFile, NameExpr, StrExpr, SymbolNode, SymbolTable, SymbolTableNode, + Block, ClassDef, Expression, MemberExpr, MypyFile, NameExpr, StrExpr, SymbolTable, SymbolTableNode, TypeInfo, Var, ) from mypy.semanal import SemanticAnalyzer @@ -28,23 +28,32 @@ def get_django_metadata(model_info: TypeInfo) -> Dict[str, Any]: return model_info.metadata.setdefault('django', {}) -def lookup_fully_qualified_sym(fullname: str, all_modules: Dict[str, MypyFile]) -> Optional[SymbolTableNode]: +def split_symbol_name(fullname: str, all_modules: Dict[str, MypyFile]) -> Optional[Tuple[str, str]]: if '.' not in fullname: return None - module_file = None + module_name = None parts = fullname.split('.') for i in range(len(parts), 0, -1): possible_module_name = '.'.join(parts[:i]) if possible_module_name in all_modules: - module_file = all_modules[possible_module_name] + module_name = possible_module_name break - if module_file is None: + if module_name is None: return None - cls_name = fullname.replace(module_file.fullname, '').lstrip('.') - sym_table = module_file.names + cls_name = fullname.replace(module_name, '').lstrip('.') + return module_name, cls_name + + +def lookup_fully_qualified_typeinfo(api: AnyPluginAPI, fullname: str) -> Optional[TypeInfo]: + split = split_symbol_name(fullname, api.modules) + if split is None: + return None + module_name, cls_name = split + + sym_table = api.modules[module_name].names # type: Dict[str, SymbolTableNode] if '.' in cls_name: parent_cls_name, _, cls_name = cls_name.rpartition('.') # nested class @@ -55,23 +64,14 @@ def lookup_fully_qualified_sym(fullname: str, all_modules: Dict[str, MypyFile]) return None sym_table = sym.node.names - return sym_table.get(cls_name) - - -def lookup_fully_qualified_generic(name: str, all_modules: Dict[str, MypyFile]) -> Optional[SymbolNode]: - sym = lookup_fully_qualified_sym(name, all_modules) - if sym is None: + sym = sym_table.get(cls_name) + if (sym is None + or sym.node is None + or not isinstance(sym.node, TypeInfo)): return None return sym.node -def lookup_fully_qualified_typeinfo(api: AnyPluginAPI, fullname: str) -> Optional[TypeInfo]: - node = lookup_fully_qualified_generic(fullname, api.modules) - if not isinstance(node, TypeInfo): - return None - return node - - def lookup_class_typeinfo(api: AnyPluginAPI, klass: type) -> Optional[TypeInfo]: fullname = get_class_fullname(klass) field_info = lookup_fully_qualified_typeinfo(api, fullname) diff --git a/mypy_django_plugin/lib/sem_helpers.py b/mypy_django_plugin/lib/sem_helpers.py index 33db42e03..139d604b9 100644 --- a/mypy_django_plugin/lib/sem_helpers.py +++ b/mypy_django_plugin/lib/sem_helpers.py @@ -4,9 +4,9 @@ from mypy.plugin import ClassDefContext, DynamicClassDefContext from mypy.plugins.common import add_method from mypy.semanal import SemanticAnalyzer -from mypy.types import AnyType, CallableType, Instance +from mypy.types import AnyType, CallableType, Instance, PlaceholderType from mypy.types import Type as MypyType -from mypy.types import TypeOfAny +from mypy.types import TypeOfAny, get_proper_type class IncompleteDefnException(Exception): @@ -54,8 +54,9 @@ def analyze_callable_signature(api: SemanticAnalyzer, method_node: FuncDef) -> S for arg_name, arg_type, original_argument in zip(method_type.arg_names[1:], method_type.arg_types[1:], method_node.arguments[1:]): - analyzed_arg_type = api.anal_type(arg_type) - if analyzed_arg_type is None: + analyzed_arg_type = api.anal_type(get_proper_type(arg_type), allow_placeholder=True) + assert analyzed_arg_type is not None + if isinstance(analyzed_arg_type, PlaceholderType): unbound = True var = Var(name=original_argument.variable.name, @@ -63,14 +64,15 @@ def analyze_callable_signature(api: SemanticAnalyzer, method_node: FuncDef) -> S var.set_line(original_argument.variable) argument = Argument(variable=var, - type_annotation=arg_type, + type_annotation=analyzed_arg_type, initializer=original_argument.initializer, kind=original_argument.kind) argument.set_line(original_argument) arguments.append(argument) - analyzed_ret_type = api.anal_type(method_type.ret_type) - if analyzed_ret_type is None: + analyzed_ret_type = api.anal_type(get_proper_type(method_type.ret_type), allow_placeholder=True) + assert analyzed_ret_type is not None + if isinstance(analyzed_ret_type, PlaceholderType): unbound = True return SignatureTuple(arguments, analyzed_ret_type, unbound) diff --git a/mypy_django_plugin/transformers/managers.py b/mypy_django_plugin/transformers/managers.py index 747e273a9..78d223088 100644 --- a/mypy_django_plugin/transformers/managers.py +++ b/mypy_django_plugin/transformers/managers.py @@ -60,15 +60,11 @@ def resolve_passed_queryset_info_or_exception(ctx: DynamicClassDefContext) -> Ty def resolve_django_manager_info_or_exception(ctx: DynamicClassDefContext) -> TypeInfo: api = sem_helpers.get_semanal_api(ctx) - - sym = api.lookup_fully_qualified_or_none(fullnames.MANAGER_CLASS_FULLNAME) - if (sym is None - or sym.node is None - or isinstance(sym.node, PlaceholderNode)): + info = helpers.lookup_fully_qualified_typeinfo(api, fullnames.MANAGER_CLASS_FULLNAME) + if info is None: raise sem_helpers.BoundNameNotFound(fullnames.MANAGER_CLASS_FULLNAME) - assert isinstance(sym.node, TypeInfo) - return sym.node + return info def new_manager_typeinfo(ctx: DynamicClassDefContext, callee_manager_info: TypeInfo) -> TypeInfo: From 585273ac7f18729b0d12f5fb1a5efa6868f1b8fa Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sun, 5 Jan 2020 17:55:16 +0300 Subject: [PATCH 06/76] add two new as_manager tests --- .../managers/querysets/test_as_manager.yml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/test-data/typecheck/managers/querysets/test_as_manager.yml b/test-data/typecheck/managers/querysets/test_as_manager.yml index e08c02168..fe2c20dc8 100644 --- a/test-data/typecheck/managers/querysets/test_as_manager.yml +++ b/test-data/typecheck/managers/querysets/test_as_manager.yml @@ -20,6 +20,34 @@ objects = MyQuerySet.as_manager() +- case: two_invocations_parametrized_with_different_models + main: | + from myapp.models import User, Blog + reveal_type(User.objects) # N: Revealed type is 'myapp.models.MyQuerySet_AsManager_User' + reveal_type(User.objects.get()) # N: Revealed type is 'myapp.models.User*' + reveal_type(User.objects.queryset_method) # N: Revealed type is 'def () -> builtins.int' + reveal_type(User.objects.queryset_method()) # N: Revealed type is 'builtins.int' + + reveal_type(Blog.objects) # N: Revealed type is 'myapp.models.MyQuerySet_AsManager_Blog' + reveal_type(Blog.objects.get()) # N: Revealed type is 'myapp.models.Blog*' + reveal_type(Blog.objects.queryset_method) # N: Revealed type is 'def () -> builtins.int' + reveal_type(Blog.objects.queryset_method()) # N: Revealed type is 'builtins.int' + installed_apps: + - myapp + files: + - path: myapp/__init__.py + - path: myapp/models.py + content: | + from django.db import models + class MyQuerySet(models.QuerySet): + def queryset_method(self) -> int: + pass + class User(models.Model): + objects = MyQuerySet.as_manager() + class Blog(models.Model): + objects = MyQuerySet.as_manager() + + - case: as_manager_outside_model_parametrized_with_any main: | from myapp.models import NotModel, outside_objects @@ -40,3 +68,28 @@ outside_objects = MyQuerySet.as_manager() class NotModel: objects = MyQuerySet.as_manager() + +- case: test_as_manager_without_name_to_bind_in_different_files + main: | + from myapp.models import MyQuerySet + reveal_type(MyQuerySet.as_manager()) # N: Revealed type is 'Any' + reveal_type(MyQuerySet.as_manager().get()) # N: Revealed type is 'Any' + reveal_type(MyQuerySet.as_manager().mymethod()) # N: Revealed type is 'Any' + + from myapp import helpers + installed_apps: + - myapp + files: + - path: myapp/__init__.py + - path: myapp/models.py + content: | + from django.db import models + class MyQuerySet(models.QuerySet): + def mymethod(self) -> int: + pass + class MyModel(models.Model): + objects = MyQuerySet.as_manager() + - path: myapp/helpers.py + content: | + from myapp.models import MyQuerySet + MyQuerySet.as_manager() \ No newline at end of file From c0f41f38c72f009aea65ce84e1d6fdf6f3299bf2 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sun, 2 Feb 2020 03:12:32 +0300 Subject: [PATCH 07/76] wip --- mypy_django_plugin/django/context.py | 8 +- mypy_django_plugin/lib/generics.py | 6 + mypy_django_plugin/lib/helpers.py | 204 ++++++++++++++- mypy_django_plugin/lib/sem_helpers.py | 8 +- mypy_django_plugin/main.py | 17 +- mypy_django_plugin/py.typed | 0 mypy_django_plugin/transformers/fields.py | 5 + mypy_django_plugin/transformers/managers.py | 56 +++-- mypy_django_plugin/transformers/models.py | 101 ++------ mypy_django_plugin/transformers2/__init__.py | 0 .../transformers2/dynamic_managers.py | 79 ++++++ mypy_django_plugin/transformers2/models.py | 236 ++++++++++++++++++ .../transformers2/new_helpers.py | 26 ++ test-data/typecheck/fields/test_related.yml | 9 +- .../managers/querysets/test_from_queryset.yml | 119 ++++++--- .../managers/querysets/test_values_list.yml | 3 + .../typecheck/managers/test_managers.yml | 193 +++++--------- 17 files changed, 778 insertions(+), 292 deletions(-) create mode 100644 mypy_django_plugin/lib/generics.py create mode 100644 mypy_django_plugin/py.typed create mode 100644 mypy_django_plugin/transformers2/__init__.py create mode 100644 mypy_django_plugin/transformers2/dynamic_managers.py create mode 100644 mypy_django_plugin/transformers2/models.py create mode 100644 mypy_django_plugin/transformers2/new_helpers.py diff --git a/mypy_django_plugin/django/context.py b/mypy_django_plugin/django/context.py index f9dd152b7..faa56b918 100644 --- a/mypy_django_plugin/django/context.py +++ b/mypy_django_plugin/django/context.py @@ -55,10 +55,10 @@ def initialize_django(settings_module: str) -> Tuple['Apps', 'LazySettings']: def noop_class_getitem(cls, key): return cls - from django.db import models + # from django.db import models - models.QuerySet.__class_getitem__ = classmethod(noop_class_getitem) # type: ignore - models.Manager.__class_getitem__ = classmethod(noop_class_getitem) # type: ignore + # models.QuerySet.__class_getitem__ = classmethod(noop_class_getitem) # type: ignore + # models.Manager.__class_getitem__ = classmethod(noop_class_getitem) # type: ignore from django.conf import settings from django.apps import apps @@ -226,7 +226,7 @@ def get_attname(self, field: Field) -> str: attname = field.attname return attname - def get_field_nullability(self, field: Union[Field, ForeignObjectRel], method: Optional[str]) -> bool: + def get_field_nullability(self, field: Union[Field, ForeignObjectRel], method: Optional[str] = None) -> bool: nullable = field.null if not nullable and isinstance(field, CharField) and field.blank: return True diff --git a/mypy_django_plugin/lib/generics.py b/mypy_django_plugin/lib/generics.py new file mode 100644 index 000000000..434a17c50 --- /dev/null +++ b/mypy_django_plugin/lib/generics.py @@ -0,0 +1,6 @@ +def make_classes_generic(*klasses: type) -> None: + for klass in klasses: + def fake_classgetitem(cls, *args, **kwargs): + return cls + + klass.__class_getitem__ = classmethod(fake_classgetitem) # type: ignore diff --git a/mypy_django_plugin/lib/helpers.py b/mypy_django_plugin/lib/helpers.py index e4ca1acfe..cc68aa465 100644 --- a/mypy_django_plugin/lib/helpers.py +++ b/mypy_django_plugin/lib/helpers.py @@ -1,8 +1,8 @@ +from abc import abstractmethod from typing import ( TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union, -) + cast) -from django.db.models.fields import Field from django.db.models.fields.related import RelatedField from django.db.models.fields.reverse_related import ForeignObjectRel from mypy.checker import TypeChecker @@ -10,20 +10,210 @@ from mypy.nodes import ( Block, ClassDef, Expression, MemberExpr, MypyFile, NameExpr, StrExpr, SymbolTable, SymbolTableNode, TypeInfo, Var, -) + CallExpr, Context, PlaceholderNode, FuncDef, FakeInfo) +from mypy.plugin import DynamicClassDefContext, ClassDefContext +from mypy.plugins.common import add_method from mypy.semanal import SemanticAnalyzer -from mypy.types import AnyType, Instance, NoneTyp +from mypy.types import AnyType, Instance, NoneTyp, TypeType from mypy.types import Type as MypyType from mypy.types import TypeOfAny, UnionType +from mypy.typetraverser import TypeTraverserVisitor +from django.db.models.fields import Field from mypy_django_plugin.lib import fullnames +from mypy_django_plugin.lib.sem_helpers import prepare_unannotated_method_signature, analyze_callable_signature +from mypy_django_plugin.transformers2 import new_helpers if TYPE_CHECKING: from mypy_django_plugin.django.context import DjangoContext + from mypy_django_plugin.main import NewSemanalDjangoPlugin AnyPluginAPI = Union[TypeChecker, SemanticAnalyzer] +class DjangoPluginCallback: + django_context: 'DjangoContext' + + def __init__(self, plugin: 'NewSemanalDjangoPlugin') -> None: + self.plugin = plugin + self.django_context = plugin.django_context + + # def lookup_fully_qualified(self, fullname: str) -> Optional[SymbolTableNode]: + # return self.plugin.lookup_fully_qualified(fullname) + + +class SemanalPluginCallback(DjangoPluginCallback): + semanal_api: SemanticAnalyzer + + def build_defer_error_message(self, message: str) -> str: + return f'{self.__class__.__name__}: {message}' + + def defer_till_next_iteration(self, deferral_context: Optional[Context] = None, + *, + reason: Optional[str] = None) -> bool: + """ Returns False if cannot be deferred. """ + if self.semanal_api.final_iteration: + return False + self.semanal_api.defer(deferral_context) + print(f'LOG: defer: {self.build_defer_error_message(reason)}') + return True + + def lookup_typeinfo_or_defer(self, fullname: str, *, + deferral_context: Optional[Context] = None, + reason_for_defer: Optional[str] = None) -> Optional[TypeInfo]: + sym = self.plugin.lookup_fully_qualified(fullname) + if sym is None or sym.node is None or isinstance(sym.node, PlaceholderNode): + deferral_context = deferral_context or self.semanal_api.cur_mod_node + reason = reason_for_defer or f'{fullname!r} is not available for lookup' + if not self.defer_till_next_iteration(deferral_context, reason=reason): + raise new_helpers.TypeInfoNotFound(fullname) + return None + + if not isinstance(sym.node, TypeInfo): + raise ValueError(f'{fullname!r} does not correspond to TypeInfo') + + return sym.node + + def new_typeinfo(self, name: str, bases: List[Instance]) -> TypeInfo: + class_def = ClassDef(name, Block([])) + class_def.fullname = self.semanal_api.qualified_name(name) + + info = TypeInfo(SymbolTable(), class_def, self.semanal_api.cur_mod_id) + info.bases = bases + calculate_mro(info) + info.metaclass_type = info.calculate_metaclass_type() + + class_def.info = info + return info + + # def add_symbol_table_node_or_defer(self, name: str, sym: SymbolTableNode) -> bool: + # return self.semanal_api.add_symbol_table_node(name, sym, + # context=self.semanal_api.cur_mod_node) + + def add_method_from_signature(self, + signature_node: FuncDef, + new_method_name: str, + new_self_type: Instance, + class_defn: ClassDef) -> bool: + if signature_node.type is None: + if self.defer_till_next_iteration(reason=signature_node.fullname): + return False + + arguments, return_type = prepare_unannotated_method_signature(signature_node) + ctx = ClassDefContext(class_defn, signature_node, self.semanal_api) + add_method(ctx, + new_method_name, + self_type=new_self_type, + args=arguments, + return_type=return_type) + return True + + # add imported objects from method signature to the current module, if not present + source_symbols = self.semanal_api.modules[signature_node.info.module_name].names + currently_imported_symbols = self.semanal_api.cur_mod_node.names + + def import_symbol_from_source(name: str) -> None: + if name in source_symbols['__builtins__'].node.names: + return + sym = source_symbols[name].copy() + self.semanal_api.add_imported_symbol(name, sym, context=self.semanal_api.cur_mod_node) + + class UnimportedTypesVisitor(TypeTraverserVisitor): + def visit_union_type(self, t: UnionType) -> None: + super().visit_union_type(t) + union_sym = currently_imported_symbols.get('Union') + if union_sym is None: + # TODO: check if it's exactly typing.Union + import_symbol_from_source('Union') + + def visit_type_type(self, t: TypeType) -> None: + super().visit_type_type(t) + type_sym = currently_imported_symbols.get('Union') + if type_sym is None: + # TODO: check if it's exactly typing.Type + import_symbol_from_source('Type') + + def visit_instance(self, t: Instance) -> None: + super().visit_instance(t) + if isinstance(t.type, FakeInfo): + return + type_name = t.type.name + sym = currently_imported_symbols.get(type_name) + if sym is None: + # TODO: check if it's exactly typing.Type + import_symbol_from_source(type_name) + + signature_node.type.accept(UnimportedTypesVisitor()) + + # # copy global SymbolTableNode objects from original class to the current node, if not present + # original_module = semanal_api.modules[method_node.info.module_name] + # for name, sym in original_module.names.items(): + # if (not sym.plugin_generated + # and name not in semanal_api.cur_mod_node.names): + # semanal_api.add_imported_symbol(name, sym, context=semanal_api.cur_mod_node) + + arguments, analyzed_return_type, unbound = analyze_callable_signature(self.semanal_api, signature_node) + if unbound: + raise new_helpers.IncompleteDefnError(f'Signature of method {signature_node.fullname!r} is not ready') + + assert len(arguments) + 1 == len(signature_node.arguments) + assert analyzed_return_type is not None + + ctx = ClassDefContext(class_defn, signature_node, self.semanal_api) + add_method(ctx, + new_method_name, + self_type=new_self_type, + args=arguments, + return_type=analyzed_return_type) + return True + + +class DynamicClassPluginCallback(SemanalPluginCallback): + class_name: str + call_expr: CallExpr + + def __call__(self, ctx: DynamicClassDefContext) -> None: + self.class_name = ctx.name + self.call_expr = ctx.call + self.semanal_api = cast(SemanticAnalyzer, ctx.api) + self.create_new_dynamic_class() + + def get_callee(self) -> MemberExpr: + callee = self.call_expr.callee + assert isinstance(callee, MemberExpr) + return callee + + def lookup_same_module_or_defer(self, name: str, *, + deferral_context: Optional[Context] = None) -> Optional[SymbolTableNode]: + sym = self.semanal_api.lookup_qualified(name, self.call_expr) + if sym is None or sym.node is None or isinstance(sym.node, PlaceholderNode): + deferral_context = deferral_context or self.call_expr + if not self.defer_till_next_iteration(deferral_context, + reason=f'{self.semanal_api.cur_mod_id}.{name} does not exist'): + raise new_helpers.NameNotFound(name) + return None + return sym + + @abstractmethod + def create_new_dynamic_class(self) -> None: + raise NotImplementedError + + +class ClassDefPluginCallback(SemanalPluginCallback): + reason: Expression + class_defn: ClassDef + + def __call__(self, ctx: ClassDefContext) -> None: + self.reason = ctx.reason + self.class_defn = ctx.cls + self.semanal_api = cast(SemanticAnalyzer, ctx.api) + self.modify_class_defn() + + @abstractmethod + def modify_class_defn(self) -> None: + raise NotImplementedError + + def get_django_metadata(model_info: TypeInfo) -> Dict[str, Any]: return model_info.metadata.setdefault('django', {}) @@ -31,7 +221,6 @@ def get_django_metadata(model_info: TypeInfo) -> Dict[str, Any]: def split_symbol_name(fullname: str, all_modules: Dict[str, MypyFile]) -> Optional[Tuple[str, str]]: if '.' not in fullname: return None - module_name = None parts = fullname.split('.') for i in range(len(parts), 0, -1): @@ -39,12 +228,11 @@ def split_symbol_name(fullname: str, all_modules: Dict[str, MypyFile]) -> Option if possible_module_name in all_modules: module_name = possible_module_name break - if module_name is None: return None - cls_name = fullname.replace(module_name, '').lstrip('.') - return module_name, cls_name + symbol_name = fullname.replace(module_name, '').lstrip('.') + return module_name, symbol_name def lookup_fully_qualified_typeinfo(api: AnyPluginAPI, fullname: str) -> Optional[TypeInfo]: diff --git a/mypy_django_plugin/lib/sem_helpers.py b/mypy_django_plugin/lib/sem_helpers.py index 139d604b9..720557ae8 100644 --- a/mypy_django_plugin/lib/sem_helpers.py +++ b/mypy_django_plugin/lib/sem_helpers.py @@ -9,12 +9,12 @@ from mypy.types import TypeOfAny, get_proper_type -class IncompleteDefnException(Exception): +class IncompleteDefnError(Exception): def __init__(self, error_message: str = '') -> None: super().__init__(error_message) -class BoundNameNotFound(IncompleteDefnException): +class BoundNameNotFound(IncompleteDefnError): def __init__(self, fullname: str) -> None: super().__init__(f'No {fullname!r} found') @@ -85,7 +85,7 @@ def copy_method_or_incomplete_defn_exception(ctx: ClassDefContext, if method_node.type is None: if not semanal_api.final_iteration: - raise IncompleteDefnException(f'Unannotated method {method_node.fullname!r}') + raise IncompleteDefnError(f'Unannotated method {method_node.fullname!r}') arguments, return_type = prepare_unannotated_method_signature(method_node) add_method(ctx, @@ -107,7 +107,7 @@ def copy_method_or_incomplete_defn_exception(ctx: ClassDefContext, arguments, analyzed_return_type, unbound = analyze_callable_signature(semanal_api, method_node) assert len(arguments) + 1 == len(method_node.arguments) if unbound: - raise IncompleteDefnException(f'Signature of method {method_node.fullname!r} is not ready') + raise IncompleteDefnError(f'Signature of method {method_node.fullname!r} is not ready') assert analyzed_return_type is not None diff --git a/mypy_django_plugin/main.py b/mypy_django_plugin/main.py index 39a3f3e4e..ca9c93007 100644 --- a/mypy_django_plugin/main.py +++ b/mypy_django_plugin/main.py @@ -17,10 +17,10 @@ fields, forms, init_create, meta, querysets, request, settings, ) from mypy_django_plugin.transformers.managers import ( - create_manager_class_from_as_manager_method, create_new_manager_class_from_from_queryset_method, - instantiate_anonymous_queryset_from_as_manager, -) + create_manager_class_from_as_manager_method, instantiate_anonymous_queryset_from_as_manager) from mypy_django_plugin.transformers.models import process_model_class +from mypy_django_plugin.transformers2.dynamic_managers import CreateNewManagerClassFrom_FromQuerySet +from mypy_django_plugin.transformers2.models import ModelCallback def transform_model_class(ctx: ClassDefContext, @@ -190,9 +190,9 @@ def get_function_hook(self, fullname: str if fullname == 'django.contrib.auth.get_user_model': return partial(settings.get_user_model_hook, django_context=self.django_context) - manager_bases = self._get_current_manager_bases() - if fullname in manager_bases: - return querysets.determine_proper_manager_type + # manager_bases = self._get_current_manager_bases() + # if fullname in manager_bases: + # return querysets.determine_proper_manager_type info = self._get_typeinfo_or_none(fullname) if info: @@ -248,7 +248,7 @@ def get_base_class_hook(self, fullname: str ) -> Optional[Callable[[ClassDefContext], None]]: if (fullname in self.django_context.all_registered_model_class_fullnames or fullname in self._get_current_model_bases()): - return partial(transform_model_class, django_context=self.django_context) + return ModelCallback(self) if fullname in self._get_current_manager_bases(): return add_new_manager_base @@ -275,7 +275,8 @@ def get_dynamic_class_hook(self, fullname: str class_name, _, _ = fullname.rpartition('.') info = self._get_typeinfo_or_none(class_name) if info and info.has_base(fullnames.BASE_MANAGER_CLASS_FULLNAME): - return create_new_manager_class_from_from_queryset_method + return CreateNewManagerClassFrom_FromQuerySet(self) + if fullname.endswith('as_manager'): class_name, _, _ = fullname.rpartition('.') info = self._get_typeinfo_or_none(class_name) diff --git a/mypy_django_plugin/py.typed b/mypy_django_plugin/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/mypy_django_plugin/transformers/fields.py b/mypy_django_plugin/transformers/fields.py index d2082b54e..d0bff20ef 100644 --- a/mypy_django_plugin/transformers/fields.py +++ b/mypy_django_plugin/transformers/fields.py @@ -110,6 +110,11 @@ def get_field_descriptor_types(field_info: TypeInfo, is_nullable: bool) -> Tuple return set_type, get_type +def get_field_type(field_info: TypeInfo, is_nullable: bool) -> Instance: + set_type, get_type = get_field_descriptor_types(field_info, is_nullable) + return Instance(field_info, [set_type, get_type]) + + def set_descriptor_types_for_field(ctx: FunctionContext) -> Instance: default_return_type = cast(Instance, ctx.default_return_type) diff --git a/mypy_django_plugin/transformers/managers.py b/mypy_django_plugin/transformers/managers.py index 78d223088..9bb7cea69 100644 --- a/mypy_django_plugin/transformers/managers.py +++ b/mypy_django_plugin/transformers/managers.py @@ -1,12 +1,13 @@ from typing import Any, Dict, Iterator, Optional, Tuple +from mypy.checker import gen_unique_name from mypy.nodes import ( GDEF, CallExpr, Context, Decorator, FuncDef, MemberExpr, NameExpr, OverloadedFuncDef, PlaceholderNode, RefExpr, StrExpr, SymbolTable, SymbolTableNode, TypeInfo, -) + MypyFile) from mypy.plugin import ClassDefContext, DynamicClassDefContext, MethodContext from mypy.semanal import SemanticAnalyzer, is_same_symbol, is_valid_replacement -from mypy.types import AnyType, CallableType, Instance +from mypy.types import AnyType, CallableType, Instance, TypeVarType, TypeVarDef from mypy.types import Type as MypyType from mypy.types import TypeOfAny @@ -26,28 +27,43 @@ def generate_from_queryset_name(base_manager_info: TypeInfo, queryset_info: Type return base_manager_info.name + 'From' + queryset_info.name -def resolve_callee_info_or_exception(ctx: DynamicClassDefContext) -> TypeInfo: - callee = ctx.call.callee +# +# def cb_resolve_callee_info_or_exception(cb: ) -> TypeInfo: +# callee = ctx.call.callee +# assert isinstance(callee, MemberExpr) +# assert isinstance(callee.expr, RefExpr) +# +# callee_info = callee.expr.node +# if (callee_info is None +# or isinstance(callee_info, PlaceholderNode)): +# raise sem_helpers.IncompleteDefnError(f'Definition of base manager {callee.fullname!r} ' +# f'is incomplete.') +# +# assert isinstance(callee_info, TypeInfo) +# return callee_info + + +def resolve_callee_info_or_exception(callback: helpers.DynamicClassPluginCallback) -> TypeInfo: + callee = callback.call_expr.callee assert isinstance(callee, MemberExpr) assert isinstance(callee.expr, RefExpr) callee_info = callee.expr.node if (callee_info is None or isinstance(callee_info, PlaceholderNode)): - raise sem_helpers.IncompleteDefnException(f'Definition of base manager {callee.fullname!r} ' - f'is incomplete.') + raise sem_helpers.IncompleteDefnError(f'Definition of base manager {callee.fullname!r} ' + f'is incomplete.') assert isinstance(callee_info, TypeInfo) return callee_info -def resolve_passed_queryset_info_or_exception(ctx: DynamicClassDefContext) -> TypeInfo: - api = sem_helpers.get_semanal_api(ctx) - - passed_queryset_name_expr = ctx.call.args[0] +def resolve_passed_queryset_info_or_exception(callback: helpers.DynamicClassPluginCallback) -> TypeInfo: + passed_queryset_name_expr = callback.call_expr.args[0] assert isinstance(passed_queryset_name_expr, NameExpr) - sym = api.lookup_qualified(passed_queryset_name_expr.name, ctx=ctx.call) + # lookup in the same module + sym = callback.semanal_api.lookup_qualified(passed_queryset_name_expr.name, ctx=callback.call_expr) if (sym is None or sym.node is None or isinstance(sym.node, PlaceholderNode)): @@ -58,9 +74,8 @@ def resolve_passed_queryset_info_or_exception(ctx: DynamicClassDefContext) -> Ty return sym.node -def resolve_django_manager_info_or_exception(ctx: DynamicClassDefContext) -> TypeInfo: - api = sem_helpers.get_semanal_api(ctx) - info = helpers.lookup_fully_qualified_typeinfo(api, fullnames.MANAGER_CLASS_FULLNAME) +def resolve_django_manager_info_or_exception(callback: helpers.DynamicClassPluginCallback) -> TypeInfo: + info = callback.lookup_typeinfo_or_defer(fullnames.MANAGER_CLASS_FULLNAME) if info is None: raise sem_helpers.BoundNameNotFound(fullnames.MANAGER_CLASS_FULLNAME) @@ -113,7 +128,7 @@ def create_new_manager_class_from_from_queryset_method(ctx: DynamicClassDefConte callee_manager_info = resolve_callee_info_or_exception(ctx) queryset_info = resolve_passed_queryset_info_or_exception(ctx) django_manager_info = resolve_django_manager_info_or_exception(ctx) - except sem_helpers.IncompleteDefnException: + except sem_helpers.IncompleteDefnError: if not semanal_api.final_iteration: semanal_api.defer() return @@ -137,7 +152,7 @@ def create_new_manager_class_from_from_queryset_method(ctx: DynamicClassDefConte self_type, new_method_name=name, method_node=method_node) - except sem_helpers.IncompleteDefnException: + except sem_helpers.IncompleteDefnError: if not semanal_api.final_iteration: semanal_api.defer() return @@ -216,12 +231,17 @@ def add_symbol_table_node(api: SemanticAnalyzer, return False +class CreateNewManagerClassFrom_AsManager(helpers.DynamicClassPluginCallback): + def create_new_dynamic_class(self) -> None: + pass + + def create_manager_class_from_as_manager_method(ctx: DynamicClassDefContext) -> None: semanal_api = sem_helpers.get_semanal_api(ctx) try: queryset_info = resolve_callee_info_or_exception(ctx) django_manager_info = resolve_django_manager_info_or_exception(ctx) - except sem_helpers.IncompleteDefnException: + except sem_helpers.IncompleteDefnError: if not semanal_api.final_iteration: semanal_api.defer() return @@ -258,7 +278,7 @@ def create_manager_class_from_as_manager_method(ctx: DynamicClassDefContext) -> self_type, new_method_name=name, method_node=method_node) - except sem_helpers.IncompleteDefnException: + except sem_helpers.IncompleteDefnError: if not semanal_api.final_iteration: semanal_api.defer() return diff --git a/mypy_django_plugin/transformers/models.py b/mypy_django_plugin/transformers/models.py index 6e22aae97..c1d0e85a8 100644 --- a/mypy_django_plugin/transformers/models.py +++ b/mypy_django_plugin/transformers/models.py @@ -7,7 +7,7 @@ ManyToManyRel, ManyToOneRel, OneToOneRel, ) from mypy.nodes import ( - ARG_STAR2, GDEF, MDEF, Argument, Context, FuncDef, SymbolTableNode, TypeInfo, Var, + ARG_STAR2, GDEF, MDEF, Argument, Context, SymbolTableNode, TypeInfo, Var, ) from mypy.plugin import ClassDefContext from mypy.plugins import common @@ -34,15 +34,15 @@ def __init__(self, ctx: ClassDefContext, django_context: DjangoContext): def lookup_typeinfo(self, fullname: str) -> Optional[TypeInfo]: return helpers.lookup_fully_qualified_typeinfo(self.api, fullname) - def lookup_typeinfo_or_incomplete_defn_error(self, fullname: str) -> TypeInfo: + def lookup_typeinfo_or_exception(self, fullname: str) -> TypeInfo: info = self.lookup_typeinfo(fullname) if info is None: - raise sem_helpers.IncompleteDefnException(f'No {fullname!r} found') + raise sem_helpers.IncompleteDefnError(f'No {fullname!r} found') return info - def lookup_class_typeinfo_or_incomplete_defn_error(self, klass: type) -> TypeInfo: + def lookup_class_typeinfo_or_exception(self, klass: type) -> TypeInfo: fullname = helpers.get_class_fullname(klass) - field_info = self.lookup_typeinfo_or_incomplete_defn_error(fullname) + field_info = self.lookup_typeinfo_or_exception(fullname) return field_info def model_class_has_attribute_defined(self, name: str, traverse_mro: bool = True) -> bool: @@ -141,7 +141,7 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: return auto_field_class_fullname = helpers.get_class_fullname(auto_field.__class__) - auto_field_info = self.lookup_typeinfo_or_incomplete_defn_error(auto_field_class_fullname) + auto_field_info = self.lookup_typeinfo_or_exception(auto_field_class_fullname) set_type, get_type = fields.get_field_descriptor_types(auto_field_info, is_nullable=False) self.add_new_node_to_model_class(primary_key_attrname, Instance(auto_field_info, @@ -168,8 +168,6 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: related_id_attr_name = field.attname if self.model_class_has_attribute_defined(related_id_attr_name): continue - # if self.get_model_class_attr(related_id_attr_name) is not None: - # continue related_model_cls = self.django_context.get_field_related_model_cls(field) if related_model_cls is None: @@ -189,8 +187,8 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: rel_primary_key_field = self.django_context.get_primary_key_field(related_model_cls) try: - field_info = self.lookup_class_typeinfo_or_incomplete_defn_error(rel_primary_key_field.__class__) - except sem_helpers.IncompleteDefnException as exc: + field_info = self.lookup_class_typeinfo_or_exception(rel_primary_key_field.__class__) + except sem_helpers.IncompleteDefnError as exc: if not self.api.final_iteration: raise exc else: @@ -203,53 +201,6 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: class AddManagers(ModelClassInitializer): - def has_any_parametrized_manager_as_base(self, info: TypeInfo) -> bool: - for base in helpers.iter_bases(info): - if self.is_any_parametrized_manager(base): - return True - return False - - def is_any_parametrized_manager(self, typ: Instance) -> bool: - return typ.type.fullname in fullnames.MANAGER_CLASSES and isinstance(typ.args[0], AnyType) - - def create_new_model_parametrized_manager(self, name: str, base_manager_info: TypeInfo) -> Instance: - bases = [] - for original_base in base_manager_info.bases: - if self.is_any_parametrized_manager(original_base): - original_base = helpers.reparametrize_instance(original_base, - [Instance(self.model_classdef.info, [])]) - bases.append(original_base) - - new_manager_info = self.add_new_class_for_current_module(name, bases, force_replace_existing=True) - # copy fields to a new manager - new_cls_def_context = ClassDefContext(cls=new_manager_info.defn, - reason=self.ctx.reason, - api=self.api) - custom_manager_type = Instance(new_manager_info, [Instance(self.model_classdef.info, [])]) - - for name, sym in base_manager_info.names.items(): - if name in new_manager_info.names: - raise ValueError(f'Name {name!r} already exists on newly-created {new_manager_info.fullname!r} class.') - - # replace self type with new class, if copying method - if isinstance(sym.node, FuncDef): - sem_helpers.copy_method_or_incomplete_defn_exception(new_cls_def_context, - self_type=custom_manager_type, - new_method_name=name, - method_node=sym.node) - continue - - new_sym = sym.copy() - if isinstance(new_sym.node, Var): - new_var = Var(name, type=sym.type) - new_var.info = new_manager_info - new_var._fullname = new_manager_info.fullname + '.' + name - new_sym.node = new_var - - new_manager_info.names[name] = new_sym - - return custom_manager_type - def run_with_model_cls(self, model_cls: Type[Model]) -> None: for manager_name, manager in model_cls._meta.managers_map.items(): if self.model_class_has_attribute_defined(manager_name, traverse_mro=False): @@ -258,8 +209,7 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: if (sym.type is not None and isinstance(sym.type, Instance) - and sym.type.type.has_base(fullnames.BASE_MANAGER_CLASS_FULLNAME) - and not self.has_any_parametrized_manager_as_base(sym.type.type)): + and sym.type.type.has_base(fullnames.BASE_MANAGER_CLASS_FULLNAME)): # already defined and parametrized properly continue @@ -270,24 +220,12 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: continue manager_fullname = self.resolve_manager_fullname(helpers.get_class_fullname(manager.__class__)) - manager_info = self.lookup_typeinfo_or_incomplete_defn_error(manager_fullname) - manager_class_name = manager_fullname.rsplit('.', maxsplit=1)[1] + manager_info = self.lookup_typeinfo_or_exception(manager_fullname) if manager_name not in self.model_classdef.info.names: # manager not yet defined, just add models.Manager[ModelName] manager_type = Instance(manager_info, [Instance(self.model_classdef.info, [])]) self.add_new_node_to_model_class(manager_name, manager_type) - else: - # creates new MODELNAME_MANAGERCLASSNAME class that represents manager parametrized with current model - if not self.has_any_parametrized_manager_as_base(manager_info): - continue - - custom_model_manager_name = manager.model.__name__ + '_' + manager_class_name - custom_manager_type = self.create_new_model_parametrized_manager(custom_model_manager_name, - base_manager_info=manager_info) - - self.add_new_node_to_model_class(manager_name, custom_manager_type, - force_replace_existing=True) class AddDefaultManagerAttribute(ModelClassInitializer): @@ -304,7 +242,7 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: default_manager_fullname = helpers.get_class_fullname(model_cls._meta.default_manager.__class__) resolved_default_manager_fullname = self.resolve_manager_fullname(default_manager_fullname) - default_manager_info = self.lookup_typeinfo_or_incomplete_defn_error(resolved_default_manager_fullname) + default_manager_info = self.lookup_typeinfo_or_exception(resolved_default_manager_fullname) default_manager = Instance(default_manager_info, [Instance(self.model_classdef.info, [])]) self.add_new_node_to_model_class('_default_manager', default_manager) @@ -323,11 +261,12 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: related_model_cls = self.django_context.get_field_related_model_cls(relation) if related_model_cls is None: + # could not find a referenced model (maybe invalid to= value) continue try: - related_model_info = self.lookup_class_typeinfo_or_incomplete_defn_error(related_model_cls) - except sem_helpers.IncompleteDefnException as exc: + related_model_info = self.lookup_class_typeinfo_or_exception(related_model_cls) + except sem_helpers.IncompleteDefnError as exc: if not self.api.final_iteration: raise exc else: @@ -339,11 +278,11 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: if isinstance(relation, (ManyToOneRel, ManyToManyRel)): try: - related_manager_info = self.lookup_typeinfo_or_incomplete_defn_error( + related_manager_info = self.lookup_typeinfo_or_exception( fullnames.RELATED_MANAGER_CLASS) # noqa: E501 if 'objects' not in related_model_info.names: - raise sem_helpers.IncompleteDefnException() - except sem_helpers.IncompleteDefnException as exc: + raise sem_helpers.IncompleteDefnError() + except sem_helpers.IncompleteDefnError as exc: if not self.api.final_iteration: raise exc else: @@ -372,7 +311,7 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: # get_FOO_display for choices for field in self.django_context.get_model_fields(model_cls): if field.choices: - info = self.lookup_typeinfo_or_incomplete_defn_error('builtins.str') + info = self.lookup_typeinfo_or_exception('builtins.str') return_type = Instance(info, []) common.add_method(self.ctx, name='get_{}_display'.format(field.attname), @@ -402,7 +341,7 @@ def run_with_model_cls(self, model_cls: Type[Model]) -> None: class AddMetaOptionsAttribute(ModelClassInitializer): def run_with_model_cls(self, model_cls: Type[Model]) -> None: if '_meta' not in self.model_classdef.info.names: - options_info = self.lookup_typeinfo_or_incomplete_defn_error(fullnames.OPTIONS_CLASS_FULLNAME) + options_info = self.lookup_typeinfo_or_exception(fullnames.OPTIONS_CLASS_FULLNAME) self.add_new_node_to_model_class('_meta', Instance(options_info, [ Instance(self.model_classdef.info, []) @@ -424,7 +363,7 @@ def process_model_class(ctx: ClassDefContext, for initializer_cls in initializers: try: initializer_cls(ctx, django_context).run() - except sem_helpers.IncompleteDefnException as exc: + except sem_helpers.IncompleteDefnError as exc: if not ctx.api.final_iteration: ctx.api.defer() continue diff --git a/mypy_django_plugin/transformers2/__init__.py b/mypy_django_plugin/transformers2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mypy_django_plugin/transformers2/dynamic_managers.py b/mypy_django_plugin/transformers2/dynamic_managers.py new file mode 100644 index 000000000..9c927e5d7 --- /dev/null +++ b/mypy_django_plugin/transformers2/dynamic_managers.py @@ -0,0 +1,79 @@ +from typing import Optional + +from mypy.checker import gen_unique_name +from mypy.nodes import NameExpr, TypeInfo, SymbolTableNode, StrExpr +from mypy.types import Type as MypyType, TypeVarType, TypeVarDef, Instance + +from mypy_django_plugin.lib import helpers, fullnames, chk_helpers, sem_helpers +from mypy_django_plugin.transformers.managers import iter_all_custom_queryset_methods + + +class CreateNewManagerClassFrom_FromQuerySet(helpers.DynamicClassPluginCallback): + def create_typevar_in_current_module(self, name: str, + upper_bound: Optional[MypyType] = None) -> TypeVarDef: + tvar_name = gen_unique_name(name, self.semanal_api.globals) + tvar_def = TypeVarDef(tvar_name, + fullname=self.semanal_api.cur_mod_id + '.' + tvar_name, + id=-1, + values=[], + upper_bound=upper_bound) + return tvar_def + + def create_new_dynamic_class(self) -> None: + # extract Manager class which will act as base + callee = self.get_callee() + fullname = callee.fullname or callee.expr.fullname + callee_manager_info = self.lookup_typeinfo_or_defer(fullname) + if callee_manager_info is None: + return None + + # extract queryset from which we're going to copy methods + passed_queryset_name_expr = self.call_expr.args[0] + assert isinstance(passed_queryset_name_expr, NameExpr) + queryset_class_name = passed_queryset_name_expr.name + sym = self.lookup_same_module_or_defer(queryset_class_name) + if sym is None: + return None + assert isinstance(sym.node, TypeInfo) + passed_queryset_info = sym.node + + # for TypeVar bound + base_model_info = self.lookup_typeinfo_or_defer(fullnames.MODEL_CLASS_FULLNAME) + if base_model_info is None: + return + model_tvar_defn = self.create_typevar_in_current_module('_M', upper_bound=Instance(base_model_info, [])) + model_tvar_type = TypeVarType(model_tvar_defn) + + # make Manager[_T] + parent_manager_type = Instance(callee_manager_info, [model_tvar_type]) + + # instantiate with a proper model, Manager[MyModel], filling all Manager type vars in process + new_manager_info = self.new_typeinfo(self.class_name, + bases=[parent_manager_type]) + new_manager_info.defn.type_vars = [model_tvar_defn] + new_manager_info.type_vars = [model_tvar_defn.name] + new_manager_info.set_line(self.call_expr) + + # copy methods from passed_queryset_info with self type replaced + self_type = Instance(new_manager_info, [model_tvar_type]) + for name, method_node in iter_all_custom_queryset_methods(passed_queryset_info): + self.add_method_from_signature(method_node, + name, + self_type, + new_manager_info.defn) + + new_manager_sym = SymbolTableNode(self.semanal_api.current_symbol_kind(), + new_manager_info, + plugin_generated=True) + self.semanal_api.add_symbol_table_node(self.class_name, new_manager_sym) + + # add mapping between generated manager and current one + runtime_manager_class_name = None + if 'class_name' in self.call_expr.arg_names: + class_name_arg = self.call_expr.args[self.call_expr.arg_names.index('class_name')] + if isinstance(class_name_arg, StrExpr): + runtime_manager_class_name = class_name_arg.value + + new_manager_name = runtime_manager_class_name or (callee_manager_info.name + 'From' + queryset_class_name) + django_generated_manager_name = 'django.db.models.manager.' + new_manager_name + base_model_info.metadata.setdefault('managers', {})[django_generated_manager_name] = new_manager_info.fullname diff --git a/mypy_django_plugin/transformers2/models.py b/mypy_django_plugin/transformers2/models.py new file mode 100644 index 000000000..12962e674 --- /dev/null +++ b/mypy_django_plugin/transformers2/models.py @@ -0,0 +1,236 @@ +from abc import abstractmethod +from typing import Type, Optional + +from django.db.models.base import Model +from django.db.models.fields.related import OneToOneField, ForeignKey +from django.db.models.fields.reverse_related import OneToOneRel, ManyToManyRel, ManyToOneRel +from mypy.checker import gen_unique_name +from mypy.nodes import TypeInfo, Var, SymbolTableNode, MDEF +from mypy.plugin import ClassDefContext +from mypy.semanal import dummy_context +from mypy.types import Instance, TypeOfAny, AnyType +from mypy.types import Type as MypyType + +from django.db import models +from mypy_django_plugin.lib import helpers, fullnames +from mypy_django_plugin.transformers import fields +from mypy_django_plugin.transformers.fields import get_field_type +from mypy_django_plugin.transformers2 import new_helpers + + +class TransformModelClassCallback(helpers.ClassDefPluginCallback): + def get_real_manager_fullname(self, manager_fullname: str) -> str: + model_info = self.lookup_typeinfo_or_defer(fullnames.MODEL_CLASS_FULLNAME) + real_manager_fullname = model_info.metadata.get('managers', {}).get(manager_fullname, manager_fullname) + return real_manager_fullname + + def modify_class_defn(self) -> None: + model_cls = self.django_context.get_model_class_by_fullname(self.class_defn.fullname) + if model_cls is None: + return None + return self.modify_model_class_defn(model_cls) + + def add_new_model_attribute(self, name: str, typ: MypyType, force_replace: bool = False) -> None: + model_info = self.class_defn.info + if name in model_info.names and not force_replace: + raise ValueError('Attribute already exists on the model') + + var = Var(name, type=typ) + var.info = model_info + var._fullname = self.semanal_api.qualified_name(name) + var.is_initialized_in_class = True + + sym = SymbolTableNode(MDEF, var, plugin_generated=True) + error_context = None if force_replace else dummy_context() + added = self.semanal_api.add_symbol_table_node(name, sym, context=error_context) + assert added + + def lookup_typeinfo_for_class_or_defer(self, klass: type, *, + reason_for_defer: Optional[str] = None) -> Optional[TypeInfo]: + manager_cls_fullname = helpers.get_class_fullname(klass) + return self.lookup_typeinfo_or_defer(manager_cls_fullname, + reason_for_defer=reason_for_defer) + + @abstractmethod + def modify_model_class_defn(self, runtime_model_cls: Type[Model]) -> None: + raise NotImplementedError + + +class AddDefaultManagerCallback(TransformModelClassCallback): + def modify_model_class_defn(self, runtime_model_cls: Type[Model]) -> None: + if ('_default_manager' in self.class_defn.info.names + or runtime_model_cls._meta.default_manager is None): + return None + + runtime_default_manager_class = runtime_model_cls._meta.default_manager.__class__ + runtime_manager_cls_fullname = new_helpers.get_class_fullname(runtime_default_manager_class) + manager_cls_fullname = self.get_real_manager_fullname(runtime_manager_cls_fullname) + + default_manager_info = self.lookup_typeinfo_or_defer(manager_cls_fullname) + if default_manager_info is None: + return + + self.add_new_model_attribute('_default_manager', + Instance(default_manager_info, [Instance(self.class_defn.info, [])])) + + +class AddManagersCallback(TransformModelClassCallback): + def modify_model_class_defn(self, runtime_model_cls: Type[models.Model]) -> None: + for manager_name, manager in runtime_model_cls._meta.managers_map.items(): + if manager_name in self.class_defn.info.names: + # already defined on the current model class, in file or at a previous iteration + continue + + manager_info = self.lookup_typeinfo_for_class_or_defer(manager.__class__) + if manager_info is None: + continue + + manager_type = Instance(manager_info, [Instance(self.class_defn.info, [])]) + self.add_new_model_attribute(manager_name, manager_type) + + +class AddPrimaryKeyIfDoesNotExist(TransformModelClassCallback): + """ + Adds default primary key to models which does not define their own. + class User(models.Model): + name = models.TextField() + """ + + def modify_model_class_defn(self, runtime_model_cls: Type[Model]) -> None: + auto_pk_field = runtime_model_cls._meta.auto_field + if auto_pk_field is None: + # defined explicitly + return None + auto_pk_field_name = auto_pk_field.attname + if auto_pk_field_name in self.class_defn.info.names: + # added on previous iteration + return None + + auto_pk_field_info = self.lookup_typeinfo_for_class_or_defer(auto_pk_field.__class__) + if auto_pk_field_info is None: + return None + + self.add_new_model_attribute(auto_pk_field_name, + fields.get_field_type(auto_pk_field_info, is_nullable=False)) + + +class AddRelatedManagersCallback(TransformModelClassCallback): + def modify_model_class_defn(self, runtime_model_cls: Type[Model]) -> None: + for relation in self.django_context.get_model_relations(runtime_model_cls): + reverse_manager_name = relation.get_accessor_name() + if (reverse_manager_name is None + or reverse_manager_name in self.class_defn.info.names): + continue + + related_model_cls = self.django_context.get_field_related_model_cls(relation) + if related_model_cls is None: + # could not find a referenced model (maybe invalid to= value, or GenericForeignKey) + continue + + related_model_info = self.lookup_typeinfo_for_class_or_defer(related_model_cls) + if related_model_info is None: + continue + + if isinstance(relation, OneToOneRel): + self.add_new_model_attribute(reverse_manager_name, + Instance(related_model_info, [])) + elif isinstance(relation, (ManyToOneRel, ManyToManyRel)): + related_manager_info = self.lookup_typeinfo_or_defer(fullnames.RELATED_MANAGER_CLASS) + if related_manager_info is None: + if not self.defer_till_next_iteration(self.class_defn, + reason=f'{fullnames.RELATED_MANAGER_CLASS!r} is not available for lookup'): + raise TypeInfoNotFound(fullnames.RELATED_MANAGER_CLASS) + continue + + # get type of default_manager for model + default_manager_fullname = helpers.get_class_fullname(related_model_cls._meta.default_manager.__class__) + reason_for_defer = (f'Trying to lookup default_manager {default_manager_fullname!r} ' + f'of model {helpers.get_class_fullname(related_model_cls)!r}') + default_manager_info = self.lookup_typeinfo_or_defer(default_manager_fullname, + reason_for_defer=reason_for_defer) + if default_manager_info is None: + continue + + default_manager_type = Instance(default_manager_info, [Instance(related_model_info, [])]) + + # related_model_cls._meta.default_manager.__class__ + # # we're making a subclass of 'objects', need to have it defined + # if 'objects' not in related_model_info.names: + # if not self.defer_till_next_iteration(self.class_defn, + # reason=f"'objects' manager is not yet defined on {related_model_info.fullname!r}"): + # raise AttributeNotFound(self.class_defn.info, 'objects') + # continue + + related_manager_type = Instance(related_manager_info, + [Instance(related_model_info, [])]) + # + # objects_sym = related_model_info.names['objects'] + # default_manager_type = objects_sym.type + # if default_manager_type is None: + # # dynamic base class, extract from django_context + # default_manager_cls = related_model_cls._meta.default_manager.__class__ + # default_manager_info = self.lookup_typeinfo_for_class_or_defer(default_manager_cls) + # if default_manager_info is None: + # continue + # default_manager_type = Instance(default_manager_info, [Instance(related_model_info, [])]) + + if (not isinstance(default_manager_type, Instance) + or default_manager_type.type.fullname == fullnames.MANAGER_CLASS_FULLNAME): + # if not defined or trivial -> just return RelatedManager[Model] + self.add_new_model_attribute(reverse_manager_name, related_manager_type) + continue + + # make anonymous class + name = gen_unique_name(related_model_cls.__name__ + '_' + 'RelatedManager', + self.semanal_api.current_symbol_table()) + bases = [related_manager_type, default_manager_type] + new_manager_info = self.new_typeinfo(name, bases) + self.add_new_model_attribute(reverse_manager_name, Instance(new_manager_info, [])) + + +class AddForeignPrimaryKeys(TransformModelClassCallback): + def modify_model_class_defn(self, runtime_model_cls: Type[Model]) -> None: + for field in runtime_model_cls._meta.get_fields(): + if not isinstance(field, (OneToOneField, ForeignKey)): + continue + rel_pk_field_name = field.attname + if rel_pk_field_name in self.class_defn.info.names: + continue + + related_model_cls = self.django_context.get_field_related_model_cls(field) + if related_model_cls is None: + field_sym = self.class_defn.info.get(field.name) + if field_sym is not None and field_sym.node is not None: + error_context = field_sym.node + else: + error_context = self.class_defn + self.semanal_api.fail(f'Cannot find model {field.related_model!r} ' + f'referenced in field {field.name!r} ', + ctx=error_context) + self.add_new_model_attribute(rel_pk_field_name, AnyType(TypeOfAny.from_error)) + continue + if related_model_cls._meta.abstract: + continue + + rel_pk_field = self.django_context.get_primary_key_field(related_model_cls) + rel_pk_field_info = self.lookup_typeinfo_for_class_or_defer(rel_pk_field.__class__) + if rel_pk_field_info is None: + continue + + field_type = get_field_type(rel_pk_field_info, + is_nullable=self.django_context.get_field_nullability(field)) + self.add_new_model_attribute(rel_pk_field_name, field_type) + + +class ModelCallback(helpers.ClassDefPluginCallback): + def __call__(self, ctx: ClassDefContext) -> None: + callback_classes = [ + AddManagersCallback, + AddPrimaryKeyIfDoesNotExist, + AddForeignPrimaryKeys, + AddDefaultManagerCallback, + AddRelatedManagersCallback, + ] + for callback_cls in callback_classes: + callback = callback_cls(self.plugin) + callback.__call__(ctx) diff --git a/mypy_django_plugin/transformers2/new_helpers.py b/mypy_django_plugin/transformers2/new_helpers.py new file mode 100644 index 000000000..87f695894 --- /dev/null +++ b/mypy_django_plugin/transformers2/new_helpers.py @@ -0,0 +1,26 @@ +from typing import Union + +from mypy.nodes import TypeInfo, MypyFile + + +class IncompleteDefnError(Exception): + pass + + +class TypeInfoNotFound(IncompleteDefnError): + def __init__(self, fullname: str) -> None: + super().__init__(f'It is final iteration and required type {fullname!r} is not ready yet.') + + +class AttributeNotFound(IncompleteDefnError): + def __init__(self, node: Union[TypeInfo, MypyFile], attrname: str) -> None: + super().__init__(f'Attribute {attrname!r} is not defined for the {node.fullname!r}.') + + +class NameNotFound(IncompleteDefnError): + def __init__(self, name: str) -> None: + super().__init__(f'Could not find {name!r} in the current activated namespaces') + + +def get_class_fullname(klass: type) -> str: + return klass.__module__ + '.' + klass.__qualname__ diff --git a/test-data/typecheck/fields/test_related.yml b/test-data/typecheck/fields/test_related.yml index 0a79d19b5..a757d37c0 100644 --- a/test-data/typecheck/fields/test_related.yml +++ b/test-data/typecheck/fields/test_related.yml @@ -36,7 +36,7 @@ pass class Book(models.Model): publisher = models.ForeignKey(to=Publisher, on_delete=models.CASCADE) - owner = models.ForeignKey(db_column='model_id', to='auth.User', on_delete=models.CASCADE) + owner = models.ForeignKey(to='auth.User', on_delete=models.CASCADE) - case: foreign_key_field_different_order_of_params main: | @@ -653,7 +653,7 @@ - case: related_manager_is_a_subclass_of_default_manager main: | from myapp.models import User - reveal_type(User().orders) # N: Revealed type is 'myapp.models.Order_RelatedManager' + reveal_type(User().orders) # N: Revealed type is 'myapp.models.User.Order_RelatedManager' reveal_type(User().orders.get()) # N: Revealed type is 'myapp.models.Order*' reveal_type(User().orders.manager_method()) # N: Revealed type is 'builtins.int' installed_apps: @@ -663,9 +663,12 @@ - path: myapp/models.py content: | from django.db import models + from mypy_django_plugin.lib import generics + generics.make_classes_generic(models.Manager) + class User(models.Model): pass - class OrderManager(models.Manager): + class OrderManager(models.Manager['Order']): def manager_method(self) -> int: pass class Order(models.Model): diff --git a/test-data/typecheck/managers/querysets/test_from_queryset.yml b/test-data/typecheck/managers/querysets/test_from_queryset.yml index 96bdf9a3a..83bd39094 100644 --- a/test-data/typecheck/managers/querysets/test_from_queryset.yml +++ b/test-data/typecheck/managers/querysets/test_from_queryset.yml @@ -1,10 +1,16 @@ - case: from_queryset_with_base_manager main: | from myapp.models import MyModel - reveal_type(MyModel().objects) # N: Revealed type is 'myapp.models.MyModel_NewManager[myapp.models.MyModel]' + + reveal_type(MyModel().objects) # N: Revealed type is 'myapp.models.NewBaseManager[myapp.models.MyModel*]' reveal_type(MyModel().objects.get()) # N: Revealed type is 'myapp.models.MyModel*' reveal_type(MyModel().objects.queryset_method) # N: Revealed type is 'def () -> builtins.str' reveal_type(MyModel().objects.queryset_method()) # N: Revealed type is 'builtins.str' + + reveal_type(MyModel().objects2) # N: Revealed type is 'myapp.models.NewManager[myapp.models.MyModel*]' + reveal_type(MyModel().objects2.get()) # N: Revealed type is 'myapp.models.MyModel*' + reveal_type(MyModel().objects2.queryset_method) # N: Revealed type is 'def () -> builtins.str' + reveal_type(MyModel().objects2.queryset_method()) # N: Revealed type is 'builtins.str' installed_apps: - myapp files: @@ -12,21 +18,25 @@ - path: myapp/models.py content: | from django.db import models - from django.db.models.manager import BaseManager + from django.db.models.manager import BaseManager, Manager + from mypy_django_plugin.lib import generics class ModelQuerySet(models.QuerySet): def queryset_method(self) -> str: return 'hello' - NewManager = BaseManager.from_queryset(ModelQuerySet) + + NewBaseManager = BaseManager.from_queryset(ModelQuerySet) + NewManager = Manager.from_queryset(ModelQuerySet) + + generics.make_classes_generic(NewBaseManager, NewManager) + class MyModel(models.Model): - objects = NewManager() + objects = NewBaseManager['MyModel']() + objects2 = NewManager['MyModel']() -- case: from_queryset_with_manager +- case: manager_without_generic_requires_annotation main: | - from myapp.models import MyModel - reveal_type(MyModel().objects) # N: Revealed type is 'myapp.models.MyModel_NewManager[myapp.models.MyModel]' - reveal_type(MyModel().objects.get()) # N: Revealed type is 'myapp.models.MyModel*' - reveal_type(MyModel().objects.queryset_method()) # N: Revealed type is 'builtins.str' + from myapp.models import ModelQuerySet installed_apps: - myapp files: @@ -34,20 +44,27 @@ - path: myapp/models.py content: | from django.db import models + from django.db.models.manager import BaseManager, Manager + from mypy_django_plugin.lib import generics class ModelQuerySet(models.QuerySet): def queryset_method(self) -> str: return 'hello' - NewManager = models.Manager.from_queryset(ModelQuerySet) + NewBaseManager = BaseManager.from_queryset(ModelQuerySet) + NewManager = Manager.from_queryset(ModelQuerySet) + + generics.make_classes_generic(NewBaseManager, NewManager) + class MyModel(models.Model): - objects = NewManager() + objects = NewBaseManager() # E: Need type annotation for 'objects' + -- case: from_queryset_returns_intersection_of_manager_and_queryset +- case: from_queryset_with_custom_manager_as_base main: | from myapp.models import MyModel, NewManager - reveal_type(NewManager()) # N: Revealed type is 'myapp.models.NewManager' - reveal_type(MyModel.objects) # N: Revealed type is 'myapp.models.MyModel_NewManager[myapp.models.MyModel]' + reveal_type(NewManager()) # N: Revealed type is 'myapp.models.NewManager[]' + reveal_type(MyModel.objects) # N: Revealed type is 'myapp.models.NewManager[myapp.models.MyModel]' reveal_type(MyModel.objects.get()) # N: Revealed type is 'Any' reveal_type(MyModel.objects.manager_only_method()) # N: Revealed type is 'builtins.int' reveal_type(MyModel.objects.manager_and_queryset_method()) # N: Revealed type is 'builtins.str' @@ -58,6 +75,8 @@ - path: myapp/models.py content: | from django.db import models + from mypy_django_plugin.lib import generics + class ModelBaseManager(models.Manager): def manager_only_method(self) -> int: return 1 @@ -66,14 +85,17 @@ return 'hello' NewManager = ModelBaseManager.from_queryset(ModelQuerySet) + generics.make_classes_generic(NewManager) + class MyModel(models.Model): - objects = NewManager() + objects = NewManager['MyModel']() + - case: from_queryset_with_class_name_provided main: | from myapp.models import MyModel, NewManager - reveal_type(NewManager()) # N: Revealed type is 'myapp.models.NewManager' - reveal_type(MyModel.objects) # N: Revealed type is 'myapp.models.MyModel_NewManager[myapp.models.MyModel]' + reveal_type(NewManager()) # N: Revealed type is 'myapp.models.NewManager[]' + reveal_type(MyModel.objects) # N: Revealed type is 'myapp.models.NewManager[myapp.models.MyModel]' reveal_type(MyModel.objects.get()) # N: Revealed type is 'Any' reveal_type(MyModel.objects.manager_only_method()) # N: Revealed type is 'builtins.int' reveal_type(MyModel.objects.manager_and_queryset_method()) # N: Revealed type is 'builtins.str' @@ -84,6 +106,8 @@ - path: myapp/models.py content: | from django.db import models + from mypy_django_plugin.lib import generics + class ModelBaseManager(models.Manager): def manager_only_method(self) -> int: return 1 @@ -92,37 +116,44 @@ return 'hello' NewManager = ModelBaseManager.from_queryset(ModelQuerySet, class_name='NewManager') + from mypy_django_plugin.lib import generics + generics.make_classes_generic(NewManager) + class MyModel(models.Model): - objects = NewManager() + objects = NewManager['MyModel']() - case: from_queryset_with_class_inheritance main: | from myapp.models import MyModel - reveal_type(MyModel().objects) # N: Revealed type is 'myapp.models.MyModel_NewManager[myapp.models.MyModel]' + reveal_type(MyModel().objects) # N: Revealed type is 'myapp.models.NewManager[myapp.models.MyModel*]' reveal_type(MyModel().objects.get()) # N: Revealed type is 'myapp.models.MyModel*' reveal_type(MyModel().objects.queryset_method()) # N: Revealed type is 'builtins.str' installed_apps: - myapp files: - - path: myapp/__init__.py - - path: myapp/models.py - content: | - from django.db import models - from django.db.models.manager import BaseManager - class BaseQuerySet(models.QuerySet): - def queryset_method(self) -> str: - return 'hello' - class ModelQuerySet(BaseQuerySet): - pass - - NewManager = BaseManager.from_queryset(ModelQuerySet) - class MyModel(models.Model): - objects = NewManager() + - path: myapp/__init__.py + - path: myapp/models.py + content: | + from django.db import models + from django.db.models.manager import BaseManager + class BaseQuerySet(models.QuerySet): + def queryset_method(self) -> str: + return 'hello' + class ModelQuerySet(BaseQuerySet): + pass + + NewManager = BaseManager.from_queryset(ModelQuerySet) + + from mypy_django_plugin.lib import generics + generics.make_classes_generic(NewManager) + + class MyModel(models.Model): + objects = NewManager['MyModel']() - case: from_queryset_with_manager_in_another_directory_and_imports main: | from myapp.models import MyModel - reveal_type(MyModel().objects) # N: Revealed type is 'myapp.models.MyModel_NewManager[myapp.models.MyModel]' + reveal_type(MyModel().objects) # N: Revealed type is 'myapp.managers.NewManager[myapp.models.MyModel*]' reveal_type(MyModel().objects.get()) # N: Revealed type is 'myapp.models.MyModel*' reveal_type(MyModel().objects.queryset_method) # N: Revealed type is 'def (param: Union[builtins.str, None] =) -> Union[builtins.str, None]' reveal_type(MyModel().objects.queryset_method('str')) # N: Revealed type is 'Union[builtins.str, None]' @@ -135,8 +166,11 @@ from django.db import models from myapp.managers import NewManager + from mypy_django_plugin.lib import generics + generics.make_classes_generic(NewManager) + class MyModel(models.Model): - objects = NewManager() + objects = NewManager['MyModel']() - path: myapp/managers.py content: | from typing import Optional @@ -152,7 +186,7 @@ disable_cache: true main: | from myapp.models import MyModel - reveal_type(MyModel().objects) # N: Revealed type is 'myapp.models.MyModel_NewManager[myapp.models.MyModel]' + reveal_type(MyModel().objects) # N: Revealed type is 'myapp.managers.NewManager[myapp.models.MyModel*]' reveal_type(MyModel().objects.get()) # N: Revealed type is 'myapp.models.MyModel*' reveal_type(MyModel().objects.base_queryset_method) # N: Revealed type is 'def (param: Union[builtins.int, builtins.str]) -> ' reveal_type(MyModel().objects.base_queryset_method(2)) # N: Revealed type is '' @@ -164,8 +198,12 @@ content: | from django.db import models from myapp.managers import NewManager + + from mypy_django_plugin.lib import generics + generics.make_classes_generic(NewManager) + class MyModel(models.Model): - objects = NewManager() + objects = NewManager['MyModel']() - path: myapp/managers.py content: | from django.db import models @@ -192,7 +230,7 @@ reveal_type(Permission().another_models) # N: Revealed type is 'django.db.models.manager.RelatedManager[myapp.models.AnotherModelInProjectWithContribAuthM2M]' from myapp.managers import NewManager - reveal_type(NewManager()) # N: Revealed type is 'myapp.managers.NewManager' + reveal_type(NewManager()) # N: Revealed type is 'myapp.managers.NewManager[]' reveal_type(NewManager().base_queryset_method) # N: Revealed type is 'def (param: builtins.dict[builtins.str, Union[builtins.int, builtins.str]]) -> Union[builtins.int, builtins.str]' from myapp.models import MyModel @@ -210,8 +248,11 @@ from myapp.managers import NewManager from django.contrib.auth.models import Permission + from mypy_django_plugin.lib import generics + generics.make_classes_generic(NewManager) + class MyModel(models.Model): - objects = NewManager() + objects = NewManager['MyModel']() class AnotherModelInProjectWithContribAuthM2M(models.Model): permissions = models.ForeignKey( diff --git a/test-data/typecheck/managers/querysets/test_values_list.yml b/test-data/typecheck/managers/querysets/test_values_list.yml index c67f2658d..a95151710 100644 --- a/test-data/typecheck/managers/querysets/test_values_list.yml +++ b/test-data/typecheck/managers/querysets/test_values_list.yml @@ -220,6 +220,9 @@ - path: myapp/models.py content: | from django.db import models + from mypy_django_plugin.lib import generics + generics.make_classes_generic(models.QuerySet) + class TransactionQuerySet(models.QuerySet['Transaction']): pass class Transaction(models.Model): diff --git a/test-data/typecheck/managers/test_managers.yml b/test-data/typecheck/managers/test_managers.yml index 21ea5a840..c08c289d7 100644 --- a/test-data/typecheck/managers/test_managers.yml +++ b/test-data/typecheck/managers/test_managers.yml @@ -30,6 +30,7 @@ class Child(Parent): pass + - case: test_model_objects_attribute_present_in_case_of_model_cls_passed_as_generic_parameter main: | from myapp.models import Base, MyModel @@ -55,23 +56,27 @@ def method(self) -> None: reveal_type(self.model_cls._default_manager) # N: Revealed type is 'django.db.models.manager.Manager[myapp.models.MyModel]' + - case: if_custom_manager_defined_it_is_set_to_default_manager main: | from myapp.models import MyModel reveal_type(MyModel._default_manager) # N: Revealed type is 'myapp.models.CustomManager[myapp.models.MyModel]' + reveal_type(MyModel._default_manager.get()) # N: Revealed type is 'myapp.models.MyModel*' installed_apps: - myapp files: - path: myapp/__init__.py - path: myapp/models.py content: | - from typing import TypeVar from django.db import models - _T = TypeVar('_T', bound=models.Model) - class CustomManager(models.Manager[_T]): + from mypy_django_plugin.lib import generics + generics.make_classes_generic(models.Manager) + + class CustomManager(models.Manager['MyModel']): pass class MyModel(models.Model): - manager = CustomManager['MyModel']() + manager = CustomManager() + - case: if_default_manager_name_is_passed_set_default_manager_to_it main: | @@ -83,40 +88,54 @@ - path: myapp/__init__.py - path: myapp/models.py content: | - from typing import TypeVar from django.db import models - _T = TypeVar('_T', bound=models.Model) - class Manager1(models.Manager[_T]): + from mypy_django_plugin.lib import generics + generics.make_classes_generic(models.Manager) + + class Manager1(models.Manager['MyModel']): pass - class Manager2(models.Manager[_T]): + class Manager2(models.Manager['MyModel']): pass class MyModel(models.Model): class Meta: default_manager_name = 'm2' - m1 = Manager1['MyModel']() - m2 = Manager2['MyModel']() + m1 = Manager1() + m2 = Manager2() -- case: test_leave_as_is_if_objects_is_set_and_fill_typevars_with_outer_class + +- case: manager_requires_type_annotation_to_be_set_if_generic_is_not_specified main: | - from myapp.models import MyUser - reveal_type(MyUser.objects) # N: Revealed type is 'myapp.models.UserManager[myapp.models.MyUser]' - reveal_type(MyUser.objects.get()) # N: Revealed type is 'myapp.models.MyUser*' - reveal_type(MyUser.objects.get_or_404()) # N: Revealed type is 'myapp.models.MyUser' + from myapp.models import MyModel + reveal_type(MyModel.objects) # N: Revealed type is 'myapp.models.MyManager' + reveal_type(MyModel.objects.get()) # N: Revealed type is 'Any' + reveal_type(MyModel.objects2) # N: Revealed type is 'myapp.models.MyGenericManager[Any]' + reveal_type(MyModel.objects2.get()) # N: Revealed type is 'Any' + reveal_type(MyModel.objects3) # N: Revealed type is 'myapp.models.MyGenericManager[myapp.models.MyModel]' + reveal_type(MyModel.objects3.get()) # N: Revealed type is 'myapp.models.MyModel*' installed_apps: - myapp files: - path: myapp/__init__.py - path: myapp/models.py content: | + from typing import TypeVar from django.db import models - class UserManager(models.Manager['MyUser']): - def get_or_404(self) -> 'MyUser': - pass + from mypy_django_plugin.lib import generics + generics.make_classes_generic(models.Manager) + + class MyManager(models.Manager): + pass + _T = TypeVar('_T', bound=models.Model) + class MyGenericManager(models.Manager[_T]): + pass + + class MyModel(models.Model): + objects = MyManager() + objects2 = MyGenericManager() # E: Need type annotation for 'objects2' + objects3: 'MyGenericManager[MyModel]' = MyGenericManager() - class MyUser(models.Model): - objects = UserManager() - case: model_imported_from_different_file main: | @@ -139,13 +158,14 @@ class Inventory(models.Model): pass + - case: managers_that_defined_on_other_models_do_not_influence main: | from myapp.models import AbstractPerson, Book reveal_type(AbstractPerson.abstract_persons) # N: Revealed type is 'django.db.models.manager.Manager[myapp.models.AbstractPerson]' - reveal_type(Book.published_objects) # N: Revealed type is 'myapp.models.PublishedBookManager[myapp.models.Book]' + reveal_type(Book.published_objects) # N: Revealed type is 'myapp.models.PublishedBookManager' Book.published_objects.create(title='hello') - reveal_type(Book.annotated_objects) # N: Revealed type is 'myapp.models.AnnotatedBookManager[myapp.models.Book]' + reveal_type(Book.annotated_objects) # N: Revealed type is 'myapp.models.AnnotatedBookManager' Book.annotated_objects.create(title='hello') installed_apps: - myapp @@ -155,6 +175,9 @@ content: | from django.db import models + from mypy_django_plugin.lib import generics + generics.make_classes_generic(models.Manager) + class AbstractPerson(models.Model): abstract_persons = models.Manager['AbstractPerson']() class PublishedBookManager(models.Manager['Book']): @@ -166,7 +189,8 @@ published_objects = PublishedBookManager() annotated_objects = AnnotatedBookManager() -- case: managers_inherited_from_abstract_classes_multiple_inheritance + +- case: managers_inherited_from_abstract_classes_multiple_inheritance_do_not_warn_on_liskov main: | installed_apps: - myapp @@ -175,6 +199,9 @@ - path: myapp/models.py content: | from django.db import models + from mypy_django_plugin.lib import generics + generics.make_classes_generic(models.Manager) + class CustomManager1(models.Manager['AbstractBase1']): pass class AbstractBase1(models.Model): @@ -193,6 +220,7 @@ class Child(AbstractBase1, AbstractBase2): pass + - case: model_has_a_manager_of_the_same_type main: | from myapp.models import UnrelatedModel, MyModel @@ -208,59 +236,21 @@ - path: myapp/models.py content: | from django.db import models + from mypy_django_plugin.lib import generics + generics.make_classes_generic(models.Manager) + class UnrelatedModel(models.Model): objects = models.Manager['UnrelatedModel']() class MyModel(models.Model): pass -- case: manager_without_annotation_of_the_model_gets_it_from_outer_one - main: | - from myapp.models import UnrelatedModel2, MyModel2 - reveal_type(UnrelatedModel2.objects) # N: Revealed type is 'django.db.models.manager.Manager[myapp.models.UnrelatedModel2]' - reveal_type(UnrelatedModel2.objects.first()) # N: Revealed type is 'Union[myapp.models.UnrelatedModel2*, None]' - - reveal_type(MyModel2.objects) # N: Revealed type is 'django.db.models.manager.Manager[myapp.models.MyModel2]' - reveal_type(MyModel2.objects.first()) # N: Revealed type is 'Union[myapp.models.MyModel2*, None]' - installed_apps: - - myapp - files: - - path: myapp/__init__.py - - path: myapp/models.py - content: | - from django.db import models - class UnrelatedModel2(models.Model): - objects = models.Manager() - - class MyModel2(models.Model): - pass - -- case: inherited_manager_has_the_proper_type_of_model - main: | - from myapp.models import ParentOfMyModel3, MyModel3 - reveal_type(ParentOfMyModel3.objects) # N: Revealed type is 'django.db.models.manager.Manager[myapp.models.ParentOfMyModel3]' - reveal_type(ParentOfMyModel3.objects.first()) # N: Revealed type is 'Union[myapp.models.ParentOfMyModel3*, None]' - - reveal_type(MyModel3.objects) # N: Revealed type is 'django.db.models.manager.Manager[myapp.models.MyModel3]' - reveal_type(MyModel3.objects.first()) # N: Revealed type is 'Union[myapp.models.MyModel3*, None]' - installed_apps: - - myapp - files: - - path: myapp/__init__.py - - path: myapp/models.py - content: | - from django.db import models - class ParentOfMyModel3(models.Model): - objects = models.Manager() - - class MyModel3(ParentOfMyModel3): - pass - case: inheritance_with_explicit_type_on_child_manager main: | from myapp.models import ParentOfMyModel4, MyModel4 - reveal_type(ParentOfMyModel4.objects) # N: Revealed type is 'django.db.models.manager.Manager[myapp.models.ParentOfMyModel4]' - reveal_type(ParentOfMyModel4.objects.first()) # N: Revealed type is 'Union[myapp.models.ParentOfMyModel4*, None]' + reveal_type(ParentOfMyModel4.objects) # N: Revealed type is 'django.db.models.manager.Manager[Any]' + reveal_type(ParentOfMyModel4.objects.first()) # N: Revealed type is 'Union[Any, None]' reveal_type(MyModel4.objects) # N: Revealed type is 'django.db.models.manager.Manager[myapp.models.MyModel4]' reveal_type(MyModel4.objects.first()) # N: Revealed type is 'Union[myapp.models.MyModel4*, None]' @@ -271,70 +261,15 @@ - path: myapp/models.py content: | from django.db import models + from mypy_django_plugin.lib import generics + generics.make_classes_generic(models.Manager) + class ParentOfMyModel4(models.Model): - objects = models.Manager() + objects = models.Manager() # E: Need type annotation for 'objects' class MyModel4(ParentOfMyModel4): objects = models.Manager['MyModel4']() -# TODO: make it work someday -#- case: inheritance_of_two_models_with_custom_objects_manager -# main: | -# from myapp.models import MyBaseUser, MyUser -# reveal_type(MyBaseUser.objects) # N: Revealed type is 'myapp.models.MyBaseManager[myapp.models.MyBaseUser]' -# reveal_type(MyBaseUser.objects.get()) # N: Revealed type is 'myapp.models.MyBaseUser' -# -# reveal_type(MyUser.objects) # N: Revealed type is 'myapp.models.MyManager[myapp.models.MyUser]' -# reveal_type(MyUser.objects.get()) # N: Revealed type is 'myapp.models.MyUser' -# installed_apps: -# - myapp -# files: -# - path: myapp/__init__.py -# - path: myapp/models.py -# content: | -# from django.db import models -# -# class MyBaseManager(models.Manager): -# pass -# class MyBaseUser(models.Model): -# objects = MyBaseManager() -# -# class MyManager(models.Manager): -# pass -# class MyUser(MyBaseUser): -# objects = MyManager() - -- case: custom_manager_returns_proper_model_types - main: | - from myapp.models import User - reveal_type(User.objects) # N: Revealed type is 'myapp.models.User_MyManager[myapp.models.User]' - reveal_type(User.objects.select_related()) # N: Revealed type is 'myapp.models.User_MyManager[myapp.models.User]' - reveal_type(User.objects.get()) # N: Revealed type is 'myapp.models.User*' - reveal_type(User.objects.get_instance()) # N: Revealed type is 'builtins.int' - reveal_type(User.objects.get_instance_untyped('hello')) # N: Revealed type is 'Any' - - from myapp.models import ChildUser - reveal_type(ChildUser.objects) # N: Revealed type is 'myapp.models.ChildUser_MyManager[myapp.models.ChildUser]' - reveal_type(ChildUser.objects.select_related()) # N: Revealed type is 'myapp.models.ChildUser_MyManager[myapp.models.ChildUser]' - reveal_type(ChildUser.objects.get()) # N: Revealed type is 'myapp.models.ChildUser*' - reveal_type(ChildUser.objects.get_instance()) # N: Revealed type is 'builtins.int' - reveal_type(ChildUser.objects.get_instance_untyped('hello')) # N: Revealed type is 'Any' - installed_apps: - - myapp - files: - - path: myapp/__init__.py - - path: myapp/models.py - content: | - from django.db import models - class MyManager(models.Manager): - def get_instance(self) -> int: - pass - def get_instance_untyped(self, name): - pass - class User(models.Model): - objects = MyManager() - class ChildUser(models.Model): - objects = MyManager() - case: custom_manager_annotate_method_before_type_declaration main: | @@ -357,10 +292,11 @@ class ModelB(models.Model): movie = models.TextField() + - case: manager_defined_in_the_nested_class main: | from myapp.models import MyModel - reveal_type(MyModel.objects) # N: Revealed type is 'myapp.models.MyModel_MyManager[myapp.models.MyModel]' + reveal_type(MyModel.objects) # N: Revealed type is 'myapp.models.MyModel.MyManager' reveal_type(MyModel.objects.get()) # N: Revealed type is 'myapp.models.MyModel*' reveal_type(MyModel.objects.mymethod()) # N: Revealed type is 'builtins.int' installed_apps: @@ -370,8 +306,11 @@ - path: myapp/models.py content: | from django.db import models + from mypy_django_plugin.lib import generics + generics.make_classes_generic(models.Manager) + class MyModel(models.Model): - class MyManager(models.Manager): + class MyManager(models.Manager['MyModel']): def mymethod(self) -> int: pass objects = MyManager() From e9bdc50541f79d42515ad0d9c9646034c10d9fc4 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sun, 15 Mar 2020 00:58:11 +0300 Subject: [PATCH 08/76] wip --- django-stubs/db/models/fields/__init__.pyi | 3 + mfile.py | 113 ++++++++++ mfile.py.gv | 13 ++ mfile.py.gv.pdf | Bin 0 -> 20039 bytes mfile.py.gv.png | Bin 0 -> 30176 bytes my.gv | 9 + my.gv.pdf | Bin 0 -> 14836 bytes mypy_django_plugin/django/context.py | 10 +- mypy_django_plugin/lib/helpers.py | 152 +++++++++++-- mypy_django_plugin/main.py | 22 +- mypy_django_plugin/transformers/managers.py | 99 --------- .../transformers2/dynamic_managers.py | 27 ++- mypy_django_plugin/transformers2/models.py | 207 ++++++++++++------ .../transformers2/new_helpers.py | 4 + .../transformers2/related_managers.py | 69 ++++++ test-data/typecheck/fields/test_related.yml | 2 +- .../managers/querysets/test_as_manager.yml | 95 -------- .../managers/querysets/test_from_queryset.yml | 6 +- test-output/round-table.gv | 3 + test-output/round-table.gv.pdf | Bin 0 -> 11474 bytes 20 files changed, 513 insertions(+), 321 deletions(-) create mode 100644 mfile.py create mode 100644 mfile.py.gv create mode 100644 mfile.py.gv.pdf create mode 100644 mfile.py.gv.png create mode 100644 my.gv create mode 100644 my.gv.pdf create mode 100644 mypy_django_plugin/transformers2/related_managers.py delete mode 100644 test-data/typecheck/managers/querysets/test_as_manager.yml create mode 100644 test-output/round-table.gv create mode 100644 test-output/round-table.gv.pdf diff --git a/django-stubs/db/models/fields/__init__.pyi b/django-stubs/db/models/fields/__init__.pyi index 6add1186b..2721a2d46 100644 --- a/django-stubs/db/models/fields/__init__.pyi +++ b/django-stubs/db/models/fields/__init__.pyi @@ -42,6 +42,9 @@ _ST = TypeVar("_ST") # __get__ return type _GT = TypeVar("_GT") +class CharField(Field[str, str]): + + class Field(RegisterLookupMixin, Generic[_ST, _GT]): _pyi_private_set_type: Any _pyi_private_get_type: Any diff --git a/mfile.py b/mfile.py new file mode 100644 index 000000000..29f688537 --- /dev/null +++ b/mfile.py @@ -0,0 +1,113 @@ +from graphviz import Digraph +from mypy.options import Options + +source = """ +from root.package import MyQuerySet + +MyQuerySet().mymethod() +""" + +from mypy import parse + +parsed = parse.parse(source, 'myfile.py', None, None, Options()) +print(parsed) + +graphattrs = { + "labelloc": "t", + "fontcolor": "blue", + # "bgcolor": "#333333", + "margin": "0", +} + +nodeattrs = { + # "color": "white", + "fontcolor": "#00008b", + # "style": "filled", + # "fillcolor": "#ffffff", + # "fillcolor": "#006699", +} + +edgeattrs = { + # "color": "white", + # "fontcolor": "white", +} + +graph = Digraph('mfile.py', graph_attr=graphattrs, node_attr=nodeattrs, edge_attr=edgeattrs) +graph.node('__builtins__') + +graph.node('django.db.models') +graph.node('django.db.models.fields') + +graph.edge('django.db.models', 'django.db.models.fields') +graph.edge('django.db.models', '__builtins__') +graph.edge('django.db.models.fields', '__builtins__') + +graph.node('mymodule') +graph.edge('mymodule', 'django.db.models') +graph.edge('mymodule', '__builtins__') +# +# graph.node('ImportFrom', label='ImportFrom(val=root.package, [MyQuerySet])') +# graph.edge('MypyFile', 'ImportFrom') + + + +# graph.node('ClassDef_MyQuerySet', label='ClassDef(name=MyQuerySet)') +# graph.edge('MypyFile', 'ClassDef_MyQuerySet') +# +# graph.node('FuncDef_mymethod', label='FuncDef(name=mymethod)') +# graph.edge('ClassDef_MyQuerySet', 'FuncDef_mymethod') +# +# graph.node('Args', label='Args') +# graph.edge('FuncDef_mymethod', 'Args') +# +# graph.node('Var_self', label='Var(name=self)') +# graph.edge('Args', 'Var_self') +# +# graph.node('Block', label='Block') +# graph.edge('FuncDef_mymethod', 'Block') +# +# graph.node('PassStmt') +# graph.edge('Block', 'PassStmt') + +# graph.node('ExpressionStmt') +# graph.edge('MypyFile', 'ExpressionStmt') +# +# graph.node('CallExpr', label='CallExpr(val="MyQuerySet()")') +# graph.edge('ExpressionStmt', 'CallExpr') +# +# graph.node('MemberExpr', label='MemberExpr(val=".mymethod()")') +# graph.edge('CallExpr', 'MemberExpr') +# +# graph.node('CallExpr_outer_Args', label='Args()') +# graph.edge('CallExpr', 'CallExpr_outer_Args') +# +# graph.node('CallExpr_inner', label='CallExpr(val="mymethod()")') +# graph.edge('MemberExpr', 'CallExpr_inner') +# +# graph.node('NameExpr', label='NameExpr(val="mymethod")') +# graph.edge('CallExpr_inner', 'NameExpr') +# +# graph.node('Expression_Args', label='Args()') +# graph.edge('CallExpr_inner', 'Expression_Args') + +graph.render(view=True, format='png') + + +# MypyFile( +# ClassDef( +# name=MyQuerySet, +# FuncDef( +# name=mymethod, +# Args( +# Var(self)) +# Block(PassStmt()) +# ) +# ) +# ExpressionStmt:6( +# CallExpr:6( +# MemberExpr:6( +# CallExpr:6( +# NameExpr(MyQuerySet) +# Args()) +# mymethod) +# Args()))) diff --git a/mfile.py.gv b/mfile.py.gv new file mode 100644 index 000000000..58ca3910d --- /dev/null +++ b/mfile.py.gv @@ -0,0 +1,13 @@ +digraph "mfile.py" { + graph [fontcolor=blue labelloc=t margin=0] + node [fontcolor="#00008b"] + __builtins__ + "django.db.models" + "django.db.models.fields" + "django.db.models" -> "django.db.models.fields" + "django.db.models" -> __builtins__ + "django.db.models.fields" -> __builtins__ + mymodule + mymodule -> "django.db.models" + mymodule -> __builtins__ +} diff --git a/mfile.py.gv.pdf b/mfile.py.gv.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2be078e63a23c7b43fe09628b8766f30dd2a51f2 GIT binary patch literal 20039 zcmZsC1CSui((Tx`?H${;ZQHgzvokxkZQHhO+qU0+_x|^f`(C_`j;yZJ*>y6zBRWor zR9;w=hJls^inQsf`U8rIfS$n4&=QK9n}AN*#MaE&oPhZ+qzFYoKtLyIVeM?<__wq+ za5fP(F|so@f#T(ba&mSwF|dJh&ur3=i6I$6>^iBTa5(3*j^DuthlA0V5?9w{x(8*i z%-90;0=v{P%J7@qp!)rF6!ZE@*Zrowc64)IVRzf4(XFyftw+7Bf-9-v?J)6W zYEmLo8QG$$l}brnje=_xd-1TZI6wSoC%C-k&eHV```cr?($WRiwH?gJR9-h{#5!VG zTjFKC=Am=5^x52fI77<>7zhjfcJsAGgI2_*d=S5 zm_ntY1htk`!^7VmKDyLs*p^7YFm-I?7b@0}?mID6T#a5m)J%nSb568t7i~4BY z3_BS1kDxxtuX(`+7r5HLr5`4i(p+zLGVd>3{F&|uMg*sG$z6d_y7aUgTQ$8WVUg@y zAaj)<)-_h$cO^YNYIL4fv*=p3*80~AnFc&BiJt)t>L_6j>b)Jp2xV7-6AluZ8TO{# z&gZX={ly4DA<3DsL{Efn1;p?}esv$?MslylE7Wftzb!%iHZY3RJ3Sh^* zJ4;TsRUbEL9XB!GGyG|I{l)!m?Q;Ft)0EY|Nr0ZqCOU@s6({7Is*QjZD`h-rM5~{4 zh1eIepJ2>kOd!Qr7>YVAO{=bP%h@(wSZs5w%ZaPVO#poG7C3NN;5use68}>!fz&ey zap~j8iAlaTp+#=p`p9J%Bt>8aHr^d;IpA2NuxdIutQU3_S$5Zl7>DIDDFVZt)+9bP zP_~A2#|=wjp3k}2yN(IJa$-}Qy4d{Zx=HV_$#}Hs1O+@E*I^G{__+k~F^&1v+2)L> z*?Y>_aAO62K|+|55c^t7F|Km%wXP`Y#$74=k%ljOl!z7l7$yuU(GmoNRcj2X+FgfA z(T<9JHFBWU+*~v(j$7R=0w=%3-AD=a!(P)P9HIwQLU3|Z*9O*NfUCP%G0hTnUAq;x zZa6=pUU6NxS`NBT0zkNdz17tZugCMh0$X>Snz&>6{0ED*zqx4a5H>pN0L|K zvK7Qd(kcrTJEsKF6+)ccQmcYc#YT)GPSu58S1TSgwcMOD z#9mA;LorO?p9e_WAn$}Z1+g(f5kUzw_mMVYD}iElh0hqWaLrHrEuNI~;HSrvRA#?O zfFzH*HcJ8Lpe{iAjJ0j?nSwL5b3axuZZqGhEOQmJVSqDeDFTR4QN}C;#tC84DjrdO zC9mj0$zWHtt3WDukX#jZvVli-&K=y|a~~{VDY@Y{>8&nFa=iq@rQ;jCsb$Zr>5N?! z>RYh6owXg=mOUFn18Vl`sXbx%jP5MftMP4=((>-D0WJ5V2wH@O)$?a_MVJ8x!xc)2 zaniBjX_%2x5|B&-85(s6^X<@-KDL=cCS;-S(q+QMurQM&mjLknf$Mal<_JOHGGaZX zS%!s8&`5t6%vuRVq60hjp~%A}#5yHz_FgE-%Dq8zc|x2n=z2$8{F$WsE~RuYU7JTv zmzlRM=aNt3*Fllj>0eM;lt#k!Y-Z=MF$99IJUHbS+O4N`Qf8o`g0fx(o+z-Ml|~jt zPdDs%)eufeYe%-mlT#@Jz|q==C`)A>Tefo!V5y?*DYlZ< zl9#aXVHL6Z_ZMplJlW>0ST96I>AW#zf>p6ADW{WJVmo`z@%X$k_a$+D_0OLYCttv!Qo(l4t(_Rn z=>w7YTr^Uilw0^kPzF{G^Cx0Xa+3te5sFI|MMo*Tdm=!pMaXq+xrUjCzDjou`1^fz zyqnIo1bo#2($C=T3*h{a4B|aXx3gqP>ce@cdNMa3;$Od+w3EZA6{0&e44epA(}N;`BTN znbE3J*GP!iMuw_LMjVKzdRQ$-drR>HAB<9S)GUI?!-M;$cRg2k0N$DkZ zLttmCn!w}8nfJR#Qvo8ZVhcbd1!2eGjzPb3W_8_hyY{rGx!tZOc)eFpu^BLfFjQ^ zAJaRzF6p>Rb+o^)-uces{V3m-to2{!{%n~iYE^L*%Nj}zL>EgK$54ma2tmtf-hwhm z!aLRGc|za!XFPq?*0L+xfa5|wA(IeIuAf)_A3^0jKe1; zKyIYOFenm3`+8eF6YWy&0k|$H1A#KJHU1yT^!Mr?vGtF_`ltM>u^5?{ng4P4r^@)h zTI^ryOT^t-Ov(AL@*?2o{s;Jb&>7JGtq{-&2@o(4&>0#0wSV>4-y8q7bfR{)&j00Y zN<&XWPr&jYJ@?n?@A*Gutp7_!K&R|sZ$dyPZ(wHfFJgHEN0YzpF#P8^IvEpV3j;wr zcLJ@y5Iq4aJu3k-Ju`vMzr2P1a{hI9BKSv2{#!SSIoi3{|5qIUZvS8Ce-csp%j#%g z>tz2AmyySRUdxrlRvvuMJz(*k~j6WAFT9bOh1n-SZMe5L6^njgcs z$2j%IB~+;UR|fpp0OGV^&CZSWPbZ)3NQvMEgyz2ZrxKoa2@Y*y5?oq?0n`GdYIvls zxv#bYQcFu`{^**Yi~`fUG%^Jeqc=6Uxi!)=f!_gqzeD6IswkEe7ZrVQ^-oSs!O}ba zxPcLXIoH!SfXaW01-+C)&#q5zgPB9`gNAp9RaMda0W@8{em<535JE=I9mD5YgsBQp z1%QOMg7)&_LI+5qc=B`gKp&U>`HsYm#vy=g1m*0;Kbz_26D2b^F$ep6pq9?0t^}ZR zLKw{si<^7oF?kn@=o?GyLxV8%zLB7{wIZ@KdkDiKXK&?;7^21r7`_Q}Gy9}tYN>B> zYI^##!&HLSO5kG!NM+w(Dqq)RX9q4C`fkG0w=;vMXK8c=0^;6ju=gvO@+Sb`LWV|W z`^5xs3INcKnZfj(Cor^y_iA=OQ=3=yyX8cWr;|oH;Kt5ATQ-!IA7FG2^YrR%X3!4_ zhl#0oupa}29w=R7wf}4DiwB1J3w;7J5Qk+WHNe9MhIb#+a%kC;Llo)!Zw_-0gN}>wBGZgfuWhF z+wlkXgJ08?-{BAA%@0%0_ng#YjI1B_lOJWGeRK1-Q?l|;J@W3BTxheun-m}~@b_(E zQ{KT3En{O{t?Lh8(pSyVd9UBr(vK3CMeN>(MX=3O54p-D&7Ugk09o0a>2D_RMwYi38_Xe_s4Hn| z>nQ;0OB?_pF)<-G_F^}o`q~Cy?liy~bjZ`^6CD70#?|$JZ7G1~X#%_6kFAi80f0(t z08wzWju&V<<1ew}83Jr%|GDj1L~|=xX3#HOlkc3IWew-VF+=7s-F+IZQmn430 zvwFke}igq`02n4_{!PGTH#5bv?hadnvxr9l9^#Uo*BZ4!?_EINy50FMNvn zcOI+7?(7Uf>8XBXZZsFaiLPI=L;V)K*3(Etk0_-=jV z8vJ%Y7N>p!ZWiiZ!Mh`C-@&^pYQMp|Cu-lY({@Z&zk&anuYb%B@Xm|NCwTYG#Us2o zKIkd_EA7DBKjz5J*kfnq3jejc<|F47|C!m<)te3GbmnBL!|i9P812Rl1i~2@Yk&i3~O_WOu_AI@&>udTMzv>*7*-otJ#Jz`R2L-Yh8~si)^E zAgQ&lUNv{@TuOKNi-6-ow9Nh5b%>1@s~4?h4=lp6e2t_MK!IHpcD;m}LMg%nY>@Lx zqZ0SAu?W+dB7QuK(GS_Yrc>Y@M?40;XSD7F)8^frvRHy5C~{bF@CCAH$ew;J2`8l( zW6bsq$4{6i(cRE0_RGqK7DNseB(s<;g{Nj^s0G~TuGkHFYO}v?LcYh{wsdrqBPc-*e@^TP)HXFpgWl93H0k-xCq?}t_ePuCBzo72?v#QIySF?eMWHxN`0 z5b|m7;m1U8w$vC${Wj--Ss4|*an1MvmzUI6=Eh_- zHeRK;_&Nk1^v;vX!_Dx7T$Z=ir`87pOI4RBi67?ynZxg3jI;c@DE*mUoFzDz~hF8&J`dqT=G+TcKxp@ya;a7=n!_s!P{1O7*_z z58HHV?XAzGC%j}JLW^T6*Lg)w;~}@w>37qZH-`AT&97XsEHd4y7u56R4l`>vJaU;0 z&Yte>;IN@9Y731@<=RSG!W2=mnj@nXz+48zjKabbt_0$nLUX3)Y*KoQ9SYLa;jDXA z^@kQ~I5cq0l&9n~fV&dF+(tS+xuIdLx@Py(jz>Nt#`MX(EX&8D5(vGq#HaA#6M4Sd zN0{9sk_IIq9>+N1y1h+@G1zIaMfzh0s-rXA`Wbdy!w8Q&rgMVuqU>Own$q7hgS)NW zSps4rAo-tlrWO#DotpcpLLjcWhJI9AFu#Rmf)Qo>reGw#>wTwlua%nDhLfSyFxM7-z}uOT z&4=qjIQ7e_U8Bc-R*-H*dIhPRVYJh=@o>yz6kP0kxu3+vcO6hrbq@)Ia5QcAC0^Un zM6jsi(T(o?<27}?<{;BpNgR? zqF4Bqs3PCR;-?jSclFA7`6-l1L)T{J6rX=6j7w)~^^o~aFV|kG3bM9IFYsR}NIA=& zDB9Y=B=G6ZufXl06$a;Xp1fCP?LH39E_%PKi&AzuAvm{au7L4)jkTTbrYCUcw z>*}eafvBvhR+d2YA#eXfTCC&>^Es?{%6OTHGV@j;2laul4QOs$Ay)BdP#bdvw>o|T z=L{F=(@K4=_SvMZAX9)$Y|tjDnD26&x zhbZ}+z#>%Q>Yt;f2_LT+&%Hj>M>QGxQZIc9aQR1h zE4>{!)qR{1?lpUn4qYv!%n>Pr#h91mhhj9C7DimP7nP}HhfcTD1B{xjlDK#y>Q0q| zTfUMDrlq#>gdDOBvtgWy-IAJQxY3nhO-qd(wW9EZvpYYgF+8>Ujg$_kk0XOko)P zVs^sxOp|$wHVnmWHSt>T32|kwEid7AJ7JBxg_gTNg*kigNK{-ZywVA};3L)8KyME@ z1Zkr9YGphM0=csZGEwm)_3YtOFj8uJ70rt^0`zAZOUP{%Q7*e|h27s|f0s7m zE$*H3Q48GRX-{Z5Wso~R`bh2D6k{UWw?I91qe3WU((0d$jmluRqR6vn=PLFd!lQX?LwcH@%WB(Da$#jSY{z(oq(C(PLWP zKII_psU+=XU;(jSG2%Ew_ccbZS#sGV*x94(i zQJ&Kz(9dPt{B@LxVt?`u39eciQtsYlz#_@Xm%^5cUTpSkRl(mQdW79npUz~q(n0`5 z^sH|Rt6FCl+-X?YBF^RZ96zjR$_VWE@}lUZq(9=Slx(O(T{}(M1CNsBitbh)rbZ|w z3;3&~k4YB!Tr{MiU3xTsuI@P_)5bY)pDAE&crGKA?n&-UKh;9o{7WX0S8t!?uyrks z+=;IcIPE4(Y@~Yty&(@jvZ9$OiZJEUkRn#1y2O2(BL|V>^p{V()_Jk+UK{kyFq;6Knaut^30|nzy z4*_?P#ax+gs=2p+pp%MtY#TCI zs3eNz#+1uwG9Fn|>a{N)3YUj$7+l6s70BM0~t!5`o_C#RZY-%ic1V&zW-q@MA?Q z5{yKKSioJKn%w$8CDBu3^?vRJkM9cc6W7-m(SqZbY@Crxb%JIqESO`%lX_6!{7+Xn z$##V~P(O@g^@->V`c1c{7>#y!x?sRreYGgDoJ5!;*YiU?{(|Qe01n7i`ZibmV)2#W zy1;_?QVyHJu-Q+X!&EUKZxITiL0~LT)kmyc&zESX(}5l$!qZknZbkPw-cB6} z;P;x;B$7rL%tjv#_#Gxjx7o;KQ-h5&tG^Hl=jRu(4Bbx25+`pJ3a+84UZ!I!4#37b zOjDrwqRHG^^s+sZh2_`sE3e9o(BV~iJXp4}+kZk|L$w2A{+N_4Bn_WC45oA_p<<3; zZiGDtp}4Z4b&)Us~I6YfjE zV|4oLx1aIOSAIM}URKNn#+#vggwFSxsfK6+V~rDUEOlw(BQufTN@=0icOxT>A0)}a zY@NcQcL(7V?I#5XJ*x}vtVa3hD}{3h!q+wdNn+Ge;8Tt`BsMFPw8=DyEDgp>+Tr_< z>AlJ zCezY$QHiRdoxl;cMdLYx2w-lMJm?I>cg6THhb)rU10_d~=knuRTp7F|)s^9X%O}dX znng%v#q>6;=IJ10!nAbbZBFB`ltlMS#u3P*vc#{u+=g!Oh(&n_Kh<;Fhhm!*dA@Yp z>Qa&eRFrm1M&u}u6@i`E@)}*qjk6}d{==cS(R7|h@eF#jj%?eu@axd=7uc?XCXS#@ zi|~r5`C;Vy75F?odIV4~i?dDA5SvK4U7eto21;dMpzBghVy5z_aRY>|4E?*v)|SZy zXcjawgZHHr6rM_m8!N8>@w?7tz$Yi00Kevra-HqNJuG5)S(k z#qRz|B=OWOe9oFB`MJa-WL$>vA4ANGZt)hzptd|_a;P%}EHNTOd>Gx_-w14}(p(yu zCPXTvPh$j4QNNtdo+0Fohsv)#R7R(d1YA=PPLWMweQbnTcQroZRO>U*n1TVAS-*@8KP=q`=Qe`XzXe1dQ{(`oHW=<{9cwPkAbDmwftmc36)D}0Jo|Qi zq#O(t__^D5B!H0+-IGl^eyaVs+yh*-UN`M8woBMWoq=tsQAX9(HTT zJOgM?D;^#gY*gOQ(pGAcXm$P4HPz>Lo1<51kYD{qzWX@DRErzGcf z9(m<(VIs_)VhR;*=)-QOgRBqLLx9XWSO9o*&RQly>n`!u`2Do{q2iFDCj zu9;M4WYI6xDv_i02~cH1u^O>en!7eswss-vU%G@diRPrJjA8Nv7s z2wpo2GIiANSSxh1ir)w`iq|+Q%vA$$z4Se~C3V`P3Id&IYn_7@TVj3}FV-|HU+@!I z;z%#aaDm;?{sB*NfM1>uX|R`>KCljrx%8UkqlE0?!U1xZc?IjiZxs^kwb^L+Ls#b* zH+1OpEKR(9u!8^cs3d@=mXB40aUlrYBMctq>n*iS?Gv;%Ys8-+7{W(x*eh?uKZId6 zZ=5$$uK%6qg%d{iBeCo3NWaZJsTc9vT?oTABt_~?x^WLEhJ)^ zkZVdhnK0(}(Hh!rLCFsFLgN<=1tjHQ$6T(hR$0~Y(n?4B_w%VZUNiF+MJk3igbyA9 z>v?#5eS0F~flVmS4D!y`gA%o$fq~2$!=88n+F|6H4R@ zuBJ{jr~-zkVyCrz`_~+qvr4)5lc}jC^3gDWcViD$&d;J1TvB2f|F;^034Y~NQo;ml zZZcv!_Gmlh?g}D(P7`>c$@hRkR5PBnX^JY&(9EFKr*vQu z$O+xkj;Ho}k1HWOltM9Zffejo)Us6~tfRQTs18K@+k<0BbW6swgv!7rlYawDROa+h z^`J3Ba+d{WX7Rw@+bO(3eu*U>d7yZT5vbU}eIr!26?mP?BeoaV-^sUNe_=ps6d&Yj zQ47mZCcBq~0-qov10u>hq@kNT>R6CD>&p<->kCy9JVKILZ`PWT=930;SM*HrouWlB zw`q6G(vKq_uCB+d%t{=B{XveVVV_&E!uWa-R&k5H0WsA=$x)?hnSokxd}d(ePj)3^ ztSBq>(3{$QwRoFnP*nRm0#%r)~Wr)a%8&+-o&!vm+$+gIkR}uWVLzMw=30 zcT^fV=lt_YeOq-X?U3jeeAytZ3(0hq;$N6wiuI1|Fi3P=&{%fk3GdX=j?D?uv3yOR z?YjrSN3DG_ihXDcMShJt(*P8dYrtY&I*h%X^&v~P23p1(KZR7lWbc}!5v()ay!!fo zvnHQHn_(^7sXtx7vTrAth^}xzIp!W5TOvfa!p*ZTI?D7PzH6H2-Zgr1!{KPmnfxNE z#ba0VBbxRkl#1@ytX!8< z;&o393@8JE1L`__xaAp9WJAfhfi=CmRUpo+5{Z}|{oU2olZv=K?H0Ni&;emS_SI^m z39l#m@ONZP?KRtS5^mpsNFR!vUA%6>%AIl1b4Y9y1CeQIT(;Z3P?~x|vHWO5+92?6^Y0+Wb$=^2Ai-4rJ7M$yh6?=RS5gr(pb? z1f~h2s?;XBMcgMxxka98@e+?FUpKnNtGpG*YTmAQ$sCzMlA2>1s}TN>kY;46PZv(KW{B3k@HZ-LXIOH9QS$YD^8Dg`*x;`2)Bp8Fflm1&CyS zeB{_clhFmJoE_*ff@(s|gUSfBafyU{`*ax;LpQg@9LiYtu=1lYIrfOokBC^rc=}Af z2O(Ad8CoB<;DNzBHT2#A?4jcmX@d!C$#{2=K{&O=gYt6pzIs)ux7ckZok};sCV{zDGS$ADNvFu9)A5)RJ_f+lP;1aF7%vWKGRq;_9< z{QJI=hO0IH$Ah3|FZ9un0m6^Ftte?jJ`r3`hEcH#*_&|Ddb{RK>F=Tbel6>1dOl#V zJ(Bq0$T=}ynQ{bk*APL3g%bHoy|wwup$K4jHweoNg{61fwMNFIbZ|Q-_q9}_b+VfN zS0;~sH9KJe!>8*D{cx%usq^D$U|B$QlU@>y-jT+5SQ*RxvmGrQzKDZ;x!AT`a8j$L zYbzag_yz9S>tx_DTRvBRTZ-iH0^qWCYdY)kRPN@gu*FbwLU{lT zPO}GWu)VD+k6uQddv}*MwA2V&4NHgid0cTg?e#Wu3Qu7d=|LVkKK1t&2&ncAjHG@f z8PZK^ay9mq2XsyD9=vgP6fi<^=m~Jbkz*xflbq(Dr}LI=O8NkeISQ@~y^PL$jY{Z( z@T2>_k_0=T<%=tQyT_V<+B?c$`NPYetw^z+Z4704pdo6Z5y<{pVx5&6U{BFpSGzjK zuN>+i#Kwbav|vVMO9B-I!3*Vku?7bU$diJ=f9H+^8bMF%KA6Z~6kX!Auxv4%_Ep~+ zuM=ywvf)v_L!0;Sw8S;td3#oX^)?R~*>;EssmSRN7(Q}J7-&|gsjypoa3mn5;-0QI z#~~Lc1Qo{4B7}DN9@>56r3rm$ytq5L73V4+CDHb)Nt22wgRWO1Q0E6lgjiRX=T|Id z>RCQw^@Ps@m(y%E$;224pY90nkZVcafgFttwfHG;1a-GfqVVuYa*dy?sQc;W zN_;&f2(c$c>zp-FpQzVi9D%vi{VaV!bbp^gSEk%V^P3S#OxO^~VWrKE`@Sg_-kGck z0?=+rrpdXfZ@eEB*>i+wHUA#8GitL8+(|u7@DQy@Z)I|Vvkkd2e^ptfqLK1_Lv-7u zJk_GWYv{D_5o~r*ruu@gq&O#hDCl&*KalAp8;}y5jMUI{`qR-}b|^vj7~r5c=tG-9 zMoZMNZnulhC&+g7DlT7dpqQp{R_1;p=s=2CiPf-OZ6#VVj7+Qeb`54_ZLvs1#XH|H zwpTSM%*YvWEVAVizpA7|;J|DkmiXC>s3T3YgN1<9>bpvuxPhQ9v7qr(IW77fP;K;- znX_f}%m{<0NM=9b0c}_spbs_EtEIF{xpxvoBdymO^}&phbKft% zW=Z)xa|vgLAEqqR3q}Tr3e4Ubxib(fU^Hhv>s-262t&C2dwNMix09i*U;%!P#dAGm z)#m4hj4OSA`f)rg0D0_Mp49@2oC{S=L^kdE&#d2wk z&Js#CJ}FZEgaAakejpw#uBIvdGD)Yz3Oq#UiYOR0AsyKB7fCJQ9O{uVoQn$NrHa)~ zhSCw4Xqe(^Y^Gy&(R$?W?Z%Z_nr!GNar$9{8K1F=^9N~>zgrXV0%$t-ZJ^XZR9!Bu zT1+;Cs%dX< zT7}^h@1b#sUffi?*0>ML!#LZ8+Ikz`4^cQl`QaIKsrQp@j%c&ZAl>q9egR!&mU0&c z4n2ae3&vIx!);*WumqmeX4#Ows-~puYcf|WP6%SvQI*K}>acA>Em)7gL)3AtYv^B3 zf31kwxFuw98!*RQ1Ltp6mPD*eH3{vYwd~<_^24HjC&8@}kscD%;#0gs)h@PY(wnTMWk$djk1g zDghtWl}J~lU#G%(Z=U?kA069;IGNgO->>1l$+%9SE%$bvXRquL%eK4h1duF6?oN$% z4#80}AXQ&?5_e{SnOjMj4HI<{K9HhB9DK>P)GPemltNoVVRV{3l3%;-XRW#~d;ma( z$SwS$QWW|W|6qOl_)6X)#+nS4j1R8QrYFLVV}tJd#KP+ko)KURA$6V*{aQk&wgi5> zJU4S95{sK2c1>-zvV4>-Lc=iyAs>IrZyx}sKXnrU=PA$%p99zp?%k~xFz)z4ySkMS z&5ds_OvRdc5V4W@K&s+$N`b2a$9pXurx*i)s8K`QV)$a~zwP?&2g z5Rc0Ck&{s$sh(CXD)?4n_Un++9JoEX}WiH%Tk zM_8U$Ds8aeT8_HI-67-JXZH-fF~u^HB*JGSkcG(!*CvjqRmNHJYNaR}Li&b8mA0Ib zJ=KAPB~hNS7;W2mnE$&!W#)$~grZzu^eNSfniSM8-x*AVhZ05?_Zk2!{%u4wKga$& zT-|A7P|#u&0Fne1JX^&6z5wNPuCFcPitunuo$o<|u%Tu8G3}XLri%~OrQz=~*IG@{ zP4YGPuJ(JdZrzJ_UV^u}%{!!T`teWt*Q3u?Hp`0B&$$9CDE7!(F4}&a>zQwcoawp_ zqmPbrUXJ7&{>lIuZ^=KP$Wl#MqI@|HYR)L78RM*dX7_sS5@EWKBR{xDNjIJxto!nH z${u{