Support forward references to decorated functions

This handles a variety of common cases of forward references to
decorated functions by effectively performing limited type inference
during semantic analysis. Also fixes forward references to
properties.

Closes #613. Closes #857.

Author: JukkaL

mypy/build.py

@@ -339,10 +339,9 @@ def __init__(self, data_dir: str,
         self.reports = reports
         self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors,
                                                   pyversion=pyversion)
-        self.semantic_analyzer_pass3 = ThirdPass(self.errors)
-        self.type_checker = TypeChecker(self.errors,
-                                        self.semantic_analyzer.modules,
-                                        self.pyversion)
+        modules = self.semantic_analyzer.modules
+        self.semantic_analyzer_pass3 = ThirdPass(modules, self.errors)
+        self.type_checker = TypeChecker(self.errors, modules, self.pyversion)
         self.states = []  # type: List[State]
         self.module_files = {}  # type: Dict[str, str]
         self.module_deps = {}  # type: Dict[Tuple[str, str], bool]

mypy/semanal.py

@@ -1357,6 +1357,7 @@ def visit_decorator(self, dec: Decorator) -> None:
         removed = []  # type: List[int]
         no_type_check = False
         for i, d in enumerate(dec.decorators):
+            # A bunch of decorators are special cased here.
             if refers_to_fullname(d, 'abc.abstractmethod'):
                 removed.append(i)
                 dec.func.is_abstract = True
@@ -1395,14 +1396,10 @@ def visit_decorator(self, dec: Decorator) -> None:
                 dec.var.is_initialized_in_class = True
                 self.add_symbol(dec.var.name(), SymbolTableNode(MDEF, dec),
                                 dec)
-        if dec.decorators and dec.var.is_property:
-            self.fail('Decorated property not supported', dec)
         if not no_type_check:
             dec.func.accept(self)
-        if not dec.decorators and not dec.var.is_property:
-            # No non-special decorators left. We can trivially infer the type
-            # of the function here.
-            dec.var.type = dec.func.type
+        if dec.decorators and dec.var.is_property:
+            self.fail('Decorated property not supported', dec)
 
     def check_decorated_function_is_method(self, decorator: str,
                                            context: Context) -> None:
@@ -2162,10 +2159,12 @@ def visit_try_stmt(self, s: TryStmt) -> None:
 class ThirdPass(TraverserVisitor[None]):
     """The third and final pass of semantic analysis.
 
-    Check type argument counts and values of generic types.
+    Check type argument counts and values of generic types, and perform some
+    straightforward type inference.
     """
 
-    def __init__(self, errors: Errors) -> None:
+    def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None:
+        self.modules = modules
         self.errors = errors
 
     def visit_file(self, file_node: MypyFile, fnam: str) -> None:
@@ -2183,6 +2182,37 @@ def visit_class_def(self, tdef: ClassDef) -> None:
             self.analyze(type)
         super().visit_class_def(tdef)
 
+    def visit_decorator(self, dec: Decorator) -> None:
+        """Try to infer the type of the decorated function.
+
+        This helps us resolve forward references to decorated
+        functions during type checking.
+        """
+        super().visit_decorator(dec)
+        if dec.var.is_property:
+            if dec.func.type is None:
+                dec.var.type = AnyType()
+            elif isinstance(dec.func.type, CallableType):
+                dec.var.type = dec.func.type.ret_type
+            return
+        decorator_preserves_type = True
+        for expr in dec.decorators:
+            preserve_type = False
+            if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef):
+                if is_identity_signature(expr.node.type):
+                    preserve_type = True
+            if not preserve_type:
+                decorator_preserves_type = False
+                break
+        if decorator_preserves_type:
+            # No non-special decorators left. We can trivially infer the type
+            # of the function here.
+            dec.var.type = function_type(dec.func, self.builtin_type('function'))
+        if dec.decorators and returns_any_if_called(dec.decorators[0]):
+            # The outermost decorator will return Any so we know the type of the
+            # decorated function.
+            dec.var.type = AnyType()
+
     def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
         self.analyze(s.type)
         super().visit_assignment_stmt(s)
@@ -2196,6 +2226,8 @@ def visit_type_application(self, e: TypeApplication) -> None:
             self.analyze(type)
         super().visit_type_application(e)
 
+    # Helpers
+
     def analyze(self, type: Type) -> None:
         if type:
             analyzer = TypeAnalyserPass3(self.fail)
@@ -2204,6 +2236,12 @@ def analyze(self, type: Type) -> None:
     def fail(self, msg: str, ctx: Context) -> None:
         self.errors.report(ctx.get_line(), msg)
 
+    def builtin_type(self, name: str, args: List[Type] = None) -> Instance:
+        names = self.modules['builtins']
+        sym = names.names[name]
+        assert isinstance(sym.node, TypeInfo)
+        return Instance(cast(TypeInfo, sym.node), args or [])
+
 
 def self_type(typ: TypeInfo) -> Union[Instance, TupleType]:
     """For a non-generic type, return instance type representing the type.
@@ -2356,3 +2394,34 @@ def visit_import_from(self, node: ImportFrom) -> None:
 
     def visit_import_all(self, node: ImportAll) -> None:
         node.is_unreachable = True
+
+
+def is_identity_signature(sig: Type) -> bool:
+    """Is type a callable of form T -> T (where T is a type variable)?"""
+    if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]:
+        if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType):
+            return sig.arg_types[0].id == sig.ret_type.id
+    return False
+
+
+def returns_any_if_called(expr: Node) -> bool:
+    """Return True if we can predict that expr will return Any if called.
+
+    This only uses information available during semantic analysis so this
+    will sometimes return False because of insufficient information (as
+    type inference hasn't run yet).
+    """
+    if isinstance(expr, RefExpr):
+        if isinstance(expr.node, FuncDef):
+            typ = expr.node.type
+            if typ is None:
+                # No signature -> default to Any.
+                return True
+            # Explicit Any return?
+            return isinstance(typ, CallableType) and isinstance(typ.ret_type, AnyType)
+        elif isinstance(expr.node, Var):
+            typ = expr.node.type
+            return typ is None or isinstance(typ, AnyType)
+    elif isinstance(expr, CallExpr):
+        return returns_any_if_called(expr.callee)
+    return False

mypy/test/data/check-functions.test

@@ -559,7 +559,6 @@ def dec2(f: Callable[[Any, Any], None]) -> Callable[[Any], None]: pass
 @dec2
 def f(x, y): pass
 
-
 [case testNoTypeCheckDecoratorOnMethod1]
 from typing import no_type_check
 
@@ -606,6 +605,214 @@ def foo(x: {1:2}) -> [1]:
     x = y
 
 
+-- Forward references to decorated functions
+-- -----------------------------------------
+
+
+[case testForwardReferenceToDynamicallyTypedDecorator]
+def f(self) -> None:
+    g()
+    g(1)
+
+def dec(f):
+    return f
+
+@dec
+def g():
+    pass
+
+[case testForwardReferenceToDecoratorWithAnyReturn]
+from typing import Any
+
+def f(self) -> None:
+    g()
+    g(1)
+
+def dec(f) -> Any:
+    return f
+
+@dec
+def g():
+    pass
+
+[case testForwardReferenceToDecoratorWithIdentityMapping]
+from typing import TypeVar
+
+def f(self) -> None:
+    g()
+    g(1) # E: Too many arguments for "g"
+    h(1).x # E: "str" has no attribute "x"
+    h('') # E: Argument 1 to "h" has incompatible type "str"; expected "int"
+
+T = TypeVar('T')
+def dec(f: T) -> T:
+    return f
+
+@dec
+def g(): pass
+@dec
+def h(x: int) -> str: pass
+[out]
+main: note: In function "f":
+
+[case testForwardReferenceToDynamicallyTypedDecoratedMethod]
+def f(self) -> None:
+    A().f(1).y
+    A().f()
+
+class A:
+    @dec
+    def f(self, x): pass
+
+def dec(f): return f
+[builtins fixtures/staticmethod.py]
+
+[case testForwardReferenceToStaticallyTypedDecoratedMethod]
+from typing import TypeVar
+
+def f(self) -> None:
+    A().f(1).y # E: "str" has no attribute "y"
+    A().f('') # E: Argument 1 to "f" of "A" has incompatible type "str"; expected "int"
+
+class A:
+    @dec
+    def f(self, a: int) -> str: return ''
+
+T = TypeVar('T')
+def dec(f: T) -> T: return f
+[builtins fixtures/staticmethod.py]
+[out]
+main: note: In function "f":
+
+[case testForwardReferenceToDynamicallyTypedProperty]
+def f(self) -> None:
+    A().x.y
+
+class A:
+    @property
+    def x(self): pass
+[builtins fixtures/property.py]
+
+[case testForwardReferenceToStaticallyTypedProperty]
+def f(self) -> None:
+    A().x.y # E: "int" has no attribute "y"
+
+class A:
+    @property
+    def x(self) -> int: return 1
+[builtins fixtures/property.py]
+[out]
+main: note: In function "f":
+
+[case testForwardReferenceToDynamicallyTypedStaticMethod]
+def f(self) -> None:
+    A.x(1).y
+    A.x() # E: Too few arguments for "x"
+
+class A:
+    @staticmethod
+    def x(x): pass
+[builtins fixtures/staticmethod.py]
+[out]
+main: note: In function "f":
+
+[case testForwardReferenceToStaticallyTypedStaticMethod]
+def f(self) -> None:
+    A.x(1).y # E: "str" has no attribute "y"
+    A.x('') # E: Argument 1 to "x" of "A" has incompatible type "str"; expected "int"
+
+class A:
+    @staticmethod
+    def x(a: int) -> str: return ''
+[builtins fixtures/staticmethod.py]
+[out]
+main: note: In function "f":
+
+[case testForwardReferenceToDynamicallyTypedClassMethod]
+def f(self) -> None:
+    A.x(1).y
+    A.x() # E: Too few arguments for "x"
+
+class A:
+    @classmethod
+    def x(cls, a): pass
+[builtins fixtures/classmethod.py]
+[out]
+main: note: In function "f":
+
+[case testForwardReferenceToStaticallyTypedClassMethod]
+def f(self) -> None:
+    A.x(1).y # E: "str" has no attribute "y"
+    A.x('') # E: Argument 1 to "x" of "A" has incompatible type "str"; expected "int"
+
+class A:
+    @classmethod
+    def x(cls, x: int) -> str: return ''
+[builtins fixtures/classmethod.py]
+[out]
+main: note: In function "f":
+
+[case testForwardReferenceToDecoratedFunctionUsingMemberExpr]
+import m
+
+def f(self) -> None:
+    g(1).x # E: "str" has no attribute "x"
+
+@m.dec
+def g(x: int) -> str: pass
+[file m.py]
+from typing import TypeVar
+T = TypeVar('T')
+def dec(f: T) -> T:
+    return f
+[out]
+main: note: In function "f":
+
+[case testForwardReferenceToFunctionWithMultipleDecorators]
+def f(self) -> None:
+    g()
+    g(1)
+
+def dec(f):
+    return f
+
+@dec
+@dec2
+def g():
+    pass
+
+def dec2(f):
+    return f
+
+[case testForwardReferenceToDynamicallyTypedDecoratedStaticMethod]
+def f(self) -> None:
+    A().f(1).y
+    A().f()
+    A().g(1).y
+    A().g()
+
+class A:
+    @dec
+    @staticmethod
+    def f(self, x): pass
+    @staticmethod
+    @dec
+    def g(self, x): pass
+
+def dec(f): return f
+[builtins fixtures/staticmethod.py]
+
+[case testForwardRefereceToDecoratedFunctionWithCallExpressionDecorator]
+def f(self) -> None:
+    g()
+    g(1)
+
+@dec(1)
+def g(): pass
+
+def dec(f): pass
+
+
 -- Conditional function definition
 -- -------------------------------
 
@@ -874,7 +1081,7 @@ def f(x, y):
     v = realtypes(x, y) # E: Argument 2 to "realtypes" has incompatible type "object"; expected "int"
     return v # E: Incompatible return value type: expected builtins.int, got builtins.str
 [out]
-main: note: In function "f": 
+main: note: In function "f":
 
 [case testCallDocstringFunction-skip]
 from typing import List