Various improvements to fine-grained incremental checking (#4423)

This includes several fixes to fine-grained incremental mode. 

These are the main ones:

1) Support --ignore-missing-imports in fine-grained incremental mode

Most of the changes are about supporting command-line options in
fine-grained incremental test cases.

2) Add test case for skipping imports in fine-grained incremental mode

Most of the changes just enable testing skipped imports.

3) Fix unannotated functions in fine-grained incremental mode

4) Fix super() in fine-grained incremental mode

5) Fix AST diff of overloaded methods and properties with setters

Author: JukkaL

mypy/semanal.py

@@ -480,6 +480,8 @@ def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None:
         first_item.is_overload = True
         first_item.accept(self)
 
+        defn._fullname = self.qualified_name(defn.name())
+
         if isinstance(first_item, Decorator) and first_item.func.is_property:
             first_item.func.is_overload = True
             self.analyze_property_with_multi_part_definition(defn)

mypy/server/astdiff.py

@@ -7,16 +7,16 @@
 Only look at detail at definitions at the current module.
 """
 
-from typing import Set, List, TypeVar, Dict, Tuple, Optional, Sequence
+from typing import Set, List, TypeVar, Dict, Tuple, Optional, Sequence, Union
 
 from mypy.nodes import (
-    SymbolTable, SymbolTableNode, FuncBase, TypeInfo, Var, MypyFile, SymbolNode, Decorator,
-    TypeVarExpr, MODULE_REF, TYPE_ALIAS, UNBOUND_IMPORTED, TVAR
+    SymbolTable, SymbolTableNode, TypeInfo, Var, MypyFile, SymbolNode, Decorator, TypeVarExpr,
+    OverloadedFuncDef, FuncItem, MODULE_REF, TYPE_ALIAS, UNBOUND_IMPORTED, TVAR
 )
 from mypy.types import (
     Type, TypeVisitor, UnboundType, TypeList, AnyType, NoneTyp, UninhabitedType,
     ErasedType, DeletedType, Instance, TypeVarType, CallableType, TupleType, TypedDictType,
-    UnionType, Overloaded, PartialType, TypeType
+    UnionType, Overloaded, PartialType, TypeType, function_type
 )
 from mypy.util import get_prefix
 
@@ -232,9 +232,13 @@ def snapshot_definition(node: Optional[SymbolNode],
     The representation is nested tuples and dicts. Only externally
     visible attributes are included.
     """
-    if isinstance(node, FuncBase):
+    if isinstance(node, (OverloadedFuncDef, FuncItem)):
         # TODO: info
-        return ('Func', common, node.is_property, snapshot_type(node.type))
+        if node.type:
+            signature = snapshot_type(node.type)
+        else:
+            signature = snapshot_untyped_signature(node)
+        return ('Func', common, node.is_property, signature)
     elif isinstance(node, Var):
         return ('Var', common, snapshot_optional_type(node.type))
     elif isinstance(node, Decorator):
@@ -373,3 +377,19 @@ def visit_partial_type(self, typ: PartialType) -> SnapshotItem:
 
     def visit_type_type(self, typ: TypeType) -> SnapshotItem:
         return ('TypeType', snapshot_type(typ.item))
+
+
+def snapshot_untyped_signature(func: Union[OverloadedFuncDef, FuncItem]) -> Tuple[object, ...]:
+    if isinstance(func, FuncItem):
+        return (tuple(func.arg_names), tuple(func.arg_kinds))
+    else:
+        result = []
+        for item in func.items:
+            if isinstance(item, Decorator):
+                if item.var.type:
+                    result.append(snapshot_type(item.var.type))
+                else:
+                    result.append(('DecoratorWithoutType',))
+            else:
+                result.append(snapshot_untyped_signature(item))
+        return tuple(result)

mypy/server/astmerge.py

@@ -7,7 +7,7 @@
 
 from mypy.nodes import (
     Node, MypyFile, SymbolTable, Block, AssignmentStmt, NameExpr, MemberExpr, RefExpr, TypeInfo,
-    FuncDef, ClassDef, NamedTupleExpr, SymbolNode, Var, Statement, MDEF
+    FuncDef, ClassDef, NamedTupleExpr, SymbolNode, Var, Statement, SuperExpr, MDEF
 )
 from mypy.traverser import TraverserVisitor
 from mypy.types import (
@@ -123,6 +123,10 @@ def visit_namedtuple_expr(self, node: NamedTupleExpr) -> None:
         super().visit_namedtuple_expr(node)
         self.process_type_info(node.info)
 
+    def visit_super_expr(self, node: SuperExpr) -> None:
+        super().visit_super_expr(node)
+        node.info = self.fixup(node.info)
+
     # Helpers
 
     def fixup(self, node: SN) -> SN:

mypy/server/update.py

@@ -513,7 +513,7 @@ def invalidate_stale_cache_entries(cache: SavedCache,
 def verify_dependencies(state: State, manager: BuildManager) -> None:
     """Report errors for import targets in module that don't exist."""
     for dep in state.dependencies + state.suppressed:  # TODO: ancestors?
-        if dep not in manager.modules:
+        if dep not in manager.modules and not manager.options.ignore_missing_imports:
             assert state.tree
             line = find_import_line(state.tree, dep) or 1
             assert state.path

mypy/test/helpers.py

@@ -7,6 +7,8 @@
 
 from mypy import defaults
 from mypy.myunit import AssertionFailure
+from mypy.main import process_options
+from mypy.options import Options
 from mypy.test.data import DataDrivenTestCase
 
 
@@ -308,3 +310,32 @@ def retry_on_error(func: Callable[[], Any], max_wait: float = 1.0) -> None:
                 # Done enough waiting, the error seems persistent.
                 raise
             time.sleep(wait_time)
+
+
+def parse_options(program_text: str, testcase: DataDrivenTestCase,
+                  incremental_step: int) -> Options:
+    """Parse comments like '# flags: --foo' in a test case."""
+    options = Options()
+    flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
+    if incremental_step > 1:
+        flags2 = re.search('# flags{}: (.*)$'.format(incremental_step), program_text,
+                           flags=re.MULTILINE)
+        if flags2:
+            flags = flags2
+
+    flag_list = None
+    if flags:
+        flag_list = flags.group(1).split()
+        targets, options = process_options(flag_list, require_targets=False)
+        if targets:
+            # TODO: support specifying targets via the flags pragma
+            raise RuntimeError('Specifying targets via the flags pragma is not supported.')
+    else:
+        options = Options()
+
+    # Allow custom python version to override testcase_pyversion
+    if (not flag_list or
+            all(flag not in flag_list for flag in ['--python-version', '-2', '--py2'])):
+        options.python_version = testcase_pyversion(testcase.file, testcase.name)
+
+    return options

mypy/test/testcheck.py

@@ -7,14 +7,13 @@
 from typing import Dict, List, Optional, Set, Tuple
 
 from mypy import build, defaults
-from mypy.main import process_options
 from mypy.build import BuildSource, find_module_clear_caches
 from mypy.myunit import AssertionFailure
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import (
     assert_string_arrays_equal, normalize_error_messages,
-    retry_on_error, testcase_pyversion, update_testcase_output,
+    retry_on_error, update_testcase_output, parse_options
 )
 from mypy.errors import CompileError
 from mypy.options import Options
@@ -155,7 +154,7 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int = 0)
                     retry_on_error(lambda: os.remove(path))
 
         # Parse options after moving files (in case mypy.ini is being moved).
-        options = self.parse_options(original_program_text, testcase, incremental_step)
+        options = parse_options(original_program_text, testcase, incremental_step)
         options.use_builtins_fixtures = True
         options.show_traceback = True
         if 'optional' in testcase.file:
@@ -328,30 +327,3 @@ def parse_module(self,
             return out
         else:
             return [('__main__', 'main', program_text)]
-
-    def parse_options(self, program_text: str, testcase: DataDrivenTestCase,
-                      incremental_step: int) -> Options:
-        options = Options()
-        flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
-        if incremental_step > 1:
-            flags2 = re.search('# flags{}: (.*)$'.format(incremental_step), program_text,
-                               flags=re.MULTILINE)
-            if flags2:
-                flags = flags2
-
-        flag_list = None
-        if flags:
-            flag_list = flags.group(1).split()
-            targets, options = process_options(flag_list, require_targets=False)
-            if targets:
-                # TODO: support specifying targets via the flags pragma
-                raise RuntimeError('Specifying targets via the flags pragma is not supported.')
-        else:
-            options = Options()
-
-        # Allow custom python version to override testcase_pyversion
-        if (not flag_list or
-                all(flag not in flag_list for flag in ['--python-version', '-2', '--py2'])):
-            options.python_version = testcase_pyversion(testcase.file, testcase.name)
-
-        return options

mypy/test/testfinegrained.py

@@ -23,8 +23,10 @@
 from mypy.server.update import FineGrainedBuildManager
 from mypy.strconv import StrConv, indent
 from mypy.test.config import test_temp_dir, test_data_prefix
-from mypy.test.data import parse_test_cases, DataDrivenTestCase, DataSuite, UpdateFile
-from mypy.test.helpers import assert_string_arrays_equal
+from mypy.test.data import (
+    parse_test_cases, DataDrivenTestCase, DataSuite, UpdateFile, module_from_path
+)
+from mypy.test.helpers import assert_string_arrays_equal, parse_options
 from mypy.test.testtypegen import ignore_node
 from mypy.types import TypeStrVisitor, Type
 from mypy.util import short_type
@@ -42,7 +44,8 @@ class FineGrainedSuite(DataSuite):
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         main_src = '\n'.join(testcase.input)
-        messages, manager, graph = self.build(main_src)
+        sources_override = self.parse_sources(main_src)
+        messages, manager, graph = self.build(main_src, testcase, sources_override)
 
         a = []
         if messages:
@@ -63,6 +66,10 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
                     # Delete file
                     os.remove(op.path)
                     modules.append((op.module, op.path))
+            if sources_override is not None:
+                modules = [(module, path)
+                           for module, path in sources_override
+                           if any(m == module for m, _ in modules)]
             new_messages = fine_grained_manager.update(modules)
             all_triggered.append(fine_grained_manager.triggered)
             new_messages = normalize_messages(new_messages)
@@ -85,16 +92,28 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
                 'Invalid active triggers ({}, line {})'.format(testcase.file,
                                                                testcase.line))
 
-    def build(self, source: str) -> Tuple[List[str], BuildManager, Graph]:
-        options = Options()
+    def build(self,
+              source: str,
+              testcase: DataDrivenTestCase,
+              sources_override: Optional[List[Tuple[str, str]]]) -> Tuple[List[str],
+                                                                          BuildManager,
+                                                                          Graph]:
+        # This handles things like '# flags: --foo'.
+        options = parse_options(source, testcase, incremental_step=1)
         options.incremental = True
         options.use_builtins_fixtures = True
         options.show_traceback = True
         main_path = os.path.join(test_temp_dir, 'main')
         with open(main_path, 'w') as f:
             f.write(source)
+        if sources_override is not None:
+            sources = [BuildSource(path, module, None)
+                       for module, path in sources_override]
+        else:
+            sources = [BuildSource(main_path, None, None)]
+        print(sources)
         try:
-            result = build.build(sources=[BuildSource(main_path, None, None)],
+            result = build.build(sources=sources,
                                  options=options,
                                  alt_lib_path=test_temp_dir)
         except CompileError as e:
@@ -112,6 +131,27 @@ def format_triggered(self, triggered: List[List[str]]) -> List[str]:
             result.append(('%d: %s' % (n + 2, ', '.join(filtered))).strip())
         return result
 
+    def parse_sources(self, program_text: str) -> Optional[List[Tuple[str, str]]]:
+        """Return target (module, path) tuples for a test case, if not using the defaults.
+
+        These are defined through a comment like '# cmd: main a.py' in the test case
+        description.
+        """
+        # TODO: Support defining separately for each incremental step.
+        m = re.search('# cmd: mypy ([a-zA-Z0-9_. ]+)$', program_text, flags=re.MULTILINE)
+        if m:
+            # The test case wants to use a non-default set of files.
+            paths = m.group(1).strip().split()
+            result = []
+            for path in paths:
+                path = os.path.join(test_temp_dir, path)
+                module = module_from_path(path)
+                if module == 'main':
+                    module = '__main__'
+                result.append((module, path))
+            return result
+        return None
+
 
 def normalize_messages(messages: List[str]) -> List[str]:
     return [re.sub('^tmp' + re.escape(os.sep), '', message)

mypy/traverser.py

@@ -10,7 +10,7 @@
     GeneratorExpr, ListComprehension, SetComprehension, DictionaryComprehension,
     ConditionalExpr, TypeApplication, ExecStmt, Import, ImportFrom,
     LambdaExpr, ComparisonExpr, OverloadedFuncDef, YieldFromExpr,
-    YieldExpr, StarExpr, BackquoteExpr, AwaitExpr, PrintStmt,
+    YieldExpr, StarExpr, BackquoteExpr, AwaitExpr, PrintStmt, SuperExpr,
 )
 
 
@@ -250,6 +250,9 @@ def visit_backquote_expr(self, o: BackquoteExpr) -> None:
     def visit_await_expr(self, o: AwaitExpr) -> None:
         o.expr.accept(self)
 
+    def visit_super_expr(self, o: SuperExpr) -> None:
+        o.call.accept(self)
+
     def visit_import(self, o: Import) -> None:
         for a in o.assignments:
             a.accept(self)

test-data/unit/diff.test

@@ -476,3 +476,76 @@ def g(x: object) -> Iterator[None]:
 [builtins fixtures/list.pyi]
 [out]
 __main__.g
+
+[case testOverloadedMethod]
+from typing import overload
+
+class A:
+    @overload
+    def f(self, x: int) -> int: pass
+    @overload
+    def f(self, x: str) -> str: pass
+    def f(self, x): pass
+
+    @overload
+    def g(self, x: int) -> int: pass
+    @overload
+    def g(self, x: str) -> str: pass
+    def g(self, x): pass
+[file next.py]
+from typing import overload
+
+class A:
+    @overload
+    def f(self, x: int) -> int: pass
+    @overload
+    def f(self, x: str) -> str: pass
+    def f(self, x): pass
+
+    @overload
+    def g(self, x: int) -> int: pass
+    @overload
+    def g(self, x: object) -> object: pass
+    def g(self, x): pass
+[out]
+__main__.A.g
+
+[case testPropertyWithSetter]
+class A:
+    @property
+    def x(self) -> int:
+        pass
+
+    @x.setter
+    def x(self, o: int) -> None:
+        pass
+
+class B:
+    @property
+    def x(self) -> int:
+        pass
+
+    @x.setter
+    def x(self, o: int) -> None:
+        pass
+[file next.py]
+class A:
+    @property
+    def x(self) -> int:
+        pass
+
+    @x.setter
+    def x(self, o: int) -> None:
+        pass
+
+class B:
+    @property
+    def x(self) -> str:
+        pass
+
+    @x.setter
+    def x(self, o: str) -> None:
+        pass
+[builtins fixtures/property.pyi]
+[out]
+__main__.B.x