[html-report] Display unreachable code as Any (#3616)

Fixes #2785

Author: ilinum

mypy/report.py

@@ -1,13 +1,12 @@
 """Classes for producing HTML reports about imprecision."""
 
 from abc import ABCMeta, abstractmethod
-import cgi
 import json
 import os
 import shutil
 import tokenize
 from operator import attrgetter
-
+from urllib.request import pathname2url
 from typing import Any, Callable, Dict, List, Optional, Tuple, cast
 
 import time
@@ -157,9 +156,12 @@ def on_file(self,
         visitor = stats.StatisticsVisitor(inferred=True, filename=tree.fullname(),
                                           typemap=type_map, all_nodes=True)
         tree.accept(visitor)
-        num_total = visitor.num_imprecise + visitor.num_precise + visitor.num_any
+        num_unanalyzed_lines = list(visitor.line_map.values()).count(stats.TYPE_UNANALYZED)
+        # count each line of dead code as one expression of type "Any"
+        num_any = visitor.num_any + num_unanalyzed_lines
+        num_total = visitor.num_imprecise + visitor.num_precise + num_any
         if num_total > 0:
-            self.counts[tree.fullname()] = (visitor.num_any, num_total)
+            self.counts[tree.fullname()] = (num_any, num_total)
 
     def on_finish(self) -> None:
         total_any = sum(num_any for num_any, _ in self.counts.values())
@@ -171,9 +173,11 @@ def on_finish(self) -> None:
         any_column_name = "Anys"
         total_column_name = "Exprs"
         file_column_name = "Name"
+        total_row_name = "Total"
         coverage_column_name = "Coverage"
         # find the longest filename all files
-        name_width = max([len(file) for file in self.counts] + [len(file_column_name)])
+        name_width = max([len(file) for file in self.counts] +
+                         [len(file_column_name), len(total_row_name)])
         # totals are the largest numbers in their column - no need to look at others
         min_column_distance = 3  # minimum distance between numbers in two columns
         any_width = max(len(str(total_any)) + min_column_distance, len(any_column_name))
@@ -199,8 +203,8 @@ def on_finish(self) -> None:
                                coverage_width=coverage_width))
             f.write(separator)
             f.write('{:{name_width}} {:{any_width}} {:{total_width}} {:>{coverage_width}.2f}%\n'
-                    .format('Total', total_any, total_expr, total_coverage, name_width=name_width,
-                            any_width=any_width, total_width=exprs_width,
+                    .format(total_row_name, total_any, total_expr, total_coverage,
+                            name_width=name_width, any_width=any_width, total_width=exprs_width,
                             coverage_width=coverage_width))
 
 
@@ -402,7 +406,7 @@ def on_file(self,
         # Assumes a layout similar to what XmlReporter uses.
         xslt_path = os.path.relpath('mypy-html.xslt', path)
         transform_pi = etree.ProcessingInstruction('xml-stylesheet',
-                'type="text/xsl" href="%s"' % cgi.escape(xslt_path, True))
+                'type="text/xsl" href="%s"' % pathname2url(xslt_path))
         root.addprevious(transform_pi)
         self.schema.assertValid(doc)
 
@@ -425,7 +429,7 @@ def on_finish(self) -> None:
                              module=file_info.module)
         xslt_path = os.path.relpath('mypy-html.xslt', '.')
         transform_pi = etree.ProcessingInstruction('xml-stylesheet',
-                'type="text/xsl" href="%s"' % cgi.escape(xslt_path, True))
+                'type="text/xsl" href="%s"' % pathname2url(xslt_path))
         root.addprevious(transform_pi)
         self.schema.assertValid(doc)
 

mypy/stats.py

@@ -3,7 +3,7 @@
 import cgi
 import os.path
 
-from typing import Any, Dict, List, cast, Tuple
+from typing import Dict, List, cast, Tuple, Set, Optional
 
 from mypy.traverser import TraverserVisitor
 from mypy.types import (
@@ -12,17 +12,19 @@
 from mypy import nodes
 from mypy.nodes import (
     Expression, FuncDef, TypeApplication, AssignmentStmt, NameExpr, CallExpr, MypyFile,
-    MemberExpr, OpExpr, ComparisonExpr, IndexExpr, UnaryExpr, YieldFromExpr, RefExpr
+    MemberExpr, OpExpr, ComparisonExpr, IndexExpr, UnaryExpr, YieldFromExpr, RefExpr, ClassDef
 )
 
 
 TYPE_EMPTY = 0
-TYPE_PRECISE = 1
-TYPE_IMPRECISE = 2
-TYPE_ANY = 3
+TYPE_UNANALYZED = 1  # type of non-typechecked code
+TYPE_PRECISE = 2
+TYPE_IMPRECISE = 3
+TYPE_ANY = 4
 
 precision_names = [
     'empty',
+    'unanalyzed',
     'precise',
     'imprecise',
     'any',
@@ -79,6 +81,15 @@ def visit_func_def(self, o: FuncDef) -> None:
                 self.record_line(self.line, TYPE_ANY)
             super().visit_func_def(o)
 
+    def visit_class_def(self, o: ClassDef) -> None:
+        # Override this method because we don't want to analyze base_type_exprs (base_type_exprs
+        # are base classes in a class declaration).
+        # While base_type_exprs are technically expressions, type analyzer does not visit them and
+        # they are not in the typemap.
+        for d in o.decorators:
+            d.accept(self)
+        o.defs.accept(self)
+
     def visit_type_application(self, o: TypeApplication) -> None:
         self.line = o.line
         for t in o.types:
@@ -155,12 +166,17 @@ def visit_unary_expr(self, o: UnaryExpr) -> None:
     def process_node(self, node: Expression) -> None:
         if self.all_nodes:
             if self.typemap is not None:
-                typ = self.typemap.get(node)
-                if typ:
-                    self.line = node.line
-                    self.type(typ)
+                self.line = node.line
+                self.type(self.typemap.get(node))
+
+    def type(self, t: Optional[Type]) -> None:
+        if not t:
+            # If an expression does not have a type, it is often due to dead code.
+            # Don't count these because there can be an unanalyzed value on a line with other
+            # analyzed expressions, which overwrite the TYPE_UNANALYZED.
+            self.record_line(self.line, TYPE_UNANALYZED)
+            return
 
-    def type(self, t: Type) -> None:
         if isinstance(t, AnyType):
             self.log('  !! Any type around line %d' % self.line)
             self.num_any += 1
@@ -197,7 +213,7 @@ def log(self, string: str) -> None:
 
     def record_line(self, line: int, precision: int) -> None:
         self.line_map[line] = max(precision,
-                                  self.line_map.get(line, TYPE_PRECISE))
+                                  self.line_map.get(line, TYPE_EMPTY))
 
 
 def dump_type_stats(tree: MypyFile, path: str, inferred: bool = False,

mypy/test/testcmdline.py

@@ -22,7 +22,10 @@
 python3_path = sys.executable
 
 # Files containing test case descriptions.
-cmdline_files = ['cmdline.test']
+cmdline_files = [
+    'cmdline.test',
+    'reports.test',
+]
 
 
 class PythonEvaluationSuite(Suite):

test-data/unit/cmdline.test

@@ -304,14 +304,6 @@ bad = 0
 [out]
 mypy.ini: [mypy]: Unrecognized option: bad = 0
 
-[case testConfigErrorUnknownReport]
-# cmd: mypy -c pass
-[file mypy.ini]
-[[mypy]
-bad_report = .
-[out]
-mypy.ini: [mypy]: Unrecognized report type: bad_report
-
 [case testConfigErrorBadBoolean]
 # cmd: mypy -c pass
 [file mypy.ini]
@@ -329,64 +321,6 @@ python_version = 3.4
 [out]
 mypy.ini: [mypy-*]: Per-module sections should only specify per-module flags (python_version)
 
-[case testAnyExprReportDivisionByZero]
-# cmd: mypy --any-exprs-report=out -c 'pass'
-
-[case testCoberturaParser]
-# cmd: mypy --cobertura-xml-report build pkg
-[file pkg/__init__.py]
-[file pkg/a.py]
-from typing import Dict
-
-def foo() -> Dict:
-  z = {'hello': 'world'}
-  return z
-[file pkg/subpkg/__init__.py]
-[file pkg/subpkg/a.py]
-def bar() -> str:
-  return 'world'
-def untyped_function():
-  return 42
-[outfile build/cobertura.xml]
-<coverage timestamp="$TIMESTAMP" version="$VERSION" line-rate="0.8000" branch-rate="0">
-  <sources>
-    <source>$PWD</source>
-  </sources>
-  <packages>
-    <package complexity="1.0" name="pkg" branch-rate="0" line-rate="1.0000">
-      <classes>
-        <class complexity="1.0" filename="pkg/__init__.py" name="__init__.py" branch-rate="0" line-rate="1.0">
-          <methods/>
-          <lines/>
-        </class>
-        <class complexity="1.0" filename="pkg/a.py" name="a.py" branch-rate="0" line-rate="1.0000">
-          <methods/>
-          <lines>
-            <line branch="true" hits="1" number="3" precision="imprecise" condition-coverage="50% (1/2)"/>
-            <line branch="false" hits="1" number="4" precision="precise"/>
-            <line branch="false" hits="1" number="5" precision="precise"/>
-          </lines>
-        </class>
-      </classes>
-    </package>
-    <package complexity="1.0" name="pkg.subpkg" branch-rate="0" line-rate="0.5000">
-      <classes>
-        <class complexity="1.0" filename="pkg/subpkg/__init__.py" name="__init__.py" branch-rate="0" line-rate="1.0">
-          <methods/>
-          <lines/>
-        </class>
-        <class complexity="1.0" filename="pkg/subpkg/a.py" name="a.py" branch-rate="0" line-rate="0.5000">
-          <methods/>
-          <lines>
-            <line branch="false" hits="1" number="1" precision="precise"/>
-            <line branch="false" hits="0" number="3" precision="any"/>
-          </lines>
-        </class>
-      </classes>
-    </package>
-  </packages>
-</coverage>
-
 [case testConfigMypyPath]
 # cmd: mypy file.py
 [file mypy.ini]