From 038b122dccb659afe18ba9fbfeea3e12c2e25104 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 25 Mar 2019 22:40:54 -0700 Subject: [PATCH 1/3] Allow type ignores after type comments This is a bit of a hack in fastparse, but allows for the following to pass typechecking: `x = 1 # type: str # type: ignore` Fixes https://github.com/python/mypy/issues/5967 --- mypy/fastparse.py | 47 ++++++++++++++++++++--------- mypy/fastparse2.py | 18 ++++++++--- test-data/unit/check-fastparse.test | 26 ++++++++++++++++ 3 files changed, 72 insertions(+), 19 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 7d890570519b..89bf57ec6e49 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1,3 +1,4 @@ +import re import sys from typing import ( @@ -124,6 +125,8 @@ def ast3_parse(source: Union[str, bytes], filename: str, mode: str, TYPE_COMMENT_SYNTAX_ERROR = 'syntax error in type comment' # type: Final +TYPE_IGNORE_PATTERN = re.compile(r'#\s*type:\s*ignore') + # Older versions of typing don't allow using overload outside stubs, # so provide a dummy. @@ -184,18 +187,19 @@ def parse_type_comment(type_comment: str, line: int, errors: Optional[Errors], assume_str_is_unicode: bool = True, - ) -> Optional[Type]: + ) -> Tuple[bool, Optional[Type]]: try: typ = ast3_parse(type_comment, '', 'eval') except SyntaxError as e: if errors is not None: errors.report(line, e.offset, TYPE_COMMENT_SYNTAX_ERROR, blocker=True) - return None + return False, None else: raise else: + extra_ignore = TYPE_IGNORE_PATTERN.search(type_comment) is not None assert isinstance(typ, ast3_Expression) - return TypeConverter(errors, line=line, + return extra_ignore, TypeConverter(errors, line=line, assume_str_is_unicode=assume_str_is_unicode).visit(typ.body) @@ -216,7 +220,7 @@ def parse_type_string(expr_string: str, expr_fallback_name: str, code with unicode_literals...) and setting `assume_str_is_unicode` accordingly. """ try: - node = parse_type_comment(expr_string.strip(), line=line, errors=None, + _, node = parse_type_comment(expr_string.strip(), line=line, errors=None, assume_str_is_unicode=assume_str_is_unicode) if isinstance(node, UnboundType) and node.original_str_expr is None: node.original_str_expr = expr_string @@ -251,6 +255,8 @@ def __init__(self, self.is_stub = is_stub self.errors = errors + self.extra_type_ignores = [] # type: List[int] + # Cache of visit_X methods keyed by type of visited object self.visitor_cache = {} # type: Dict[type, Callable[[Optional[AST]], Any]] @@ -393,11 +399,11 @@ def translate_module_id(self, id: str) -> str: def visit_Module(self, mod: ast3.Module) -> MypyFile: body = self.fix_function_overloads(self.translate_stmt_list(mod.body)) - + ignores = [ti.lineno for ti in mod.type_ignores] + self.extra_type_ignores return MypyFile(body, self.imports, False, - {ti.lineno for ti in mod.type_ignores}, + {*ignores}, ) # --- stmt --- @@ -591,7 +597,10 @@ def make_argument(self, arg: ast3.arg, default: Optional[ast3.expr], kind: int, if annotation is not None: arg_type = TypeConverter(self.errors, line=arg.lineno).visit(annotation) elif type_comment is not None: - arg_type = parse_type_comment(type_comment, arg.lineno, self.errors) + extra_ignore, arg_type = parse_type_comment(type_comment, arg.lineno, self.errors) + if extra_ignore: + self.extra_type_ignores.append(arg.lineno) + return Argument(Var(arg.arg), arg_type, self.visit(default), kind) def fail_arg(self, msg: str, arg: ast3.arg) -> None: @@ -646,7 +655,9 @@ def visit_Assign(self, n: ast3.Assign) -> AssignmentStmt: lvalues = self.translate_expr_list(n.targets) rvalue = self.visit(n.value) if n.type_comment is not None: - typ = parse_type_comment(n.type_comment, n.lineno, self.errors) + extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: typ = None s = AssignmentStmt(lvalues, rvalue, type=typ, new_syntax=False) @@ -678,7 +689,9 @@ def visit_NamedExpr(self, n: NamedExpr) -> None: # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) def visit_For(self, n: ast3.For) -> ForStmt: if n.type_comment is not None: - target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: target_type = None node = ForStmt(self.visit(n.target), @@ -691,7 +704,9 @@ def visit_For(self, n: ast3.For) -> ForStmt: # AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) def visit_AsyncFor(self, n: ast3.AsyncFor) -> ForStmt: if n.type_comment is not None: - target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: target_type = None node = ForStmt(self.visit(n.target), @@ -720,7 +735,9 @@ def visit_If(self, n: ast3.If) -> IfStmt: # With(withitem* items, stmt* body, string? type_comment) def visit_With(self, n: ast3.With) -> WithStmt: if n.type_comment is not None: - target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: target_type = None node = WithStmt([self.visit(i.context_expr) for i in n.items], @@ -732,7 +749,9 @@ def visit_With(self, n: ast3.With) -> WithStmt: # AsyncWith(withitem* items, stmt* body, string? type_comment) def visit_AsyncWith(self, n: ast3.AsyncWith) -> WithStmt: if n.type_comment is not None: - target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: target_type = None s = WithStmt([self.visit(i.context_expr) for i in n.items], @@ -1215,11 +1234,11 @@ def visit_raw_str(self, s: str) -> Type: # An escape hatch that allows the AST walker in fastparse2 to # directly hook into the Python 3.5 type converter in some cases # without needing to create an intermediary `Str` object. - return (parse_type_comment(s.strip(), + _, typ = parse_type_comment(s.strip(), self.line, self.errors, self.assume_str_is_unicode) - or AnyType(TypeOfAny.from_error)) + return typ or AnyType(TypeOfAny.from_error) def visit_Call(self, e: Call) -> Type: # Parse the arg constructor diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 016a137c50a3..d32c3ca347fb 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -166,6 +166,8 @@ def __init__(self, # Cache of visit_X methods keyed by type of visited object self.visitor_cache = {} # type: Dict[type, Callable[[Optional[AST]], Any]] + self.extra_type_ignores = [] # type: List[int] + def fail(self, msg: str, line: int, column: int) -> None: self.errors.report(line, column, msg, blocker=True) @@ -305,11 +307,11 @@ def translate_module_id(self, id: str) -> str: def visit_Module(self, mod: ast27.Module) -> MypyFile: body = self.fix_function_overloads(self.translate_stmt_list(mod.body)) - + ignores = [ti.lineno for ti in mod.type_ignores] + self.extra_type_ignores return MypyFile(body, self.imports, False, - {ti.lineno for ti in mod.type_ignores}, + {*ignores}, ) # --- stmt --- @@ -547,8 +549,10 @@ def visit_Delete(self, n: ast27.Delete) -> DelStmt: def visit_Assign(self, n: ast27.Assign) -> AssignmentStmt: typ = None if n.type_comment: - typ = parse_type_comment(n.type_comment, n.lineno, self.errors, + extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors, assume_str_is_unicode=self.unicode_literals) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) stmt = AssignmentStmt(self.translate_expr_list(n.targets), self.visit(n.value), @@ -565,8 +569,10 @@ def visit_AugAssign(self, n: ast27.AugAssign) -> OperatorAssignmentStmt: # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) def visit_For(self, n: ast27.For) -> ForStmt: if n.type_comment is not None: - target_type = parse_type_comment(n.type_comment, n.lineno, self.errors, + extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors, assume_str_is_unicode=self.unicode_literals) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: target_type = None stmt = ForStmt(self.visit(n.target), @@ -593,8 +599,10 @@ def visit_If(self, n: ast27.If) -> IfStmt: # With(withitem* items, stmt* body, string? type_comment) def visit_With(self, n: ast27.With) -> WithStmt: if n.type_comment is not None: - target_type = parse_type_comment(n.type_comment, n.lineno, self.errors, + extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors, assume_str_is_unicode=self.unicode_literals) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: target_type = None stmt = WithStmt([self.visit(n.context_expr)], diff --git a/test-data/unit/check-fastparse.test b/test-data/unit/check-fastparse.test index f3f5403ffc35..f32b1f0526bd 100644 --- a/test-data/unit/check-fastparse.test +++ b/test-data/unit/check-fastparse.test @@ -88,6 +88,32 @@ def f4(x: Iterable[x][x]) -> None: pass # E: Invalid type comment or annotation def f5(x: Callable[..., int][x]) -> None: pass # E: Invalid type comment or annotation def f6(x: Callable[..., int].x) -> None: pass # E: Invalid type comment or annotation +[case testFastParseTypeWithIgnore] +def f(x, # type: x # type: ignore + ): + # type: (...) -> None + pass + +[case testFastParseVariableTypeWithIgnore] + +x = 1 # type: str # type: ignore + +[case testFastParseVariableTypeWithIgnoreNoSpace] + +x = 1 # type: str #type:ignore + +[case testFastParseVariableTypeWithIgnoreAndComment] + +x = 1 # type: str # type: ignore # comment + +[case testFastParseTypeWithIgnoreWithStmt] +with open('test', 'r') as f: # type: int # type: ignore + pass + +[case testFastParseTypeWithIgnoreForStmt] +for i in (1, 2, 3, 100): # type: str # type: ignore + pass + [case testFastParseProperty] class C: From 223de15c896e305aed27257431d833d8ed243fb4 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 25 Mar 2019 23:39:58 -0700 Subject: [PATCH 2/3] Don't pick up type ignores in comments --- mypy/fastparse.py | 13 +++++++++++-- test-data/unit/check-fastparse.test | 3 +++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 89bf57ec6e49..84a17c9b318b 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -125,7 +125,7 @@ def ast3_parse(source: Union[str, bytes], filename: str, mode: str, TYPE_COMMENT_SYNTAX_ERROR = 'syntax error in type comment' # type: Final -TYPE_IGNORE_PATTERN = re.compile(r'#\s*type:\s*ignore') +TYPE_IGNORE_PATTERN = re.compile(r'\s*type:\s*ignore') # Older versions of typing don't allow using overload outside stubs, @@ -197,7 +197,16 @@ def parse_type_comment(type_comment: str, else: raise else: - extra_ignore = TYPE_IGNORE_PATTERN.search(type_comment) is not None + # We need to handle the case where an + # extra # type: ignore is in a comment, like this line. + # Therefore we split on comment delineations and check + # only the first possibility for a type ignore. + parts = type_comment.split('#') + if len(parts) > 1: + _, possible_ignore, *rest = parts + else: + possible_ignore = type_comment + extra_ignore = TYPE_IGNORE_PATTERN.search(possible_ignore) is not None assert isinstance(typ, ast3_Expression) return extra_ignore, TypeConverter(errors, line=line, assume_str_is_unicode=assume_str_is_unicode).visit(typ.body) diff --git a/test-data/unit/check-fastparse.test b/test-data/unit/check-fastparse.test index f32b1f0526bd..4815f1f45d87 100644 --- a/test-data/unit/check-fastparse.test +++ b/test-data/unit/check-fastparse.test @@ -114,6 +114,9 @@ with open('test', 'r') as f: # type: int # type: ignore for i in (1, 2, 3, 100): # type: str # type: ignore pass +[case testFastParseVariableCommentThenIgnore] +a="test" # type: int #comment # type: ignore # E: Incompatible types in assignment (expression has type "str", variable has type "int") + [case testFastParseProperty] class C: From d0cf678e3988d7386bb008f6f1266a7b22087317 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 26 Mar 2019 20:18:31 -0700 Subject: [PATCH 3/3] Use better regex to recognize extra type ignore, fix indents --- mypy/fastparse.py | 24 ++++++++---------------- mypy/fastparse2.py | 21 +++++++++++---------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index b5f8826f46ad..63a2ac1095ae 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -121,7 +121,7 @@ def ast3_parse(source: Union[str, bytes], filename: str, mode: str, TYPE_COMMENT_SYNTAX_ERROR = 'syntax error in type comment' # type: Final -TYPE_IGNORE_PATTERN = re.compile(r'\s*type:\s*ignore') +TYPE_IGNORE_PATTERN = re.compile(r'[^#]*#\s*type:\s*ignore\s*($|#)') # Older versions of typing don't allow using overload outside stubs, @@ -193,16 +193,7 @@ def parse_type_comment(type_comment: str, else: raise else: - # We need to handle the case where an - # extra # type: ignore is in a comment, like this line. - # Therefore we split on comment delineations and check - # only the first possibility for a type ignore. - parts = type_comment.split('#') - if len(parts) > 1: - _, possible_ignore, *rest = parts - else: - possible_ignore = type_comment - extra_ignore = TYPE_IGNORE_PATTERN.search(possible_ignore) is not None + extra_ignore = TYPE_IGNORE_PATTERN.match(type_comment) is not None assert isinstance(typ, ast3_Expression) return extra_ignore, TypeConverter(errors, line=line, assume_str_is_unicode=assume_str_is_unicode).visit(typ.body) @@ -226,7 +217,7 @@ def parse_type_string(expr_string: str, expr_fallback_name: str, """ try: _, node = parse_type_comment(expr_string.strip(), line=line, errors=None, - assume_str_is_unicode=assume_str_is_unicode) + assume_str_is_unicode=assume_str_is_unicode) if isinstance(node, UnboundType) and node.original_str_expr is None: node.original_str_expr = expr_string node.original_str_fallback = expr_fallback_name @@ -404,7 +395,8 @@ def translate_module_id(self, id: str) -> str: def visit_Module(self, mod: ast3.Module) -> MypyFile: body = self.fix_function_overloads(self.translate_stmt_list(mod.body)) - ignores = [ti.lineno for ti in mod.type_ignores] + self.extra_type_ignores + ignores = [ti.lineno for ti in mod.type_ignores] + ignores.extend(self.extra_type_ignores) return MypyFile(body, self.imports, False, @@ -1240,9 +1232,9 @@ def visit_raw_str(self, s: str) -> Type: # directly hook into the Python 3.5 type converter in some cases # without needing to create an intermediary `Str` object. _, typ = parse_type_comment(s.strip(), - self.line, - self.errors, - self.assume_str_is_unicode) + self.line, + self.errors, + self.assume_str_is_unicode) return typ or AnyType(TypeOfAny.from_error) def visit_Call(self, e: Call) -> Type: diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index eae229d12cb5..6865fd85a13a 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -303,7 +303,8 @@ def translate_module_id(self, id: str) -> str: def visit_Module(self, mod: ast27.Module) -> MypyFile: body = self.fix_function_overloads(self.translate_stmt_list(mod.body)) - ignores = [ti.lineno for ti in mod.type_ignores] + self.extra_type_ignores + ignores = [ti.lineno for ti in mod.type_ignores] + ignores.extend(self.extra_type_ignores) return MypyFile(body, self.imports, False, @@ -546,7 +547,7 @@ def visit_Assign(self, n: ast27.Assign) -> AssignmentStmt: typ = None if n.type_comment: extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors, - assume_str_is_unicode=self.unicode_literals) + assume_str_is_unicode=self.unicode_literals) if extra_ignore: self.extra_type_ignores.append(n.lineno) @@ -565,17 +566,17 @@ def visit_AugAssign(self, n: ast27.AugAssign) -> OperatorAssignmentStmt: # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) def visit_For(self, n: ast27.For) -> ForStmt: if n.type_comment is not None: - extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors, - assume_str_is_unicode=self.unicode_literals) + extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors, + assume_str_is_unicode=self.unicode_literals) if extra_ignore: self.extra_type_ignores.append(n.lineno) else: - target_type = None + typ = None stmt = ForStmt(self.visit(n.target), self.visit(n.iter), self.as_required_block(n.body, n.lineno), self.as_block(n.orelse, n.lineno), - target_type) + typ) return self.set_line(stmt, n) # While(expr test, stmt* body, stmt* orelse) @@ -595,16 +596,16 @@ def visit_If(self, n: ast27.If) -> IfStmt: # With(withitem* items, stmt* body, string? type_comment) def visit_With(self, n: ast27.With) -> WithStmt: if n.type_comment is not None: - extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors, - assume_str_is_unicode=self.unicode_literals) + extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors, + assume_str_is_unicode=self.unicode_literals) if extra_ignore: self.extra_type_ignores.append(n.lineno) else: - target_type = None + typ = None stmt = WithStmt([self.visit(n.context_expr)], [self.visit(n.optional_vars)], self.as_required_block(n.body, n.lineno), - target_type) + typ) return self.set_line(stmt, n) def visit_Raise(self, n: ast27.Raise) -> RaiseStmt: