From 148b6d9002d4606313acf1c5d2c2a420ddbc3716 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 16 May 2024 16:56:36 +0100 Subject: [PATCH 01/12] [WIP] Proof-of-concept scoping implementation for new type param syntax --- mypy/semanal.py | 52 ++++++++++++++++++++++------- test-data/unit/check-python312.test | 12 +++++++ 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index f92471c159de..3c998846d56f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -317,6 +317,14 @@ CORE_BUILTIN_CLASSES: Final = ["object", "bool", "function"] +# Python has these different scope/namespace kinds with subtly different semantics +SCOPE_GLOBAL: Final = 0 +SCOPE_CLASS: Final = 1 +SCOPE_FUNC: Final = 2 +SCOPE_COMPREHENSION: Final = 3 +SCOPE_TYPE_PARAM: Final = 4 + + # Used for tracking incomplete references Tag: _TypeAlias = int @@ -417,6 +425,7 @@ def __init__( errors: Report analysis errors using this instance """ self.locals = [None] + self.scope_stack = [SCOPE_GLOBAL] self.is_comprehension_stack = [False] # Saved namespaces from previous iteration. Every top-level function/method body is # analyzed in several iterations until all names are resolved. We need to save @@ -880,6 +889,7 @@ def analyze_func_def(self, defn: FuncDef) -> None: # Don't store not ready types (including placeholders). if self.found_incomplete_ref(tag) or has_placeholder(result): self.defer(defn) + # TODO: pop type args return assert isinstance(result, ProperType) if isinstance(result, CallableType): @@ -1645,6 +1655,8 @@ def push_type_args( ) -> list[tuple[str, TypeVarLikeExpr]] | None: if not type_args: return [] + self.locals.append(SymbolTable()) + self.scope_stack.append(SCOPE_TYPE_PARAM) tvs: list[tuple[str, TypeVarLikeExpr]] = [] for p in type_args: tv = self.analyze_type_param(p) @@ -1653,7 +1665,7 @@ def push_type_args( tvs.append((p.name, tv)) for name, tv in tvs: - self.add_symbol(name, tv, context, no_progress=True) + self.add_symbol(name, tv, context, no_progress=True, type_param=True) return tvs @@ -1701,9 +1713,8 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None: def pop_type_args(self, type_args: list[TypeParam] | None) -> None: if not type_args: return - for tv in type_args: - names = self.current_symbol_table() - del names[tv.name] + self.locals.pop() + self.scope_stack.pop() def analyze_class(self, defn: ClassDef) -> None: fullname = self.qualified_name(defn.name) @@ -1938,6 +1949,7 @@ def enter_class(self, info: TypeInfo) -> None: # Remember previous active class self.type_stack.append(self.type) self.locals.append(None) # Add class scope + self.scope_stack.append(SCOPE_CLASS) self.is_comprehension_stack.append(False) self.block_depth.append(-1) # The class body increments this to 0 self.loop_depth.append(0) @@ -1949,6 +1961,7 @@ def leave_class(self) -> None: self.block_depth.pop() self.loop_depth.pop() self.locals.pop() + self.scope_stack.pop() self.is_comprehension_stack.pop() self._type = self.type_stack.pop() self.missing_names.pop() @@ -6281,6 +6294,7 @@ def add_symbol( can_defer: bool = True, escape_comprehensions: bool = False, no_progress: bool = False, + type_param: bool = False, ) -> bool: """Add symbol to the currently active symbol table. @@ -6303,7 +6317,7 @@ def add_symbol( kind, node, module_public=module_public, module_hidden=module_hidden ) return self.add_symbol_table_node( - name, symbol, context, can_defer, escape_comprehensions, no_progress + name, symbol, context, can_defer, escape_comprehensions, no_progress, type_param ) def add_symbol_skip_local(self, name: str, node: SymbolNode) -> None: @@ -6336,6 +6350,7 @@ def add_symbol_table_node( can_defer: bool = True, escape_comprehensions: bool = False, no_progress: bool = False, + type_param: bool = False, ) -> bool: """Add symbol table node to the currently active symbol table. @@ -6355,7 +6370,7 @@ def add_symbol_table_node( can_defer: if True, defer current target if adding a placeholder context: error context (see above about None value) """ - names = self.current_symbol_table(escape_comprehensions=escape_comprehensions) + names = self.current_symbol_table(escape_comprehensions=escape_comprehensions, type_param=type_param) existing = names.get(name) if isinstance(symbol.node, PlaceholderNode) and can_defer: if context is not None: @@ -6673,6 +6688,7 @@ def enter( names = self.saved_locals.setdefault(function, SymbolTable()) self.locals.append(names) is_comprehension = isinstance(function, (GeneratorExpr, DictionaryComprehension)) + self.scope_stack.append(SCOPE_FUNC if not is_comprehension else SCOPE_COMPREHENSION) self.is_comprehension_stack.append(is_comprehension) self.global_decls.append(set()) self.nonlocal_decls.append(set()) @@ -6684,6 +6700,7 @@ def enter( yield finally: self.locals.pop() + self.scope_stack.pop() self.is_comprehension_stack.pop() self.global_decls.pop() self.nonlocal_decls.pop() @@ -6692,11 +6709,14 @@ def enter( self.missing_names.pop() def is_func_scope(self) -> bool: - return self.locals[-1] is not None + scope_type = self.scope_stack[-1] + if scope_type == SCOPE_TYPE_PARAM: + scope_type = self.scope_stack[-2] + return scope_type in (SCOPE_FUNC, SCOPE_COMPREHENSION) def is_nested_within_func_scope(self) -> bool: """Are we underneath a function scope, even if we are in a nested class also?""" - return any(l is not None for l in self.locals) + return any(s in (SCOPE_FUNC, SCOPE_COMPREHENSION) for s in self.scope_stack) def is_class_scope(self) -> bool: return self.type is not None and not self.is_func_scope() @@ -6713,9 +6733,17 @@ def current_symbol_kind(self) -> int: kind = GDEF return kind - def current_symbol_table(self, escape_comprehensions: bool = False) -> SymbolTable: - if self.is_func_scope(): - assert self.locals[-1] is not None + def current_symbol_table(self, escape_comprehensions: bool = False, type_param: bool = False) -> SymbolTable: + if type_param and self.scope_stack[-1] == SCOPE_TYPE_PARAM: + n = self.locals[-1] + assert n is not None + return n + elif self.is_func_scope(): + if self.scope_stack[-1] == SCOPE_TYPE_PARAM: + n = self.locals[-2] + else: + n = self.locals[-1] + assert n is not None if escape_comprehensions: assert len(self.locals) == len(self.is_comprehension_stack) # Retrieve the symbol table from the enclosing non-comprehension scope. @@ -6734,7 +6762,7 @@ def current_symbol_table(self, escape_comprehensions: bool = False) -> SymbolTab else: assert False, "Should have at least one non-comprehension scope" else: - names = self.locals[-1] + names = n assert names is not None elif self.type is not None: names = self.type.names diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 53656ae5e3fb..cbea6b8b6b61 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -958,3 +958,15 @@ def f[T](x: T) -> T: class C: def m[T](self, x: T) -> T: return unknown() # E: Name "unknown" is not defined + +[case testPEP695FunctionTypeVarAccessInFunction] +# mypy: enable-incomplete-feature=NewGenericSyntax +from typing import cast + +class C: + def m[T](self, x: T) -> T: + y: T = x + reveal_type(y) # N: Revealed type is "T`-1" + return cast(T, y) + +reveal_type(C().m(1)) # N: Revealed type is "builtins.int" From 0fc961bf73ee697208aa4962af78de39d4e2210e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 17 May 2024 10:58:41 +0100 Subject: [PATCH 02/12] Refactor is_comprehension_stack away --- mypy/semanal.py | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 3c998846d56f..003617c2ec38 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -317,12 +317,12 @@ CORE_BUILTIN_CLASSES: Final = ["object", "bool", "function"] -# Python has these different scope/namespace kinds with subtly different semantics -SCOPE_GLOBAL: Final = 0 -SCOPE_CLASS: Final = 1 -SCOPE_FUNC: Final = 2 -SCOPE_COMPREHENSION: Final = 3 -SCOPE_TYPE_PARAM: Final = 4 +# Python has several different scope/namespace kinds with subtly different semantics. +SCOPE_GLOBAL: Final = 0 # Module top level +SCOPE_CLASS: Final = 1 # Class body +SCOPE_FUNC: Final = 2 # Function or lambda +SCOPE_COMPREHENSION: Final = 3 # Comprehension or generator expression +SCOPE_TYPE_PARAM: Final = 4 # Python 3.12 new-style type parameter scope (PEP 695) # Used for tracking incomplete references @@ -350,8 +350,8 @@ class SemanticAnalyzer( nonlocal_decls: list[set[str]] # Local names of function scopes; None for non-function scopes. locals: list[SymbolTable | None] - # Whether each scope is a comprehension scope. - is_comprehension_stack: list[bool] + # Type of each scope (SCOPE_*, indexes match locals) + scope_stack: list[int] # Nested block depths of scopes block_depth: list[int] # TypeInfo of directly enclosing class (or None) @@ -426,7 +426,6 @@ def __init__( """ self.locals = [None] self.scope_stack = [SCOPE_GLOBAL] - self.is_comprehension_stack = [False] # Saved namespaces from previous iteration. Every top-level function/method body is # analyzed in several iterations until all names are resolved. We need to save # the local namespaces for the top level function and all nested functions between @@ -1950,7 +1949,6 @@ def enter_class(self, info: TypeInfo) -> None: self.type_stack.append(self.type) self.locals.append(None) # Add class scope self.scope_stack.append(SCOPE_CLASS) - self.is_comprehension_stack.append(False) self.block_depth.append(-1) # The class body increments this to 0 self.loop_depth.append(0) self._type = info @@ -1962,7 +1960,6 @@ def leave_class(self) -> None: self.loop_depth.pop() self.locals.pop() self.scope_stack.pop() - self.is_comprehension_stack.pop() self._type = self.type_stack.pop() self.missing_names.pop() @@ -2936,8 +2933,8 @@ class C: [(j := i) for i in [1, 2, 3]] is a syntax error that is not enforced by Python parser, but at later steps. """ - for i, is_comprehension in enumerate(reversed(self.is_comprehension_stack)): - if not is_comprehension and i < len(self.locals) - 1: + for i, scope_type in enumerate(reversed(self.scope_stack)): + if scope_type != SCOPE_COMPREHENSION and i < len(self.locals) - 1: if self.locals[-1 - i] is None: self.fail( "Assignment expression within a comprehension" @@ -5363,7 +5360,7 @@ def visit_star_expr(self, expr: StarExpr) -> None: def visit_yield_from_expr(self, e: YieldFromExpr) -> None: if not self.is_func_scope(): self.fail('"yield from" outside function', e, serious=True, blocker=True) - elif self.is_comprehension_stack[-1]: + elif self.scope_stack[-1] == SCOPE_COMPREHENSION: self.fail( '"yield from" inside comprehension or generator expression', e, @@ -5861,7 +5858,7 @@ def visit__promote_expr(self, expr: PromoteExpr) -> None: def visit_yield_expr(self, e: YieldExpr) -> None: if not self.is_func_scope(): self.fail('"yield" outside function', e, serious=True, blocker=True) - elif self.is_comprehension_stack[-1]: + elif self.scope_stack[-1] == SCOPE_COMPREHENSION: self.fail( '"yield" inside comprehension or generator expression', e, @@ -6370,7 +6367,9 @@ def add_symbol_table_node( can_defer: if True, defer current target if adding a placeholder context: error context (see above about None value) """ - names = self.current_symbol_table(escape_comprehensions=escape_comprehensions, type_param=type_param) + names = self.current_symbol_table( + escape_comprehensions=escape_comprehensions, type_param=type_param + ) existing = names.get(name) if isinstance(symbol.node, PlaceholderNode) and can_defer: if context is not None: @@ -6689,7 +6688,6 @@ def enter( self.locals.append(names) is_comprehension = isinstance(function, (GeneratorExpr, DictionaryComprehension)) self.scope_stack.append(SCOPE_FUNC if not is_comprehension else SCOPE_COMPREHENSION) - self.is_comprehension_stack.append(is_comprehension) self.global_decls.append(set()) self.nonlocal_decls.append(set()) # -1 since entering block will increment this to 0. @@ -6701,7 +6699,6 @@ def enter( finally: self.locals.pop() self.scope_stack.pop() - self.is_comprehension_stack.pop() self.global_decls.pop() self.nonlocal_decls.pop() self.block_depth.pop() @@ -6733,7 +6730,9 @@ def current_symbol_kind(self) -> int: kind = GDEF return kind - def current_symbol_table(self, escape_comprehensions: bool = False, type_param: bool = False) -> SymbolTable: + def current_symbol_table( + self, escape_comprehensions: bool = False, type_param: bool = False + ) -> SymbolTable: if type_param and self.scope_stack[-1] == SCOPE_TYPE_PARAM: n = self.locals[-1] assert n is not None @@ -6745,10 +6744,10 @@ def current_symbol_table(self, escape_comprehensions: bool = False, type_param: n = self.locals[-1] assert n is not None if escape_comprehensions: - assert len(self.locals) == len(self.is_comprehension_stack) + assert len(self.locals) == len(self.scope_stack) # Retrieve the symbol table from the enclosing non-comprehension scope. - for i, is_comprehension in enumerate(reversed(self.is_comprehension_stack)): - if not is_comprehension: + for i, scope_type in enumerate(reversed(self.scope_stack)): + if scope_type != SCOPE_COMPREHENSION: if i == len(self.locals) - 1: # The last iteration. # The caller of the comprehension is in the global space. names = self.globals From 3a546913132e78b73a9a82bbbaa0d339d47c963d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 17 May 2024 11:42:28 +0100 Subject: [PATCH 03/12] Add scoping test cases --- test-data/unit/check-python312.test | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index cbea6b8b6b61..cfa224a0f91d 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -970,3 +970,33 @@ class C: return cast(T, y) reveal_type(C().m(1)) # N: Revealed type is "builtins.int" + +[case testPEP695ScopingBasics] +# mypy: enable-incomplete-feature=NewGenericSyntax + +T = 1 + +def f[T](x: T) -> T: + T = 'a' + reveal_type(T) # N: Revealed type is "builtins.str" + return x + +reveal_type(T) # N: Revealed type is "builtins.int" + +class C[T]: + T = 1.2 + reveal_type(T) # N: Revealed type is "builtins.float" + +reveal_type(T) # N: Revealed type is "builtins.int" + +[case testPEP695ClassScoping] +# mypy: enable-incomplete-feature=NewGenericSyntax + +class C: + class D: pass + + def m[T: D](self, x: T, y: D) -> T: + return x + +C().m(C.D(), C.D()) +C().m(1, C.D()) # E: Value of type variable "T" of "m" of "C" cannot be "int" From 23ef8b098e39640e7642043bc0773857f25b2396 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 17 May 2024 12:10:24 +0100 Subject: [PATCH 04/12] Fix nonlocal --- mypy/semanal.py | 8 +++++++- test-data/unit/check-python312.test | 31 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 003617c2ec38..6386898f9d0b 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5198,8 +5198,14 @@ def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: self.fail("nonlocal declaration not allowed at module level", d) else: for name in d.names: - for table in reversed(self.locals[:-1]): + for table, scope_type in zip( + reversed(self.locals[:-1]), reversed(self.scope_stack[:-1]) + ): if table is not None and name in table: + if scope_type == SCOPE_TYPE_PARAM: + self.fail( + f'nonlocal binding not allowed for type parameter "{name}"', d + ) break else: self.fail(f'No binding for nonlocal "{name}" found', d) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index cfa224a0f91d..2ee9a9b4793f 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1000,3 +1000,34 @@ class C: C().m(C.D(), C.D()) C().m(1, C.D()) # E: Value of type variable "T" of "m" of "C" cannot be "int" + +[case testPEP695NestedGenericFunction] +# mypy: enable-incomplete-feature=NewGenericSyntax +def f[T](x: T) -> T: + def g[S](a: S) -> S: + return a + reveal_type(g(1)) # N: Revealed type is "builtins.int" + reveal_type(g(x)) # N: Revealed type is "T`-1" + + def h[S](a: S) -> S: + return a + reveal_type(h(1)) # N: Revealed type is "builtins.int" + reveal_type(h(x)) # N: Revealed type is "T`-1" + return x + +[case testPEP695NonLocal] +# mypy: enable-incomplete-feature=NewGenericSyntax +def f() -> None: + T = 1 + def g[T](x: T) -> T: + nonlocal T # E: nonlocal binding not allowed for type parameter "T" + T = 'x' # E: "T" is a type variable and only valid in type context + return x + reveal_type(T) # N: Revealed type is "builtins.int" + +def g() -> None: + a = 1 + def g[T](x: T) -> T: + nonlocal a + a = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + return x From 20a40578dfc61f7c767be163e8d7221b7b181d6a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 17 May 2024 12:38:12 +0100 Subject: [PATCH 05/12] Update test case --- test-data/unit/check-python312.test | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 2ee9a9b4793f..17bfeb28a6dd 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1004,7 +1004,16 @@ C().m(1, C.D()) # E: Value of type variable "T" of "m" of "C" cannot be "int" [case testPEP695NestedGenericFunction] # mypy: enable-incomplete-feature=NewGenericSyntax def f[T](x: T) -> T: + reveal_type(f(x)) # N: Revealed type is "T`-1" + reveal_type(f(1)) # N: Revealed type is "builtins.int" + + def ff(x: T) -> T: + return x + reveal_type(ff(x)) # N: Revealed type is "T`-1" + ff(1) # E: Argument 1 to "ff" has incompatible type "int"; expected "T" + def g[S](a: S) -> S: + ff(a) # E: Argument 1 to "ff" has incompatible type "S"; expected "T" return a reveal_type(g(1)) # N: Revealed type is "builtins.int" reveal_type(g(x)) # N: Revealed type is "T`-1" From b047363479e0c75a324814d7386c34cbdfe852a2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 17 May 2024 13:00:11 +0100 Subject: [PATCH 06/12] Fix error message when using unbound type parameter in default --- mypy/nodes.py | 10 +++++--- mypy/semanal.py | 8 ++++++- mypy/typeanal.py | 37 ++++++++++++++++++----------- test-data/unit/check-python312.test | 18 ++++++++++++++ 4 files changed, 55 insertions(+), 18 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 4c83d8081f6c..6657ab8cb65f 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2502,7 +2502,7 @@ class TypeVarLikeExpr(SymbolNode, Expression): Note that they are constructed by the semantic analyzer. """ - __slots__ = ("_name", "_fullname", "upper_bound", "default", "variance") + __slots__ = ("_name", "_fullname", "upper_bound", "default", "variance", "is_new_style") _name: str _fullname: str @@ -2525,6 +2525,7 @@ def __init__( upper_bound: mypy.types.Type, default: mypy.types.Type, variance: int = INVARIANT, + is_new_style: bool = False, ) -> None: super().__init__() self._name = name @@ -2532,6 +2533,7 @@ def __init__( self.upper_bound = upper_bound self.default = default self.variance = variance + self.is_new_style = is_new_style @property def name(self) -> str: @@ -2570,8 +2572,9 @@ def __init__( upper_bound: mypy.types.Type, default: mypy.types.Type, variance: int = INVARIANT, + is_new_style: bool = False, ) -> None: - super().__init__(name, fullname, upper_bound, default, variance) + super().__init__(name, fullname, upper_bound, default, variance, is_new_style) self.values = values def accept(self, visitor: ExpressionVisitor[T]) -> T: @@ -2648,8 +2651,9 @@ def __init__( tuple_fallback: mypy.types.Instance, default: mypy.types.Type, variance: int = INVARIANT, + is_new_style: bool = False, ) -> None: - super().__init__(name, fullname, upper_bound, default, variance) + super().__init__(name, fullname, upper_bound, default, variance, is_new_style) self.tuple_fallback = tuple_fallback def accept(self, visitor: ExpressionVisitor[T]) -> T: diff --git a/mypy/semanal.py b/mypy/semanal.py index 6386898f9d0b..fb337ce6f32f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1692,10 +1692,15 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None: upper_bound=upper_bound, default=default, variance=VARIANCE_NOT_READY, + is_new_style=True, ) elif type_param.kind == PARAM_SPEC_KIND: return ParamSpecExpr( - name=type_param.name, fullname=fullname, upper_bound=upper_bound, default=default + name=type_param.name, + fullname=fullname, + upper_bound=upper_bound, + default=default, + is_new_style=True, ) else: assert type_param.kind == TYPE_VAR_TUPLE_KIND @@ -1707,6 +1712,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None: upper_bound=tuple_fallback.copy_modified(), tuple_fallback=tuple_fallback, default=default, + is_new_style=True, ) def pop_type_args(self, type_args: list[TypeParam] | None) -> None: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 5cde7da721ec..31d451b0831a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -894,6 +894,7 @@ def analyze_unbound_type_without_type_info( t = t.copy_modified(args=self.anal_array(t.args)) # TODO: Move this message building logic to messages.py. notes: list[str] = [] + error_code = codes.VALID_TYPE if isinstance(sym.node, Var): notes.append( "See https://mypy.readthedocs.io/en/" @@ -912,25 +913,33 @@ def analyze_unbound_type_without_type_info( message = 'Module "{}" is not valid as a type' notes.append("Perhaps you meant to use a protocol matching the module structure?") elif unbound_tvar: - message = 'Type variable "{}" is unbound' - short = name.split(".")[-1] - notes.append( - ( - '(Hint: Use "Generic[{}]" or "Protocol[{}]" base class' - ' to bind "{}" inside a class)' - ).format(short, short, short) - ) - notes.append( - '(Hint: Use "{}" in function signature to bind "{}"' - " inside a function)".format(short, short) - ) + assert isinstance(sym.node, TypeVarLikeExpr) + if sym.node.is_new_style: + # PEP 695 type paramaters are never considered unbound -- they are undefined + # in contexts where they aren't valid, such as in argument default values. + message = 'Name "{}" is not defined' + name = name.split(".")[-1] + error_code = codes.NAME_DEFINED + else: + message = 'Type variable "{}" is unbound' + short = name.split(".")[-1] + notes.append( + ( + '(Hint: Use "Generic[{}]" or "Protocol[{}]" base class' + ' to bind "{}" inside a class)' + ).format(short, short, short) + ) + notes.append( + '(Hint: Use "{}" in function signature to bind "{}"' + " inside a function)".format(short, short) + ) else: message = 'Cannot interpret reference "{}" as a type' if not defining_literal: # Literal check already gives a custom error. Avoid duplicating errors. - self.fail(message.format(name), t, code=codes.VALID_TYPE) + self.fail(message.format(name), t, code=error_code) for note in notes: - self.note(note, t, code=codes.VALID_TYPE) + self.note(note, t, code=error_code) # TODO: Would it be better to always return Any instead of UnboundType # in case of an error? On one hand, UnboundType has a name so error messages diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 17bfeb28a6dd..685bbfb27315 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1040,3 +1040,21 @@ def g() -> None: nonlocal a a = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") return x + +[case testPEP695ArgumentDefault] +# mypy: enable-incomplete-feature=NewGenericSyntax +from typing import cast + +def f[T]( + x: T = + T # E: Name "T" is not defined \ + # E: Incompatible default for argument "x" (default has type "object", argument has type "T") +) -> T: + return x + +def g[T](x: T = cast(T, None)) -> T: # E: Name "T" is not defined + return x + +class C: + def m[T](self, x: T = cast(T, None)) -> T: # E: Name "T" is not defined + return x From dda6804728fd5ff285b9bf536f15dd3f39908bd4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 17 May 2024 13:15:43 +0100 Subject: [PATCH 07/12] Add test case --- test-data/unit/check-python312.test | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 685bbfb27315..c9f3c9e993fd 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1008,7 +1008,8 @@ def f[T](x: T) -> T: reveal_type(f(1)) # N: Revealed type is "builtins.int" def ff(x: T) -> T: - return x + y: T = x + return y reveal_type(ff(x)) # N: Revealed type is "T`-1" ff(1) # E: Argument 1 to "ff" has incompatible type "int"; expected "T" @@ -1058,3 +1059,12 @@ def g[T](x: T = cast(T, None)) -> T: # E: Name "T" is not defined class C: def m[T](self, x: T = cast(T, None)) -> T: # E: Name "T" is not defined return x + +[case testPEP695ListComprehension] +# mypy: enable-incomplete-feature=NewGenericSyntax +from typing import cast + +def f[T](x: T) -> T: + b = [cast(T, a) for a in [1, 2]] + reveal_type(b) # N: Revealed type is "builtins.list[T`-1]" + return x From 37f44886294ae1e924282a05462bc7afb315bdf3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 17 May 2024 13:37:09 +0100 Subject: [PATCH 08/12] More tests --- test-data/unit/check-python312.test | 38 ++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index c9f3c9e993fd..eb3b5c6cc3ec 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1025,7 +1025,7 @@ def f[T](x: T) -> T: reveal_type(h(x)) # N: Revealed type is "T`-1" return x -[case testPEP695NonLocal] +[case testPEP695NonLocalAndGlobal] # mypy: enable-incomplete-feature=NewGenericSyntax def f() -> None: T = 1 @@ -1042,6 +1042,19 @@ def g() -> None: a = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") return x +x = 1 + +def h[T](a: T) -> T: + global x + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + return a + +class C[T]: + def m[S](self, a: S) -> S: + global x + x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") + return a + [case testPEP695ArgumentDefault] # mypy: enable-incomplete-feature=NewGenericSyntax from typing import cast @@ -1068,3 +1081,26 @@ def f[T](x: T) -> T: b = [cast(T, a) for a in [1, 2]] reveal_type(b) # N: Revealed type is "builtins.list[T`-1]" return x + +[case testPEP695ReuseNameInSameScope] +# mypy: enable-incomplete-feature=NewGenericSyntax + +class C[T]: + def m[S](self, x: S, y: T) -> S | T: + return x + + def m2[S](self, x: S, y: T) -> S | T: + return x + +class D[T]: + pass + +def f[T](x: T) -> T: + return x + +def g[T](x: T) -> T: + def nested[S](y: S) -> S: + return y + def nested2[S](y: S) -> S: + return y + return x From b82c9cd4f4307951de2148e84aca898d5a8c1792 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 17 May 2024 13:56:37 +0100 Subject: [PATCH 09/12] Add test cases --- test-data/unit/check-python312.test | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index eb3b5c6cc3ec..97a6efbd64dc 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1104,3 +1104,36 @@ def g[T](x: T) -> T: def nested2[S](y: S) -> S: return y return x + +[case testPEP695NestedScopingSpecialCases] +# mypy: enable-incomplete-feature=NewGenericSyntax +# This is adapted from PEP 695 +S = 0 + +def outer1[S]() -> None: + S = 1 + T = 1 + + def outer2[T]() -> None: + def inner1() -> None: + nonlocal S + nonlocal T # E: nonlocal binding not allowed for type parameter "T" + + def inner2() -> None: + global S + +[case testPEP695ScopingWithBaseClasses] +# mypy: enable-incomplete-feature=NewGenericSyntax +# This is adapted from PEP 695 +class Outer: + class Private: + pass + + # If the type parameter scope was like a traditional scope, + # the base class 'Private' would not be accessible here. + class Inner[T](Private, list[T]): + pass + + # Likewise, 'Inner' would not be available in these type annotations. + def method1[T](self, a: Inner[T]) -> Inner[T]: + return a From 441e0437391f4ca640bd1c8d49dcc5df474c5c97 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 17 May 2024 14:08:23 +0100 Subject: [PATCH 10/12] Detect redefinitions of type parameters --- mypy/semanal.py | 15 ++++++++++++++- test-data/unit/check-python312.test | 12 ++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index fb337ce6f32f..1c90a3ea9b7a 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1664,10 +1664,23 @@ def push_type_args( tvs.append((p.name, tv)) for name, tv in tvs: - self.add_symbol(name, tv, context, no_progress=True, type_param=True) + if self.is_defined_type_param(name): + self.fail(f'"{name}" already defined as a type parameter', context) + else: + self.add_symbol(name, tv, context, no_progress=True, type_param=True) return tvs + def is_defined_type_param(self, name: str) -> bool: + for names in self.locals: + if names is None: + continue + if name in names: + node = names[name].node + if isinstance(node, TypeVarLikeExpr): + return True + return False + def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None: fullname = self.qualified_name(type_param.name) if type_param.upper_bound: diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 97a6efbd64dc..3090c520f46f 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1137,3 +1137,15 @@ class Outer: # Likewise, 'Inner' would not be available in these type annotations. def method1[T](self, a: Inner[T]) -> Inner[T]: return a + +[case testPEP695RedefineTypeParameterInScope] +# mypy: enable-incomplete-feature=NewGenericSyntax +class C[T]: + def m[T](self, x: T) -> T: # E: "T" already defined as a type parameter + return x + def m2(self) -> None: + def nested[T](x: T) -> T: # E: "T" already defined as a type parameter + return x + +def f[S, S](x: S) -> S: # E: "S" already defined as a type parameter + return x From 1717ac11a6dfb8592def260045769133c174e527 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 17 May 2024 14:25:43 +0100 Subject: [PATCH 11/12] Rename to SCOPE_ANNOTATION as suggested by Jelle --- mypy/semanal.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 1c90a3ea9b7a..74678fba1e9e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -322,7 +322,7 @@ SCOPE_CLASS: Final = 1 # Class body SCOPE_FUNC: Final = 2 # Function or lambda SCOPE_COMPREHENSION: Final = 3 # Comprehension or generator expression -SCOPE_TYPE_PARAM: Final = 4 # Python 3.12 new-style type parameter scope (PEP 695) +SCOPE_ANNOTATION: Final = 4 # Annotation scopes for type parameters and aliases (PEP 695) # Used for tracking incomplete references @@ -1655,7 +1655,7 @@ def push_type_args( if not type_args: return [] self.locals.append(SymbolTable()) - self.scope_stack.append(SCOPE_TYPE_PARAM) + self.scope_stack.append(SCOPE_ANNOTATION) tvs: list[tuple[str, TypeVarLikeExpr]] = [] for p in type_args: tv = self.analyze_type_param(p) @@ -5221,7 +5221,7 @@ def visit_nonlocal_decl(self, d: NonlocalDecl) -> None: reversed(self.locals[:-1]), reversed(self.scope_stack[:-1]) ): if table is not None and name in table: - if scope_type == SCOPE_TYPE_PARAM: + if scope_type == SCOPE_ANNOTATION: self.fail( f'nonlocal binding not allowed for type parameter "{name}"', d ) @@ -6732,7 +6732,7 @@ def enter( def is_func_scope(self) -> bool: scope_type = self.scope_stack[-1] - if scope_type == SCOPE_TYPE_PARAM: + if scope_type == SCOPE_ANNOTATION: scope_type = self.scope_stack[-2] return scope_type in (SCOPE_FUNC, SCOPE_COMPREHENSION) @@ -6758,12 +6758,12 @@ def current_symbol_kind(self) -> int: def current_symbol_table( self, escape_comprehensions: bool = False, type_param: bool = False ) -> SymbolTable: - if type_param and self.scope_stack[-1] == SCOPE_TYPE_PARAM: + if type_param and self.scope_stack[-1] == SCOPE_ANNOTATION: n = self.locals[-1] assert n is not None return n elif self.is_func_scope(): - if self.scope_stack[-1] == SCOPE_TYPE_PARAM: + if self.scope_stack[-1] == SCOPE_ANNOTATION: n = self.locals[-2] else: n = self.locals[-1] From 007f4a4c2ac073d4abc2362c2c502a245a9d7668 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 17 May 2024 14:33:54 +0100 Subject: [PATCH 12/12] Type parameters are not in scope in class decorators --- mypy/semanal.py | 10 ++++++++++ test-data/unit/check-python312.test | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index 74678fba1e9e..a66f43e17dd2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1814,8 +1814,18 @@ def analyze_class(self, defn: ClassDef) -> None: defn.info.is_protocol = is_protocol self.recalculate_metaclass(defn, declared_metaclass) defn.info.runtime_protocol = False + + if defn.type_args: + # PEP 695 type parameters are not in scope in class decorators, so + # temporarily disable type parameter namespace. + type_params_names = self.locals.pop() + self.scope_stack.pop() for decorator in defn.decorators: self.analyze_class_decorator(defn, decorator) + if defn.type_args: + self.locals.append(type_params_names) + self.scope_stack.append(SCOPE_ANNOTATION) + self.analyze_class_body_common(defn) def setup_type_vars(self, defn: ClassDef, tvar_defs: list[TypeVarLikeType]) -> None: diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 3090c520f46f..cce22634df6d 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1149,3 +1149,15 @@ class C[T]: def f[S, S](x: S) -> S: # E: "S" already defined as a type parameter return x + +[case testPEP695ClassDecorator] +# mypy: enable-incomplete-feature=NewGenericSyntax +from typing import Any + +T = 0 + +def decorator(x: str) -> Any: ... + +@decorator(T) # E: Argument 1 to "decorator" has incompatible type "int"; expected "str" +class C[T]: + pass