From 29abf398d6a9e88e899df8a1941019105821f9f0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 26 Aug 2023 21:30:13 +0100 Subject: [PATCH] Support PEP 646 syntax for Callable (#15951) Fixes https://github.com/python/mypy/issues/15412 Two new things here as specified by PEP 646: * Using star for an (explicit) type unpaking in callables, like `Callable[[str, *tuple[int, ...]], None]` * Allowing suffix items after a variadic item, like `Callable[[X, Unpack[Ys], Z], bool]` Implementation is straightforward. Btw while working in this I accidentally fixed a nasty bug, tuple types were often not given any line/column numbers, so if such type becomes a location of an error, it is impossible to ignore. --- mypy/exprtotype.py | 10 +++- mypy/fastparse.py | 14 ++++- mypy/typeanal.py | 73 +++++++++++++++++++------ test-data/unit/check-typevar-tuple.test | 58 +++++++++++++++----- 4 files changed, 123 insertions(+), 32 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index bbc284a5188a..b82d35607ef1 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -17,6 +17,7 @@ NameExpr, OpExpr, RefExpr, + StarExpr, StrExpr, TupleExpr, UnaryExpr, @@ -35,6 +36,7 @@ TypeOfAny, UnboundType, UnionType, + UnpackType, ) @@ -56,6 +58,7 @@ def expr_to_unanalyzed_type( options: Options | None = None, allow_new_syntax: bool = False, _parent: Expression | None = None, + allow_unpack: bool = False, ) -> ProperType: """Translate an expression to the corresponding type. @@ -163,7 +166,10 @@ def expr_to_unanalyzed_type( return CallableArgument(typ, name, arg_const, expr.line, expr.column) elif isinstance(expr, ListExpr): return TypeList( - [expr_to_unanalyzed_type(t, options, allow_new_syntax, expr) for t in expr.items], + [ + expr_to_unanalyzed_type(t, options, allow_new_syntax, expr, allow_unpack=True) + for t in expr.items + ], line=expr.line, column=expr.column, ) @@ -189,5 +195,7 @@ def expr_to_unanalyzed_type( return RawExpressionType(None, "builtins.complex", line=expr.line, column=expr.column) elif isinstance(expr, EllipsisExpr): return EllipsisType(expr.line) + elif allow_unpack and isinstance(expr, StarExpr): + return UnpackType(expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax)) else: raise TypeTranslationError() diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 3a26cfe7d6ff..6aa626afb81e 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -115,6 +115,7 @@ TypeOfAny, UnboundType, UnionType, + UnpackType, ) from mypy.util import bytes_to_human_readable_repr, unnamed_function @@ -1730,6 +1731,7 @@ def __init__( self.override_column = override_column self.node_stack: list[AST] = [] self.is_evaluated = is_evaluated + self.allow_unpack = False def convert_column(self, column: int) -> int: """Apply column override if defined; otherwise return column. @@ -2006,10 +2008,20 @@ def visit_Attribute(self, n: Attribute) -> Type: else: return self.invalid_type(n) + # Used for Callable[[X *Ys, Z], R] + def visit_Starred(self, n: ast3.Starred) -> Type: + return UnpackType(self.visit(n.value)) + # List(expr* elts, expr_context ctx) def visit_List(self, n: ast3.List) -> Type: assert isinstance(n.ctx, ast3.Load) - return self.translate_argument_list(n.elts) + old_allow_unpack = self.allow_unpack + # We specifically only allow starred expressions in a list to avoid + # confusing errors for top-level unpacks (e.g. in base classes). + self.allow_unpack = True + result = self.translate_argument_list(n.elts) + self.allow_unpack = old_allow_unpack + return result def stringify_name(n: AST) -> str | None: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index e29cca09be63..1955d2bc3c43 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -568,7 +568,9 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ instance = self.named_type("builtins.tuple", [self.anal_type(t.args[0])]) instance.line = t.line return instance - return self.tuple_type(self.anal_array(t.args, allow_unpack=True)) + return self.tuple_type( + self.anal_array(t.args, allow_unpack=True), line=t.line, column=t.column + ) elif fullname == "typing.Union": items = self.anal_array(t.args) return UnionType.make_union(items) @@ -968,7 +970,10 @@ def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: return t def visit_unpack_type(self, t: UnpackType) -> Type: - raise NotImplementedError + if not self.allow_unpack: + self.fail(message_registry.INVALID_UNPACK_POSITION, t.type, code=codes.VALID_TYPE) + return AnyType(TypeOfAny.from_error) + return UnpackType(self.anal_type(t.type)) def visit_parameters(self, t: Parameters) -> Type: raise NotImplementedError("ParamSpec literals cannot have unbound TypeVars") @@ -1364,12 +1369,22 @@ def analyze_callable_type(self, t: UnboundType) -> Type: assert isinstance(ret, CallableType) return ret.accept(self) + def refers_to_full_names(self, arg: UnboundType, names: Sequence[str]) -> bool: + sym = self.lookup_qualified(arg.name, arg) + if sym is not None: + if sym.fullname in names: + return True + return False + def analyze_callable_args( self, arglist: TypeList ) -> tuple[list[Type], list[ArgKind], list[str | None]] | None: args: list[Type] = [] kinds: list[ArgKind] = [] names: list[str | None] = [] + seen_unpack = False + unpack_types: list[Type] = [] + invalid_unpacks = [] for arg in arglist.items: if isinstance(arg, CallableArgument): args.append(arg.typ) @@ -1390,20 +1405,42 @@ def analyze_callable_args( if arg.name is not None and kind.is_star(): self.fail(f"{arg.constructor} arguments should not have names", arg) return None - elif isinstance(arg, UnboundType): - kind = ARG_POS - # Potentially a unpack. - sym = self.lookup_qualified(arg.name, arg) - if sym is not None: - if sym.fullname in ("typing_extensions.Unpack", "typing.Unpack"): - kind = ARG_STAR - args.append(arg) - kinds.append(kind) - names.append(None) + elif ( + isinstance(arg, UnboundType) + and self.refers_to_full_names(arg, ("typing_extensions.Unpack", "typing.Unpack")) + or isinstance(arg, UnpackType) + ): + if seen_unpack: + # Multiple unpacks, preserve them, so we can give an error later. + invalid_unpacks.append(arg) + continue + seen_unpack = True + unpack_types.append(arg) + else: + if seen_unpack: + unpack_types.append(arg) + else: + args.append(arg) + kinds.append(ARG_POS) + names.append(None) + if seen_unpack: + if len(unpack_types) == 1: + args.append(unpack_types[0]) else: - args.append(arg) - kinds.append(ARG_POS) - names.append(None) + first = unpack_types[0] + if isinstance(first, UnpackType): + # UnpackType doesn't have its own line/column numbers, + # so use the unpacked type for error messages. + first = first.type + args.append( + UnpackType(self.tuple_type(unpack_types, line=first.line, column=first.column)) + ) + kinds.append(ARG_STAR) + names.append(None) + for arg in invalid_unpacks: + args.append(arg) + kinds.append(ARG_STAR) + names.append(None) # Note that arglist below is only used for error context. check_arg_names(names, [arglist] * len(args), self.fail, "Callable") check_arg_kinds(kinds, [arglist] * len(args), self.fail) @@ -1713,9 +1750,11 @@ def check_unpacks_in_list(self, items: list[Type]) -> list[Type]: self.fail("More than one Unpack in a type is not allowed", final_unpack) return new_items - def tuple_type(self, items: list[Type]) -> TupleType: + def tuple_type(self, items: list[Type], line: int, column: int) -> TupleType: any_type = AnyType(TypeOfAny.special_form) - return TupleType(items, fallback=self.named_type("builtins.tuple", [any_type])) + return TupleType( + items, fallback=self.named_type("builtins.tuple", [any_type]), line=line, column=column + ) TypeVarLikeList = List[Tuple[str, TypeVarLikeExpr]] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index ee81597edadf..c7716f3e8346 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -509,6 +509,51 @@ call_prefix(target=func_prefix, args=(0, 'foo')) call_prefix(target=func2_prefix, args=(0, 'foo')) # E: Argument "target" to "call_prefix" has incompatible type "Callable[[str, int, str], None]"; expected "Callable[[bytes, int, str], None]" [builtins fixtures/tuple.pyi] +[case testTypeVarTuplePep646CallableSuffixSyntax] +from typing import Callable, Tuple, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +x: Callable[[str, Unpack[Tuple[int, ...]], bool], None] +reveal_type(x) # N: Revealed type is "def (builtins.str, *Unpack[Tuple[Unpack[builtins.tuple[builtins.int, ...]], builtins.bool]])" + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +A = Callable[[T, Unpack[Ts], S], int] +y: A[int, str, bool] +reveal_type(y) # N: Revealed type is "def (builtins.int, builtins.str, builtins.bool) -> builtins.int" +z: A[Unpack[Tuple[int, ...]]] +reveal_type(z) # N: Revealed type is "def (builtins.int, *Unpack[Tuple[Unpack[builtins.tuple[builtins.int, ...]], builtins.int]]) -> builtins.int" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646CallableInvalidSyntax] +from typing import Callable, Tuple, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") +Us = TypeVarTuple("Us") +a: Callable[[Unpack[Ts], Unpack[Us]], int] # E: Var args may not appear after named or var args \ + # E: More than one Unpack in a type is not allowed +reveal_type(a) # N: Revealed type is "def [Ts, Us] (*Unpack[Ts`-1]) -> builtins.int" +b: Callable[[Unpack], int] # E: Unpack[...] requires exactly one type argument +reveal_type(b) # N: Revealed type is "def (*Any) -> builtins.int" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTuplePep646CallableNewSyntax] +from typing import Callable, Generic, Tuple +from typing_extensions import ParamSpec + +x: Callable[[str, *Tuple[int, ...]], None] +reveal_type(x) # N: Revealed type is "def (builtins.str, *builtins.int)" +y: Callable[[str, *Tuple[int, ...], bool], None] +reveal_type(y) # N: Revealed type is "def (builtins.str, *Unpack[Tuple[Unpack[builtins.tuple[builtins.int, ...]], builtins.bool]])" + +P = ParamSpec("P") +class C(Generic[P]): ... +bad: C[[int, *Tuple[int, ...], int]] # E: Unpack is only valid in a variadic position +reveal_type(bad) # N: Revealed type is "__main__.C[[builtins.int, *Any]]" +[builtins fixtures/tuple.pyi] + [case testTypeVarTuplePep646UnspecifiedParameters] from typing import Tuple, Generic, TypeVar from typing_extensions import Unpack, TypeVarTuple @@ -635,19 +680,6 @@ x: A[str, str] reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.str, builtins.str]" [builtins fixtures/tuple.pyi] -[case testVariadicAliasWrongCallable] -from typing import TypeVar, Callable -from typing_extensions import Unpack, TypeVarTuple - -T = TypeVar("T") -S = TypeVar("S") -Ts = TypeVarTuple("Ts") - -A = Callable[[T, Unpack[Ts], S], int] # E: Required positional args may not appear after default, named or var args -x: A[int, str, int, str] -reveal_type(x) # N: Revealed type is "def (builtins.int, builtins.str, builtins.int, builtins.str) -> builtins.int" -[builtins fixtures/tuple.pyi] - [case testVariadicAliasMultipleUnpacks] from typing import Tuple, Generic, Callable from typing_extensions import Unpack, TypeVarTuple