Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

gh-99344, gh-99379, gh-99382: Fix issues in substitution of ParamSpec and TypeVarTuple #99412

Merged
merged 1 commit into from
Nov 29, 2022
Merged

gh-99344, gh-99379, gh-99382: Fix issues in substitution of ParamSpec and TypeVarTuple #99412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 81 additions & 0 deletions Lib/test/test_typing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -772,20 +772,42 @@ class C(Generic[*Ts]): pass
('generic[*Ts]', '[*Ts]', 'generic[*Ts]'),
('generic[*Ts]', '[T, *Ts]', 'generic[T, *Ts]'),
('generic[*Ts]', '[*Ts, T]', 'generic[*Ts, T]'),
('generic[T, *Ts]', '[()]', 'TypeError'),
('generic[T, *Ts]', '[int]', 'generic[int]'),
('generic[T, *Ts]', '[int, str]', 'generic[int, str]'),
('generic[T, *Ts]', '[int, str, bool]', 'generic[int, str, bool]'),
('generic[list[T], *Ts]', '[()]', 'TypeError'),
('generic[list[T], *Ts]', '[int]', 'generic[list[int]]'),
('generic[list[T], *Ts]', '[int, str]', 'generic[list[int], str]'),
('generic[list[T], *Ts]', '[int, str, bool]', 'generic[list[int], str, bool]'),

('generic[*Ts, T]', '[()]', 'TypeError'),
('generic[*Ts, T]', '[int]', 'generic[int]'),
('generic[*Ts, T]', '[int, str]', 'generic[int, str]'),
('generic[*Ts, T]', '[int, str, bool]', 'generic[int, str, bool]'),
('generic[*Ts, list[T]]', '[()]', 'TypeError'),
('generic[*Ts, list[T]]', '[int]', 'generic[list[int]]'),
('generic[*Ts, list[T]]', '[int, str]', 'generic[int, list[str]]'),
('generic[*Ts, list[T]]', '[int, str, bool]', 'generic[int, str, list[bool]]'),

('generic[T1, T2, *Ts]', '[()]', 'TypeError'),
('generic[T1, T2, *Ts]', '[int]', 'TypeError'),
('generic[T1, T2, *Ts]', '[int, str]', 'generic[int, str]'),
('generic[T1, T2, *Ts]', '[int, str, bool]', 'generic[int, str, bool]'),
('generic[T1, T2, *Ts]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'),

('generic[*Ts, T1, T2]', '[()]', 'TypeError'),
('generic[*Ts, T1, T2]', '[int]', 'TypeError'),
('generic[*Ts, T1, T2]', '[int, str]', 'generic[int, str]'),
('generic[*Ts, T1, T2]', '[int, str, bool]', 'generic[int, str, bool]'),
('generic[*Ts, T1, T2]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'),

('generic[T1, *Ts, T2]', '[()]', 'TypeError'),
('generic[T1, *Ts, T2]', '[int]', 'TypeError'),
('generic[T1, *Ts, T2]', '[int, str]', 'generic[int, str]'),
('generic[T1, *Ts, T2]', '[int, str, bool]', 'generic[int, str, bool]'),
('generic[T1, *Ts, T2]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'),

('generic[T, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...]]'),
('generic[T, *Ts]', '[str, *tuple_type[int, ...]]', 'generic[str, *tuple_type[int, ...]]'),
('generic[T, *Ts]', '[*tuple_type[int, ...], str]', 'generic[int, *tuple_type[int, ...], str]'),
Expand Down Expand Up @@ -7241,6 +7263,65 @@ class X(Generic[P, P2]):
self.assertEqual(G1.__args__, ((int, str), (bytes,)))
self.assertEqual(G2.__args__, ((int,), (str, bytes)))

def test_typevartuple_and_paramspecs_in_user_generics(self):
Ts = TypeVarTuple("Ts")
P = ParamSpec("P")

class X(Generic[*Ts, P]):
f: Callable[P, int]
g: Tuple[*Ts]

G1 = X[int, [bytes]]
self.assertEqual(G1.__args__, (int, (bytes,)))
G2 = X[int, str, [bytes]]
self.assertEqual(G2.__args__, (int, str, (bytes,)))
G3 = X[[bytes]]
self.assertEqual(G3.__args__, ((bytes,),))
G4 = X[[]]
self.assertEqual(G4.__args__, ((),))
with self.assertRaises(TypeError):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add tests for X[int] and Y[int]? (I guess those should raise TypeError.)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, unfortunately it does not raise TypeError. The type check for ParamSpec and TypeVar substitution are too lenient and accept any nonsense.

>>> from typing import *
>>> T = TypeVar('T')
>>> P = ParamSpec('P')
>>> class A(Generic[P, T]): pass
... 
>>> A[int, [str]]
__main__.A[int, [<class 'str'>]]
>>> A[1, 2]
__main__.A[1, 2]

More strict type checks can be introduced in a separate issue.

X[()]

class Y(Generic[P, *Ts]):
f: Callable[P, int]
g: Tuple[*Ts]

G1 = Y[[bytes], int]
self.assertEqual(G1.__args__, ((bytes,), int))
G2 = Y[[bytes], int, str]
self.assertEqual(G2.__args__, ((bytes,), int, str))
G3 = Y[[bytes]]
self.assertEqual(G3.__args__, ((bytes,),))
G4 = Y[[]]
self.assertEqual(G4.__args__, ((),))
with self.assertRaises(TypeError):
Y[()]

def test_typevartuple_and_paramspecs_in_generic_aliases(self):
P = ParamSpec('P')
T = TypeVar('T')
Ts = TypeVarTuple('Ts')

for C in Callable, collections.abc.Callable:
with self.subTest(generic=C):
A = C[P, Tuple[*Ts]]
B = A[[int, str], bytes, float]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wow, this is an interesting case. Would it be worth checking C[Tuple[*Ts], P] too?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

C is Callable. Callable[Tuple[*Ts], P] does not make sense. It is an error for collections.abc.Callable, and accepted for typing.Callable (producing a meaningless result), but it is a different issue.

self.assertEqual(B.__args__, (int, str, Tuple[bytes, float]))

class X(Generic[T, P]):
pass

A = X[Tuple[*Ts], P]
B = A[bytes, float, [int, str]]
self.assertEqual(B.__args__, (Tuple[bytes, float], (int, str,)))

class Y(Generic[P, T]):
pass

A = Y[P, Tuple[*Ts]]
B = A[[int, str], bytes, float]
self.assertEqual(B.__args__, ((int, str,), Tuple[bytes, float]))

def test_var_substitution(self):
T = TypeVar("T")
P = ParamSpec("P")
Expand Down
71 changes: 30 additions & 41 deletions Lib/typing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,25 +284,6 @@ def _unpack_args(args):
newargs.append(arg)
return newargs

def _prepare_paramspec_params(cls, params):
"""Prepares the parameters for a Generic containing ParamSpec
variables (internal helper).
"""
# Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612.
if (len(cls.__parameters__) == 1
and params and not _is_param_expr(params[0])):
assert isinstance(cls.__parameters__[0], ParamSpec)
return (params,)
else:
_check_generic(cls, params, len(cls.__parameters__))
_params = []
# Convert lists to tuples to help other libraries cache the results.
for p, tvar in zip(params, cls.__parameters__):
if isinstance(tvar, ParamSpec) and isinstance(p, list):
p = tuple(p)
_params.append(p)
return tuple(_params)

def _deduplicate(params):
# Weed out strict duplicates, preserving the first of each occurrence.
all_params = set(params)
Expand Down Expand Up @@ -1238,7 +1219,18 @@ def __typing_subst__(self, arg):
return arg

def __typing_prepare_subst__(self, alias, args):
return _prepare_paramspec_params(alias, args)
params = alias.__parameters__
i = params.index(self)
if i >= len(args):
raise TypeError(f"Too few arguments for {alias}")
# Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612.
if len(params) == 1 and not _is_param_expr(args[0]):
assert i == 0
args = (args,)
# Convert lists to tuples to help other libraries cache the results.
elif isinstance(args[i], list):
args = (*args[:i], tuple(args[i]), *args[i+1:])
return args

def _is_dunder(attr):
return attr.startswith('__') and attr.endswith('__')
Expand Down Expand Up @@ -1801,23 +1793,13 @@ def __class_getitem__(cls, params):
if not isinstance(params, tuple):
params = (params,)

if not params:
# We're only ok with `params` being empty if the class's only type
# parameter is a `TypeVarTuple` (which can contain zero types).
class_params = getattr(cls, "__parameters__", None)
only_class_parameter_is_typevartuple = (
class_params is not None
and len(class_params) == 1
and isinstance(class_params[0], TypeVarTuple)
)
if not only_class_parameter_is_typevartuple:
raise TypeError(
f"Parameter list to {cls.__qualname__}[...] cannot be empty"
)

params = tuple(_type_convert(p) for p in params)
if cls in (Generic, Protocol):
# Generic and Protocol can only be subscripted with unique type variables.
if not params:
raise TypeError(
f"Parameter list to {cls.__qualname__}[...] cannot be empty"
)
if not all(_is_typevar_like(p) for p in params):
raise TypeError(
f"Parameters to {cls.__name__}[...] must all be type variables "
Expand All @@ -1827,13 +1809,20 @@ def __class_getitem__(cls, params):
f"Parameters to {cls.__name__}[...] must all be unique")
else:
# Subscripting a regular Generic subclass.
if any(isinstance(t, ParamSpec) for t in cls.__parameters__):
params = _prepare_paramspec_params(cls, params)
elif not any(isinstance(p, TypeVarTuple) for p in cls.__parameters__):
# We only run this if there are no TypeVarTuples, because we
# don't check variadic generic arity at runtime (to reduce
# complexity of typing.py).
_check_generic(cls, params, len(cls.__parameters__))
for param in cls.__parameters__:
prepare = getattr(param, '__typing_prepare_subst__', None)
if prepare is not None:
params = prepare(cls, params)
_check_generic(cls, params, len(cls.__parameters__))

new_args = []
for param, new_arg in zip(cls.__parameters__, params):
if isinstance(param, TypeVarTuple):
new_args.extend(new_arg)
else:
new_args.append(new_arg)
params = tuple(new_args)

return _GenericAlias(cls, params,
_paramspec_tvars=True)

Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix substitution of :class:`~typing.TypeVarTuple` and
:class:`~typing.ParamSpec` together in user generics.
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix substitution of :class:`~typing.ParamSpec` followed by
:class:`~typing.TypeVarTuple` in generic aliases.
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Check the number of arguments in substitution in user generics containing a
:class:`~typing.TypeVarTuple` and one or more :class:`~typing.TypeVar`.