From 16bce9735a9238cafda1e97c89f80fe9f57b382f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 12 Jun 2022 16:22:01 +0300 Subject: [PATCH] gh-91162: Support splitting of unpacked arbitrary-length tuple over TypeVar and TypeVarTuple parameters (alt) (GH-93412) For example: A[T, *Ts][*tuple[int, ...]] -> A[int, *tuple[int, ...]] A[*Ts, T][*tuple[int, ...]] -> A[*tuple[int, ...], int] (cherry picked from commit 3473817106c23eca7341c931453da0341c367e1d) Co-authored-by: Serhiy Storchaka --- Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_runtime_init.h | 1 + Lib/test/test_typing.py | 17 +-- Lib/typing.py | 110 ++++++++---------- ...2-06-01-11-24-13.gh-issue-91162.NxvU_u.rst | 5 + Objects/genericaliasobject.c | 99 +++++++--------- 6 files changed, 106 insertions(+), 127 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-06-01-11-24-13.gh-issue-91162.NxvU_u.rst diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index cfa8ae99d1b6d9..ca970627cb2e17 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -202,6 +202,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__truediv__) STRUCT_FOR_ID(__trunc__) STRUCT_FOR_ID(__typing_is_unpacked_typevartuple__) + STRUCT_FOR_ID(__typing_prepare_subst__) STRUCT_FOR_ID(__typing_subst__) STRUCT_FOR_ID(__typing_unpacked_tuple_args__) STRUCT_FOR_ID(__warningregistry__) diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 57cacb97bcf1a8..13eae1e4c88d8f 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -825,6 +825,7 @@ extern "C" { INIT_ID(__truediv__), \ INIT_ID(__trunc__), \ INIT_ID(__typing_is_unpacked_typevartuple__), \ + INIT_ID(__typing_prepare_subst__), \ INIT_ID(__typing_subst__), \ INIT_ID(__typing_unpacked_tuple_args__), \ INIT_ID(__warningregistry__), \ diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index d6cd3d9bdd6a48..dfbe2d9d34c8ff 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -753,14 +753,11 @@ class C(Generic[*Ts]): pass ('generic[*Ts]', '[*tuple_type[int]]', 'generic[int]'), ('generic[*Ts]', '[*tuple_type[*Ts]]', 'generic[*Ts]'), ('generic[*Ts]', '[*tuple_type[int, str]]', 'generic[int, str]'), + ('generic[*Ts]', '[str, *tuple_type[int, ...], bool]', 'generic[str, *tuple_type[int, ...], bool]'), ('generic[*Ts]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'), ('generic[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'), ('generic[*Ts]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...]]'), - - # Technically, multiple unpackings are forbidden by PEP 646, but we - # choose to be less restrictive at runtime, to allow folks room - # to experiment. So all three of these should be valid. - ('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'generic[*tuple_type[int, ...], *tuple_type[str, ...]]'), + ('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'), ('generic[*Ts]', '[*Ts]', 'generic[*Ts]'), ('generic[*Ts]', '[T, *Ts]', 'generic[T, *Ts]'), @@ -772,8 +769,6 @@ class C(Generic[*Ts]): pass ('generic[list[T], *Ts]', '[int, str]', 'generic[list[int], str]'), ('generic[list[T], *Ts]', '[int, str, bool]', 'generic[list[int], str, bool]'), - ('generic[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be generic[int, *tuple[int, ...]] - ('generic[*Ts, T]', '[int]', 'generic[int]'), ('generic[*Ts, T]', '[int, str]', 'generic[int, str]'), ('generic[*Ts, T]', '[int, str, bool]', 'generic[int, str, bool]'), @@ -781,6 +776,14 @@ class C(Generic[*Ts]): pass ('generic[*Ts, list[T]]', '[int, str]', 'generic[int, list[str]]'), ('generic[*Ts, list[T]]', '[int, str, bool]', 'generic[int, str, list[bool]]'), + ('generic[T, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...]]'), + ('generic[*Ts, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], int]'), + ('generic[T1, *Ts, T2]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...], int]'), + ('generic[T, str, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, str, *tuple_type[int, ...]]'), + ('generic[*Ts, str, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], str, int]'), + ('generic[list[T], *Ts]', '[*tuple_type[int, ...]]', 'generic[list[int], *tuple_type[int, ...]]'), + ('generic[*Ts, list[T]]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], list[int]]'), + ('generic[T, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'), ('generic[T1, T2, *tuple_type[int, ...]]', '[str, bool]', 'generic[str, bool, *tuple_type[int, ...]]'), ('generic[T1, *tuple_type[int, ...], T2]', '[str, bool]', 'generic[str, *tuple_type[int, ...], bool]'), diff --git a/Lib/typing.py b/Lib/typing.py index 40ab516f7c8ff7..25cae7ffb8d788 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1065,6 +1065,42 @@ def __repr__(self): def __typing_subst__(self, arg): raise TypeError("Substitution of bare TypeVarTuple is not supported") + def __typing_prepare_subst__(self, alias, args): + params = alias.__parameters__ + typevartuple_index = params.index(self) + for param in enumerate(params[typevartuple_index + 1:]): + if isinstance(param, TypeVarTuple): + raise TypeError(f"More than one TypeVarTuple parameter in {alias}") + + alen = len(args) + plen = len(params) + left = typevartuple_index + right = plen - typevartuple_index - 1 + var_tuple_index = None + fillarg = None + for k, arg in enumerate(args): + if not (isinstance(arg, type) and not isinstance(arg, GenericAlias)): + subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) + if subargs and len(subargs) == 2 and subargs[-1] is ...: + if var_tuple_index is not None: + raise TypeError("More than one unpacked arbitrary-length tuple argument") + var_tuple_index = k + fillarg = subargs[0] + if var_tuple_index is not None: + left = min(left, var_tuple_index) + right = min(right, alen - var_tuple_index - 1) + elif left + right > alen: + raise TypeError(f"Too few arguments for {alias};" + f" actual {alen}, expected at least {plen-1}") + + return ( + *args[:left], + *([fillarg]*(typevartuple_index - left)), + tuple(args[left: alen - right]), + *([fillarg]*(plen - right - left - typevartuple_index - 1)), + *args[alen - right:], + ) + class ParamSpecArgs(_Final, _Immutable, _root=True): """The args for a ParamSpec object. @@ -1184,6 +1220,8 @@ def __typing_subst__(self, arg): f"ParamSpec, or Concatenate. Got {arg}") return arg + def __typing_prepare_subst__(self, alias, args): + return _prepare_paramspec_params(alias, args) def _is_dunder(attr): return attr.startswith('__') and attr.endswith('__') @@ -1255,44 +1293,6 @@ def __dir__(self): + [attr for attr in dir(self.__origin__) if not _is_dunder(attr)])) -def _is_unpacked_tuple(x: Any) -> bool: - # Is `x` something like `*tuple[int]` or `*tuple[int, ...]`? - if not isinstance(x, _UnpackGenericAlias): - return False - # Alright, `x` is `Unpack[something]`. - - # `x` will always have `__args__`, because Unpack[] and Unpack[()] - # aren't legal. - unpacked_type = x.__args__[0] - - return getattr(unpacked_type, '__origin__', None) is tuple - - -def _is_unpacked_arbitrary_length_tuple(x: Any) -> bool: - if not _is_unpacked_tuple(x): - return False - unpacked_tuple = x.__args__[0] - - if not hasattr(unpacked_tuple, '__args__'): - # It's `Unpack[tuple]`. We can't make any assumptions about the length - # of the tuple, so it's effectively an arbitrary-length tuple. - return True - - tuple_args = unpacked_tuple.__args__ - if not tuple_args: - # It's `Unpack[tuple[()]]`. - return False - - last_arg = tuple_args[-1] - if last_arg is Ellipsis: - # It's `Unpack[tuple[something, ...]]`, which is arbitrary-length. - return True - - # If the arguments didn't end with an ellipsis, then it's not an - # arbitrary-length tuple. - return False - - # Special typing constructs Union, Optional, Generic, Callable and Tuple # use three special attributes for internal bookkeeping of generic types: # * __parameters__ is a tuple of unique free type parameters of a generic @@ -1385,10 +1385,6 @@ def __getitem__(self, args): args = (args,) args = tuple(_type_convert(p) for p in args) args = _unpack_args(args) - if (self._paramspec_tvars - and any(isinstance(t, ParamSpec) for t in self.__parameters__)): - args = _prepare_paramspec_params(self, args) - new_args = self._determine_new_args(args) r = self.copy_with(new_args) return r @@ -1410,30 +1406,16 @@ def _determine_new_args(self, args): params = self.__parameters__ # In the example above, this would be {T3: str} - new_arg_by_param = {} - typevartuple_index = None - for i, param in enumerate(params): - if isinstance(param, TypeVarTuple): - if typevartuple_index is not None: - raise TypeError(f"More than one TypeVarTuple parameter in {self}") - typevartuple_index = i - + for param in params: + prepare = getattr(param, '__typing_prepare_subst__', None) + if prepare is not None: + args = prepare(self, args) alen = len(args) plen = len(params) - if typevartuple_index is not None: - i = typevartuple_index - j = alen - (plen - i - 1) - if j < i: - raise TypeError(f"Too few arguments for {self};" - f" actual {alen}, expected at least {plen-1}") - new_arg_by_param.update(zip(params[:i], args[:i])) - new_arg_by_param[params[i]] = tuple(args[i: j]) - new_arg_by_param.update(zip(params[i + 1:], args[j:])) - else: - if alen != plen: - raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};" - f" actual {alen}, expected {plen}") - new_arg_by_param.update(zip(params, args)) + if alen != plen: + raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};" + f" actual {alen}, expected {plen}") + new_arg_by_param = dict(zip(params, args)) new_args = [] for old_arg in self.__args__: diff --git a/Misc/NEWS.d/next/Library/2022-06-01-11-24-13.gh-issue-91162.NxvU_u.rst b/Misc/NEWS.d/next/Library/2022-06-01-11-24-13.gh-issue-91162.NxvU_u.rst new file mode 100644 index 00000000000000..09fa47c0d23840 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-06-01-11-24-13.gh-issue-91162.NxvU_u.rst @@ -0,0 +1,5 @@ +Support splitting of unpacked arbitrary-length tuple over ``TypeVar`` and +``TypeVarTuple`` parameters. For example: + +* ``A[T, *Ts][*tuple[int, ...]]`` -> ``A[int, *tuple[int, ...]]`` +* ``A[*Ts, T][*tuple[int, ...]]`` -> ``A[*tuple[int, ...], int]`` diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 59420816496f09..0a0d0cc4c15b68 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -269,7 +269,7 @@ _Py_make_parameters(PyObject *args) a non-empty tuple, return a new reference to obj. */ static PyObject * subs_tvars(PyObject *obj, PyObject *params, - PyObject **argitems, Py_ssize_t nargs, Py_ssize_t varparam) + PyObject **argitems, Py_ssize_t nargs) { PyObject *subparams; if (_PyObject_LookupAttr(obj, &_Py_ID(__parameters__), &subparams) < 0) { @@ -283,28 +283,28 @@ subs_tvars(PyObject *obj, PyObject *params, Py_DECREF(subparams); return NULL; } - for (Py_ssize_t i = 0, j = 0; i < nsubargs; ++i) { + Py_ssize_t j = 0; + for (Py_ssize_t i = 0; i < nsubargs; ++i) { PyObject *arg = PyTuple_GET_ITEM(subparams, i); Py_ssize_t iparam = tuple_index(params, nparams, arg); - if (iparam == varparam) { - j = tuple_extend(&subargs, j, - argitems + iparam, nargs - nparams + 1); - if (j < 0) { - return NULL; - } - } - else { - if (iparam >= 0) { - if (iparam > varparam) { - iparam += nargs - nparams; + if (iparam >= 0) { + PyObject *param = PyTuple_GET_ITEM(params, iparam); + arg = argitems[iparam]; + if (Py_TYPE(param)->tp_iter && PyTuple_Check(arg)) { // TypeVarTuple + j = tuple_extend(&subargs, j, + &PyTuple_GET_ITEM(arg, 0), + PyTuple_GET_SIZE(arg)); + if (j < 0) { + return NULL; } - arg = argitems[iparam]; + continue; } - Py_INCREF(arg); - PyTuple_SET_ITEM(subargs, j, arg); - j++; } + Py_INCREF(arg); + PyTuple_SET_ITEM(subargs, j, arg); + j++; } + assert(j == PyTuple_GET_SIZE(subargs)); obj = PyObject_GetItem(obj, subargs); @@ -409,39 +409,37 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje self); } item = _unpack_args(item); - int is_tuple = PyTuple_Check(item); - Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1; - PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item; - Py_ssize_t varparam = nparams; for (Py_ssize_t i = 0; i < nparams; i++) { PyObject *param = PyTuple_GET_ITEM(parameters, i); - if (Py_TYPE(param)->tp_iter) { // TypeVarTuple - if (varparam < nparams) { - Py_DECREF(item); - return PyErr_Format(PyExc_TypeError, - "More than one TypeVarTuple parameter in %S", - self); - } - varparam = i; - } - } - if (varparam < nparams) { - if (nitems < nparams - 1) { + PyObject *prepare, *tmp; + if (_PyObject_LookupAttr(param, &_Py_ID(__typing_prepare_subst__), &prepare) < 0) { Py_DECREF(item); - return PyErr_Format(PyExc_TypeError, - "Too few arguments for %R", - self); + return NULL; } - } - else { - if (nitems != nparams) { - Py_DECREF(item); - return PyErr_Format(PyExc_TypeError, - "Too %s arguments for %R; actual %zd, expected %zd", - nitems > nparams ? "many" : "few", - self, nitems, nparams); + if (prepare && prepare != Py_None) { + if (PyTuple_Check(item)) { + tmp = PyObject_CallFunction(prepare, "OO", self, item); + } + else { + tmp = PyObject_CallFunction(prepare, "O(O)", self, item); + } + Py_DECREF(prepare); + Py_SETREF(item, tmp); + if (item == NULL) { + return NULL; + } } } + int is_tuple = PyTuple_Check(item); + Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1; + PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item; + if (nitems != nparams) { + Py_DECREF(item); + return PyErr_Format(PyExc_TypeError, + "Too %s arguments for %R; actual %zd, expected %zd", + nitems > nparams ? "many" : "few", + self, nitems, nparams); + } /* Replace all type variables (specified by parameters) with corresponding values specified by argitems. t = list[T]; t[int] -> newargs = [int] @@ -471,22 +469,11 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje if (subst) { Py_ssize_t iparam = tuple_index(parameters, nparams, arg); assert(iparam >= 0); - if (iparam == varparam) { - Py_DECREF(subst); - Py_DECREF(newargs); - Py_DECREF(item); - PyErr_SetString(PyExc_TypeError, - "Substitution of bare TypeVarTuple is not supported"); - return NULL; - } - if (iparam > varparam) { - iparam += nitems - nparams; - } arg = PyObject_CallOneArg(subst, argitems[iparam]); Py_DECREF(subst); } else { - arg = subs_tvars(arg, parameters, argitems, nitems, varparam); + arg = subs_tvars(arg, parameters, argitems, nitems); } if (arg == NULL) { Py_DECREF(newargs);