[Python-checkins] bpo-43224: Implement substitution of unpacked TypeVarTuple in C (GH-31828)
serhiy-storchaka
webhook-mailer at python.org
Sat Apr 30 01:22:50 EDT 2022
https://github.com/python/cpython/commit/e8c2f72b94ae5dfba50c0f2e2c06b7ee975c8acb
commit: e8c2f72b94ae5dfba50c0f2e2c06b7ee975c8acb
branch: main
author: Serhiy Storchaka <storchaka at gmail.com>
committer: serhiy-storchaka <storchaka at gmail.com>
date: 2022-04-30T08:22:46+03:00
summary:
bpo-43224: Implement substitution of unpacked TypeVarTuple in C (GH-31828)
Co-authored-by: Matthew Rahtz <mrahtz at gmail.com>
files:
M Include/internal/pycore_global_strings.h
M Include/internal/pycore_runtime_init.h
M Lib/test/test_typing.py
M Lib/typing.py
M Objects/genericaliasobject.c
diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h
index cc94662256e87..28fffa95071a0 100644
--- a/Include/internal/pycore_global_strings.h
+++ b/Include/internal/pycore_global_strings.h
@@ -202,6 +202,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(__truediv__)
STRUCT_FOR_ID(__trunc__)
STRUCT_FOR_ID(__typing_subst__)
+ STRUCT_FOR_ID(__typing_unpacked__)
STRUCT_FOR_ID(__warningregistry__)
STRUCT_FOR_ID(__weakref__)
STRUCT_FOR_ID(__xor__)
diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h
index 9a3a9d04324ba..941badfc8cb6a 100644
--- a/Include/internal/pycore_runtime_init.h
+++ b/Include/internal/pycore_runtime_init.h
@@ -825,6 +825,7 @@ extern "C" {
INIT_ID(__truediv__), \
INIT_ID(__trunc__), \
INIT_ID(__typing_subst__), \
+ INIT_ID(__typing_unpacked__), \
INIT_ID(__warningregistry__), \
INIT_ID(__weakref__), \
INIT_ID(__xor__), \
diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index 412d117c238da..88be2850acb1c 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -754,89 +754,89 @@ class C(Generic[*Ts]): pass
tests = [
# Alias # Args # Expected result
('C[*Ts]', '[()]', 'C[()]'),
- ('tuple[*Ts]', '[()]', 'TypeError'), # Should be tuple[()]
+ ('tuple[*Ts]', '[()]', 'tuple[()]'),
('Tuple[*Ts]', '[()]', 'Tuple[()]'),
('C[*Ts]', '[int]', 'C[int]'),
- ('tuple[*Ts]', '[int]', 'tuple[(int,),]'), # Should be tuple[int]
+ ('tuple[*Ts]', '[int]', 'tuple[int]'),
('Tuple[*Ts]', '[int]', 'Tuple[int]'),
('C[*Ts]', '[int, str]', 'C[int, str]'),
- ('tuple[*Ts]', '[int, str]', 'TypeError'), # Should be tuple[int, str]
+ ('tuple[*Ts]', '[int, str]', 'tuple[int, str]'),
('Tuple[*Ts]', '[int, str]', 'Tuple[int, str]'),
('C[*Ts]', '[*tuple_type[int]]', 'C[*tuple_type[int]]'), # Should be C[int]
- ('tuple[*Ts]', '[*tuple_type[int]]', 'tuple[(*tuple_type[int],),]'), # Should be tuple[int]
+ ('tuple[*Ts]', '[*tuple_type[int]]', 'tuple[*tuple_type[int]]'), # Should be tuple[int]
('Tuple[*Ts]', '[*tuple_type[int]]', 'Tuple[*tuple_type[int]]'), # Should be Tuple[int]
('C[*Ts]', '[*tuple_type[*Ts]]', 'C[*tuple_type[*Ts]]'), # Should be C[*Ts]
- ('tuple[*Ts]', '[*tuple_type[*Ts]]', 'tuple[(*tuple_type[*Ts],),]'), # Should be tuple[*Ts]
+ ('tuple[*Ts]', '[*tuple_type[*Ts]]', 'tuple[*tuple_type[*Ts]]'), # Should be tuple[*Ts]
('Tuple[*Ts]', '[*tuple_type[*Ts]]', 'Tuple[*tuple_type[*Ts]]'), # Should be Tuple[*Ts]
('C[*Ts]', '[*tuple_type[int, str]]', 'C[*tuple_type[int, str]]'), # Should be C[int, str]
- ('tuple[*Ts]', '[*tuple_type[int, str]]', 'tuple[(*tuple_type[int, str],),]'), # Should be tuple[int, str]
+ ('tuple[*Ts]', '[*tuple_type[int, str]]', 'tuple[*tuple_type[int, str]]'), # Should be tuple[int, str]
('Tuple[*Ts]', '[*tuple_type[int, str]]', 'Tuple[*tuple_type[int, str]]'), # Should be Tuple[int, str]
('C[*Ts]', '[tuple_type[int, ...]]', 'C[tuple_type[int, ...]]'),
- ('tuple[*Ts]', '[tuple_type[int, ...]]', 'tuple[(tuple_type[int, ...],),]'), # Should be tuple[tuple_type[int, ...]]
+ ('tuple[*Ts]', '[tuple_type[int, ...]]', 'tuple[tuple_type[int, ...]]'),
('Tuple[*Ts]', '[tuple_type[int, ...]]', 'Tuple[tuple_type[int, ...]]'),
('C[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'C[tuple_type[int, ...], tuple_type[str, ...]]'),
- ('tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'TypeError'), # Should be tuple[tuple_type[int, ...], tuple_type[str, ...]]
+ ('tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'tuple[tuple_type[int, ...], tuple_type[str, ...]]'),
('Tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'Tuple[tuple_type[int, ...], tuple_type[str, ...]]'),
('C[*Ts]', '[*tuple_type[int, ...]]', 'C[*tuple_type[int, ...]]'),
- ('tuple[*Ts]', '[*tuple_type[int, ...]]', 'tuple[(*tuple_type[int, ...],),]'), # Should be tuple[*tuple_type[int, ...]]
+ ('tuple[*Ts]', '[*tuple_type[int, ...]]', 'tuple[*tuple_type[int, ...]]'),
('Tuple[*Ts]', '[*tuple_type[int, ...]]', 'Tuple[*tuple_type[int, ...]]'),
# Technically, multiple unpackings are forbidden by PEP 646, but we
# choose to be less restrictive at runtime, to allow folks room
# to experiment. So all three of these should be valid.
('C[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'C[*tuple_type[int, ...], *tuple_type[str, ...]]'),
- # Should be tuple[*tuple_type[int, ...], *tuple_type[str, ...]], to match the other two.
- ('tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'),
+ ('tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'tuple[*tuple_type[int, ...], *tuple_type[str, ...]]'),
('Tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'Tuple[*tuple_type[int, ...], *tuple_type[str, ...]]'),
('C[*Ts]', '[*Ts]', 'C[*Ts]'),
- ('tuple[*Ts]', '[*Ts]', 'tuple[(*Ts,),]'), # Should be tuple[*Ts]
+ ('tuple[*Ts]', '[*Ts]', 'tuple[*Ts]'),
('Tuple[*Ts]', '[*Ts]', 'Tuple[*Ts]'),
('C[*Ts]', '[T, *Ts]', 'C[T, *Ts]'),
- ('tuple[*Ts]', '[T, *Ts]', 'TypeError'), # Should be tuple[T, *Ts]
+ ('tuple[*Ts]', '[T, *Ts]', 'tuple[T, *Ts]'),
('Tuple[*Ts]', '[T, *Ts]', 'Tuple[T, *Ts]'),
('C[*Ts]', '[*Ts, T]', 'C[*Ts, T]'),
- ('tuple[*Ts]', '[*Ts, T]', 'TypeError'), # Should be tuple[*Ts, T]
+ ('tuple[*Ts]', '[*Ts, T]', 'tuple[*Ts, T]'),
('Tuple[*Ts]', '[*Ts, T]', 'Tuple[*Ts, T]'),
('C[T, *Ts]', '[int]', 'C[int]'),
- ('tuple[T, *Ts]', '[int]', 'TypeError'), # Should be tuple[int]
+ ('tuple[T, *Ts]', '[int]', 'tuple[int]'),
('Tuple[T, *Ts]', '[int]', 'Tuple[int]'),
('C[T, *Ts]', '[int, str]', 'C[int, str]'),
- ('tuple[T, *Ts]', '[int, str]', 'tuple[int, (str,)]'), # Should be tuple[int, str]
+ ('tuple[T, *Ts]', '[int, str]', 'tuple[int, str]'),
('Tuple[T, *Ts]', '[int, str]', 'Tuple[int, str]'),
('C[T, *Ts]', '[int, str, bool]', 'C[int, str, bool]'),
- ('tuple[T, *Ts]', '[int, str, bool]', 'TypeError'), # Should be tuple[int, str, bool]
+ ('tuple[T, *Ts]', '[int, str, bool]', 'tuple[int, str, bool]'),
('Tuple[T, *Ts]', '[int, str, bool]', 'Tuple[int, str, bool]'),
('C[T, *Ts]', '[*tuple[int, ...]]', 'C[*tuple[int, ...]]'), # Should be C[int, *tuple[int, ...]]
('C[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Ditto
- ('tuple[T, *Ts]', '[*tuple_type[int, ...]]', 'TypeError'), # Should be tuple[int, *tuple[int, ...]]
- ('Tuple[T, *Ts]', '[*tuple[int, ...]]', 'Tuple[*tuple[int, ...]]'), # Ditto
- ('Tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Ditto
+ ('tuple[T, *Ts]', '[*tuple[int, ...]]', 'tuple[*tuple[int, ...]]'), # Should be tuple[int, *tuple[int, ...]]
+ ('tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Should be tuple[int, *Tuple[int, ...]]
+ ('Tuple[T, *Ts]', '[*tuple[int, ...]]', 'Tuple[*tuple[int, ...]]'), # Should be Tuple[int, *tuple[int, ...]]
+ ('Tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Should be Tuple[int, *Tuple[int, ...]]
('C[*Ts, T]', '[int]', 'C[int]'),
- ('tuple[*Ts, T]', '[int]', 'TypeError'), # Should be tuple[int]
+ ('tuple[*Ts, T]', '[int]', 'tuple[int]'),
('Tuple[*Ts, T]', '[int]', 'Tuple[int]'),
('C[*Ts, T]', '[int, str]', 'C[int, str]'),
- ('tuple[*Ts, T]', '[int, str]', 'tuple[(int,), str]'), # Should be tuple[int, str]
+ ('tuple[*Ts, T]', '[int, str]', 'tuple[int, str]'),
('Tuple[*Ts, T]', '[int, str]', 'Tuple[int, str]'),
('C[*Ts, T]', '[int, str, bool]', 'C[int, str, bool]'),
- ('tuple[*Ts, T]', '[int, str, bool]', 'TypeError'), # Should be tuple[int, str, bool]
+ ('tuple[*Ts, T]', '[int, str, bool]', 'tuple[int, str, bool]'),
('Tuple[*Ts, T]', '[int, str, bool]', 'Tuple[int, str, bool]'),
('generic[T, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'),
@@ -945,7 +945,7 @@ def test_var_substitution(self):
T2 = TypeVar('T2')
class G(Generic[Unpack[Ts]]): pass
- for A in G, Tuple:
+ for A in G, Tuple, tuple:
B = A[Unpack[Ts]]
self.assertEqual(B[()], A[()])
self.assertEqual(B[float], A[float])
@@ -984,6 +984,21 @@ def test_bad_var_substitution(self):
T2 = TypeVar('T2')
class G(Generic[Unpack[Ts]]): pass
+ for A in G, Tuple, tuple:
+ B = A[Ts]
+ with self.assertRaises(TypeError):
+ B[int, str]
+
+ C = A[T, T2]
+ with self.assertRaises(TypeError):
+ C[Unpack[Ts]]
+
+ def test_repr_is_correct(self):
+ Ts = TypeVarTuple('Ts')
+ T = TypeVar('T')
+ T2 = TypeVar('T2')
+ class G(Generic[Unpack[Ts]]): pass
+
for A in G, Tuple:
B = A[T, Unpack[Ts], str, T2]
with self.assertRaises(TypeError):
diff --git a/Lib/typing.py b/Lib/typing.py
index b250f2992dfff..35deb32220eb8 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -1019,7 +1019,7 @@ def __repr__(self):
return self.__name__
def __typing_subst__(self, arg):
- raise AssertionError
+ raise TypeError("Substitution of bare TypeVarTuple is not supported")
class ParamSpecArgs(_Final, _Immutable, _root=True):
@@ -1686,11 +1686,15 @@ def __repr__(self):
return '*' + repr(self.__args__[0])
def __getitem__(self, args):
- if (len(self.__parameters__) == 1 and
- isinstance(self.__parameters__[0], TypeVarTuple)):
+ if self.__typing_unpacked__():
return args
return super().__getitem__(args)
+ def __typing_unpacked__(self):
+ # If x is Unpack[tuple[...]], __parameters__ will be empty.
+ return bool(self.__parameters__ and
+ isinstance(self.__parameters__[0], TypeVarTuple))
+
class Generic:
"""Abstract base class for generic types.
diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c
index 7059c4018303e..7b689912dffc1 100644
--- a/Objects/genericaliasobject.c
+++ b/Objects/genericaliasobject.c
@@ -190,6 +190,23 @@ tuple_add(PyObject *self, Py_ssize_t len, PyObject *item)
return 0;
}
+static Py_ssize_t
+tuple_extend(PyObject **dst, Py_ssize_t dstindex,
+ PyObject **src, Py_ssize_t count)
+{
+ assert(count >= 0);
+ if (_PyTuple_Resize(dst, PyTuple_GET_SIZE(*dst) + count - 1) != 0) {
+ return -1;
+ }
+ assert(dstindex + count <= PyTuple_GET_SIZE(*dst));
+ for (Py_ssize_t i = 0; i < count; ++i) {
+ PyObject *item = src[i];
+ Py_INCREF(item);
+ PyTuple_SET_ITEM(*dst, dstindex + i, item);
+ }
+ return dstindex + count;
+}
+
PyObject *
_Py_make_parameters(PyObject *args)
{
@@ -251,7 +268,8 @@ _Py_make_parameters(PyObject *args)
If obj doesn't have a __parameters__ attribute or that's not
a non-empty tuple, return a new reference to obj. */
static PyObject *
-subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems)
+subs_tvars(PyObject *obj, PyObject *params,
+ PyObject **argitems, Py_ssize_t nargs, Py_ssize_t varparam)
{
PyObject *subparams;
if (_PyObject_LookupAttr(obj, &_Py_ID(__parameters__), &subparams) < 0) {
@@ -265,14 +283,27 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems)
Py_DECREF(subparams);
return NULL;
}
- for (Py_ssize_t i = 0; i < nsubargs; ++i) {
+ for (Py_ssize_t i = 0, j = 0; i < nsubargs; ++i) {
PyObject *arg = PyTuple_GET_ITEM(subparams, i);
Py_ssize_t iparam = tuple_index(params, nparams, arg);
- if (iparam >= 0) {
- arg = argitems[iparam];
+ if (iparam == varparam) {
+ j = tuple_extend(&subargs, j,
+ argitems + iparam, nargs - nparams + 1);
+ if (j < 0) {
+ return NULL;
+ }
+ }
+ else {
+ if (iparam >= 0) {
+ if (iparam > varparam) {
+ iparam += nargs - nsubargs;
+ }
+ arg = argitems[iparam];
+ }
+ Py_INCREF(arg);
+ PyTuple_SET_ITEM(subargs, j, arg);
+ j++;
}
- Py_INCREF(arg);
- PyTuple_SET_ITEM(subargs, i, arg);
}
obj = PyObject_GetItem(obj, subargs);
@@ -286,6 +317,23 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems)
return obj;
}
+static int
+_is_unpacked_typevartuple(PyObject *arg)
+{
+ PyObject *meth;
+ int res = _PyObject_LookupAttr(arg, &_Py_ID(__typing_unpacked__), &meth);
+ if (res > 0) {
+ PyObject *tmp = PyObject_CallNoArgs(meth);
+ Py_DECREF(meth);
+ if (tmp == NULL) {
+ return -1;
+ }
+ res = PyObject_IsTrue(tmp);
+ Py_DECREF(tmp);
+ }
+ return res;
+}
+
PyObject *
_Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObject *item)
{
@@ -298,11 +346,27 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
int is_tuple = PyTuple_Check(item);
Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1;
PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item;
- if (nitems != nparams) {
- return PyErr_Format(PyExc_TypeError,
- "Too %s arguments for %R",
- nitems > nparams ? "many" : "few",
- self);
+ Py_ssize_t varparam = 0;
+ for (; varparam < nparams; varparam++) {
+ PyObject *param = PyTuple_GET_ITEM(parameters, varparam);
+ if (Py_TYPE(param)->tp_iter) { // TypeVarTuple
+ break;
+ }
+ }
+ if (varparam < nparams) {
+ if (nitems < nparams - 1) {
+ return PyErr_Format(PyExc_TypeError,
+ "Too few arguments for %R",
+ self);
+ }
+ }
+ else {
+ if (nitems != nparams) {
+ return PyErr_Format(PyExc_TypeError,
+ "Too %s arguments for %R",
+ nitems > nparams ? "many" : "few",
+ self);
+ }
}
/* Replace all type variables (specified by parameters)
with corresponding values specified by argitems.
@@ -315,8 +379,13 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
if (newargs == NULL) {
return NULL;
}
- for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) {
+ for (Py_ssize_t iarg = 0, jarg = 0; iarg < nargs; iarg++) {
PyObject *arg = PyTuple_GET_ITEM(args, iarg);
+ int unpack = _is_unpacked_typevartuple(arg);
+ if (unpack < 0) {
+ Py_DECREF(newargs);
+ return NULL;
+ }
PyObject *subst;
if (_PyObject_LookupAttr(arg, &_Py_ID(__typing_subst__), &subst) < 0) {
Py_DECREF(newargs);
@@ -325,17 +394,38 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
if (subst) {
Py_ssize_t iparam = tuple_index(parameters, nparams, arg);
assert(iparam >= 0);
+ if (iparam == varparam) {
+ Py_DECREF(subst);
+ Py_DECREF(newargs);
+ PyErr_SetString(PyExc_TypeError,
+ "Substitution of bare TypeVarTuple is not supported");
+ return NULL;
+ }
+ if (iparam > varparam) {
+ iparam += nitems - nparams;
+ }
arg = PyObject_CallOneArg(subst, argitems[iparam]);
Py_DECREF(subst);
}
else {
- arg = subs_tvars(arg, parameters, argitems);
+ arg = subs_tvars(arg, parameters, argitems, nitems, varparam);
}
if (arg == NULL) {
Py_DECREF(newargs);
return NULL;
}
- PyTuple_SET_ITEM(newargs, iarg, arg);
+ if (unpack) {
+ jarg = tuple_extend(&newargs, jarg,
+ &PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg));
+ Py_DECREF(arg);
+ if (jarg < 0) {
+ return NULL;
+ }
+ }
+ else {
+ PyTuple_SET_ITEM(newargs, jarg, arg);
+ jarg++;
+ }
}
return newargs;
More information about the Python-checkins
mailing list