Mailman 3 November 2023 - Python-checkins

gh-112509: Fix keys being present in both required_keys and optional_keys in TypedDict (#112512)
by JelleZijlstra Nov. 29, 2023

Nov. 29, 2023

https://github.com/python/cpython/commit/403886942376210662610627b01fea6acd… commit: 403886942376210662610627b01fea6acd77d331 branch: main author: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2023-11-29T09:36:48-08:00 summary: gh-112509: Fix keys being present in both required_keys and optional_keys in TypedDict (#112512) Co-authored-by: Alex Waygood <Alex.Waygood(a)Gmail.com> files: A Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst M Lib/test/test_typing.py M Lib/typing.py diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 669803177315e..4fbb410f26ab8 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7769,6 +7769,46 @@ class Cat(Animal): 'voice': str, }) + def test_keys_inheritance_with_same_name(self): + class NotTotal(TypedDict, total=False): + a: int + + class Total(NotTotal): + a: int + + self.assertEqual(NotTotal.__required_keys__, frozenset()) + self.assertEqual(NotTotal.__optional_keys__, frozenset(['a'])) + self.assertEqual(Total.__required_keys__, frozenset(['a'])) + self.assertEqual(Total.__optional_keys__, frozenset()) + + class Base(TypedDict): + a: NotRequired[int] + b: Required[int] + + class Child(Base): + a: Required[int] + b: NotRequired[int] + + self.assertEqual(Base.__required_keys__, frozenset(['b'])) + self.assertEqual(Base.__optional_keys__, frozenset(['a'])) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset(['b'])) + + def test_multiple_inheritance_with_same_key(self): + class Base1(TypedDict): + a: NotRequired[int] + + class Base2(TypedDict): + a: Required[str] + + class Child(Base1, Base2): + pass + + # Last base wins + self.assertEqual(Child.__annotations__, {'a': Required[str]}) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset()) + def test_required_notrequired_keys(self): self.assertEqual(NontotalMovie.__required_keys__, frozenset({"title"})) diff --git a/Lib/typing.py b/Lib/typing.py index 216f0c141b62a..b3af701f8d543 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2884,8 +2884,14 @@ def __new__(cls, name, bases, ns, total=True): for base in bases: annotations.update(base.__dict__.get('__annotations__', {})) - required_keys.update(base.__dict__.get('__required_keys__', ())) - optional_keys.update(base.__dict__.get('__optional_keys__', ())) + + base_required = base.__dict__.get('__required_keys__', set()) + required_keys |= base_required + optional_keys -= base_required + + base_optional = base.__dict__.get('__optional_keys__', set()) + required_keys -= base_optional + optional_keys |= base_optional annotations.update(own_annotations) for annotation_key, annotation_type in own_annotations.items(): @@ -2897,14 +2903,23 @@ def __new__(cls, name, bases, ns, total=True): annotation_origin = get_origin(annotation_type) if annotation_origin is Required: - required_keys.add(annotation_key) + is_required = True elif annotation_origin is NotRequired: - optional_keys.add(annotation_key) - elif total: + is_required = False + else: + is_required = total + + if is_required: required_keys.add(annotation_key) + optional_keys.discard(annotation_key) else: optional_keys.add(annotation_key) + required_keys.discard(annotation_key) + assert required_keys.isdisjoint(optional_keys), ( + f"Required keys overlap with optional keys in {name}:" + f" {required_keys=}, {optional_keys=}" + ) tp_dict.__annotations__ = annotations tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) diff --git a/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst b/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst new file mode 100644 index 0000000000000..a16d67e7776bc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst @@ -0,0 +1,3 @@ +Fix edge cases that could cause a key to be present in both the +``__required_keys__`` and ``__optional_keys__`` attributes of a +:class:`typing.TypedDict`. Patch by Jelle Zijlstra.

1 0

[3.11] gh-110930: Correct book title by Alan D. Moore (GH-112490) (#112524)
by hugovk Nov. 29, 2023

Nov. 29, 2023

https://github.com/python/cpython/commit/0a0cf45f1bae94b605e99b4f6581f120d7… commit: 0a0cf45f1bae94b605e99b4f6581f120d75ae7ad branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-11-29T15:56:20Z summary: [3.11] gh-110930: Correct book title by Alan D. Moore (GH-112490) (#112524) Co-authored-by: Hugo van Kemenade <hugovk(a)users.noreply.github.com> files: M Doc/library/tkinter.rst diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index aa78a0fe93d53d..5703b1fc050b1f 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -58,8 +58,8 @@ details that are unchanged. * `Modern Tkinter for Busy Python Developers <https://tkdocs.com/book.html>`_ By Mark Roseman. (ISBN 978-1999149567) - * `Python and Tkinter Programming <https://www.packtpub.com/product/python-gui-programming-with-tkinter/978178…>`_ - By Alan Moore. (ISBN 978-1788835886) + * `Python GUI programming with Tkinter <https://www.packtpub.com/product/python-gui-programming-with-tkinter/978178…>`_ + By Alan D. Moore. (ISBN 978-1788835886) * `Programming Python <https://learning-python.com/about-pp4e.html>`_ By Mark Lutz; has excellent coverage of Tkinter. (ISBN 978-0596158101)

1 0

[3.12] gh-110930: Correct book title by Alan D. Moore (GH-112490) (#112523)
by hugovk Nov. 29, 2023

Nov. 29, 2023

https://github.com/python/cpython/commit/01b882b39043f5c9f594c85b12e215df7c… commit: 01b882b39043f5c9f594c85b12e215df7c5af69c branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-11-29T15:55:46Z summary: [3.12] gh-110930: Correct book title by Alan D. Moore (GH-112490) (#112523) Co-authored-by: Hugo van Kemenade <hugovk(a)users.noreply.github.com> files: M Doc/library/tkinter.rst diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index aa78a0fe93d53d..5703b1fc050b1f 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -58,8 +58,8 @@ details that are unchanged. * `Modern Tkinter for Busy Python Developers <https://tkdocs.com/book.html>`_ By Mark Roseman. (ISBN 978-1999149567) - * `Python and Tkinter Programming <https://www.packtpub.com/product/python-gui-programming-with-tkinter/978178…>`_ - By Alan Moore. (ISBN 978-1788835886) + * `Python GUI programming with Tkinter <https://www.packtpub.com/product/python-gui-programming-with-tkinter/978178…>`_ + By Alan D. Moore. (ISBN 978-1788835886) * `Programming Python <https://learning-python.com/about-pp4e.html>`_ By Mark Lutz; has excellent coverage of Tkinter. (ISBN 978-0596158101)

1 0

Add more C API tests (GH-112522)
by serhiy-storchaka Nov. 29, 2023

Nov. 29, 2023

https://github.com/python/cpython/commit/e0449b9a7fffc0c0eed806bf4cbb8f1f65… commit: e0449b9a7fffc0c0eed806bf4cbb8f1f65397bbb branch: main author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2023-11-29T17:37:05+02:00 summary: Add more C API tests (GH-112522) Add tests for PyObject_Str(), PyObject_Repr(), PyObject_ASCII() and PyObject_Bytes(). files: M Lib/test/test_capi/test_abstract.py M Modules/_testcapi/abstract.c diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index 26152c3049848c..97ed939928c360 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -8,6 +8,30 @@ NULL = None +class StrSubclass(str): + pass + +class BytesSubclass(bytes): + pass + +class WithStr: + def __init__(self, value): + self.value = value + def __str__(self): + return self.value + +class WithRepr: + def __init__(self, value): + self.value = value + def __repr__(self): + return self.value + +class WithBytes: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + class TestObject: @property def evil(self): @@ -44,6 +68,68 @@ def gen(): class CAPITest(unittest.TestCase): + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + + def test_object_str(self): + # Test PyObject_Str() + object_str = _testcapi.object_str + self.assertTypedEqual(object_str(''), '') + self.assertTypedEqual(object_str('abc'), 'abc') + self.assertTypedEqual(object_str('\U0001f40d'), '\U0001f40d') + self.assertTypedEqual(object_str(StrSubclass('abc')), 'abc') + self.assertTypedEqual(object_str(WithStr('abc')), 'abc') + self.assertTypedEqual(object_str(WithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(object_str(WithRepr('<abc>')), '<abc>') + self.assertTypedEqual(object_str(WithRepr(StrSubclass('<abc>'))), StrSubclass('<abc>')) + self.assertTypedEqual(object_str(NULL), '<NULL>') + + def test_object_repr(self): + # Test PyObject_Repr() + object_repr = _testcapi.object_repr + self.assertTypedEqual(object_repr(''), "''") + self.assertTypedEqual(object_repr('abc'), "'abc'") + self.assertTypedEqual(object_repr('\U0001f40d'), "'\U0001f40d'") + self.assertTypedEqual(object_repr(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(object_repr(WithRepr('<abc>')), '<abc>') + self.assertTypedEqual(object_repr(WithRepr(StrSubclass('<abc>'))), StrSubclass('<abc>')) + self.assertTypedEqual(object_repr(WithRepr('<\U0001f40d>')), '<\U0001f40d>') + self.assertTypedEqual(object_repr(WithRepr(StrSubclass('<\U0001f40d>'))), StrSubclass('<\U0001f40d>')) + self.assertTypedEqual(object_repr(NULL), '<NULL>') + + def test_object_ascii(self): + # Test PyObject_ASCII() + object_ascii = _testcapi.object_ascii + self.assertTypedEqual(object_ascii(''), "''") + self.assertTypedEqual(object_ascii('abc'), "'abc'") + self.assertTypedEqual(object_ascii('\U0001f40d'), r"'\U0001f40d'") + self.assertTypedEqual(object_ascii(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(object_ascii(WithRepr('<abc>')), '<abc>') + self.assertTypedEqual(object_ascii(WithRepr(StrSubclass('<abc>'))), StrSubclass('<abc>')) + self.assertTypedEqual(object_ascii(WithRepr('<\U0001f40d>')), r'<\U0001f40d>') + self.assertTypedEqual(object_ascii(WithRepr(StrSubclass('<\U0001f40d>'))), r'<\U0001f40d>') + self.assertTypedEqual(object_ascii(NULL), '<NULL>') + + def test_object_bytes(self): + # Test PyObject_Bytes() + object_bytes = _testcapi.object_bytes + self.assertTypedEqual(object_bytes(b''), b'') + self.assertTypedEqual(object_bytes(b'abc'), b'abc') + self.assertTypedEqual(object_bytes(BytesSubclass(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(WithBytes(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(WithBytes(BytesSubclass(b'abc'))), BytesSubclass(b'abc')) + self.assertTypedEqual(object_bytes(bytearray(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(memoryview(b'abc')), b'abc') + self.assertTypedEqual(object_bytes([97, 98, 99]), b'abc') + self.assertTypedEqual(object_bytes((97, 98, 99)), b'abc') + self.assertTypedEqual(object_bytes(iter([97, 98, 99])), b'abc') + self.assertRaises(TypeError, object_bytes, WithBytes(bytearray(b'abc'))) + self.assertRaises(TypeError, object_bytes, WithBytes([97, 98, 99])) + self.assertRaises(TypeError, object_bytes, 3) + self.assertRaises(TypeError, object_bytes, 'abc') + self.assertRaises(TypeError, object_bytes, object()) + self.assertTypedEqual(object_bytes(NULL), b'<NULL>') def test_object_getattr(self): xgetattr = _testcapi.object_getattr diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c index 4a9144e66f0fcd..a8ba009eb6a54b 100644 --- a/Modules/_testcapi/abstract.c +++ b/Modules/_testcapi/abstract.c @@ -2,6 +2,34 @@ #include "util.h" +static PyObject * +object_repr(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Repr(arg); +} + +static PyObject * +object_ascii(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_ASCII(arg); +} + +static PyObject * +object_str(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Str(arg); +} + +static PyObject * +object_bytes(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Bytes(arg); +} + static PyObject * object_getattr(PyObject *self, PyObject *args) { @@ -616,6 +644,11 @@ sequence_tuple(PyObject *self, PyObject *obj) static PyMethodDef test_methods[] = { + {"object_repr", object_repr, METH_O}, + {"object_ascii", object_ascii, METH_O}, + {"object_str", object_str, METH_O}, + {"object_bytes", object_bytes, METH_O}, + {"object_getattr", object_getattr, METH_VARARGS}, {"object_getattrstring", object_getattrstring, METH_VARARGS}, {"object_getoptionalattr", object_getoptionalattr, METH_VARARGS},

1 0

[Enum] update class creation for RuntimeError changes (GH-111815)
by ethanfurman Nov. 29, 2023

Nov. 29, 2023

https://github.com/python/cpython/commit/f9e6ce03953e9ee988d55324dc715b0ef2… commit: f9e6ce03953e9ee988d55324dc715b0ef2303cfb branch: main author: Ethan Furman <ethan(a)stoneleaf.us> committer: ethanfurman <ethan(a)stoneleaf.us> date: 2023-11-28T20:40:12-08:00 summary: [Enum] update class creation for RuntimeError changes (GH-111815) files: M Doc/howto/enum.rst M Lib/enum.py diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index a136c76303c8ef..ffdafb749c73a9 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -1439,7 +1439,6 @@ alias:: Traceback (most recent call last): ... ValueError: aliases not allowed in DuplicateFreeEnum: 'GRENE' --> 'GREEN' - Error calling __set_name__ on '_proto_member' instance 'GRENE' in 'Color' .. note:: diff --git a/Lib/enum.py b/Lib/enum.py index 4e76e96d2c916e..648401e80be685 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -568,12 +568,16 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k try: exc = None enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) - except RuntimeError as e: - # any exceptions raised by member.__new__ will get converted to a - # RuntimeError, so get that original exception back and raise it instead - exc = e.__cause__ or e + except Exception as e: + # since 3.12 the line "Error calling __set_name__ on '_proto_member' instance ..." + # is tacked on to the error instead of raising a RuntimeError + # recreate the exception to discard + exc = type(e)(str(e)) + exc.__cause__ = e.__cause__ + exc.__context__ = e.__context__ + tb = e.__traceback__ if exc is not None: - raise exc + raise exc.with_traceback(tb) # # update classdict with any changes made by __init_subclass__ classdict.update(enum_class.__dict__)

1 0

Rename ...Uop... to ...UOp... (uppercase O) for consistency (#112327)
by gvanrossum Nov. 29, 2023

Nov. 29, 2023

https://github.com/python/cpython/commit/e723700190ba497d1601cb423ee48d5d22… commit: e723700190ba497d1601cb423ee48d5d222a9d26 branch: main author: Guido van Rossum <guido(a)python.org> committer: gvanrossum <gvanrossum(a)gmail.com> date: 2023-11-28T17:10:11-08:00 summary: Rename ...Uop... to ...UOp... (uppercase O) for consistency (#112327) * Rename _PyUopExecute to _PyUOpExecute (uppercase O) for consistency * Also rename _PyUopName and _PyUOp_Replacements, and some output strings files: M Include/internal/pycore_uops.h M Python/bytecodes.c M Python/ceval.c M Python/generated_cases.c.h M Python/optimizer.c diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index e2b94894681f44..ea8f90bf8c1d8f 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -24,7 +24,7 @@ typedef struct { _PyUOpInstruction trace[1]; } _PyUOpExecutorObject; -_PyInterpreterFrame *_PyUopExecute( +_PyInterpreterFrame *_PyUOpExecute( _PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a1ca66e457e47d..2e5f6c8d0d10e2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2357,7 +2357,7 @@ dummy_func( JUMPBY(1-original_oparg); frame->instr_ptr = next_instr; Py_INCREF(executor); - if (executor->execute == _PyUopExecute) { + if (executor->execute == _PyUOpExecute) { current_executor = (_PyUOpExecutorObject *)executor; GOTO_TIER_TWO(); } diff --git a/Python/ceval.c b/Python/ceval.c index def75fd114cb86..1806ceb7fa9681 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -647,7 +647,7 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { extern const struct _PyCode_DEF(8) _Py_InitCleanup; -extern const char *_PyUopName(int index); +extern const char *_PyUOpName(int index); /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ @@ -1002,7 +1002,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DPRINTF(3, "%4d: uop %s, oparg %d, operand %" PRIu64 ", target %d, stack_level %d\n", (int)(next_uop - current_executor->trace), - _PyUopName(uopcode), + _PyUOpName(uopcode), next_uop->oparg, next_uop->operand, next_uop->target, @@ -1051,8 +1051,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int pop_1_error_tier_two: STACK_SHRINK(1); error_tier_two: - DPRINTF(2, "Error: [Uop %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n", - uopcode, _PyUopName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, + DPRINTF(2, "Error: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n", + uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, (int)(next_uop - current_executor->trace - 1)); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); frame->return_offset = 0; // Don't leave this random @@ -1064,8 +1064,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int deoptimize: // On DEOPT_IF we just repeat the last instruction. // This presumes nothing was popped from the stack (nor pushed). - DPRINTF(2, "DEOPT: [Uop %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n", - uopcode, _PyUopName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, + DPRINTF(2, "DEOPT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n", + uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, (int)(next_uop - current_executor->trace - 1)); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dedd793111b7ff..0ac99e759deb12 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2331,7 +2331,7 @@ JUMPBY(1-original_oparg); frame->instr_ptr = next_instr; Py_INCREF(executor); - if (executor->execute == _PyUopExecute) { + if (executor->execute == _PyUOpExecute) { current_executor = (_PyUOpExecutorObject *)executor; GOTO_TIER_TWO(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index ec59fea26fbc70..dd24fbebbfd2a9 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -325,7 +325,7 @@ uop_dealloc(_PyUOpExecutorObject *self) { } const char * -_PyUopName(int index) +_PyUOpName(int index) { if (index <= MAX_REAL_OPCODE) { return _PyOpcode_OpName[index]; @@ -347,7 +347,7 @@ uop_item(_PyUOpExecutorObject *self, Py_ssize_t index) PyErr_SetNone(PyExc_IndexError); return NULL; } - const char *name = _PyUopName(self->trace[index].opcode); + const char *name = _PyUOpName(self->trace[index].opcode); if (name == NULL) { name = "<nil>"; } @@ -388,7 +388,7 @@ PyTypeObject _PyUOpExecutor_Type = { /* TO DO -- Generate these tables */ static const uint16_t -_PyUop_Replacements[OPCODE_METADATA_SIZE] = { +_PyUOp_Replacements[OPCODE_METADATA_SIZE] = { [_ITER_JUMP_RANGE] = _GUARD_NOT_EXHAUSTED_RANGE, [_ITER_JUMP_LIST] = _GUARD_NOT_EXHAUSTED_LIST, [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE, @@ -451,7 +451,7 @@ translate_bytecode_to_trace( #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ DPRINTF(2, \ " ADD_TO_TRACE(%s, %d, %" PRIu64 ")\n", \ - _PyUopName(OPCODE), \ + _PyUOpName(OPCODE), \ (OPARG), \ (uint64_t)(OPERAND)); \ assert(trace_length < max_length); \ @@ -474,7 +474,7 @@ translate_bytecode_to_trace( } // Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE -#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUopName(opcode)) +#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) // Trace stack operations (used by _PUSH_FRAME, _POP_FRAME) #define TRACE_STACK_PUSH() \ @@ -546,8 +546,8 @@ translate_bytecode_to_trace( uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; DPRINTF(4, "%s(%d): counter=%x, bitcount=%d, likely=%d, uopcode=%s\n", - _PyUopName(opcode), oparg, - counter, bitcount, jump_likely, _PyUopName(uopcode)); + _PyUOpName(opcode), oparg, + counter, bitcount, jump_likely, _PyUOpName(uopcode)); ADD_TO_TRACE(uopcode, max_length, 0, target); if (jump_likely) { _Py_CODEUNIT *target_instr = next_instr + oparg; @@ -615,8 +615,8 @@ translate_bytecode_to_trace( oparg += extras; } } - if (_PyUop_Replacements[uop]) { - uop = _PyUop_Replacements[uop]; + if (_PyUOp_Replacements[uop]) { + uop = _PyUOp_Replacements[uop]; if (uop == _FOR_ITER_TIER_TWO) { target += 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; assert(_PyCode_CODE(code)[target-1].op.code == END_FOR || @@ -712,7 +712,7 @@ translate_bytecode_to_trace( } break; } - DPRINTF(2, "Unsupported opcode %s\n", _PyUopName(opcode)); + DPRINTF(2, "Unsupported opcode %s\n", _PyUOpName(opcode)); OPT_UNSUPPORTED_OPCODE(opcode); goto done; // Break out of loop } // End default @@ -832,7 +832,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) dest--; } assert(dest == -1); - executor->base.execute = _PyUopExecute; + executor->base.execute = _PyUOpExecute; _Py_ExecutorInit((_PyExecutorObject *)executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -845,7 +845,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) for (int i = 0; i < length; i++) { printf("%4d %s(%d, %d, %" PRIu64 ")\n", i, - _PyUopName(executor->trace[i].opcode), + _PyUOpName(executor->trace[i].opcode), executor->trace[i].oparg, executor->trace[i].target, executor->trace[i].operand); @@ -888,11 +888,11 @@ uop_optimize( return 1; } -/* Dummy execute() function for Uop Executor. +/* Dummy execute() function for UOp Executor. * The actual implementation is inlined in ceval.c, * in _PyEval_EvalFrameDefault(). */ _PyInterpreterFrame * -_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) +_PyUOpExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) { Py_FatalError("Tier 2 is now inlined into Tier 1"); }

1 0

[3.12] gh-112503: Fix test_sys on WASI (gh-112505)
by ericsnowcurrently Nov. 29, 2023

Nov. 29, 2023

https://github.com/python/cpython/commit/3e6b32875207c39325ed8eded5819cfa5d… commit: 3e6b32875207c39325ed8eded5819cfa5dcebc82 branch: 3.12 author: Eric Snow <ericsnowcurrently(a)gmail.com> committer: ericsnowcurrently <ericsnowcurrently(a)gmail.com> date: 2023-11-29T00:36:07Z summary: [3.12] gh-112503: Fix test_sys on WASI (gh-112505) Skip tests if no interpreters module.gh-110713 added some tests that use the interpreters module but did not accommodated builds that don't support subinterpreters (incl. an interpreters module). We fix that here by catching ImportError and skipping tests as appropriate. files: M Lib/test/test_sys.py diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 446dc531b1d82..f1aab43c12969 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -15,11 +15,21 @@ from test.support.script_helper import assert_python_ok, assert_python_failure from test.support import threading_helper from test.support import import_helper -from test.support import interpreters import textwrap import unittest import warnings +try: + from test.support import interpreters +except ImportError: + interpreters = None + + +def requires_subinterpreters(func): + deco = unittest.skipIf(interpreters is None, + 'Test requires subinterpreters') + return deco(func) + DICT_KEY_STRUCT_FORMAT = 'n2BI2n' @@ -700,6 +710,7 @@ def __hash__(self): self.assertRaises(TypeError, sys.intern, S("abc")) + @requires_subinterpreters def test_subinterp_intern_dynamically_allocated(self): s = "never interned before" + str(random.randrange(0, 10**9)) t = sys.intern(s) @@ -713,6 +724,7 @@ def test_subinterp_intern_dynamically_allocated(self): assert id(t) != {id(t)}, (id(t), {id(t)}) ''')) + @requires_subinterpreters def test_subinterp_intern_statically_allocated(self): # See Tools/build/generate_global_objects.py for the list # of strings that are always statically allocated.

1 0

gh-112454: Disable TLS-PSK if OpenSSL was built without PSK support (#112491)
by gpshead Nov. 29, 2023

Nov. 29, 2023

https://github.com/python/cpython/commit/e413daf5f6b983bdb4e1965d76b5313cb9… commit: e413daf5f6b983bdb4e1965d76b5313cb93b266e branch: main author: Grant Ramsay <grant.ramsay(a)hotmail.com> committer: gpshead <greg(a)krypto.org> date: 2023-11-28T16:15:39-08:00 summary: gh-112454: Disable TLS-PSK if OpenSSL was built without PSK support (#112491) If OpenSSL was built without PSK support, the python TLS-PSK methods will raise "NotImplementedError" if called. Add a constant "ssl.HAS_PSK" to check if TLS-PSK is supported files: M Doc/library/ssl.rst M Lib/ssl.py M Lib/test/test_ssl.py M Modules/_ssl.c diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 206294528e001..0db233e2dde33 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -908,6 +908,12 @@ Constants .. versionadded:: 3.7 +.. data:: HAS_PSK + + Whether the OpenSSL library has built-in support for TLS-PSK. + + .. versionadded:: 3.13 + .. data:: CHANNEL_BINDING_TYPES List of supported TLS channel binding types. Strings in this list @@ -2050,6 +2056,9 @@ to speed up repeated connections from the same clients. return 'ClientId_1', psk_table.get(hint, b'') context.set_psk_client_callback(callback) + This method will raise :exc:`NotImplementedError` if :data:`HAS_PSK` is + ``False``. + .. versionadded:: 3.13 .. method:: SSLContext.set_psk_server_callback(callback, identity_hint=None) @@ -2092,6 +2101,9 @@ to speed up repeated connections from the same clients. return psk_table.get(identity, b'') context.set_psk_server_callback(callback, 'ServerId_1') + This method will raise :exc:`NotImplementedError` if :data:`HAS_PSK` is + ``False``. + .. versionadded:: 3.13 .. index:: single: certificates diff --git a/Lib/ssl.py b/Lib/ssl.py index 36fca9d4aa93d..d01484964b689 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -116,7 +116,7 @@ from _ssl import ( HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, - HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3 + HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK ) from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index aecba89cde149..3fdfa2960503b 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4259,6 +4259,7 @@ def test_session_handling(self): 'Session refers to a different SSLContext.') @requires_tls_version('TLSv1_2') + @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') def test_psk(self): psk = bytes.fromhex('deadbeef') @@ -4326,6 +4327,7 @@ def server_callback(identity): s.connect((HOST, server.port)) @requires_tls_version('TLSv1_3') + @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') def test_psk_tls1_3(self): psk = bytes.fromhex('deadbeef') identity_hint = 'identity-hint' diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 707e7ad9543ac..90b600f4b776a 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -301,8 +301,10 @@ typedef struct { BIO *keylog_bio; /* Cached module state, also used in SSLSocket and SSLSession code. */ _sslmodulestate *state; +#ifndef OPENSSL_NO_PSK PyObject *psk_client_callback; PyObject *psk_server_callback; +#endif } PySSLContext; typedef struct { @@ -3125,8 +3127,10 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) self->alpn_protocols = NULL; self->set_sni_cb = NULL; self->state = get_ssl_state(module); +#ifndef OPENSSL_NO_PSK self->psk_client_callback = NULL; self->psk_server_callback = NULL; +#endif /* Don't check host name by default */ if (proto_version == PY_SSL_VERSION_TLS_CLIENT) { @@ -3239,8 +3243,10 @@ context_clear(PySSLContext *self) Py_CLEAR(self->set_sni_cb); Py_CLEAR(self->msg_cb); Py_CLEAR(self->keylog_filename); +#ifndef OPENSSL_NO_PSK Py_CLEAR(self->psk_client_callback); Py_CLEAR(self->psk_server_callback); +#endif if (self->keylog_bio != NULL) { PySSL_BEGIN_ALLOW_THREADS BIO_free_all(self->keylog_bio); @@ -4668,6 +4674,7 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) return NULL; } +#ifndef OPENSSL_NO_PSK static unsigned int psk_client_callback(SSL *s, const char *hint, char *identity, @@ -4735,6 +4742,7 @@ static unsigned int psk_client_callback(SSL *s, PyGILState_Release(gstate); return 0; } +#endif /*[clinic input] _ssl._SSLContext.set_psk_client_callback @@ -4747,6 +4755,7 @@ _ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, PyObject *callback) /*[clinic end generated code: output=0aba86f6ed75119e input=7627bae0e5ee7635]*/ { +#ifndef OPENSSL_NO_PSK if (self->protocol == PY_SSL_VERSION_TLS_SERVER) { _setSSLError(get_state_ctx(self), "Cannot add PSK client callback to a " @@ -4774,8 +4783,14 @@ _ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, SSL_CTX_set_psk_client_callback(self->ctx, ssl_callback); Py_RETURN_NONE; +#else + PyErr_SetString(PyExc_NotImplementedError, + "TLS-PSK is not supported by your OpenSSL version."); + return NULL; +#endif } +#ifndef OPENSSL_NO_PSK static unsigned int psk_server_callback(SSL *s, const char *identity, unsigned char *psk, @@ -4835,6 +4850,7 @@ static unsigned int psk_server_callback(SSL *s, PyGILState_Release(gstate); return 0; } +#endif /*[clinic input] _ssl._SSLContext.set_psk_server_callback @@ -4849,6 +4865,7 @@ _ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, const char *identity_hint) /*[clinic end generated code: output=1f4d6a4e09a92b03 input=65d4b6022aa85ea3]*/ { +#ifndef OPENSSL_NO_PSK if (self->protocol == PY_SSL_VERSION_TLS_CLIENT) { _setSSLError(get_state_ctx(self), "Cannot add PSK server callback to a " @@ -4882,6 +4899,11 @@ _ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, SSL_CTX_set_psk_server_callback(self->ctx, ssl_callback); Py_RETURN_NONE; +#else + PyErr_SetString(PyExc_NotImplementedError, + "TLS-PSK is not supported by your OpenSSL version."); + return NULL; +#endif } @@ -6243,6 +6265,12 @@ sslmodule_init_constants(PyObject *m) addbool(m, "HAS_TLSv1_3", 0); #endif +#ifdef OPENSSL_NO_PSK + addbool(m, "HAS_PSK", 0); +#else + addbool(m, "HAS_PSK", 1); +#endif + #undef addbool #undef ADD_INT_CONST

1 0

[3.12] gh-105716: Fix Refleaks in test_interpreters (gh-112500)
by ericsnowcurrently Nov. 28, 2023

Nov. 28, 2023

https://github.com/python/cpython/commit/5b6858c22049b467c844620bf591c3b1bc… commit: 5b6858c22049b467c844620bf591c3b1bca629d6 branch: 3.12 author: Eric Snow <ericsnowcurrently(a)gmail.com> committer: ericsnowcurrently <ericsnowcurrently(a)gmail.com> date: 2023-11-28T19:42:49Z summary: [3.12] gh-105716: Fix Refleaks in test_interpreters (gh-112500) gh-110707 (0122b4d) added some tests that didn't close file descriptors they created, leading to failures on the refleak buildbots. This closes the leaking file descriptors, resolving the failure. files: M Lib/test/test_interpreters.py diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index cc4f4003b0359..2b0cff596da00 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -68,6 +68,20 @@ def run(): class TestBase(unittest.TestCase): + def os_pipe(self): + r, w = os.pipe() + def cleanup(): + try: + os.close(w) + except Exception: + pass + try: + os.close(r) + except Exception: + pass + self.addCleanup(cleanup) + return r, w + def tearDown(self): clean_up_interpreters() @@ -262,7 +276,7 @@ def test_subinterpreter(self): self.assertFalse(interp.is_running()) def test_finished(self): - r, w = os.pipe() + r, w = self.os_pipe() interp = interpreters.create() interp.run(f"""if True: import os @@ -299,8 +313,8 @@ def test_bad_id(self): interp.is_running() def test_with_only_background_threads(self): - r_interp, w_interp = os.pipe() - r_thread, w_thread = os.pipe() + r_interp, w_interp = self.os_pipe() + r_thread, w_thread = self.os_pipe() DONE = b'D' FINISHED = b'F' @@ -425,8 +439,8 @@ def test_still_running(self): self.assertTrue(interp.is_running()) def test_subthreads_still_running(self): - r_interp, w_interp = os.pipe() - r_thread, w_thread = os.pipe() + r_interp, w_interp = self.os_pipe() + r_thread, w_thread = self.os_pipe() FINISHED = b'F' @@ -532,8 +546,8 @@ def test_bytes_for_script(self): interp.run(b'print("spam")') def test_with_background_threads_still_running(self): - r_interp, w_interp = os.pipe() - r_thread, w_thread = os.pipe() + r_interp, w_interp = self.os_pipe() + r_thread, w_thread = self.os_pipe() RAN = b'R' DONE = b'D'

1 0

GH-112245: Promote free threaded CI (#112246)
by carljm Nov. 28, 2023

Nov. 28, 2023

https://github.com/python/cpython/commit/48dfd74a9db9d4aa9c6f23b4a67b461e5d… commit: 48dfd74a9db9d4aa9c6f23b4a67b461e5d977173 branch: main author: Itamar Oren <itamarost(a)gmail.com> committer: carljm <carl(a)oddbird.net> date: 2023-11-28T08:45:03-07:00 summary: GH-112245: Promote free threaded CI (#112246) files: M .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4210194b978749..cfb36c8c32e18d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -190,7 +190,7 @@ jobs: build_windows_free_threaded: name: 'Windows (free-threaded)' needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && contains(github.event.pull_request.labels.*.name, 'topic-free-threaded') + if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-windows.yml with: free-threaded: true @@ -206,7 +206,7 @@ jobs: build_macos_free_threaded: name: 'macOS (free-threaded)' needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && contains(github.event.pull_request.labels.*.name, 'topic-free-threaded') + if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-macos.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} @@ -228,7 +228,7 @@ jobs: build_ubuntu_free_threaded: name: 'Ubuntu (free-threaded)' needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && contains(github.event.pull_request.labels.*.name, 'topic-free-threaded') + if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-ubuntu.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} @@ -521,10 +521,7 @@ jobs: uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: allowed-failures: >- - build_macos_free_threaded, - build_ubuntu_free_threaded, build_ubuntu_ssltests, - build_windows_free_threaded, cifuzz, test_hypothesis, allowed-skips: >-

1 0