Mailman 3 June 2024 - Python-checkins

gh-121137: Add missing Py_DECREF calls for ADDITEMS opcode of _pickle.c (#121136)
by gaogaotiantian 28 Jun '24

28 Jun '24

https://github.com/python/cpython/commit/92893fd8dc803ed7cdde55d29d25f84ccb… commit: 92893fd8dc803ed7cdde55d29d25f84ccb5e3ef0 branch: main author: Justin Applegate <70449145+Legoclones(a)users.noreply.github.com> committer: gaogaotiantian <gaogaotiantian(a)hotmail.com> date: 2024-06-28T14:43:45-07:00 summary: gh-121137: Add missing Py_DECREF calls for ADDITEMS opcode of _pickle.c (#121136) PyObject_GetAttr returns a new reference, but this reference is never decremented using Py_DECREF, so Py_DECREF calls to this referece are added files: M Modules/_pickle.c diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 21be88a79d8705..4a2191db0cf984 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -6524,11 +6524,13 @@ load_additems(PickleState *state, UnpicklerObject *self) if (result == NULL) { Pdata_clear(self->stack, i + 1); Py_SET_SIZE(self->stack, mark); + Py_DECREF(add_func); return -1; } Py_DECREF(result); } Py_SET_SIZE(self->stack, mark); + Py_DECREF(add_func); } return 0;

1 0

gh-117139: Fix a few wrong steals in bytecodes.c (GH-121127)
by Fidget-Spinner 28 Jun '24

28 Jun '24

https://github.com/python/cpython/commit/e6543daf12051e9c660a5c0437683e8d27… commit: e6543daf12051e9c660a5c0437683e8d2706a3c7 branch: main author: Ken Jin <kenjin(a)python.org> committer: Fidget-Spinner <kenjin4096(a)gmail.com> date: 2024-06-29T02:14:48+08:00 summary: gh-117139: Fix a few wrong steals in bytecodes.c (GH-121127) Fix a few wrong steals in bytecodes.c files: M Python/bytecodes.c M Python/ceval.c M Python/executor_cases.c.h M Python/generated_cases.c.h diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8dfce77dfca297..50978a0dc87694 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -230,7 +230,7 @@ dummy_func( } replicate(8) pure inst(LOAD_FAST, (-- value)) { - assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); } @@ -673,7 +673,7 @@ dummy_func( err = 1; } else { - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v)); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); } PyStackRef_CLOSE(v); @@ -789,7 +789,7 @@ dummy_func( inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) { int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectSteal(v)); + PyStackRef_AsPyObjectBorrow(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -813,7 +813,7 @@ dummy_func( op(_STORE_SUBSCR, (v, container, sub -- )) { /* container[sub] = v */ - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v)); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1235,7 +1235,7 @@ dummy_func( inst(POP_EXCEPT, (exc_value -- )) { _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, - PyStackRef_AsPyObjectBorrow(exc_value) == Py_None + PyStackRef_Is(exc_value, PyStackRef_None) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); } @@ -1330,9 +1330,9 @@ dummy_func( ERROR_IF(true, error); } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); else - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1450,7 +1450,7 @@ dummy_func( op(_STORE_ATTR, (v, owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectSteal(v)); + name, PyStackRef_AsPyObjectBorrow(v)); DECREF_INPUTS(); ERROR_IF(err, error); } diff --git a/Python/ceval.c b/Python/ceval.c index 2412256ace8c8d..f4b3a417025c14 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1619,7 +1619,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, goto kw_fail; } - if (PyDict_SetItem(kwdict, keyword, PyStackRef_AsPyObjectSteal(value_stackref)) == -1) { + if (PyDict_SetItem(kwdict, keyword, PyStackRef_AsPyObjectBorrow(value_stackref)) == -1) { goto kw_fail; } PyStackRef_CLOSE(value_stackref); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 38437c6f2c087c..76b7a9b4b15ae9 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -54,7 +54,7 @@ _PyStackRef value; oparg = 0; assert(oparg == CURRENT_OPARG()); - assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; @@ -66,7 +66,7 @@ _PyStackRef value; oparg = 1; assert(oparg == CURRENT_OPARG()); - assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; @@ -78,7 +78,7 @@ _PyStackRef value; oparg = 2; assert(oparg == CURRENT_OPARG()); - assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; @@ -90,7 +90,7 @@ _PyStackRef value; oparg = 3; assert(oparg == CURRENT_OPARG()); - assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; @@ -102,7 +102,7 @@ _PyStackRef value; oparg = 4; assert(oparg == CURRENT_OPARG()); - assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; @@ -114,7 +114,7 @@ _PyStackRef value; oparg = 5; assert(oparg == CURRENT_OPARG()); - assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; @@ -126,7 +126,7 @@ _PyStackRef value; oparg = 6; assert(oparg == CURRENT_OPARG()); - assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; @@ -138,7 +138,7 @@ _PyStackRef value; oparg = 7; assert(oparg == CURRENT_OPARG()); - assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; @@ -149,7 +149,7 @@ case _LOAD_FAST: { _PyStackRef value; oparg = CURRENT_OPARG(); - assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; @@ -757,7 +757,7 @@ err = 1; } else { - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v)); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); } PyStackRef_CLOSE(v); @@ -939,7 +939,7 @@ v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectSteal(v)); + PyStackRef_AsPyObjectBorrow(v)); PyStackRef_CLOSE(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; @@ -955,7 +955,7 @@ container = stack_pointer[-2]; v = stack_pointer[-3]; /* container[sub] = v */ - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v)); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); @@ -1281,7 +1281,7 @@ exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, - PyStackRef_AsPyObjectBorrow(exc_value) == Py_None + PyStackRef_Is(exc_value, PyStackRef_None) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1338,9 +1338,9 @@ if (true) JUMP_TO_ERROR(); } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); else - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); PyStackRef_CLOSE(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; @@ -1483,7 +1483,7 @@ v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectSteal(v)); + name, PyStackRef_AsPyObjectBorrow(v)); PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); if (err) JUMP_TO_ERROR(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 14959fb31be201..32b22aff14a768 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4667,7 +4667,7 @@ next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); _PyStackRef value; - assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; @@ -5359,7 +5359,7 @@ exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, - PyStackRef_AsPyObjectBorrow(exc_value) == Py_None + PyStackRef_Is(exc_value, PyStackRef_None) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -5876,7 +5876,7 @@ v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectSteal(v)); + PyStackRef_AsPyObjectBorrow(v)); PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; @@ -5977,7 +5977,7 @@ { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectSteal(v)); + name, PyStackRef_AsPyObjectBorrow(v)); PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); if (err) goto pop_2_error; @@ -6215,9 +6215,9 @@ if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); else - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; @@ -6244,7 +6244,7 @@ err = 1; } else { - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v)); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); } PyStackRef_CLOSE(v); @@ -6285,7 +6285,7 @@ v = stack_pointer[-3]; { /* container[sub] = v */ - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v)); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub);

1 0

[3.12] gh-115986 Improve pprint docs formatting (GH-117401) (GH-121099)
by encukou 28 Jun '24

28 Jun '24

https://github.com/python/cpython/commit/3a420de986efef4bae039e2aad9feaa265… commit: 3a420de986efef4bae039e2aad9feaa26505d24b branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: encukou <encukou(a)gmail.com> date: 2024-06-28T18:21:56+02:00 summary: [3.12] gh-115986 Improve pprint docs formatting (GH-117401) (GH-121099) gh-115986 Improve pprint docs formatting (GH-117401) * Move pprinter parameters description to the table The change improves readability. Suggested in the GHGH-116085 PR discussion. * Make pprint doc with params markup * Fix formatting Indentation of code blocks made them nested "Version changed" is better placed after the code block * Fix formatting for tests * fix code indentation for autotests * Fix identation for autotests * Remove duplication of the parameters' description * Rearrange parameters description in a correct order --------- (cherry picked from commit 0890ad7c024ccf29614849b6ffadcb92c0e91ce7) Co-authored-by: Kerim Kabirov <the.privat33r+gh(a)pm.me> Co-authored-by: Hugo van Kemenade <1324225+hugovk(a)users.noreply.github.com> Co-authored-by: Petr Viktorin <encukou(a)gmail.com> files: M Doc/library/pprint.rst diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index df706c10ce9ec4..1b3498e51f766d 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -35,24 +35,66 @@ Dictionaries are sorted by key before the display is computed. Functions --------- -.. function:: pp(object, *args, sort_dicts=False, **kwargs) - - Prints the formatted representation of *object* followed by a newline. - If *sort_dicts* is false (the default), dictionaries will be displayed with - their keys in insertion order, otherwise the dict keys will be sorted. - *args* and *kwargs* will be passed to :func:`~pprint.pprint` as formatting - parameters. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff) - >>> pprint.pp(stuff) - [<Recursion on list with id=...>, - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] +.. function:: pp(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=False, underscore_numbers=False) + + Prints the formatted representation of *object*, followed by a newline. + This function may be used in the interactive interpreter + instead of the :func:`print` function for inspecting values. + Tip: you can reassign ``print = pprint.pp`` for use within a scope. + + :param object: + The object to be printed. + + :param stream: + A file-like object to which the output will be written + by calling its :meth:`!write` method. + If ``None`` (the default), :data:`sys.stdout` is used. + :type stream: :term:`file-like object` | None + + :param int indent: + The amount of indentation added for each nesting level. + + :param int width: + The desired maximum number of characters per line in the output. + If a structure cannot be formatted within the width constraint, + a best effort will be made. + + :param depth: + The number of nesting levels which may be printed. + If the data structure being printed is too deep, + the next contained level is replaced by ``...``. + If ``None`` (the default), there is no constraint + on the depth of the objects being formatted. + :type depth: int | None + + :param bool compact: + Control the way long :term:`sequences <sequence>` are formatted. + If ``False`` (the default), + each item of a sequence will be formatted on a separate line, + otherwise as many items as will fit within the *width* + will be formatted on each output line. + + :param bool sort_dicts: + If ``True``, dictionaries will be formatted with + their keys sorted, otherwise + they will be displayed in insertion order (the default). + + :param bool underscore_numbers: + If ``True``, + integers will be formatted with the ``_`` character for a thousands separator, + otherwise underscores are not displayed (the default). + + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff) + >>> pprint.pp(stuff) + [<Recursion on list with id=...>, + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] .. versionadded:: 3.8 @@ -60,19 +102,10 @@ Functions .. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ compact=False, sort_dicts=True, underscore_numbers=False) - Prints the formatted representation of *object* on *stream*, followed by a - newline. If *stream* is ``None``, :data:`sys.stdout` is used. This may be used - in the interactive interpreter instead of the :func:`print` function for - inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). - - The configuration parameters *stream*, *indent*, *width*, *depth*, - *compact*, *sort_dicts* and *underscore_numbers* are passed to the - :class:`PrettyPrinter` constructor and their meanings are as - described in its documentation below. + Alias for :func:`~pprint.pp` with *sort_dicts* set to ``True`` by default, + which would automatically sort the dictionaries' keys, + you might want to use :func:`~pprint.pp` instead where it is ``False`` by default. - Note that *sort_dicts* is ``True`` by default and you might want to use - :func:`~pprint.pp` instead where it is ``False`` by default. .. function:: pformat(object, indent=1, width=80, depth=None, *, \ compact=False, sort_dicts=True, underscore_numbers=False) @@ -80,7 +113,7 @@ Functions Return the formatted representation of *object* as a string. *indent*, *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are passed to the :class:`PrettyPrinter` constructor as formatting parameters - and their meanings are as described in its documentation below. + and their meanings are as described in the documentation above. .. function:: isreadable(object) @@ -119,51 +152,39 @@ Functions PrettyPrinter Objects --------------------- -This module defines one class: - -.. First the implementation class: - - .. index:: single: ...; placeholder .. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ compact=False, sort_dicts=True, underscore_numbers=False) - Construct a :class:`PrettyPrinter` instance. This constructor understands - several keyword parameters. - - *stream* (default :data:`!sys.stdout`) is a :term:`file-like object` to - which the output will be written by calling its :meth:`!write` method. - If both *stream* and :data:`!sys.stdout` are ``None``, then - :meth:`~PrettyPrinter.pprint` silently returns. + Construct a :class:`PrettyPrinter` instance. - Other values configure the manner in which nesting of complex data - structures is displayed. + Arguments have the same meaning as for :func:`~pprint.pp`. + Note that they are in a different order, and that *sort_dicts* defaults to ``True``. - *indent* (default 1) specifies the amount of indentation added for - each nesting level. - - *depth* controls the number of nesting levels which may be printed; if - the data structure being printed is too deep, the next contained level - is replaced by ``...``. By default, there is no constraint on the - depth of the objects being formatted. - - *width* (default 80) specifies the desired maximum number of characters per - line in the output. If a structure cannot be formatted within the width - constraint, a best effort will be made. - - *compact* impacts the way that long sequences (lists, tuples, sets, etc) - are formatted. If *compact* is false (the default) then each item of a - sequence will be formatted on a separate line. If *compact* is true, as - many items as will fit within the *width* will be formatted on each output - line. - - If *sort_dicts* is true (the default), dictionaries will be formatted with - their keys sorted, otherwise they will display in insertion order. + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff[:]) + >>> pp = pprint.PrettyPrinter(indent=4) + >>> pp.pprint(stuff) + [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] + >>> pp = pprint.PrettyPrinter(width=41, compact=True) + >>> pp.pprint(stuff) + [['spam', 'eggs', 'lumberjack', + 'knights', 'ni'], + 'spam', 'eggs', 'lumberjack', 'knights', + 'ni'] + >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', + ... ('parrot', ('fresh fruit',)))))))) + >>> pp = pprint.PrettyPrinter(depth=6) + >>> pp.pprint(tup) + ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) - If *underscore_numbers* is true, integers will be formatted with the - ``_`` character for a thousands separator, otherwise underscores are not - displayed (the default). .. versionchanged:: 3.4 Added the *compact* parameter. @@ -177,29 +198,6 @@ This module defines one class: .. versionchanged:: 3.11 No longer attempts to write to :data:`!sys.stdout` if it is ``None``. - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff[:]) - >>> pp = pprint.PrettyPrinter(indent=4) - >>> pp.pprint(stuff) - [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] - >>> pp = pprint.PrettyPrinter(width=41, compact=True) - >>> pp.pprint(stuff) - [['spam', 'eggs', 'lumberjack', - 'knights', 'ni'], - 'spam', 'eggs', 'lumberjack', 'knights', - 'ni'] - >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', - ... ('parrot', ('fresh fruit',)))))))) - >>> pp = pprint.PrettyPrinter(depth=6) - >>> pp.pprint(tup) - ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) - :class:`PrettyPrinter` instances have the following methods:

1 0

[3.12] gh-120868: Fix breaking change in `logging.config` when using `QueueHandler` (GH-120872) (GH-121077)
by vsajip 28 Jun '24

28 Jun '24

https://github.com/python/cpython/commit/b31f7e2e90a811f46f1934dd3604412d13… commit: b31f7e2e90a811f46f1934dd3604412d1381eede branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: vsajip <vinay_sajip(a)yahoo.co.uk> date: 2024-06-28T17:10:53+01:00 summary: [3.12] gh-120868: Fix breaking change in `logging.config` when using `QueueHandler` (GH-120872) (GH-121077) (cherry picked from commit 7d9c68513d112823a9a6cdc7453b998b2c24eb4c) files: M Lib/logging/config.py M Lib/test/test_logging.py M Misc/ACKS diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 1824d0aa747225..1b59876a955a0b 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -787,25 +787,44 @@ def configure_handler(self, config): # if 'handlers' not in config: # raise ValueError('No handlers specified for a QueueHandler') if 'queue' in config: - from multiprocessing.queues import Queue as MPQueue - from multiprocessing import Manager as MM - proxy_queue = MM().Queue() - proxy_joinable_queue = MM().JoinableQueue() qspec = config['queue'] - if not isinstance(qspec, (queue.Queue, MPQueue, - type(proxy_queue), type(proxy_joinable_queue))): - if isinstance(qspec, str): - q = self.resolve(qspec) - if not callable(q): - raise TypeError('Invalid queue specifier %r' % qspec) - q = q() - elif isinstance(qspec, dict): - if '()' not in qspec: - raise TypeError('Invalid queue specifier %r' % qspec) - q = self.configure_custom(dict(qspec)) - else: + + if isinstance(qspec, str): + q = self.resolve(qspec) + if not callable(q): raise TypeError('Invalid queue specifier %r' % qspec) - config['queue'] = q + config['queue'] = q() + elif isinstance(qspec, dict): + if '()' not in qspec: + raise TypeError('Invalid queue specifier %r' % qspec) + config['queue'] = self.configure_custom(dict(qspec)) + else: + from multiprocessing.queues import Queue as MPQueue + + if not isinstance(qspec, (queue.Queue, MPQueue)): + # Safely check if 'qspec' is an instance of Manager.Queue + # / Manager.JoinableQueue + + from multiprocessing import Manager as MM + from multiprocessing.managers import BaseProxy + + # if it's not an instance of BaseProxy, it also can't be + # an instance of Manager.Queue / Manager.JoinableQueue + if isinstance(qspec, BaseProxy): + # Sometimes manager or queue creation might fail + # (e.g. see issue gh-120868). In that case, any + # exception during the creation of these queues will + # propagate up to the caller and be wrapped in a + # `ValueError`, whose cause will indicate the details of + # the failure. + mm = MM() + proxy_queue = mm.Queue() + proxy_joinable_queue = mm.JoinableQueue() + if not isinstance(qspec, (type(proxy_queue), type(proxy_joinable_queue))): + raise TypeError('Invalid queue specifier %r' % qspec) + else: + raise TypeError('Invalid queue specifier %r' % qspec) + if 'listener' in config: lspec = config['listener'] if isinstance(lspec, type): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index a51f60a0b18e3e..951c08a6e4b50e 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -60,6 +60,7 @@ import weakref from http.server import HTTPServer, BaseHTTPRequestHandler +from unittest.mock import patch from urllib.parse import urlparse, parse_qs from socketserver import (ThreadingUDPServer, DatagramRequestHandler, ThreadingTCPServer, StreamRequestHandler) @@ -3895,6 +3896,50 @@ def test_config_queue_handler(self): msg = str(ctx.exception) self.assertEqual(msg, "Unable to configure handler 'ah'") + @threading_helper.requires_working_threading() + @support.requires_subprocess() + @patch("multiprocessing.Manager") + def test_config_queue_handler_does_not_create_multiprocessing_manager(self, manager): + # gh-120868 + + from multiprocessing import Queue as MQ + + q1 = {"()": "queue.Queue", "maxsize": -1} + q2 = MQ() + q3 = queue.Queue() + + for qspec in (q1, q2, q3): + self.apply_config( + { + "version": 1, + "handlers": { + "queue_listener": { + "class": "logging.handlers.QueueHandler", + "queue": qspec, + }, + }, + } + ) + manager.assert_not_called() + + @patch("multiprocessing.Manager") + def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_manager(self, manager): + # gh-120868 + + with self.assertRaises(ValueError): + self.apply_config( + { + "version": 1, + "handlers": { + "queue_listener": { + "class": "logging.handlers.QueueHandler", + "queue": object(), + }, + }, + } + ) + manager.assert_not_called() + @support.requires_subprocess() def test_multiprocessing_queues(self): # See gh-119819 diff --git a/Misc/ACKS b/Misc/ACKS index f6285ca23f32a8..66a1a95063b5fa 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1304,6 +1304,7 @@ Hrvoje Nikšić Gregory Nofi Jesse Noller Bill Noon +Janek Nouvertné Stefan Norberg Tim Northover Joe Norton

1 0

gh-121115: Skip __index__ in PyLong_AsNativeBytes by default (GH-121118)
by zooba 28 Jun '24

28 Jun '24

https://github.com/python/cpython/commit/58a3c3c0ad1f691810054fbc0feb8d9260… commit: 58a3c3c0ad1f691810054fbc0feb8d9260e55fc2 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: zooba <steve.dower(a)microsoft.com> date: 2024-06-28T15:52:26Z summary: gh-121115: Skip __index__ in PyLong_AsNativeBytes by default (GH-121118) (cherry picked from commit 2894aa14f22430e9b6d4676afead6da7c79209ca) Co-authored-by: Steve Dower <steve.dower(a)python.org> files: A Misc/NEWS.d/next/Core and Builtins/2024-06-28-10-02-58.gh-issue-121115.EeSLfc.rst M Doc/c-api/long.rst M Include/cpython/longobject.h M Lib/test/test_capi/test_long.py M Objects/longobject.c diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 522c028cfb8d40..c502a825182889 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -405,14 +405,13 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Passing zero to *n_bytes* will return the size of a buffer that would be large enough to hold the value. This may be larger than technically - necessary, but not unreasonably so. + necessary, but not unreasonably so. If *n_bytes=0*, *buffer* may be + ``NULL``. .. note:: Passing *n_bytes=0* to this function is not an accurate way to determine - the bit length of a value. - - If *n_bytes=0*, *buffer* may be ``NULL``. + the bit length of the value. To get at the entire Python value of an unknown size, the function can be called twice: first to determine the buffer size, then to fill it:: @@ -462,6 +461,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:macro:: Py_ASNATIVEBYTES_NATIVE_ENDIAN ``3`` .. c:macro:: Py_ASNATIVEBYTES_UNSIGNED_BUFFER ``4`` .. c:macro:: Py_ASNATIVEBYTES_REJECT_NEGATIVE ``8`` + .. c:macro:: Py_ASNATIVEBYTES_ALLOW_INDEX ``16`` ============================================= ====== Specifying ``Py_ASNATIVEBYTES_NATIVE_ENDIAN`` will override any other endian @@ -483,6 +483,13 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. provided there is enough space for at least one sign bit, regardless of whether ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` was specified. + If ``Py_ASNATIVEBYTES_ALLOW_INDEX`` is specified and a non-integer value is + passed, its :meth:`~object.__index__` method will be called first. This may + result in Python code executing and other threads being allowed to run, which + could cause changes to other objects or values in use. When *flags* is + ``-1``, this option is not set, and non-integer values will raise + :exc:`TypeError`. + .. note:: With the default *flags* (``-1``, or *UNSIGNED_BUFFER* without diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 96815938c8277a..0d49242ff6808c 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -10,6 +10,7 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); #define Py_ASNATIVEBYTES_NATIVE_ENDIAN 3 #define Py_ASNATIVEBYTES_UNSIGNED_BUFFER 4 #define Py_ASNATIVEBYTES_REJECT_NEGATIVE 8 +#define Py_ASNATIVEBYTES_ALLOW_INDEX 16 /* PyLong_AsNativeBytes: Copy the integer value to a native variable. buffer points to the first byte of the variable. @@ -20,8 +21,10 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); * 2 - native endian * 4 - unsigned destination (e.g. don't reject copying 255 into one byte) * 8 - raise an exception for negative inputs - If flags is -1 (all bits set), native endian is used and value truncation - behaves most like C (allows negative inputs and allow MSB set). + * 16 - call __index__ on non-int types + If flags is -1 (all bits set), native endian is used, value truncation + behaves most like C (allows negative inputs and allow MSB set), and non-int + objects will raise a TypeError. Big endian mode will write the most significant byte into the address directly referenced by buffer; little endian will write the least significant byte into that address. diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 83f894e552f983..dd456f40ae6ea1 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -496,8 +496,9 @@ def test_long_asnativebytes(self): "PyLong_AsNativeBytes(v, <unknown>, 0, -1)") self.assertEqual(buffer, b"\x5a", "buffer overwritten when it should not have been") - # Also check via the __index__ path - self.assertEqual(expect, asnativebytes(Index(v), buffer, 0, -1), + # Also check via the __index__ path. + # We pass Py_ASNATIVEBYTES_NATIVE_ENDIAN | ALLOW_INDEX + self.assertEqual(expect, asnativebytes(Index(v), buffer, 0, 3 | 16), "PyLong_AsNativeBytes(Index(v), <unknown>, 0, -1)") self.assertEqual(buffer, b"\x5a", "buffer overwritten when it should not have been") @@ -607,6 +608,12 @@ def test_long_asnativebytes(self): with self.assertRaises(ValueError): asnativebytes(-1, buffer, 0, 8) + # Ensure omitting Py_ASNATIVEBYTES_ALLOW_INDEX raises on __index__ value + with self.assertRaises(TypeError): + asnativebytes(Index(1), buffer, 0, -1) + with self.assertRaises(TypeError): + asnativebytes(Index(1), buffer, 0, 3) + # Check a few error conditions. These are validated in code, but are # unspecified in docs, so if we make changes to the implementation, it's # fine to just update these tests rather than preserve the behaviour. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-28-10-02-58.gh-issue-121115.EeSLfc.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-28-10-02-58.gh-issue-121115.EeSLfc.rst new file mode 100644 index 00000000000000..aaecc873551cc7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-28-10-02-58.gh-issue-121115.EeSLfc.rst @@ -0,0 +1,3 @@ +:c:func:`PyLong_AsNativeBytes` no longer uses :meth:`~object.__index__` +methods by default. The ``Py_ASNATIVEBYTES_ALLOW_INDEX`` flag has been added +to allow it. diff --git a/Objects/longobject.c b/Objects/longobject.c index ce3fd6b711dcd4..d4c11e9c27e6f5 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1116,13 +1116,17 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) if (PyLong_Check(vv)) { v = (PyLongObject *)vv; } - else { + else if (flags != -1 && (flags & Py_ASNATIVEBYTES_ALLOW_INDEX)) { v = (PyLongObject *)_PyNumber_Index(vv); if (v == NULL) { return -1; } do_decref = 1; } + else { + PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); + return -1; + } if ((flags != -1 && (flags & Py_ASNATIVEBYTES_REJECT_NEGATIVE)) && _PyLong_IsNegative(v)) {

1 0

gh-121115: Skip __index__ in PyLong_AsNativeBytes by default (GH-121118)
by zooba 28 Jun '24

28 Jun '24

https://github.com/python/cpython/commit/2894aa14f22430e9b6d4676afead6da7c7… commit: 2894aa14f22430e9b6d4676afead6da7c79209ca branch: main author: Steve Dower <steve.dower(a)python.org> committer: zooba <steve.dower(a)microsoft.com> date: 2024-06-28T16:26:21+01:00 summary: gh-121115: Skip __index__ in PyLong_AsNativeBytes by default (GH-121118) files: A Misc/NEWS.d/next/Core and Builtins/2024-06-28-10-02-58.gh-issue-121115.EeSLfc.rst M Doc/c-api/long.rst M Include/cpython/longobject.h M Lib/test/test_capi/test_long.py M Objects/longobject.c diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index a0e111af5996d7..42162914c0aec8 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -405,14 +405,13 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Passing zero to *n_bytes* will return the size of a buffer that would be large enough to hold the value. This may be larger than technically - necessary, but not unreasonably so. + necessary, but not unreasonably so. If *n_bytes=0*, *buffer* may be + ``NULL``. .. note:: Passing *n_bytes=0* to this function is not an accurate way to determine - the bit length of a value. - - If *n_bytes=0*, *buffer* may be ``NULL``. + the bit length of the value. To get at the entire Python value of an unknown size, the function can be called twice: first to determine the buffer size, then to fill it:: @@ -462,6 +461,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:macro:: Py_ASNATIVEBYTES_NATIVE_ENDIAN ``3`` .. c:macro:: Py_ASNATIVEBYTES_UNSIGNED_BUFFER ``4`` .. c:macro:: Py_ASNATIVEBYTES_REJECT_NEGATIVE ``8`` + .. c:macro:: Py_ASNATIVEBYTES_ALLOW_INDEX ``16`` ============================================= ====== Specifying ``Py_ASNATIVEBYTES_NATIVE_ENDIAN`` will override any other endian @@ -483,6 +483,13 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. provided there is enough space for at least one sign bit, regardless of whether ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` was specified. + If ``Py_ASNATIVEBYTES_ALLOW_INDEX`` is specified and a non-integer value is + passed, its :meth:`~object.__index__` method will be called first. This may + result in Python code executing and other threads being allowed to run, which + could cause changes to other objects or values in use. When *flags* is + ``-1``, this option is not set, and non-integer values will raise + :exc:`TypeError`. + .. note:: With the default *flags* (``-1``, or *UNSIGNED_BUFFER* without diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 19a6722d07734a..e7e0c3d9764f20 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -10,6 +10,7 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); #define Py_ASNATIVEBYTES_NATIVE_ENDIAN 3 #define Py_ASNATIVEBYTES_UNSIGNED_BUFFER 4 #define Py_ASNATIVEBYTES_REJECT_NEGATIVE 8 +#define Py_ASNATIVEBYTES_ALLOW_INDEX 16 /* PyLong_AsNativeBytes: Copy the integer value to a native variable. buffer points to the first byte of the variable. @@ -20,8 +21,10 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); * 2 - native endian * 4 - unsigned destination (e.g. don't reject copying 255 into one byte) * 8 - raise an exception for negative inputs - If flags is -1 (all bits set), native endian is used and value truncation - behaves most like C (allows negative inputs and allow MSB set). + * 16 - call __index__ on non-int types + If flags is -1 (all bits set), native endian is used, value truncation + behaves most like C (allows negative inputs and allow MSB set), and non-int + objects will raise a TypeError. Big endian mode will write the most significant byte into the address directly referenced by buffer; little endian will write the least significant byte into that address. diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 06a29b5a0505b4..7e8d571ae234d1 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -496,8 +496,9 @@ def test_long_asnativebytes(self): "PyLong_AsNativeBytes(v, <unknown>, 0, -1)") self.assertEqual(buffer, b"\x5a", "buffer overwritten when it should not have been") - # Also check via the __index__ path - self.assertEqual(expect, asnativebytes(Index(v), buffer, 0, -1), + # Also check via the __index__ path. + # We pass Py_ASNATIVEBYTES_NATIVE_ENDIAN | ALLOW_INDEX + self.assertEqual(expect, asnativebytes(Index(v), buffer, 0, 3 | 16), "PyLong_AsNativeBytes(Index(v), <unknown>, 0, -1)") self.assertEqual(buffer, b"\x5a", "buffer overwritten when it should not have been") @@ -607,6 +608,12 @@ def test_long_asnativebytes(self): with self.assertRaises(ValueError): asnativebytes(-1, buffer, 0, 8) + # Ensure omitting Py_ASNATIVEBYTES_ALLOW_INDEX raises on __index__ value + with self.assertRaises(TypeError): + asnativebytes(Index(1), buffer, 0, -1) + with self.assertRaises(TypeError): + asnativebytes(Index(1), buffer, 0, 3) + # Check a few error conditions. These are validated in code, but are # unspecified in docs, so if we make changes to the implementation, it's # fine to just update these tests rather than preserve the behaviour. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-28-10-02-58.gh-issue-121115.EeSLfc.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-28-10-02-58.gh-issue-121115.EeSLfc.rst new file mode 100644 index 00000000000000..aaecc873551cc7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-28-10-02-58.gh-issue-121115.EeSLfc.rst @@ -0,0 +1,3 @@ +:c:func:`PyLong_AsNativeBytes` no longer uses :meth:`~object.__index__` +methods by default. The ``Py_ASNATIVEBYTES_ALLOW_INDEX`` flag has been added +to allow it. diff --git a/Objects/longobject.c b/Objects/longobject.c index 86afec9a414134..4ca259fb08e8c7 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1128,13 +1128,17 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) if (PyLong_Check(vv)) { v = (PyLongObject *)vv; } - else { + else if (flags != -1 && (flags & Py_ASNATIVEBYTES_ALLOW_INDEX)) { v = (PyLongObject *)_PyNumber_Index(vv); if (v == NULL) { return -1; } do_decref = 1; } + else { + PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); + return -1; + } if ((flags != -1 && (flags & Py_ASNATIVEBYTES_REJECT_NEGATIVE)) && _PyLong_IsNegative(v)) {

1 0

[3.13] gh-121018: Fix more cases of exiting in argparse when exit_on_error=False (GH-121056) (GH-121128)
by serhiy-storchaka 28 Jun '24

28 Jun '24

https://github.com/python/cpython/commit/99de20d729c14c9ea062c205b55925305e… commit: 99de20d729c14c9ea062c205b55925305ed797b6 branch: 3.13 author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2024-06-28T15:04:44Z summary: [3.13] gh-121018: Fix more cases of exiting in argparse when exit_on_error=False (GH-121056) (GH-121128) * parse_intermixed_args() now raises ArgumentError instead of calling error() if exit_on_error is false. * Internal code now always raises ArgumentError instead of calling error(). It is then caught at the higher level and error() is called if exit_on_error is true. (cherry picked from commit 81a654a3425eaa05a51342509089533c1f623f1b) files: A Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst M Lib/argparse.py M Lib/test/test_argparse.py diff --git a/Lib/argparse.py b/Lib/argparse.py index 64893b53efbfee..7e5e3129dbd47b 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1819,7 +1819,7 @@ def _get_kwargs(self): # ================================== def add_subparsers(self, **kwargs): if self._subparsers is not None: - self.error(_('cannot have multiple subparser arguments')) + raise ArgumentError(None, _('cannot have multiple subparser arguments')) # add the parser class to the arguments if it's not present kwargs.setdefault('parser_class', type(self)) @@ -1874,7 +1874,8 @@ def parse_args(self, args=None, namespace=None): msg = _('unrecognized arguments: %s') % ' '.join(argv) if self.exit_on_error: self.error(msg) - raise ArgumentError(None, msg) + else: + raise ArgumentError(None, msg) return args def parse_known_args(self, args=None, namespace=None): @@ -2163,7 +2164,7 @@ def consume_positionals(start_index): self._get_value(action, action.default)) if required_actions: - self.error(_('the following arguments are required: %s') % + raise ArgumentError(None, _('the following arguments are required: %s') % ', '.join(required_actions)) # make sure all required groups had one option present @@ -2179,7 +2180,7 @@ def consume_positionals(start_index): for action in group._group_actions if action.help is not SUPPRESS] msg = _('one of the arguments %s is required') - self.error(msg % ' '.join(names)) + raise ArgumentError(None, msg % ' '.join(names)) # return the updated namespace and the extra arguments return namespace, extras @@ -2206,7 +2207,7 @@ def _read_args_from_files(self, arg_strings): arg_strings = self._read_args_from_files(arg_strings) new_arg_strings.extend(arg_strings) except OSError as err: - self.error(str(err)) + raise ArgumentError(None, str(err)) # return the modified argument list return new_arg_strings @@ -2286,7 +2287,7 @@ def _parse_optional(self, arg_string): for action, option_string, sep, explicit_arg in option_tuples]) args = {'option': arg_string, 'matches': options} msg = _('ambiguous option: %(option)s could match %(matches)s') - self.error(msg % args) + raise ArgumentError(None, msg % args) # if exactly one action matched, this segmentation is good, # so return the parsed action @@ -2346,7 +2347,7 @@ def _get_option_tuples(self, option_string): # shouldn't ever get here else: - self.error(_('unexpected option string: %s') % option_string) + raise ArgumentError(None, _('unexpected option string: %s') % option_string) # return the collected option tuples return result @@ -2403,8 +2404,11 @@ def _get_nargs_pattern(self, action): def parse_intermixed_args(self, args=None, namespace=None): args, argv = self.parse_known_intermixed_args(args, namespace) if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) + msg = _('unrecognized arguments: %s') % ' '.join(argv) + if self.exit_on_error: + self.error(msg) + else: + raise ArgumentError(None, msg) return args def parse_known_intermixed_args(self, args=None, namespace=None): diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 374852949e3e25..d461911307703c 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2190,7 +2190,9 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, else: subparsers_kwargs['help'] = 'command help' subparsers = parser.add_subparsers(**subparsers_kwargs) - self.assertArgumentParserError(parser.add_subparsers) + self.assertRaisesRegex(argparse.ArgumentError, + 'cannot have multiple subparser arguments', + parser.add_subparsers) # add first sub-parser parser1_kwargs = dict(description='1 description') @@ -6085,7 +6087,8 @@ def test_help_with_metavar(self): class TestExitOnError(TestCase): def setUp(self): - self.parser = argparse.ArgumentParser(exit_on_error=False) + self.parser = argparse.ArgumentParser(exit_on_error=False, + fromfile_prefix_chars='@') self.parser.add_argument('--integers', metavar='N', type=int) def test_exit_on_error_with_good_args(self): @@ -6096,9 +6099,44 @@ def test_exit_on_error_with_bad_args(self): with self.assertRaises(argparse.ArgumentError): self.parser.parse_args('--integers a'.split()) - def test_exit_on_error_with_unrecognized_args(self): - with self.assertRaises(argparse.ArgumentError): - self.parser.parse_args('--foo bar'.split()) + def test_unrecognized_args(self): + self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --foo bar', + self.parser.parse_args, '--foo bar'.split()) + + def test_unrecognized_intermixed_args(self): + self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --foo bar', + self.parser.parse_intermixed_args, '--foo bar'.split()) + + def test_required_args(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz', + self.parser.parse_args, []) + + def test_required_mutually_exclusive_args(self): + group = self.parser.add_mutually_exclusive_group(required=True) + group.add_argument('--bar') + group.add_argument('--baz') + self.assertRaisesRegex(argparse.ArgumentError, + 'one of the arguments --bar --baz is required', + self.parser.parse_args, []) + + def test_ambiguous_option(self): + self.parser.add_argument('--foobaz') + self.parser.add_argument('--fooble', action='store_true') + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob could match --foobaz, --fooble", + self.parser.parse_args, ['--foob']) + + def test_os_error(self): + self.parser.add_argument('file') + self.assertRaisesRegex(argparse.ArgumentError, + "No such file or directory: 'no-such-file'", + self.parser.parse_args, ['@no-such-file']) + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. diff --git a/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst b/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst new file mode 100644 index 00000000000000..98a1044f887a7a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst @@ -0,0 +1,2 @@ +Fixed other issues where :class:`argparse.ArgumentParser` did not honor +``exit_on_error=False``.

1 0

[3.12] gh-121018: Fix more cases of exiting in argparse when exit_on_error=False (GH-121056) (GH-121129)
by serhiy-storchaka 28 Jun '24

28 Jun '24

https://github.com/python/cpython/commit/8ea6cc14a5178b456278c35f63b0859c6b… commit: 8ea6cc14a5178b456278c35f63b0859c6b5f4f64 branch: 3.12 author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2024-06-28T14:52:07Z summary: [3.12] gh-121018: Fix more cases of exiting in argparse when exit_on_error=False (GH-121056) (GH-121129) * parse_intermixed_args() now raises ArgumentError instead of calling error() if exit_on_error is false. * Internal code now always raises ArgumentError instead of calling error(). It is then caught at the higher level and error() is called if exit_on_error is true. (cherry picked from commit 81a654a3425eaa05a51342509089533c1f623f1b) files: M Lib/argparse.py M Lib/test/test_argparse.py M Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 0890718270f46e..e4892955e4ff09 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1843,7 +1843,7 @@ def _get_kwargs(self): # ================================== def add_subparsers(self, **kwargs): if self._subparsers is not None: - self.error(_('cannot have multiple subparser arguments')) + raise ArgumentError(None, _('cannot have multiple subparser arguments')) # add the parser class to the arguments if it's not present kwargs.setdefault('parser_class', type(self)) @@ -1898,7 +1898,8 @@ def parse_args(self, args=None, namespace=None): msg = _('unrecognized arguments: %s') % ' '.join(argv) if self.exit_on_error: self.error(msg) - raise ArgumentError(None, msg) + else: + raise ArgumentError(None, msg) return args def parse_known_args(self, args=None, namespace=None): @@ -2177,7 +2178,7 @@ def consume_positionals(start_index): self._get_value(action, action.default)) if required_actions: - self.error(_('the following arguments are required: %s') % + raise ArgumentError(None, _('the following arguments are required: %s') % ', '.join(required_actions)) # make sure all required groups had one option present @@ -2193,7 +2194,7 @@ def consume_positionals(start_index): for action in group._group_actions if action.help is not SUPPRESS] msg = _('one of the arguments %s is required') - self.error(msg % ' '.join(names)) + raise ArgumentError(None, msg % ' '.join(names)) # return the updated namespace and the extra arguments return namespace, extras @@ -2220,7 +2221,7 @@ def _read_args_from_files(self, arg_strings): arg_strings = self._read_args_from_files(arg_strings) new_arg_strings.extend(arg_strings) except OSError as err: - self.error(str(err)) + raise ArgumentError(None, str(err)) # return the modified argument list return new_arg_strings @@ -2300,7 +2301,7 @@ def _parse_optional(self, arg_string): for action, option_string, sep, explicit_arg in option_tuples]) args = {'option': arg_string, 'matches': options} msg = _('ambiguous option: %(option)s could match %(matches)s') - self.error(msg % args) + raise ArgumentError(None, msg % args) # if exactly one action matched, this segmentation is good, # so return the parsed action @@ -2360,7 +2361,7 @@ def _get_option_tuples(self, option_string): # shouldn't ever get here else: - self.error(_('unexpected option string: %s') % option_string) + raise ArgumentError(None, _('unexpected option string: %s') % option_string) # return the collected option tuples return result @@ -2417,8 +2418,11 @@ def _get_nargs_pattern(self, action): def parse_intermixed_args(self, args=None, namespace=None): args, argv = self.parse_known_intermixed_args(args, namespace) if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) + msg = _('unrecognized arguments: %s') % ' '.join(argv) + if self.exit_on_error: + self.error(msg) + else: + raise ArgumentError(None, msg) return args def parse_known_intermixed_args(self, args=None, namespace=None): diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index a4d6c4a35256f9..6a7723e8cd1e26 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2126,7 +2126,9 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, else: subparsers_kwargs['help'] = 'command help' subparsers = parser.add_subparsers(**subparsers_kwargs) - self.assertArgumentParserError(parser.add_subparsers) + self.assertRaisesRegex(argparse.ArgumentError, + 'cannot have multiple subparser arguments', + parser.add_subparsers) # add first sub-parser parser1_kwargs = dict(description='1 description') @@ -5732,7 +5734,8 @@ def test_help_with_metavar(self): class TestExitOnError(TestCase): def setUp(self): - self.parser = argparse.ArgumentParser(exit_on_error=False) + self.parser = argparse.ArgumentParser(exit_on_error=False, + fromfile_prefix_chars='@') self.parser.add_argument('--integers', metavar='N', type=int) def test_exit_on_error_with_good_args(self): @@ -5743,9 +5746,44 @@ def test_exit_on_error_with_bad_args(self): with self.assertRaises(argparse.ArgumentError): self.parser.parse_args('--integers a'.split()) - def test_exit_on_error_with_unrecognized_args(self): - with self.assertRaises(argparse.ArgumentError): - self.parser.parse_args('--foo bar'.split()) + def test_unrecognized_args(self): + self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --foo bar', + self.parser.parse_args, '--foo bar'.split()) + + def test_unrecognized_intermixed_args(self): + self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --foo bar', + self.parser.parse_intermixed_args, '--foo bar'.split()) + + def test_required_args(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz', + self.parser.parse_args, []) + + def test_required_mutually_exclusive_args(self): + group = self.parser.add_mutually_exclusive_group(required=True) + group.add_argument('--bar') + group.add_argument('--baz') + self.assertRaisesRegex(argparse.ArgumentError, + 'one of the arguments --bar --baz is required', + self.parser.parse_args, []) + + def test_ambiguous_option(self): + self.parser.add_argument('--foobaz') + self.parser.add_argument('--fooble', action='store_true') + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob could match --foobaz, --fooble", + self.parser.parse_args, ['--foob']) + + def test_os_error(self): + self.parser.add_argument('file') + self.assertRaisesRegex(argparse.ArgumentError, + "No such file or directory: 'no-such-file'", + self.parser.parse_args, ['@no-such-file']) + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. diff --git a/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst b/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst index fdc3c5f9e459af..b1d219dea3d36a 100644 --- a/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst +++ b/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst @@ -1,2 +1,3 @@ -Fixed an issue where :meth:`!argparse.ArgumentParser.parses_args` did not honor ``exit_on_error=False`` when given unrecognized arguments. -Patch by Ben Hsing. +Fixed issues where :meth:`!argparse.ArgumentParser.parses_args` did not honor +``exit_on_error=False``. +Based on patch by Ben Hsing.

1 0

[3.13] gh-115986 Improve pprint docs formatting (GH-117401) (GH-121098)
by encukou 28 Jun '24

28 Jun '24

https://github.com/python/cpython/commit/394dc93bf99d01f085b6a76ba057b1941c… commit: 394dc93bf99d01f085b6a76ba057b1941cc2ebc7 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: encukou <encukou(a)gmail.com> date: 2024-06-28T16:23:08+02:00 summary: [3.13] gh-115986 Improve pprint docs formatting (GH-117401) (GH-121098) * Move pprinter parameter descriptions to a table * Make pprint doc with params markup * Remove duplication of the parameters' description --------- (cherry picked from commit 0890ad7c024ccf29614849b6ffadcb92c0e91ce7) Co-authored-by: Kerim Kabirov <the.privat33r+gh(a)pm.me> Co-authored-by: Hugo van Kemenade <1324225+hugovk(a)users.noreply.github.com> files: M Doc/library/pprint.rst diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index df706c10ce9ec4..1b3498e51f766d 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -35,24 +35,66 @@ Dictionaries are sorted by key before the display is computed. Functions --------- -.. function:: pp(object, *args, sort_dicts=False, **kwargs) - - Prints the formatted representation of *object* followed by a newline. - If *sort_dicts* is false (the default), dictionaries will be displayed with - their keys in insertion order, otherwise the dict keys will be sorted. - *args* and *kwargs* will be passed to :func:`~pprint.pprint` as formatting - parameters. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff) - >>> pprint.pp(stuff) - [<Recursion on list with id=...>, - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] +.. function:: pp(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=False, underscore_numbers=False) + + Prints the formatted representation of *object*, followed by a newline. + This function may be used in the interactive interpreter + instead of the :func:`print` function for inspecting values. + Tip: you can reassign ``print = pprint.pp`` for use within a scope. + + :param object: + The object to be printed. + + :param stream: + A file-like object to which the output will be written + by calling its :meth:`!write` method. + If ``None`` (the default), :data:`sys.stdout` is used. + :type stream: :term:`file-like object` | None + + :param int indent: + The amount of indentation added for each nesting level. + + :param int width: + The desired maximum number of characters per line in the output. + If a structure cannot be formatted within the width constraint, + a best effort will be made. + + :param depth: + The number of nesting levels which may be printed. + If the data structure being printed is too deep, + the next contained level is replaced by ``...``. + If ``None`` (the default), there is no constraint + on the depth of the objects being formatted. + :type depth: int | None + + :param bool compact: + Control the way long :term:`sequences <sequence>` are formatted. + If ``False`` (the default), + each item of a sequence will be formatted on a separate line, + otherwise as many items as will fit within the *width* + will be formatted on each output line. + + :param bool sort_dicts: + If ``True``, dictionaries will be formatted with + their keys sorted, otherwise + they will be displayed in insertion order (the default). + + :param bool underscore_numbers: + If ``True``, + integers will be formatted with the ``_`` character for a thousands separator, + otherwise underscores are not displayed (the default). + + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff) + >>> pprint.pp(stuff) + [<Recursion on list with id=...>, + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] .. versionadded:: 3.8 @@ -60,19 +102,10 @@ Functions .. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ compact=False, sort_dicts=True, underscore_numbers=False) - Prints the formatted representation of *object* on *stream*, followed by a - newline. If *stream* is ``None``, :data:`sys.stdout` is used. This may be used - in the interactive interpreter instead of the :func:`print` function for - inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). - - The configuration parameters *stream*, *indent*, *width*, *depth*, - *compact*, *sort_dicts* and *underscore_numbers* are passed to the - :class:`PrettyPrinter` constructor and their meanings are as - described in its documentation below. + Alias for :func:`~pprint.pp` with *sort_dicts* set to ``True`` by default, + which would automatically sort the dictionaries' keys, + you might want to use :func:`~pprint.pp` instead where it is ``False`` by default. - Note that *sort_dicts* is ``True`` by default and you might want to use - :func:`~pprint.pp` instead where it is ``False`` by default. .. function:: pformat(object, indent=1, width=80, depth=None, *, \ compact=False, sort_dicts=True, underscore_numbers=False) @@ -80,7 +113,7 @@ Functions Return the formatted representation of *object* as a string. *indent*, *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are passed to the :class:`PrettyPrinter` constructor as formatting parameters - and their meanings are as described in its documentation below. + and their meanings are as described in the documentation above. .. function:: isreadable(object) @@ -119,51 +152,39 @@ Functions PrettyPrinter Objects --------------------- -This module defines one class: - -.. First the implementation class: - - .. index:: single: ...; placeholder .. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ compact=False, sort_dicts=True, underscore_numbers=False) - Construct a :class:`PrettyPrinter` instance. This constructor understands - several keyword parameters. - - *stream* (default :data:`!sys.stdout`) is a :term:`file-like object` to - which the output will be written by calling its :meth:`!write` method. - If both *stream* and :data:`!sys.stdout` are ``None``, then - :meth:`~PrettyPrinter.pprint` silently returns. + Construct a :class:`PrettyPrinter` instance. - Other values configure the manner in which nesting of complex data - structures is displayed. + Arguments have the same meaning as for :func:`~pprint.pp`. + Note that they are in a different order, and that *sort_dicts* defaults to ``True``. - *indent* (default 1) specifies the amount of indentation added for - each nesting level. - - *depth* controls the number of nesting levels which may be printed; if - the data structure being printed is too deep, the next contained level - is replaced by ``...``. By default, there is no constraint on the - depth of the objects being formatted. - - *width* (default 80) specifies the desired maximum number of characters per - line in the output. If a structure cannot be formatted within the width - constraint, a best effort will be made. - - *compact* impacts the way that long sequences (lists, tuples, sets, etc) - are formatted. If *compact* is false (the default) then each item of a - sequence will be formatted on a separate line. If *compact* is true, as - many items as will fit within the *width* will be formatted on each output - line. - - If *sort_dicts* is true (the default), dictionaries will be formatted with - their keys sorted, otherwise they will display in insertion order. + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff[:]) + >>> pp = pprint.PrettyPrinter(indent=4) + >>> pp.pprint(stuff) + [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] + >>> pp = pprint.PrettyPrinter(width=41, compact=True) + >>> pp.pprint(stuff) + [['spam', 'eggs', 'lumberjack', + 'knights', 'ni'], + 'spam', 'eggs', 'lumberjack', 'knights', + 'ni'] + >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', + ... ('parrot', ('fresh fruit',)))))))) + >>> pp = pprint.PrettyPrinter(depth=6) + >>> pp.pprint(tup) + ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) - If *underscore_numbers* is true, integers will be formatted with the - ``_`` character for a thousands separator, otherwise underscores are not - displayed (the default). .. versionchanged:: 3.4 Added the *compact* parameter. @@ -177,29 +198,6 @@ This module defines one class: .. versionchanged:: 3.11 No longer attempts to write to :data:`!sys.stdout` if it is ``None``. - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff[:]) - >>> pp = pprint.PrettyPrinter(indent=4) - >>> pp.pprint(stuff) - [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] - >>> pp = pprint.PrettyPrinter(width=41, compact=True) - >>> pp.pprint(stuff) - [['spam', 'eggs', 'lumberjack', - 'knights', 'ni'], - 'spam', 'eggs', 'lumberjack', 'knights', - 'ni'] - >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', - ... ('parrot', ('fresh fruit',)))))))) - >>> pp = pprint.PrettyPrinter(depth=6) - >>> pp.pprint(tup) - ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) - :class:`PrettyPrinter` instances have the following methods:

1 0