Mailman 3 June 2019 - Python-checkins

bpo-33569 Preserve type information with dataclasses.InitVar (GH-8927)
by Eric V. Smith June 3, 2019

June 3, 2019

https://github.com/python/cpython/commit/01ee12ba35a333e8a6a25c4153c4a21838… commit: 01ee12ba35a333e8a6a25c4153c4a21838e9585c branch: master author: Augusto Hack <hack.augusto(a)gmail.com> committer: Eric V. Smith <ericvsmith(a)users.noreply.github.com> date: 2019-06-02T22:14:48-04:00 summary: bpo-33569 Preserve type information with dataclasses.InitVar (GH-8927) files: A Misc/NEWS.d/next/Library/2018-08-28-03-00-12.bpo-33569.45YlGG.rst M Lib/dataclasses.py M Lib/test/… [View More]test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 75113f123b3a..b035cbb809f8 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -201,10 +201,16 @@ def __repr__(self): class _InitVarMeta(type): def __getitem__(self, params): - return self + return InitVar(params) class InitVar(metaclass=_InitVarMeta): - pass + __slots__ = ('type', ) + + def __init__(self, type): + self.type = type + + def __repr__(self): + return f'dataclasses.InitVar[{self.type.__name__}]' # Instances of Field are only ever created from within this module, @@ -586,7 +592,8 @@ def _is_classvar(a_type, typing): def _is_initvar(a_type, dataclasses): # The module we're checking against is the module we're # currently in (dataclasses.py). - return a_type is dataclasses.InitVar + return (a_type is dataclasses.InitVar + or type(a_type) is dataclasses.InitVar) def _is_type(annotation, cls, a_module, a_type, is_type_predicate): diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 867210688f57..53e8443c2adf 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -1097,6 +1097,12 @@ def __post_init__(self, init_param): c = C(init_param=10) self.assertEqual(c.x, 20) + def test_init_var_preserve_type(self): + self.assertEqual(InitVar[int].type, int) + + # Make sure the repr is correct. + self.assertEqual(repr(InitVar[int]), 'dataclasses.InitVar[int]') + def test_init_var_inheritance(self): # Note that this deliberately tests that a dataclass need not # have a __post_init__ function if it has an InitVar field. diff --git a/Misc/NEWS.d/next/Library/2018-08-28-03-00-12.bpo-33569.45YlGG.rst b/Misc/NEWS.d/next/Library/2018-08-28-03-00-12.bpo-33569.45YlGG.rst new file mode 100644 index 000000000000..adafa2803ae3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-08-28-03-00-12.bpo-33569.45YlGG.rst @@ -0,0 +1 @@ +dataclasses.InitVar: Exposes the type used to create the init var. [View Less]

1 0

bpo-37069: tests use catch_unraisable_exception() (GH-13762)
by Victor Stinner June 3, 2019

June 3, 2019

https://github.com/python/cpython/commit/0025350294959594e7f57aef4fc9579c77… commit: 0025350294959594e7f57aef4fc9579c77a0ed1c branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2019-06-03T03:51:43+02:00 summary: bpo-37069: tests use catch_unraisable_exception() (GH-13762) Modify test_coroutines, test_cprofile, test_generators, test_raise, test_ssl and test_yield_from to use support.catch_unraisable_exception() rather than … [View More]support.captured_stderr(). test_thread: remove test_save_exception_state_on_error() which is now updated. test_unraisable_exception() checks that sys.unraisablehook() is called to handle _thread.start_new_thread() exception. test_cprofile now rely on unittest for test discovery: replace support.run_unittest() with unittest.main(). files: A Misc/NEWS.d/next/Tests/2019-06-03-02-30-36.bpo-37069.rVtdLk.rst M Lib/test/test_coroutines.py M Lib/test/test_cprofile.py M Lib/test/test_generators.py M Lib/test/test_raise.py M Lib/test/test_ssl.py M Lib/test/test_thread.py M Lib/test/test_yield_from.py diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index b406b1c3ebf8..208b5c2ccf5c 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2032,11 +2032,17 @@ def test_pickle(self): def test_fatal_coro_warning(self): # Issue 27811 async def func(): pass - with warnings.catch_warnings(), support.captured_stderr() as stderr: + with warnings.catch_warnings(), \ + support.catch_unraisable_exception() as cm: warnings.filterwarnings("error") - func() + coro = func() + # only store repr() to avoid keeping the coroutine alive + coro_repr = repr(coro) + coro = None support.gc_collect() - self.assertIn("was never awaited", stderr.getvalue()) + + self.assertIn("was never awaited", str(cm.unraisable.exc_value)) + self.assertEqual(repr(cm.unraisable.object), coro_repr) def test_for_assign_raising_stop_async_iteration(self): class BadTarget: diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index efcf6bc92803..5c70037f39a2 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -1,13 +1,13 @@ """Test suite for the cProfile module.""" import sys -from test.support import run_unittest, TESTFN, unlink import unittest # rip off all interesting stuff from test_profile import cProfile from test.test_profile import ProfileTest, regenerate_expected_output from test.support.script_helper import assert_python_failure, assert_python_ok +from test import support class CProfileTest(ProfileTest): @@ -18,24 +18,18 @@ class CProfileTest(ProfileTest): def get_expected_output(self): return _ProfileOutput - # Issue 3895. def test_bad_counter_during_dealloc(self): + # bpo-3895 import _lsprof - # Must use a file as StringIO doesn't trigger the bug. - orig_stderr = sys.stderr - try: - with open(TESTFN, 'w') as file: - sys.stderr = file - try: - obj = _lsprof.Profiler(lambda: int) - obj.enable() - obj = _lsprof.Profiler(1) - obj.disable() - obj.clear() - finally: - sys.stderr = orig_stderr - finally: - unlink(TESTFN) + + with support.catch_unraisable_exception() as cm: + obj = _lsprof.Profiler(lambda: int) + obj.enable() + obj = _lsprof.Profiler(1) + obj.disable() + obj.clear() + + self.assertEqual(cm.unraisable.exc_type, TypeError) def test_profile_enable_disable(self): prof = self.profilerclass() @@ -70,12 +64,10 @@ def test_sort(self): self.assertGreater(rc, 0) self.assertIn(b"option -s: invalid choice: 'demo'", err) -def test_main(): - run_unittest(CProfileTest, TestCommandLine) def main(): if '-r' not in sys.argv: - test_main() + unittest.main() else: regenerate_expected_output(__file__, CProfileTest) diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 7f1472fa03ac..a34e4ec2eda7 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -2051,15 +2051,17 @@ def printsolution(self, x): Our ill-behaved code should be invoked during GC: ->>> import sys, io ->>> old, sys.stderr = sys.stderr, io.StringIO() ->>> g = f() ->>> next(g) ->>> del g ->>> "RuntimeError: generator ignored GeneratorExit" in sys.stderr.getvalue() +>>> with support.catch_unraisable_exception() as cm: +... g = f() +... next(g) +... del g +... +... cm.unraisable.exc_type == RuntimeError +... "generator ignored GeneratorExit" in str(cm.unraisable.exc_value) +... cm.unraisable.exc_traceback is not None +True +True True ->>> sys.stderr = old - And errors thrown during closing should propagate: diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index c1ef154a9a9f..57da0e15a756 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -459,9 +459,12 @@ def f(): self.assertNotEqual(e.__context__, None) self.assertIsInstance(e.__context__, AttributeError) - with support.captured_output("stderr"): + with support.catch_unraisable_exception() as cm: f() + self.assertEqual(ZeroDivisionError, cm.unraisable.exc_type) + + class TestRemovedFunctionality(unittest.TestCase): def test_tuples(self): try: diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index f368906c8a94..a72d79132181 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4051,13 +4051,15 @@ def cb_raising(ssl_sock, server_name, initial_context): 1/0 server_context.set_servername_callback(cb_raising) - with self.assertRaises(ssl.SSLError) as cm, \ - support.captured_stderr() as stderr: - stats = server_params_test(client_context, server_context, - chatty=False, - sni_name='supermessage') - self.assertEqual(cm.exception.reason, 'SSLV3_ALERT_HANDSHAKE_FAILURE') - self.assertIn("ZeroDivisionError", stderr.getvalue()) + with support.catch_unraisable_exception() as catch: + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + + self.assertEqual(cm.exception.reason, + 'SSLV3_ALERT_HANDSHAKE_FAILURE') + self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) @needs_sni def test_sni_callback_wrong_return_type(self): @@ -4069,13 +4071,15 @@ def cb_wrong_return_type(ssl_sock, server_name, initial_context): return "foo" server_context.set_servername_callback(cb_wrong_return_type) - with self.assertRaises(ssl.SSLError) as cm, \ - support.captured_stderr() as stderr: - stats = server_params_test(client_context, server_context, - chatty=False, - sni_name='supermessage') - self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') - self.assertIn("TypeError", stderr.getvalue()) + with support.catch_unraisable_exception() as catch: + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + + + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') + self.assertEqual(catch.unraisable.exc_type, TypeError) def test_shared_ciphers(self): client_context, server_context, hostname = testing_context() diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index f946f7bc8399..9f4801f47e3a 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -133,27 +133,6 @@ def task(): time.sleep(POLL_SLEEP) self.assertEqual(thread._count(), orig) - def test_save_exception_state_on_error(self): - # See issue #14474 - def task(): - started.release() - raise SyntaxError - def mywrite(self, *args): - try: - raise ValueError - except ValueError: - pass - real_write(self, *args) - started = thread.allocate_lock() - with support.captured_output("stderr") as stderr: - real_write = stderr.write - stderr.write = mywrite - started.acquire() - with support.wait_threads_exit(): - thread.start_new_thread(task, ()) - started.acquire() - self.assertIn("Traceback", stderr.getvalue()) - def test_unraisable_exception(self): def task(): started.release() diff --git a/Lib/test/test_yield_from.py b/Lib/test/test_yield_from.py index ce21c3df8140..4735ef4bee3b 100644 --- a/Lib/test/test_yield_from.py +++ b/Lib/test/test_yield_from.py @@ -11,6 +11,7 @@ import inspect from test.support import captured_stderr, disable_gc, gc_collect +from test import support class TestPEP380Operation(unittest.TestCase): """ @@ -562,11 +563,12 @@ def g(): self.assertEqual(next(gi), 1) gi.throw(AttributeError) - with captured_stderr() as output: + with support.catch_unraisable_exception() as cm: gi = g() self.assertEqual(next(gi), 1) gi.close() - self.assertIn('ZeroDivisionError', output.getvalue()) + + self.assertEqual(ZeroDivisionError, cm.unraisable.exc_type) def test_exception_in_initial_next_call(self): """ diff --git a/Misc/NEWS.d/next/Tests/2019-06-03-02-30-36.bpo-37069.rVtdLk.rst b/Misc/NEWS.d/next/Tests/2019-06-03-02-30-36.bpo-37069.rVtdLk.rst new file mode 100644 index 000000000000..566ff5150d5f --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2019-06-03-02-30-36.bpo-37069.rVtdLk.rst @@ -0,0 +1,3 @@ +Modify test_coroutines, test_cprofile, test_generators, test_raise, test_ssl +and test_yield_from to use :func:`test.support.catch_unraisable_exception` +rather than :func:`test.support.captured_stderr`. [View Less]

1 0

[3.7] bpo-27987: align PyGC_Head to alignof(long double) (GH-13335) (GH-13581)
by Gregory P. Smith June 3, 2019

June 3, 2019

https://github.com/python/cpython/commit/8766cb74e186d3820db0a855ccd780d6d8… commit: 8766cb74e186d3820db0a855ccd780d6d84461f7 branch: 3.7 author: Inada Naoki <songofacandy(a)gmail.com> committer: Gregory P. Smith <greg(a)krypto.org> date: 2019-06-02T18:51:31-07:00 summary: [3.7] bpo-27987: align PyGC_Head to alignof(long double) (GH-13335) (GH-13581) This reverts commit 2156fec1f7a8f9972e90cdbaf404e3fd9eaccb35. Now that https://github.com/python/cpython/commit/… [View More]

1 0

bpo-36896: Clarify that some types constructors are unstable (GH-13271)
by Petr Viktorin June 3, 2019

June 3, 2019

https://github.com/python/cpython/commit/13136e83a637a9f1cfbada7e9309700529… commit: 13136e83a637a9f1cfbada7e93097005296659b4 branch: master author: Matthias Bussonnier <bussonniermatthias(a)gmail.com> committer: Petr Viktorin <encukou(a)gmail.com> date: 2019-06-03T02:43:21+02:00 summary: bpo-36896: Clarify that some types constructors are unstable (GH-13271) files: A Misc/NEWS.d/next/Documentation/2019-05-31-10-46-36.bpo-36896.wkXTW9.rst M Doc/library/types.rst diff --git a/Doc/… [View More]

1 0

bpo-36027: Really fix "incompatible pointer type" compiler warning (GH-13761)
by Petr Viktorin June 3, 2019

June 3, 2019

https://github.com/python/cpython/commit/1e375c6269e9de4f3d05d4aa6d6d74e00f… commit: 1e375c6269e9de4f3d05d4aa6d6d74e00f522d63 branch: master author: Petr Viktorin <encukou(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-03T02:28:29+02:00 summary: bpo-36027: Really fix "incompatible pointer type" compiler warning (GH-13761) Apologies for the earlier hasty attempt. files: M Objects/longobject.c diff --git a/Objects/longobject.c b/Objects/longobject.c index … [View More]

1 0

bpo-36974: add some assertions for PEP 590 (GH-13682)
by Petr Viktorin June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/be718c33f06b3496faa61142df24fb071f… commit: be718c33f06b3496faa61142df24fb071fd5d1f1 branch: master author: Jeroen Demeyer <J.Demeyer(a)UGent.be> committer: Petr Viktorin <encukou(a)gmail.com> date: 2019-06-03T01:57:22+02:00 summary: bpo-36974: add some assertions for PEP 590 (GH-13682) files: M Objects/typeobject.c diff --git a/Objects/typeobject.c b/Objects/typeobject.c index beb0ddd82400..006df8d1f090 100644 --- a/Objects/typeobject.c ++… [View More]

1 0

bpo-36974: document PEP 590 (GH-13450)
by Petr Viktorin June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/9e3e06e582accec82eb29cf665c3b4c7d8… commit: 9e3e06e582accec82eb29cf665c3b4c7d84d2eb0 branch: master author: Jeroen Demeyer <J.Demeyer(a)UGent.be> committer: Petr Viktorin <encukou(a)gmail.com> date: 2019-06-03T01:43:13+02:00 summary: bpo-36974: document PEP 590 (GH-13450) files: M Doc/c-api/object.rst M Doc/c-api/typeobj.rst M Doc/includes/typestruct.h M Doc/whatsnew/3.8.rst diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index … [View More]ffc35241e7a4..ce0d05942f4e 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -335,6 +335,83 @@ Object Protocol *NULL* on failure. +.. c:function:: PyObject* _PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + + Call a callable Python object *callable*, using + :c:data:`vectorcall <PyTypeObject.tp_vectorcall_offset>` if possible. + + *args* is a C array with the positional arguments. + + *nargsf* is the number of positional arguments plus optionally the flag + :const:`PY_VECTORCALL_ARGUMENTS_OFFSET` (see below). + To get actual number of arguments, use + :c:func:`PyVectorcall_NARGS(nargsf) <PyVectorcall_NARGS>`. + + *kwnames* can be either NULL (no keyword arguments) or a tuple of keyword + names. In the latter case, the values of the keyword arguments are stored + in *args* after the positional arguments. + The number of keyword arguments does not influence *nargsf*. + + *kwnames* must contain only objects of type ``str`` (not a subclass), + and all keys must be unique. + + Return the result of the call on success, or *NULL* on failure. + + This uses the vectorcall protocol if the callable supports it; + otherwise, the arguments are converted to use + :c:member:`~PyTypeObject.tp_call`. + + .. note:: + + This function is provisional and expected to become public in Python 3.9, + with a different name and, possibly, changed semantics. + If you use the function, plan for updating your code for Python 3.9. + + .. versionadded:: 3.8 + +.. c:var:: PY_VECTORCALL_ARGUMENTS_OFFSET + + If set in a vectorcall *nargsf* argument, the callee is allowed to + temporarily change ``args[-1]``. In other words, *args* points to + argument 1 (not 0) in the allocated vector. + The callee must restore the value of ``args[-1]`` before returning. + + Whenever they can do so cheaply (without additional allocation), callers + are encouraged to use :const:`PY_VECTORCALL_ARGUMENTS_OFFSET`. + Doing so will allow callables such as bound methods to make their onward + calls (which include a prepended *self* argument) cheaply. + + .. versionadded:: 3.8 + +.. c:function:: Py_ssize_t PyVectorcall_NARGS(size_t nargsf) + + Given a vectorcall *nargsf* argument, return the actual number of + arguments. + Currently equivalent to ``nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET``. + + .. versionadded:: 3.8 + +.. c:function:: PyObject* _PyObject_FastCallDict(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwdict) + + Same as :c:func:`_PyObject_Vectorcall` except that the keyword arguments + are passed as a dictionary in *kwdict*. This may be *NULL* if there + are no keyword arguments. + + For callables supporting :c:data:`vectorcall <PyTypeObject.tp_vectorcall_offset>`, + the arguments are internally converted to the vectorcall convention. + Therefore, this function adds some overhead compared to + :c:func:`_PyObject_Vectorcall`. + It should only be used if the caller already has a dictionary ready to use. + + .. note:: + + This function is provisional and expected to become public in Python 3.9, + with a different name and, possibly, changed semantics. + If you use the function, plan for updating your code for Python 3.9. + + .. versionadded:: 3.8 + + .. c:function:: Py_hash_t PyObject_Hash(PyObject *o) .. index:: builtin: hash diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index e2f8f54be79a..83fcc5abed70 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -36,115 +36,115 @@ Quick Reference .. table:: :widths: 18,18,18,1,1,1,1 - +---------------------------------------------+-----------------------------------+-------------------+---------------+ - | PyTypeObject Slot [#slots]_ | :ref:`Type <slot-typedefs-table>` | special | Info [#cols]_ | - | | | methods/attrs +---+---+---+---+ - | | | | O | T | D | I | - +=============================================+===================================+===================+===+===+===+===+ - | <R> :c:member:`~PyTypeObject.tp_name` | const char * | __name__ | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_basicsize` | Py_ssize_t | | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_itemsize` | Py_ssize_t | | | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dealloc` | :c:type:`destructor` | | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_print`) | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_getattr`) | :c:type:`getattrfunc` | __getattribute__, | | | | G | - | | | __getattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_setattr`) | :c:type:`setattrfunc` | __setattr__, | | | | G | - | | | __delattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_async` | :c:type:`PyAsyncMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_repr` | :c:type:`reprfunc` | __repr__ | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_number` | :c:type:`PyNumberMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_sequence` | :c:type:`PySequenceMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_mapping` | :c:type:`PyMappingMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_hash` | :c:type:`hashfunc` | __hash__ | X | | | G | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_call` | :c:type:`ternaryfunc` | __call__ | | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_str` | :c:type:`reprfunc` | __str__ | X | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_getattro` | :c:type:`getattrofunc` | __getattribute__, | X | X | | G | - | | | __getattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | - | | | __delattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_doc` | const char * | __doc__ | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_traverse` | :c:type:`traverseproc` | | | X | | G | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_clear` | :c:type:`inquiry` | | | X | | G | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_richcompare` | :c:type:`richcmpfunc` | __lt__, | X | | | G | - | | | __le__, | | | | | - | | | __eq__, | | | | | - | | | __ne__, | | | | | - | | | __gt__, | | | | | - | | | __ge__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_weaklistoffset` | Py_ssize_t | | | X | | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_iter` | :c:type:`getiterfunc` | __iter__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_iternext` | :c:type:`iternextfunc` | __next__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_methods` | :c:type:`PyMethodDef` [] | | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_members` | :c:type:`PyMemberDef` [] | | | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_getset` | :c:type:`PyGetSetDef` [] | | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_base` | :c:type:`PyTypeObject` * | __base__ | | | X | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dict` | :c:type:`PyObject` * | __dict__ | | | ? | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_descr_get` | :c:type:`descrgetfunc` | __get__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_descr_set` | :c:type:`descrsetfunc` | __set__, | | | | X | - | | | __delete__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dictoffset` | Py_ssize_t | | | X | | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_init` | :c:type:`initproc` | __init__ | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_alloc` | :c:type:`allocfunc` | | X | | ? | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_new` | :c:type:`newfunc` | __new__ | X | X | ? | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_free` | :c:type:`freefunc` | | X | X | ? | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_is_gc` | :c:type:`inquiry` | | | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | <:c:member:`~PyTypeObject.tp_bases`> | :c:type:`PyObject` * | __bases__ | | | ~ | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | <:c:member:`~PyTypeObject.tp_mro`> | :c:type:`PyObject` * | __mro__ | | | ~ | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_cache`] | :c:type:`PyObject` * | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_subclasses`] | :c:type:`PyObject` * | __subclasses__ | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_weaklist`] | :c:type:`PyObject` * | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_del`) | :c:type:`destructor` | | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_version_tag`] | unsigned int | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_finalize` | :c:type:`destructor` | __del__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + +------------------------------------------------+-----------------------------------+-------------------+---------------+ + | PyTypeObject Slot [#slots]_ | :ref:`Type <slot-typedefs-table>` | special | Info [#cols]_ | + | | | methods/attrs +---+---+---+---+ + | | | | O | T | D | I | + +================================================+===================================+===================+===+===+===+===+ + | <R> :c:member:`~PyTypeObject.tp_name` | const char * | __name__ | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_basicsize` | Py_ssize_t | | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_itemsize` | Py_ssize_t | | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_dealloc` | :c:type:`destructor` | | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_vectorcall_offset` | Py_ssize_t | | | | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | (:c:member:`~PyTypeObject.tp_getattr`) | :c:type:`getattrfunc` | __getattribute__, | | | | G | + | | | __getattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | (:c:member:`~PyTypeObject.tp_setattr`) | :c:type:`setattrfunc` | __setattr__, | | | | G | + | | | __delattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_async` | :c:type:`PyAsyncMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_repr` | :c:type:`reprfunc` | __repr__ | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_number` | :c:type:`PyNumberMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_sequence` | :c:type:`PySequenceMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_mapping` | :c:type:`PyMappingMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_hash` | :c:type:`hashfunc` | __hash__ | X | | | G | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_call` | :c:type:`ternaryfunc` | __call__ | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_str` | :c:type:`reprfunc` | __str__ | X | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_getattro` | :c:type:`getattrofunc` | __getattribute__, | X | X | | G | + | | | __getattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | + | | | __delattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_doc` | const char * | __doc__ | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_traverse` | :c:type:`traverseproc` | | | X | | G | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_clear` | :c:type:`inquiry` | | | X | | G | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_richcompare` | :c:type:`richcmpfunc` | __lt__, | X | | | G | + | | | __le__, | | | | | + | | | __eq__, | | | | | + | | | __ne__, | | | | | + | | | __gt__, | | | | | + | | | __ge__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_weaklistoffset` | Py_ssize_t | | | X | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_iter` | :c:type:`getiterfunc` | __iter__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_iternext` | :c:type:`iternextfunc` | __next__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_methods` | :c:type:`PyMethodDef` [] | | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_members` | :c:type:`PyMemberDef` [] | | | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_getset` | :c:type:`PyGetSetDef` [] | | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_base` | :c:type:`PyTypeObject` * | __base__ | | | X | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_dict` | :c:type:`PyObject` * | __dict__ | | | ? | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_descr_get` | :c:type:`descrgetfunc` | __get__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_descr_set` | :c:type:`descrsetfunc` | __set__, | | | | X | + | | | __delete__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_dictoffset` | Py_ssize_t | | | X | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_init` | :c:type:`initproc` | __init__ | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_alloc` | :c:type:`allocfunc` | | X | | ? | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_new` | :c:type:`newfunc` | __new__ | X | X | ? | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_free` | :c:type:`freefunc` | | X | X | ? | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_is_gc` | :c:type:`inquiry` | | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | <:c:member:`~PyTypeObject.tp_bases`> | :c:type:`PyObject` * | __bases__ | | | ~ | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | <:c:member:`~PyTypeObject.tp_mro`> | :c:type:`PyObject` * | __mro__ | | | ~ | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_cache`] | :c:type:`PyObject` * | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_subclasses`] | :c:type:`PyObject` * | __subclasses__ | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_weaklist`] | :c:type:`PyObject` * | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | (:c:member:`~PyTypeObject.tp_del`) | :c:type:`destructor` | | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_version_tag`] | unsigned int | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_finalize` | :c:type:`destructor` | __del__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ If :const:`COUNT_ALLOCS` is defined then the following (internal-only) fields exist as well: @@ -364,12 +364,6 @@ slot typedefs +-----------------------------+-----------------------------+----------------------+ | :c:type:`reprfunc` | :c:type:`PyObject` * | :c:type:`PyObject` * | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`printfunc` | .. line-block:: | int | -| | | | -| | :c:type:`PyObject` * | | -| | FILE * | | -| | int | | -+-----------------------------+-----------------------------+----------------------+ | :c:type:`getattrfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | | | :c:type:`PyObject` * | | @@ -675,9 +669,66 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes. -.. c:member:: printfunc PyTypeObject.tp_print +.. c:member:: Py_ssize_t PyTypeObject.tp_vectorcall_offset + + An optional offset to a per-instance function that implements calling + the object using the *vectorcall* protocol, a more efficient alternative + of the simpler :c:member:`~PyTypeObject.tp_call`. + + This field is only used if the flag :const:`_Py_TPFLAGS_HAVE_VECTORCALL` + is set. If so, this must be a positive integer containing the offset in the + instance of a :c:type:`vectorcallfunc` pointer. + The signature is the same as for :c:func:`_PyObject_Vectorcall`:: + + PyObject *vectorcallfunc(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + + The *vectorcallfunc* pointer may be zero, in which case the instance behaves + as if :const:`_Py_TPFLAGS_HAVE_VECTORCALL` was not set: calling the instance + falls back to :c:member:`~PyTypeObject.tp_call`. + + Any class that sets ``_Py_TPFLAGS_HAVE_VECTORCALL`` must also set + :c:member:`~PyTypeObject.tp_call` and make sure its behaviour is consistent + with the *vectorcallfunc* function. + This can be done by setting *tp_call* to ``PyVectorcall_Call``: + + .. c:function:: PyObject *PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict) + + Call *callable*'s *vectorcallfunc* with positional and keyword + arguments given in a tuple and dict, respectively. + + This function is intended to be used in the ``tp_call`` slot. + It does not fall back to ``tp_call`` and it currently does not check the + ``_Py_TPFLAGS_HAVE_VECTORCALL`` flag. + To call an object, use one of the :c:func:`PyObject_Call <PyObject_Call>` + functions instead. + + .. note:: + + It is not recommended for :ref:`heap types <heap-types>` to implement + the vectorcall protocol. + When a user sets ``__call__`` in Python code, only ``tp_call`` is updated, + possibly making it inconsistent with the vectorcall function. + + .. note:: + + The semantics of the ``tp_vectorcall_offset`` slot are provisional and + expected to be finalized in Python 3.9. + If you use vectorcall, plan for updating your code for Python 3.9. + + .. versionchanged:: 3.8 + + This slot was used for print formatting in Python 2.x. + In Python 3.0 to 3.7, it was reserved and named ``tp_print``. - Reserved slot, formerly used for print formatting in Python 2.x. + **Inheritance:** + + This field is inherited by subtypes together with + :c:member:`~PyTypeObject.tp_call`: a subtype inherits + :c:member:`~PyTypeObject.tp_vectorcall_offset` from its base type when + the subtype’s :c:member:`~PyTypeObject.tp_call` is NULL. + + Note that `heap types`_ (including subclasses defined in Python) do not + inherit the :const:`_Py_TPFLAGS_HAVE_VECTORCALL` flag. .. c:member:: getattrfunc PyTypeObject.tp_getattr @@ -1104,6 +1155,28 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_finalize` slot is always present in the type structure. + .. data:: _Py_TPFLAGS_HAVE_VECTORCALL + + This bit is set when the class implements the vectorcall protocol. + See :c:member:`~PyTypeObject.tp_vectorcall_offset` for details. + + **Inheritance:** + + This bit is set on *static* subtypes if ``tp_flags`` is not overridden: + a subtype inherits ``_Py_TPFLAGS_HAVE_VECTORCALL`` from its base type + when the subtype’s :c:member:`~PyTypeObject.tp_call` is NULL + and the subtype's ``Py_TPFLAGS_HEAPTYPE`` is not set. + + `Heap types`_ do not inherit ``_Py_TPFLAGS_HAVE_VECTORCALL``. + + .. note:: + + This flag is provisional and expected to become public in Python 3.9, + with a different name and, possibly, changed semantics. + If you use vectorcall, plan for updating your code for Python 3.9. + + .. versionadded:: 3.8 + .. c:member:: const char* PyTypeObject.tp_doc @@ -2286,6 +2359,14 @@ Slot Type typedefs .. c:type:: void (*destructor)(PyObject *) +.. c:type:: PyObject *(*vectorcallfunc)(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + + See :c:member:`~PyTypeObject.tp_vectorcall_offset`. + + Arguments to ``vectorcallfunc`` are the same as for :c:func:`_PyObject_Vectorcall`. + + .. versionadded:: 3.8 + .. c:type:: void (*freefunc)(void *) See :c:member:`~PyTypeObject.tp_free`. @@ -2302,10 +2383,6 @@ Slot Type typedefs See :c:member:`~PyTypeObject.tp_repr`. -.. c:type:: int (*printfunc)(PyObject *, FILE *, int) - - This is hidden if :const:`PY_LIMITED_API` is set. - .. c:type:: PyObject *(*getattrfunc)(PyObject *self, char *attr) Return the value of the named attribute for the object. @@ -2409,7 +2486,7 @@ with a more verbose initializer:: sizeof(MyObject), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)myobj_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index 9f47899a198e..9ada03cfc4a4 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -6,7 +6,7 @@ typedef struct _typeobject { /* Methods to implement standard operations */ destructor tp_dealloc; - printfunc tp_print; + Py_ssize_t tp_vectorcall_offset; getattrfunc tp_getattr; setattrfunc tp_setattr; PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index e9c9c814c69a..9474a2f4aafe 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -238,6 +238,22 @@ See :pep:`587` for a full description. (Contributed by Victor Stinner in :issue:`36763`.) +Vectorcall: a fast calling protocol for CPython +----------------------------------------------- + +The "vectorcall" protocol is added to the Python/C API. +It is meant to formalize existing optimizations which were already done +for various classes. +Any extension type implementing a callable can use this protocol. + +This is currently provisional, +the aim is to make it fully public in Python 3.9. + +See :pep:`590` for a full description. + +(Contributed by Jeroen Demeyer and Mark Shannon in :issue:`36974`.) + + Other Language Changes ====================== [View Less]

1 0

Update the annotated assignment docs (GH-13757)
by Ivan Levkivskyi June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/82eac26a73107ded733110cf11e59e95f4… commit: 82eac26a73107ded733110cf11e59e95f41c197e branch: master author: Ivan Levkivskyi <levkivskyi(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-03T00:41:00+01:00 summary: Update the annotated assignment docs (GH-13757) files: M Doc/reference/simple_stmts.rst diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index af7c0caff627..9c0430da1fb2 100644 --- a/… [View More]

1 0

bpo-37100: Fix test_coroutines with -Werror (GH-13756)
by Victor Stinner June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/3cf7ea1272fbc921a89acdbe40ca152813… commit: 3cf7ea1272fbc921a89acdbe40ca152813028cb5 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2019-06-03T01:35:37+02:00 summary: bpo-37100: Fix test_coroutines with -Werror (GH-13756) test_coroutines: test_unawaited_warning_when_module_broken() now uses support.check_warnings() to catch expected RuntimeWarning. files: M Lib/test/… [View More]

1 0

bpo-37012: Clean up special cases in PyType_FromSpecWithBases slot assignments (GH-13496)
by Petr Viktorin June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/7f4ae1b2cc60cb69938e7c88793b9e9a2d… commit: 7f4ae1b2cc60cb69938e7c88793b9e9a2dd36d93 branch: master author: Petr Viktorin <encukou(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-03T01:31:12+02:00 summary: bpo-37012: Clean up special cases in PyType_FromSpecWithBases slot assignments (GH-13496) The main slot assignment loop is now if-else if ladder, making the control flow clearer. Based on suggestion by Victor Stinner … [View More]in: https://github.com/python/cpython/pull/10304/#issuecomment-491123026 files: M Objects/typeobject.c diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 76e06aa31d64..beb0ddd82400 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2941,14 +2941,13 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) PyErr_SetString(PyExc_RuntimeError, "invalid slot offset"); goto fail; } - if (slot->slot == Py_tp_base || slot->slot == Py_tp_bases) + else if (slot->slot == Py_tp_base || slot->slot == Py_tp_bases) { /* Processed above */ continue; - *(void**)(res_start + slotoffsets[slot->slot]) = slot->pfunc; - - /* need to make a copy of the docstring slot, which usually - points to a static string literal */ - if (slot->slot == Py_tp_doc) { + } + else if (slot->slot == Py_tp_doc) { + /* For the docstring slot, which usually points to a static string + literal, we need to make a copy */ const char *old_doc = _PyType_DocWithoutSignature(type->tp_name, slot->pfunc); size_t len = strlen(old_doc)+1; char *tp_doc = PyObject_MALLOC(len); @@ -2960,13 +2959,16 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) memcpy(tp_doc, old_doc, len); type->tp_doc = tp_doc; } - - /* Move the slots to the heap type itself */ - if (slot->slot == Py_tp_members) { + else if (slot->slot == Py_tp_members) { + /* Move the slots to the heap type itself */ size_t len = Py_TYPE(type)->tp_itemsize * nmembers; memcpy(PyHeapType_GET_MEMBERS(res), slot->pfunc, len); type->tp_members = PyHeapType_GET_MEMBERS(res); } + else { + /* Copy other slots directly */ + *(void**)(res_start + slotoffsets[slot->slot]) = slot->pfunc; + } } if (type->tp_dealloc == NULL) { /* It's a heap type, so needs the heap types' dealloc. [View Less]

1 0

bpo-19184: Update the documentation of dis module. (GH-13652) (GH-13755)
by Ezio Melotti June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/9390e98c3ed9eb9fa414030a2feec19261… commit: 9390e98c3ed9eb9fa414030a2feec1926193af94 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Ezio Melotti <ezio.melotti(a)gmail.com> date: 2019-06-03T01:18:50+02:00 summary: bpo-19184: Update the documentation of dis module. (GH-13652) (GH-13755) * bpo-19184: Update the documentation of dis module * Explain the behavior of the number of … [View More]

1 0

Fix typos in docs and docstrings (GH-13745)
by Petr Viktorin June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/0d70227e419ab78c44d81b4ea6ae8aaf76… commit: 0d70227e419ab78c44d81b4ea6ae8aaf769470e6 branch: master author: Xtreak <tir.karthi(a)gmail.com> committer: Petr Viktorin <encukou(a)gmail.com> date: 2019-06-03T01:12:33+02:00 summary: Fix typos in docs and docstrings (GH-13745) files: M Doc/library/dis.rst M Doc/using/windows.rst M Doc/whatsnew/3.8.rst M Lib/asyncio/unix_events.py M Lib/bdb.py M Lib/distutils/ccompiler.py M Lib/email/… [View More]_header_value_parser.py M Lib/encodings/__init__.py M Lib/lib2to3/fixes/fix_metaclass.py M Lib/pstats.py M Lib/pyclbr.py M Lib/ssl.py M Lib/turtle.py M Lib/turtledemo/paint.py M Lib/zipfile.py diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 15e707ae49f8..2a3ffb5e8271 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -708,7 +708,7 @@ iterations of the loop. Cleans up the value stack and the block stack. If *preserve_tos* is not ``0`` TOS first is popped from the stack and pushed on the stack after - perfoming other stack operations: + performing other stack operations: * If TOS is ``NULL`` or an integer (pushed by :opcode:`BEGIN_FINALLY` or :opcode:`CALL_FINALLY`) it is popped from the stack. diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index a1b25ffd25f0..462e4c2b6c63 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -815,7 +815,7 @@ Customizing default Python versions In some cases, a version qualifier can be included in a command to dictate which version of Python will be used by the command. A version qualifier starts with a major version number and can optionally be followed by a period -('.') and a minor version specifier. Furthermore it is possible to specifiy +('.') and a minor version specifier. Furthermore it is possible to specify if a 32 or 64 bit implementation shall be requested by adding "-32" or "-64". For example, a shebang line of ``#!python`` has no version qualifier, while diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 74d0079a53db..e9c9c814c69a 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -1286,7 +1286,7 @@ Changes in the C API (Contributed by Zackery Spytz in :issue:`33407`.) * The interpreter does not pretend to support binary compatibility of - extension types accross feature releases, anymore. A :c:type:`PyTypeObject` + extension types across feature releases, anymore. A :c:type:`PyTypeObject` exported by a third-party extension module is supposed to have all the slots expected in the current Python version, including :c:member:`~PyTypeObject.tp_finalize` (:const:`Py_TPFLAGS_HAVE_FINALIZE` diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 6714542e4e33..b943845d9363 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -193,7 +193,7 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None, # prevents subprocess execution if the watcher # is not ready to handle it. raise RuntimeError("asyncio.get_child_watcher() is not activated, " - "subproccess support is not installed.") + "subprocess support is not installed.") waiter = self.create_future() transp = _UnixSubprocessTransport(self, protocol, args, shell, stdin, stdout, stderr, bufsize, diff --git a/Lib/bdb.py b/Lib/bdb.py index 69174364c46a..96e7d18d718d 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -384,7 +384,7 @@ def set_break(self, filename, lineno, temporary=False, cond=None, return None def _prune_breaks(self, filename, lineno): - """Prune breakpoints for filname:lineno. + """Prune breakpoints for filename:lineno. A list of breakpoints is maintained in the Bdb instance and in the Breakpoint class. If a breakpoint in the Bdb instance no diff --git a/Lib/distutils/ccompiler.py b/Lib/distutils/ccompiler.py index b71d1d39bcda..1a411ed11132 100644 --- a/Lib/distutils/ccompiler.py +++ b/Lib/distutils/ccompiler.py @@ -545,7 +545,7 @@ def compile(self, sources, output_dir=None, macros=None, 'extra_preargs' and 'extra_postargs' are implementation- dependent. On platforms that have the notion of a command-line (e.g. Unix, DOS/Windows), they are most likely lists of strings: extra - command-line arguments to prepand/append to the compiler command + command-line arguments to prepend/append to the compiler command line. On other platforms, consult the implementation class documentation. In any event, they are intended as an escape hatch for those occasions when the abstract compiler framework doesn't diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 649f1539fa02..14cc00c61e07 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -2385,7 +2385,7 @@ def parse_mime_parameters(value): the formal RFC grammar, but it is more convenient for us for the set of parameters to be treated as its own TokenList. - This is 'parse' routine because it consumes the reminaing value, but it + This is 'parse' routine because it consumes the remaining value, but it would never be called to parse a full header. Instead it is called to parse everything after the non-parameter value of a specific MIME header. diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py index d737d5339dce..ddd5afdcf2da 100644 --- a/Lib/encodings/__init__.py +++ b/Lib/encodings/__init__.py @@ -12,7 +12,7 @@ * getregentry() -> codecs.CodecInfo object The getregentry() API must return a CodecInfo object with encoder, decoder, incrementalencoder, incrementaldecoder, streamwriter and streamreader - atttributes which adhere to the Python Codec Interface Standard. + attributes which adhere to the Python Codec Interface Standard. In addition, a module may optionally also define the following APIs which are then used by the package's codec search function: diff --git a/Lib/lib2to3/fixes/fix_metaclass.py b/Lib/lib2to3/fixes/fix_metaclass.py index 8e34463bd8ab..d1cd10d32758 100644 --- a/Lib/lib2to3/fixes/fix_metaclass.py +++ b/Lib/lib2to3/fixes/fix_metaclass.py @@ -1,6 +1,6 @@ """Fixer for __metaclass__ = X -> (metaclass=X) methods. - The various forms of classef (inherits nothing, inherits once, inherints + The various forms of classef (inherits nothing, inherits once, inherits many) don't parse the same in the CST so we look at ALL classes for a __metaclass__ and if we find one normalize the inherits to all be an arglist. diff --git a/Lib/pstats.py b/Lib/pstats.py index b7649ebc6f1c..4b419a8ecdb6 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -632,12 +632,12 @@ def do_EOF(self, line): print("", file=self.stream) return 1 def help_EOF(self): - print("Leave the profile brower.", file=self.stream) + print("Leave the profile browser.", file=self.stream) def do_quit(self, line): return 1 def help_quit(self): - print("Leave the profile brower.", file=self.stream) + print("Leave the profile browser.", file=self.stream) def do_read(self, line): if line: diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py index 8fd0523b7e3b..99a17343fb61 100644 --- a/Lib/pyclbr.py +++ b/Lib/pyclbr.py @@ -50,7 +50,7 @@ class _Object: - "Informaton about Python class or function." + "Information about Python class or function." def __init__(self, module, name, file, lineno, parent): self.module = module self.name = name diff --git a/Lib/ssl.py b/Lib/ssl.py index f5fa6aeec2d2..4afa46e5da5c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -862,7 +862,7 @@ def server_side(self): @property def server_hostname(self): """The currently set server hostname (for SNI), or ``None`` if no - server hostame is set.""" + server hostname is set.""" return self._sslobj.server_hostname def read(self, len=1024, buffer=None): diff --git a/Lib/turtle.py b/Lib/turtle.py index 044d91cf6d83..ee67a351b54f 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -833,7 +833,7 @@ def numinput(self, title, prompt, default=None, minval=None, maxval=None): Arguments: title is the title of the dialog window, prompt is a text mostly describing what numerical information to input. default: default value - minval: minimum value for imput + minval: minimum value for input maxval: maximum value for input The number input must be in the range minval .. maxval if these are diff --git a/Lib/turtledemo/paint.py b/Lib/turtledemo/paint.py index dde16912dfed..fc6852a20082 100755 --- a/Lib/turtledemo/paint.py +++ b/Lib/turtledemo/paint.py @@ -7,7 +7,7 @@ - left mouse button moves turtle - middle mouse button changes color -- right mouse button toogles betweem pen up +- right mouse button toggles between pen up (no line drawn when the turtle moves) and pen down (line is drawn). If pen up follows at least two pen-down moves, the polygon that diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 5496f6eb1867..62f2fd27d3ce 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -2163,7 +2163,7 @@ class Path: >>> (b / 'missing.txt').exists() False - Coersion to string: + Coercion to string: >>> str(c) 'abcde.zip/b/c.txt' [View Less]

1 0

bpo-36027 bpo-36974: Fix "incompatible pointer type" compiler warnings (GH-13758)
by Petr Viktorin June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/e584cbff1ea78e700cf9943d50467e3b58… commit: e584cbff1ea78e700cf9943d50467e3b58301ccc branch: master author: Petr Viktorin <encukou(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-03T01:08:14+02:00 summary: bpo-36027 bpo-36974: Fix "incompatible pointer type" compiler warnings (GH-13758) files: M Modules/_testcapimodule.c M Objects/longobject.c diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index … [View More]

1 0

bpo-35621: Fix tests when SafeChildWatcher is expected instead of ThreadedChildWatcher (GH-13754)
by Miss Islington (bot) June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/c6789d6c85a290a35f3839efb52a3d3453… commit: c6789d6c85a290a35f3839efb52a3d34536dcebe branch: master author: Andrew Svetlov <andrew.svetlov(a)gmail.com> committer: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> date: 2019-06-02T15:45:54-07:00 summary: bpo-35621: Fix tests when SafeChildWatcher is expected instead of ThreadedChildWatcher (GH-13754) https://bugs.python.org/issue35621 files: M Lib/test/… [View More]

1 0

bpo-36974: Make tp_call=PyVectorcall_Call work for inherited types (GH-13699)
by Petr Viktorin June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/fb9423fd0a85f06affb8c3a8f25dd598a6… commit: fb9423fd0a85f06affb8c3a8f25dd598a649aa42 branch: master author: Petr Viktorin <encukou(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T23:52:20+02:00 summary: bpo-36974: Make tp_call=PyVectorcall_Call work for inherited types (GH-13699) When inheriting a heap subclass from a vectorcall class that sets `.tp_call=PyVectorcall_Call` (as recommended in PEP 590), the subclass … [View More]does not inherit `_Py_TPFLAGS_HAVE_VECTORCALL`, and thus `PyVectorcall_Call` does not work for it. This attempts to solve the issue by: * always inheriting `tp_vectorcall_offset` unless `tp_call` is overridden in the subclass * inheriting _Py_TPFLAGS_HAVE_VECTORCALL for static types, unless `tp_call` is overridden * making `PyVectorcall_Call` ignore `_Py_TPFLAGS_HAVE_VECTORCALL` This means it'll be ever more important to only call `PyVectorcall_Call` on classes that support vectorcall. In `PyVectorcall_Call`'s intended role as `tp_call` filler, that's not a problem. files: M Lib/test/test_capi.py M Modules/_testcapimodule.c M Objects/call.c M Objects/typeobject.c diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index fabc821e5c3a..88bda057ed6a 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -515,9 +515,10 @@ def test_vectorcall_override(self): def test_vectorcall(self): # Test a bunch of different ways to call objects: - # 1. normal call - # 2. vectorcall using _PyObject_Vectorcall() - # 3. vectorcall using PyVectorcall_Call() + # 1. vectorcall using PyVectorcall_Call() + # (only for objects that support vectorcall directly) + # 2. normal call + # 3. vectorcall using _PyObject_Vectorcall() # 4. call as bound method # 5. call using functools.partial @@ -541,6 +542,27 @@ def vectorcall(func, args, kwargs): kwnames = tuple(kwargs) return pyobject_vectorcall(func, args, kwnames) + for (func, args, kwargs, expected) in calls: + with self.subTest(str(func)): + if not kwargs: + self.assertEqual(expected, pyvectorcall_call(func, args)) + self.assertEqual(expected, pyvectorcall_call(func, args, kwargs)) + + # Add derived classes (which do not support vectorcall directly, + # but do support all other ways of calling). + + class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): + pass + + class MethodDescriptorOverridden(_testcapi.MethodDescriptorBase): + def __call__(self, n): + return 'new' + + calls += [ + (MethodDescriptorHeap(), (0,), {}, True), + (MethodDescriptorOverridden(), (0,), {}, 'new'), + ] + for (func, args, kwargs, expected) in calls: with self.subTest(str(func)): args1 = args[1:] @@ -549,12 +571,10 @@ def vectorcall(func, args, kwargs): if not kwargs: self.assertEqual(expected, func(*args)) self.assertEqual(expected, pyobject_vectorcall(func, args, None)) - self.assertEqual(expected, pyvectorcall_call(func, args)) self.assertEqual(expected, meth(*args1)) self.assertEqual(expected, wrapped(*args)) self.assertEqual(expected, func(*args, **kwargs)) self.assertEqual(expected, vectorcall(func, args, kwargs)) - self.assertEqual(expected, pyvectorcall_call(func, args, kwargs)) self.assertEqual(expected, meth(*args1, **kwargs)) self.assertEqual(expected, wrapped(*args, **kwargs)) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index bf20e81a4ce8..eed34c9802c3 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5854,7 +5854,7 @@ MethodDescriptor_vectorcall(PyObject *callable, PyObject *const *args, static PyObject * MethodDescriptor_new(PyTypeObject* type, PyObject* args, PyObject *kw) { - MethodDescriptorObject *op = PyObject_New(MethodDescriptorObject, type); + MethodDescriptorObject *op = type->tp_alloc(type, 0); op->vectorcall = MethodDescriptor_vectorcall; return (PyObject *)op; } diff --git a/Objects/call.c b/Objects/call.c index c0d14567e430..578e1b3ab619 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -173,12 +173,22 @@ _PyObject_MakeTpCall(PyObject *callable, PyObject *const *args, Py_ssize_t nargs PyObject * PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *kwargs) { - vectorcallfunc func = _PyVectorcall_Function(callable); + /* get vectorcallfunc as in _PyVectorcall_Function, but without + * the _Py_TPFLAGS_HAVE_VECTORCALL check */ + Py_ssize_t offset = Py_TYPE(callable)->tp_vectorcall_offset; + if ((offset <= 0) || (!Py_TYPE(callable)->tp_call)) { + PyErr_Format(PyExc_TypeError, "'%.200s' object does not support vectorcall", + Py_TYPE(callable)->tp_name); + return NULL; + } + vectorcallfunc func = *(vectorcallfunc *)(((char *)callable) + offset); if (func == NULL) { PyErr_Format(PyExc_TypeError, "'%.200s' object does not support vectorcall", Py_TYPE(callable)->tp_name); return NULL; } + + /* Convert arguments & call */ PyObject *const *args; Py_ssize_t nargs = PyTuple_GET_SIZE(tuple); PyObject *kwnames; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b6d925c1442e..76e06aa31d64 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5145,17 +5145,21 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) } COPYSLOT(tp_repr); /* tp_hash see tp_richcompare */ - COPYSLOT(tp_call); - /* Inherit tp_vectorcall_offset and _Py_TPFLAGS_HAVE_VECTORCALL if tp_call - * was inherited, but only for extension types */ - if ((base->tp_flags & _Py_TPFLAGS_HAVE_VECTORCALL) && - !(type->tp_flags & _Py_TPFLAGS_HAVE_VECTORCALL) && - !(type->tp_flags & Py_TPFLAGS_HEAPTYPE) && - base->tp_call && - type->tp_call == base->tp_call) { - type->tp_vectorcall_offset = base->tp_vectorcall_offset; - type->tp_flags |= _Py_TPFLAGS_HAVE_VECTORCALL; + /* Inherit tp_vectorcall_offset only if tp_call is not overridden */ + if (!type->tp_call) { + COPYSLOT(tp_vectorcall_offset); + } + /* Inherit_Py_TPFLAGS_HAVE_VECTORCALL for non-heap types + * if tp_call is not overridden */ + if (!type->tp_call && + (base->tp_flags & _Py_TPFLAGS_HAVE_VECTORCALL) && + !(type->tp_flags & _Py_TPFLAGS_HAVE_VECTORCALL) && + !(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) + { + type->tp_flags |= _Py_TPFLAGS_HAVE_VECTORCALL; + } + COPYSLOT(tp_call); } COPYSLOT(tp_str); { [View Less]

1 0

bpo-37014: Update docstring and Documentation of fileinput.FileInput(). (GH-13545) (GH-13753)
by Ezio Melotti June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/6bd438e137a0618b8db949a4751304f541… commit: 6bd438e137a0618b8db949a4751304f541b6674d branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Ezio Melotti <ezio.melotti(a)gmail.com> date: 2019-06-02T23:36:34+02:00 summary: bpo-37014: Update docstring and Documentation of fileinput.FileInput(). (GH-13545) (GH-13753) * bpo-37014: Update docstring and Documentation of fileinput.FileInput() * … [View More]Explain the behavior of fileinput.FileInput() when reading stdin. * Update blurb. * bpo-37014: Fix typo in the docstring and documentation. (cherry picked from commit aca273e2401ca3151e15e984f400233b7f255e15) Co-authored-by: Michele Angrisano <michele.angrisano(a)gmail.com> files: M Doc/library/fileinput.rst M Lib/fileinput.py M Misc/ACKS diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 1fc11ffce25e..bf81749f9456 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -23,8 +23,9 @@ The typical use is:: This iterates over the lines of all files listed in ``sys.argv[1:]``, defaulting to ``sys.stdin`` if the list is empty. If a filename is ``'-'``, it is also -replaced by ``sys.stdin``. To specify an alternative list of filenames, pass it -as the first argument to :func:`.input`. A single file name is also allowed. +replaced by ``sys.stdin`` and the optional arguments *mode* and *openhook* +are ignored. To specify an alternative list of filenames, pass it as the +first argument to :func:`.input`. A single file name is also allowed. All files are opened in text mode by default, but you can override this by specifying the *mode* parameter in the call to :func:`.input` or diff --git a/Lib/fileinput.py b/Lib/fileinput.py index c6fc9a1981a1..c8b8b23674c9 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -8,9 +8,9 @@ This iterates over the lines of all files listed in sys.argv[1:], defaulting to sys.stdin if the list is empty. If a filename is '-' it -is also replaced by sys.stdin. To specify an alternative list of -filenames, pass it as the argument to input(). A single file name is -also allowed. +is also replaced by sys.stdin and the optional arguments mode and +openhook are ignored. To specify an alternative list of filenames, +pass it as the argument to input(). A single file name is also allowed. Functions filename(), lineno() return the filename and cumulative line number of the line that has just been read; filelineno() returns its diff --git a/Misc/ACKS b/Misc/ACKS index 40e799088645..f7fc70afd173 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -55,6 +55,7 @@ Juancarlo Añez Chris Angelico Jérémy Anger Jon Anglin +Michele Angrisano Ankur Ankan Heidi Annexstad Ramchandra Apte [View Less]

1 0

bpo-19184: Update the documentation of dis module. (GH-13652)
by Ezio Melotti June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/e1179a5096fb12297ececd7a1c79969aa5… commit: e1179a5096fb12297ececd7a1c79969aa5747e28 branch: master author: Michele Angrisano <michele.angrisano(a)gmail.com> committer: Ezio Melotti <ezio.melotti(a)gmail.com> date: 2019-06-02T23:34:12+02:00 summary: bpo-19184: Update the documentation of dis module. (GH-13652) * bpo-19184: Update the documentation of dis module * Explain the behavior of the number of arguments of RAISE_VARGARGS opcode. *… [View More]

1 0

test_gdb.test_pycfunction: test more calling conventions (GH-13668)
by Petr Viktorin June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/64e2c64f7f0111c52834155becc0c6134f… commit: 64e2c64f7f0111c52834155becc0c6134f9d8750 branch: master author: Petr Viktorin <encukou(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T23:11:24+02:00 summary: test_gdb.test_pycfunction: test more calling conventions (GH-13668) As the code paths for various METH_* conventions are diverging due to optimizations, we should check they continue to be covered by GDB integration. … [View More]files: M Lib/test/test_gdb.py diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 3127e69ca9ba..f57e348b6c14 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -867,27 +867,40 @@ def test_gc(self): # unless we add LD_PRELOAD=PATH-TO-libpthread.so.1 as a workaround def test_pycfunction(self): 'Verify that "py-bt" displays invocations of PyCFunction instances' - # Tested function must not be defined with METH_NOARGS or METH_O, - # otherwise call_function() doesn't call PyCFunction_Call() - cmd = ('from time import gmtime\n' - 'def foo():\n' - ' gmtime(1)\n' - 'def bar():\n' - ' foo()\n' - 'bar()\n') - # Verify with "py-bt": - gdb_output = self.get_stack_trace(cmd, - breakpoint='time_gmtime', - cmds_after_breakpoint=['bt', 'py-bt'], - ) - self.assertIn('<built-in method gmtime', gdb_output) - - # Verify with "py-bt-full": - gdb_output = self.get_stack_trace(cmd, - breakpoint='time_gmtime', - cmds_after_breakpoint=['py-bt-full'], - ) - self.assertIn('#1 <built-in method gmtime', gdb_output) + # Various optimizations multiply the code paths by which these are + # called, so test a variety of calling conventions. + for py_name, py_args, c_name, expected_frame_number in ( + ('gmtime', '', 'time_gmtime', 1), # METH_VARARGS + ('len', '[]', 'builtin_len', 2), # METH_O + ('locals', '', 'builtin_locals', 2), # METH_NOARGS + ('iter', '[]', 'builtin_iter', 2), # METH_FASTCALL + ('sorted', '[]', 'builtin_sorted', 2), # METH_FASTCALL|METH_KEYWORDS + ): + with self.subTest(c_name): + cmd = ('from time import gmtime\n' # (not always needed) + 'def foo():\n' + f' {py_name}({py_args})\n' + 'def bar():\n' + ' foo()\n' + 'bar()\n') + # Verify with "py-bt": + gdb_output = self.get_stack_trace( + cmd, + breakpoint=c_name, + cmds_after_breakpoint=['bt', 'py-bt'], + ) + self.assertIn(f'<built-in method {py_name}', gdb_output) + + # Verify with "py-bt-full": + gdb_output = self.get_stack_trace( + cmd, + breakpoint=c_name, + cmds_after_breakpoint=['py-bt-full'], + ) + self.assertIn( + f'#{expected_frame_number} <built-in method {py_name}', + gdb_output, + ) @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") [View Less]

1 0

bpo-36829: test_threading: Fix a ref cycle (GH-13752)
by Victor Stinner June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/cdce0574d03005e27b843fc110c54c99c7… commit: cdce0574d03005e27b843fc110c54c99c7a76412 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T23:08:41+02:00 summary: bpo-36829: test_threading: Fix a ref cycle (GH-13752) files: M Lib/test/test_threading.py diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 8c8cc128b051..6ac4ea9623de 100644 --- a/Lib/test/… [View More]

1 0

bpo-37014: Update docstring and Documentation of fileinput.FileInput(). (GH-13545)
by Ezio Melotti June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/aca273e2401ca3151e15e984f400233b7f… commit: aca273e2401ca3151e15e984f400233b7f255e15 branch: master author: Michele Angrisano <michele.angrisano(a)gmail.com> committer: Ezio Melotti <ezio.melotti(a)gmail.com> date: 2019-06-02T23:01:49+02:00 summary: bpo-37014: Update docstring and Documentation of fileinput.FileInput(). (GH-13545) * bpo-37014: Update docstring and Documentation of fileinput.FileInput() * Explain the behavior of fileinput.… [View More]FileInput() when reading stdin. * Update blurb. * bpo-37014: Fix typo in the docstring and documentation. files: M Doc/library/fileinput.rst M Lib/fileinput.py M Misc/ACKS diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 14be492f55a6..f5e5280a1363 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -23,8 +23,9 @@ The typical use is:: This iterates over the lines of all files listed in ``sys.argv[1:]``, defaulting to ``sys.stdin`` if the list is empty. If a filename is ``'-'``, it is also -replaced by ``sys.stdin``. To specify an alternative list of filenames, pass it -as the first argument to :func:`.input`. A single file name is also allowed. +replaced by ``sys.stdin`` and the optional arguments *mode* and *openhook* +are ignored. To specify an alternative list of filenames, pass it as the +first argument to :func:`.input`. A single file name is also allowed. All files are opened in text mode by default, but you can override this by specifying the *mode* parameter in the call to :func:`.input` or diff --git a/Lib/fileinput.py b/Lib/fileinput.py index d868e74cd5e9..c1b0ec9a8ed0 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -8,9 +8,9 @@ This iterates over the lines of all files listed in sys.argv[1:], defaulting to sys.stdin if the list is empty. If a filename is '-' it -is also replaced by sys.stdin. To specify an alternative list of -filenames, pass it as the argument to input(). A single file name is -also allowed. +is also replaced by sys.stdin and the optional arguments mode and +openhook are ignored. To specify an alternative list of filenames, +pass it as the argument to input(). A single file name is also allowed. Functions filename(), lineno() return the filename and cumulative line number of the line that has just been read; filelineno() returns its diff --git a/Misc/ACKS b/Misc/ACKS index 5c23df8c5899..082fa567f23a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -55,6 +55,7 @@ Juancarlo Añez Chris Angelico Jérémy Anger Jon Anglin +Michele Angrisano Ankur Ankan Heidi Annexstad Ramchandra Apte [View Less]

1 0

Call PyObject_GC_UnTrack in structseq dealloc (GH-13751)
by Pablo Galindo June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/3caf4de6f05f68c3a175f4d8ce870d7a00… commit: 3caf4de6f05f68c3a175f4d8ce870d7a0016622a branch: master author: Pablo Galindo <Pablogsal(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T21:52:49+01:00 summary: Call PyObject_GC_UnTrack in structseq dealloc (GH-13751) files: M Objects/structseq.c diff --git a/Objects/structseq.c b/Objects/structseq.c index 3d857f734be8..2c25e1646a2a 100644 --- a/Objects/structseq.c +++ b/… [View More]

1 0

bpo-37124: Fix reference leak in test_msilib (GH-13750)
by Pablo Galindo June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/c0295dba259accc4b247beb22a0b2cc2f3… commit: c0295dba259accc4b247beb22a0b2cc2f31d9850 branch: master author: Pablo Galindo <Pablogsal(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T21:36:21+01:00 summary: bpo-37124: Fix reference leak in test_msilib (GH-13750) files: M Lib/test/test_msilib.py diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py index fa0be581613d..f9bd0da7498e 100644 --- a/Lib/test/… [View More]

1 0

bpo-35610: IDLE - Replace .context_use_ps1 with .prompt_last_line (GH-11307)
by Miss Islington (bot) June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/b4e0bfd4778e142f037f50c19c4bb5bd12… commit: b4e0bfd4778e142f037f50c19c4bb5bd123b4641 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T12:35:53-07:00 summary: bpo-35610: IDLE - Replace .context_use_ps1 with .prompt_last_line (GH-11307) Changes in bpo- 31858 made the less informative 'context_use_ps1' redundant. (cherry picked from commit … [View More]6bdc4dee01788599808c7858e2fe9fdd72cf6792) Co-authored-by: Cheryl Sabella <cheryl.sabella(a)gmail.com> files: A Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst M Lib/idlelib/NEWS.txt M Lib/idlelib/editor.py M Lib/idlelib/hyperparser.py M Lib/idlelib/idle_test/test_autocomplete.py M Lib/idlelib/idle_test/test_hyperparser.py M Lib/idlelib/idle_test/test_parenmatch.py M Lib/idlelib/pyshell.py diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 808c236bb590..172fdc00d237 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,9 @@ Released on 2019-06-24? ====================================== +bpo-35610: Replace now redundant editor.context_use_ps1 with +.prompt_last_line. This finishes change started in bpo-31858. + bpo-32411: Stop sorting dict created with desired line order. bpo-37038: Make idlelib.run runnable; add test clause. diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 83260329640e..89b7239a96ea 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -228,10 +228,6 @@ def __init__(self, flist=None, filename=None, key=None, root=None): self.indentwidth = self.tabwidth self.set_notabs_indentwidth() - # If context_use_ps1 is true, parsing searches back for a ps1 line; - # else searches for a popular (if, def, ...) Python stmt. - self.context_use_ps1 = False - # When searching backwards for a reliable place to begin parsing, # first start num_context_lines[0] lines back, then # num_context_lines[1] lines back if that didn't work, and so on. @@ -1337,14 +1333,13 @@ def newline_and_indent_event(self, event): # open/close first need to find the last stmt lno = index2line(text.index('insert')) y = pyparse.Parser(self.indentwidth, self.tabwidth) - if not self.context_use_ps1: + if not self.prompt_last_line: for context in self.num_context_lines: startat = max(lno - context, 1) startatindex = repr(startat) + ".0" rawtext = text.get(startatindex, "insert") y.set_code(rawtext) bod = y.find_good_parse_start( - self.context_use_ps1, self._build_char_in_string_func(startatindex)) if bod is not None or startat == 1: break diff --git a/Lib/idlelib/hyperparser.py b/Lib/idlelib/hyperparser.py index 7e7e0ae80247..77baca782b3f 100644 --- a/Lib/idlelib/hyperparser.py +++ b/Lib/idlelib/hyperparser.py @@ -35,7 +35,7 @@ def index2line(index): return int(float(index)) lno = index2line(text.index(index)) - if not editwin.context_use_ps1: + if not editwin.prompt_last_line: for context in editwin.num_context_lines: startat = max(lno - context, 1) startatindex = repr(startat) + ".0" diff --git a/Lib/idlelib/idle_test/test_autocomplete.py b/Lib/idlelib/idle_test/test_autocomplete.py index 398cb359e093..6181b29ec250 100644 --- a/Lib/idlelib/idle_test/test_autocomplete.py +++ b/Lib/idlelib/idle_test/test_autocomplete.py @@ -19,7 +19,7 @@ def __init__(self, root, text): self.text = text self.indentwidth = 8 self.tabwidth = 8 - self.context_use_ps1 = True + self.prompt_last_line = '>>>' # Currently not used by autocomplete. class AutoCompleteTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_hyperparser.py b/Lib/idlelib/idle_test/test_hyperparser.py index 8dbfc63779d3..343843c4166e 100644 --- a/Lib/idlelib/idle_test/test_hyperparser.py +++ b/Lib/idlelib/idle_test/test_hyperparser.py @@ -11,7 +11,7 @@ def __init__(self, text): self.text = text self.indentwidth = 8 self.tabwidth = 8 - self.context_use_ps1 = True + self.prompt_last_line = '>>>' self.num_context_lines = 50, 500, 1000 _build_char_in_string_func = EditorWindow._build_char_in_string_func @@ -53,7 +53,7 @@ def setUp(self): def tearDown(self): self.text.delete('1.0', 'end') - self.editwin.context_use_ps1 = True + self.editwin.prompt_last_line = '>>>' def get_parser(self, index): """ @@ -71,7 +71,7 @@ def test_init(self): self.assertIn('precedes', str(ve.exception)) # test without ps1 - self.editwin.context_use_ps1 = False + self.editwin.prompt_last_line = '' # number of lines lesser than 50 p = self.get_parser('end') diff --git a/Lib/idlelib/idle_test/test_parenmatch.py b/Lib/idlelib/idle_test/test_parenmatch.py index f58819abf112..4a41d8433d54 100644 --- a/Lib/idlelib/idle_test/test_parenmatch.py +++ b/Lib/idlelib/idle_test/test_parenmatch.py @@ -17,7 +17,7 @@ def __init__(self, text): self.text = text self.indentwidth = 8 self.tabwidth = 8 - self.context_use_ps1 = True + self.prompt_last_line = '>>>' # Currently not used by parenmatch. class ParenMatchTest(unittest.TestCase): diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 2de42658b01c..6e0707d68bb6 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -881,7 +881,7 @@ def __init__(self, flist=None): self.usetabs = True # indentwidth must be 8 when using tabs. See note in EditorWindow: self.indentwidth = 8 - self.context_use_ps1 = True + self.sys_ps1 = sys.ps1 if hasattr(sys, 'ps1') else '>>> ' self.prompt_last_line = self.sys_ps1.split('\n')[-1] self.prompt = self.sys_ps1 # Changes when debug active diff --git a/Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst b/Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst new file mode 100644 index 000000000000..0042ab70497a --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst @@ -0,0 +1,2 @@ +Replace now redundant .context_use_ps1 with .prompt_last_line. This finishes +change started in bpo-31858. [View Less]

1 0

bpo-35610: IDLE - Replace .context_use_ps1 with .prompt_last_line (GH-11307)
by Terry Jan Reedy June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/6bdc4dee01788599808c7858e2fe9fdd72… commit: 6bdc4dee01788599808c7858e2fe9fdd72cf6792 branch: master author: Cheryl Sabella <cheryl.sabella(a)gmail.com> committer: Terry Jan Reedy <tjreedy(a)udel.edu> date: 2019-06-02T14:56:47-04:00 summary: bpo-35610: IDLE - Replace .context_use_ps1 with .prompt_last_line (GH-11307) Changes in bpo- 31858 made the less informative 'context_use_ps1' redundant. files: A Misc/NEWS.d/next/IDLE/2019-06-02-14-10-… [View More]52.bpo-35610.0w_v6Y.rst M Lib/idlelib/NEWS.txt M Lib/idlelib/editor.py M Lib/idlelib/hyperparser.py M Lib/idlelib/idle_test/test_autocomplete.py M Lib/idlelib/idle_test/test_hyperparser.py M Lib/idlelib/idle_test/test_parenmatch.py M Lib/idlelib/pyshell.py diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index b260d4e5ef1b..982af7767251 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,9 @@ Released on 2019-10-20? ====================================== +bpo-35610: Replace now redundant editor.context_use_ps1 with +.prompt_last_line. This finishes change started in bpo-31858. + bpo-32411: Stop sorting dict created with desired line order. bpo-37038: Make idlelib.run runnable; add test clause. diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 83260329640e..89b7239a96ea 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -228,10 +228,6 @@ def __init__(self, flist=None, filename=None, key=None, root=None): self.indentwidth = self.tabwidth self.set_notabs_indentwidth() - # If context_use_ps1 is true, parsing searches back for a ps1 line; - # else searches for a popular (if, def, ...) Python stmt. - self.context_use_ps1 = False - # When searching backwards for a reliable place to begin parsing, # first start num_context_lines[0] lines back, then # num_context_lines[1] lines back if that didn't work, and so on. @@ -1337,14 +1333,13 @@ def newline_and_indent_event(self, event): # open/close first need to find the last stmt lno = index2line(text.index('insert')) y = pyparse.Parser(self.indentwidth, self.tabwidth) - if not self.context_use_ps1: + if not self.prompt_last_line: for context in self.num_context_lines: startat = max(lno - context, 1) startatindex = repr(startat) + ".0" rawtext = text.get(startatindex, "insert") y.set_code(rawtext) bod = y.find_good_parse_start( - self.context_use_ps1, self._build_char_in_string_func(startatindex)) if bod is not None or startat == 1: break diff --git a/Lib/idlelib/hyperparser.py b/Lib/idlelib/hyperparser.py index 7e7e0ae80247..77baca782b3f 100644 --- a/Lib/idlelib/hyperparser.py +++ b/Lib/idlelib/hyperparser.py @@ -35,7 +35,7 @@ def index2line(index): return int(float(index)) lno = index2line(text.index(index)) - if not editwin.context_use_ps1: + if not editwin.prompt_last_line: for context in editwin.num_context_lines: startat = max(lno - context, 1) startatindex = repr(startat) + ".0" diff --git a/Lib/idlelib/idle_test/test_autocomplete.py b/Lib/idlelib/idle_test/test_autocomplete.py index 398cb359e093..6181b29ec250 100644 --- a/Lib/idlelib/idle_test/test_autocomplete.py +++ b/Lib/idlelib/idle_test/test_autocomplete.py @@ -19,7 +19,7 @@ def __init__(self, root, text): self.text = text self.indentwidth = 8 self.tabwidth = 8 - self.context_use_ps1 = True + self.prompt_last_line = '>>>' # Currently not used by autocomplete. class AutoCompleteTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_hyperparser.py b/Lib/idlelib/idle_test/test_hyperparser.py index 8dbfc63779d3..343843c4166e 100644 --- a/Lib/idlelib/idle_test/test_hyperparser.py +++ b/Lib/idlelib/idle_test/test_hyperparser.py @@ -11,7 +11,7 @@ def __init__(self, text): self.text = text self.indentwidth = 8 self.tabwidth = 8 - self.context_use_ps1 = True + self.prompt_last_line = '>>>' self.num_context_lines = 50, 500, 1000 _build_char_in_string_func = EditorWindow._build_char_in_string_func @@ -53,7 +53,7 @@ def setUp(self): def tearDown(self): self.text.delete('1.0', 'end') - self.editwin.context_use_ps1 = True + self.editwin.prompt_last_line = '>>>' def get_parser(self, index): """ @@ -71,7 +71,7 @@ def test_init(self): self.assertIn('precedes', str(ve.exception)) # test without ps1 - self.editwin.context_use_ps1 = False + self.editwin.prompt_last_line = '' # number of lines lesser than 50 p = self.get_parser('end') diff --git a/Lib/idlelib/idle_test/test_parenmatch.py b/Lib/idlelib/idle_test/test_parenmatch.py index f58819abf112..4a41d8433d54 100644 --- a/Lib/idlelib/idle_test/test_parenmatch.py +++ b/Lib/idlelib/idle_test/test_parenmatch.py @@ -17,7 +17,7 @@ def __init__(self, text): self.text = text self.indentwidth = 8 self.tabwidth = 8 - self.context_use_ps1 = True + self.prompt_last_line = '>>>' # Currently not used by parenmatch. class ParenMatchTest(unittest.TestCase): diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 2de42658b01c..6e0707d68bb6 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -881,7 +881,7 @@ def __init__(self, flist=None): self.usetabs = True # indentwidth must be 8 when using tabs. See note in EditorWindow: self.indentwidth = 8 - self.context_use_ps1 = True + self.sys_ps1 = sys.ps1 if hasattr(sys, 'ps1') else '>>> ' self.prompt_last_line = self.sys_ps1.split('\n')[-1] self.prompt = self.sys_ps1 # Changes when debug active diff --git a/Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst b/Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst new file mode 100644 index 000000000000..0042ab70497a --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst @@ -0,0 +1,2 @@ +Replace now redundant .context_use_ps1 with .prompt_last_line. This finishes +change started in bpo-31858. [View Less]

1 0

Add description to the command line help of the argument clinic (GH-8518)
by Pablo Galindo June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/5df4025f42b30aca72f441899b361f748c… commit: 5df4025f42b30aca72f441899b361f748c304c57 branch: master author: Tim Hoffmann <2836374+timhoffm(a)users.noreply.github.com> committer: Pablo Galindo <Pablogsal(a)gmail.com> date: 2019-06-02T17:58:10+01:00 summary: Add description to the command line help of the argument clinic (GH-8518) files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index cb2ded4649dc..… [View More]

1 0

bpo-37126: Allow structseq objects to be tracked by the GC (GH-13729)
by Pablo Galindo June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/7ffcf848df214135abeea7f6c6faa4135f… commit: 7ffcf848df214135abeea7f6c6faa4135fd0928f branch: master author: Pablo Galindo <Pablogsal(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T15:45:13+01:00 summary: bpo-37126: Allow structseq objects to be tracked by the GC (GH-13729) files: A Misc/NEWS.d/next/Core and Builtins/2019-06-01-20-03-13.bpo-37126.tP6lL4.rst M Objects/structseq.c diff --git a/Misc/NEWS.d/next/Core and … [View More]Builtins/2019-06-01-20-03-13.bpo-37126.tP6lL4.rst b/Misc/NEWS.d/next/Core and Builtins/2019-06-01-20-03-13.bpo-37126.tP6lL4.rst new file mode 100644 index 000000000000..069b064dfa61 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-06-01-20-03-13.bpo-37126.tP6lL4.rst @@ -0,0 +1,2 @@ +All structseq objects are now tracked by the garbage collector. Patch by +Pablo Galindo. diff --git a/Objects/structseq.c b/Objects/structseq.c index a5046c42cbc3..3d857f734be8 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_tupleobject.h" +#include "pycore_object.h" #include "structmember.h" static const char visible_length_key[] = "n_sequence_fields"; @@ -59,6 +60,18 @@ PyStructSequence_GetItem(PyObject* op, Py_ssize_t i) return PyStructSequence_GET_ITEM(op, i); } + +static int +structseq_traverse(PyStructSequence *obj, visitproc visit, void *arg) +{ + Py_ssize_t i, size; + size = REAL_SIZE(obj); + for (i = 0; i < size; ++i) { + Py_VISIT(obj->ob_item[i]); + } + return 0; +} + static void structseq_dealloc(PyStructSequence *obj) { @@ -166,6 +179,7 @@ structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict) } Py_DECREF(arg); + _PyObject_GC_TRACK(res); return (PyObject*) res; } @@ -388,6 +402,7 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) type->tp_methods = structseq_methods; type->tp_new = structseq_new; type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC; + type->tp_traverse = (traverseproc) structseq_traverse; n_members = count_members(desc, &n_unnamed_members); members = PyMem_NEW(PyMemberDef, n_members - n_unnamed_members + 1); @@ -426,7 +441,7 @@ PyStructSequence_NewType(PyStructSequence_Desc *desc) PyMemberDef *members; PyObject *bases; PyTypeObject *type; - PyType_Slot slots[7]; + PyType_Slot slots[8]; PyType_Spec spec; Py_ssize_t n_members, n_unnamed_members; @@ -446,7 +461,8 @@ PyStructSequence_NewType(PyStructSequence_Desc *desc) slots[3] = (PyType_Slot){Py_tp_methods, structseq_methods}; slots[4] = (PyType_Slot){Py_tp_new, structseq_new}; slots[5] = (PyType_Slot){Py_tp_members, members}; - slots[6] = (PyType_Slot){0, 0}; + slots[6] = (PyType_Slot){Py_tp_traverse, (traverseproc)structseq_traverse}; + slots[7] = (PyType_Slot){0, 0}; /* Initialize Spec */ /* The name in this PyType_Spec is statically allocated so it is */ [View Less]

1 0

bpo-35621: Support running subprocesses in asyncio when loop is executed in non-main thread (#13630)
by Andrew Svetlov June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/13ed07998ad93dbdd94991ba0451b9b559… commit: 13ed07998ad93dbdd94991ba0451b9b559f07972 branch: master author: Andrew Svetlov <andrew.svetlov(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T13:56:38+03:00 summary: bpo-35621: Support running subprocesses in asyncio when loop is executed in non-main thread (#13630) files: A Misc/NEWS.d/next/Library/2019-05-28-19-03-46.bpo-35621.Abc1lf.rst M Lib/asyncio/unix_events.py M Lib/… [View More]test/test_asyncio/test_subprocess.py M Lib/test/test_asyncio/test_unix_events.py diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 28128d2977df..6714542e4e33 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -2,6 +2,7 @@ import errno import io +import itertools import os import selectors import signal @@ -29,7 +30,9 @@ __all__ = ( 'SelectorEventLoop', 'AbstractChildWatcher', 'SafeChildWatcher', - 'FastChildWatcher', 'DefaultEventLoopPolicy', + 'FastChildWatcher', + 'MultiLoopChildWatcher', 'ThreadedChildWatcher', + 'DefaultEventLoopPolicy', ) @@ -184,6 +187,13 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None, stdin, stdout, stderr, bufsize, extra=None, **kwargs): with events.get_child_watcher() as watcher: + if not watcher.is_active(): + # Check early. + # Raising exception before process creation + # prevents subprocess execution if the watcher + # is not ready to handle it. + raise RuntimeError("asyncio.get_child_watcher() is not activated, " + "subproccess support is not installed.") waiter = self.create_future() transp = _UnixSubprocessTransport(self, protocol, args, shell, stdin, stdout, stderr, bufsize, @@ -838,6 +848,15 @@ def close(self): """ raise NotImplementedError() + def is_active(self): + """Watcher status. + + Return True if the watcher is installed and ready to handle process exit + notifications. + + """ + raise NotImplementedError() + def __enter__(self): """Enter the watcher's context and allow starting new processes @@ -849,6 +868,20 @@ def __exit__(self, a, b, c): raise NotImplementedError() +def _compute_returncode(status): + if os.WIFSIGNALED(status): + # The child process died because of a signal. + return -os.WTERMSIG(status) + elif os.WIFEXITED(status): + # The child process exited (e.g sys.exit()). + return os.WEXITSTATUS(status) + else: + # The child exited, but we don't understand its status. + # This shouldn't happen, but if it does, let's just + # return that status; perhaps that helps debug it. + return status + + class BaseChildWatcher(AbstractChildWatcher): def __init__(self): @@ -858,6 +891,9 @@ def __init__(self): def close(self): self.attach_loop(None) + def is_active(self): + return self._loop is not None and self._loop.is_running() + def _do_waitpid(self, expected_pid): raise NotImplementedError() @@ -898,19 +934,6 @@ def _sig_chld(self): 'exception': exc, }) - def _compute_returncode(self, status): - if os.WIFSIGNALED(status): - # The child process died because of a signal. - return -os.WTERMSIG(status) - elif os.WIFEXITED(status): - # The child process exited (e.g sys.exit()). - return os.WEXITSTATUS(status) - else: - # The child exited, but we don't understand its status. - # This shouldn't happen, but if it does, let's just - # return that status; perhaps that helps debug it. - return status - class SafeChildWatcher(BaseChildWatcher): """'Safe' child watcher implementation. @@ -934,11 +957,6 @@ def __exit__(self, a, b, c): pass def add_child_handler(self, pid, callback, *args): - if self._loop is None: - raise RuntimeError( - "Cannot add child handler, " - "the child watcher does not have a loop attached") - self._callbacks[pid] = (callback, args) # Prevent a race condition in case the child is already terminated. @@ -974,7 +992,7 @@ def _do_waitpid(self, expected_pid): # The child process is still alive. return - returncode = self._compute_returncode(status) + returncode = _compute_returncode(status) if self._loop.get_debug(): logger.debug('process %s exited with returncode %s', expected_pid, returncode) @@ -1035,11 +1053,6 @@ def __exit__(self, a, b, c): def add_child_handler(self, pid, callback, *args): assert self._forks, "Must use the context manager" - if self._loop is None: - raise RuntimeError( - "Cannot add child handler, " - "the child watcher does not have a loop attached") - with self._lock: try: returncode = self._zombies.pop(pid) @@ -1072,7 +1085,7 @@ def _do_waitpid_all(self): # A child process is still alive. return - returncode = self._compute_returncode(status) + returncode = _compute_returncode(status) with self._lock: try: @@ -1101,6 +1114,177 @@ def _do_waitpid_all(self): callback(pid, returncode, *args) +class MultiLoopChildWatcher(AbstractChildWatcher): + # The class keeps compatibility with AbstractChildWatcher ABC + # To achieve this it has empty attach_loop() method + # and doesn't accept explicit loop argument + # for add_child_handler()/remove_child_handler() + # but retrieves the current loop by get_running_loop() + + def __init__(self): + self._callbacks = {} + self._saved_sighandler = None + + def is_active(self): + return self._saved_sighandler is not None + + def close(self): + self._callbacks.clear() + if self._saved_sighandler is not None: + handler = signal.getsignal(signal.SIGCHLD) + if handler != self._sig_chld: + logger.warning("SIGCHLD handler was changed by outside code") + else: + signal.signal(signal.SIGCHLD, self._saved_sighandler) + self._saved_sighandler = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def add_child_handler(self, pid, callback, *args): + loop = events.get_running_loop() + self._callbacks[pid] = (loop, callback, args) + + # Prevent a race condition in case the child is already terminated. + self._do_waitpid(pid) + + def remove_child_handler(self, pid): + try: + del self._callbacks[pid] + return True + except KeyError: + return False + + def attach_loop(self, loop): + # Don't save the loop but initialize itself if called first time + # The reason to do it here is that attach_loop() is called from + # unix policy only for the main thread. + # Main thread is required for subscription on SIGCHLD signal + if self._saved_sighandler is None: + self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) + if self._saved_sighandler is None: + logger.warning("Previous SIGCHLD handler was set by non-Python code, " + "restore to default handler on watcher close.") + self._saved_sighandler = signal.SIG_DFL + + # Set SA_RESTART to limit EINTR occurrences. + signal.siginterrupt(signal.SIGCHLD, False) + + def _do_waitpid_all(self): + for pid in list(self._callbacks): + self._do_waitpid(pid) + + def _do_waitpid(self, expected_pid): + assert expected_pid > 0 + + try: + pid, status = os.waitpid(expected_pid, os.WNOHANG) + except ChildProcessError: + # The child process is already reaped + # (may happen if waitpid() is called elsewhere). + pid = expected_pid + returncode = 255 + logger.warning( + "Unknown child process pid %d, will report returncode 255", + pid) + debug_log = False + else: + if pid == 0: + # The child process is still alive. + return + + returncode = _compute_returncode(status) + debug_log = True + try: + loop, callback, args = self._callbacks.pop(pid) + except KeyError: # pragma: no cover + # May happen if .remove_child_handler() is called + # after os.waitpid() returns. + logger.warning("Child watcher got an unexpected pid: %r", + pid, exc_info=True) + else: + if loop.is_closed(): + logger.warning("Loop %r that handles pid %r is closed", loop, pid) + else: + if debug_log and loop.get_debug(): + logger.debug('process %s exited with returncode %s', + expected_pid, returncode) + loop.call_soon_threadsafe(callback, pid, returncode, *args) + + def _sig_chld(self, signum, frame): + try: + self._do_waitpid_all() + except (SystemExit, KeyboardInterrupt): + raise + except BaseException: + logger.warning('Unknown exception in SIGCHLD handler', exc_info=True) + + +class ThreadedChildWatcher(AbstractChildWatcher): + # The watcher uses a thread per process + # for waiting for the process finish. + # It doesn't require subscription on POSIX signal + + def __init__(self): + self._pid_counter = itertools.count(0) + + def is_active(self): + return True + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def add_child_handler(self, pid, callback, *args): + loop = events.get_running_loop() + thread = threading.Thread(target=self._do_waitpid, + name=f"waitpid-{next(self._pid_counter)}", + args=(loop, pid, callback, args), + daemon=True) + thread.start() + + def remove_child_handler(self, pid): + # asyncio never calls remove_child_handler() !!! + # The method is no-op but is implemented because + # abstract base classe requires it + return True + + def attach_loop(self, loop): + pass + + def _do_waitpid(self, loop, expected_pid, callback, args): + assert expected_pid > 0 + + try: + pid, status = os.waitpid(expected_pid, 0) + except ChildProcessError: + # The child process is already reaped + # (may happen if waitpid() is called elsewhere). + pid = expected_pid + returncode = 255 + logger.warning( + "Unknown child process pid %d, will report returncode 255", + pid) + else: + returncode = _compute_returncode(status) + if loop.get_debug(): + logger.debug('process %s exited with returncode %s', + expected_pid, returncode) + + if loop.is_closed(): + logger.warning("Loop %r that handles pid %r is closed", loop, pid) + else: + loop.call_soon_threadsafe(callback, pid, returncode, *args) + + class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): """UNIX event loop policy with a watcher for child processes.""" _loop_factory = _UnixSelectorEventLoop @@ -1112,7 +1296,7 @@ def __init__(self): def _init_watcher(self): with events._lock: if self._watcher is None: # pragma: no branch - self._watcher = SafeChildWatcher() + self._watcher = ThreadedChildWatcher() if isinstance(threading.current_thread(), threading._MainThread): self._watcher.attach_loop(self._local._loop) @@ -1134,7 +1318,7 @@ def set_event_loop(self, loop): def get_child_watcher(self): """Get the watcher for child processes. - If not yet set, a SafeChildWatcher object is automatically created. + If not yet set, a ThreadedChildWatcher object is automatically created. """ if self._watcher is None: self._init_watcher() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 7d72e6cde4e7..582e17202460 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -633,6 +633,7 @@ def test_create_subprocess_exec_with_path(self): self.assertIsNone(self.loop.run_until_complete(execute())) + if sys.platform != 'win32': # Unix class SubprocessWatcherMixin(SubprocessMixin): @@ -648,7 +649,24 @@ def setUp(self): watcher = self.Watcher() watcher.attach_loop(self.loop) policy.set_child_watcher(watcher) - self.addCleanup(policy.set_child_watcher, None) + + def tearDown(self): + super().setUp() + policy = asyncio.get_event_loop_policy() + watcher = policy.get_child_watcher() + policy.set_child_watcher(None) + watcher.attach_loop(None) + watcher.close() + + class SubprocessThreadedWatcherTests(SubprocessWatcherMixin, + test_utils.TestCase): + + Watcher = unix_events.ThreadedChildWatcher + + class SubprocessMultiLoopWatcherTests(SubprocessWatcherMixin, + test_utils.TestCase): + + Watcher = unix_events.MultiLoopChildWatcher class SubprocessSafeWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): @@ -670,5 +688,25 @@ def setUp(self): self.set_event_loop(self.loop) +class GenericWatcherTests: + + def test_create_subprocess_fails_with_inactive_watcher(self): + + async def execute(): + watcher = mock.create_authspec(asyncio.AbstractChildWatcher) + watcher.is_active.return_value = False + asyncio.set_child_watcher(watcher) + + with self.assertRaises(RuntimeError): + await subprocess.create_subprocess_exec( + support.FakePath(sys.executable), '-c', 'pass') + + watcher.add_child_handler.assert_not_called() + + self.assertIsNone(self.loop.run_until_complete(execute())) + + + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index ac84304ec99d..f7f992fcea49 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1082,6 +1082,8 @@ def test_not_implemented(self): NotImplementedError, watcher.attach_loop, f) self.assertRaises( NotImplementedError, watcher.close) + self.assertRaises( + NotImplementedError, watcher.is_active) self.assertRaises( NotImplementedError, watcher.__enter__) self.assertRaises( @@ -1784,15 +1786,6 @@ def test_close(self, m): if isinstance(self.watcher, asyncio.FastChildWatcher): self.assertFalse(self.watcher._zombies) - @waitpid_mocks - def test_add_child_handler_with_no_loop_attached(self, m): - callback = mock.Mock() - with self.create_watcher() as watcher: - with self.assertRaisesRegex( - RuntimeError, - 'the child watcher does not have a loop attached'): - watcher.add_child_handler(100, callback) - class SafeChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): def create_watcher(self): @@ -1809,17 +1802,16 @@ class PolicyTests(unittest.TestCase): def create_policy(self): return asyncio.DefaultEventLoopPolicy() - def test_get_child_watcher(self): + def test_get_default_child_watcher(self): policy = self.create_policy() self.assertIsNone(policy._watcher) watcher = policy.get_child_watcher() - self.assertIsInstance(watcher, asyncio.SafeChildWatcher) + self.assertIsInstance(watcher, asyncio.ThreadedChildWatcher) self.assertIs(policy._watcher, watcher) self.assertIs(watcher, policy.get_child_watcher()) - self.assertIsNone(watcher._loop) def test_get_child_watcher_after_set(self): policy = self.create_policy() @@ -1829,18 +1821,6 @@ def test_get_child_watcher_after_set(self): self.assertIs(policy._watcher, watcher) self.assertIs(watcher, policy.get_child_watcher()) - def test_get_child_watcher_with_mainloop_existing(self): - policy = self.create_policy() - loop = policy.get_event_loop() - - self.assertIsNone(policy._watcher) - watcher = policy.get_child_watcher() - - self.assertIsInstance(watcher, asyncio.SafeChildWatcher) - self.assertIs(watcher._loop, loop) - - loop.close() - def test_get_child_watcher_thread(self): def f(): @@ -1865,7 +1845,11 @@ def test_child_watcher_replace_mainloop_existing(self): policy = self.create_policy() loop = policy.get_event_loop() - watcher = policy.get_child_watcher() + # Explicitly setup SafeChildWatcher, + # default ThreadedChildWatcher has no _loop property + watcher = asyncio.SafeChildWatcher() + policy.set_child_watcher(watcher) + watcher.attach_loop(loop) self.assertIs(watcher._loop, loop) diff --git a/Misc/NEWS.d/next/Library/2019-05-28-19-03-46.bpo-35621.Abc1lf.rst b/Misc/NEWS.d/next/Library/2019-05-28-19-03-46.bpo-35621.Abc1lf.rst new file mode 100644 index 000000000000..c492e1de6d5c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-28-19-03-46.bpo-35621.Abc1lf.rst @@ -0,0 +1,2 @@ +Support running asyncio subprocesses when execution event loop in a thread +on UNIX. [View Less]

1 0

bpo-36027: Extend three-argument pow to negative second argument (GH-13266)
by Mark Dickinson June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/c52996785a45d4693857ea219e040777a1… commit: c52996785a45d4693857ea219e040777a14584f8 branch: master author: Mark Dickinson <mdickinson(a)enthought.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T10:24:06+01:00 summary: bpo-36027: Extend three-argument pow to negative second argument (GH-13266) files: A Misc/NEWS.d/next/Core and Builtins/2019-05-12-18-46-50.bpo-36027.Q4YatQ.rst M Doc/library/functions.rst M Doc/whatsnew/3.8.… [View More]rst M Lib/test/test_builtin.py M Lib/test/test_pow.py M Objects/longobject.c diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index d5c9f18c79b7..415a65b4946f 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1277,9 +1277,24 @@ are always available. They are listed here in alphabetical order. operands, the result has the same type as the operands (after coercion) unless the second argument is negative; in that case, all arguments are converted to float and a float result is delivered. For example, ``10**2`` - returns ``100``, but ``10**-2`` returns ``0.01``. If the second argument is - negative, the third argument must be omitted. If *z* is present, *x* and *y* - must be of integer types, and *y* must be non-negative. + returns ``100``, but ``10**-2`` returns ``0.01``. + + For :class:`int` operands *x* and *y*, if *z* is present, *z* must also be + of integer type and *z* must be nonzero. If *z* is present and *y* is + negative, *x* must be relatively prime to *z*. In that case, ``pow(inv_x, + -y, z)`` is returned, where *inv_x* is an inverse to *x* modulo *z*. + + Here's an example of computing an inverse for ``38`` modulo ``97``:: + + >>> pow(38, -1, 97) + 23 + >>> 23 * 38 % 97 == 1 + True + + .. versionchanged:: 3.8 + For :class:`int` operands, the three-argument form of ``pow`` now allows + the second argument to be negative, permitting computation of modular + inverses. .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 591b45488372..74d0079a53db 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -304,6 +304,12 @@ Other Language Changes * Added new ``replace()`` method to the code type (:class:`types.CodeType`). (Contributed by Victor Stinner in :issue:`37032`.) +* For integers, the three-argument form of the :func:`pow` function now permits + the exponent to be negative in the case where the base is relatively prime to + the modulus. It then computes a modular inverse to the base when the exponent + is ``-1``, and a suitable power of that inverse for other negative exponents. + (Contributed by Mark Dickinson in :issue:`36027`.) + New Modules =========== diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index e32fb75d8191..b536cec06487 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1195,7 +1195,8 @@ def test_pow(self): self.assertAlmostEqual(pow(-1, 0.5), 1j) self.assertAlmostEqual(pow(-1, 1/3), 0.5 + 0.8660254037844386j) - self.assertRaises(ValueError, pow, -1, -2, 3) + # See test_pow for additional tests for three-argument pow. + self.assertEqual(pow(-1, -2, 3), 1) self.assertRaises(ValueError, pow, 1, 2, 0) self.assertRaises(TypeError, pow) diff --git a/Lib/test/test_pow.py b/Lib/test/test_pow.py index cac1ae5ea2d8..660ff80bbf52 100644 --- a/Lib/test/test_pow.py +++ b/Lib/test/test_pow.py @@ -1,3 +1,4 @@ +import math import unittest class PowTest(unittest.TestCase): @@ -119,5 +120,30 @@ def test_bug705231(self): eq(pow(a, -fiveto), expected) eq(expected, 1.0) # else we didn't push fiveto to evenness + def test_negative_exponent(self): + for a in range(-50, 50): + for m in range(-50, 50): + with self.subTest(a=a, m=m): + if m != 0 and math.gcd(a, m) == 1: + # Exponent -1 should give an inverse, with the + # same sign as m. + inv = pow(a, -1, m) + self.assertEqual(inv, inv % m) + self.assertEqual((inv * a - 1) % m, 0) + + # Larger exponents + self.assertEqual(pow(a, -2, m), pow(inv, 2, m)) + self.assertEqual(pow(a, -3, m), pow(inv, 3, m)) + self.assertEqual(pow(a, -1001, m), pow(inv, 1001, m)) + + else: + with self.assertRaises(ValueError): + pow(a, -1, m) + with self.assertRaises(ValueError): + pow(a, -2, m) + with self.assertRaises(ValueError): + pow(a, -1001, m) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-12-18-46-50.bpo-36027.Q4YatQ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-12-18-46-50.bpo-36027.Q4YatQ.rst new file mode 100644 index 000000000000..866309cddc68 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-12-18-46-50.bpo-36027.Q4YatQ.rst @@ -0,0 +1,3 @@ +Allow computation of modular inverses via three-argument ``pow``: the second +argument is now permitted to be negative in the case where the first and +third arguments are relatively prime. diff --git a/Objects/longobject.c b/Objects/longobject.c index 5d2b595621f3..49f1420bf64f 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -4174,6 +4174,98 @@ long_divmod(PyObject *a, PyObject *b) return z; } + +/* Compute an inverse to a modulo n, or raise ValueError if a is not + invertible modulo n. Assumes n is positive. The inverse returned + is whatever falls out of the extended Euclidean algorithm: it may + be either positive or negative, but will be smaller than n in + absolute value. + + Pure Python equivalent for long_invmod: + + def invmod(a, n): + b, c = 1, 0 + while n: + q, r = divmod(a, n) + a, b, c, n = n, c, b - q*c, r + + # at this point a is the gcd of the original inputs + if a == 1: + return b + raise ValueError("Not invertible") +*/ + +static PyLongObject * +long_invmod(PyLongObject *a, PyLongObject *n) +{ + PyLongObject *b, *c; + + /* Should only ever be called for positive n */ + assert(Py_SIZE(n) > 0); + + b = (PyLongObject *)PyLong_FromLong(1L); + if (b == NULL) { + return NULL; + } + c = (PyLongObject *)PyLong_FromLong(0L); + if (c == NULL) { + Py_DECREF(b); + return NULL; + } + Py_INCREF(a); + Py_INCREF(n); + + /* references now owned: a, b, c, n */ + while (Py_SIZE(n) != 0) { + PyLongObject *q, *r, *s, *t; + + if (l_divmod(a, n, &q, &r) == -1) { + goto Error; + } + Py_DECREF(a); + a = n; + n = r; + t = (PyLongObject *)long_mul(q, c); + Py_DECREF(q); + if (t == NULL) { + goto Error; + } + s = (PyLongObject *)long_sub(b, t); + Py_DECREF(t); + if (s == NULL) { + goto Error; + } + Py_DECREF(b); + b = c; + c = s; + } + /* references now owned: a, b, c, n */ + + Py_DECREF(c); + Py_DECREF(n); + if (long_compare(a, _PyLong_One)) { + /* a != 1; we don't have an inverse. */ + Py_DECREF(a); + Py_DECREF(b); + PyErr_SetString(PyExc_ValueError, + "base is not invertible for the given modulus"); + return NULL; + } + else { + /* a == 1; b gives an inverse modulo n */ + Py_DECREF(a); + return b; + } + + Error: + Py_DECREF(a); + Py_DECREF(b); + Py_DECREF(c); + Py_DECREF(n); + return NULL; +} + + /* pow(v, w, x) */ static PyObject * long_pow(PyObject *v, PyObject *w, PyObject *x) @@ -4207,20 +4299,14 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) Py_RETURN_NOTIMPLEMENTED; } - if (Py_SIZE(b) < 0) { /* if exponent is negative */ - if (c) { - PyErr_SetString(PyExc_ValueError, "pow() 2nd argument " - "cannot be negative when 3rd argument specified"); - goto Error; - } - else { - /* else return a float. This works because we know + if (Py_SIZE(b) < 0 && c == NULL) { + /* if exponent is negative and there's no modulus: + return a float. This works because we know that this calls float_pow() which converts its arguments to double. */ - Py_DECREF(a); - Py_DECREF(b); - return PyFloat_Type.tp_as_number->nb_power(v, w, x); - } + Py_DECREF(a); + Py_DECREF(b); + return PyFloat_Type.tp_as_number->nb_power(v, w, x); } if (c) { @@ -4255,6 +4341,26 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) goto Done; } + /* if exponent is negative, negate the exponent and + replace the base with a modular inverse */ + if (Py_SIZE(b) < 0) { + temp = (PyLongObject *)_PyLong_Copy(b); + if (temp == NULL) + goto Error; + Py_DECREF(b); + b = temp; + temp = NULL; + _PyLong_Negate(&b); + if (b == NULL) + goto Error; + + temp = long_invmod(a, c); + if (temp == NULL) + goto Error; + Py_DECREF(a); + a = temp; + } + /* Reduce base by modulus in some cases: 1. If base < 0. Forcing the base non-negative makes things easier. 2. If base is obviously larger than the modulus. The "small [View Less]

1 0

bpo-37128: Add math.perm(). (GH-13731)
by Serhiy Storchaka June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/5ae299ac78abb628803ab7dee099736454… commit: 5ae299ac78abb628803ab7dee0997364547f5cc8 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T11:16:49+03:00 summary: bpo-37128: Add math.perm(). (GH-13731) files: A Misc/NEWS.d/next/Library/2019-06-01-22-54-03.bpo-37128.oGXBWN.rst M Doc/library/math.rst M Lib/test/test_math.py M Modules/clinic/mathmodule.c.h M Modules/mathmodule.… [View More]c diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 8c6837050319..c5a77f1fab9f 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -207,6 +207,19 @@ Number-theoretic and representation functions of *x* and are floats. +.. function:: perm(n, k) + + Return the number of ways to choose *k* items from *n* items + without repetition and with order. + + It is mathematically equal to the expression ``n! / (n - k)!``. + + Raises :exc:`TypeError` if the arguments not integers. + Raises :exc:`ValueError` if the arguments are negative or if *k* > *n*. + + .. versionadded:: 3.8 + + .. function:: prod(iterable, *, start=1) Calculate the product of all the elements in the input *iterable*. diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index e27092eefd6e..96e0cf2fe671 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -240,6 +240,9 @@ def result_check(expected, got, ulp_tol=5, abs_tol=0.0): else: return None +class IntSubclass(int): + pass + # Class providing an __index__ method. class MyIndexable(object): def __init__(self, value): @@ -1862,6 +1865,64 @@ def test_fractions(self): self.assertAllClose(fraction_examples, rel_tol=1e-8) self.assertAllNotClose(fraction_examples, rel_tol=1e-9) + def testPerm(self): + perm = math.perm + factorial = math.factorial + # Test if factorial defintion is satisfied + for n in range(100): + for k in range(n + 1): + self.assertEqual(perm(n, k), + factorial(n) // factorial(n - k)) + + # Test for Pascal's identity + for n in range(1, 100): + for k in range(1, n): + self.assertEqual(perm(n, k), perm(n - 1, k - 1) * k + perm(n - 1, k)) + + # Test corner cases + for n in range(1, 100): + self.assertEqual(perm(n, 0), 1) + self.assertEqual(perm(n, 1), n) + self.assertEqual(perm(n, n), factorial(n)) + + # Raises TypeError if any argument is non-integer or argument count is + # not 2 + self.assertRaises(TypeError, perm, 10, 1.0) + self.assertRaises(TypeError, perm, 10, decimal.Decimal(1.0)) + self.assertRaises(TypeError, perm, 10, "1") + self.assertRaises(TypeError, perm, 10.0, 1) + self.assertRaises(TypeError, perm, decimal.Decimal(10.0), 1) + self.assertRaises(TypeError, perm, "10", 1) + + self.assertRaises(TypeError, perm, 10) + self.assertRaises(TypeError, perm, 10, 1, 3) + self.assertRaises(TypeError, perm) + + # Raises Value error if not k or n are negative numbers + self.assertRaises(ValueError, perm, -1, 1) + self.assertRaises(ValueError, perm, -2**1000, 1) + self.assertRaises(ValueError, perm, 1, -1) + self.assertRaises(ValueError, perm, 1, -2**1000) + + # Raises value error if k is greater than n + self.assertRaises(ValueError, perm, 1, 2) + self.assertRaises(ValueError, perm, 1, 2**1000) + + n = 2**1000 + self.assertEqual(perm(n, 0), 1) + self.assertEqual(perm(n, 1), n) + self.assertEqual(perm(n, 2), n * (n-1)) + self.assertRaises((OverflowError, MemoryError), perm, n, n) + + for n, k in (True, True), (True, False), (False, False): + self.assertEqual(perm(n, k), 1) + self.assertIs(type(perm(n, k)), int) + self.assertEqual(perm(IntSubclass(5), IntSubclass(2)), 20) + self.assertEqual(perm(MyIndexable(5), MyIndexable(2)), 20) + for k in range(3): + self.assertIs(type(perm(IntSubclass(5), IntSubclass(k))), int) + self.assertIs(type(perm(MyIndexable(5), MyIndexable(k))), int) + def testComb(self): comb = math.comb factorial = math.factorial @@ -1925,8 +1986,11 @@ def testComb(self): for n, k in (True, True), (True, False), (False, False): self.assertEqual(comb(n, k), 1) self.assertIs(type(comb(n, k)), int) + self.assertEqual(comb(IntSubclass(5), IntSubclass(2)), 10) self.assertEqual(comb(MyIndexable(5), MyIndexable(2)), 10) - self.assertIs(type(comb(MyIndexable(5), MyIndexable(2))), int) + for k in range(3): + self.assertIs(type(comb(IntSubclass(5), IntSubclass(k))), int) + self.assertIs(type(comb(MyIndexable(5), MyIndexable(k))), int) def test_main(): diff --git a/Misc/NEWS.d/next/Library/2019-06-01-22-54-03.bpo-37128.oGXBWN.rst b/Misc/NEWS.d/next/Library/2019-06-01-22-54-03.bpo-37128.oGXBWN.rst new file mode 100644 index 000000000000..f1b825890231 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-06-01-22-54-03.bpo-37128.oGXBWN.rst @@ -0,0 +1 @@ +Added :func:`math.perm`. diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index 92ec4bec9bf1..0efe5cc409ce 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -638,6 +638,41 @@ math_prod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k return return_value; } +PyDoc_STRVAR(math_perm__doc__, +"perm($module, n, k, /)\n" +"--\n" +"\n" +"Number of ways to choose k items from n items without repetition and with order.\n" +"\n" +"It is mathematically equal to the expression n! / (n - k)!.\n" +"\n" +"Raises TypeError if the arguments are not integers.\n" +"Raises ValueError if the arguments are negative or if k > n."); + +#define MATH_PERM_METHODDEF \ + {"perm", (PyCFunction)(void(*)(void))math_perm, METH_FASTCALL, math_perm__doc__}, + +static PyObject * +math_perm_impl(PyObject *module, PyObject *n, PyObject *k); + +static PyObject * +math_perm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *n; + PyObject *k; + + if (!_PyArg_CheckPositional("perm", nargs, 2, 2)) { + goto exit; + } + n = args[0]; + k = args[1]; + return_value = math_perm_impl(module, n, k); + +exit: + return return_value; +} + PyDoc_STRVAR(math_comb__doc__, "comb($module, n, k, /)\n" "--\n" @@ -674,4 +709,4 @@ math_comb(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=6709521e5e1d90ec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a82b0e705b6d0ec0 input=a9049054013a1b77]*/ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index bea4607b9be1..6e1099321c54 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -2998,6 +2998,120 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) } +/*[clinic input] +math.perm + + n: object + k: object + / + +Number of ways to choose k items from n items without repetition and with order. + +It is mathematically equal to the expression n! / (n - k)!. + +Raises TypeError if the arguments are not integers. +Raises ValueError if the arguments are negative or if k > n. +[clinic start generated code]*/ + +static PyObject * +math_perm_impl(PyObject *module, PyObject *n, PyObject *k) +/*[clinic end generated code: output=e021a25469653e23 input=f71ee4f6ff26be24]*/ +{ + PyObject *result = NULL, *factor = NULL; + int overflow, cmp; + long long i, factors; + + n = PyNumber_Index(n); + if (n == NULL) { + return NULL; + } + if (!PyLong_CheckExact(n)) { + Py_SETREF(n, _PyLong_Copy((PyLongObject *)n)); + if (n == NULL) { + return NULL; + } + } + k = PyNumber_Index(k); + if (k == NULL) { + Py_DECREF(n); + return NULL; + } + if (!PyLong_CheckExact(k)) { + Py_SETREF(k, _PyLong_Copy((PyLongObject *)k)); + if (k == NULL) { + Py_DECREF(n); + return NULL; + } + } + + if (Py_SIZE(n) < 0) { + PyErr_SetString(PyExc_ValueError, + "n must be a non-negative integer"); + goto error; + } + cmp = PyObject_RichCompareBool(n, k, Py_LT); + if (cmp != 0) { + if (cmp > 0) { + PyErr_SetString(PyExc_ValueError, + "k must be an integer less than or equal to n"); + } + goto error; + } + + factors = PyLong_AsLongLongAndOverflow(k, &overflow); + if (overflow > 0) { + PyErr_Format(PyExc_OverflowError, + "k must not exceed %lld", + LLONG_MAX); + goto error; + } + else if (overflow < 0 || factors < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "k must be a non-negative integer"); + } + goto error; + } + + if (factors == 0) { + result = PyLong_FromLong(1); + goto done; + } + + result = n; + Py_INCREF(result); + if (factors == 1) { + goto done; + } + + factor = n; + Py_INCREF(factor); + for (i = 1; i < factors; ++i) { + Py_SETREF(factor, PyNumber_Subtract(factor, _PyLong_One)); + if (factor == NULL) { + goto error; + } + Py_SETREF(result, PyNumber_Multiply(result, factor)); + if (result == NULL) { + goto error; + } + } + Py_DECREF(factor); + +done: + Py_DECREF(n); + Py_DECREF(k); + return result; + +error: + Py_XDECREF(factor); + Py_XDECREF(result); + Py_DECREF(n); + Py_DECREF(k); + return NULL; +} + + /*[clinic input] math.comb @@ -3028,11 +3142,24 @@ math_comb_impl(PyObject *module, PyObject *n, PyObject *k) if (n == NULL) { return NULL; } + if (!PyLong_CheckExact(n)) { + Py_SETREF(n, _PyLong_Copy((PyLongObject *)n)); + if (n == NULL) { + return NULL; + } + } k = PyNumber_Index(k); if (k == NULL) { Py_DECREF(n); return NULL; } + if (!PyLong_CheckExact(k)) { + Py_SETREF(k, _PyLong_Copy((PyLongObject *)k)); + if (k == NULL) { + Py_DECREF(n); + return NULL; + } + } if (Py_SIZE(n) < 0) { PyErr_SetString(PyExc_ValueError, @@ -3050,7 +3177,7 @@ math_comb_impl(PyObject *module, PyObject *n, PyObject *k) "k must be an integer less than or equal to n"); goto error; } - cmp = PyObject_RichCompareBool(k, temp, Py_GT); + cmp = PyObject_RichCompareBool(temp, k, Py_LT); if (cmp > 0) { Py_SETREF(k, temp); } @@ -3174,6 +3301,7 @@ static PyMethodDef math_methods[] = { {"tanh", math_tanh, METH_O, math_tanh_doc}, MATH_TRUNC_METHODDEF MATH_PROD_METHODDEF + MATH_PERM_METHODDEF MATH_COMB_METHODDEF {NULL, NULL} /* sentinel */ }; [View Less]

1 0

Add more tests for preserving identity in marshal. (GH-13736)
by Serhiy Storchaka June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/d71f3170ac9c850f6d1f9bffaa628dc473… commit: d71f3170ac9c850f6d1f9bffaa628dc473df5e75 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T09:03:59+03:00 summary: Add more tests for preserving identity in marshal. (GH-13736) files: M Lib/test/test_marshal.py diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index a3bd350c77b9..ace1593999d4 100644 --- a/Lib/… [View More]test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -383,10 +383,7 @@ def CollectObjectIDs(ids, obj): return len(ids) class InstancingTestCase(unittest.TestCase, HelperMixin): - intobj = 123321 - floatobj = 1.2345 - strobj = "abcde"*3 - dictobj = {"hello":floatobj, "goodbye":floatobj, floatobj:"hello"} + keys = (123, 1.2345, 'abc', (123, 'abc'), frozenset({123, 'abc'})) def helper3(self, rsample, recursive=False, simple=False): #we have two instances @@ -394,11 +391,12 @@ def helper3(self, rsample, recursive=False, simple=False): n0 = CollectObjectIDs(set(), sample) - s3 = marshal.dumps(sample, 3) - n3 = CollectObjectIDs(set(), marshal.loads(s3)) + for v in range(3, marshal.version + 1): + s3 = marshal.dumps(sample, v) + n3 = CollectObjectIDs(set(), marshal.loads(s3)) - #same number of instances generated - self.assertEqual(n3, n0) + #same number of instances generated + self.assertEqual(n3, n0) if not recursive: #can compare with version 2 @@ -414,20 +412,54 @@ def helper3(self, rsample, recursive=False, simple=False): self.assertGreaterEqual(len(s2), len(s3)) def testInt(self): - self.helper(self.intobj) - self.helper3(self.intobj, simple=True) + intobj = 123321 + self.helper(intobj) + self.helper3(intobj, simple=True) def testFloat(self): - self.helper(self.floatobj) - self.helper3(self.floatobj) + floatobj = 1.2345 + self.helper(floatobj) + self.helper3(floatobj) def testStr(self): - self.helper(self.strobj) - self.helper3(self.strobj) + strobj = "abcde"*3 + self.helper(strobj) + self.helper3(strobj) + + def testBytes(self): + bytesobj = b"abcde"*3 + self.helper(bytesobj) + self.helper3(bytesobj) + + def testList(self): + for obj in self.keys: + listobj = [obj, obj] + self.helper(listobj) + self.helper3(listobj) + + def testTuple(self): + for obj in self.keys: + tupleobj = (obj, obj) + self.helper(tupleobj) + self.helper3(tupleobj) + + def testSet(self): + for obj in self.keys: + setobj = {(obj, 1), (obj, 2)} + self.helper(setobj) + self.helper3(setobj) + + def testFrozenSet(self): + for obj in self.keys: + frozensetobj = frozenset({(obj, 1), (obj, 2)}) + self.helper(frozensetobj) + self.helper3(frozensetobj) def testDict(self): - self.helper(self.dictobj) - self.helper3(self.dictobj) + for obj in self.keys: + dictobj = {"hello": obj, "goodbye": obj, obj: "hello"} + self.helper(dictobj) + self.helper3(dictobj) def testModule(self): with open(__file__, "rb") as f: @@ -438,10 +470,11 @@ def testModule(self): self.helper3(code) def testRecursion(self): - d = dict(self.dictobj) + obj = 1.2345 + d = {"hello": obj, "goodbye": obj, obj: "hello"} d["self"] = d self.helper3(d, recursive=True) - l = [self.dictobj] + l = [obj, obj] l.append(l) self.helper3(l, recursive=True) [View Less]

1 0

Improve version added references in `typing` module docs (GH-13457)
by Miss Islington (bot) June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/c76add7afd68387aa2481d672e1c0d7e7b… commit: c76add7afd68387aa2481d672e1c0d7e7b4c9afc branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T17:23:40-07:00 summary: Improve version added references in `typing` module docs (GH-13457) (cherry picked from commit b7daabd711274a009e70556020efeae502a85f0b) Co-authored-by: Anthony Sottile <asottile(… [View More]a)umich.edu> files: A Misc/NEWS.d/next/Documentation/2019-05-20-22-21-17.bpo-36984.IjZlmS.rst M Doc/library/typing.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 21258a58d0aa..12f4c03f4232 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -555,7 +555,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.Collection` - .. versionadded:: 3.6 + .. versionadded:: 3.6.0 .. class:: AbstractSet(Sized, Collection[T_co]) @@ -599,6 +599,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.deque`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: List(list, MutableSequence[T]) @@ -648,6 +649,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.Awaitable`. + .. versionadded:: 3.5.2 + .. class:: Coroutine(Awaitable[V_co], Generic[T_co T_contra, V_co]) A generic version of :class:`collections.abc.Coroutine`. @@ -661,25 +664,33 @@ The module defines the following classes, functions and decorators: async def bar() -> None: x = await c # type: int + .. versionadded:: 3.5.3 + .. class:: AsyncIterable(Generic[T_co]) A generic version of :class:`collections.abc.AsyncIterable`. + .. versionadded:: 3.5.2 + .. class:: AsyncIterator(AsyncIterable[T_co]) A generic version of :class:`collections.abc.AsyncIterator`. + .. versionadded:: 3.5.2 + .. class:: ContextManager(Generic[T_co]) A generic version of :class:`contextlib.AbstractContextManager`. - .. versionadded:: 3.6 + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.0 .. class:: AsyncContextManager(Generic[T_co]) A generic version of :class:`contextlib.AbstractAsyncContextManager`. - .. versionadded:: 3.6 + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 .. class:: Dict(dict, MutableMapping[KT, VT]) @@ -708,12 +719,14 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.Counter`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) A generic version of :class:`collections.ChainMap`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -778,7 +791,7 @@ The module defines the following classes, functions and decorators: yield start start = await increment(start) - .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 .. class:: Text @@ -977,6 +990,7 @@ The module defines the following classes, functions and decorators: raise RuntimeError('no way') .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 .. data:: Union diff --git a/Misc/NEWS.d/next/Documentation/2019-05-20-22-21-17.bpo-36984.IjZlmS.rst b/Misc/NEWS.d/next/Documentation/2019-05-20-22-21-17.bpo-36984.IjZlmS.rst new file mode 100644 index 000000000000..b26eeadb924a --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2019-05-20-22-21-17.bpo-36984.IjZlmS.rst @@ -0,0 +1 @@ +Improve version added references in ``typing`` module - by Anthony Sottile. [View Less]

1 0

Improve version added references in `typing` module docs (GH-13457)
by Ivan Levkivskyi June 2, 2019

June 2, 2019

https://github.com/python/cpython/commit/b7daabd711274a009e70556020efeae502… commit: b7daabd711274a009e70556020efeae502a85f0b branch: master author: Anthony Sottile <asottile(a)umich.edu> committer: Ivan Levkivskyi <levkivskyi(a)gmail.com> date: 2019-06-02T01:13:25+01:00 summary: Improve version added references in `typing` module docs (GH-13457) files: A Misc/NEWS.d/next/Documentation/2019-05-20-22-21-17.bpo-36984.IjZlmS.rst M Doc/library/typing.rst diff --git a/Doc/library/… [View More]typing.rst b/Doc/library/typing.rst index 2575a995817d..1a766c29a57a 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -637,7 +637,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.Collection` - .. versionadded:: 3.6 + .. versionadded:: 3.6.0 .. class:: AbstractSet(Sized, Collection[T_co]) @@ -681,6 +681,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.deque`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: List(list, MutableSequence[T]) @@ -730,6 +731,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.Awaitable`. + .. versionadded:: 3.5.2 + .. class:: Coroutine(Awaitable[V_co], Generic[T_co T_contra, V_co]) A generic version of :class:`collections.abc.Coroutine`. @@ -743,25 +746,33 @@ The module defines the following classes, functions and decorators: async def bar() -> None: x = await c # type: int + .. versionadded:: 3.5.3 + .. class:: AsyncIterable(Generic[T_co]) A generic version of :class:`collections.abc.AsyncIterable`. + .. versionadded:: 3.5.2 + .. class:: AsyncIterator(AsyncIterable[T_co]) A generic version of :class:`collections.abc.AsyncIterator`. + .. versionadded:: 3.5.2 + .. class:: ContextManager(Generic[T_co]) A generic version of :class:`contextlib.AbstractContextManager`. - .. versionadded:: 3.6 + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.0 .. class:: AsyncContextManager(Generic[T_co]) A generic version of :class:`contextlib.AbstractAsyncContextManager`. - .. versionadded:: 3.6 + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 .. class:: Dict(dict, MutableMapping[KT, VT]) @@ -790,12 +801,14 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.Counter`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) A generic version of :class:`collections.ChainMap`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -860,7 +873,7 @@ The module defines the following classes, functions and decorators: yield start start = await increment(start) - .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 .. class:: Text @@ -1166,6 +1179,7 @@ The module defines the following classes, functions and decorators: raise RuntimeError('no way') .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 .. data:: Union diff --git a/Misc/NEWS.d/next/Documentation/2019-05-20-22-21-17.bpo-36984.IjZlmS.rst b/Misc/NEWS.d/next/Documentation/2019-05-20-22-21-17.bpo-36984.IjZlmS.rst new file mode 100644 index 000000000000..b26eeadb924a --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2019-05-20-22-21-17.bpo-36984.IjZlmS.rst @@ -0,0 +1 @@ +Improve version added references in ``typing`` module - by Anthony Sottile. [View Less]

1 0

bpo-32411: IDLE: Remove line number sort in browser.py (GH-5011)
by Miss Islington (bot) June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/ac60d1afd2b04f61fe4c965740fa32809f… commit: ac60d1afd2b04f61fe4c965740fa32809f2b84ed branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T15:26:00-07:00 summary: bpo-32411: IDLE: Remove line number sort in browser.py (GH-5011) Insertion in line order makes sorting keys by line order unneeded. (cherry picked from commit … [View More]1a4d9ffa1aecd7e750195f2be06d3d16c7a3a88f) Co-authored-by: Cheryl Sabella <cheryl.sabella(a)gmail.com> files: A Misc/NEWS.d/next/IDLE/2017-12-25-18-48-50.bpo-32411.vNwDhe.rst M Lib/idlelib/NEWS.txt M Lib/idlelib/browser.py M Lib/idlelib/idle_test/test_browser.py diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 7b5a13a21524..808c236bb590 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,8 @@ Released on 2019-06-24? ====================================== +bpo-32411: Stop sorting dict created with desired line order. + bpo-37038: Make idlelib.run runnable; add test clause. bpo-36958: Print any argument other than None or int passed to diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 234883fe8605..e5b0bc53c662 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -29,9 +29,10 @@ def transform_children(child_dict, modname=None): The dictionary maps names to pyclbr information objects. Filter out imported objects. Augment class names with bases. - Sort objects by line number. + The insertion order of the dictonary is assumed to have been in line + number order, so sorting is not necessary. - The current tree only calls this once per child_dic as it saves + The current tree only calls this once per child_dict as it saves TreeItems once created. A future tree and tests might violate this, so a check prevents multiple in-place augmentations. """ @@ -51,7 +52,7 @@ def transform_children(child_dict, modname=None): supers.append(sname) obj.name += '({})'.format(', '.join(supers)) obs.append(obj) - return sorted(obs, key=lambda o: o.lineno) + return obs class ModuleBrowser: diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index dfbab6dd6b5e..25d6dc663036 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -61,16 +61,16 @@ def test_close(self): # Nested tree same as in test_pyclbr.py except for supers on C0. C1. mb = pyclbr module, fname = 'test', 'test.py' -f0 = mb.Function(module, 'f0', fname, 1) -f1 = mb._nest_function(f0, 'f1', 2) -f2 = mb._nest_function(f1, 'f2', 3) -c1 = mb._nest_class(f0, 'c1', 5) -C0 = mb.Class(module, 'C0', ['base'], fname, 6) -F1 = mb._nest_function(C0, 'F1', 8) -C1 = mb._nest_class(C0, 'C1', 11, ['']) -C2 = mb._nest_class(C1, 'C2', 12) -F3 = mb._nest_function(C2, 'F3', 14) -mock_pyclbr_tree = {'f0': f0, 'C0': C0} +C0 = mb.Class(module, 'C0', ['base'], fname, 1) +F1 = mb._nest_function(C0, 'F1', 3) +C1 = mb._nest_class(C0, 'C1', 6, ['']) +C2 = mb._nest_class(C1, 'C2', 7) +F3 = mb._nest_function(C2, 'F3', 9) +f0 = mb.Function(module, 'f0', fname, 11) +f1 = mb._nest_function(f0, 'f1', 12) +f2 = mb._nest_function(f1, 'f2', 13) +c1 = mb._nest_class(f0, 'c1', 15) +mock_pyclbr_tree = {'C0': C0, 'f0': f0} # Adjust C0.name, C1.name so tests do not depend on order. browser.transform_children(mock_pyclbr_tree, 'test') # C0(base) @@ -87,12 +87,12 @@ def test_transform_module_children(self): transform = browser.transform_children # Parameter matches tree module. tcl = list(transform(mock_pyclbr_tree, 'test')) - eq(tcl, [f0, C0]) - eq(tcl[0].name, 'f0') - eq(tcl[1].name, 'C0(base)') + eq(tcl, [C0, f0]) + eq(tcl[0].name, 'C0(base)') + eq(tcl[1].name, 'f0') # Check that second call does not change suffix. tcl = list(transform(mock_pyclbr_tree, 'test')) - eq(tcl[1].name, 'C0(base)') + eq(tcl[0].name, 'C0(base)') # Nothing to traverse if parameter name isn't same as tree module. tcl = list(transform(mock_pyclbr_tree, 'different name')) eq(tcl, []) diff --git a/Misc/NEWS.d/next/IDLE/2017-12-25-18-48-50.bpo-32411.vNwDhe.rst b/Misc/NEWS.d/next/IDLE/2017-12-25-18-48-50.bpo-32411.vNwDhe.rst new file mode 100644 index 000000000000..a5522012923a --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-12-25-18-48-50.bpo-32411.vNwDhe.rst @@ -0,0 +1,2 @@ +In browser.py, remove extraneous sorting by line number since dictionary was +created in line number order. [View Less]

1 0

Put math.comb() docs is correct place alphabetically (GH-13734)
by Raymond Hettinger June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/b7fade4f87e0d37d1686a4e8596141e55e… commit: b7fade4f87e0d37d1686a4e8596141e55ecef099 branch: master author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T15:01:46-07:00 summary: Put math.comb() docs is correct place alphabetically (GH-13734) files: M Doc/library/math.rst diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 206b06edd2a2..8c6837050319 100644 --- … [View More]

1 0

Fix the error handling in bytesio_sizeof(). (GH-10459)
by Miss Islington (bot) June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/fefdc009906c5ea8fb57383817b3c42d3b… commit: fefdc009906c5ea8fb57383817b3c42d3b0634ad branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T14:58:33-07:00 summary: Fix the error handling in bytesio_sizeof(). (GH-10459) bytesio_sizeof() must check if an error has occurred in _PySys_GetSizeOf(). (cherry picked from commit … [View More]

1 0

bpo-33608: Factor out a private, per-interpreter _Py_AddPendingCall(). (gh-13714)
by Eric Snow June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/6a150bcaeb190d1731b38ab9c7a5d1a352… commit: 6a150bcaeb190d1731b38ab9c7a5d1a352847ddc branch: master author: Eric Snow <ericsnowcurrently(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T15:39:46-06:00 summary: bpo-33608: Factor out a private, per-interpreter _Py_AddPendingCall(). (gh-13714) files: A Misc/NEWS.d/next/Core and Builtins/2018-09-15-12-13-46.bpo-33608.avmvVP.rst M Include/internal/pycore_ceval.h M Include/… [View More]internal/pycore_pystate.h M Lib/test/test_capi.py M Modules/_testcapimodule.c M Modules/signalmodule.c M Python/ceval.c M Python/ceval_gil.h M Python/pylifecycle.c M Python/pystate.c diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 37170ed438f8..d44afdf4fa46 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -12,19 +12,22 @@ extern "C" { #include "pycore_pystate.h" #include "pythread.h" -PyAPI_FUNC(void) _Py_FinishPendingCalls(_PyRuntimeState *runtime); PyAPI_FUNC(void) _PyEval_Initialize(struct _ceval_runtime_state *); PyAPI_FUNC(void) _PyEval_FiniThreads( - struct _ceval_runtime_state *ceval); + struct _ceval_runtime_state *); PyAPI_FUNC(void) _PyEval_SignalReceived( - struct _ceval_runtime_state *ceval); + struct _ceval_runtime_state *); PyAPI_FUNC(int) _PyEval_AddPendingCall( PyThreadState *tstate, - struct _ceval_runtime_state *ceval, + struct _ceval_runtime_state *, + struct _ceval_interpreter_state *, + unsigned long thread_id, int (*func)(void *), void *arg); +PyAPI_FUNC(void) _PyEval_FinishPendingCalls(PyInterpreterState *); PyAPI_FUNC(void) _PyEval_SignalAsyncExc( - struct _ceval_runtime_state *ceval); + struct _ceval_runtime_state *, + struct _ceval_interpreter_state *); PyAPI_FUNC(void) _PyEval_ReInitThreads( _PyRuntimeState *runtime); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 520a74b8a61f..aca5533022e3 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -25,7 +25,7 @@ struct pyruntimestate; /* ceval state */ -struct _pending_calls { +struct _ceval_pending_calls { int finishing; PyThread_type_lock lock; /* Request for running pending calls. */ @@ -36,6 +36,7 @@ struct _pending_calls { int async_exc; #define NPENDINGCALLS 32 struct { + unsigned long thread_id; int (*func)(void *); void *arg; } calls[NPENDINGCALLS]; @@ -53,15 +54,21 @@ struct _ceval_runtime_state { int tracing_possible; /* This single variable consolidates all requests to break out of the fast path in the eval loop. */ + // XXX This can move to _ceval_interpreter_state once all parts + // from COMPUTE_EVAL_BREAKER have moved under PyInterpreterState. _Py_atomic_int eval_breaker; /* Request for dropping the GIL */ _Py_atomic_int gil_drop_request; - struct _pending_calls pending; /* Request for checking signals. */ _Py_atomic_int signals_pending; struct _gil_runtime_state gil; }; +struct _ceval_interpreter_state { + struct _ceval_pending_calls pending; +}; + + /* interpreter state */ typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int); @@ -136,6 +143,7 @@ struct _is { uint64_t tstate_next_unique_id; + struct _ceval_interpreter_state ceval; struct _warnings_runtime_state warnings; PyObject *audit_hooks; diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 4dd78bb9a2fd..fabc821e5c3a 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -431,7 +431,7 @@ def pendingcalls_wait(self, l, n, context = None): def test_pendingcalls_threaded(self): #do every callback on a separate thread - n = 32 #total callbacks + n = 32 #total callbacks (see NPENDINGCALLS in pycore_ceval.h) threads = [] class foo(object):pass context = foo() diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-09-15-12-13-46.bpo-33608.avmvVP.rst b/Misc/NEWS.d/next/Core and Builtins/2018-09-15-12-13-46.bpo-33608.avmvVP.rst new file mode 100644 index 000000000000..73a01a1f46bd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-09-15-12-13-46.bpo-33608.avmvVP.rst @@ -0,0 +1,5 @@ +We added a new internal _Py_AddPendingCall() that operates relative to the +provided interpreter. This allows us to use the existing implementation to +ask another interpreter to do work that cannot be done in the current +interpreter, like decref an object the other interpreter owns. The existing +Py_AddPendingCall() only operates relative to the main interpreter. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b42f41cc8d8f..bf20e81a4ce8 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2677,6 +2677,7 @@ pending_threadfunc(PyObject *self, PyObject *arg) Py_INCREF(callable); Py_BEGIN_ALLOW_THREADS + /* XXX Use the internal _Py_AddPendingCall(). */ r = Py_AddPendingCall(&_pending_callback, callable); Py_END_ALLOW_THREADS diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 7698984ff3af..1964646da252 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -21,6 +21,7 @@ #include <process.h> #endif #endif +#include "internal/pycore_pystate.h" #ifdef HAVE_SIGNAL_H #include <signal.h> @@ -259,6 +260,7 @@ trip_signal(int sig_num) /* Notify ceval.c */ _PyRuntimeState *runtime = &_PyRuntime; PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyInterpreterState *interp = runtime->interpreters.main; _PyEval_SignalReceived(&runtime->ceval); /* And then write to the wakeup fd *after* setting all the globals and @@ -299,7 +301,10 @@ trip_signal(int sig_num) { /* Py_AddPendingCall() isn't signal-safe, but we still use it for this exceptional case. */ - _PyEval_AddPendingCall(tstate, &runtime->ceval, + _PyEval_AddPendingCall(tstate, + &runtime->ceval, + &interp->ceval, + runtime->main_thread, report_wakeup_send_error, (void *)(intptr_t) last_error); } @@ -318,7 +323,10 @@ trip_signal(int sig_num) { /* Py_AddPendingCall() isn't signal-safe, but we still use it for this exceptional case. */ - _PyEval_AddPendingCall(tstate, &runtime->ceval, + _PyEval_AddPendingCall(tstate, + &runtime->ceval, + &interp->ceval, + runtime->main_thread, report_wakeup_write_error, (void *)(intptr_t)errno); } diff --git a/Python/ceval.c b/Python/ceval.c index d9a71e942153..a092a2355641 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -101,66 +101,64 @@ static long dxp[256]; #endif #endif -#define GIL_REQUEST _Py_atomic_load_relaxed(&ceval->gil_drop_request) - /* This can set eval_breaker to 0 even though gil_drop_request became 1. We believe this is all right because the eval loop will release the GIL eventually anyway. */ -#define COMPUTE_EVAL_BREAKER(ceval) \ +#define COMPUTE_EVAL_BREAKER(ceval_r, ceval_i) \ _Py_atomic_store_relaxed( \ - &(ceval)->eval_breaker, \ - GIL_REQUEST | \ - _Py_atomic_load_relaxed(&(ceval)->signals_pending) | \ - _Py_atomic_load_relaxed(&(ceval)->pending.calls_to_do) | \ - (ceval)->pending.async_exc) + &(ceval_r)->eval_breaker, \ + _Py_atomic_load_relaxed(&(ceval_r)->gil_drop_request) | \ + _Py_atomic_load_relaxed(&(ceval_r)->signals_pending) | \ + _Py_atomic_load_relaxed(&(ceval_i)->pending.calls_to_do) | \ + (ceval_i)->pending.async_exc) -#define SET_GIL_DROP_REQUEST(ceval) \ +#define SET_GIL_DROP_REQUEST(ceval_r) \ do { \ - _Py_atomic_store_relaxed(&(ceval)->gil_drop_request, 1); \ - _Py_atomic_store_relaxed(&(ceval)->eval_breaker, 1); \ + _Py_atomic_store_relaxed(&(ceval_r)->gil_drop_request, 1); \ + _Py_atomic_store_relaxed(&(ceval_r)->eval_breaker, 1); \ } while (0) -#define RESET_GIL_DROP_REQUEST(ceval) \ +#define RESET_GIL_DROP_REQUEST(ceval_r, ceval_i) \ do { \ - _Py_atomic_store_relaxed(&(ceval)->gil_drop_request, 0); \ - COMPUTE_EVAL_BREAKER(ceval); \ + _Py_atomic_store_relaxed(&(ceval_r)->gil_drop_request, 0); \ + COMPUTE_EVAL_BREAKER(ceval_r, ceval_i); \ } while (0) /* Pending calls are only modified under pending_lock */ -#define SIGNAL_PENDING_CALLS(ceval) \ +#define SIGNAL_PENDING_CALLS(ceval_r, ceval_i) \ do { \ - _Py_atomic_store_relaxed(&(ceval)->pending.calls_to_do, 1); \ - _Py_atomic_store_relaxed(&(ceval)->eval_breaker, 1); \ + _Py_atomic_store_relaxed(&(ceval_i)->pending.calls_to_do, 1); \ + _Py_atomic_store_relaxed(&(ceval_r)->eval_breaker, 1); \ } while (0) -#define UNSIGNAL_PENDING_CALLS(ceval) \ +#define UNSIGNAL_PENDING_CALLS(ceval_r, ceval_i) \ do { \ - _Py_atomic_store_relaxed(&(ceval)->pending.calls_to_do, 0); \ - COMPUTE_EVAL_BREAKER(ceval); \ + _Py_atomic_store_relaxed(&(ceval_i)->pending.calls_to_do, 0); \ + COMPUTE_EVAL_BREAKER(ceval_r, ceval_i); \ } while (0) -#define SIGNAL_PENDING_SIGNALS(ceval) \ +#define SIGNAL_PENDING_SIGNALS(ceval_r) \ do { \ - _Py_atomic_store_relaxed(&(ceval)->signals_pending, 1); \ - _Py_atomic_store_relaxed(&(ceval)->eval_breaker, 1); \ + _Py_atomic_store_relaxed(&(ceval_r)->signals_pending, 1); \ + _Py_atomic_store_relaxed(&(ceval_r)->eval_breaker, 1); \ } while (0) -#define UNSIGNAL_PENDING_SIGNALS(ceval) \ +#define UNSIGNAL_PENDING_SIGNALS(ceval_r, ceval_i) \ do { \ - _Py_atomic_store_relaxed(&(ceval)->signals_pending, 0); \ - COMPUTE_EVAL_BREAKER(ceval); \ + _Py_atomic_store_relaxed(&(ceval_r)->signals_pending, 0); \ + COMPUTE_EVAL_BREAKER(ceval_r, ceval_i); \ } while (0) -#define SIGNAL_ASYNC_EXC(ceval) \ +#define SIGNAL_ASYNC_EXC(ceval_r, ceval_i) \ do { \ - (ceval)->pending.async_exc = 1; \ - _Py_atomic_store_relaxed(&(ceval)->eval_breaker, 1); \ + (ceval_i)->pending.async_exc = 1; \ + _Py_atomic_store_relaxed(&(ceval_r)->eval_breaker, 1); \ } while (0) -#define UNSIGNAL_ASYNC_EXC(ceval) \ +#define UNSIGNAL_ASYNC_EXC(ceval_r, ceval_i) \ do { \ - (ceval)->pending.async_exc = 0; \ - COMPUTE_EVAL_BREAKER(ceval); \ + (ceval_i)->pending.async_exc = 0; \ + COMPUTE_EVAL_BREAKER(ceval_r, ceval_i); \ } while (0) @@ -180,8 +178,8 @@ void PyEval_InitThreads(void) { _PyRuntimeState *runtime = &_PyRuntime; - struct _ceval_runtime_state *ceval = &runtime->ceval; - struct _gil_runtime_state *gil = &ceval->gil; + struct _ceval_runtime_state *ceval_r = &runtime->ceval; + struct _gil_runtime_state *gil = &ceval_r->gil; if (gil_created(gil)) { return; } @@ -189,19 +187,15 @@ PyEval_InitThreads(void) PyThread_init_thread(); create_gil(gil); PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); - take_gil(ceval, tstate); + take_gil(ceval_r, tstate); - struct _pending_calls *pending = &ceval->pending; - pending->lock = PyThread_allocate_lock(); - if (pending->lock == NULL) { - Py_FatalError("Can't initialize threads for pending calls"); - } + // The pending calls mutex is initialized in PyInterpreterState_New(). } void -_PyEval_FiniThreads(struct _ceval_runtime_state *ceval) +_PyEval_FiniThreads(struct _ceval_runtime_state *ceval_r) { - struct _gil_runtime_state *gil = &ceval->gil; + struct _gil_runtime_state *gil = &ceval_r->gil; if (!gil_created(gil)) { return; } @@ -209,20 +203,24 @@ _PyEval_FiniThreads(struct _ceval_runtime_state *ceval) destroy_gil(gil); assert(!gil_created(gil)); - struct _pending_calls *pending = &ceval->pending; - if (pending->lock != NULL) { - PyThread_free_lock(pending->lock); - pending->lock = NULL; - } + // The pending calls mutex is freed in PyInterpreterState_Delete(). } static inline void exit_thread_if_finalizing(PyThreadState *tstate) { - _PyRuntimeState *runtime = tstate->interp->runtime; - /* _Py_Finalizing is protected by the GIL */ + PyInterpreterState *interp = tstate->interp; + // Stop if thread/interpreter inalization already stated. + if (interp == NULL) { + return; + } + _PyRuntimeState *runtime = interp->runtime; + if (runtime == NULL) { + return; + } + // Don't exit if the main thread (i.e. of the main interpreter). if (runtime->finalizing != NULL && !_Py_CURRENTLY_FINALIZING(runtime, tstate)) { - drop_gil(&runtime->ceval, tstate); + drop_gil(&runtime->ceval, &interp->ceval, tstate); PyThread_exit_thread(); } } @@ -231,12 +229,12 @@ void PyEval_AcquireLock(void) { _PyRuntimeState *runtime = &_PyRuntime; - struct _ceval_runtime_state *ceval = &runtime->ceval; + struct _ceval_runtime_state *ceval_r = &runtime->ceval; PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); if (tstate == NULL) { Py_FatalError("PyEval_AcquireLock: current thread state is NULL"); } - take_gil(ceval, tstate); + take_gil(ceval_r, tstate); exit_thread_if_finalizing(tstate); } @@ -244,12 +242,21 @@ void PyEval_ReleaseLock(void) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); /* This function must succeed when the current thread state is NULL. We therefore avoid PyThreadState_Get() which dumps a fatal error in debug mode. */ - drop_gil(&runtime->ceval, tstate); + PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + // Fall back to the main interpreter if there is not active Python + // thread. This only affects the eval_breaker. + PyInterpreterState *interp = runtime->interpreters.main; + if (tstate != NULL) { + interp = tstate->interp; + if (interp == NULL) { + Py_FatalError("PyEval_ReleaseLock: NULL interpreter state"); + } + } + drop_gil(&runtime->ceval, &interp->ceval, tstate); } void @@ -258,14 +265,19 @@ PyEval_AcquireThread(PyThreadState *tstate) if (tstate == NULL) { Py_FatalError("PyEval_AcquireThread: NULL new thread state"); } - assert(tstate->interp != NULL); - - _PyRuntimeState *runtime = tstate->interp->runtime; - struct _ceval_runtime_state *ceval = &runtime->ceval; + PyInterpreterState *interp = tstate->interp; + if (interp == NULL) { + Py_FatalError("PyEval_AcquireThread: NULL interpreter state"); + } + _PyRuntimeState *runtime = interp->runtime; + if (runtime == NULL) { + Py_FatalError("PyEval_AcquireThread: NULL runtime state"); + } + struct _ceval_runtime_state *ceval_r = &runtime->ceval; /* Check someone has called PyEval_InitThreads() to create the lock */ - assert(gil_created(&ceval->gil)); - take_gil(ceval, tstate); + assert(gil_created(&ceval_r->gil)); + take_gil(ceval_r, tstate); exit_thread_if_finalizing(tstate); if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) { Py_FatalError("PyEval_AcquireThread: non-NULL old thread state"); @@ -278,14 +290,20 @@ PyEval_ReleaseThread(PyThreadState *tstate) if (tstate == NULL) { Py_FatalError("PyEval_ReleaseThread: NULL thread state"); } - assert(tstate->interp != NULL); + PyInterpreterState *interp = tstate->interp; + if (interp == NULL) { + Py_FatalError("PyEval_ReleaseThread: NULL interpreter state"); + } + _PyRuntimeState *runtime = interp->runtime; + if (runtime == NULL) { + Py_FatalError("PyEval_ReleaseThread: NULL runtime state"); + } - _PyRuntimeState *runtime = tstate->interp->runtime; PyThreadState *new_tstate = _PyThreadState_Swap(&runtime->gilstate, NULL); if (new_tstate != tstate) { Py_FatalError("PyEval_ReleaseThread: wrong thread state"); } - drop_gil(&runtime->ceval, tstate); + drop_gil(&runtime->ceval, &interp->ceval, tstate); } /* This function is called from PyOS_AfterFork_Child to destroy all threads @@ -296,15 +314,17 @@ PyEval_ReleaseThread(PyThreadState *tstate) void _PyEval_ReInitThreads(_PyRuntimeState *runtime) { - struct _ceval_runtime_state *ceval = &runtime->ceval; - if (!gil_created(&ceval->gil)) { + struct _ceval_runtime_state *ceval_r = &runtime->ceval; + if (!gil_created(&ceval_r->gil)) { return; } - recreate_gil(&ceval->gil); + recreate_gil(&ceval_r->gil); PyThreadState *current_tstate = _PyRuntimeState_GetThreadState(runtime); - take_gil(ceval, current_tstate); + take_gil(ceval_r, current_tstate); - struct _pending_calls *pending = &ceval->pending; + // Only the main interpreter remains, so ignore the rest. + PyInterpreterState *interp = _PyRuntime.interpreters.main; + struct _ceval_pending_calls *pending = &interp->ceval.pending; pending->lock = PyThread_allocate_lock(); if (pending->lock == NULL) { Py_FatalError("Can't initialize threads for pending calls"); @@ -318,22 +338,28 @@ _PyEval_ReInitThreads(_PyRuntimeState *runtime) raised. */ void -_PyEval_SignalAsyncExc(struct _ceval_runtime_state *ceval) +_PyEval_SignalAsyncExc(struct _ceval_runtime_state *ceval_r, + struct _ceval_interpreter_state *ceval_i) { - SIGNAL_ASYNC_EXC(ceval); + SIGNAL_ASYNC_EXC(ceval_r, ceval_i); } PyThreadState * PyEval_SaveThread(void) { _PyRuntimeState *runtime = &_PyRuntime; - struct _ceval_runtime_state *ceval = &runtime->ceval; + struct _ceval_runtime_state *ceval_r = &runtime->ceval; PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, NULL); if (tstate == NULL) { Py_FatalError("PyEval_SaveThread: NULL tstate"); } - assert(gil_created(&ceval->gil)); - drop_gil(ceval, tstate); + PyInterpreterState *interp = tstate->interp; + if (interp == NULL) { + Py_FatalError("PyEval_SaveThread: NULL interpreter state"); + } + + assert(gil_created(&ceval_r->gil)); + drop_gil(ceval_r, &interp->ceval, tstate); return tstate; } @@ -343,14 +369,20 @@ PyEval_RestoreThread(PyThreadState *tstate) if (tstate == NULL) { Py_FatalError("PyEval_RestoreThread: NULL tstate"); } - assert(tstate->interp != NULL); + PyInterpreterState *interp = tstate->interp; + if (interp == NULL) { + Py_FatalError("PyEval_RestoreThread: NULL interpreter state"); + } + _PyRuntimeState *runtime = interp->runtime; + if (runtime == NULL) { + Py_FatalError("PyEval_RestoreThread: NULL runtime state"); + } + struct _ceval_runtime_state *ceval_r = &runtime->ceval; - _PyRuntimeState *runtime = tstate->interp->runtime; - struct _ceval_runtime_state *ceval = &runtime->ceval; - assert(gil_created(&ceval->gil)); + assert(gil_created(&ceval_r->gil)); int err = errno; - take_gil(ceval, tstate); + take_gil(ceval_r, tstate); exit_thread_if_finalizing(tstate); errno = err; @@ -381,17 +413,17 @@ PyEval_RestoreThread(PyThreadState *tstate) */ void -_PyEval_SignalReceived(struct _ceval_runtime_state *ceval) +_PyEval_SignalReceived(struct _ceval_runtime_state *ceval_r) { /* bpo-30703: Function called when the C signal handler of Python gets a signal. We cannot queue a callback using Py_AddPendingCall() since that function is not async-signal-safe. */ - SIGNAL_PENDING_SIGNALS(ceval); + SIGNAL_PENDING_SIGNALS(ceval_r); } /* Push one item onto the queue while holding the lock. */ static int -_push_pending_call(struct _pending_calls *pending, +_push_pending_call(struct _ceval_pending_calls *pending, unsigned long thread_id, int (*func)(void *), void *arg) { int i = pending->last; @@ -399,6 +431,7 @@ _push_pending_call(struct _pending_calls *pending, if (j == pending->first) { return -1; /* Queue full */ } + pending->calls[i].thread_id = thread_id; pending->calls[i].func = func; pending->calls[i].arg = arg; pending->last = j; @@ -407,7 +440,7 @@ _push_pending_call(struct _pending_calls *pending, /* Pop one item off the queue while holding the lock. */ static void -_pop_pending_call(struct _pending_calls *pending, +_pop_pending_call(struct _ceval_pending_calls *pending, unsigned long *thread_id, int (**func)(void *), void **arg) { int i = pending->first; @@ -417,6 +450,7 @@ _pop_pending_call(struct _pending_calls *pending, *func = pending->calls[i].func; *arg = pending->calls[i].arg; + *thread_id = pending->calls[i].thread_id; pending->first = (i + 1) % NPENDINGCALLS; } @@ -427,10 +461,12 @@ _pop_pending_call(struct _pending_calls *pending, int _PyEval_AddPendingCall(PyThreadState *tstate, - struct _ceval_runtime_state *ceval, + struct _ceval_runtime_state *ceval_r, + struct _ceval_interpreter_state *ceval_i, + unsigned long thread_id, int (*func)(void *), void *arg) { - struct _pending_calls *pending = &ceval->pending; + struct _ceval_pending_calls *pending = &ceval_i->pending; PyThread_acquire_lock(pending->lock, WAIT_LOCK); if (pending->finishing) { @@ -445,20 +481,27 @@ _PyEval_AddPendingCall(PyThreadState *tstate, _PyErr_Restore(tstate, exc, val, tb); return -1; } - int result = _push_pending_call(pending, func, arg); + int result = _push_pending_call(pending, thread_id, func, arg); + + /* signal loop */ + SIGNAL_PENDING_CALLS(ceval_r, ceval_i); PyThread_release_lock(pending->lock); - /* signal main loop */ - SIGNAL_PENDING_CALLS(ceval); return result; } +/* Py_AddPendingCall() is a simple wrapper for the sake + of backward-compatibility. */ int Py_AddPendingCall(int (*func)(void *), void *arg) { _PyRuntimeState *runtime = &_PyRuntime; + PyInterpreterState *interp = runtime->interpreters.main; PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); - return _PyEval_AddPendingCall(tstate, &runtime->ceval, func, arg); + return _PyEval_AddPendingCall(tstate, + &runtime->ceval, &interp->ceval, + runtime->main_thread, + func, arg); } static int @@ -479,47 +522,69 @@ handle_signals(_PyRuntimeState *runtime) return 0; } - struct _ceval_runtime_state *ceval = &runtime->ceval; - UNSIGNAL_PENDING_SIGNALS(ceval); + struct _ceval_runtime_state *ceval_r = &runtime->ceval; + struct _ceval_interpreter_state *ceval_i = &interp->ceval; + UNSIGNAL_PENDING_SIGNALS(ceval_r, ceval_i); if (_PyErr_CheckSignals() < 0) { - SIGNAL_PENDING_SIGNALS(ceval); /* We're not done yet */ + SIGNAL_PENDING_SIGNALS(ceval_r); /* We're not done yet */ return -1; } return 0; } static int -make_pending_calls(_PyRuntimeState *runtime) +make_pending_calls(PyInterpreterState *interp) { - static int busy = 0; - - /* only service pending calls on main thread */ - if (PyThread_get_thread_ident() != runtime->main_thread) { - return 0; + if (interp == NULL) { + Py_FatalError("make_pending_calls: NULL interpreter state"); + } + _PyRuntimeState *runtime = interp->runtime; + if (runtime == NULL) { + Py_FatalError("make_pending_calls: NULL runtime state"); + } + PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + if (tstate == NULL) { + Py_FatalError("make_pending_calls: NULL thread state"); } + if (tstate->interp == NULL || tstate->interp != interp) { + Py_FatalError("make_pending_calls: thread state mismatch"); + } + static int busy = 0; /* don't perform recursive pending calls */ if (busy) { return 0; } busy = 1; - struct _ceval_runtime_state *ceval = &runtime->ceval; + struct _ceval_runtime_state *ceval_r = &runtime->ceval; + struct _ceval_interpreter_state *ceval_i = &interp->ceval; /* unsignal before starting to call callbacks, so that any callback added in-between re-signals */ - UNSIGNAL_PENDING_CALLS(ceval); + UNSIGNAL_PENDING_CALLS(ceval_r, ceval_i); int res = 0; /* perform a bounded number of calls, in case of recursion */ - struct _pending_calls *pending = &ceval->pending; + struct _ceval_pending_calls *pending = &ceval_i->pending; + unsigned long thread_id = 0; for (int i=0; i<NPENDINGCALLS; i++) { int (*func)(void *) = NULL; void *arg = NULL; /* pop one item off the queue while holding the lock */ PyThread_acquire_lock(pending->lock, WAIT_LOCK); - _pop_pending_call(pending, &func, &arg); + _pop_pending_call(pending, &thread_id, &func, &arg); PyThread_release_lock(pending->lock); + if (thread_id && PyThread_get_thread_ident() != thread_id) { + // Thread mismatch, so move it to the end of the list + // and start over. + _PyEval_AddPendingCall(tstate, + &runtime->ceval, &interp->ceval, + thread_id, + func, arg); + goto error; + } + /* having released the lock, perform the callback */ if (func == NULL) { break; @@ -535,17 +600,16 @@ make_pending_calls(_PyRuntimeState *runtime) error: busy = 0; - SIGNAL_PENDING_CALLS(ceval); + SIGNAL_PENDING_CALLS(ceval_r, ceval_i); return res; } void -_Py_FinishPendingCalls(_PyRuntimeState *runtime) +_PyEval_FinishPendingCalls(PyInterpreterState *interp) { assert(PyGILState_Check()); - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); - struct _pending_calls *pending = &runtime->ceval.pending; + struct _ceval_pending_calls *pending = &interp->ceval.pending; PyThread_acquire_lock(pending->lock, WAIT_LOCK); pending->finishing = 1; @@ -555,12 +619,19 @@ _Py_FinishPendingCalls(_PyRuntimeState *runtime) return; } - if (make_pending_calls(runtime) < 0) { - PyObject *exc, *val, *tb; - _PyErr_Fetch(tstate, &exc, &val, &tb); - PyErr_BadInternalCall(); - _PyErr_ChainExceptions(exc, val, tb); - _PyErr_Print(tstate); + if (make_pending_calls(interp) < 0) { + _PyRuntimeState *runtime = interp->runtime; + if (runtime == NULL) { + runtime = &_PyRuntime; + } + PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + if (tstate != NULL) { + PyObject *exc, *val, *tb; + _PyErr_Fetch(tstate, &exc, &val, &tb); + PyErr_BadInternalCall(); + _PyErr_ChainExceptions(exc, val, tb); + _PyErr_Print(tstate); + } } } @@ -579,7 +650,8 @@ Py_MakePendingCalls(void) return res; } - res = make_pending_calls(runtime); + PyInterpreterState *interp = _PyRuntime.interpreters.main; + res = make_pending_calls(interp); if (res != 0) { return res; } @@ -596,11 +668,11 @@ Py_MakePendingCalls(void) int _Py_CheckRecursionLimit = Py_DEFAULT_RECURSION_LIMIT; void -_PyEval_Initialize(struct _ceval_runtime_state *state) +_PyEval_Initialize(struct _ceval_runtime_state *ceval_r) { - state->recursion_limit = Py_DEFAULT_RECURSION_LIMIT; + ceval_r->recursion_limit = Py_DEFAULT_RECURSION_LIMIT; _Py_CheckRecursionLimit = Py_DEFAULT_RECURSION_LIMIT; - _gil_initialize(&state->gil); + _gil_initialize(&ceval_r->gil); } int @@ -612,9 +684,9 @@ Py_GetRecursionLimit(void) void Py_SetRecursionLimit(int new_limit) { - struct _ceval_runtime_state *ceval = &_PyRuntime.ceval; - ceval->recursion_limit = new_limit; - _Py_CheckRecursionLimit = ceval->recursion_limit; + struct _ceval_runtime_state *ceval_r = &_PyRuntime.ceval; + ceval_r->recursion_limit = new_limit; + _Py_CheckRecursionLimit = ceval_r->recursion_limit; } /* the macro Py_EnterRecursiveCall() only calls _Py_CheckRecursiveCall() @@ -663,7 +735,7 @@ _Py_CheckRecursiveCall(const char *where) static int do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause); static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **); -#define _Py_TracingPossible(ceval) ((ceval)->tracing_possible) +#define _Py_TracingPossible(ceval_r) ((ceval_r)->tracing_possible) PyObject * @@ -709,8 +781,10 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) PyObject *retval = NULL; /* Return value */ _PyRuntimeState * const runtime = &_PyRuntime; PyThreadState * const tstate = _PyRuntimeState_GetThreadState(runtime); - struct _ceval_runtime_state * const ceval = &runtime->ceval; - _Py_atomic_int * const eval_breaker = &ceval->eval_breaker; + PyInterpreterState * const interp = tstate->interp; + struct _ceval_runtime_state * const ceval_r = &runtime->ceval; + struct _ceval_interpreter_state * const ceval_i = &interp->ceval; + _Py_atomic_int * const eval_breaker = &ceval_r->eval_breaker; PyCodeObject *co; /* when tracing we set things up so that @@ -797,7 +871,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) #ifdef LLTRACE #define FAST_DISPATCH() \ { \ - if (!lltrace && !_Py_TracingPossible(ceval) && !PyDTrace_LINE_ENABLED()) { \ + if (!lltrace && !_Py_TracingPossible(ceval_r) && !PyDTrace_LINE_ENABLED()) { \ f->f_lasti = INSTR_OFFSET(); \ NEXTOPARG(); \ goto *opcode_targets[opcode]; \ @@ -807,7 +881,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) #else #define FAST_DISPATCH() \ { \ - if (!_Py_TracingPossible(ceval) && !PyDTrace_LINE_ENABLED()) { \ + if (!_Py_TracingPossible(ceval_r) && !PyDTrace_LINE_ENABLED()) { \ f->f_lasti = INSTR_OFFSET(); \ NEXTOPARG(); \ goto *opcode_targets[opcode]; \ @@ -1122,27 +1196,27 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) goto fast_next_opcode; } - if (_Py_atomic_load_relaxed(&ceval->signals_pending)) { + if (_Py_atomic_load_relaxed(&ceval_r->signals_pending)) { if (handle_signals(runtime) != 0) { goto error; } } - if (_Py_atomic_load_relaxed(&ceval->pending.calls_to_do)) { - if (make_pending_calls(runtime) != 0) { + if (_Py_atomic_load_relaxed(&ceval_i->pending.calls_to_do)) { + if (make_pending_calls(interp) != 0) { goto error; } } - if (_Py_atomic_load_relaxed(&ceval->gil_drop_request)) { + if (_Py_atomic_load_relaxed(&ceval_r->gil_drop_request)) { /* Give another thread a chance */ if (_PyThreadState_Swap(&runtime->gilstate, NULL) != tstate) { Py_FatalError("ceval: tstate mix-up"); } - drop_gil(ceval, tstate); + drop_gil(ceval_r, ceval_i, tstate); /* Other threads may run now */ - take_gil(ceval, tstate); + take_gil(ceval_r, tstate); /* Check if we should make a quick exit. */ exit_thread_if_finalizing(tstate); @@ -1155,7 +1229,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) if (tstate->async_exc != NULL) { PyObject *exc = tstate->async_exc; tstate->async_exc = NULL; - UNSIGNAL_ASYNC_EXC(ceval); + UNSIGNAL_ASYNC_EXC(ceval_r, ceval_i); _PyErr_SetNone(tstate, exc); Py_DECREF(exc); goto error; @@ -1170,7 +1244,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) /* line-by-line tracing support */ - if (_Py_TracingPossible(ceval) && + if (_Py_TracingPossible(ceval_r) && tstate->c_tracefunc != NULL && !tstate->tracing) { int err; /* see maybe_call_line_trace diff --git a/Python/ceval_gil.h b/Python/ceval_gil.h index 34d48c990c44..b44d0abad36b 100644 --- a/Python/ceval_gil.h +++ b/Python/ceval_gil.h @@ -141,9 +141,11 @@ static void recreate_gil(struct _gil_runtime_state *gil) } static void -drop_gil(struct _ceval_runtime_state *ceval, PyThreadState *tstate) +drop_gil(struct _ceval_runtime_state *ceval_r, + struct _ceval_interpreter_state *ceval_i, + PyThreadState *tstate) { - struct _gil_runtime_state *gil = &ceval->gil; + struct _gil_runtime_state *gil = &ceval_r->gil; if (!_Py_atomic_load_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } @@ -163,12 +165,12 @@ drop_gil(struct _ceval_runtime_state *ceval, PyThreadState *tstate) MUTEX_UNLOCK(gil->mutex); #ifdef FORCE_SWITCHING - if (_Py_atomic_load_relaxed(&ceval->gil_drop_request) && tstate != NULL) { + if (_Py_atomic_load_relaxed(&ceval_r->gil_drop_request) && tstate != NULL) { MUTEX_LOCK(gil->switch_mutex); /* Not switched yet => wait */ if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate) { - RESET_GIL_DROP_REQUEST(ceval); + RESET_GIL_DROP_REQUEST(ceval_r, ceval_i); /* NOTE: if COND_WAIT does not atomically start waiting when releasing the mutex, another thread can run through, take the GIL and drop it again, and reset the condition @@ -181,13 +183,19 @@ drop_gil(struct _ceval_runtime_state *ceval, PyThreadState *tstate) } static void -take_gil(struct _ceval_runtime_state *ceval, PyThreadState *tstate) +take_gil(struct _ceval_runtime_state *ceval_r, + PyThreadState *tstate) { if (tstate == NULL) { Py_FatalError("take_gil: NULL tstate"); } + PyInterpreterState *interp = tstate->interp; + if (interp == NULL) { + Py_FatalError("take_gil: NULL interp"); + } + struct _ceval_interpreter_state *ceval_i = &interp->ceval; - struct _gil_runtime_state *gil = &ceval->gil; + struct _gil_runtime_state *gil = &ceval_r->gil; int err = errno; MUTEX_LOCK(gil->mutex); @@ -210,7 +218,7 @@ take_gil(struct _ceval_runtime_state *ceval, PyThreadState *tstate) _Py_atomic_load_relaxed(&gil->locked) && gil->switch_number == saved_switchnum) { - SET_GIL_DROP_REQUEST(ceval); + SET_GIL_DROP_REQUEST(ceval_r); } } _ready: @@ -232,11 +240,11 @@ take_gil(struct _ceval_runtime_state *ceval, PyThreadState *tstate) COND_SIGNAL(gil->switch_cond); MUTEX_UNLOCK(gil->switch_mutex); #endif - if (_Py_atomic_load_relaxed(&ceval->gil_drop_request)) { - RESET_GIL_DROP_REQUEST(ceval); + if (_Py_atomic_load_relaxed(&ceval_r->gil_drop_request)) { + RESET_GIL_DROP_REQUEST(ceval_r, ceval_i); } if (tstate->async_exc != NULL) { - _PyEval_SignalAsyncExc(ceval); + _PyEval_SignalAsyncExc(ceval_r, ceval_i); } MUTEX_UNLOCK(gil->mutex); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 6590ef8e9a27..3de5528811ae 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1147,15 +1147,31 @@ Py_FinalizeEx(void) return status; } + /* Get current thread state and interpreter pointer */ + PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyInterpreterState *interp = tstate->interp; + // Wrap up existing "threading"-module-created, non-daemon threads. wait_for_thread_shutdown(); // Make any remaining pending calls. - _Py_FinishPendingCalls(runtime); - - /* Get current thread state and interpreter pointer */ - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); - PyInterpreterState *interp = tstate->interp; + /* XXX For the moment we are going to ignore lingering pending calls. + * We've seen sporadic on some of the buildbots during finalization + * with the changes for per-interpreter pending calls (see bpo-33608), + * meaning the previous _PyEval_FinishPendincCalls() call here is + * a trigger, if not responsible. + * + * Ignoring pending calls at this point in the runtime lifecycle + * is okay (for now) for the following reasons: + * + * * pending calls are still not a widely-used feature + * * this only affects runtime finalization, where the process is + * likely to end soon anyway (except for some embdding cases) + * + * See bpo-37127 about resolving the problem. Ultimately the call + * here should be re-enabled. + */ + //_PyEval_FinishPendingCalls(interp); /* The interpreter is still entirely intact at this point, and the * exit funcs may be relying on that. In particular, if some thread @@ -1580,6 +1596,9 @@ Py_EndInterpreter(PyThreadState *tstate) // Wrap up existing "threading"-module-created, non-daemon threads. wait_for_thread_shutdown(); + // Make any remaining pending calls. + _PyEval_FinishPendingCalls(interp); + call_py_exitfuncs(interp); if (tstate != interp->tstate_head || tstate->next != NULL) diff --git a/Python/pystate.c b/Python/pystate.c index 2b7db0e48deb..a9f3389a0d83 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -218,6 +218,13 @@ PyInterpreterState_New(void) return NULL; } + interp->ceval.pending.lock = PyThread_allocate_lock(); + if (interp->ceval.pending.lock == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "failed to create interpreter ceval pending mutex"); + return NULL; + } + interp->eval_frame = _PyEval_EvalFrameDefault; #ifdef HAVE_DLOPEN #if HAVE_DECL_RTLD_NOW @@ -345,6 +352,10 @@ PyInterpreterState_Delete(PyInterpreterState *interp) if (interp->id_mutex != NULL) { PyThread_free_lock(interp->id_mutex); } + if (interp->ceval.pending.lock != NULL) { + PyThread_free_lock(interp->ceval.pending.lock); + interp->ceval.pending.lock = NULL; + } PyMem_RawFree(interp); } @@ -1014,7 +1025,7 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) p->async_exc = exc; HEAD_UNLOCK(runtime); Py_XDECREF(old_exc); - _PyEval_SignalAsyncExc(&runtime->ceval); + _PyEval_SignalAsyncExc(&runtime->ceval, &interp->ceval); return 1; } } @@ -1444,7 +1455,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) return 0; } -static void +static int _release_xidata(void *arg) { _PyCrossInterpreterData *data = (_PyCrossInterpreterData *)arg; @@ -1452,42 +1463,21 @@ _release_xidata(void *arg) data->free(data->data); } Py_XDECREF(data->obj); -} - -static void -_call_in_interpreter(struct _gilstate_runtime_state *gilstate, - PyInterpreterState *interp, - void (*func)(void *), void *arg) -{ - /* We would use Py_AddPendingCall() if it weren't specific to the - * main interpreter (see bpo-33608). In the meantime we take a - * naive approach. - */ - PyThreadState *save_tstate = NULL; - if (interp != _PyRuntimeGILState_GetThreadState(gilstate)->interp) { - // XXX Using the "head" thread isn't strictly correct. - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - // XXX Possible GILState issues? - save_tstate = _PyThreadState_Swap(gilstate, tstate); - } - - func(arg); - - // Switch back. - if (save_tstate != NULL) { - _PyThreadState_Swap(gilstate, save_tstate); - } + PyMem_Free(data); + return 0; } void _PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) { + _PyRuntimeState *runtime = &_PyRuntime; + if (data->data == NULL && data->obj == NULL) { // Nothing to release! return; } - // Switch to the original interpreter. + // Get the original interpreter. PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interp); if (interp == NULL) { // The intepreter was already destroyed. @@ -1496,10 +1486,28 @@ _PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) } return; } + // XXX There's an ever-so-slight race here... + if (interp->finalizing) { + // XXX Someone leaked some memory... + return; + } // "Release" the data and/or the object. - struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; - _call_in_interpreter(gilstate, interp, _release_xidata, data); + _PyCrossInterpreterData *copied = PyMem_Malloc(sizeof(_PyCrossInterpreterData)); + if (copied == NULL) { + PyErr_SetString(PyExc_MemoryError, + "Not enough memory to preserve cross-interpreter data"); + PyErr_Print(); + return; + } + memcpy(copied, data, sizeof(_PyCrossInterpreterData)); + PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + int res = _PyEval_AddPendingCall(tstate, + &runtime->ceval, &interp->ceval, + 0, _release_xidata, copied); + if (res != 0) { + // XXX Queue full or couldn't get lock. Try again somehow? + } } PyObject * [View Less]

1 0

bpo-29414: Change 'the for statement is such an iterator' in Tutorial (GH-273)
by Raymond Hettinger June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/218e47b61862470477922e9aba1a23fd3d… commit: 218e47b61862470477922e9aba1a23fd3dab18ae branch: master author: Marco Buttu <marco.buttu(a)gmail.com> committer: Raymond Hettinger <rhettinger(a)users.noreply.github.com> date: 2019-06-01T14:11:47-07:00 summary: bpo-29414: Change 'the for statement is such an iterator' in Tutorial (GH-273) files: M Doc/tutorial/controlflow.rst diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.… [View More]rst index 81a28a6e5325..79111f8518d9 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -139,18 +139,24 @@ but in fact it isn't. It is an object which returns the successive items of the desired sequence when you iterate over it, but it doesn't really make the list, thus saving space. -We say such an object is *iterable*, that is, suitable as a target for +We say such an object is :term:`iterable`, that is, suitable as a target for functions and constructs that expect something from which they can -obtain successive items until the supply is exhausted. We have seen that -the :keyword:`for` statement is such an *iterator*. The function :func:`list` -is another; it creates lists from iterables:: +obtain successive items until the supply is exhausted. We have seen that +the :keyword:`for` statement is such a construct, while an example of function +that takes an iterable is :func:`sum`:: + >>> sum(range(4)) # 0 + 1 + 2 + 3 + 6 - >>> list(range(5)) - [0, 1, 2, 3, 4] +Later we will see more functions that return iterables and take iterables as +arguments. Lastly, maybe you are curious about how to get a list from a range. +Here is the solution:: -Later we will see more functions that return iterables and take iterables as argument. + >>> list(range(4)) + [0, 1, 2, 3] +In chapter :ref:`tut-structures`, we will discuss in more detail about +:func:`list`. .. _tut-break: @@ -161,7 +167,7 @@ The :keyword:`break` statement, like in C, breaks out of the innermost enclosing :keyword:`for` or :keyword:`while` loop. Loop statements may have an :keyword:`!else` clause; it is executed when the loop -terminates through exhaustion of the list (with :keyword:`for`) or when the +terminates through exhaustion of the iterable (with :keyword:`for`) or when the condition becomes false (with :keyword:`while`), but not when the loop is terminated by a :keyword:`break` statement. This is exemplified by the following loop, which searches for prime numbers:: @@ -188,8 +194,8 @@ following loop, which searches for prime numbers:: the :keyword:`for` loop, **not** the :keyword:`if` statement.) When used with a loop, the ``else`` clause has more in common with the -``else`` clause of a :keyword:`try` statement than it does that of -:keyword:`if` statements: a :keyword:`!try` statement's ``else`` clause runs +``else`` clause of a :keyword:`try` statement than it does with that of +:keyword:`if` statements: a :keyword:`try` statement's ``else`` clause runs when no exception occurs, and a loop's ``else`` clause runs when no ``break`` occurs. For more on the :keyword:`!try` statement and exceptions, see :ref:`tut-handling`. [View Less]

1 0

Fix the error handling in bytesio_sizeof(). (GH-10459)
by Serhiy Storchaka June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/36dcaab7fde5d2e54cdeff5b705b5adcb2… commit: 36dcaab7fde5d2e54cdeff5b705b5adcb27726dd branch: master author: Zackery Spytz <zspytz(a)gmail.com> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2019-06-02T00:07:45+03:00 summary: Fix the error handling in bytesio_sizeof(). (GH-10459) bytesio_sizeof() must check if an error has occurred in _PySys_GetSizeOf(). files: M Modules/_io/bytesio.c diff --git a/Modules/_io/bytesio.c b/Modules/… [View More]

1 0

bpo-20092. Use __index__ in constructors of int, float and complex. (GH-13108)
by Serhiy Storchaka June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/bdbad71b9def0b86433de12cecca022eee… commit: bdbad71b9def0b86433de12cecca022eee91bd9f branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T00:05:48+03:00 summary: bpo-20092. Use __index__ in constructors of int, float and complex. (GH-13108) files: A Misc/NEWS.d/next/Core and Builtins/2019-05-31-11-55-49.bpo-20092.KIMjBW.rst M Doc/c-api/complex.rst M Doc/c-api/float.rst M … [View More]Doc/library/functions.rst M Doc/reference/datamodel.rst M Doc/whatsnew/3.8.rst M Lib/test/test_cmath.py M Lib/test/test_complex.py M Lib/test/test_float.py M Lib/test/test_getargs2.py M Lib/test/test_index.py M Lib/test/test_int.py M Objects/abstract.c M Objects/complexobject.c M Objects/floatobject.c diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 675bd013e892..06dbb2572725 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -129,4 +129,10 @@ Complex Numbers as Python Objects If *op* is not a Python complex number object but has a :meth:`__complex__` method, this method will first be called to convert *op* to a Python complex - number object. Upon failure, this method returns ``-1.0`` as a real value. + number object. If ``__complex__()`` is not defined then it falls back to + :meth:`__float__`. If ``__float__()`` is not defined then it falls back + to :meth:`__index__`. Upon failure, this method returns ``-1.0`` as a real + value. + + .. versionchanged:: 3.8 + Use :meth:`__index__` if available. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 8a996422ce48..057ff522516a 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -47,9 +47,13 @@ Floating Point Objects Return a C :c:type:`double` representation of the contents of *pyfloat*. If *pyfloat* is not a Python floating point object but has a :meth:`__float__` method, this method will first be called to convert *pyfloat* into a float. + If ``__float__()`` is not defined then it falls back to :meth:`__index__`. This method returns ``-1.0`` upon failure, so one should call :c:func:`PyErr_Occurred` to check for errors. + .. versionchanged:: 3.8 + Use :meth:`__index__` if available. + .. c:function:: double PyFloat_AS_DOUBLE(PyObject *pyfloat) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 425a985320fc..d5c9f18c79b7 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -318,6 +318,11 @@ are always available. They are listed here in alphabetical order. :class:`int` and :class:`float`. If both arguments are omitted, returns ``0j``. + For a general Python object ``x``, ``complex(x)`` delegates to + ``x.__complex__()``. If ``__complex__()`` is not defined then it falls back + to :meth:`__float__`. If ``__float__()`` is not defined then it falls back + to :meth:`__index__`. + .. note:: When converting from a string, the string must not contain whitespace @@ -330,6 +335,10 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.6 Grouping digits with underscores as in code literals is allowed. + .. versionchanged:: 3.8 + Falls back to :meth:`__index__` if :meth:`__complex__` and + :meth:`__float__` are not defined. + .. function:: delattr(object, name) @@ -584,7 +593,8 @@ are always available. They are listed here in alphabetical order. float, an :exc:`OverflowError` will be raised. For a general Python object ``x``, ``float(x)`` delegates to - ``x.__float__()``. + ``x.__float__()``. If ``__float__()`` is not defined then it falls back + to :meth:`__index__`. If no argument is given, ``0.0`` is returned. @@ -609,6 +619,9 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.7 *x* is now a positional-only parameter. + .. versionchanged:: 3.8 + Falls back to :meth:`__index__` if :meth:`__float__` is not defined. + .. index:: single: __format__ @@ -780,7 +793,8 @@ are always available. They are listed here in alphabetical order. Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* defines :meth:`__int__`, - ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`__trunc__`, + ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`__index__`, + it returns ``x.__index__()``. If *x* defines :meth:`__trunc__`, it returns ``x.__trunc__()``. For floating point numbers, this truncates towards zero. @@ -812,6 +826,9 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.7 *x* is now a positional-only parameter. + .. versionchanged:: 3.8 + Falls back to :meth:`__index__` if :meth:`__int__` is not defined. + .. function:: isinstance(object, classinfo) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 44017d8a55df..fa47bf1c1619 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2394,11 +2394,9 @@ left undefined. functions). Presence of this method indicates that the numeric object is an integer type. Must return an integer. - .. note:: - - In order to have a coherent integer type class, when :meth:`__index__` is - defined :meth:`__int__` should also be defined, and both should return - the same value. + If :meth:`__int__`, :meth:`__float__` and :meth:`__complex__` are not + defined then corresponding built-in functions :func:`int`, :func:`float` + and :func:`complex` fall back to :meth:`__index__`. .. method:: object.__round__(self, [,ndigits]) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 4c5a9bb0cdb9..591b45488372 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -250,6 +250,12 @@ Other Language Changes compatible with the existing :meth:`float.as_integer_ratio` method. (Contributed by Lisa Roach in :issue:`33073`.) +* Constructors of :class:`int`, :class:`float` and :class:`complex` will now + use the :meth:`~object.__index__` special method, if available and the + corresponding method :meth:`~object.__int__`, :meth:`~object.__float__` + or :meth:`~object.__complex__` is not available. + (Contributed by Serhiy Storchaka in :issue:`20092`.) + * Added support of ``\N{name}`` escapes in :mod:`regular expressions <re>`. (Contributed by Jonathan Eunice and Serhiy Storchaka in :issue:`30688`.) @@ -868,7 +874,10 @@ Build and C API Changes ``__index__()`` method (like :class:`~decimal.Decimal` and :class:`~fractions.Fraction`). :c:func:`PyNumber_Check` will now return ``1`` for objects implementing ``__index__()``. - (Contributed by Serhiy Storchaka in :issue:`36048`.) + :c:func:`PyNumber_Long`, :c:func:`PyNumber_Float` and + :c:func:`PyFloat_AsDouble` also now use the ``__index__()`` method if + available. + (Contributed by Serhiy Storchaka in :issue:`36048` and :issue:`20092`.) * Heap-allocated type objects will now increase their reference count in :c:func:`PyObject_Init` (and its parallel macro ``PyObject_INIT``) diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index 43a074b4b663..a00185f43dbf 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -220,12 +220,11 @@ class NeitherComplexNorFloat(object): pass class NeitherComplexNorFloatOS: pass - class MyInt(object): + class Index: def __int__(self): return 2 def __index__(self): return 2 - class MyIntOS: + class MyInt: def __int__(self): return 2 - def __index__(self): return 2 # other possible combinations of __float__ and __complex__ # that should work @@ -255,6 +254,7 @@ def __float__(self): self.assertEqual(f(FloatAndComplexOS()), f(cx_arg)) self.assertEqual(f(JustFloat()), f(flt_arg)) self.assertEqual(f(JustFloatOS()), f(flt_arg)) + self.assertEqual(f(Index()), f(int(Index()))) # TypeError should be raised for classes not providing # either __complex__ or __float__, even if they provide # __int__ or __index__. An old-style class @@ -263,7 +263,6 @@ def __float__(self): self.assertRaises(TypeError, f, NeitherComplexNorFloat()) self.assertRaises(TypeError, f, MyInt()) self.assertRaises(Exception, f, NeitherComplexNorFloatOS()) - self.assertRaises(Exception, f, MyIntOS()) # non-complex return value from __complex__ -> TypeError for bad_complex in non_complexes: self.assertRaises(TypeError, f, MyComplex(bad_complex)) diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 21c6eaed6054..fe1e566562de 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -368,6 +368,24 @@ def __float__(self): self.assertAlmostEqual(complex(real=float2(17.), imag=float2(23.)), 17+23j) self.assertRaises(TypeError, complex, float2(None)) + class MyIndex: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + + self.assertAlmostEqual(complex(MyIndex(42)), 42.0+0.0j) + self.assertAlmostEqual(complex(123, MyIndex(42)), 123.0+42.0j) + self.assertRaises(OverflowError, complex, MyIndex(2**2000)) + self.assertRaises(OverflowError, complex, 123, MyIndex(2**2000)) + + class MyInt: + def __int__(self): + return 42 + + self.assertRaises(TypeError, complex, MyInt()) + self.assertRaises(TypeError, complex, 123, MyInt()) + class complex0(complex): """Test usage of __complex__() when inheriting from 'complex'""" def __complex__(self): diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 5278d716de23..b656582538e8 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -223,6 +223,21 @@ def __float__(self): with self.assertWarns(DeprecationWarning): self.assertIs(type(FloatSubclass(F())), FloatSubclass) + class MyIndex: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + + self.assertEqual(float(MyIndex(42)), 42.0) + self.assertRaises(OverflowError, float, MyIndex(2**2000)) + + class MyInt: + def __int__(self): + return 42 + + self.assertRaises(TypeError, float, MyInt()) + def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): float(x='3.14') diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py index 07e2d1513791..1a73fa461580 100644 --- a/Lib/test/test_getargs2.py +++ b/Lib/test/test_getargs2.py @@ -457,6 +457,8 @@ def test_f(self): with self.assertWarns(DeprecationWarning): self.assertEqual(getargs_f(BadFloat2()), 4.25) self.assertEqual(getargs_f(BadFloat3(7.5)), 7.5) + self.assertEqual(getargs_f(Index()), 99.0) + self.assertRaises(TypeError, getargs_f, Int()) for x in (FLT_MIN, -FLT_MIN, FLT_MAX, -FLT_MAX, INF, -INF): self.assertEqual(getargs_f(x), x) @@ -489,6 +491,8 @@ def test_d(self): with self.assertWarns(DeprecationWarning): self.assertEqual(getargs_d(BadFloat2()), 4.25) self.assertEqual(getargs_d(BadFloat3(7.5)), 7.5) + self.assertEqual(getargs_d(Index()), 99.0) + self.assertRaises(TypeError, getargs_d, Int()) for x in (DBL_MIN, -DBL_MIN, DBL_MAX, -DBL_MAX, INF, -INF): self.assertEqual(getargs_d(x), x) @@ -511,6 +515,8 @@ def test_D(self): with self.assertWarns(DeprecationWarning): self.assertEqual(getargs_D(BadComplex2()), 4.25+0.5j) self.assertEqual(getargs_D(BadComplex3(7.5+0.25j)), 7.5+0.25j) + self.assertEqual(getargs_D(Index()), 99.0+0j) + self.assertRaises(TypeError, getargs_D, Int()) for x in (DBL_MIN, -DBL_MIN, DBL_MAX, -DBL_MAX, INF, -INF): c = complex(x, 1.0) diff --git a/Lib/test/test_index.py b/Lib/test/test_index.py index a2ac32132e23..cbdc56c801a4 100644 --- a/Lib/test/test_index.py +++ b/Lib/test/test_index.py @@ -60,7 +60,7 @@ def test_int_subclass_with_index(self): # subclasses. See issue #17576. class MyInt(int): def __index__(self): - return int(self) + 1 + return int(str(self)) + 1 my_int = MyInt(7) direct_index = my_int.__index__() diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 307ca36bb4fa..6fdf52ef23f6 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -378,15 +378,23 @@ def __trunc__(self): int(ExceptionalTrunc()) for trunc_result_base in (object, Classic): - class Integral(trunc_result_base): - def __int__(self): + class Index(trunc_result_base): + def __index__(self): return 42 class TruncReturnsNonInt(base): def __trunc__(self): - return Integral() - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(TruncReturnsNonInt()), 42) + return Index() + self.assertEqual(int(TruncReturnsNonInt()), 42) + + class Intable(trunc_result_base): + def __int__(self): + return 42 + + class TruncReturnsNonIndex(base): + def __trunc__(self): + return Intable() + self.assertEqual(int(TruncReturnsNonInt()), 42) class NonIntegral(trunc_result_base): def __trunc__(self): @@ -418,6 +426,21 @@ def __trunc__(self): with self.assertRaises(TypeError): int(TruncReturnsBadInt()) + def test_int_subclass_with_index(self): + class MyIndex(int): + def __index__(self): + return 42 + + class BadIndex(int): + def __index__(self): + return 42.0 + + my_int = MyIndex(7) + self.assertEqual(my_int, 7) + self.assertEqual(int(my_int), 7) + + self.assertEqual(int(BadIndex()), 0) + def test_int_subclass_with_int(self): class MyInt(int): def __int__(self): @@ -431,9 +454,19 @@ def __int__(self): self.assertEqual(my_int, 7) self.assertEqual(int(my_int), 42) - self.assertRaises(TypeError, int, BadInt()) + my_int = BadInt(7) + self.assertEqual(my_int, 7) + self.assertRaises(TypeError, int, my_int) def test_int_returns_int_subclass(self): + class BadIndex: + def __index__(self): + return True + + class BadIndex2(int): + def __index__(self): + return True + class BadInt: def __int__(self): return True @@ -442,6 +475,10 @@ class BadInt2(int): def __int__(self): return True + class TruncReturnsBadIndex: + def __trunc__(self): + return BadIndex() + class TruncReturnsBadInt: def __trunc__(self): return BadInt() @@ -450,6 +487,17 @@ class TruncReturnsIntSubclass: def __trunc__(self): return True + bad_int = BadIndex() + with self.assertWarns(DeprecationWarning): + n = int(bad_int) + self.assertEqual(n, 1) + self.assertIs(type(n), int) + + bad_int = BadIndex2() + n = int(bad_int) + self.assertEqual(n, 0) + self.assertIs(type(n), int) + bad_int = BadInt() with self.assertWarns(DeprecationWarning): n = int(bad_int) @@ -462,6 +510,12 @@ def __trunc__(self): self.assertEqual(n, 1) self.assertIs(type(n), int) + bad_int = TruncReturnsBadIndex() + with self.assertWarns(DeprecationWarning): + n = int(bad_int) + self.assertEqual(n, 1) + self.assertIs(type(n), int) + bad_int = TruncReturnsBadInt() with self.assertWarns(DeprecationWarning): n = int(bad_int) diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-31-11-55-49.bpo-20092.KIMjBW.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-31-11-55-49.bpo-20092.KIMjBW.rst new file mode 100644 index 000000000000..7536dc33c9f1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-31-11-55-49.bpo-20092.KIMjBW.rst @@ -0,0 +1,4 @@ +Constructors of :class:`int`, :class:`float` and :class:`complex` will now +use the :meth:`~object.__index__` special method, if available and the +corresponding method :meth:`~object.__int__`, :meth:`~object.__float__` +or :meth:`~object.__complex__` is not available. diff --git a/Objects/abstract.c b/Objects/abstract.c index 68d06edfa600..77d09143aa07 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1373,6 +1373,13 @@ PyNumber_Long(PyObject *o) } return result; } + if (m && m->nb_index) { + result = _PyLong_FromNbIndexOrNbInt(o); + if (result != NULL && !PyLong_CheckExact(result)) { + Py_SETREF(result, _PyLong_Copy((PyLongObject *)result)); + } + return result; + } trunc_func = _PyObject_LookupSpecial(o, &PyId___trunc__); if (trunc_func) { result = _PyObject_CallNoArg(trunc_func); @@ -1480,6 +1487,18 @@ PyNumber_Float(PyObject *o) Py_DECREF(res); return PyFloat_FromDouble(val); } + if (m && m->nb_index) { + PyObject *res = PyNumber_Index(o); + if (!res) { + return NULL; + } + double val = PyLong_AsDouble(res); + Py_DECREF(res); + if (val == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyFloat_FromDouble(val); + } if (PyFloat_Check(o)) { /* A float subclass with nb_float == NULL */ return PyFloat_FromDouble(PyFloat_AS_DOUBLE(o)); } diff --git a/Objects/complexobject.c b/Objects/complexobject.c index a5f95186d625..f78c0fdf78de 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -984,7 +984,7 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) } nbr = r->ob_type->tp_as_number; - if (nbr == NULL || nbr->nb_float == NULL) { + if (nbr == NULL || (nbr->nb_float == NULL && nbr->nb_index == NULL)) { PyErr_Format(PyExc_TypeError, "complex() first argument must be a string or a number, " "not '%.200s'", @@ -996,7 +996,7 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) } if (i != NULL) { nbi = i->ob_type->tp_as_number; - if (nbi == NULL || nbi->nb_float == NULL) { + if (nbi == NULL || (nbi->nb_float == NULL && nbi->nb_index == NULL)) { PyErr_Format(PyExc_TypeError, "complex() second argument must be a number, " "not '%.200s'", @@ -1052,7 +1052,7 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) /* The "imag" part really is entirely imaginary, and contributes nothing in the real direction. Just treat it as a double. */ - tmp = (*nbi->nb_float)(i); + tmp = PyNumber_Float(i); if (tmp == NULL) return NULL; ci.real = PyFloat_AsDouble(tmp); diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 2bf7061d4f62..15cbe5c9d8ba 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -246,6 +246,15 @@ PyFloat_AsDouble(PyObject *op) nb = Py_TYPE(op)->tp_as_number; if (nb == NULL || nb->nb_float == NULL) { + if (nb && nb->nb_index) { + PyObject *res = PyNumber_Index(op); + if (!res) { + return -1; + } + double val = PyLong_AsDouble(res); + Py_DECREF(res); + return val; + } PyErr_Format(PyExc_TypeError, "must be real number, not %.50s", op->ob_type->tp_name); return -1; [View Less]

1 0

bpo-32411: IDLE: Remove line number sort in browser.py (#5011)
by Terry Jan Reedy June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/1a4d9ffa1aecd7e750195f2be06d3d16c7… commit: 1a4d9ffa1aecd7e750195f2be06d3d16c7a3a88f branch: master author: Cheryl Sabella <cheryl.sabella(a)gmail.com> committer: Terry Jan Reedy <tjreedy(a)udel.edu> date: 2019-06-01T17:03:22-04:00 summary: bpo-32411: IDLE: Remove line number sort in browser.py (#5011) Insertion in line order makes sorting keys by line order unneeded. files: A Misc/NEWS.d/next/IDLE/2017-12-25-18-48-50.bpo-32411.vNwDhe.rst … [View More]M Lib/idlelib/NEWS.txt M Lib/idlelib/browser.py M Lib/idlelib/idle_test/test_browser.py diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index e1bc009ae933..b260d4e5ef1b 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,8 @@ Released on 2019-10-20? ====================================== +bpo-32411: Stop sorting dict created with desired line order. + bpo-37038: Make idlelib.run runnable; add test clause. bpo-36958: Print any argument other than None or int passed to diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 234883fe8605..e5b0bc53c662 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -29,9 +29,10 @@ def transform_children(child_dict, modname=None): The dictionary maps names to pyclbr information objects. Filter out imported objects. Augment class names with bases. - Sort objects by line number. + The insertion order of the dictonary is assumed to have been in line + number order, so sorting is not necessary. - The current tree only calls this once per child_dic as it saves + The current tree only calls this once per child_dict as it saves TreeItems once created. A future tree and tests might violate this, so a check prevents multiple in-place augmentations. """ @@ -51,7 +52,7 @@ def transform_children(child_dict, modname=None): supers.append(sname) obj.name += '({})'.format(', '.join(supers)) obs.append(obj) - return sorted(obs, key=lambda o: o.lineno) + return obs class ModuleBrowser: diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index dfbab6dd6b5e..25d6dc663036 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -61,16 +61,16 @@ def test_close(self): # Nested tree same as in test_pyclbr.py except for supers on C0. C1. mb = pyclbr module, fname = 'test', 'test.py' -f0 = mb.Function(module, 'f0', fname, 1) -f1 = mb._nest_function(f0, 'f1', 2) -f2 = mb._nest_function(f1, 'f2', 3) -c1 = mb._nest_class(f0, 'c1', 5) -C0 = mb.Class(module, 'C0', ['base'], fname, 6) -F1 = mb._nest_function(C0, 'F1', 8) -C1 = mb._nest_class(C0, 'C1', 11, ['']) -C2 = mb._nest_class(C1, 'C2', 12) -F3 = mb._nest_function(C2, 'F3', 14) -mock_pyclbr_tree = {'f0': f0, 'C0': C0} +C0 = mb.Class(module, 'C0', ['base'], fname, 1) +F1 = mb._nest_function(C0, 'F1', 3) +C1 = mb._nest_class(C0, 'C1', 6, ['']) +C2 = mb._nest_class(C1, 'C2', 7) +F3 = mb._nest_function(C2, 'F3', 9) +f0 = mb.Function(module, 'f0', fname, 11) +f1 = mb._nest_function(f0, 'f1', 12) +f2 = mb._nest_function(f1, 'f2', 13) +c1 = mb._nest_class(f0, 'c1', 15) +mock_pyclbr_tree = {'C0': C0, 'f0': f0} # Adjust C0.name, C1.name so tests do not depend on order. browser.transform_children(mock_pyclbr_tree, 'test') # C0(base) @@ -87,12 +87,12 @@ def test_transform_module_children(self): transform = browser.transform_children # Parameter matches tree module. tcl = list(transform(mock_pyclbr_tree, 'test')) - eq(tcl, [f0, C0]) - eq(tcl[0].name, 'f0') - eq(tcl[1].name, 'C0(base)') + eq(tcl, [C0, f0]) + eq(tcl[0].name, 'C0(base)') + eq(tcl[1].name, 'f0') # Check that second call does not change suffix. tcl = list(transform(mock_pyclbr_tree, 'test')) - eq(tcl[1].name, 'C0(base)') + eq(tcl[0].name, 'C0(base)') # Nothing to traverse if parameter name isn't same as tree module. tcl = list(transform(mock_pyclbr_tree, 'different name')) eq(tcl, []) diff --git a/Misc/NEWS.d/next/IDLE/2017-12-25-18-48-50.bpo-32411.vNwDhe.rst b/Misc/NEWS.d/next/IDLE/2017-12-25-18-48-50.bpo-32411.vNwDhe.rst new file mode 100644 index 000000000000..a5522012923a --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-12-25-18-48-50.bpo-32411.vNwDhe.rst @@ -0,0 +1,2 @@ +In browser.py, remove extraneous sorting by line number since dictionary was +created in line number order. [View Less]

1 0

bpo-34303: Micro-optimizations in functools.reduce() (GH-8598)
by Raymond Hettinger June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/e5f6207ba6cb510d9370519ba869296be0… commit: e5f6207ba6cb510d9370519ba869296be01787be branch: master author: Sergey Fedoseev <fedoseev.sergey(a)gmail.com> committer: Raymond Hettinger <rhettinger(a)users.noreply.github.com> date: 2019-06-01T13:32:17-07:00 summary: bpo-34303: Micro-optimizations in functools.reduce() (GH-8598) files: A Misc/NEWS.d/next/Library/2018-08-03-09-47-20.bpo-34303.tOE2HP.rst M Modules/_functoolsmodule.c diff --git a/… [View More]

1 0

bpo-36842: Pass positional only parameters to code_new audit hook (GH-13707)
by Pablo Galindo June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/3b57f50efc16c65df96914ec53bc8d3dc2… commit: 3b57f50efc16c65df96914ec53bc8d3dc28e18b6 branch: master author: Pablo Galindo <Pablogsal(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T21:18:48+01:00 summary: bpo-36842: Pass positional only parameters to code_new audit hook (GH-13707) files: M Doc/c-api/code.rst M Objects/codeobject.c diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 92baa4c7df5c..48428109e6f2 … [View More]

1 0

Fix compiler warnings in the pystrehex module (GH-13730)
by Pablo Galindo June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/938d9a0167f7fa4ed109484d269902021d… commit: 938d9a0167f7fa4ed109484d269902021d274c64 branch: master author: Pablo Galindo <Pablogsal(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T21:02:08+01:00 summary: Fix compiler warnings in the pystrehex module (GH-13730) files: M Python/pystrhex.c diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 695a3c394e9b..5dc7c9613796 100644 --- a/Python/pystrhex.c +++ b/Python/… [View More]

1 0

Clean up and reduce visual clutter in the makeunicode.py script. (GH-7558)
by Stefan Behnel June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/faa2948654d15a859bc4317e00730ff213… commit: faa2948654d15a859bc4317e00730ff213295764 branch: master author: Stefan Behnel <stefan_ml(a)behnel.de> committer: GitHub <noreply(a)github.com> date: 2019-06-01T21:49:03+02:00 summary: Clean up and reduce visual clutter in the makeunicode.py script. (GH-7558) files: M Tools/unicode/makeunicodedata.py diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index … [View More]5418eec588c8..5b9427acd390 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -31,6 +31,7 @@ import zipfile from textwrap import dedent +from functools import partial SCRIPT = sys.argv[0] VERSION = "3.3" @@ -106,11 +107,11 @@ ('2CEB0', '2EBE0'), ] + def maketables(trace=0): print("--- Reading", UNICODE_DATA % "", "...") - version = "" unicode = UnicodeData(UNIDATA_VERSION) print(len(list(filter(None, unicode.table))), "characters") @@ -125,6 +126,7 @@ def maketables(trace=0): makeunicodedata(unicode, trace) makeunicodetype(unicode, trace) + # -------------------------------------------------------------------- # unicode character properties @@ -258,124 +260,125 @@ def makeunicodedata(unicode, trace): print("--- Writing", FILE, "...") - fp = open(FILE, "w") - print("/* this file was generated by %s %s */" % (SCRIPT, VERSION), file=fp) - print(file=fp) - print('#define UNIDATA_VERSION "%s"' % UNIDATA_VERSION, file=fp) - print("/* a list of unique database records */", file=fp) - print("const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = {", file=fp) - for item in table: - print(" {%d, %d, %d, %d, %d, %d}," % item, file=fp) - print("};", file=fp) - print(file=fp) - - print("/* Reindexing of NFC first characters. */", file=fp) - print("#define TOTAL_FIRST",total_first, file=fp) - print("#define TOTAL_LAST",total_last, file=fp) - print("struct reindex{int start;short count,index;};", file=fp) - print("static struct reindex nfc_first[] = {", file=fp) - for start,end in comp_first_ranges: - print(" { %d, %d, %d}," % (start,end-start,comp_first[start]), file=fp) - print(" {0,0,0}", file=fp) - print("};\n", file=fp) - print("static struct reindex nfc_last[] = {", file=fp) - for start,end in comp_last_ranges: - print(" { %d, %d, %d}," % (start,end-start,comp_last[start]), file=fp) - print(" {0,0,0}", file=fp) - print("};\n", file=fp) - - # FIXME: <fl> the following tables could be made static, and - # the support code moved into unicodedatabase.c - - print("/* string literals */", file=fp) - print("const char *_PyUnicode_CategoryNames[] = {", file=fp) - for name in CATEGORY_NAMES: - print(" \"%s\"," % name, file=fp) - print(" NULL", file=fp) - print("};", file=fp) - - print("const char *_PyUnicode_BidirectionalNames[] = {", file=fp) - for name in BIDIRECTIONAL_NAMES: - print(" \"%s\"," % name, file=fp) - print(" NULL", file=fp) - print("};", file=fp) - - print("const char *_PyUnicode_EastAsianWidthNames[] = {", file=fp) - for name in EASTASIANWIDTH_NAMES: - print(" \"%s\"," % name, file=fp) - print(" NULL", file=fp) - print("};", file=fp) - - print("static const char *decomp_prefix[] = {", file=fp) - for name in decomp_prefix: - print(" \"%s\"," % name, file=fp) - print(" NULL", file=fp) - print("};", file=fp) - - # split record index table - index1, index2, shift = splitbins(index, trace) - - print("/* index tables for the database records */", file=fp) - print("#define SHIFT", shift, file=fp) - Array("index1", index1).dump(fp, trace) - Array("index2", index2).dump(fp, trace) - - # split decomposition index table - index1, index2, shift = splitbins(decomp_index, trace) - - print("/* decomposition data */", file=fp) - Array("decomp_data", decomp_data).dump(fp, trace) - - print("/* index tables for the decomposition data */", file=fp) - print("#define DECOMP_SHIFT", shift, file=fp) - Array("decomp_index1", index1).dump(fp, trace) - Array("decomp_index2", index2).dump(fp, trace) - - index, index2, shift = splitbins(comp_data, trace) - print("/* NFC pairs */", file=fp) - print("#define COMP_SHIFT", shift, file=fp) - Array("comp_index", index).dump(fp, trace) - Array("comp_data", index2).dump(fp, trace) - - # Generate delta tables for old versions - for version, table, normalization in unicode.changed: - cversion = version.replace(".","_") - records = [table[0]] - cache = {table[0]:0} - index = [0] * len(table) - for i, record in enumerate(table): - try: - index[i] = cache[record] - except KeyError: - index[i] = cache[record] = len(records) - records.append(record) + with open(FILE, "w") as fp: + fprint = partial(print, file=fp) + + fprint("/* this file was generated by %s %s */" % (SCRIPT, VERSION)) + fprint() + fprint('#define UNIDATA_VERSION "%s"' % UNIDATA_VERSION) + fprint("/* a list of unique database records */") + fprint("const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = {") + for item in table: + fprint(" {%d, %d, %d, %d, %d, %d}," % item) + fprint("};") + fprint() + + fprint("/* Reindexing of NFC first characters. */") + fprint("#define TOTAL_FIRST",total_first) + fprint("#define TOTAL_LAST",total_last) + fprint("struct reindex{int start;short count,index;};") + fprint("static struct reindex nfc_first[] = {") + for start,end in comp_first_ranges: + fprint(" { %d, %d, %d}," % (start,end-start,comp_first[start])) + fprint(" {0,0,0}") + fprint("};\n") + fprint("static struct reindex nfc_last[] = {") + for start,end in comp_last_ranges: + fprint(" { %d, %d, %d}," % (start,end-start,comp_last[start])) + fprint(" {0,0,0}") + fprint("};\n") + + # FIXME: <fl> the following tables could be made static, and + # the support code moved into unicodedatabase.c + + fprint("/* string literals */") + fprint("const char *_PyUnicode_CategoryNames[] = {") + for name in CATEGORY_NAMES: + fprint(" \"%s\"," % name) + fprint(" NULL") + fprint("};") + + fprint("const char *_PyUnicode_BidirectionalNames[] = {") + for name in BIDIRECTIONAL_NAMES: + fprint(" \"%s\"," % name) + fprint(" NULL") + fprint("};") + + fprint("const char *_PyUnicode_EastAsianWidthNames[] = {") + for name in EASTASIANWIDTH_NAMES: + fprint(" \"%s\"," % name) + fprint(" NULL") + fprint("};") + + fprint("static const char *decomp_prefix[] = {") + for name in decomp_prefix: + fprint(" \"%s\"," % name) + fprint(" NULL") + fprint("};") + + # split record index table index1, index2, shift = splitbins(index, trace) - print("static const change_record change_records_%s[] = {" % cversion, file=fp) - for record in records: - print(" { %s }," % ", ".join(map(str,record)), file=fp) - print("};", file=fp) - Array("changes_%s_index" % cversion, index1).dump(fp, trace) - Array("changes_%s_data" % cversion, index2).dump(fp, trace) - print("static const change_record* get_change_%s(Py_UCS4 n)" % cversion, file=fp) - print("{", file=fp) - print(" int index;", file=fp) - print(" if (n >= 0x110000) index = 0;", file=fp) - print(" else {", file=fp) - print(" index = changes_%s_index[n>>%d];" % (cversion, shift), file=fp) - print(" index = changes_%s_data[(index<<%d)+(n & %d)];" % \ - (cversion, shift, ((1<<shift)-1)), file=fp) - print(" }", file=fp) - print(" return change_records_%s+index;" % cversion, file=fp) - print("}\n", file=fp) - print("static Py_UCS4 normalization_%s(Py_UCS4 n)" % cversion, file=fp) - print("{", file=fp) - print(" switch(n) {", file=fp) - for k, v in normalization: - print(" case %s: return 0x%s;" % (hex(k), v), file=fp) - print(" default: return 0;", file=fp) - print(" }\n}\n", file=fp) - - fp.close() + + fprint("/* index tables for the database records */") + fprint("#define SHIFT", shift) + Array("index1", index1).dump(fp, trace) + Array("index2", index2).dump(fp, trace) + + # split decomposition index table + index1, index2, shift = splitbins(decomp_index, trace) + + fprint("/* decomposition data */") + Array("decomp_data", decomp_data).dump(fp, trace) + + fprint("/* index tables for the decomposition data */") + fprint("#define DECOMP_SHIFT", shift) + Array("decomp_index1", index1).dump(fp, trace) + Array("decomp_index2", index2).dump(fp, trace) + + index, index2, shift = splitbins(comp_data, trace) + fprint("/* NFC pairs */") + fprint("#define COMP_SHIFT", shift) + Array("comp_index", index).dump(fp, trace) + Array("comp_data", index2).dump(fp, trace) + + # Generate delta tables for old versions + for version, table, normalization in unicode.changed: + cversion = version.replace(".","_") + records = [table[0]] + cache = {table[0]:0} + index = [0] * len(table) + for i, record in enumerate(table): + try: + index[i] = cache[record] + except KeyError: + index[i] = cache[record] = len(records) + records.append(record) + index1, index2, shift = splitbins(index, trace) + fprint("static const change_record change_records_%s[] = {" % cversion) + for record in records: + fprint(" { %s }," % ", ".join(map(str,record))) + fprint("};") + Array("changes_%s_index" % cversion, index1).dump(fp, trace) + Array("changes_%s_data" % cversion, index2).dump(fp, trace) + fprint("static const change_record* get_change_%s(Py_UCS4 n)" % cversion) + fprint("{") + fprint(" int index;") + fprint(" if (n >= 0x110000) index = 0;") + fprint(" else {") + fprint(" index = changes_%s_index[n>>%d];" % (cversion, shift)) + fprint(" index = changes_%s_data[(index<<%d)+(n & %d)];" % \ + (cversion, shift, ((1<<shift)-1))) + fprint(" }") + fprint(" return change_records_%s+index;" % cversion) + fprint("}\n") + fprint("static Py_UCS4 normalization_%s(Py_UCS4 n)" % cversion) + fprint("{") + fprint(" switch(n) {") + for k, v in normalization: + fprint(" case %s: return 0x%s;" % (hex(k), v)) + fprint(" default: return 0;") + fprint(" }\n}\n") + # -------------------------------------------------------------------- # unicode character type tables @@ -404,7 +407,6 @@ def makeunicodetype(unicode, trace): bidirectional = record[4] properties = record[16] flags = 0 - delta = True if category in ["Lm", "Lt", "Lu", "Ll", "Lo"]: flags |= ALPHA_MASK if "Lowercase" in properties: @@ -505,90 +507,91 @@ def makeunicodetype(unicode, trace): print("--- Writing", FILE, "...") - fp = open(FILE, "w") - print("/* this file was generated by %s %s */" % (SCRIPT, VERSION), file=fp) - print(file=fp) - print("/* a list of unique character type descriptors */", file=fp) - print("const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = {", file=fp) - for item in table: - print(" {%d, %d, %d, %d, %d, %d}," % item, file=fp) - print("};", file=fp) - print(file=fp) - - print("/* extended case mappings */", file=fp) - print(file=fp) - print("const Py_UCS4 _PyUnicode_ExtendedCase[] = {", file=fp) - for c in extra_casing: - print(" %d," % c, file=fp) - print("};", file=fp) - print(file=fp) - - # split decomposition index table - index1, index2, shift = splitbins(index, trace) - - print("/* type indexes */", file=fp) - print("#define SHIFT", shift, file=fp) - Array("index1", index1).dump(fp, trace) - Array("index2", index2).dump(fp, trace) - - # Generate code for _PyUnicode_ToNumeric() - numeric_items = sorted(numeric.items()) - print('/* Returns the numeric value as double for Unicode characters', file=fp) - print(' * having this property, -1.0 otherwise.', file=fp) - print(' */', file=fp) - print('double _PyUnicode_ToNumeric(Py_UCS4 ch)', file=fp) - print('{', file=fp) - print(' switch (ch) {', file=fp) - for value, codepoints in numeric_items: - # Turn text into float literals - parts = value.split('/') - parts = [repr(float(part)) for part in parts] - value = '/'.join(parts) - - codepoints.sort() - for codepoint in codepoints: - print(' case 0x%04X:' % (codepoint,), file=fp) - print(' return (double) %s;' % (value,), file=fp) - print(' }', file=fp) - print(' return -1.0;', file=fp) - print('}', file=fp) - print(file=fp) - - # Generate code for _PyUnicode_IsWhitespace() - print("/* Returns 1 for Unicode characters having the bidirectional", file=fp) - print(" * type 'WS', 'B' or 'S' or the category 'Zs', 0 otherwise.", file=fp) - print(" */", file=fp) - print('int _PyUnicode_IsWhitespace(const Py_UCS4 ch)', file=fp) - print('{', file=fp) - print(' switch (ch) {', file=fp) - - for codepoint in sorted(spaces): - print(' case 0x%04X:' % (codepoint,), file=fp) - print(' return 1;', file=fp) - - print(' }', file=fp) - print(' return 0;', file=fp) - print('}', file=fp) - print(file=fp) - - # Generate code for _PyUnicode_IsLinebreak() - print("/* Returns 1 for Unicode characters having the line break", file=fp) - print(" * property 'BK', 'CR', 'LF' or 'NL' or having bidirectional", file=fp) - print(" * type 'B', 0 otherwise.", file=fp) - print(" */", file=fp) - print('int _PyUnicode_IsLinebreak(const Py_UCS4 ch)', file=fp) - print('{', file=fp) - print(' switch (ch) {', file=fp) - for codepoint in sorted(linebreaks): - print(' case 0x%04X:' % (codepoint,), file=fp) - print(' return 1;', file=fp) - - print(' }', file=fp) - print(' return 0;', file=fp) - print('}', file=fp) - print(file=fp) - - fp.close() + with open(FILE, "w") as fp: + fprint = partial(print, file=fp) + + fprint("/* this file was generated by %s %s */" % (SCRIPT, VERSION)) + fprint() + fprint("/* a list of unique character type descriptors */") + fprint("const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = {") + for item in table: + fprint(" {%d, %d, %d, %d, %d, %d}," % item) + fprint("};") + fprint() + + fprint("/* extended case mappings */") + fprint() + fprint("const Py_UCS4 _PyUnicode_ExtendedCase[] = {") + for c in extra_casing: + fprint(" %d," % c) + fprint("};") + fprint() + + # split decomposition index table + index1, index2, shift = splitbins(index, trace) + + fprint("/* type indexes */") + fprint("#define SHIFT", shift) + Array("index1", index1).dump(fp, trace) + Array("index2", index2).dump(fp, trace) + + # Generate code for _PyUnicode_ToNumeric() + numeric_items = sorted(numeric.items()) + fprint('/* Returns the numeric value as double for Unicode characters') + fprint(' * having this property, -1.0 otherwise.') + fprint(' */') + fprint('double _PyUnicode_ToNumeric(Py_UCS4 ch)') + fprint('{') + fprint(' switch (ch) {') + for value, codepoints in numeric_items: + # Turn text into float literals + parts = value.split('/') + parts = [repr(float(part)) for part in parts] + value = '/'.join(parts) + + codepoints.sort() + for codepoint in codepoints: + fprint(' case 0x%04X:' % (codepoint,)) + fprint(' return (double) %s;' % (value,)) + fprint(' }') + fprint(' return -1.0;') + fprint('}') + fprint() + + # Generate code for _PyUnicode_IsWhitespace() + fprint("/* Returns 1 for Unicode characters having the bidirectional") + fprint(" * type 'WS', 'B' or 'S' or the category 'Zs', 0 otherwise.") + fprint(" */") + fprint('int _PyUnicode_IsWhitespace(const Py_UCS4 ch)') + fprint('{') + fprint(' switch (ch) {') + + for codepoint in sorted(spaces): + fprint(' case 0x%04X:' % (codepoint,)) + fprint(' return 1;') + + fprint(' }') + fprint(' return 0;') + fprint('}') + fprint() + + # Generate code for _PyUnicode_IsLinebreak() + fprint("/* Returns 1 for Unicode characters having the line break") + fprint(" * property 'BK', 'CR', 'LF' or 'NL' or having bidirectional") + fprint(" * type 'B', 0 otherwise.") + fprint(" */") + fprint('int _PyUnicode_IsLinebreak(const Py_UCS4 ch)') + fprint('{') + fprint(' switch (ch) {') + for codepoint in sorted(linebreaks): + fprint(' case 0x%04X:' % (codepoint,)) + fprint(' return 1;') + + fprint(' }') + fprint(' return 0;') + fprint('}') + fprint() + # -------------------------------------------------------------------- # unicode name database @@ -727,63 +730,63 @@ def word_key(a): print("--- Writing", FILE, "...") - fp = open(FILE, "w") - print("/* this file was generated by %s %s */" % (SCRIPT, VERSION), file=fp) - print(file=fp) - print("#define NAME_MAXLEN", 256, file=fp) - print(file=fp) - print("/* lexicon */", file=fp) - Array("lexicon", lexicon).dump(fp, trace) - Array("lexicon_offset", lexicon_offset).dump(fp, trace) - - # split decomposition index table - offset1, offset2, shift = splitbins(phrasebook_offset, trace) - - print("/* code->name phrasebook */", file=fp) - print("#define phrasebook_shift", shift, file=fp) - print("#define phrasebook_short", short, file=fp) - - Array("phrasebook", phrasebook).dump(fp, trace) - Array("phrasebook_offset1", offset1).dump(fp, trace) - Array("phrasebook_offset2", offset2).dump(fp, trace) - - print("/* name->code dictionary */", file=fp) - codehash.dump(fp, trace) - - print(file=fp) - print('static const unsigned int aliases_start = %#x;' % - NAME_ALIASES_START, file=fp) - print('static const unsigned int aliases_end = %#x;' % - (NAME_ALIASES_START + len(unicode.aliases)), file=fp) - - print('static const unsigned int name_aliases[] = {', file=fp) - for name, codepoint in unicode.aliases: - print(' 0x%04X,' % codepoint, file=fp) - print('};', file=fp) - - # In Unicode 6.0.0, the sequences contain at most 4 BMP chars, - # so we are using Py_UCS2 seq[4]. This needs to be updated if longer - # sequences or sequences with non-BMP chars are added. - # unicodedata_lookup should be adapted too. - print(dedent(""" - typedef struct NamedSequence { - int seqlen; - Py_UCS2 seq[4]; - } named_sequence; - """), file=fp) - - print('static const unsigned int named_sequences_start = %#x;' % - NAMED_SEQUENCES_START, file=fp) - print('static const unsigned int named_sequences_end = %#x;' % - (NAMED_SEQUENCES_START + len(unicode.named_sequences)), file=fp) - - print('static const named_sequence named_sequences[] = {', file=fp) - for name, sequence in unicode.named_sequences: - seq_str = ', '.join('0x%04X' % cp for cp in sequence) - print(' {%d, {%s}},' % (len(sequence), seq_str), file=fp) - print('};', file=fp) - - fp.close() + with open(FILE, "w") as fp: + fprint = partial(print, file=fp) + + fprint("/* this file was generated by %s %s */" % (SCRIPT, VERSION)) + fprint() + fprint("#define NAME_MAXLEN", 256) + fprint() + fprint("/* lexicon */") + Array("lexicon", lexicon).dump(fp, trace) + Array("lexicon_offset", lexicon_offset).dump(fp, trace) + + # split decomposition index table + offset1, offset2, shift = splitbins(phrasebook_offset, trace) + + fprint("/* code->name phrasebook */") + fprint("#define phrasebook_shift", shift) + fprint("#define phrasebook_short", short) + + Array("phrasebook", phrasebook).dump(fp, trace) + Array("phrasebook_offset1", offset1).dump(fp, trace) + Array("phrasebook_offset2", offset2).dump(fp, trace) + + fprint("/* name->code dictionary */") + codehash.dump(fp, trace) + + fprint() + fprint('static const unsigned int aliases_start = %#x;' % + NAME_ALIASES_START) + fprint('static const unsigned int aliases_end = %#x;' % + (NAME_ALIASES_START + len(unicode.aliases))) + + fprint('static const unsigned int name_aliases[] = {') + for name, codepoint in unicode.aliases: + fprint(' 0x%04X,' % codepoint) + fprint('};') + + # In Unicode 6.0.0, the sequences contain at most 4 BMP chars, + # so we are using Py_UCS2 seq[4]. This needs to be updated if longer + # sequences or sequences with non-BMP chars are added. + # unicodedata_lookup should be adapted too. + fprint(dedent(""" + typedef struct NamedSequence { + int seqlen; + Py_UCS2 seq[4]; + } named_sequence; + """)) + + fprint('static const unsigned int named_sequences_start = %#x;' % + NAMED_SEQUENCES_START) + fprint('static const unsigned int named_sequences_end = %#x;' % + (NAMED_SEQUENCES_START + len(unicode.named_sequences))) + + fprint('static const named_sequence named_sequences[] = {') + for name, sequence in unicode.named_sequences: + seq_str = ', '.join('0x%04X' % cp for cp in sequence) + fprint(' {%d, {%s}},' % (len(sequence), seq_str)) + fprint('};') def merge_old_version(version, new, old): @@ -882,6 +885,7 @@ class Difference(Exception):pass numeric_changes)), normalization_changes)) + def open_data(template, version): local = template % ('-'+version,) if not os.path.exists(local): @@ -898,6 +902,7 @@ def open_data(template, version): # Unihan.zip return open(local, 'rb') + # -------------------------------------------------------------------- # the following support code is taken from the unidb utilities # Copyright (c) 1999-2000 by Secret Labs AB @@ -1150,6 +1155,7 @@ def uselatin1(self): # restrict character range to ISO Latin 1 self.chars = list(range(256)) + # hash table tools # this is a straight-forward reimplementation of Python's built-in @@ -1165,6 +1171,7 @@ def myhash(s, magic): h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff return h + SIZES = [ (4,3), (8,3), (16,3), (32,5), (64,3), (128,3), (256,29), (512,17), (1024,9), (2048,5), (4096,83), (8192,27), (16384,43), (32768,3), @@ -1172,6 +1179,7 @@ def myhash(s, magic): (2097152,5), (4194304,3), (8388608,33), (16777216,27) ] + class Hash: def __init__(self, name, data, magic): # turn a (key, value) list into a static hash table structure @@ -1202,7 +1210,7 @@ def __init__(self, name, data, magic): if v is None: table[i] = value continue - incr = (h ^ (h >> 3)) & mask; + incr = (h ^ (h >> 3)) & mask if not incr: incr = mask while 1: @@ -1236,6 +1244,7 @@ def dump(self, file, trace): file.write("#define %s_size %d\n" % (self.name, self.size)) file.write("#define %s_poly %d\n" % (self.name, self.poly)) + # stuff to deal with arrays of unsigned integers class Array: @@ -1270,6 +1279,7 @@ def dump(self, file, trace=0): file.write(s.rstrip() + "\n") file.write("};\n\n") + def getsize(data): # return smallest possible integer size for the given array maxdata = max(data) @@ -1280,6 +1290,7 @@ def getsize(data): else: return 4 + def splitbins(t, trace=0): """t, trace=0 -> (t1, t2, shift). Split a table to save space. @@ -1299,8 +1310,8 @@ def splitbins(t, trace=0): def dump(t1, t2, shift, bytes): print("%d+%d bins at shift %d; %d bytes" % ( len(t1), len(t2), shift, bytes), file=sys.stderr) - print("Size of original table:", len(t)*getsize(t), \ - "bytes", file=sys.stderr) + print("Size of original table:", len(t)*getsize(t), "bytes", + file=sys.stderr) n = len(t)-1 # last valid index maxshift = 0 # the most we can shift n and still have something left if n > 0: @@ -1341,5 +1352,6 @@ def dump(t1, t2, shift, bytes): assert t[i] == t2[(t1[i >> shift] << shift) + (i & mask)] return best + if __name__ == "__main__": maketables(1) [View Less]

1 0

bpo-28595: Allow shlex whitespace_split with punctuation_chars (GH-2071)
by Vinay Sajip June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/56624a99a916fd27152d5b23364303acc0… commit: 56624a99a916fd27152d5b23364303acc0d707de branch: master author: Evan <evanunderscore(a)gmail.com> committer: Vinay Sajip <vinay_sajip(a)yahoo.co.uk> date: 2019-06-01T20:09:22+01:00 summary: bpo-28595: Allow shlex whitespace_split with punctuation_chars (GH-2071) files: M Doc/library/shlex.rst M Lib/shlex.py M Lib/test/test_shlex.py diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index … [View More]8c5b0239d1f0..a8421fdb7008 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -225,7 +225,8 @@ variables which either control lexical analysis or can be used for debugging: appear in filename specifications and command line parameters, will also be included in this attribute, and any characters which appear in ``punctuation_chars`` will be removed from ``wordchars`` if they are present - there. + there. If :attr:`whitespace_split` is set to ``True``, this will have no + effect. .. attribute:: shlex.whitespace @@ -258,11 +259,13 @@ variables which either control lexical analysis or can be used for debugging: If ``True``, tokens will only be split in whitespaces. This is useful, for example, for parsing command lines with :class:`~shlex.shlex`, getting - tokens in a similar way to shell arguments. If this attribute is ``True``, - :attr:`punctuation_chars` will have no effect, and splitting will happen - only on whitespaces. When using :attr:`punctuation_chars`, which is - intended to provide parsing closer to that implemented by shells, it is - advisable to leave ``whitespace_split`` as ``False`` (the default value). + tokens in a similar way to shell arguments. When used in combination with + :attr:`punctuation_chars`, tokens will be split on whitespace in addition to + those characters. + + .. versionchanged:: 3.8 + The :attr:`punctuation_chars` attribute was made compatible with the + :attr:`whitespace_split` attribute. .. attribute:: shlex.infile @@ -398,12 +401,15 @@ otherwise. To illustrate, you can see the difference in the following snippet: >>> import shlex >>> text = "a && b; c && d || e; f >'abc'; (def \"ghi\")" - >>> list(shlex.shlex(text)) - ['a', '&', '&', 'b', ';', 'c', '&', '&', 'd', '|', '|', 'e', ';', 'f', '>', - "'abc'", ';', '(', 'def', '"ghi"', ')'] - >>> list(shlex.shlex(text, punctuation_chars=True)) - ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', "'abc'", - ';', '(', 'def', '"ghi"', ')'] + >>> s = shlex.shlex(text, posix=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b;', 'c', '&&', 'd', '||', 'e;', 'f', '>abc;', '(def', 'ghi)'] + >>> s = shlex.shlex(text, posix=True, punctuation_chars=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', 'abc', ';', + '(', 'def', 'ghi', ')'] Of course, tokens will be returned which are not valid for shells, and you'll need to implement your own error checks on the returned tokens. @@ -428,6 +434,11 @@ which characters constitute punctuation. For example:: >>> list(s) ['~/a', '&&', 'b-c', '--color=auto', '||', 'd', '*.py?'] + However, to match the shell as closely as possible, it is recommended to + always use ``posix`` and :attr:`~shlex.whitespace_split` when using + :attr:`~shlex.punctuation_chars`, which will negate + :attr:`~shlex.wordchars` entirely. + For best effect, ``punctuation_chars`` should be set in conjunction with ``posix=True``. (Note that ``posix=False`` is the default for :class:`~shlex.shlex`.) diff --git a/Lib/shlex.py b/Lib/shlex.py index fb1130d4eac2..edea07794860 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -246,7 +246,8 @@ def read_token(self): escapedstate = 'a' self.state = nextchar elif (nextchar in self.wordchars or nextchar in self.quotes - or self.whitespace_split): + or (self.whitespace_split and + nextchar not in self.punctuation_chars)): self.token += nextchar else: if self.punctuation_chars: diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index a432610d3af4..376c5e88d381 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -1,4 +1,5 @@ import io +import itertools import shlex import string import unittest @@ -183,10 +184,12 @@ def testSyntaxSplitAmpersandAndPipe(self): src = ['echo hi %s echo bye' % delimiter, 'echo hi%secho bye' % delimiter] ref = ['echo', 'hi', delimiter, 'echo', 'bye'] - for ss in src: + for ss, ws in itertools.product(src, (False, True)): s = shlex.shlex(ss, punctuation_chars=True) + s.whitespace_split = ws result = list(s) - self.assertEqual(ref, result, "While splitting '%s'" % ss) + self.assertEqual(ref, result, + "While splitting '%s' [ws=%s]" % (ss, ws)) def testSyntaxSplitSemicolon(self): """Test handling of syntax splitting of ;""" @@ -197,10 +200,12 @@ def testSyntaxSplitSemicolon(self): 'echo hi%s echo bye' % delimiter, 'echo hi%secho bye' % delimiter] ref = ['echo', 'hi', delimiter, 'echo', 'bye'] - for ss in src: + for ss, ws in itertools.product(src, (False, True)): s = shlex.shlex(ss, punctuation_chars=True) + s.whitespace_split = ws result = list(s) - self.assertEqual(ref, result, "While splitting '%s'" % ss) + self.assertEqual(ref, result, + "While splitting '%s' [ws=%s]" % (ss, ws)) def testSyntaxSplitRedirect(self): """Test handling of syntax splitting of >""" @@ -211,10 +216,11 @@ def testSyntaxSplitRedirect(self): 'echo hi%s out' % delimiter, 'echo hi%sout' % delimiter] ref = ['echo', 'hi', delimiter, 'out'] - for ss in src: + for ss, ws in itertools.product(src, (False, True)): s = shlex.shlex(ss, punctuation_chars=True) result = list(s) - self.assertEqual(ref, result, "While splitting '%s'" % ss) + self.assertEqual(ref, result, + "While splitting '%s' [ws=%s]" % (ss, ws)) def testSyntaxSplitParen(self): """Test handling of syntax splitting of ()""" @@ -222,18 +228,25 @@ def testSyntaxSplitParen(self): src = ['( echo hi )', '(echo hi)'] ref = ['(', 'echo', 'hi', ')'] - for ss in src: + for ss, ws in itertools.product(src, (False, True)): s = shlex.shlex(ss, punctuation_chars=True) + s.whitespace_split = ws result = list(s) - self.assertEqual(ref, result, "While splitting '%s'" % ss) + self.assertEqual(ref, result, + "While splitting '%s' [ws=%s]" % (ss, ws)) def testSyntaxSplitCustom(self): """Test handling of syntax splitting with custom chars""" + ss = "~/a&&b-c --color=auto||d *.py?" ref = ['~/a', '&', '&', 'b-c', '--color=auto', '||', 'd', '*.py?'] - ss = "~/a && b-c --color=auto || d *.py?" s = shlex.shlex(ss, punctuation_chars="|") result = list(s) - self.assertEqual(ref, result, "While splitting '%s'" % ss) + self.assertEqual(ref, result, "While splitting '%s' [ws=False]" % ss) + ref = ['~/a&&b-c', '--color=auto', '||', 'd', '*.py?'] + s = shlex.shlex(ss, punctuation_chars="|") + s.whitespace_split = True + result = list(s) + self.assertEqual(ref, result, "While splitting '%s' [ws=True]" % ss) def testTokenTypes(self): """Test that tokens are split with types as expected.""" @@ -293,6 +306,19 @@ def testEmptyStringHandling(self): s = shlex.shlex("'')abc", punctuation_chars=True) self.assertEqual(list(s), expected) + def testUnicodeHandling(self): + """Test punctuation_chars and whitespace_split handle unicode.""" + ss = "\u2119\u01b4\u2602\u210c\u00f8\u1f24" + # Should be parsed as one complete token (whitespace_split=True). + ref = ['\u2119\u01b4\u2602\u210c\u00f8\u1f24'] + s = shlex.shlex(ss, punctuation_chars=True) + s.whitespace_split = True + self.assertEqual(list(s), ref) + # Without whitespace_split, uses wordchars and splits on all. + ref = ['\u2119', '\u01b4', '\u2602', '\u210c', '\u00f8', '\u1f24'] + s = shlex.shlex(ss, punctuation_chars=True) + self.assertEqual(list(s), ref) + def testQuote(self): safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./' unicode_sample = '\xe9\xe0\xdf' # e + acute accent, a + grave, sharp s [View Less]

1 0

bpo-35431: Refactor math.comb() implementation. (GH-13725)
by Serhiy Storchaka June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/2b843ac0ae745026ce39514573c5d07513… commit: 2b843ac0ae745026ce39514573c5d075137bef65 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T22:09:02+03:00 summary: bpo-35431: Refactor math.comb() implementation. (GH-13725) * Fixed some bugs. * Added support for index-likes objects. * Improved error messages. * Cleaned up and optimized the code. * Added more tests. files: M … [View More]Doc/library/math.rst M Lib/test/test_math.py M Modules/clinic/mathmodule.c.h M Modules/mathmodule.c diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 5243970df806..206b06edd2a2 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -238,11 +238,11 @@ Number-theoretic and representation functions and without order. Also called the binomial coefficient. It is mathematically equal to the expression - ``n! / (k! (n - k)!)``. It is equivalent to the coefficient of k-th term in + ``n! / (k! (n - k)!)``. It is equivalent to the coefficient of the *k*-th term in the polynomial expansion of the expression ``(1 + x) ** n``. Raises :exc:`TypeError` if the arguments not integers. - Raises :exc:`ValueError` if the arguments are negative or if k > n. + Raises :exc:`ValueError` if the arguments are negative or if *k* > *n*. .. versionadded:: 3.8 diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 9da7f7c4e6e2..e27092eefd6e 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -1893,9 +1893,11 @@ def testComb(self): # Raises TypeError if any argument is non-integer or argument count is # not 2 self.assertRaises(TypeError, comb, 10, 1.0) + self.assertRaises(TypeError, comb, 10, decimal.Decimal(1.0)) self.assertRaises(TypeError, comb, 10, "1") - self.assertRaises(TypeError, comb, "10", 1) self.assertRaises(TypeError, comb, 10.0, 1) + self.assertRaises(TypeError, comb, decimal.Decimal(10.0), 1) + self.assertRaises(TypeError, comb, "10", 1) self.assertRaises(TypeError, comb, 10) self.assertRaises(TypeError, comb, 10, 1, 3) @@ -1903,15 +1905,28 @@ def testComb(self): # Raises Value error if not k or n are negative numbers self.assertRaises(ValueError, comb, -1, 1) - self.assertRaises(ValueError, comb, -10*10, 1) + self.assertRaises(ValueError, comb, -2**1000, 1) self.assertRaises(ValueError, comb, 1, -1) - self.assertRaises(ValueError, comb, 1, -10*10) + self.assertRaises(ValueError, comb, 1, -2**1000) # Raises value error if k is greater than n - self.assertRaises(ValueError, comb, 1, 10**10) - self.assertRaises(ValueError, comb, 0, 1) - - + self.assertRaises(ValueError, comb, 1, 2) + self.assertRaises(ValueError, comb, 1, 2**1000) + + n = 2**1000 + self.assertEqual(comb(n, 0), 1) + self.assertEqual(comb(n, 1), n) + self.assertEqual(comb(n, 2), n * (n-1) // 2) + self.assertEqual(comb(n, n), 1) + self.assertEqual(comb(n, n-1), n) + self.assertEqual(comb(n, n-2), n * (n-1) // 2) + self.assertRaises((OverflowError, MemoryError), comb, n, n//2) + + for n, k in (True, True), (True, False), (False, False): + self.assertEqual(comb(n, k), 1) + self.assertIs(type(comb(n, k)), int) + self.assertEqual(comb(MyIndexable(5), MyIndexable(2)), 10) + self.assertIs(type(comb(MyIndexable(5), MyIndexable(2))), int) def test_main(): diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index cba791e2098f..92ec4bec9bf1 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -639,10 +639,10 @@ math_prod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k } PyDoc_STRVAR(math_comb__doc__, -"comb($module, /, n, k)\n" +"comb($module, n, k, /)\n" "--\n" "\n" -"Number of ways to choose *k* items from *n* items without repetition and without order.\n" +"Number of ways to choose k items from n items without repetition and without order.\n" "\n" "Also called the binomial coefficient. It is mathematically equal to the expression\n" "n! / (k! * (n - k)!). It is equivalent to the coefficient of k-th term in\n" @@ -652,38 +652,26 @@ PyDoc_STRVAR(math_comb__doc__, "Raises ValueError if the arguments are negative or if k > n."); #define MATH_COMB_METHODDEF \ - {"comb", (PyCFunction)(void(*)(void))math_comb, METH_FASTCALL|METH_KEYWORDS, math_comb__doc__}, + {"comb", (PyCFunction)(void(*)(void))math_comb, METH_FASTCALL, math_comb__doc__}, static PyObject * math_comb_impl(PyObject *module, PyObject *n, PyObject *k); static PyObject * -math_comb(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +math_comb(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - static const char * const _keywords[] = {"n", "k", NULL}; - static _PyArg_Parser _parser = {NULL, _keywords, "comb", 0}; - PyObject *argsbuf[2]; PyObject *n; PyObject *k; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyLong_Check(args[0])) { - _PyArg_BadArgument("comb", 1, "int", args[0]); + if (!_PyArg_CheckPositional("comb", nargs, 2, 2)) { goto exit; } n = args[0]; - if (!PyLong_Check(args[1])) { - _PyArg_BadArgument("comb", 2, "int", args[1]); - goto exit; - } k = args[1]; return_value = math_comb_impl(module, n, k); exit: return return_value; } -/*[clinic end generated code: output=00aa76356759617a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6709521e5e1d90ec input=a9049054013a1b77]*/ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 007a8801429c..bea4607b9be1 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -3001,10 +3001,11 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) /*[clinic input] math.comb - n: object(subclass_of='&PyLong_Type') - k: object(subclass_of='&PyLong_Type') + n: object + k: object + / -Number of ways to choose *k* items from *n* items without repetition and without order. +Number of ways to choose k items from n items without repetition and without order. Also called the binomial coefficient. It is mathematically equal to the expression n! / (k! * (n - k)!). It is equivalent to the coefficient of k-th term in @@ -3017,103 +3018,109 @@ Raises ValueError if the arguments are negative or if k > n. static PyObject * math_comb_impl(PyObject *module, PyObject *n, PyObject *k) -/*[clinic end generated code: output=bd2cec8d854f3493 input=565f340f98efb5b5]*/ +/*[clinic end generated code: output=bd2cec8d854f3493 input=2f336ac9ec8242f9]*/ { - PyObject *val = NULL, - *temp_obj1 = NULL, - *temp_obj2 = NULL, - *dump_var = NULL; + PyObject *result = NULL, *factor = NULL, *temp; int overflow, cmp; - long long i, terms; + long long i, factors; - cmp = PyObject_RichCompareBool(n, k, Py_LT); - if (cmp < 0) { - goto fail_comb; + n = PyNumber_Index(n); + if (n == NULL) { + return NULL; } - else if (cmp > 0) { - PyErr_Format(PyExc_ValueError, - "n must be an integer greater than or equal to k"); - goto fail_comb; + k = PyNumber_Index(k); + if (k == NULL) { + Py_DECREF(n); + return NULL; } - /* b = min(b, a - b) */ - dump_var = PyNumber_Subtract(n, k); - if (dump_var == NULL) { - goto fail_comb; + if (Py_SIZE(n) < 0) { + PyErr_SetString(PyExc_ValueError, + "n must be a non-negative integer"); + goto error; } - cmp = PyObject_RichCompareBool(k, dump_var, Py_GT); - if (cmp < 0) { - goto fail_comb; + /* k = min(k, n - k) */ + temp = PyNumber_Subtract(n, k); + if (temp == NULL) { + goto error; } - else if (cmp > 0) { - k = dump_var; - dump_var = NULL; + if (Py_SIZE(temp) < 0) { + Py_DECREF(temp); + PyErr_SetString(PyExc_ValueError, + "k must be an integer less than or equal to n"); + goto error; + } + cmp = PyObject_RichCompareBool(k, temp, Py_GT); + if (cmp > 0) { + Py_SETREF(k, temp); } else { - Py_DECREF(dump_var); - dump_var = NULL; + Py_DECREF(temp); + if (cmp < 0) { + goto error; + } } - terms = PyLong_AsLongLongAndOverflow(k, &overflow); - if (terms < 0 && PyErr_Occurred()) { - goto fail_comb; - } - else if (overflow > 0) { + factors = PyLong_AsLongLongAndOverflow(k, &overflow); + if (overflow > 0) { PyErr_Format(PyExc_OverflowError, - "minimum(n - k, k) must not exceed %lld", + "min(n - k, k) must not exceed %lld", LLONG_MAX); - goto fail_comb; + goto error; } - else if (overflow < 0 || terms < 0) { - PyErr_Format(PyExc_ValueError, - "k must be a positive integer"); - goto fail_comb; + else if (overflow < 0 || factors < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "k must be a non-negative integer"); + } + goto error; } - if (terms == 0) { - return PyNumber_Long(_PyLong_One); + if (factors == 0) { + result = PyLong_FromLong(1); + goto done; } - val = PyNumber_Long(n); - for (i = 1; i < terms; ++i) { - temp_obj1 = PyLong_FromSsize_t(i); - if (temp_obj1 == NULL) { - goto fail_comb; - } - temp_obj2 = PyNumber_Subtract(n, temp_obj1); - if (temp_obj2 == NULL) { - goto fail_comb; + result = n; + Py_INCREF(result); + if (factors == 1) { + goto done; + } + + factor = n; + Py_INCREF(factor); + for (i = 1; i < factors; ++i) { + Py_SETREF(factor, PyNumber_Subtract(factor, _PyLong_One)); + if (factor == NULL) { + goto error; } - dump_var = val; - val = PyNumber_Multiply(val, temp_obj2); - if (val == NULL) { - goto fail_comb; + Py_SETREF(result, PyNumber_Multiply(result, factor)); + if (result == NULL) { + goto error; } - Py_DECREF(dump_var); - dump_var = NULL; - Py_DECREF(temp_obj2); - temp_obj2 = PyLong_FromUnsignedLongLong((unsigned long long)(i + 1)); - if (temp_obj2 == NULL) { - goto fail_comb; + + temp = PyLong_FromUnsignedLongLong((unsigned long long)i + 1); + if (temp == NULL) { + goto error; } - dump_var = val; - val = PyNumber_FloorDivide(val, temp_obj2); - if (val == NULL) { - goto fail_comb; + Py_SETREF(result, PyNumber_FloorDivide(result, temp)); + Py_DECREF(temp); + if (result == NULL) { + goto error; } - Py_DECREF(dump_var); - Py_DECREF(temp_obj1); - Py_DECREF(temp_obj2); } + Py_DECREF(factor); - return val; - -fail_comb: - Py_XDECREF(val); - Py_XDECREF(dump_var); - Py_XDECREF(temp_obj1); - Py_XDECREF(temp_obj2); +done: + Py_DECREF(n); + Py_DECREF(k); + return result; +error: + Py_XDECREF(factor); + Py_XDECREF(result); + Py_DECREF(n); + Py_DECREF(k); return NULL; } [View Less]

1 0

Improve exception message for str.format (GH-12675)
by Raymond Hettinger June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/9843bc110dc4241ba7cb05f3d3ef74ac6c… commit: 9843bc110dc4241ba7cb05f3d3ef74ac6c77caf2 branch: master author: Francisco Couzo <franciscouzo(a)gmail.com> committer: Raymond Hettinger <rhettinger(a)users.noreply.github.com> date: 2019-06-01T10:14:00-07:00 summary: Improve exception message for str.format (GH-12675) files: M Objects/stringlib/unicode_format.h diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h … [View More]

1 0

bpo-37122: Make co->co_argcount represent the total number of positonal arguments in the code object (GH-13726)
by Pablo Galindo June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/cd74e66a8c420be675fd2fbf3fe708ac02… commit: cd74e66a8c420be675fd2fbf3fe708ac02ee9f21 branch: master author: Pablo Galindo <Pablogsal(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T18:08:04+01:00 summary: bpo-37122: Make co->co_argcount represent the total number of positonal arguments in the code object (GH-13726) files: A Misc/NEWS.d/next/Core and Builtins/2019-06-01-16-53-41.bpo-37122.dZ3-NY.rst M Doc/c-api/code.… [View More]rst M Doc/reference/datamodel.rst M Doc/whatsnew/3.8.rst M Lib/inspect.py M Lib/pdb.py M Lib/test/test_code.py M Lib/test/test_dis.py M Lib/test/test_positional_only_arg.py M Objects/call.c M Objects/codeobject.c M Objects/typeobject.c M Python/ceval.c M Python/compile.c diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 7aa91ee84d2e..92baa4c7df5c 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -42,6 +42,8 @@ bound into a function. .. versionchanged:: 3.8 An extra parameter is required (*posonlyargcount*) to support :PEP:`570`. + The first parameter (*argcount*) now represents the total number of positional arguments, + including positional-only. .. audit-event:: code.__new__ "code filename name argcount kwonlyargcount nlocals stacksize flags" diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index c566dfdf856d..44017d8a55df 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -907,24 +907,26 @@ Internal types single: co_freevars (code object attribute) Special read-only attributes: :attr:`co_name` gives the function name; - :attr:`co_argcount` is the number of positional arguments (including arguments - with default values); :attr:`co_posonlyargcount` is the number of - positional-only arguments (including arguments with default values); - :attr:`co_kwonlyargcount` is the number of keyword-only arguments (including - arguments with default values); :attr:`co_nlocals` is the number of local - variables used by the function (including arguments); :attr:`co_varnames` is a - tuple containing the names of the local variables (starting with the argument - names); :attr:`co_cellvars` is a tuple containing the names of local variables + :attr:`co_argcount` is the total number of positional arguments + (including positional-only arguments and arguments with default values); + :attr:`co_posonlyargcount` is the number of positional-only arguments + (including arguments with default values); :attr:`co_kwonlyargcount` is + the number of keyword-only arguments (including arguments with default + values); :attr:`co_nlocals` is the number of local variables used by the + function (including arguments); :attr:`co_varnames` is a tuple containing + the names of the local variables (starting with the argument names); + :attr:`co_cellvars` is a tuple containing the names of local variables that are referenced by nested functions; :attr:`co_freevars` is a tuple - containing the names of free variables; :attr:`co_code` is a string representing - the sequence of bytecode instructions; :attr:`co_consts` is a tuple containing - the literals used by the bytecode; :attr:`co_names` is a tuple containing the - names used by the bytecode; :attr:`co_filename` is the filename from which the - code was compiled; :attr:`co_firstlineno` is the first line number of the - function; :attr:`co_lnotab` is a string encoding the mapping from bytecode - offsets to line numbers (for details see the source code of the interpreter); - :attr:`co_stacksize` is the required stack size (including local variables); - :attr:`co_flags` is an integer encoding a number of flags for the interpreter. + containing the names of free variables; :attr:`co_code` is a string + representing the sequence of bytecode instructions; :attr:`co_consts` is + a tuple containing the literals used by the bytecode; :attr:`co_names` is + a tuple containing the names used by the bytecode; :attr:`co_filename` is + the filename from which the code was compiled; :attr:`co_firstlineno` is + the first line number of the function; :attr:`co_lnotab` is a string + encoding the mapping from bytecode offsets to line numbers (for details + see the source code of the interpreter); :attr:`co_stacksize` is the + required stack size (including local variables); :attr:`co_flags` is an + integer encoding a number of flags for the interpreter. .. index:: object: generator diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 2d5291102d2a..4c5a9bb0cdb9 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -1177,8 +1177,10 @@ Changes in the Python API * :class:`types.CodeType` has a new parameter in the second position of the constructor (*posonlyargcount*) to support positional-only arguments defined - in :pep:`570`. A new ``replace()`` method of :class:`types.CodeType` can be - used to make the code future-proof. + in :pep:`570`. The first argument (*argcount*) now represents the total + number of positional arguments (including positional-only arguments). A new + ``replace()`` method of :class:`types.CodeType` can be used to make the code + future-proof. Changes in the C API diff --git a/Lib/inspect.py b/Lib/inspect.py index ca81a24f0641..91d209dc64bc 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1037,15 +1037,11 @@ def getargs(co): names = co.co_varnames nargs = co.co_argcount - nposonlyargs = co.co_posonlyargcount nkwargs = co.co_kwonlyargcount - nposargs = nargs + nposonlyargs - posonlyargs = list(names[:nposonlyargs]) - args = list(names[nposonlyargs:nposonlyargs+nargs]) - kwonlyargs = list(names[nposargs:nposargs+nkwargs]) + args = list(names[:nargs]) + kwonlyargs = list(names[nargs:nargs+nkwargs]) step = 0 - nargs += nposonlyargs nargs += nkwargs varargs = None if co.co_flags & CO_VARARGS: @@ -1054,7 +1050,7 @@ def getargs(co): varkw = None if co.co_flags & CO_VARKEYWORDS: varkw = co.co_varnames[nargs] - return Arguments(posonlyargs + args + kwonlyargs, varargs, varkw) + return Arguments(args + kwonlyargs, varargs, varkw) ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') @@ -2136,11 +2132,9 @@ def _signature_from_function(cls, func, skip_bound_arg=True): pos_count = func_code.co_argcount arg_names = func_code.co_varnames posonly_count = func_code.co_posonlyargcount - positional_count = posonly_count + pos_count - positional_only = tuple(arg_names[:posonly_count]) - positional = tuple(arg_names[posonly_count:positional_count]) + positional = arg_names[:pos_count] keyword_only_count = func_code.co_kwonlyargcount - keyword_only = arg_names[positional_count:(positional_count + keyword_only_count)] + keyword_only = arg_names[pos_count:pos_count + keyword_only_count] annotations = func.__annotations__ defaults = func.__defaults__ kwdefaults = func.__kwdefaults__ @@ -2152,13 +2146,11 @@ def _signature_from_function(cls, func, skip_bound_arg=True): parameters = [] - non_default_count = positional_count - pos_default_count - all_positional = positional_only + positional - + non_default_count = pos_count - pos_default_count posonly_left = posonly_count # Non-keyword-only parameters w/o defaults. - for name in all_positional[:non_default_count]: + for name in positional[:non_default_count]: kind = _POSITIONAL_ONLY if posonly_left else _POSITIONAL_OR_KEYWORD annotation = annotations.get(name, _empty) parameters.append(Parameter(name, annotation=annotation, @@ -2167,7 +2159,7 @@ def _signature_from_function(cls, func, skip_bound_arg=True): posonly_left -= 1 # ... w/ defaults. - for offset, name in enumerate(all_positional[non_default_count:]): + for offset, name in enumerate(positional[non_default_count:]): kind = _POSITIONAL_ONLY if posonly_left else _POSITIONAL_OR_KEYWORD annotation = annotations.get(name, _empty) parameters.append(Parameter(name, annotation=annotation, @@ -2178,7 +2170,7 @@ def _signature_from_function(cls, func, skip_bound_arg=True): # *args if func_code.co_flags & CO_VARARGS: - name = arg_names[positional_count + keyword_only_count] + name = arg_names[pos_count + keyword_only_count] annotation = annotations.get(name, _empty) parameters.append(Parameter(name, annotation=annotation, kind=_VAR_POSITIONAL)) @@ -2195,7 +2187,7 @@ def _signature_from_function(cls, func, skip_bound_arg=True): default=default)) # **kwargs if func_code.co_flags & CO_VARKEYWORDS: - index = positional_count + keyword_only_count + index = pos_count + keyword_only_count if func_code.co_flags & CO_VARARGS: index += 1 diff --git a/Lib/pdb.py b/Lib/pdb.py index 13068ce27bd1..0e7609e43d4e 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1132,7 +1132,7 @@ def do_args(self, arg): """ co = self.curframe.f_code dict = self.curframe_locals - n = co.co_argcount + co.co_posonlyargcount + co.co_kwonlyargcount + n = co.co_argcount + co.co_kwonlyargcount if co.co_flags & inspect.CO_VARARGS: n = n+1 if co.co_flags & inspect.CO_VARKEYWORDS: n = n+1 for i in range(n): diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 91008c04f7fd..0d80af44d9f1 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -112,7 +112,7 @@ >>> dump(posonly_args.__code__) name: posonly_args -argcount: 1 +argcount: 3 posonlyargcount: 2 kwonlyargcount: 0 names: () diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 1561021f5bc7..652af45d55ad 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -640,7 +640,7 @@ def f(c=c): code_info_tricky = """\ Name: tricky Filename: (.*) -Argument count: 3 +Argument count: 5 Positional-only arguments: 2 Kw-only arguments: 3 Number of locals: 10 diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index 0aaad84cb3bf..59b0b8fb5562 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -100,14 +100,14 @@ def test_pos_only_definition(self): def f(a, b, c, /, d, e=1, *, f, g=2): pass - self.assertEqual(2, f.__code__.co_argcount) # 2 "standard args" + self.assertEqual(5, f.__code__.co_argcount) # 3 posonly + 2 "standard args" self.assertEqual(3, f.__code__.co_posonlyargcount) self.assertEqual((1,), f.__defaults__) def f(a, b, c=1, /, d=2, e=3, *, f, g=4): pass - self.assertEqual(2, f.__code__.co_argcount) # 2 "standard args" + self.assertEqual(5, f.__code__.co_argcount) # 3 posonly + 2 "standard args" self.assertEqual(3, f.__code__.co_posonlyargcount) self.assertEqual((1, 2, 3), f.__defaults__) diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-06-01-16-53-41.bpo-37122.dZ3-NY.rst b/Misc/NEWS.d/next/Core and Builtins/2019-06-01-16-53-41.bpo-37122.dZ3-NY.rst new file mode 100644 index 000000000000..b9584b50e61d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-06-01-16-53-41.bpo-37122.dZ3-NY.rst @@ -0,0 +1,5 @@ +Make the *co_argcount* attribute of code objects represent the total number +of positional arguments (including positional-only arguments). The value of +*co_posonlyargcount* can be used to distinguish which arguments are +positional only, and the difference (*co_argcount* - *co_posonlyargcount*) +is the number of positional-or-keyword arguments. Patch by Pablo Galindo. diff --git a/Objects/call.c b/Objects/call.c index acd1f26dcbb5..c0d14567e430 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -308,11 +308,11 @@ _PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs (co->co_flags & ~PyCF_MASK) == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { /* Fast paths */ - if (argdefs == NULL && co->co_argcount + co->co_posonlyargcount == nargs) { + if (argdefs == NULL && co->co_argcount == nargs) { return function_code_fastcall(co, args, nargs, globals); } else if (nargs == 0 && argdefs != NULL - && co->co_argcount + co->co_posonlyargcount == PyTuple_GET_SIZE(argdefs)) { + && co->co_argcount == PyTuple_GET_SIZE(argdefs)) { /* function called with no arguments, but all parameters have a default value: use default values as arguments .*/ args = _PyTuple_ITEMS(argdefs); @@ -396,11 +396,11 @@ _PyFunction_Vectorcall(PyObject *func, PyObject* const* stack, if (co->co_kwonlyargcount == 0 && nkwargs == 0 && (co->co_flags & ~PyCF_MASK) == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - if (argdefs == NULL && co->co_argcount + co->co_posonlyargcount== nargs) { + if (argdefs == NULL && co->co_argcount == nargs) { return function_code_fastcall(co, stack, nargs, globals); } else if (nargs == 0 && argdefs != NULL - && co->co_argcount + co->co_posonlyargcount == PyTuple_GET_SIZE(argdefs)) { + && co->co_argcount == PyTuple_GET_SIZE(argdefs)) { /* function called with no arguments, but all parameters have a default value: use default values as arguments .*/ stack = _PyTuple_ITEMS(argdefs); diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 886ce4194438..bf68e54f42ec 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -114,8 +114,9 @@ PyCode_New(int argcount, int posonlyargcount, int kwonlyargcount, Py_ssize_t i, n_cellvars, n_varnames, total_args; /* Check argument types */ - if (argcount < 0 || posonlyargcount < 0 || kwonlyargcount < 0 || - nlocals < 0 || stacksize < 0 || flags < 0 || + if (argcount < posonlyargcount || posonlyargcount < 0 || + kwonlyargcount < 0 || nlocals < 0 || + stacksize < 0 || flags < 0 || code == NULL || !PyBytes_Check(code) || consts == NULL || !PyTuple_Check(consts) || names == NULL || !PyTuple_Check(names) || @@ -152,11 +153,9 @@ PyCode_New(int argcount, int posonlyargcount, int kwonlyargcount, } n_varnames = PyTuple_GET_SIZE(varnames); - if (posonlyargcount + argcount <= n_varnames - && kwonlyargcount <= n_varnames) { + if (argcount <= n_varnames && kwonlyargcount <= n_varnames) { /* Never overflows. */ - total_args = (Py_ssize_t)posonlyargcount + (Py_ssize_t)argcount - + (Py_ssize_t)kwonlyargcount + + total_args = (Py_ssize_t)argcount + (Py_ssize_t)kwonlyargcount + ((flags & CO_VARARGS) != 0) + ((flags & CO_VARKEYWORDS) != 0); } else { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index da249b569ad2..b6d925c1442e 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7807,7 +7807,7 @@ super_init(PyObject *self, PyObject *args, PyObject *kwds) "super(): no code object"); return -1; } - if (co->co_posonlyargcount + co->co_argcount == 0) { + if (co->co_argcount == 0) { PyErr_SetString(PyExc_RuntimeError, "super(): no arguments"); return -1; diff --git a/Python/ceval.c b/Python/ceval.c index f9ff4e09f17e..d9a71e942153 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3757,10 +3757,10 @@ missing_arguments(PyThreadState *tstate, PyCodeObject *co, return; if (positional) { start = 0; - end = co->co_posonlyargcount + co->co_argcount - defcount; + end = co->co_argcount - defcount; } else { - start = co->co_posonlyargcount + co->co_argcount; + start = co->co_argcount; end = start + co->co_kwonlyargcount; } for (i = start; i < end; i++) { @@ -3788,25 +3788,23 @@ too_many_positional(PyThreadState *tstate, PyCodeObject *co, Py_ssize_t kwonly_given = 0; Py_ssize_t i; PyObject *sig, *kwonly_sig; - Py_ssize_t co_posonlyargcount = co->co_posonlyargcount; Py_ssize_t co_argcount = co->co_argcount; - Py_ssize_t total_positional = co_argcount + co_posonlyargcount; assert((co->co_flags & CO_VARARGS) == 0); /* Count missing keyword-only args. */ - for (i = total_positional; i < total_positional + co->co_kwonlyargcount; i++) { + for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) { if (GETLOCAL(i) != NULL) { kwonly_given++; } } if (defcount) { - Py_ssize_t atleast = total_positional - defcount; + Py_ssize_t atleast = co_argcount - defcount; plural = 1; - sig = PyUnicode_FromFormat("from %zd to %zd", atleast, total_positional); + sig = PyUnicode_FromFormat("from %zd to %zd", atleast, co_argcount); } else { - plural = (total_positional != 1); - sig = PyUnicode_FromFormat("%zd", total_positional); + plural = (co_argcount != 1); + sig = PyUnicode_FromFormat("%zd", co_argcount); } if (sig == NULL) return; @@ -3917,7 +3915,7 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, PyObject *retval = NULL; PyObject **fastlocals, **freevars; PyObject *x, *u; - const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount + co->co_posonlyargcount; + const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; Py_ssize_t i, j, n; PyObject *kwdict; @@ -3953,9 +3951,9 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, kwdict = NULL; } - /* Copy positional only arguments into local variables */ - if (argcount > co->co_argcount + co->co_posonlyargcount) { - n = co->co_posonlyargcount; + /* Copy all positional arguments into local variables */ + if (argcount > co->co_argcount) { + n = co->co_argcount; } else { n = argcount; @@ -3966,20 +3964,6 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, SETLOCAL(j, x); } - - /* Copy positional arguments into local variables */ - if (argcount > co->co_argcount + co->co_posonlyargcount) { - n += co->co_argcount; - } - else { - n = argcount; - } - for (i = j; i < n; i++) { - x = args[i]; - Py_INCREF(x); - SETLOCAL(i, x); - } - /* Pack other positional arguments into the *args argument */ if (co->co_flags & CO_VARARGS) { u = _PyTuple_FromArray(args + n, argcount - n); @@ -4059,14 +4043,14 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, } /* Check the number of positional arguments */ - if ((argcount > co->co_argcount + co->co_posonlyargcount) && !(co->co_flags & CO_VARARGS)) { + if ((argcount > co->co_argcount) && !(co->co_flags & CO_VARARGS)) { too_many_positional(tstate, co, argcount, defcount, fastlocals); goto fail; } /* Add missing positional arguments (copy default values from defs) */ - if (argcount < co->co_posonlyargcount + co->co_argcount) { - Py_ssize_t m = co->co_posonlyargcount + co->co_argcount - defcount; + if (argcount < co->co_argcount) { + Py_ssize_t m = co->co_argcount - defcount; Py_ssize_t missing = 0; for (i = argcount; i < m; i++) { if (GETLOCAL(i) == NULL) { @@ -4093,7 +4077,7 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, /* Add missing keyword arguments (copy default values from kwdefs) */ if (co->co_kwonlyargcount > 0) { Py_ssize_t missing = 0; - for (i = co->co_posonlyargcount + co->co_argcount; i < total_args; i++) { + for (i = co->co_argcount; i < total_args; i++) { PyObject *name; if (GETLOCAL(i) != NULL) continue; diff --git a/Python/compile.c b/Python/compile.c index f6ec929b3ca4..9e4a2094ac9b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5764,7 +5764,7 @@ makecode(struct compiler *c, struct assembler *a) Py_ssize_t nlocals; int nlocals_int; int flags; - int argcount, posonlyargcount, kwonlyargcount, maxdepth; + int posorkeywordargcount, posonlyargcount, kwonlyargcount, maxdepth; consts = consts_dict_keys_inorder(c->u->u_consts); names = dict_keys_inorder(c->u->u_names, 0); @@ -5808,15 +5808,15 @@ makecode(struct compiler *c, struct assembler *a) goto error; } - argcount = Py_SAFE_DOWNCAST(c->u->u_argcount, Py_ssize_t, int); posonlyargcount = Py_SAFE_DOWNCAST(c->u->u_posonlyargcount, Py_ssize_t, int); + posorkeywordargcount = Py_SAFE_DOWNCAST(c->u->u_argcount, Py_ssize_t, int); kwonlyargcount = Py_SAFE_DOWNCAST(c->u->u_kwonlyargcount, Py_ssize_t, int); maxdepth = stackdepth(c); if (maxdepth < 0) { goto error; } - co = PyCode_New(argcount, posonlyargcount, kwonlyargcount, - nlocals_int, maxdepth, flags, + co = PyCode_New(posonlyargcount+posorkeywordargcount, posonlyargcount, + kwonlyargcount, nlocals_int, maxdepth, flags, bytecode, consts, names, varnames, freevars, cellvars, c->c_filename, c->u->u_name, [View Less]

1 0

bpo-31968: Documentation -- add clarification on the globals dict for exec() (GH-13140)
by Raymond Hettinger June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/059b9ea5ac98f432e41b05d1fa5aab4ffa… commit: 059b9ea5ac98f432e41b05d1fa5aab4ffa22df4d branch: master author: Anthony Shaw <anthony.p.shaw(a)gmail.com> committer: Raymond Hettinger <rhettinger(a)users.noreply.github.com> date: 2019-06-01T08:51:58-07:00 summary: bpo-31968: Documentation -- add clarification on the globals dict for exec() (GH-13140) files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/… [View More]

1 0

Doc: Correct the creation year and the credits of the Logo Programming language (GH-13520)
by Miss Islington (bot) June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/fc914dd5e03db9188b6d28d1c48574dc78… commit: fc914dd5e03db9188b6d28d1c48574dc78ee4325 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T04:47:15-07:00 summary: Doc: Correct the creation year and the credits of the Logo Programming language (GH-13520) (cherry picked from commit 66501058fef76a5d77e6879f6da3282f0a9eef1b) Co-authored-by: Sté… [View More]

1 0

Doc: Correct the creation year and the credits of the Logo Programming language (GH-13520)
by Cheryl Sabella June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/66501058fef76a5d77e6879f6da3282f0a… commit: 66501058fef76a5d77e6879f6da3282f0a9eef1b branch: master author: Stéphane Wirtel <stephane(a)wirtel.be> committer: Cheryl Sabella <cheryl.sabella(a)gmail.com> date: 2019-06-01T07:41:33-04:00 summary: Doc: Correct the creation year and the credits of the Logo Programming language (GH-13520) files: M Doc/library/turtle.rst diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index … [View More]

1 0

Move whats-new entry for math.factorial to the math module section. (GH-13723)
by Mark Dickinson June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/a0adffb90287900a9e2c809ce2c1bbe481… commit: a0adffb90287900a9e2c809ce2c1bbe481ac7dd0 branch: master author: Mark Dickinson <mdickinson(a)enthought.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T12:21:53+01:00 summary: Move whats-new entry for math.factorial to the math module section. (GH-13723) files: M Doc/whatsnew/3.8.rst diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index bc7c9ef37442..2d5291102d2a 100644 --- … [View More]

1 0

bpo-36813: Fix QueueListener to call task_done() upon termination. (GH-13113)
by Miss Islington (bot) June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/f286e0373feda0955c910a9fe4ef99cd2c… commit: f286e0373feda0955c910a9fe4ef99cd2c40969e branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T02:36:29-07:00 summary: bpo-36813: Fix QueueListener to call task_done() upon termination. (GH-13113) Fixed QueueListener in order to avoid random deadlocks. Unable to add regression tests atm due to time … [View More]constraints, will add it in a bit. Regarding implementation, although it's nested, it does not cause performance issues whatsoever, and does not call task_done() in case of an exception (which is the right thing to do IMHO). https://bugs.python.org/issue36813 (cherry picked from commit 6b282e18877ec84e927b381b4ce187eaf4ba3dd7) Co-authored-by: Bar Harel <bzvi7919(a)gmail.com> files: A Misc/NEWS.d/next/Library/2019-05-06-18-28-38.bpo-36813.NXD0KZ.rst M Lib/logging/handlers.py M Lib/test/test_logging.py diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 3727bf0677cd..a913d27389ab 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1477,6 +1477,8 @@ def _monitor(self): try: record = self.dequeue(True) if record is self._sentinel: + if has_task_done: + q.task_done() break self.handle(record) if has_task_done: diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index d12e1e57455d..97c13a4c5214 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3495,6 +3495,16 @@ def test_no_messages_in_queue_after_stop(self): [m.msg if isinstance(m, logging.LogRecord) else m for m in items])) + def test_calls_task_done_after_stop(self): + # Issue 36813: Make sure queue.join does not deadlock. + log_queue = queue.Queue() + listener = logging.handlers.QueueListener(log_queue) + listener.start() + listener.stop() + with self.assertRaises(ValueError): + # Make sure all tasks are done and .join won't block. + log_queue.task_done() + ZERO = datetime.timedelta(0) diff --git a/Misc/NEWS.d/next/Library/2019-05-06-18-28-38.bpo-36813.NXD0KZ.rst b/Misc/NEWS.d/next/Library/2019-05-06-18-28-38.bpo-36813.NXD0KZ.rst new file mode 100644 index 000000000000..e89358aa4051 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-06-18-28-38.bpo-36813.NXD0KZ.rst @@ -0,0 +1,2 @@ +Fix :class:`~logging.handlers.QueueListener` to call ``queue.task_done()`` +upon stopping. Patch by Bar Harel. [View Less]

1 0

bpo-36813: Fix QueueListener to call task_done() upon termination. (GH-13113)
by Miss Islington (bot) June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/6b282e18877ec84e927b381b4ce187eaf4… commit: 6b282e18877ec84e927b381b4ce187eaf4ba3dd7 branch: master author: Bar Harel <bzvi7919(a)gmail.com> committer: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> date: 2019-06-01T02:19:09-07:00 summary: bpo-36813: Fix QueueListener to call task_done() upon termination. (GH-13113) Fixed QueueListener in order to avoid random deadlocks. Unable to add regression tests atm due to … [View More]time constraints, will add it in a bit. Regarding implementation, although it's nested, it does not cause performance issues whatsoever, and does not call task_done() in case of an exception (which is the right thing to do IMHO). https://bugs.python.org/issue36813 files: A Misc/NEWS.d/next/Library/2019-05-06-18-28-38.bpo-36813.NXD0KZ.rst M Lib/logging/handlers.py M Lib/test/test_logging.py diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 3727bf0677cd..a913d27389ab 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1477,6 +1477,8 @@ def _monitor(self): try: record = self.dequeue(True) if record is self._sentinel: + if has_task_done: + q.task_done() break self.handle(record) if has_task_done: diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index b884753ad397..50148dc2f252 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3633,6 +3633,16 @@ def test_no_messages_in_queue_after_stop(self): [m.msg if isinstance(m, logging.LogRecord) else m for m in items])) + def test_calls_task_done_after_stop(self): + # Issue 36813: Make sure queue.join does not deadlock. + log_queue = queue.Queue() + listener = logging.handlers.QueueListener(log_queue) + listener.start() + listener.stop() + with self.assertRaises(ValueError): + # Make sure all tasks are done and .join won't block. + log_queue.task_done() + ZERO = datetime.timedelta(0) diff --git a/Misc/NEWS.d/next/Library/2019-05-06-18-28-38.bpo-36813.NXD0KZ.rst b/Misc/NEWS.d/next/Library/2019-05-06-18-28-38.bpo-36813.NXD0KZ.rst new file mode 100644 index 000000000000..e89358aa4051 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-06-18-28-38.bpo-36813.NXD0KZ.rst @@ -0,0 +1,2 @@ +Fix :class:`~logging.handlers.QueueListener` to call ``queue.task_done()`` +upon stopping. Patch by Bar Harel. [View Less]

1 0

Use more PEP 570 syntax in the documentation. (GH-13720)
by Serhiy Storchaka June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/70c5f2ae6e6a07d44a8d3f3202ea01bf69… commit: 70c5f2ae6e6a07d44a8d3f3202ea01bf697e05db branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T11:38:24+03:00 summary: Use more PEP 570 syntax in the documentation. (GH-13720) files: M Doc/faq/programming.rst M Doc/howto/logging-cookbook.rst M Doc/library/functools.rst M Doc/reference/datamodel.rst diff --git a/Doc/faq/… [View More]programming.rst b/Doc/faq/programming.rst index 9660a701427f..a00c6a053ef1 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -554,8 +554,8 @@ desired effect in a number of ways. 5) Or bundle up values in a class instance:: class callByRef: - def __init__(self, **args): - for (key, value) in args.items(): + def __init__(self, /, **args): + for key, value in args.items(): setattr(self, key, value) def func4(args): diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 4956aa0dd957..71f9fc920fdf 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -579,7 +579,7 @@ information. When you call one of the logging methods on an instance of information in the delegated call. Here's a snippet from the code of :class:`LoggerAdapter`:: - def debug(self, msg, *args, **kwargs): + def debug(self, msg, /, *args, **kwargs): """ Delegate a debug call to the underlying logger, after adding contextual information from this adapter instance. @@ -1079,7 +1079,7 @@ call ``str()`` on that object to get the actual format string. Consider the following two classes:: class BraceMessage: - def __init__(self, fmt, *args, **kwargs): + def __init__(self, fmt, /, *args, **kwargs): self.fmt = fmt self.args = args self.kwargs = kwargs @@ -1088,7 +1088,7 @@ following two classes:: return self.fmt.format(*self.args, **self.kwargs) class DollarMessage: - def __init__(self, fmt, **kwargs): + def __init__(self, fmt, /, **kwargs): self.fmt = fmt self.kwargs = kwargs @@ -1143,7 +1143,7 @@ to the above, as in the following example:: import logging - class Message(object): + class Message: def __init__(self, fmt, args): self.fmt = fmt self.args = args @@ -1155,7 +1155,7 @@ to the above, as in the following example:: def __init__(self, logger, extra=None): super(StyleAdapter, self).__init__(logger, extra or {}) - def log(self, level, msg, *args, **kwargs): + def log(self, level, msg, /, *args, **kwargs): if self.isEnabledFor(level): msg, kwargs = self.process(msg, kwargs) self.logger._log(level, Message(msg, args), (), **kwargs) @@ -1301,7 +1301,7 @@ You can also subclass :class:`QueueListener` to get messages from other kinds of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: class ZeroMQSocketListener(QueueListener): - def __init__(self, uri, *handlers, **kwargs): + def __init__(self, uri, /, *handlers, **kwargs): self.ctx = kwargs.get('ctx') or zmq.Context() socket = zmq.Socket(self.ctx, zmq.SUB) socket.setsockopt_string(zmq.SUBSCRIBE, '') # subscribe to everything @@ -1706,8 +1706,8 @@ which uses JSON to serialise the event in a machine-parseable manner:: import json import logging - class StructuredMessage(object): - def __init__(self, message, **kwargs): + class StructuredMessage: + def __init__(self, message, /, **kwargs): self.message = message self.kwargs = kwargs @@ -1750,8 +1750,8 @@ as in the following complete example:: return o.encode('unicode_escape').decode('ascii') return super(Encoder, self).default(o) - class StructuredMessage(object): - def __init__(self, message, **kwargs): + class StructuredMessage: + def __init__(self, message, /, **kwargs): self.message = message self.kwargs = kwargs @@ -1982,8 +1982,8 @@ object as a message format string, and that the logging package will call :func:`str` on that object to get the actual format string. Consider the following two classes:: - class BraceMessage(object): - def __init__(self, fmt, *args, **kwargs): + class BraceMessage: + def __init__(self, fmt, /, *args, **kwargs): self.fmt = fmt self.args = args self.kwargs = kwargs @@ -1991,8 +1991,8 @@ following two classes:: def __str__(self): return self.fmt.format(*self.args, **self.kwargs) - class DollarMessage(object): - def __init__(self, fmt, **kwargs): + class DollarMessage: + def __init__(self, fmt, /, **kwargs): self.fmt = fmt self.kwargs = kwargs @@ -2457,7 +2457,7 @@ scope of the context manager:: import logging import sys - class LoggingContext(object): + class LoggingContext: def __init__(self, logger, level=None, handler=None, close=True): self.logger = logger self.level = level diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 654a3efa214b..3a0b554e923c 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -252,7 +252,7 @@ The :mod:`functools` module defines the following functions: 18 -.. class:: partialmethod(func, *args, **keywords) +.. class:: partialmethod(func, /, *args, **keywords) Return a new :class:`partialmethod` descriptor which behaves like :class:`partial` except that it is designed to be used as a method diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 8b4d889535fb..c566dfdf856d 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1809,7 +1809,7 @@ class defining the method. class, as in:: class Philosopher: - def __init_subclass__(cls, default_name, **kwargs): + def __init_subclass__(cls, /, default_name, **kwargs): super().__init_subclass__(**kwargs) cls.default_name = default_name [View Less]

1 0

bpo-37116: Use PEP 570 syntax for positional-only parameters. (GH-13700)
by Serhiy Storchaka June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/2085bd0877e17ad4d98a4586d5eabb6fae… commit: 2085bd0877e17ad4d98a4586d5eabb6faecbb190 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-01T11:00:15+03:00 summary: bpo-37116: Use PEP 570 syntax for positional-only parameters. (GH-13700) files: M Doc/library/collections.rst M Doc/library/contextlib.rst M Doc/library/email.headerregistry.rst M Doc/library/functools.rst M Doc/… [View More]library/inspect.rst M Doc/library/operator.rst M Doc/library/string.rst M Doc/library/types.rst M Doc/library/unittest.mock-examples.rst M Doc/library/unittest.rst M Doc/library/weakref.rst M Doc/whatsnew/3.8.rst M Lib/_collections_abc.py M Lib/_py_abc.py M Lib/_threading_local.py M Lib/collections/__init__.py M Lib/contextlib.py M Lib/dataclasses.py M Lib/functools.py M Lib/idlelib/debugger_r.py M Lib/idlelib/rpc.py M Lib/inspect.py M Lib/multiprocessing/dummy/__init__.py M Lib/multiprocessing/managers.py M Lib/multiprocessing/pool.py M Lib/operator.py M Lib/random.py M Lib/string.py M Lib/test/support/script_helper.py M Lib/typing.py M Lib/unittest/case.py M Lib/unittest/mock.py M Lib/unittest/test/test_runner.py M Lib/weakref.py diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index ec921d79d0c4..90a3f4bea9a4 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -1140,7 +1140,7 @@ variants of :func:`functools.lru_cache`:: class LRU(OrderedDict): 'Limit size, evicting the least recently looked-up key when full' - def __init__(self, maxsize=128, *args, **kwds): + def __init__(self, maxsize=128, /, *args, **kwds): self.maxsize = maxsize super().__init__(*args, **kwds) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 017a87a5648c..73b24e5f251a 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -637,7 +637,7 @@ even further by means of a small helper class:: from contextlib import ExitStack class Callback(ExitStack): - def __init__(self, callback, *args, **kwds): + def __init__(self, callback, /, *args, **kwds): super(Callback, self).__init__() self.callback(callback, *args, **kwds) diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index ce283c6b596c..c3ce90c2d6d9 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -107,7 +107,7 @@ headers. method if it wishes to set additional attributes beyond those provided by ``BaseHeader`` itself. Such an ``init`` method should look like this:: - def init(self, *args, **kw): + def init(self, /, *args, **kw): self._myattr = kw.pop('myattr') super().init(*args, **kw) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 8b8b1f80a622..654a3efa214b 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -221,7 +221,7 @@ The :mod:`functools` module defines the following functions: Returning NotImplemented from the underlying comparison function for unrecognised types is now supported. -.. function:: partial(func, *args, **keywords) +.. function:: partial(func, /, *args, **keywords) Return a new :ref:`partial object<partial-objects>` which when called will behave like *func* called with the positional arguments *args* @@ -230,7 +230,7 @@ The :mod:`functools` module defines the following functions: supplied, they extend and override *keywords*. Roughly equivalent to:: - def partial(func, *args, **keywords): + def partial(func, /, *args, **keywords): def newfunc(*fargs, **fkeywords): newkeywords = {**keywords, **fkeywords} return func(*args, *fargs, **newkeywords) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 1cc503a8e94b..2a71201a80b2 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1022,7 +1022,7 @@ Classes and functions metatype is in use, cls will be the first element of the tuple. -.. function:: getcallargs(func, *args, **kwds) +.. function:: getcallargs(func, /, *args, **kwds) Bind the *args* and *kwds* to the argument names of the Python function or method *func*, as if it was called with them. For bound methods, bind also the diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 5d0ea7dfdd89..fa02bde84650 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -339,7 +339,7 @@ expect a function argument. [('orange', 1), ('banana', 2), ('apple', 3), ('pear', 5)] -.. function:: methodcaller(name[, args...]) +.. function:: methodcaller(name, /, *args, **kwargs) Return a callable object that calls the method *name* on its operand. If additional arguments and/or keyword arguments are given, they will be given @@ -352,7 +352,7 @@ expect a function argument. Equivalent to:: - def methodcaller(name, *args, **kwargs): + def methodcaller(name, /, *args, **kwargs): def caller(obj): return getattr(obj, name)(*args, **kwargs) return caller diff --git a/Doc/library/string.rst b/Doc/library/string.rst index c2f65224bc8d..288dde6b3fe4 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -88,7 +88,7 @@ implementation as the built-in :meth:`~str.format` method. The :class:`Formatter` class has the following public methods: - .. method:: format(format_string, *args, **kwargs) + .. method:: format(format_string, /, *args, **kwargs) The primary API method. It takes a format string and an arbitrary set of positional and keyword arguments. @@ -720,7 +720,7 @@ these rules. The methods of :class:`Template` are: The constructor takes a single argument which is the template string. - .. method:: substitute(mapping, **kwds) + .. method:: substitute(mapping={}, /, **kwds) Performs the template substitution, returning a new string. *mapping* is any dictionary-like object with keys that match the placeholders in the @@ -729,7 +729,7 @@ these rules. The methods of :class:`Template` are: and there are duplicates, the placeholders from *kwds* take precedence. - .. method:: safe_substitute(mapping, **kwds) + .. method:: safe_substitute(mapping={}, /, **kwds) Like :meth:`substitute`, except that if placeholders are missing from *mapping* and *kwds*, instead of raising a :exc:`KeyError` exception, the diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 07c3a2e7f682..e629c2935f27 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -327,7 +327,7 @@ Additional Utility Classes and Functions The type is roughly equivalent to the following code:: class SimpleNamespace: - def __init__(self, **kwargs): + def __init__(self, /, **kwargs): self.__dict__.update(kwargs) def __repr__(self): diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 16690f349822..811f0fb1ce93 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -848,7 +848,7 @@ Here's an example implementation: >>> from copy import deepcopy >>> class CopyingMock(MagicMock): - ... def __call__(self, *args, **kwargs): + ... def __call__(self, /, *args, **kwargs): ... args = deepcopy(args) ... kwargs = deepcopy(kwargs) ... return super(CopyingMock, self).__call__(*args, **kwargs) @@ -1042,7 +1042,7 @@ that it takes arbitrary keyword arguments (``**kwargs``) which are then passed onto the mock constructor: >>> class Subclass(MagicMock): - ... def _get_child_mock(self, **kwargs): + ... def _get_child_mock(self, /, **kwargs): ... return MagicMock(**kwargs) ... >>> mymock = Subclass() diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 54a9f2c6f735..5ec4b40856ae 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1456,7 +1456,7 @@ Test cases .. versionadded:: 3.1 - .. classmethod:: addClassCleanup(function, *args, **kwargs) + .. classmethod:: addClassCleanup(function, /, *args, **kwargs) Add a function to be called after :meth:`tearDownClass` to cleanup resources used during the test class. Functions will be called in reverse @@ -2313,7 +2313,7 @@ To add cleanup code that must be run even in the case of an exception, use ``addModuleCleanup``: -.. function:: addModuleCleanup(function, *args, **kwargs) +.. function:: addModuleCleanup(function, /, *args, **kwargs) Add a function to be called after :func:`tearDownModule` to cleanup resources used during the test class. Functions will be called in reverse diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 80a908bbd83b..a28d71060f38 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -396,7 +396,7 @@ the referent is accessed:: import weakref class ExtendedRef(weakref.ref): - def __init__(self, ob, callback=None, **annotations): + def __init__(self, ob, callback=None, /, **annotations): super(ExtendedRef, self).__init__(ob, callback) self.__counter = 0 for k, v in annotations.items(): diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 76d00938dbec..bc7c9ef37442 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -941,8 +941,7 @@ Deprecated :meth:`profile.Profile.runcall`, :meth:`cProfile.Profile.runcall`, :meth:`bdb.Bdb.runcall`, :meth:`trace.Trace.runfunc` and :func:`curses.wrapper`. - - *function* in :func:`unittest.addModuleCleanup` and - :meth:`unittest.TestCase.addCleanup`. + - *function* in :meth:`unittest.TestCase.addCleanup`. - *fn* in the :meth:`~concurrent.futures.Executor.submit` method of :class:`concurrent.futures.ThreadPoolExecutor` and :class:`concurrent.futures.ProcessPoolExecutor`. diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index c363987970b4..2b2ddba170e1 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -821,30 +821,21 @@ def clear(self): except KeyError: pass - def update(*args, **kwds): + def update(self, other=(), /, **kwds): ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. If E present and has a .keys() method, does: for k in E: D[k] = E[k] If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v In either case, this is followed by: for k, v in F.items(): D[k] = v ''' - if not args: - raise TypeError("descriptor 'update' of 'MutableMapping' object " - "needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('update expected at most 1 arguments, got %d' % - len(args)) - if args: - other = args[0] - if isinstance(other, Mapping): - for key in other: - self[key] = other[key] - elif hasattr(other, "keys"): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value + if isinstance(other, Mapping): + for key in other: + self[key] = other[key] + elif hasattr(other, "keys"): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value for key, value in kwds.items(): self[key] = value diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index 3c3aa8e3d61b..c870ae9048b4 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -32,7 +32,7 @@ class ABCMeta(type): # external code. _abc_invalidation_counter = 0 - def __new__(mcls, name, bases, namespace, **kwargs): + def __new__(mcls, name, bases, namespace, /, **kwargs): cls = super().__new__(mcls, name, bases, namespace, **kwargs) # Compute set of abstract method names abstracts = {name diff --git a/Lib/_threading_local.py b/Lib/_threading_local.py index 245bd0ac91b7..b006d76c4e23 100644 --- a/Lib/_threading_local.py +++ b/Lib/_threading_local.py @@ -56,7 +56,7 @@ >>> class MyLocal(local): ... number = 2 - ... def __init__(self, **kw): + ... def __init__(self, /, **kw): ... self.__dict__.update(kw) ... def squared(self): ... return self.number ** 2 @@ -204,7 +204,7 @@ def _patch(self): class local: __slots__ = '_local__impl', '__dict__' - def __new__(cls, *args, **kw): + def __new__(cls, /, *args, **kw): if (args or kw) and (cls.__init__ is object.__init__): raise TypeError("Initialization arguments are not supported") self = object.__new__(cls) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 5b740d84c275..e9999e27d5f5 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -93,16 +93,10 @@ class OrderedDict(dict): # Individual links are kept alive by the hard reference in self.__map. # Those hard references disappear when a key is deleted from an OrderedDict. - def __init__(*args, **kwds): + def __init__(self, other=(), /, **kwds): '''Initialize an ordered dictionary. The signature is the same as regular dictionaries. Keyword argument order is preserved. ''' - if not args: - raise TypeError("descriptor '__init__' of 'OrderedDict' object " - "needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) try: self.__root except AttributeError: @@ -110,7 +104,7 @@ def __init__(*args, **kwds): self.__root = root = _proxy(self.__hardroot) root.prev = root.next = root self.__map = {} - self.__update(*args, **kwds) + self.__update(other, **kwds) def __setitem__(self, key, value, dict_setitem=dict.__setitem__, proxy=_proxy, Link=_Link): @@ -413,8 +407,8 @@ def _make(cls, iterable): _make.__func__.__doc__ = (f'Make a new {typename} object from a sequence ' 'or iterable') - def _replace(_self, **kwds): - result = _self._make(_map(kwds.pop, field_names, _self)) + def _replace(self, /, **kwds): + result = self._make(_map(kwds.pop, field_names, self)) if kwds: raise ValueError(f'Got unexpected field names: {list(kwds)!r}') return result @@ -543,7 +537,7 @@ class Counter(dict): # http://code.activestate.com/recipes/259174/ # Knuth, TAOCP Vol. II section 4.6.3 - def __init__(*args, **kwds): + def __init__(self, iterable=None, /, **kwds): '''Create a new, empty Counter object. And if given, count elements from an input iterable. Or, initialize the count from another mapping of elements to their counts. @@ -554,14 +548,8 @@ def __init__(*args, **kwds): >>> c = Counter(a=4, b=2) # a new counter from keyword args ''' - if not args: - raise TypeError("descriptor '__init__' of 'Counter' object " - "needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) super(Counter, self).__init__() - self.update(*args, **kwds) + self.update(iterable, **kwds) def __missing__(self, key): 'The count of elements not in the Counter is zero.' @@ -617,7 +605,7 @@ def fromkeys(cls, iterable, v=None): raise NotImplementedError( 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.') - def update(*args, **kwds): + def update(self, iterable=None, /, **kwds): '''Like dict.update() but add counts instead of replacing them. Source can be an iterable, a dictionary, or another Counter instance. @@ -637,13 +625,6 @@ def update(*args, **kwds): # contexts. Instead, we implement straight-addition. Both the inputs # and outputs are allowed to contain zero and negative counts. - if not args: - raise TypeError("descriptor 'update' of 'Counter' object " - "needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - iterable = args[0] if args else None if iterable is not None: if isinstance(iterable, _collections_abc.Mapping): if self: @@ -657,7 +638,7 @@ def update(*args, **kwds): if kwds: self.update(kwds) - def subtract(*args, **kwds): + def subtract(self, iterable=None, /, **kwds): '''Like dict.update() but subtracts counts instead of replacing them. Counts can be reduced below zero. Both the inputs and outputs are allowed to contain zero and negative counts. @@ -673,13 +654,6 @@ def subtract(*args, **kwds): -1 ''' - if not args: - raise TypeError("descriptor 'subtract' of 'Counter' object " - "needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - iterable = args[0] if args else None if iterable is not None: self_get = self.get if isinstance(iterable, _collections_abc.Mapping): @@ -1141,7 +1115,7 @@ def copy(self): return self.__class__(self) def count(self, item): return self.data.count(item) def index(self, item, *args): return self.data.index(item, *args) def reverse(self): self.data.reverse() - def sort(self, *args, **kwds): self.data.sort(*args, **kwds) + def sort(self, /, *args, **kwds): self.data.sort(*args, **kwds) def extend(self, other): if isinstance(other, UserList): self.data.extend(other.data) @@ -1240,7 +1214,7 @@ def find(self, sub, start=0, end=_sys.maxsize): if isinstance(sub, UserString): sub = sub.data return self.data.find(sub, start, end) - def format(self, *args, **kwds): + def format(self, /, *args, **kwds): return self.data.format(*args, **kwds) def format_map(self, mapping): return self.data.format_map(mapping) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index de989a001c6d..94dc2bfed06c 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -377,8 +377,7 @@ def _create_exit_wrapper(cm, cm_exit): return MethodType(cm_exit, cm) @staticmethod - def _create_cb_wrapper(*args, **kwds): - callback, *args = args + def _create_cb_wrapper(callback, /, *args, **kwds): def _exit_wrapper(exc_type, exc, tb): callback(*args, **kwds) return _exit_wrapper @@ -553,8 +552,7 @@ def _create_async_exit_wrapper(cm, cm_exit): return MethodType(cm_exit, cm) @staticmethod - def _create_async_cb_wrapper(*args, **kwds): - callback, *args = args + def _create_async_cb_wrapper(callback, /, *args, **kwds): async def _exit_wrapper(exc_type, exc, tb): await callback(*args, **kwds) return _exit_wrapper diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 325b822d9f06..75113f123b3a 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -962,10 +962,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): return cls -# _cls should never be specified by keyword, so start it with an -# underscore. The presence of _cls is used to detect if this -# decorator is being called with parameters or not. -def dataclass(_cls=None, *, init=True, repr=True, eq=True, order=False, +def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False): """Returns the same class as was passed in, with dunder methods added based on the fields defined in the class. @@ -983,12 +980,12 @@ def wrap(cls): return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen) # See if we're being called as @dataclass or @dataclass(). - if _cls is None: + if cls is None: # We're called with parens. return wrap # We're called as @dataclass without parens. - return wrap(_cls) + return wrap(cls) def fields(class_or_instance): diff --git a/Lib/functools.py b/Lib/functools.py index 30964a6fe3d8..64d120182bb0 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -273,15 +273,9 @@ class partial: __slots__ = "func", "args", "keywords", "__dict__", "__weakref__" - def __new__(*args, **keywords): - if not args: - raise TypeError("descriptor '__new__' of partial needs an argument") - if len(args) < 2: - raise TypeError("type 'partial' takes at least one argument") - cls, func, *args = args + def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") - args = tuple(args) if hasattr(func, "func"): args = func.args + args @@ -295,10 +289,7 @@ def __new__(*args, **keywords): self.keywords = keywords return self - def __call__(*args, **keywords): - if not args: - raise TypeError("descriptor '__call__' of partial needs an argument") - self, *args = args + def __call__(self, /, *args, **keywords): keywords = {**self.keywords, **keywords} return self.func(*self.args, *args, **keywords) @@ -402,8 +393,7 @@ def __repr__(self): keywords=keywords) def _make_unbound_method(self): - def _method(*args, **keywords): - cls_or_self, *args = args + def _method(cls_or_self, /, *args, **keywords): keywords = {**self.keywords, **keywords} return self.func(cls_or_self, *self.args, *args, **keywords) _method.__isabstractmethod__ = self.__isabstractmethod__ diff --git a/Lib/idlelib/debugger_r.py b/Lib/idlelib/debugger_r.py index 0e6dcfbd12c2..9dcfc56414c0 100644 --- a/Lib/idlelib/debugger_r.py +++ b/Lib/idlelib/debugger_r.py @@ -299,7 +299,7 @@ def __init__(self, conn, shell, oid): self.conn = conn self.shell = shell - def call(self, methodname, *args, **kwargs): + def call(self, methodname, /, *args, **kwargs): ##print("*** IdbProxy.call %s %s %s" % (methodname, args, kwargs)) value = self.conn.remotecall(self.oid, methodname, args, kwargs) ##print("*** IdbProxy.call %s returns %r" % (methodname, value)) diff --git a/Lib/idlelib/rpc.py b/Lib/idlelib/rpc.py index f035bde4a0a0..aa8cbd36c479 100644 --- a/Lib/idlelib/rpc.py +++ b/Lib/idlelib/rpc.py @@ -603,7 +603,7 @@ def __init__(self, sockio, oid, name): self.oid = oid self.name = name - def __call__(self, *args, **kwargs): + def __call__(self, /, *args, **kwargs): value = self.sockio.remotecall(self.oid, self.name, args, kwargs) return value diff --git a/Lib/inspect.py b/Lib/inspect.py index a4f28f755705..ca81a24f0641 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1329,14 +1329,12 @@ def _too_many(f_name, args, kwonly, varargs, defcount, given, values): (f_name, sig, "s" if plural else "", given, kwonly_sig, "was" if given == 1 and not kwonly_given else "were")) -def getcallargs(*func_and_positional, **named): +def getcallargs(func, /, *positional, **named): """Get the mapping of arguments to values. A dict is returned, with keys the function argument names (including the names of the * and ** arguments, if any), and values the respective bound values from 'positional' and 'named'.""" - func = func_and_positional[0] - positional = func_and_positional[1:] spec = getfullargspec(func) args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = spec f_name = func.__name__ @@ -3027,19 +3025,19 @@ def _bind(self, args, kwargs, *, partial=False): return self._bound_arguments_cls(self, arguments) - def bind(*args, **kwargs): + def bind(self, /, *args, **kwargs): """Get a BoundArguments object, that maps the passed `args` and `kwargs` to the function's signature. Raises `TypeError` if the passed arguments can not be bound. """ - return args[0]._bind(args[1:], kwargs) + return self._bind(args, kwargs) - def bind_partial(*args, **kwargs): + def bind_partial(self, /, *args, **kwargs): """Get a BoundArguments object, that partially maps the passed `args` and `kwargs` to the function's signature. Raises `TypeError` if the passed arguments can not be bound. """ - return args[0]._bind(args[1:], kwargs, partial=True) + return self._bind(args, kwargs, partial=True) def __reduce__(self): return (type(self), diff --git a/Lib/multiprocessing/dummy/__init__.py b/Lib/multiprocessing/dummy/__init__.py index 403f5e5198e4..6a1468609e34 100644 --- a/Lib/multiprocessing/dummy/__init__.py +++ b/Lib/multiprocessing/dummy/__init__.py @@ -80,7 +80,7 @@ def freeze_support(): # class Namespace(object): - def __init__(self, **kwds): + def __init__(self, /, **kwds): self.__dict__.update(kwds) def __repr__(self): items = list(self.__dict__.items()) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 514152298b09..7e1818bb0996 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -615,13 +615,10 @@ def _run_server(cls, registry, address, authkey, serializer, writer, util.info('manager serving at %r', server.address) server.serve_forever() - def _create(*args, **kwds): + def _create(self, typeid, /, *args, **kwds): ''' Create a new shared object; return the token and exposed tuple ''' - self, typeid, *args = args - args = tuple(args) - assert self._state.value == State.STARTED, 'server not yet started' conn = self._Client(self._address, authkey=self._authkey) try: @@ -738,7 +735,7 @@ def register(cls, typeid, callable=None, proxytype=None, exposed=None, ) if create_method: - def temp(self, *args, **kwds): + def temp(self, /, *args, **kwds): util.debug('requesting creation of a shared %r object', typeid) token, exp = self._create(typeid, *args, **kwds) proxy = proxytype( @@ -978,7 +975,7 @@ def MakeProxyType(name, exposed, _cache={}): dic = {} for meth in exposed: - exec('''def %s(self, *args, **kwds): + exec('''def %s(self, /, *args, **kwds): return self._callmethod(%r, args, kwds)''' % (meth, meth), dic) ProxyType = type(name, (BaseProxy,), dic) @@ -1017,7 +1014,7 @@ def AutoProxy(token, serializer, manager=None, authkey=None, # class Namespace(object): - def __init__(self, **kwds): + def __init__(self, /, **kwds): self.__dict__.update(kwds) def __repr__(self): items = list(self.__dict__.items()) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 665ca067fa07..b223d6aa724b 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -154,7 +154,7 @@ class _PoolCache(dict): notification is done by the use of a queue that is provided when instantiating the cache. """ - def __init__(self, *args, notifier=None, **kwds): + def __init__(self, /, *args, notifier=None, **kwds): self.notifier = notifier super().__init__(*args, **kwds) diff --git a/Lib/operator.py b/Lib/operator.py index 0e2e53efc69a..fb58851fa6ef 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -302,15 +302,11 @@ class methodcaller: """ __slots__ = ('_name', '_args', '_kwargs') - def __init__(*args, **kwargs): - if len(args) < 2: - msg = "methodcaller needs at least one argument, the method name" - raise TypeError(msg) - self = args[0] - self._name = args[1] + def __init__(self, name, /, *args, **kwargs): + self._name = name if not isinstance(self._name, str): raise TypeError('method name must be a string') - self._args = args[2:] + self._args = args self._kwargs = kwargs def __call__(self, obj): diff --git a/Lib/random.py b/Lib/random.py index 53981f3e4f89..365a01957203 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -100,7 +100,7 @@ def __init__(self, x=None): self.seed(x) self.gauss_next = None - def __init_subclass__(cls, **kwargs): + def __init_subclass__(cls, /, **kwargs): """Control how subclasses generate random integers. The algorithm a subclass can use depends on the random() and/or diff --git a/Lib/string.py b/Lib/string.py index b9d6f5eb5675..b423ff5dc6f6 100644 --- a/Lib/string.py +++ b/Lib/string.py @@ -52,6 +52,8 @@ def capwords(s, sep=None): import re as _re from collections import ChainMap as _ChainMap +_sentinel_dict = {} + class _TemplateMetaclass(type): pattern = r""" %(delim)s(?: @@ -104,19 +106,11 @@ def _invalid(self, mo): raise ValueError('Invalid placeholder in string: line %d, col %d' % (lineno, colno)) - def substitute(*args, **kws): - if not args: - raise TypeError("descriptor 'substitute' of 'Template' object " - "needs an argument") - self, *args = args # allow the "self" keyword be passed - if len(args) > 1: - raise TypeError('Too many positional arguments') - if not args: + def substitute(self, mapping=_sentinel_dict, /, **kws): + if mapping is _sentinel_dict: mapping = kws elif kws: - mapping = _ChainMap(kws, args[0]) - else: - mapping = args[0] + mapping = _ChainMap(kws, mapping) # Helper function for .sub() def convert(mo): # Check the most common path first. @@ -131,19 +125,11 @@ def convert(mo): self.pattern) return self.pattern.sub(convert, self.template) - def safe_substitute(*args, **kws): - if not args: - raise TypeError("descriptor 'safe_substitute' of 'Template' object " - "needs an argument") - self, *args = args # allow the "self" keyword be passed - if len(args) > 1: - raise TypeError('Too many positional arguments') - if not args: + def safe_substitute(self, mapping=_sentinel_dict, /, **kws): + if mapping is _sentinel_dict: mapping = kws elif kws: - mapping = _ChainMap(kws, args[0]) - else: - mapping = args[0] + mapping = _ChainMap(kws, mapping) # Helper function for .sub() def convert(mo): named = mo.group('named') or mo.group('braced') @@ -173,16 +159,7 @@ def convert(mo): # The field name parser is implemented in _string.formatter_field_name_split class Formatter: - def format(*args, **kwargs): - if not args: - raise TypeError("descriptor 'format' of 'Formatter' object " - "needs an argument") - self, *args = args # allow the "self" keyword be passed - try: - format_string, *args = args # allow the "format_string" keyword be passed - except ValueError: - raise TypeError("format() missing 1 required positional " - "argument: 'format_string'") from None + def format(self, format_string, /, *args, **kwargs): return self.vformat(format_string, args, kwargs) def vformat(self, format_string, args, kwargs): diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index 27a47f2c4e66..83519988e394 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -137,7 +137,7 @@ def run_python_until_end(*args, **env_vars): err = strip_python_stderr(err) return _PythonRunResult(rc, out, err), cmd_line -def _assert_python(expected_success, *args, **env_vars): +def _assert_python(expected_success, /, *args, **env_vars): res, cmd_line = run_python_until_end(*args, **env_vars) if (res.rc and expected_success) or (not res.rc and not expected_success): res.fail(cmd_line) diff --git a/Lib/typing.py b/Lib/typing.py index 16ccfad049f4..5f1a0ad3d637 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -283,7 +283,7 @@ class _Final: __slots__ = ('__weakref__',) - def __init_subclass__(self, *args, **kwds): + def __init_subclass__(self, /, *args, **kwds): if '_root' not in kwds: raise TypeError("Cannot subclass special typing classes") diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 7b1e86941315..b363c6351007 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -86,23 +86,10 @@ def _id(obj): _module_cleanups = [] -def addModuleCleanup(*args, **kwargs): +def addModuleCleanup(function, /, *args, **kwargs): """Same as addCleanup, except the cleanup items are called even if setUpModule fails (unlike tearDownModule).""" - if args: - function, *args = args - elif 'function' in kwargs: - function = kwargs.pop('function') - import warnings - warnings.warn("Passing 'function' as keyword argument is deprecated", - DeprecationWarning, stacklevel=2) - else: - raise TypeError('addModuleCleanup expected at least 1 positional ' - 'argument, got %d' % (len(args)-1)) - args = tuple(args) - _module_cleanups.append((function, args, kwargs)) -addModuleCleanup.__text_signature__ = '(function, /, *args, **kwargs)' def doModuleCleanups(): @@ -501,22 +488,11 @@ def addCleanup(*args, **kwargs): self._cleanups.append((function, args, kwargs)) addCleanup.__text_signature__ = '($self, function, /, *args, **kwargs)' - def addClassCleanup(*args, **kwargs): + @classmethod + def addClassCleanup(cls, function, /, *args, **kwargs): """Same as addCleanup, except the cleanup items are called even if setUpClass fails (unlike tearDownClass).""" - if len(args) >= 2: - cls, function, *args = args - elif not args: - raise TypeError("descriptor 'addClassCleanup' of 'TestCase' object " - "needs an argument") - else: - raise TypeError('addClassCleanup expected at least 1 positional ' - 'argument, got %d' % (len(args)-1)) - args = tuple(args) - cls._class_cleanups.append((function, args, kwargs)) - addClassCleanup.__text_signature__ = '($cls, function, /, *args, **kwargs)' - addClassCleanup = classmethod(addClassCleanup) def setUp(self): "Hook method for setting up the test fixture before exercising it." diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index be96194793ef..c2802726d75d 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -106,7 +106,7 @@ def _check_signature(func, mock, skipfirst, instance=False): if sig is None: return func, sig = sig - def checksig(_mock_self, *args, **kwargs): + def checksig(self, /, *args, **kwargs): sig.bind(*args, **kwargs) _copy_func_details(func, checksig) type(mock)._mock_check_sig = checksig @@ -243,7 +243,7 @@ def _setup_async_mock(mock): # Mock is not configured yet so the attributes are set # to a function and then the corresponding mock helper function # is called when the helper is accessed similar to _setup_func. - def wrapper(attr, *args, **kwargs): + def wrapper(attr, /, *args, **kwargs): return getattr(mock.mock, attr)(*args, **kwargs) for attribute in ('assert_awaited', @@ -387,7 +387,7 @@ def __next__(self): class Base(object): _mock_return_value = DEFAULT _mock_side_effect = None - def __init__(self, *args, **kwargs): + def __init__(self, /, *args, **kwargs): pass @@ -395,7 +395,7 @@ def __init__(self, *args, **kwargs): class NonCallableMock(Base): """A non-callable version of `Mock`""" - def __new__(cls, *args, **kw): + def __new__(cls, /, *args, **kw): # every instance has its own class # so we can create magic methods on the # class without stomping on other mocks @@ -602,7 +602,7 @@ def reset_mock(self, visited=None,*, return_value=False, side_effect=False): ret.reset_mock(visited) - def configure_mock(self, **kwargs): + def configure_mock(self, /, **kwargs): """Set attributes on the mock through keyword arguments. Attributes plus return values and side effects can be set on child @@ -820,10 +820,9 @@ def _call_matcher(self, _call): else: return _call - def assert_not_called(_mock_self): + def assert_not_called(self): """assert that the mock was never called. """ - self = _mock_self if self.call_count != 0: msg = ("Expected '%s' to not have been called. Called %s times.%s" % (self._mock_name or 'mock', @@ -831,19 +830,17 @@ def assert_not_called(_mock_self): self._calls_repr())) raise AssertionError(msg) - def assert_called(_mock_self): + def assert_called(self): """assert that the mock was called at least once """ - self = _mock_self if self.call_count == 0: msg = ("Expected '%s' to have been called." % self._mock_name or 'mock') raise AssertionError(msg) - def assert_called_once(_mock_self): + def assert_called_once(self): """assert that the mock was called only once. """ - self = _mock_self if not self.call_count == 1: msg = ("Expected '%s' to have been called once. Called %s times.%s" % (self._mock_name or 'mock', @@ -851,12 +848,11 @@ def assert_called_once(_mock_self): self._calls_repr())) raise AssertionError(msg) - def assert_called_with(_mock_self, *args, **kwargs): + def assert_called_with(self, /, *args, **kwargs): """assert that the mock was called with the specified arguments. Raises an AssertionError if the args and keyword args passed in are different to the last call to the mock.""" - self = _mock_self if self.call_args is None: expected = self._format_mock_call_signature(args, kwargs) actual = 'not called.' @@ -874,10 +870,9 @@ def _error_message(): raise AssertionError(_error_message()) from cause - def assert_called_once_with(_mock_self, *args, **kwargs): + def assert_called_once_with(self, /, *args, **kwargs): """assert that the mock was called exactly once and that that call was with the specified arguments.""" - self = _mock_self if not self.call_count == 1: msg = ("Expected '%s' to be called once. Called %s times.%s" % (self._mock_name or 'mock', @@ -924,7 +919,7 @@ def assert_has_calls(self, calls, any_order=False): ) from cause - def assert_any_call(self, *args, **kwargs): + def assert_any_call(self, /, *args, **kwargs): """assert the mock has been called with the specified arguments. The assert passes if the mock has *ever* been called, unlike @@ -940,7 +935,7 @@ def assert_any_call(self, *args, **kwargs): ) from cause - def _get_child_mock(self, **kw): + def _get_child_mock(self, /, **kw): """Create the child mocks for attributes and return value. By default child mocks will be the same type as the parent. Subclasses of Mock may want to override this to customize the way @@ -1016,20 +1011,19 @@ def __init__(self, spec=None, side_effect=None, return_value=DEFAULT, self.side_effect = side_effect - def _mock_check_sig(self, *args, **kwargs): + def _mock_check_sig(self, /, *args, **kwargs): # stub method that can be replaced with one with a specific signature pass - def __call__(_mock_self, *args, **kwargs): + def __call__(self, /, *args, **kwargs): # can't use self in-case a function / method we are mocking uses self # in the signature - _mock_self._mock_check_sig(*args, **kwargs) - return _mock_self._mock_call(*args, **kwargs) + self._mock_check_sig(*args, **kwargs) + return self._mock_call(*args, **kwargs) - def _mock_call(_mock_self, *args, **kwargs): - self = _mock_self + def _mock_call(self, /, *args, **kwargs): self.called = True self.call_count += 1 @@ -1840,7 +1834,7 @@ def _patch_stopall(): def _get_method(name, func): "Turns a callable object (like a mock) into a real function" - def method(self, *args, **kw): + def method(self, /, *args, **kw): return func(self, *args, **kw) method.__name__ = name return method @@ -1954,7 +1948,7 @@ def _set_return_value(mock, method, name): class MagicMixin(object): - def __init__(self, *args, **kw): + def __init__(self, /, *args, **kw): self._mock_set_magics() # make magic work for kwargs in init _safe_super(MagicMixin, self).__init__(*args, **kw) self._mock_set_magics() # fix magic broken by upper level init @@ -1996,7 +1990,7 @@ def mock_add_spec(self, spec, spec_set=False): class AsyncMagicMixin: - def __init__(self, *args, **kw): + def __init__(self, /, *args, **kw): self._mock_set_async_magics() # make magic work for kwargs in init _safe_super(AsyncMagicMixin, self).__init__(*args, **kw) self._mock_set_async_magics() # fix magic broken by upper level init @@ -2067,7 +2061,7 @@ class AsyncMockMixin(Base): await_args = _delegating_property('await_args') await_args_list = _delegating_property('await_args_list') - def __init__(self, *args, **kwargs): + def __init__(self, /, *args, **kwargs): super().__init__(*args, **kwargs) # asyncio.iscoroutinefunction() checks _is_coroutine property to say if an # object is a coroutine. Without this check it looks to see if it is a @@ -2084,8 +2078,7 @@ def __init__(self, *args, **kwargs): code_mock.co_flags = inspect.CO_COROUTINE self.__dict__['__code__'] = code_mock - async def _mock_call(_mock_self, *args, **kwargs): - self = _mock_self + async def _mock_call(self, /, *args, **kwargs): try: result = super()._mock_call(*args, **kwargs) except (BaseException, StopIteration) as e: @@ -2110,30 +2103,27 @@ def __init__(self, *args, **kwargs): return await proxy() - def assert_awaited(_mock_self): + def assert_awaited(self): """ Assert that the mock was awaited at least once. """ - self = _mock_self if self.await_count == 0: msg = f"Expected {self._mock_name or 'mock'} to have been awaited." raise AssertionError(msg) - def assert_awaited_once(_mock_self): + def assert_awaited_once(self): """ Assert that the mock was awaited exactly once. """ - self = _mock_self if not self.await_count == 1: msg = (f"Expected {self._mock_name or 'mock'} to have been awaited once." f" Awaited {self.await_count} times.") raise AssertionError(msg) - def assert_awaited_with(_mock_self, *args, **kwargs): + def assert_awaited_with(self, /, *args, **kwargs): """ Assert that the last await was with the specified arguments. """ - self = _mock_self if self.await_args is None: expected = self._format_mock_call_signature(args, kwargs) raise AssertionError(f'Expected await: {expected}\nNot awaited') @@ -2148,23 +2138,21 @@ def _error_message(): cause = expected if isinstance(expected, Exception) else None raise AssertionError(_error_message()) from cause - def assert_awaited_once_with(_mock_self, *args, **kwargs): + def assert_awaited_once_with(self, /, *args, **kwargs): """ Assert that the mock was awaited exactly once and with the specified arguments. """ - self = _mock_self if not self.await_count == 1: msg = (f"Expected {self._mock_name or 'mock'} to have been awaited once." f" Awaited {self.await_count} times.") raise AssertionError(msg) return self.assert_awaited_with(*args, **kwargs) - def assert_any_await(_mock_self, *args, **kwargs): + def assert_any_await(self, /, *args, **kwargs): """ Assert the mock has ever been awaited with the specified arguments. """ - self = _mock_self expected = self._call_matcher((args, kwargs)) actual = [self._call_matcher(c) for c in self.await_args_list] if expected not in actual: @@ -2174,7 +2162,7 @@ def assert_any_await(_mock_self, *args, **kwargs): '%s await not found' % expected_string ) from cause - def assert_has_awaits(_mock_self, calls, any_order=False): + def assert_has_awaits(self, calls, any_order=False): """ Assert the mock has been awaited with the specified calls. The :attr:`await_args_list` list is checked for the awaits. @@ -2186,7 +2174,6 @@ def assert_has_awaits(_mock_self, calls, any_order=False): If `any_order` is True then the awaits can be in any order, but they must all appear in :attr:`await_args_list`. """ - self = _mock_self expected = [self._call_matcher(c) for c in calls] cause = expected if isinstance(expected, Exception) else None all_awaits = _CallList(self._call_matcher(c) for c in self.await_args_list) @@ -2211,17 +2198,16 @@ def assert_has_awaits(_mock_self, calls, any_order=False): '%r not all found in await list' % (tuple(not_found),) ) from cause - def assert_not_awaited(_mock_self): + def assert_not_awaited(self): """ Assert that the mock was never awaited. """ - self = _mock_self if self.await_count != 0: msg = (f"Expected {self._mock_name or 'mock'} to not have been awaited." f" Awaited {self.await_count} times.") raise AssertionError(msg) - def reset_mock(self, *args, **kwargs): + def reset_mock(self, /, *args, **kwargs): """ See :func:`.Mock.reset_mock()` """ @@ -2424,7 +2410,7 @@ def __eq__(self, other): __ne__ = object.__ne__ - def __call__(self, *args, **kwargs): + def __call__(self, /, *args, **kwargs): if self._mock_name is None: return _Call(('', args, kwargs), name='()') @@ -2439,10 +2425,10 @@ def __getattr__(self, attr): return _Call(name=name, parent=self, from_kall=False) - def count(self, *args, **kwargs): + def count(self, /, *args, **kwargs): return self.__getattr__('count')(*args, **kwargs) - def index(self, *args, **kwargs): + def index(self, /, *args, **kwargs): return self.__getattr__('index')(*args, **kwargs) def _get_call_arguments(self): @@ -2778,7 +2764,7 @@ class PropertyMock(Mock): Fetching a `PropertyMock` instance from an object calls the mock, with no args. Setting it calls the mock with the value being set. """ - def _get_child_mock(self, **kwargs): + def _get_child_mock(self, /, **kwargs): return MagicMock(**kwargs) def __get__(self, obj, obj_type): diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py index 443b689dbea0..7d36340741f4 100644 --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -410,14 +410,13 @@ def cleanup(*args, **kwargs): class Module(object): unittest.addModuleCleanup(cleanup, 1, 2, function='hello') - with self.assertWarns(DeprecationWarning): + with self.assertRaises(TypeError): unittest.addModuleCleanup(function=cleanup, arg='hello') with self.assertRaises(TypeError): unittest.addModuleCleanup() unittest.case.doModuleCleanups() self.assertEqual(cleanups, - [((), {'arg': 'hello'}), - ((1, 2), {'function': 'hello'})]) + [((1, 2), {'function': 'hello'})]) def test_run_module_cleanUp(self): blowUp = True diff --git a/Lib/weakref.py b/Lib/weakref.py index 1eeb7b0a0b44..8d71af653b7e 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -99,13 +99,7 @@ class WeakValueDictionary(_collections_abc.MutableMapping): # objects are unwrapped on the way out, and we always wrap on the # way in). - def __init__(*args, **kw): - if not args: - raise TypeError("descriptor '__init__' of 'WeakValueDictionary' " - "object needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) + def __init__(self, other=(), /, **kw): def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref): self = selfref() if self is not None: @@ -120,7 +114,7 @@ def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref): self._pending_removals = [] self._iterating = set() self.data = d = {} - self.update(*args, **kw) + self.update(other, **kw) def _commit_removals(self): l = self._pending_removals @@ -287,24 +281,17 @@ def setdefault(self, key, default=None): else: return o - def update(*args, **kwargs): - if not args: - raise TypeError("descriptor 'update' of 'WeakValueDictionary' " - "object needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - dict = args[0] if args else None + def update(self, other=None, /, **kwargs): if self._pending_removals: self._commit_removals() d = self.data - if dict is not None: - if not hasattr(dict, "items"): - dict = type({})(dict) - for key, o in dict.items(): + if other is not None: + if not hasattr(other, "items"): + other = dict(other) + for key, o in other.items(): d[key] = KeyedRef(o, self._remove, key) - if len(kwargs): - self.update(kwargs) + for key, o in kwargs.items(): + d[key] = KeyedRef(o, self._remove, key) def valuerefs(self): """Return a list of weak references to the values. @@ -488,7 +475,7 @@ def pop(self, key, *args): def setdefault(self, key, default=None): return self.data.setdefault(ref(key, self._remove),default) - def update(self, dict=None, **kwargs): + def update(self, dict=None, /, **kwargs): d = self.data if dict is not None: if not hasattr(dict, "items"): [View Less]

1 0

bpo-35431: Implemented math.comb (GH-11414)
by Raymond Hettinger June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/4a686504eb2bbf69adf78077458508a7ba… commit: 4a686504eb2bbf69adf78077458508a7ba131667 branch: master author: Yash Aggarwal <Aggarwal.yash2011(a)gmail.com> committer: Raymond Hettinger <rhettinger(a)users.noreply.github.com> date: 2019-06-01T00:21:27-07:00 summary: bpo-35431: Implemented math.comb (GH-11414) files: A Misc/NEWS.d/next/Library/2019-01-02-19-48-23.bpo-35431.FhG6QA.rst M Doc/library/math.rst M Lib/test/test_math.py M Modules/… [View More]clinic/mathmodule.c.h M Modules/mathmodule.c diff --git a/Doc/library/math.rst b/Doc/library/math.rst index b51e96bc4074..5243970df806 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -232,6 +232,21 @@ Number-theoretic and representation functions :meth:`x.__trunc__() <object.__trunc__>`. +.. function:: comb(n, k) + + Return the number of ways to choose *k* items from *n* items without repetition + and without order. + + Also called the binomial coefficient. It is mathematically equal to the expression + ``n! / (k! (n - k)!)``. It is equivalent to the coefficient of k-th term in + polynomial expansion of the expression ``(1 + x) ** n``. + + Raises :exc:`TypeError` if the arguments not integers. + Raises :exc:`ValueError` if the arguments are negative or if k > n. + + .. versionadded:: 3.8 + + Note that :func:`frexp` and :func:`modf` have a different call/return pattern than their C equivalents: they take a single argument and return a pair of values, rather than returning their second return value through an 'output diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 853a0e62f823..9da7f7c4e6e2 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -1862,6 +1862,57 @@ def test_fractions(self): self.assertAllClose(fraction_examples, rel_tol=1e-8) self.assertAllNotClose(fraction_examples, rel_tol=1e-9) + def testComb(self): + comb = math.comb + factorial = math.factorial + # Test if factorial defintion is satisfied + for n in range(100): + for k in range(n + 1): + self.assertEqual(comb(n, k), factorial(n) + // (factorial(k) * factorial(n - k))) + + # Test for Pascal's identity + for n in range(1, 100): + for k in range(1, n): + self.assertEqual(comb(n, k), comb(n - 1, k - 1) + comb(n - 1, k)) + + # Test corner cases + for n in range(100): + self.assertEqual(comb(n, 0), 1) + self.assertEqual(comb(n, n), 1) + + for n in range(1, 100): + self.assertEqual(comb(n, 1), n) + self.assertEqual(comb(n, n - 1), n) + + # Test Symmetry + for n in range(100): + for k in range(n // 2): + self.assertEqual(comb(n, k), comb(n, n - k)) + + # Raises TypeError if any argument is non-integer or argument count is + # not 2 + self.assertRaises(TypeError, comb, 10, 1.0) + self.assertRaises(TypeError, comb, 10, "1") + self.assertRaises(TypeError, comb, "10", 1) + self.assertRaises(TypeError, comb, 10.0, 1) + + self.assertRaises(TypeError, comb, 10) + self.assertRaises(TypeError, comb, 10, 1, 3) + self.assertRaises(TypeError, comb) + + # Raises Value error if not k or n are negative numbers + self.assertRaises(ValueError, comb, -1, 1) + self.assertRaises(ValueError, comb, -10*10, 1) + self.assertRaises(ValueError, comb, 1, -1) + self.assertRaises(ValueError, comb, 1, -10*10) + + # Raises value error if k is greater than n + self.assertRaises(ValueError, comb, 1, 10**10) + self.assertRaises(ValueError, comb, 0, 1) + + + def test_main(): from doctest import DocFileSuite diff --git a/Misc/NEWS.d/next/Library/2019-01-02-19-48-23.bpo-35431.FhG6QA.rst b/Misc/NEWS.d/next/Library/2019-01-02-19-48-23.bpo-35431.FhG6QA.rst new file mode 100644 index 000000000000..34687bdb8a25 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-01-02-19-48-23.bpo-35431.FhG6QA.rst @@ -0,0 +1,4 @@ +Implement :func:`math.comb` that returns binomial coefficient, that computes +the number of ways to choose k items from n items without repetition and +without order. +Patch by Yash Aggarwal and Keller Fuchs. diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index e677bd896cd8..cba791e2098f 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -637,4 +637,53 @@ math_prod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k exit: return return_value; } -/*[clinic end generated code: output=aeed62f403b90199 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(math_comb__doc__, +"comb($module, /, n, k)\n" +"--\n" +"\n" +"Number of ways to choose *k* items from *n* items without repetition and without order.\n" +"\n" +"Also called the binomial coefficient. It is mathematically equal to the expression\n" +"n! / (k! * (n - k)!). It is equivalent to the coefficient of k-th term in\n" +"polynomial expansion of the expression (1 + x)**n.\n" +"\n" +"Raises TypeError if the arguments are not integers.\n" +"Raises ValueError if the arguments are negative or if k > n."); + +#define MATH_COMB_METHODDEF \ + {"comb", (PyCFunction)(void(*)(void))math_comb, METH_FASTCALL|METH_KEYWORDS, math_comb__doc__}, + +static PyObject * +math_comb_impl(PyObject *module, PyObject *n, PyObject *k); + +static PyObject * +math_comb(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"n", "k", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "comb", 0}; + PyObject *argsbuf[2]; + PyObject *n; + PyObject *k; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyLong_Check(args[0])) { + _PyArg_BadArgument("comb", 1, "int", args[0]); + goto exit; + } + n = args[0]; + if (!PyLong_Check(args[1])) { + _PyArg_BadArgument("comb", 2, "int", args[1]); + goto exit; + } + k = args[1]; + return_value = math_comb_impl(module, n, k); + +exit: + return return_value; +} +/*[clinic end generated code: output=00aa76356759617a input=a9049054013a1b77]*/ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index a153e984ca59..007a8801429c 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -2998,6 +2998,126 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) } +/*[clinic input] +math.comb + + n: object(subclass_of='&PyLong_Type') + k: object(subclass_of='&PyLong_Type') + +Number of ways to choose *k* items from *n* items without repetition and without order. + +Also called the binomial coefficient. It is mathematically equal to the expression +n! / (k! * (n - k)!). It is equivalent to the coefficient of k-th term in +polynomial expansion of the expression (1 + x)**n. + +Raises TypeError if the arguments are not integers. +Raises ValueError if the arguments are negative or if k > n. + +[clinic start generated code]*/ + +static PyObject * +math_comb_impl(PyObject *module, PyObject *n, PyObject *k) +/*[clinic end generated code: output=bd2cec8d854f3493 input=565f340f98efb5b5]*/ +{ + PyObject *val = NULL, + *temp_obj1 = NULL, + *temp_obj2 = NULL, + *dump_var = NULL; + int overflow, cmp; + long long i, terms; + + cmp = PyObject_RichCompareBool(n, k, Py_LT); + if (cmp < 0) { + goto fail_comb; + } + else if (cmp > 0) { + PyErr_Format(PyExc_ValueError, + "n must be an integer greater than or equal to k"); + goto fail_comb; + } + + /* b = min(b, a - b) */ + dump_var = PyNumber_Subtract(n, k); + if (dump_var == NULL) { + goto fail_comb; + } + cmp = PyObject_RichCompareBool(k, dump_var, Py_GT); + if (cmp < 0) { + goto fail_comb; + } + else if (cmp > 0) { + k = dump_var; + dump_var = NULL; + } + else { + Py_DECREF(dump_var); + dump_var = NULL; + } + + terms = PyLong_AsLongLongAndOverflow(k, &overflow); + if (terms < 0 && PyErr_Occurred()) { + goto fail_comb; + } + else if (overflow > 0) { + PyErr_Format(PyExc_OverflowError, + "minimum(n - k, k) must not exceed %lld", + LLONG_MAX); + goto fail_comb; + } + else if (overflow < 0 || terms < 0) { + PyErr_Format(PyExc_ValueError, + "k must be a positive integer"); + goto fail_comb; + } + + if (terms == 0) { + return PyNumber_Long(_PyLong_One); + } + + val = PyNumber_Long(n); + for (i = 1; i < terms; ++i) { + temp_obj1 = PyLong_FromSsize_t(i); + if (temp_obj1 == NULL) { + goto fail_comb; + } + temp_obj2 = PyNumber_Subtract(n, temp_obj1); + if (temp_obj2 == NULL) { + goto fail_comb; + } + dump_var = val; + val = PyNumber_Multiply(val, temp_obj2); + if (val == NULL) { + goto fail_comb; + } + Py_DECREF(dump_var); + dump_var = NULL; + Py_DECREF(temp_obj2); + temp_obj2 = PyLong_FromUnsignedLongLong((unsigned long long)(i + 1)); + if (temp_obj2 == NULL) { + goto fail_comb; + } + dump_var = val; + val = PyNumber_FloorDivide(val, temp_obj2); + if (val == NULL) { + goto fail_comb; + } + Py_DECREF(dump_var); + Py_DECREF(temp_obj1); + Py_DECREF(temp_obj2); + } + + return val; + +fail_comb: + Py_XDECREF(val); + Py_XDECREF(dump_var); + Py_XDECREF(temp_obj1); + Py_XDECREF(temp_obj2); + + return NULL; +} + + static PyMethodDef math_methods[] = { {"acos", math_acos, METH_O, math_acos_doc}, {"acosh", math_acosh, METH_O, math_acosh_doc}, @@ -3047,6 +3167,7 @@ static PyMethodDef math_methods[] = { {"tanh", math_tanh, METH_O, math_tanh_doc}, MATH_TRUNC_METHODDEF MATH_PROD_METHODDEF + MATH_COMB_METHODDEF {NULL, NULL} /* sentinel */ }; [View Less]

1 0

bpo-18911: clarify that the minidom XML writer receives texts but not bytes (GH-13718)
by Stefan Behnel June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/18e23f227be59241cbb1eeb6d6669771dd… commit: 18e23f227be59241cbb1eeb6d6669771dd7275fb branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Stefan Behnel <stefan_ml(a)behnel.de> date: 2019-06-01T08:58:54+02:00 summary: bpo-18911: clarify that the minidom XML writer receives texts but not bytes (GH-13718) (cherry picked from commit 5ac0b988fd5f1428efe35329c531c7b5c74d37f6) Co-authored-by: … [View More]

1 0

bpo-18911: clarify that the minidom XML writer receives texts but not bytes (GH-13352)
by Stefan Behnel June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/5ac0b988fd5f1428efe35329c531c7b5c7… commit: 5ac0b988fd5f1428efe35329c531c7b5c74d37f6 branch: master author: Windson yang <wiwindson(a)outlook.com> committer: Stefan Behnel <stefan_ml(a)behnel.de> date: 2019-06-01T08:33:16+02:00 summary: bpo-18911: clarify that the minidom XML writer receives texts but not bytes (GH-13352) files: M Doc/library/xml.dom.minidom.rst diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst … [View More]

1 0

Add option to trace to run modules (GH-5134)
by Nick Coghlan June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/354227a1e90036d8c1481a211746de912c… commit: 354227a1e90036d8c1481a211746de912c6c7c33 branch: master author: Mario Corchero <mariocj89(a)gmail.com> committer: Nick Coghlan <ncoghlan(a)gmail.com> date: 2019-06-01T14:49:09+10:00 summary: Add option to trace to run modules (GH-5134) Adds a new option in trace that allows tracing runnable modules. It is exposed as `--module module_name` as `-m` is already in use for another argument. files: A … [View More]Misc/NEWS.d/next/Library/2018-01-07-21-04-50.bpo-32515.D8_Wcb.rst M Doc/library/trace.rst M Lib/test/test_trace.py M Lib/trace.py diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index 5cb7029adf5e..85fec6830006 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -42,6 +42,9 @@ all Python modules imported during the execution into the current directory. Display the version of the module and exit. +.. versionadded:: 3.8 + Added ``--module`` option that allows to run an executable module. + Main options ^^^^^^^^^^^^ diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index afe790267661..4bc21eae02ce 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -474,7 +474,7 @@ class TestCommandLine(unittest.TestCase): def test_failures(self): _errors = ( - (b'filename is missing: required with the main options', '-l', '-T'), + (b'progname is missing: required with the main options', '-l', '-T'), (b'cannot specify both --listfuncs and (--trace or --count)', '-lc'), (b'argument -R/--no-report: not allowed with argument -r/--report', '-rR'), (b'must specify one of --trace, --count, --report, --listfuncs, or --trackcalls', '-g'), @@ -524,5 +524,10 @@ def f(): self.assertIn('lines cov% module (path)', stdout) self.assertIn(f'6 100% {TESTFN} ({filename})', stdout) + def test_run_as_module(self): + assert_python_ok('-m', 'trace', '-l', '--module', 'timeit', '-n', '1') + assert_python_failure('-m', 'trace', '-l', '--module', 'not_a_module_zzz') + + if __name__ == '__main__': unittest.main() diff --git a/Lib/trace.py b/Lib/trace.py index 63008a134a8a..62325d3f238a 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -666,7 +666,9 @@ def main(): help='Ignore files in the given directory ' '(multiple directories can be joined by os.pathsep).') - parser.add_argument('filename', nargs='?', + parser.add_argument('--module', action='store_true', default=False, + help='Trace a module. ') + parser.add_argument('progname', nargs='?', help='file to run as main program') parser.add_argument('arguments', nargs=argparse.REMAINDER, help='arguments to the program') @@ -704,26 +706,40 @@ def parse_ignore_dir(s): if opts.summary and not opts.count: parser.error('--summary can only be used with --count or --report') - if opts.filename is None: - parser.error('filename is missing: required with the main options') - - sys.argv = [opts.filename, *opts.arguments] - sys.path[0] = os.path.dirname(opts.filename) + if opts.progname is None: + parser.error('progname is missing: required with the main options') t = Trace(opts.count, opts.trace, countfuncs=opts.listfuncs, countcallers=opts.trackcalls, ignoremods=opts.ignore_module, ignoredirs=opts.ignore_dir, infile=opts.file, outfile=opts.file, timing=opts.timing) try: - with open(opts.filename) as fp: - code = compile(fp.read(), opts.filename, 'exec') - # try to emulate __main__ namespace as much as possible - globs = { - '__file__': opts.filename, - '__name__': '__main__', - '__package__': None, - '__cached__': None, - } + if opts.module: + import runpy + module_name = opts.progname + mod_name, mod_spec, code = runpy._get_module_details(module_name) + sys.argv = [code.co_filename, *opts.arguments] + globs = { + '__name__': '__main__', + '__file__': code.co_filename, + '__package__': mod_spec.parent, + '__loader__': mod_spec.loader, + '__spec__': mod_spec, + '__cached__': None, + } + else: + sys.argv = [opts.progname, *opts.arguments] + sys.path[0] = os.path.dirname(opts.progname) + + with open(opts.progname) as fp: + code = compile(fp.read(), opts.progname, 'exec') + # try to emulate __main__ namespace as much as possible + globs = { + '__file__': opts.progname, + '__name__': '__main__', + '__package__': None, + '__cached__': None, + } t.runctx(code, globs, globs) except OSError as err: sys.exit("Cannot run file %r because: %s" % (sys.argv[0], err)) diff --git a/Misc/NEWS.d/next/Library/2018-01-07-21-04-50.bpo-32515.D8_Wcb.rst b/Misc/NEWS.d/next/Library/2018-01-07-21-04-50.bpo-32515.D8_Wcb.rst new file mode 100644 index 000000000000..ad585b3ab5ed --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-01-07-21-04-50.bpo-32515.D8_Wcb.rst @@ -0,0 +1 @@ +trace.py can now run modules via python3 -m trace -t --module module_name [View Less]

1 0

bpo-29984: Improve 'heapq' test coverage (GH-992)
by Raymond Hettinger June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/664fe3996f7e05ae351526f02b21504bb0… commit: 664fe3996f7e05ae351526f02b21504bb065bcf8 branch: master author: Rob Day <rkd(a)rkd.me.uk> committer: Raymond Hettinger <rhettinger(a)users.noreply.github.com> date: 2019-05-31T21:13:57-07:00 summary: bpo-29984: Improve 'heapq' test coverage (GH-992) files: M Lib/heapq.py M Lib/test/test_heapq.py diff --git a/Lib/heapq.py b/Lib/heapq.py index 0e3555cf9118..fabefd87f8bf 100644 --- a/Lib/heapq.py +++… [View More] b/Lib/heapq.py @@ -597,5 +597,5 @@ def nlargest(n, iterable, key=None): if __name__ == "__main__": - import doctest - print(doctest.testmod()) + import doctest # pragma: no cover + print(doctest.testmod()) # pragma: no cover diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py index 2f8c648d84a5..6c20b6297dfc 100644 --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -2,6 +2,7 @@ import random import unittest +import doctest from test import support from unittest import TestCase, skipUnless @@ -26,6 +27,23 @@ def test_c_functions(self): self.assertEqual(getattr(c_heapq, fname).__module__, '_heapq') +def load_tests(loader, tests, ignore): + # The 'merge' function has examples in its docstring which we should test + # with 'doctest'. + # + # However, doctest can't easily find all docstrings in the module (loading + # it through import_fresh_module seems to confuse it), so we specifically + # create a finder which returns the doctests from the merge method. + + class HeapqMergeDocTestFinder: + def find(self, *args, **kwargs): + dtf = doctest.DocTestFinder() + return dtf.find(py_heapq.merge) + + tests.addTests(doctest.DocTestSuite(py_heapq, + test_finder=HeapqMergeDocTestFinder())) + return tests + class TestHeap: def test_push_pop(self): @@ -135,6 +153,13 @@ def test_heappushpop(self): x = self.module.heappushpop(h, 11) self.assertEqual((h, x), ([11], 10)) + def test_heappop_max(self): + # _heapop_max has an optimization for one-item lists which isn't + # covered in other tests, so test that case explicitly here + h = [3, 2] + self.assertEqual(self.module._heappop_max(h), 3) + self.assertEqual(self.module._heappop_max(h), 2) + def test_heapsort(self): # Exercise everything with repeated heapsort checks for trial in range(100): @@ -168,6 +193,12 @@ def test_merge(self): list(self.module.merge(*seqs, key=key, reverse=reverse))) self.assertEqual(list(self.module.merge()), []) + def test_empty_merges(self): + # Merging two empty lists (with or without a key) should produce + # another empty list. + self.assertEqual(list(self.module.merge([], [])), []) + self.assertEqual(list(self.module.merge([], [], key=lambda: 6)), []) + def test_merge_does_not_suppress_index_error(self): # Issue 19018: Heapq.merge suppresses IndexError from user generator def iterable(): [View Less]

1 0

Improve docstring of list.sort (GH-8516)
by Raymond Hettinger June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/5c22476c01622f11b7745ee693f8b296a9… commit: 5c22476c01622f11b7745ee693f8b296a9d6a761 branch: master author: Tim Hoffmann <2836374+timhoffm(a)users.noreply.github.com> committer: Raymond Hettinger <rhettinger(a)users.noreply.github.com> date: 2019-05-31T21:10:02-07:00 summary: Improve docstring of list.sort (GH-8516) files: M Objects/clinic/listobject.c.h M Objects/listobject.c diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/… [View More]listobject.c.h index 7b8e2d9905f9..57f0a48eb083 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -156,7 +156,15 @@ PyDoc_STRVAR(list_sort__doc__, "sort($self, /, *, key=None, reverse=False)\n" "--\n" "\n" -"Stable sort *IN PLACE*."); +"Sort the list in ascending order and return None.\n" +"\n" +"The sort is in-place (i.e. the list itself is modified) and stable (i.e. the\n" +"order of two equal elements is maintained).\n" +"\n" +"If a key function is given, apply it once to each list item and sort them,\n" +"ascending or descending, according to their function values.\n" +"\n" +"The reverse flag can be set to sort in descending order."); #define LIST_SORT_METHODDEF \ {"sort", (PyCFunction)(void(*)(void))list_sort, METH_FASTCALL|METH_KEYWORDS, list_sort__doc__}, @@ -359,4 +367,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=d1d5078edb7d3cf4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=73718c0c33798c62 input=a9049054013a1b77]*/ diff --git a/Objects/listobject.c b/Objects/listobject.c index 233f13dbab0e..f8bf45e5f8cd 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2197,12 +2197,20 @@ list.sort key as keyfunc: object = None reverse: bool(accept={int}) = False -Stable sort *IN PLACE*. +Sort the list in ascending order and return None. + +The sort is in-place (i.e. the list itself is modified) and stable (i.e. the +order of two equal elements is maintained). + +If a key function is given, apply it once to each list item and sort them, +ascending or descending, according to their function values. + +The reverse flag can be set to sort in descending order. [clinic start generated code]*/ static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=b0fcf743982c5b90]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=cb56cd179a713060]*/ { MergeState ms; Py_ssize_t nremaining; [View Less]

1 0

bpo-36818: Add PyInterpreterState.runtime field. (gh-13129)
by Eric Snow June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/396e0a8d9dc65453cb9d53500d0a620602… commit: 396e0a8d9dc65453cb9d53500d0a620602656cfe branch: master author: Eric Snow <ericsnowcurrently(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-05-31T21:16:47-06:00 summary: bpo-36818: Add PyInterpreterState.runtime field. (gh-13129) https://bugs.python.org/issue36818 files: A Misc/NEWS.d/next/Core and Builtins/2019-05-06-14-46-48.bpo-36818.5UDDLj.rst M Include/cpython/pystate.h M … [View More]Include/internal/pycore_object.h M Include/internal/pycore_pylifecycle.h M Include/internal/pycore_pystate.h M Modules/_threadmodule.c M Python/ceval.c M Python/import.c M Python/pylifecycle.c M Python/pystate.c M Python/sysmodule.c diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 94b0809cd4f0..74e7fc96bec9 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -110,9 +110,9 @@ struct _ts { * if the thread holds the last reference to the lock, decref'ing the * lock will delete the lock, and that may trigger arbitrary Python code * if there's a weakref, with a callback, to the lock. But by this time - * _PyRuntime.gilstate.tstate_current is already NULL, so only the simplest - * of C code can be allowed to run (in particular it must not be possible to - * release the GIL). + * _PyRuntimeState.gilstate.tstate_current is already NULL, so only the + * simplest of C code can be allowed to run (in particular it must not be + * possible to release the GIL). * So instead of holding the lock directly, the tstate holds a weakref to * the lock: that's the value of on_delete_data below. Decref'ing a * weakref is harmless. diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 81548f819198..1c5beb01f458 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -19,9 +19,10 @@ PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content); * NB: While the object is tracked by the collector, it must be safe to call the * ob_traverse method. * - * Internal note: _PyRuntime.gc.generation0->_gc_prev doesn't have any bit flags - * because it's not object header. So we don't use _PyGCHead_PREV() and - * _PyGCHead_SET_PREV() for it to avoid unnecessary bitwise operations. + * Internal note: _PyRuntimeState.gc.generation0->_gc_prev doesn't have + * any bit flags because it's not object header. So we don't use + * _PyGCHead_PREV() and _PyGCHead_SET_PREV() for it to avoid unnecessary + * bitwise operations. * * The PyObject_GC_Track() function is the public version of this macro. */ diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 8a692ea16495..e30341710c20 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -39,13 +39,10 @@ extern PyStatus _PyFaulthandler_Init(int enable); extern int _PyTraceMalloc_Init(int enable); extern PyObject * _PyBuiltin_Init(void); extern PyStatus _PySys_Create( - _PyRuntimeState *runtime, PyInterpreterState *interp, PyObject **sysmod_p); extern PyStatus _PySys_SetPreliminaryStderr(PyObject *sysdict); -extern int _PySys_InitMain( - _PyRuntimeState *runtime, - PyInterpreterState *interp); +extern int _PySys_InitMain(PyInterpreterState *interp); extern PyStatus _PyImport_Init(PyInterpreterState *interp); extern PyStatus _PyExc_Init(void); extern PyStatus _PyErr_Init(void); @@ -86,10 +83,7 @@ extern void _PyHash_Fini(void); extern int _PyTraceMalloc_Fini(void); extern void _PyWarnings_Fini(PyInterpreterState *interp); -extern void _PyGILState_Init( - _PyRuntimeState *runtime, - PyInterpreterState *interp, - PyThreadState *tstate); +extern void _PyGILState_Init(PyThreadState *tstate); extern void _PyGILState_Fini(_PyRuntimeState *runtime); PyAPI_FUNC(void) _PyGC_DumpShutdownStats(_PyRuntimeState *runtime); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 3ab4009770c9..520a74b8a61f 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -19,6 +19,9 @@ extern "C" { #include "pycore_pymem.h" #include "pycore_warnings.h" +// forward +struct pyruntimestate; + /* ceval state */ @@ -68,6 +71,7 @@ struct _is { struct _is *next; struct _ts *tstate_head; + struct pyruntimestate *runtime; int64_t id; int64_t id_refcount; @@ -296,12 +300,8 @@ PyAPI_FUNC(void) _PyRuntime_Finalize(void); /* Other */ -PyAPI_FUNC(void) _PyThreadState_Init( - _PyRuntimeState *runtime, - PyThreadState *tstate); -PyAPI_FUNC(void) _PyThreadState_DeleteExcept( - _PyRuntimeState *runtime, - PyThreadState *tstate); +PyAPI_FUNC(void) _PyThreadState_Init(PyThreadState *tstate); +PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate); PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap( struct _gilstate_runtime_state *gilstate, diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-06-14-46-48.bpo-36818.5UDDLj.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-06-14-46-48.bpo-36818.5UDDLj.rst new file mode 100644 index 000000000000..bb6c56a628e9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-06-14-46-48.bpo-36818.5UDDLj.rst @@ -0,0 +1 @@ +Add PyInterpreterState.runtime (and use it). diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index d5e40ef999e3..099afd86d055 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -996,7 +996,7 @@ t_bootstrap(void *boot_raw) tstate = boot->tstate; tstate->thread_id = PyThread_get_thread_ident(); - _PyThreadState_Init(&_PyRuntime, tstate); + _PyThreadState_Init(tstate); PyEval_AcquireThread(tstate); tstate->interp->num_threads++; res = PyObject_Call(boot->func, boot->args, boot->keyw); diff --git a/Python/ceval.c b/Python/ceval.c index 71e6eb8ebcfd..f9ff4e09f17e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -217,8 +217,9 @@ _PyEval_FiniThreads(struct _ceval_runtime_state *ceval) } static inline void -exit_thread_if_finalizing(_PyRuntimeState *runtime, PyThreadState *tstate) +exit_thread_if_finalizing(PyThreadState *tstate) { + _PyRuntimeState *runtime = tstate->interp->runtime; /* _Py_Finalizing is protected by the GIL */ if (runtime->finalizing != NULL && !_Py_CURRENTLY_FINALIZING(runtime, tstate)) { drop_gil(&runtime->ceval, tstate); @@ -236,7 +237,7 @@ PyEval_AcquireLock(void) Py_FatalError("PyEval_AcquireLock: current thread state is NULL"); } take_gil(ceval, tstate); - exit_thread_if_finalizing(runtime, tstate); + exit_thread_if_finalizing(tstate); } void @@ -257,14 +258,15 @@ PyEval_AcquireThread(PyThreadState *tstate) if (tstate == NULL) { Py_FatalError("PyEval_AcquireThread: NULL new thread state"); } + assert(tstate->interp != NULL); - _PyRuntimeState *runtime = &_PyRuntime; + _PyRuntimeState *runtime = tstate->interp->runtime; struct _ceval_runtime_state *ceval = &runtime->ceval; /* Check someone has called PyEval_InitThreads() to create the lock */ assert(gil_created(&ceval->gil)); take_gil(ceval, tstate); - exit_thread_if_finalizing(runtime, tstate); + exit_thread_if_finalizing(tstate); if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) { Py_FatalError("PyEval_AcquireThread: non-NULL old thread state"); } @@ -276,8 +278,9 @@ PyEval_ReleaseThread(PyThreadState *tstate) if (tstate == NULL) { Py_FatalError("PyEval_ReleaseThread: NULL thread state"); } + assert(tstate->interp != NULL); - _PyRuntimeState *runtime = &_PyRuntime; + _PyRuntimeState *runtime = tstate->interp->runtime; PyThreadState *new_tstate = _PyThreadState_Swap(&runtime->gilstate, NULL); if (new_tstate != tstate) { Py_FatalError("PyEval_ReleaseThread: wrong thread state"); @@ -308,7 +311,7 @@ _PyEval_ReInitThreads(_PyRuntimeState *runtime) } /* Destroy all threads except the current one */ - _PyThreadState_DeleteExcept(runtime, current_tstate); + _PyThreadState_DeleteExcept(current_tstate); } /* This function is used to signal that async exceptions are waiting to be @@ -337,17 +340,18 @@ PyEval_SaveThread(void) void PyEval_RestoreThread(PyThreadState *tstate) { - _PyRuntimeState *runtime = &_PyRuntime; - struct _ceval_runtime_state *ceval = &runtime->ceval; - if (tstate == NULL) { Py_FatalError("PyEval_RestoreThread: NULL tstate"); } + assert(tstate->interp != NULL); + + _PyRuntimeState *runtime = tstate->interp->runtime; + struct _ceval_runtime_state *ceval = &runtime->ceval; assert(gil_created(&ceval->gil)); int err = errno; take_gil(ceval, tstate); - exit_thread_if_finalizing(runtime, tstate); + exit_thread_if_finalizing(tstate); errno = err; _PyThreadState_Swap(&runtime->gilstate, tstate); @@ -1141,7 +1145,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) take_gil(ceval, tstate); /* Check if we should make a quick exit. */ - exit_thread_if_finalizing(runtime, tstate); + exit_thread_if_finalizing(tstate); if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) { Py_FatalError("ceval: orphan tstate"); diff --git a/Python/import.c b/Python/import.c index ab7db6bc17f6..68d1f4003abc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -541,7 +541,8 @@ PyImport_Cleanup(void) _PyGC_CollectNoFail(); /* Dump GC stats before it's too late, since it uses the warnings machinery. */ - _PyGC_DumpShutdownStats(&_PyRuntime); + _PyRuntimeState *runtime = interp->runtime; + _PyGC_DumpShutdownStats(runtime); /* Now, if there are any modules left alive, clear their globals to minimize potential leaks. All C extension modules actually end diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 10a28813faa8..6590ef8e9a27 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -545,7 +545,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, _PyEval_FiniThreads(&runtime->ceval); /* Auto-thread-state API */ - _PyGILState_Init(runtime, interp, tstate); + _PyGILState_Init(tstate); /* Create the GIL */ PyEval_InitThreads(); @@ -683,7 +683,7 @@ pyinit_config(_PyRuntimeState *runtime, } PyObject *sysmod; - status = _PySys_Create(runtime, interp, &sysmod); + status = _PySys_Create(interp, &sysmod); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -892,8 +892,9 @@ _Py_ReconfigureMainInterpreter(PyInterpreterState *interp) * non-zero return code. */ static PyStatus -pyinit_main(_PyRuntimeState *runtime, PyInterpreterState *interp) +pyinit_main(PyInterpreterState *interp) { + _PyRuntimeState *runtime = interp->runtime; if (!runtime->core_initialized) { return _PyStatus_ERR("runtime core not initialized"); } @@ -919,7 +920,7 @@ pyinit_main(_PyRuntimeState *runtime, PyInterpreterState *interp) return _PyStatus_ERR("can't initialize time"); } - if (_PySys_InitMain(runtime, interp) < 0) { + if (_PySys_InitMain(interp) < 0) { return _PyStatus_ERR("can't finish initializing sys"); } @@ -999,7 +1000,7 @@ _Py_InitializeMain(void) _PyRuntimeState *runtime = &_PyRuntime; PyInterpreterState *interp = _PyRuntimeState_GetThreadState(runtime)->interp; - return pyinit_main(runtime, interp); + return pyinit_main(interp); } @@ -1026,7 +1027,7 @@ Py_InitializeFromConfig(const PyConfig *config) config = &interp->config; if (config->_init_main) { - status = pyinit_main(runtime, interp); + status = pyinit_main(interp); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -1453,7 +1454,7 @@ new_interpreter(PyThreadState **tstate_p) } Py_INCREF(interp->sysdict); PyDict_SetItemString(interp->sysdict, "modules", modules); - if (_PySys_InitMain(runtime, interp) < 0) { + if (_PySys_InitMain(interp) < 0) { return _PyStatus_ERR("can't finish initializing sys"); } } diff --git a/Python/pystate.c b/Python/pystate.c index 833e0fb30dcb..2b7db0e48deb 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -39,7 +39,6 @@ extern "C" { /* Forward declarations */ static PyThreadState *_PyGILState_GetThisThreadState(struct _gilstate_runtime_state *gilstate); -static void _PyThreadState_Delete(_PyRuntimeState *runtime, PyThreadState *tstate); static PyStatus @@ -192,6 +191,8 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime) PyInterpreterState * PyInterpreterState_New(void) { + _PyRuntimeState *runtime = &_PyRuntime; + if (PySys_Audit("cpython.PyInterpreterState_New", NULL) < 0) { return NULL; } @@ -202,6 +203,9 @@ PyInterpreterState_New(void) } memset(interp, 0, sizeof(*interp)); + + interp->runtime = runtime; + interp->id_refcount = -1; interp->check_interval = 100; @@ -223,7 +227,6 @@ PyInterpreterState_New(void) #endif #endif - _PyRuntimeState *runtime = &_PyRuntime; struct pyinterpreters *interpreters = &runtime->interpreters; HEAD_LOCK(runtime); @@ -257,9 +260,11 @@ PyInterpreterState_New(void) } -static void -_PyInterpreterState_Clear(_PyRuntimeState *runtime, PyInterpreterState *interp) +void +PyInterpreterState_Clear(PyInterpreterState *interp) { + _PyRuntimeState *runtime = interp->runtime; + if (PySys_Audit("cpython.PyInterpreterState_Clear", NULL) < 0) { PyErr_Clear(); } @@ -297,31 +302,25 @@ _PyInterpreterState_Clear(_PyRuntimeState *runtime, PyInterpreterState *interp) // objects have been cleaned up at the point. } -void -PyInterpreterState_Clear(PyInterpreterState *interp) -{ - _PyInterpreterState_Clear(&_PyRuntime, interp); -} - static void -zapthreads(_PyRuntimeState *runtime, PyInterpreterState *interp) +zapthreads(PyInterpreterState *interp) { - PyThreadState *p; + PyThreadState *ts; /* No need to lock the mutex here because this should only happen when the threads are all really dead (XXX famous last words). */ - while ((p = interp->tstate_head) != NULL) { - _PyThreadState_Delete(runtime, p); + while ((ts = interp->tstate_head) != NULL) { + PyThreadState_Delete(ts); } } -static void -_PyInterpreterState_Delete(_PyRuntimeState *runtime, - PyInterpreterState *interp) +void +PyInterpreterState_Delete(PyInterpreterState *interp) { + _PyRuntimeState *runtime = interp->runtime; struct pyinterpreters *interpreters = &runtime->interpreters; - zapthreads(runtime, interp); + zapthreads(interp); HEAD_LOCK(runtime); PyInterpreterState **p; for (p = &interpreters->head; ; p = &(*p)->next) { @@ -350,13 +349,6 @@ _PyInterpreterState_Delete(_PyRuntimeState *runtime, } -void -PyInterpreterState_Delete(PyInterpreterState *interp) -{ - _PyInterpreterState_Delete(&_PyRuntime, interp); -} - - /* * Delete all interpreter states except the main interpreter. If there * is a current interpreter state, it *must* be the main interpreter. @@ -383,8 +375,8 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) continue; } - _PyInterpreterState_Clear(runtime, interp); // XXX must activate? - zapthreads(runtime, interp); + PyInterpreterState_Clear(interp); // XXX must activate? + zapthreads(interp); if (interp->id_mutex != NULL) { PyThread_free_lock(interp->id_mutex); } @@ -497,7 +489,8 @@ _PyInterpreterState_IDDecref(PyInterpreterState *interp) if (interp->id_mutex == NULL) { return; } - struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; + _PyRuntimeState *runtime = interp->runtime; + struct _gilstate_runtime_state *gilstate = &runtime->gilstate; PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK); assert(interp->id_refcount != 0); interp->id_refcount -= 1; @@ -559,7 +552,7 @@ threadstate_getframe(PyThreadState *self) static PyThreadState * new_threadstate(PyInterpreterState *interp, int init) { - _PyRuntimeState *runtime = &_PyRuntime; + _PyRuntimeState *runtime = interp->runtime; PyThreadState *tstate = (PyThreadState *)PyMem_RawMalloc(sizeof(PyThreadState)); if (tstate == NULL) { return NULL; @@ -615,7 +608,7 @@ new_threadstate(PyInterpreterState *interp, int init) tstate->id = ++interp->tstate_next_unique_id; if (init) { - _PyThreadState_Init(runtime, tstate); + _PyThreadState_Init(tstate); } HEAD_LOCK(runtime); @@ -642,8 +635,9 @@ _PyThreadState_Prealloc(PyInterpreterState *interp) } void -_PyThreadState_Init(_PyRuntimeState *runtime, PyThreadState *tstate) +_PyThreadState_Init(PyThreadState *tstate) { + _PyRuntimeState *runtime = tstate->interp->runtime; _PyGILState_NoteThreadState(&runtime->gilstate, tstate); } @@ -808,7 +802,7 @@ PyThreadState_Clear(PyThreadState *tstate) /* Common code for PyThreadState_Delete() and PyThreadState_DeleteCurrent() */ static void -tstate_delete_common(_PyRuntimeState *runtime, PyThreadState *tstate) +tstate_delete_common(PyThreadState *tstate) { if (tstate == NULL) { Py_FatalError("PyThreadState_Delete: NULL tstate"); @@ -817,6 +811,7 @@ tstate_delete_common(_PyRuntimeState *runtime, PyThreadState *tstate) if (interp == NULL) { Py_FatalError("PyThreadState_Delete: NULL interp"); } + _PyRuntimeState *runtime = interp->runtime; HEAD_LOCK(runtime); if (tstate->prev) tstate->prev->next = tstate->next; @@ -832,9 +827,10 @@ tstate_delete_common(_PyRuntimeState *runtime, PyThreadState *tstate) } -static void -_PyThreadState_Delete(_PyRuntimeState *runtime, PyThreadState *tstate) +void +PyThreadState_Delete(PyThreadState *tstate) { + _PyRuntimeState *runtime = tstate->interp->runtime; struct _gilstate_runtime_state *gilstate = &runtime->gilstate; if (tstate == _PyRuntimeGILState_GetThreadState(gilstate)) { Py_FatalError("PyThreadState_Delete: tstate is still current"); @@ -844,14 +840,7 @@ _PyThreadState_Delete(_PyRuntimeState *runtime, PyThreadState *tstate) { PyThread_tss_set(&gilstate->autoTSSkey, NULL); } - tstate_delete_common(runtime, tstate); -} - - -void -PyThreadState_Delete(PyThreadState *tstate) -{ - _PyThreadState_Delete(&_PyRuntime, tstate); + tstate_delete_common(tstate); } @@ -863,7 +852,7 @@ _PyThreadState_DeleteCurrent(_PyRuntimeState *runtime) if (tstate == NULL) Py_FatalError( "PyThreadState_DeleteCurrent: no current tstate"); - tstate_delete_common(runtime, tstate); + tstate_delete_common(tstate); if (gilstate->autoInterpreterState && PyThread_tss_get(&gilstate->autoTSSkey) == tstate) { @@ -888,9 +877,10 @@ PyThreadState_DeleteCurrent() * be kept in those other interpreteres. */ void -_PyThreadState_DeleteExcept(_PyRuntimeState *runtime, PyThreadState *tstate) +_PyThreadState_DeleteExcept(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; + _PyRuntimeState *runtime = interp->runtime; PyThreadState *p, *next, *garbage; HEAD_LOCK(runtime); /* Remove all thread states, except tstate, from the linked list of @@ -1129,8 +1119,9 @@ _PyThread_CurrentFrames(void) static int PyThreadState_IsCurrent(PyThreadState *tstate) { + _PyRuntimeState *runtime = tstate->interp->runtime; /* Must be the tstate for this thread */ - struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; + struct _gilstate_runtime_state *gilstate = &runtime->gilstate; assert(_PyGILState_GetThisThreadState(gilstate) == tstate); return tstate == _PyRuntimeGILState_GetThreadState(gilstate); } @@ -1139,12 +1130,14 @@ PyThreadState_IsCurrent(PyThreadState *tstate) Py_Initialize/Py_FinalizeEx */ void -_PyGILState_Init(_PyRuntimeState *runtime, - PyInterpreterState *interp, PyThreadState *tstate) +_PyGILState_Init(PyThreadState *tstate) { /* must init with valid states */ - assert(interp != NULL); assert(tstate != NULL); + PyInterpreterState *interp = tstate->interp; + assert(interp != NULL); + _PyRuntimeState *runtime = interp->runtime; + assert(runtime != NULL); struct _gilstate_runtime_state *gilstate = &runtime->gilstate; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 12b1bd7711d5..97bff94d8b41 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -120,8 +120,9 @@ should_audit(void) if (!ts) { return 0; } - PyInterpreterState *is = ts ? ts->interp : NULL; - return _PyRuntime.audit_hook_head + PyInterpreterState *is = ts->interp; + _PyRuntimeState *runtime = is->runtime; + return runtime->audit_hook_head || (is && is->audit_hooks) || PyDTrace_AUDIT_ENABLED(); } @@ -280,8 +281,8 @@ void _PySys_ClearAuditHooks(void) { PySys_Audit("cpython._PySys_ClearAuditHooks", NULL); PyErr_Clear(); - _Py_AuditHookEntry *e = _PyRuntime.audit_hook_head, *n; - _PyRuntime.audit_hook_head = NULL; + _Py_AuditHookEntry *e = runtime->audit_hook_head, *n; + runtime->audit_hook_head = NULL; while (e) { n = e->next; PyMem_RawFree(e); @@ -292,6 +293,7 @@ void _PySys_ClearAuditHooks(void) { int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) { + _PyRuntimeState *runtime = &_PyRuntime; /* Invoke existing audit hooks to allow them an opportunity to abort. */ /* Cannot invoke hooks until we are initialized */ if (Py_IsInitialized()) { @@ -305,10 +307,10 @@ PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) } } - _Py_AuditHookEntry *e = _PyRuntime.audit_hook_head; + _Py_AuditHookEntry *e = runtime->audit_hook_head; if (!e) { e = (_Py_AuditHookEntry*)PyMem_RawMalloc(sizeof(_Py_AuditHookEntry)); - _PyRuntime.audit_hook_head = e; + runtime->audit_hook_head = e; } else { while (e->next) e = e->next; @@ -2413,8 +2415,9 @@ static PyStructSequence_Desc flags_desc = { }; static PyObject* -make_flags(_PyRuntimeState *runtime, PyInterpreterState *interp) +make_flags(PyInterpreterState *interp) { + _PyRuntimeState *runtime = interp->runtime; int pos = 0; PyObject *seq; const PyPreConfig *preconfig = &runtime->preconfig; @@ -2633,8 +2636,7 @@ static struct PyModuleDef sysmodule = { } while (0) static PyStatus -_PySys_InitCore(_PyRuntimeState *runtime, PyInterpreterState *interp, - PyObject *sysdict) +_PySys_InitCore(PyInterpreterState *interp, PyObject *sysdict) { PyObject *version_info; int res; @@ -2728,7 +2730,7 @@ _PySys_InitCore(_PyRuntimeState *runtime, PyInterpreterState *interp, } } /* Set flags to their default values (updated by _PySys_InitMain()) */ - SET_SYS_FROM_STRING("flags", make_flags(runtime, interp)); + SET_SYS_FROM_STRING("flags", make_flags(interp)); #if defined(MS_WINDOWS) /* getwindowsversion */ @@ -2849,7 +2851,7 @@ sys_create_xoptions_dict(const PyConfig *config) int -_PySys_InitMain(_PyRuntimeState *runtime, PyInterpreterState *interp) +_PySys_InitMain(PyInterpreterState *interp) { PyObject *sysdict = interp->sysdict; const PyConfig *config = &interp->config; @@ -2903,7 +2905,7 @@ _PySys_InitMain(_PyRuntimeState *runtime, PyInterpreterState *interp) #undef SET_SYS_FROM_WSTR /* Set flags to their final values */ - SET_SYS_FROM_STRING_INT_RESULT("flags", make_flags(runtime, interp)); + SET_SYS_FROM_STRING_INT_RESULT("flags", make_flags(interp)); /* prevent user from creating new instances */ FlagsType.tp_init = NULL; FlagsType.tp_new = NULL; @@ -2970,8 +2972,7 @@ _PySys_SetPreliminaryStderr(PyObject *sysdict) /* Create sys module without all attributes: _PySys_InitMain() should be called later to add remaining attributes. */ PyStatus -_PySys_Create(_PyRuntimeState *runtime, PyInterpreterState *interp, - PyObject **sysmod_p) +_PySys_Create(PyInterpreterState *interp, PyObject **sysmod_p) { PyObject *modules = PyDict_New(); if (modules == NULL) { @@ -3000,7 +3001,7 @@ _PySys_Create(_PyRuntimeState *runtime, PyInterpreterState *interp, return status; } - status = _PySys_InitCore(runtime, interp, sysdict); + status = _PySys_InitCore(interp, sysdict); if (_PyStatus_EXCEPTION(status)) { return status; } [View Less]

1 0

bpo-37029: keep usable_arenas in sorted order without searching (#13612)
by Tim Peters June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/1c263e39c4ed28225a7dc8ca1f24953225… commit: 1c263e39c4ed28225a7dc8ca1f24953225ac48ca branch: master author: Tim Peters <tim.peters(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-05-31T21:16:04-05:00 summary: bpo-37029: keep usable_arenas in sorted order without searching (#13612) This adds a vector of "search fingers" so that usable_arenas can be kept in sorted order (by number of free pools) via constant-time operations … [View More]instead of linear search. This should reduce worst-case time for reclaiming a great many objects from O(A**2) to O(A), where A is the number of arenas. See bpo-37029. files: A Misc/NEWS.d/next/Core and Builtins/2019-05-28-17-02-46.bpo-37029.MxpgfJ.rst M Objects/obmalloc.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-28-17-02-46.bpo-37029.MxpgfJ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-28-17-02-46.bpo-37029.MxpgfJ.rst new file mode 100644 index 000000000000..c18f5d23eaa2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-28-17-02-46.bpo-37029.MxpgfJ.rst @@ -0,0 +1 @@ +Freeing a great many small objects could take time quadratic in the number of arenas, due to using linear search to keep ``obmalloc.c``'s list of usable arenas sorted by order of number of free memory pools. This is accomplished without search now, leaving the worst-case time linear in the number of arenas. For programs where this quite visibly matters (typically with more than 100 thousand small objects alive simultaneously), this can greatly reduce the time needed to release their memory. \ No newline at end of file diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index f54856dcfe71..fc7bef619946 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -918,6 +918,11 @@ static int running_on_valgrind = -1; #define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */ #define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK +#define MAX_POOLS_IN_ARENA (ARENA_SIZE / POOL_SIZE) +#if MAX_POOLS_IN_ARENA * POOL_SIZE != ARENA_SIZE +# error "arena size not an exact multiple of pool size" +#endif + /* * -- End of tunable settings section -- */ @@ -1155,6 +1160,18 @@ usable_arenas Note that an arena_object associated with an arena all of whose pools are currently in use isn't on either list. + +Changed in Python 3.8: keeping usable_arenas sorted by number of free pools +used to be done by one-at-a-time linear search when an arena's number of +free pools changed. That could, overall, consume time quadratic in the +number of arenas. That didn't really matter when there were only a few +hundred arenas (typical!), but could be a timing disaster when there were +hundreds of thousands. See bpo-37029. + +Now we have a vector of "search fingers" to eliminate the need to search: +nfp2lasta[nfp] returns the last ("rightmost") arena in usable_arenas +with nfp free pools. This is NULL if and only if there is no arena with +nfp free pools in usable_arenas. */ /* Array of objects used to track chunks of memory (arenas). */ @@ -1172,6 +1189,9 @@ static struct arena_object* unused_arena_objects = NULL; */ static struct arena_object* usable_arenas = NULL; +/* nfp2lasta[nfp] is the last arena in usable_arenas with nfp free pools */ +static struct arena_object* nfp2lasta[MAX_POOLS_IN_ARENA + 1] = { NULL }; + /* How many arena_objects do we initially allocate? * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the * `arenas` vector. @@ -1281,8 +1301,7 @@ new_arena(void) /* pool_address <- first pool-aligned address in the arena nfreepools <- number of whole pools that fit after alignment */ arenaobj->pool_address = (block*)arenaobj->address; - arenaobj->nfreepools = ARENA_SIZE / POOL_SIZE; - assert(POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE); + arenaobj->nfreepools = MAX_POOLS_IN_ARENA; excess = (uint)(arenaobj->address & POOL_SIZE_MASK); if (excess != 0) { --arenaobj->nfreepools; @@ -1478,22 +1497,32 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) } usable_arenas->nextarena = usable_arenas->prevarena = NULL; + assert(nfp2lasta[usable_arenas->nfreepools] == NULL); + nfp2lasta[usable_arenas->nfreepools] = usable_arenas; } assert(usable_arenas->address != 0); + /* This arena already had the smallest nfreepools value, so decreasing + * nfreepools doesn't change that, and we don't need to rearrange the + * usable_arenas list. However, if the arena becomes wholly allocated, + * we need to remove its arena_object from usable_arenas. + */ + assert(usable_arenas->nfreepools > 0); + if (nfp2lasta[usable_arenas->nfreepools] == usable_arenas) { + /* It's the last of this size, so there won't be any. */ + nfp2lasta[usable_arenas->nfreepools] = NULL; + } + /* If any free pools will remain, it will be the new smallest. */ + if (usable_arenas->nfreepools > 1) { + assert(nfp2lasta[usable_arenas->nfreepools - 1] == NULL); + nfp2lasta[usable_arenas->nfreepools - 1] = usable_arenas; + } + /* Try to get a cached free pool. */ pool = usable_arenas->freepools; if (pool != NULL) { /* Unlink from cached pools. */ usable_arenas->freepools = pool->nextpool; - - /* This arena already had the smallest nfreepools - * value, so decreasing nfreepools doesn't change - * that, and we don't need to rearrange the - * usable_arenas list. However, if the arena has - * become wholly allocated, we need to remove its - * arena_object from usable_arenas. - */ --usable_arenas->nfreepools; if (usable_arenas->nfreepools == 0) { /* Wholly allocated: remove. */ @@ -1501,7 +1530,6 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) assert(usable_arenas->nextarena == NULL || usable_arenas->nextarena->prevarena == usable_arenas); - usable_arenas = usable_arenas->nextarena; if (usable_arenas != NULL) { usable_arenas->prevarena = NULL; @@ -1709,7 +1737,23 @@ pymalloc_free(void *ctx, void *p) ao = &arenas[pool->arenaindex]; pool->nextpool = ao->freepools; ao->freepools = pool; - nf = ++ao->nfreepools; + nf = ao->nfreepools; + /* If this is the rightmost arena with this number of free pools, + * nfp2lasta[nf] needs to change. Caution: if nf is 0, there + * are no arenas in usable_arenas with that value. + */ + struct arena_object* lastnf = nfp2lasta[nf]; + assert((nf == 0 && lastnf == NULL) || + (nf > 0 && + lastnf != NULL && + lastnf->nfreepools == nf && + (lastnf->nextarena == NULL || + nf < lastnf->nextarena->nfreepools))); + if (lastnf == ao) { /* it is the rightmost */ + struct arena_object* p = ao->prevarena; + nfp2lasta[nf] = (p != NULL && p->nfreepools == nf) ? p : NULL; + } + ao->nfreepools = ++nf; /* All the rest is arena management. We just freed * a pool, and there are 4 cases for arena mgmt: @@ -1777,6 +1821,9 @@ pymalloc_free(void *ctx, void *p) usable_arenas->prevarena = ao; usable_arenas = ao; assert(usable_arenas->address != 0); + if (nfp2lasta[1] == NULL) { + nfp2lasta[1] = ao; + } goto success; } @@ -1788,14 +1835,23 @@ pymalloc_free(void *ctx, void *p) * a few un-scientific tests, it seems like this * approach allowed a lot more memory to be freed. */ - if (ao->nextarena == NULL || - nf <= ao->nextarena->nfreepools) { + /* If this is the only arena with nf, record that. */ + if (nfp2lasta[nf] == NULL) { + nfp2lasta[nf] = ao; + } /* else the rightmost with nf doesn't change */ + /* If this was the rightmost of the old size, it remains in place. */ + if (ao == lastnf) { /* Case 4. Nothing to do. */ goto success; } - /* Case 3: We have to move the arena towards the end - * of the list, because it has more free pools than - * the arena to its right. + /* If ao were the only arena in the list, the last block would have + * gotten us out. + */ + assert(ao->nextarena != NULL); + + /* Case 3: We have to move the arena towards the end of the list, + * because it has more free pools than the arena to its right. It needs + * to move to follow lastnf. * First unlink ao from usable_arenas. */ if (ao->prevarena != NULL) { @@ -1809,24 +1865,13 @@ pymalloc_free(void *ctx, void *p) usable_arenas = ao->nextarena; } ao->nextarena->prevarena = ao->prevarena; - - /* Locate the new insertion point by iterating over - * the list, using our nextarena pointer. - */ - while (ao->nextarena != NULL && nf > ao->nextarena->nfreepools) { - ao->prevarena = ao->nextarena; - ao->nextarena = ao->nextarena->nextarena; - } - - /* Insert ao at this point. */ - assert(ao->nextarena == NULL || ao->prevarena == ao->nextarena->prevarena); - assert(ao->prevarena->nextarena == ao->nextarena); - - ao->prevarena->nextarena = ao; + /* And insert after lastnf. */ + ao->prevarena = lastnf; + ao->nextarena = lastnf->nextarena; if (ao->nextarena != NULL) { ao->nextarena->prevarena = ao; } - + lastnf->nextarena = ao; /* Verify that the swaps worked. */ assert(ao->nextarena == NULL || nf <= ao->nextarena->nfreepools); assert(ao->prevarena == NULL || nf > ao->prevarena->nfreepools); [View Less]

1 0

bpo-12202: Properly check MsiSummaryInfoGetProperty() calls in msilib (GH-13711)
by Berker Peksag June 1, 2019

June 1, 2019

https://github.com/python/cpython/commit/549e55a3086d04c13da9b6f33214f63996… commit: 549e55a3086d04c13da9b6f33214f6399681292a branch: master author: Zackery Spytz <zspytz(a)gmail.com> committer: Berker Peksag <berker.peksag(a)gmail.com> date: 2019-06-01T03:16:20+03:00 summary: bpo-12202: Properly check MsiSummaryInfoGetProperty() calls in msilib (GH-13711) files: A Misc/NEWS.d/next/Library/2019-05-31-15-53-34.bpo-12202.nobzc9.rst M Lib/test/test_msilib.py M PC/_msi.c diff --git a/… [View More]

1 0