Mailman 3 June 2019 - Python-checkins

Call PyObject_GC_UnTrack in structseq dealloc (GH-13751)
by Pablo Galindo 02 Jun '19

02 Jun '19

https://github.com/python/cpython/commit/3caf4de6f05f68c3a175f4d8ce870d7a00… commit: 3caf4de6f05f68c3a175f4d8ce870d7a0016622a branch: master author: Pablo Galindo <Pablogsal(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T21:52:49+01:00 summary: Call PyObject_GC_UnTrack in structseq dealloc (GH-13751) files: M Objects/structseq.c diff --git a/Objects/structseq.c b/Objects/structseq.c index 3d857f734be8..2c25e1646a2a 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -77,6 +77,7 @@ structseq_dealloc(PyStructSequence *obj) { Py_ssize_t i, size; PyTypeObject *tp; + PyObject_GC_UnTrack(obj); tp = (PyTypeObject *) Py_TYPE(obj); size = REAL_SIZE(obj);

1 0

bpo-37124: Fix reference leak in test_msilib (GH-13750)
by Pablo Galindo 02 Jun '19

02 Jun '19

https://github.com/python/cpython/commit/c0295dba259accc4b247beb22a0b2cc2f3… commit: c0295dba259accc4b247beb22a0b2cc2f31d9850 branch: master author: Pablo Galindo <Pablogsal(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T21:36:21+01:00 summary: bpo-37124: Fix reference leak in test_msilib (GH-13750) files: M Lib/test/test_msilib.py diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py index fa0be581613d..f9bd0da7498e 100644 --- a/Lib/test/test_msilib.py +++ b/Lib/test/test_msilib.py @@ -87,6 +87,7 @@ def test_directory_start_component_keyfile(self): db, db_path = init_database() self.addCleanup(unlink, db_path) self.addCleanup(db.Close) + self.addCleanup(msilib._directories.clear) feature = msilib.Feature(db, 0, 'Feature', 'A feature', 'Python') cab = msilib.CAB('CAB') dir = msilib.Directory(db, cab, None, TESTFN, 'TARGETDIR',

1 0

bpo-35610: IDLE - Replace .context_use_ps1 with .prompt_last_line (GH-11307)
by Miss Islington (bot) 02 Jun '19

02 Jun '19

https://github.com/python/cpython/commit/b4e0bfd4778e142f037f50c19c4bb5bd12… commit: b4e0bfd4778e142f037f50c19c4bb5bd123b4641 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T12:35:53-07:00 summary: bpo-35610: IDLE - Replace .context_use_ps1 with .prompt_last_line (GH-11307) Changes in bpo- 31858 made the less informative 'context_use_ps1' redundant. (cherry picked from commit 6bdc4dee01788599808c7858e2fe9fdd72cf6792) Co-authored-by: Cheryl Sabella <cheryl.sabella(a)gmail.com> files: A Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst M Lib/idlelib/NEWS.txt M Lib/idlelib/editor.py M Lib/idlelib/hyperparser.py M Lib/idlelib/idle_test/test_autocomplete.py M Lib/idlelib/idle_test/test_hyperparser.py M Lib/idlelib/idle_test/test_parenmatch.py M Lib/idlelib/pyshell.py diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 808c236bb590..172fdc00d237 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,9 @@ Released on 2019-06-24? ====================================== +bpo-35610: Replace now redundant editor.context_use_ps1 with +.prompt_last_line. This finishes change started in bpo-31858. + bpo-32411: Stop sorting dict created with desired line order. bpo-37038: Make idlelib.run runnable; add test clause. diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 83260329640e..89b7239a96ea 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -228,10 +228,6 @@ def __init__(self, flist=None, filename=None, key=None, root=None): self.indentwidth = self.tabwidth self.set_notabs_indentwidth() - # If context_use_ps1 is true, parsing searches back for a ps1 line; - # else searches for a popular (if, def, ...) Python stmt. - self.context_use_ps1 = False - # When searching backwards for a reliable place to begin parsing, # first start num_context_lines[0] lines back, then # num_context_lines[1] lines back if that didn't work, and so on. @@ -1337,14 +1333,13 @@ def newline_and_indent_event(self, event): # open/close first need to find the last stmt lno = index2line(text.index('insert')) y = pyparse.Parser(self.indentwidth, self.tabwidth) - if not self.context_use_ps1: + if not self.prompt_last_line: for context in self.num_context_lines: startat = max(lno - context, 1) startatindex = repr(startat) + ".0" rawtext = text.get(startatindex, "insert") y.set_code(rawtext) bod = y.find_good_parse_start( - self.context_use_ps1, self._build_char_in_string_func(startatindex)) if bod is not None or startat == 1: break diff --git a/Lib/idlelib/hyperparser.py b/Lib/idlelib/hyperparser.py index 7e7e0ae80247..77baca782b3f 100644 --- a/Lib/idlelib/hyperparser.py +++ b/Lib/idlelib/hyperparser.py @@ -35,7 +35,7 @@ def index2line(index): return int(float(index)) lno = index2line(text.index(index)) - if not editwin.context_use_ps1: + if not editwin.prompt_last_line: for context in editwin.num_context_lines: startat = max(lno - context, 1) startatindex = repr(startat) + ".0" diff --git a/Lib/idlelib/idle_test/test_autocomplete.py b/Lib/idlelib/idle_test/test_autocomplete.py index 398cb359e093..6181b29ec250 100644 --- a/Lib/idlelib/idle_test/test_autocomplete.py +++ b/Lib/idlelib/idle_test/test_autocomplete.py @@ -19,7 +19,7 @@ def __init__(self, root, text): self.text = text self.indentwidth = 8 self.tabwidth = 8 - self.context_use_ps1 = True + self.prompt_last_line = '>>>' # Currently not used by autocomplete. class AutoCompleteTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_hyperparser.py b/Lib/idlelib/idle_test/test_hyperparser.py index 8dbfc63779d3..343843c4166e 100644 --- a/Lib/idlelib/idle_test/test_hyperparser.py +++ b/Lib/idlelib/idle_test/test_hyperparser.py @@ -11,7 +11,7 @@ def __init__(self, text): self.text = text self.indentwidth = 8 self.tabwidth = 8 - self.context_use_ps1 = True + self.prompt_last_line = '>>>' self.num_context_lines = 50, 500, 1000 _build_char_in_string_func = EditorWindow._build_char_in_string_func @@ -53,7 +53,7 @@ def setUp(self): def tearDown(self): self.text.delete('1.0', 'end') - self.editwin.context_use_ps1 = True + self.editwin.prompt_last_line = '>>>' def get_parser(self, index): """ @@ -71,7 +71,7 @@ def test_init(self): self.assertIn('precedes', str(ve.exception)) # test without ps1 - self.editwin.context_use_ps1 = False + self.editwin.prompt_last_line = '' # number of lines lesser than 50 p = self.get_parser('end') diff --git a/Lib/idlelib/idle_test/test_parenmatch.py b/Lib/idlelib/idle_test/test_parenmatch.py index f58819abf112..4a41d8433d54 100644 --- a/Lib/idlelib/idle_test/test_parenmatch.py +++ b/Lib/idlelib/idle_test/test_parenmatch.py @@ -17,7 +17,7 @@ def __init__(self, text): self.text = text self.indentwidth = 8 self.tabwidth = 8 - self.context_use_ps1 = True + self.prompt_last_line = '>>>' # Currently not used by parenmatch. class ParenMatchTest(unittest.TestCase): diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 2de42658b01c..6e0707d68bb6 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -881,7 +881,7 @@ def __init__(self, flist=None): self.usetabs = True # indentwidth must be 8 when using tabs. See note in EditorWindow: self.indentwidth = 8 - self.context_use_ps1 = True + self.sys_ps1 = sys.ps1 if hasattr(sys, 'ps1') else '>>> ' self.prompt_last_line = self.sys_ps1.split('\n')[-1] self.prompt = self.sys_ps1 # Changes when debug active diff --git a/Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst b/Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst new file mode 100644 index 000000000000..0042ab70497a --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst @@ -0,0 +1,2 @@ +Replace now redundant .context_use_ps1 with .prompt_last_line. This finishes +change started in bpo-31858.

1 0

bpo-35610: IDLE - Replace .context_use_ps1 with .prompt_last_line (GH-11307)
by Terry Jan Reedy 02 Jun '19

02 Jun '19

https://github.com/python/cpython/commit/6bdc4dee01788599808c7858e2fe9fdd72… commit: 6bdc4dee01788599808c7858e2fe9fdd72cf6792 branch: master author: Cheryl Sabella <cheryl.sabella(a)gmail.com> committer: Terry Jan Reedy <tjreedy(a)udel.edu> date: 2019-06-02T14:56:47-04:00 summary: bpo-35610: IDLE - Replace .context_use_ps1 with .prompt_last_line (GH-11307) Changes in bpo- 31858 made the less informative 'context_use_ps1' redundant. files: A Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst M Lib/idlelib/NEWS.txt M Lib/idlelib/editor.py M Lib/idlelib/hyperparser.py M Lib/idlelib/idle_test/test_autocomplete.py M Lib/idlelib/idle_test/test_hyperparser.py M Lib/idlelib/idle_test/test_parenmatch.py M Lib/idlelib/pyshell.py diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index b260d4e5ef1b..982af7767251 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,9 @@ Released on 2019-10-20? ====================================== +bpo-35610: Replace now redundant editor.context_use_ps1 with +.prompt_last_line. This finishes change started in bpo-31858. + bpo-32411: Stop sorting dict created with desired line order. bpo-37038: Make idlelib.run runnable; add test clause. diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 83260329640e..89b7239a96ea 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -228,10 +228,6 @@ def __init__(self, flist=None, filename=None, key=None, root=None): self.indentwidth = self.tabwidth self.set_notabs_indentwidth() - # If context_use_ps1 is true, parsing searches back for a ps1 line; - # else searches for a popular (if, def, ...) Python stmt. - self.context_use_ps1 = False - # When searching backwards for a reliable place to begin parsing, # first start num_context_lines[0] lines back, then # num_context_lines[1] lines back if that didn't work, and so on. @@ -1337,14 +1333,13 @@ def newline_and_indent_event(self, event): # open/close first need to find the last stmt lno = index2line(text.index('insert')) y = pyparse.Parser(self.indentwidth, self.tabwidth) - if not self.context_use_ps1: + if not self.prompt_last_line: for context in self.num_context_lines: startat = max(lno - context, 1) startatindex = repr(startat) + ".0" rawtext = text.get(startatindex, "insert") y.set_code(rawtext) bod = y.find_good_parse_start( - self.context_use_ps1, self._build_char_in_string_func(startatindex)) if bod is not None or startat == 1: break diff --git a/Lib/idlelib/hyperparser.py b/Lib/idlelib/hyperparser.py index 7e7e0ae80247..77baca782b3f 100644 --- a/Lib/idlelib/hyperparser.py +++ b/Lib/idlelib/hyperparser.py @@ -35,7 +35,7 @@ def index2line(index): return int(float(index)) lno = index2line(text.index(index)) - if not editwin.context_use_ps1: + if not editwin.prompt_last_line: for context in editwin.num_context_lines: startat = max(lno - context, 1) startatindex = repr(startat) + ".0" diff --git a/Lib/idlelib/idle_test/test_autocomplete.py b/Lib/idlelib/idle_test/test_autocomplete.py index 398cb359e093..6181b29ec250 100644 --- a/Lib/idlelib/idle_test/test_autocomplete.py +++ b/Lib/idlelib/idle_test/test_autocomplete.py @@ -19,7 +19,7 @@ def __init__(self, root, text): self.text = text self.indentwidth = 8 self.tabwidth = 8 - self.context_use_ps1 = True + self.prompt_last_line = '>>>' # Currently not used by autocomplete. class AutoCompleteTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_hyperparser.py b/Lib/idlelib/idle_test/test_hyperparser.py index 8dbfc63779d3..343843c4166e 100644 --- a/Lib/idlelib/idle_test/test_hyperparser.py +++ b/Lib/idlelib/idle_test/test_hyperparser.py @@ -11,7 +11,7 @@ def __init__(self, text): self.text = text self.indentwidth = 8 self.tabwidth = 8 - self.context_use_ps1 = True + self.prompt_last_line = '>>>' self.num_context_lines = 50, 500, 1000 _build_char_in_string_func = EditorWindow._build_char_in_string_func @@ -53,7 +53,7 @@ def setUp(self): def tearDown(self): self.text.delete('1.0', 'end') - self.editwin.context_use_ps1 = True + self.editwin.prompt_last_line = '>>>' def get_parser(self, index): """ @@ -71,7 +71,7 @@ def test_init(self): self.assertIn('precedes', str(ve.exception)) # test without ps1 - self.editwin.context_use_ps1 = False + self.editwin.prompt_last_line = '' # number of lines lesser than 50 p = self.get_parser('end') diff --git a/Lib/idlelib/idle_test/test_parenmatch.py b/Lib/idlelib/idle_test/test_parenmatch.py index f58819abf112..4a41d8433d54 100644 --- a/Lib/idlelib/idle_test/test_parenmatch.py +++ b/Lib/idlelib/idle_test/test_parenmatch.py @@ -17,7 +17,7 @@ def __init__(self, text): self.text = text self.indentwidth = 8 self.tabwidth = 8 - self.context_use_ps1 = True + self.prompt_last_line = '>>>' # Currently not used by parenmatch. class ParenMatchTest(unittest.TestCase): diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 2de42658b01c..6e0707d68bb6 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -881,7 +881,7 @@ def __init__(self, flist=None): self.usetabs = True # indentwidth must be 8 when using tabs. See note in EditorWindow: self.indentwidth = 8 - self.context_use_ps1 = True + self.sys_ps1 = sys.ps1 if hasattr(sys, 'ps1') else '>>> ' self.prompt_last_line = self.sys_ps1.split('\n')[-1] self.prompt = self.sys_ps1 # Changes when debug active diff --git a/Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst b/Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst new file mode 100644 index 000000000000..0042ab70497a --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2019-06-02-14-10-52.bpo-35610.0w_v6Y.rst @@ -0,0 +1,2 @@ +Replace now redundant .context_use_ps1 with .prompt_last_line. This finishes +change started in bpo-31858.

1 0

Add description to the command line help of the argument clinic (GH-8518)
by Pablo Galindo 02 Jun '19

02 Jun '19

https://github.com/python/cpython/commit/5df4025f42b30aca72f441899b361f748c… commit: 5df4025f42b30aca72f441899b361f748c304c57 branch: master author: Tim Hoffmann <2836374+timhoffm(a)users.noreply.github.com> committer: Pablo Galindo <Pablogsal(a)gmail.com> date: 2019-06-02T17:58:10+01:00 summary: Add description to the command line help of the argument clinic (GH-8518) files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index cb2ded4649dc..9880b3951339 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4900,7 +4900,14 @@ def main(argv): sys.exit("Error: clinic.py requires Python 3.3 or greater.") import argparse - cmdline = argparse.ArgumentParser() + cmdline = argparse.ArgumentParser( + description="""Preprocessor for CPython C files. + +The purpose of the Argument Clinic is automating all the boilerplate involved +with writing argument parsing code for builtins and providing introspection +signatures ("docstrings") for CPython builtins. + +For more information see https://docs.python.org/3/howto/clinic.html""") cmdline.add_argument("-f", "--force", action='store_true') cmdline.add_argument("-o", "--output", type=str) cmdline.add_argument("-v", "--verbose", action='store_true')

1 0

bpo-37126: Allow structseq objects to be tracked by the GC (GH-13729)
by Pablo Galindo 02 Jun '19

02 Jun '19

https://github.com/python/cpython/commit/7ffcf848df214135abeea7f6c6faa4135f… commit: 7ffcf848df214135abeea7f6c6faa4135fd0928f branch: master author: Pablo Galindo <Pablogsal(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T15:45:13+01:00 summary: bpo-37126: Allow structseq objects to be tracked by the GC (GH-13729) files: A Misc/NEWS.d/next/Core and Builtins/2019-06-01-20-03-13.bpo-37126.tP6lL4.rst M Objects/structseq.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-06-01-20-03-13.bpo-37126.tP6lL4.rst b/Misc/NEWS.d/next/Core and Builtins/2019-06-01-20-03-13.bpo-37126.tP6lL4.rst new file mode 100644 index 000000000000..069b064dfa61 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-06-01-20-03-13.bpo-37126.tP6lL4.rst @@ -0,0 +1,2 @@ +All structseq objects are now tracked by the garbage collector. Patch by +Pablo Galindo. diff --git a/Objects/structseq.c b/Objects/structseq.c index a5046c42cbc3..3d857f734be8 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_tupleobject.h" +#include "pycore_object.h" #include "structmember.h" static const char visible_length_key[] = "n_sequence_fields"; @@ -59,6 +60,18 @@ PyStructSequence_GetItem(PyObject* op, Py_ssize_t i) return PyStructSequence_GET_ITEM(op, i); } + +static int +structseq_traverse(PyStructSequence *obj, visitproc visit, void *arg) +{ + Py_ssize_t i, size; + size = REAL_SIZE(obj); + for (i = 0; i < size; ++i) { + Py_VISIT(obj->ob_item[i]); + } + return 0; +} + static void structseq_dealloc(PyStructSequence *obj) { @@ -166,6 +179,7 @@ structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict) } Py_DECREF(arg); + _PyObject_GC_TRACK(res); return (PyObject*) res; } @@ -388,6 +402,7 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) type->tp_methods = structseq_methods; type->tp_new = structseq_new; type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC; + type->tp_traverse = (traverseproc) structseq_traverse; n_members = count_members(desc, &n_unnamed_members); members = PyMem_NEW(PyMemberDef, n_members - n_unnamed_members + 1); @@ -426,7 +441,7 @@ PyStructSequence_NewType(PyStructSequence_Desc *desc) PyMemberDef *members; PyObject *bases; PyTypeObject *type; - PyType_Slot slots[7]; + PyType_Slot slots[8]; PyType_Spec spec; Py_ssize_t n_members, n_unnamed_members; @@ -446,7 +461,8 @@ PyStructSequence_NewType(PyStructSequence_Desc *desc) slots[3] = (PyType_Slot){Py_tp_methods, structseq_methods}; slots[4] = (PyType_Slot){Py_tp_new, structseq_new}; slots[5] = (PyType_Slot){Py_tp_members, members}; - slots[6] = (PyType_Slot){0, 0}; + slots[6] = (PyType_Slot){Py_tp_traverse, (traverseproc)structseq_traverse}; + slots[7] = (PyType_Slot){0, 0}; /* Initialize Spec */ /* The name in this PyType_Spec is statically allocated so it is */

1 0

bpo-35621: Support running subprocesses in asyncio when loop is executed in non-main thread (#13630)
by Andrew Svetlov 02 Jun '19

02 Jun '19

https://github.com/python/cpython/commit/13ed07998ad93dbdd94991ba0451b9b559… commit: 13ed07998ad93dbdd94991ba0451b9b559f07972 branch: master author: Andrew Svetlov <andrew.svetlov(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T13:56:38+03:00 summary: bpo-35621: Support running subprocesses in asyncio when loop is executed in non-main thread (#13630) files: A Misc/NEWS.d/next/Library/2019-05-28-19-03-46.bpo-35621.Abc1lf.rst M Lib/asyncio/unix_events.py M Lib/test/test_asyncio/test_subprocess.py M Lib/test/test_asyncio/test_unix_events.py diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 28128d2977df..6714542e4e33 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -2,6 +2,7 @@ import errno import io +import itertools import os import selectors import signal @@ -29,7 +30,9 @@ __all__ = ( 'SelectorEventLoop', 'AbstractChildWatcher', 'SafeChildWatcher', - 'FastChildWatcher', 'DefaultEventLoopPolicy', + 'FastChildWatcher', + 'MultiLoopChildWatcher', 'ThreadedChildWatcher', + 'DefaultEventLoopPolicy', ) @@ -184,6 +187,13 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None, stdin, stdout, stderr, bufsize, extra=None, **kwargs): with events.get_child_watcher() as watcher: + if not watcher.is_active(): + # Check early. + # Raising exception before process creation + # prevents subprocess execution if the watcher + # is not ready to handle it. + raise RuntimeError("asyncio.get_child_watcher() is not activated, " + "subproccess support is not installed.") waiter = self.create_future() transp = _UnixSubprocessTransport(self, protocol, args, shell, stdin, stdout, stderr, bufsize, @@ -838,6 +848,15 @@ def close(self): """ raise NotImplementedError() + def is_active(self): + """Watcher status. + + Return True if the watcher is installed and ready to handle process exit + notifications. + + """ + raise NotImplementedError() + def __enter__(self): """Enter the watcher's context and allow starting new processes @@ -849,6 +868,20 @@ def __exit__(self, a, b, c): raise NotImplementedError() +def _compute_returncode(status): + if os.WIFSIGNALED(status): + # The child process died because of a signal. + return -os.WTERMSIG(status) + elif os.WIFEXITED(status): + # The child process exited (e.g sys.exit()). + return os.WEXITSTATUS(status) + else: + # The child exited, but we don't understand its status. + # This shouldn't happen, but if it does, let's just + # return that status; perhaps that helps debug it. + return status + + class BaseChildWatcher(AbstractChildWatcher): def __init__(self): @@ -858,6 +891,9 @@ def __init__(self): def close(self): self.attach_loop(None) + def is_active(self): + return self._loop is not None and self._loop.is_running() + def _do_waitpid(self, expected_pid): raise NotImplementedError() @@ -898,19 +934,6 @@ def _sig_chld(self): 'exception': exc, }) - def _compute_returncode(self, status): - if os.WIFSIGNALED(status): - # The child process died because of a signal. - return -os.WTERMSIG(status) - elif os.WIFEXITED(status): - # The child process exited (e.g sys.exit()). - return os.WEXITSTATUS(status) - else: - # The child exited, but we don't understand its status. - # This shouldn't happen, but if it does, let's just - # return that status; perhaps that helps debug it. - return status - class SafeChildWatcher(BaseChildWatcher): """'Safe' child watcher implementation. @@ -934,11 +957,6 @@ def __exit__(self, a, b, c): pass def add_child_handler(self, pid, callback, *args): - if self._loop is None: - raise RuntimeError( - "Cannot add child handler, " - "the child watcher does not have a loop attached") - self._callbacks[pid] = (callback, args) # Prevent a race condition in case the child is already terminated. @@ -974,7 +992,7 @@ def _do_waitpid(self, expected_pid): # The child process is still alive. return - returncode = self._compute_returncode(status) + returncode = _compute_returncode(status) if self._loop.get_debug(): logger.debug('process %s exited with returncode %s', expected_pid, returncode) @@ -1035,11 +1053,6 @@ def __exit__(self, a, b, c): def add_child_handler(self, pid, callback, *args): assert self._forks, "Must use the context manager" - if self._loop is None: - raise RuntimeError( - "Cannot add child handler, " - "the child watcher does not have a loop attached") - with self._lock: try: returncode = self._zombies.pop(pid) @@ -1072,7 +1085,7 @@ def _do_waitpid_all(self): # A child process is still alive. return - returncode = self._compute_returncode(status) + returncode = _compute_returncode(status) with self._lock: try: @@ -1101,6 +1114,177 @@ def _do_waitpid_all(self): callback(pid, returncode, *args) +class MultiLoopChildWatcher(AbstractChildWatcher): + # The class keeps compatibility with AbstractChildWatcher ABC + # To achieve this it has empty attach_loop() method + # and doesn't accept explicit loop argument + # for add_child_handler()/remove_child_handler() + # but retrieves the current loop by get_running_loop() + + def __init__(self): + self._callbacks = {} + self._saved_sighandler = None + + def is_active(self): + return self._saved_sighandler is not None + + def close(self): + self._callbacks.clear() + if self._saved_sighandler is not None: + handler = signal.getsignal(signal.SIGCHLD) + if handler != self._sig_chld: + logger.warning("SIGCHLD handler was changed by outside code") + else: + signal.signal(signal.SIGCHLD, self._saved_sighandler) + self._saved_sighandler = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def add_child_handler(self, pid, callback, *args): + loop = events.get_running_loop() + self._callbacks[pid] = (loop, callback, args) + + # Prevent a race condition in case the child is already terminated. + self._do_waitpid(pid) + + def remove_child_handler(self, pid): + try: + del self._callbacks[pid] + return True + except KeyError: + return False + + def attach_loop(self, loop): + # Don't save the loop but initialize itself if called first time + # The reason to do it here is that attach_loop() is called from + # unix policy only for the main thread. + # Main thread is required for subscription on SIGCHLD signal + if self._saved_sighandler is None: + self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) + if self._saved_sighandler is None: + logger.warning("Previous SIGCHLD handler was set by non-Python code, " + "restore to default handler on watcher close.") + self._saved_sighandler = signal.SIG_DFL + + # Set SA_RESTART to limit EINTR occurrences. + signal.siginterrupt(signal.SIGCHLD, False) + + def _do_waitpid_all(self): + for pid in list(self._callbacks): + self._do_waitpid(pid) + + def _do_waitpid(self, expected_pid): + assert expected_pid > 0 + + try: + pid, status = os.waitpid(expected_pid, os.WNOHANG) + except ChildProcessError: + # The child process is already reaped + # (may happen if waitpid() is called elsewhere). + pid = expected_pid + returncode = 255 + logger.warning( + "Unknown child process pid %d, will report returncode 255", + pid) + debug_log = False + else: + if pid == 0: + # The child process is still alive. + return + + returncode = _compute_returncode(status) + debug_log = True + try: + loop, callback, args = self._callbacks.pop(pid) + except KeyError: # pragma: no cover + # May happen if .remove_child_handler() is called + # after os.waitpid() returns. + logger.warning("Child watcher got an unexpected pid: %r", + pid, exc_info=True) + else: + if loop.is_closed(): + logger.warning("Loop %r that handles pid %r is closed", loop, pid) + else: + if debug_log and loop.get_debug(): + logger.debug('process %s exited with returncode %s', + expected_pid, returncode) + loop.call_soon_threadsafe(callback, pid, returncode, *args) + + def _sig_chld(self, signum, frame): + try: + self._do_waitpid_all() + except (SystemExit, KeyboardInterrupt): + raise + except BaseException: + logger.warning('Unknown exception in SIGCHLD handler', exc_info=True) + + +class ThreadedChildWatcher(AbstractChildWatcher): + # The watcher uses a thread per process + # for waiting for the process finish. + # It doesn't require subscription on POSIX signal + + def __init__(self): + self._pid_counter = itertools.count(0) + + def is_active(self): + return True + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def add_child_handler(self, pid, callback, *args): + loop = events.get_running_loop() + thread = threading.Thread(target=self._do_waitpid, + name=f"waitpid-{next(self._pid_counter)}", + args=(loop, pid, callback, args), + daemon=True) + thread.start() + + def remove_child_handler(self, pid): + # asyncio never calls remove_child_handler() !!! + # The method is no-op but is implemented because + # abstract base classe requires it + return True + + def attach_loop(self, loop): + pass + + def _do_waitpid(self, loop, expected_pid, callback, args): + assert expected_pid > 0 + + try: + pid, status = os.waitpid(expected_pid, 0) + except ChildProcessError: + # The child process is already reaped + # (may happen if waitpid() is called elsewhere). + pid = expected_pid + returncode = 255 + logger.warning( + "Unknown child process pid %d, will report returncode 255", + pid) + else: + returncode = _compute_returncode(status) + if loop.get_debug(): + logger.debug('process %s exited with returncode %s', + expected_pid, returncode) + + if loop.is_closed(): + logger.warning("Loop %r that handles pid %r is closed", loop, pid) + else: + loop.call_soon_threadsafe(callback, pid, returncode, *args) + + class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): """UNIX event loop policy with a watcher for child processes.""" _loop_factory = _UnixSelectorEventLoop @@ -1112,7 +1296,7 @@ def __init__(self): def _init_watcher(self): with events._lock: if self._watcher is None: # pragma: no branch - self._watcher = SafeChildWatcher() + self._watcher = ThreadedChildWatcher() if isinstance(threading.current_thread(), threading._MainThread): self._watcher.attach_loop(self._local._loop) @@ -1134,7 +1318,7 @@ def set_event_loop(self, loop): def get_child_watcher(self): """Get the watcher for child processes. - If not yet set, a SafeChildWatcher object is automatically created. + If not yet set, a ThreadedChildWatcher object is automatically created. """ if self._watcher is None: self._init_watcher() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 7d72e6cde4e7..582e17202460 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -633,6 +633,7 @@ def test_create_subprocess_exec_with_path(self): self.assertIsNone(self.loop.run_until_complete(execute())) + if sys.platform != 'win32': # Unix class SubprocessWatcherMixin(SubprocessMixin): @@ -648,7 +649,24 @@ def setUp(self): watcher = self.Watcher() watcher.attach_loop(self.loop) policy.set_child_watcher(watcher) - self.addCleanup(policy.set_child_watcher, None) + + def tearDown(self): + super().setUp() + policy = asyncio.get_event_loop_policy() + watcher = policy.get_child_watcher() + policy.set_child_watcher(None) + watcher.attach_loop(None) + watcher.close() + + class SubprocessThreadedWatcherTests(SubprocessWatcherMixin, + test_utils.TestCase): + + Watcher = unix_events.ThreadedChildWatcher + + class SubprocessMultiLoopWatcherTests(SubprocessWatcherMixin, + test_utils.TestCase): + + Watcher = unix_events.MultiLoopChildWatcher class SubprocessSafeWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): @@ -670,5 +688,25 @@ def setUp(self): self.set_event_loop(self.loop) +class GenericWatcherTests: + + def test_create_subprocess_fails_with_inactive_watcher(self): + + async def execute(): + watcher = mock.create_authspec(asyncio.AbstractChildWatcher) + watcher.is_active.return_value = False + asyncio.set_child_watcher(watcher) + + with self.assertRaises(RuntimeError): + await subprocess.create_subprocess_exec( + support.FakePath(sys.executable), '-c', 'pass') + + watcher.add_child_handler.assert_not_called() + + self.assertIsNone(self.loop.run_until_complete(execute())) + + + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index ac84304ec99d..f7f992fcea49 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1082,6 +1082,8 @@ def test_not_implemented(self): NotImplementedError, watcher.attach_loop, f) self.assertRaises( NotImplementedError, watcher.close) + self.assertRaises( + NotImplementedError, watcher.is_active) self.assertRaises( NotImplementedError, watcher.__enter__) self.assertRaises( @@ -1784,15 +1786,6 @@ def test_close(self, m): if isinstance(self.watcher, asyncio.FastChildWatcher): self.assertFalse(self.watcher._zombies) - @waitpid_mocks - def test_add_child_handler_with_no_loop_attached(self, m): - callback = mock.Mock() - with self.create_watcher() as watcher: - with self.assertRaisesRegex( - RuntimeError, - 'the child watcher does not have a loop attached'): - watcher.add_child_handler(100, callback) - class SafeChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): def create_watcher(self): @@ -1809,17 +1802,16 @@ class PolicyTests(unittest.TestCase): def create_policy(self): return asyncio.DefaultEventLoopPolicy() - def test_get_child_watcher(self): + def test_get_default_child_watcher(self): policy = self.create_policy() self.assertIsNone(policy._watcher) watcher = policy.get_child_watcher() - self.assertIsInstance(watcher, asyncio.SafeChildWatcher) + self.assertIsInstance(watcher, asyncio.ThreadedChildWatcher) self.assertIs(policy._watcher, watcher) self.assertIs(watcher, policy.get_child_watcher()) - self.assertIsNone(watcher._loop) def test_get_child_watcher_after_set(self): policy = self.create_policy() @@ -1829,18 +1821,6 @@ def test_get_child_watcher_after_set(self): self.assertIs(policy._watcher, watcher) self.assertIs(watcher, policy.get_child_watcher()) - def test_get_child_watcher_with_mainloop_existing(self): - policy = self.create_policy() - loop = policy.get_event_loop() - - self.assertIsNone(policy._watcher) - watcher = policy.get_child_watcher() - - self.assertIsInstance(watcher, asyncio.SafeChildWatcher) - self.assertIs(watcher._loop, loop) - - loop.close() - def test_get_child_watcher_thread(self): def f(): @@ -1865,7 +1845,11 @@ def test_child_watcher_replace_mainloop_existing(self): policy = self.create_policy() loop = policy.get_event_loop() - watcher = policy.get_child_watcher() + # Explicitly setup SafeChildWatcher, + # default ThreadedChildWatcher has no _loop property + watcher = asyncio.SafeChildWatcher() + policy.set_child_watcher(watcher) + watcher.attach_loop(loop) self.assertIs(watcher._loop, loop) diff --git a/Misc/NEWS.d/next/Library/2019-05-28-19-03-46.bpo-35621.Abc1lf.rst b/Misc/NEWS.d/next/Library/2019-05-28-19-03-46.bpo-35621.Abc1lf.rst new file mode 100644 index 000000000000..c492e1de6d5c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-28-19-03-46.bpo-35621.Abc1lf.rst @@ -0,0 +1,2 @@ +Support running asyncio subprocesses when execution event loop in a thread +on UNIX.

1 0

bpo-36027: Extend three-argument pow to negative second argument (GH-13266)
by Mark Dickinson 02 Jun '19

02 Jun '19

https://github.com/python/cpython/commit/c52996785a45d4693857ea219e040777a1… commit: c52996785a45d4693857ea219e040777a14584f8 branch: master author: Mark Dickinson <mdickinson(a)enthought.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T10:24:06+01:00 summary: bpo-36027: Extend three-argument pow to negative second argument (GH-13266) files: A Misc/NEWS.d/next/Core and Builtins/2019-05-12-18-46-50.bpo-36027.Q4YatQ.rst M Doc/library/functions.rst M Doc/whatsnew/3.8.rst M Lib/test/test_builtin.py M Lib/test/test_pow.py M Objects/longobject.c diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index d5c9f18c79b7..415a65b4946f 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1277,9 +1277,24 @@ are always available. They are listed here in alphabetical order. operands, the result has the same type as the operands (after coercion) unless the second argument is negative; in that case, all arguments are converted to float and a float result is delivered. For example, ``10**2`` - returns ``100``, but ``10**-2`` returns ``0.01``. If the second argument is - negative, the third argument must be omitted. If *z* is present, *x* and *y* - must be of integer types, and *y* must be non-negative. + returns ``100``, but ``10**-2`` returns ``0.01``. + + For :class:`int` operands *x* and *y*, if *z* is present, *z* must also be + of integer type and *z* must be nonzero. If *z* is present and *y* is + negative, *x* must be relatively prime to *z*. In that case, ``pow(inv_x, + -y, z)`` is returned, where *inv_x* is an inverse to *x* modulo *z*. + + Here's an example of computing an inverse for ``38`` modulo ``97``:: + + >>> pow(38, -1, 97) + 23 + >>> 23 * 38 % 97 == 1 + True + + .. versionchanged:: 3.8 + For :class:`int` operands, the three-argument form of ``pow`` now allows + the second argument to be negative, permitting computation of modular + inverses. .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 591b45488372..74d0079a53db 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -304,6 +304,12 @@ Other Language Changes * Added new ``replace()`` method to the code type (:class:`types.CodeType`). (Contributed by Victor Stinner in :issue:`37032`.) +* For integers, the three-argument form of the :func:`pow` function now permits + the exponent to be negative in the case where the base is relatively prime to + the modulus. It then computes a modular inverse to the base when the exponent + is ``-1``, and a suitable power of that inverse for other negative exponents. + (Contributed by Mark Dickinson in :issue:`36027`.) + New Modules =========== diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index e32fb75d8191..b536cec06487 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1195,7 +1195,8 @@ def test_pow(self): self.assertAlmostEqual(pow(-1, 0.5), 1j) self.assertAlmostEqual(pow(-1, 1/3), 0.5 + 0.8660254037844386j) - self.assertRaises(ValueError, pow, -1, -2, 3) + # See test_pow for additional tests for three-argument pow. + self.assertEqual(pow(-1, -2, 3), 1) self.assertRaises(ValueError, pow, 1, 2, 0) self.assertRaises(TypeError, pow) diff --git a/Lib/test/test_pow.py b/Lib/test/test_pow.py index cac1ae5ea2d8..660ff80bbf52 100644 --- a/Lib/test/test_pow.py +++ b/Lib/test/test_pow.py @@ -1,3 +1,4 @@ +import math import unittest class PowTest(unittest.TestCase): @@ -119,5 +120,30 @@ def test_bug705231(self): eq(pow(a, -fiveto), expected) eq(expected, 1.0) # else we didn't push fiveto to evenness + def test_negative_exponent(self): + for a in range(-50, 50): + for m in range(-50, 50): + with self.subTest(a=a, m=m): + if m != 0 and math.gcd(a, m) == 1: + # Exponent -1 should give an inverse, with the + # same sign as m. + inv = pow(a, -1, m) + self.assertEqual(inv, inv % m) + self.assertEqual((inv * a - 1) % m, 0) + + # Larger exponents + self.assertEqual(pow(a, -2, m), pow(inv, 2, m)) + self.assertEqual(pow(a, -3, m), pow(inv, 3, m)) + self.assertEqual(pow(a, -1001, m), pow(inv, 1001, m)) + + else: + with self.assertRaises(ValueError): + pow(a, -1, m) + with self.assertRaises(ValueError): + pow(a, -2, m) + with self.assertRaises(ValueError): + pow(a, -1001, m) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-12-18-46-50.bpo-36027.Q4YatQ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-12-18-46-50.bpo-36027.Q4YatQ.rst new file mode 100644 index 000000000000..866309cddc68 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-12-18-46-50.bpo-36027.Q4YatQ.rst @@ -0,0 +1,3 @@ +Allow computation of modular inverses via three-argument ``pow``: the second +argument is now permitted to be negative in the case where the first and +third arguments are relatively prime. diff --git a/Objects/longobject.c b/Objects/longobject.c index 5d2b595621f3..49f1420bf64f 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -4174,6 +4174,98 @@ long_divmod(PyObject *a, PyObject *b) return z; } + +/* Compute an inverse to a modulo n, or raise ValueError if a is not + invertible modulo n. Assumes n is positive. The inverse returned + is whatever falls out of the extended Euclidean algorithm: it may + be either positive or negative, but will be smaller than n in + absolute value. + + Pure Python equivalent for long_invmod: + + def invmod(a, n): + b, c = 1, 0 + while n: + q, r = divmod(a, n) + a, b, c, n = n, c, b - q*c, r + + # at this point a is the gcd of the original inputs + if a == 1: + return b + raise ValueError("Not invertible") +*/ + +static PyLongObject * +long_invmod(PyLongObject *a, PyLongObject *n) +{ + PyLongObject *b, *c; + + /* Should only ever be called for positive n */ + assert(Py_SIZE(n) > 0); + + b = (PyLongObject *)PyLong_FromLong(1L); + if (b == NULL) { + return NULL; + } + c = (PyLongObject *)PyLong_FromLong(0L); + if (c == NULL) { + Py_DECREF(b); + return NULL; + } + Py_INCREF(a); + Py_INCREF(n); + + /* references now owned: a, b, c, n */ + while (Py_SIZE(n) != 0) { + PyLongObject *q, *r, *s, *t; + + if (l_divmod(a, n, &q, &r) == -1) { + goto Error; + } + Py_DECREF(a); + a = n; + n = r; + t = (PyLongObject *)long_mul(q, c); + Py_DECREF(q); + if (t == NULL) { + goto Error; + } + s = (PyLongObject *)long_sub(b, t); + Py_DECREF(t); + if (s == NULL) { + goto Error; + } + Py_DECREF(b); + b = c; + c = s; + } + /* references now owned: a, b, c, n */ + + Py_DECREF(c); + Py_DECREF(n); + if (long_compare(a, _PyLong_One)) { + /* a != 1; we don't have an inverse. */ + Py_DECREF(a); + Py_DECREF(b); + PyErr_SetString(PyExc_ValueError, + "base is not invertible for the given modulus"); + return NULL; + } + else { + /* a == 1; b gives an inverse modulo n */ + Py_DECREF(a); + return b; + } + + Error: + Py_DECREF(a); + Py_DECREF(b); + Py_DECREF(c); + Py_DECREF(n); + return NULL; +} + + /* pow(v, w, x) */ static PyObject * long_pow(PyObject *v, PyObject *w, PyObject *x) @@ -4207,20 +4299,14 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) Py_RETURN_NOTIMPLEMENTED; } - if (Py_SIZE(b) < 0) { /* if exponent is negative */ - if (c) { - PyErr_SetString(PyExc_ValueError, "pow() 2nd argument " - "cannot be negative when 3rd argument specified"); - goto Error; - } - else { - /* else return a float. This works because we know + if (Py_SIZE(b) < 0 && c == NULL) { + /* if exponent is negative and there's no modulus: + return a float. This works because we know that this calls float_pow() which converts its arguments to double. */ - Py_DECREF(a); - Py_DECREF(b); - return PyFloat_Type.tp_as_number->nb_power(v, w, x); - } + Py_DECREF(a); + Py_DECREF(b); + return PyFloat_Type.tp_as_number->nb_power(v, w, x); } if (c) { @@ -4255,6 +4341,26 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) goto Done; } + /* if exponent is negative, negate the exponent and + replace the base with a modular inverse */ + if (Py_SIZE(b) < 0) { + temp = (PyLongObject *)_PyLong_Copy(b); + if (temp == NULL) + goto Error; + Py_DECREF(b); + b = temp; + temp = NULL; + _PyLong_Negate(&b); + if (b == NULL) + goto Error; + + temp = long_invmod(a, c); + if (temp == NULL) + goto Error; + Py_DECREF(a); + a = temp; + } + /* Reduce base by modulus in some cases: 1. If base < 0. Forcing the base non-negative makes things easier. 2. If base is obviously larger than the modulus. The "small

1 0

bpo-37128: Add math.perm(). (GH-13731)
by Serhiy Storchaka 02 Jun '19

02 Jun '19

https://github.com/python/cpython/commit/5ae299ac78abb628803ab7dee099736454… commit: 5ae299ac78abb628803ab7dee0997364547f5cc8 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T11:16:49+03:00 summary: bpo-37128: Add math.perm(). (GH-13731) files: A Misc/NEWS.d/next/Library/2019-06-01-22-54-03.bpo-37128.oGXBWN.rst M Doc/library/math.rst M Lib/test/test_math.py M Modules/clinic/mathmodule.c.h M Modules/mathmodule.c diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 8c6837050319..c5a77f1fab9f 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -207,6 +207,19 @@ Number-theoretic and representation functions of *x* and are floats. +.. function:: perm(n, k) + + Return the number of ways to choose *k* items from *n* items + without repetition and with order. + + It is mathematically equal to the expression ``n! / (n - k)!``. + + Raises :exc:`TypeError` if the arguments not integers. + Raises :exc:`ValueError` if the arguments are negative or if *k* > *n*. + + .. versionadded:: 3.8 + + .. function:: prod(iterable, *, start=1) Calculate the product of all the elements in the input *iterable*. diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index e27092eefd6e..96e0cf2fe671 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -240,6 +240,9 @@ def result_check(expected, got, ulp_tol=5, abs_tol=0.0): else: return None +class IntSubclass(int): + pass + # Class providing an __index__ method. class MyIndexable(object): def __init__(self, value): @@ -1862,6 +1865,64 @@ def test_fractions(self): self.assertAllClose(fraction_examples, rel_tol=1e-8) self.assertAllNotClose(fraction_examples, rel_tol=1e-9) + def testPerm(self): + perm = math.perm + factorial = math.factorial + # Test if factorial defintion is satisfied + for n in range(100): + for k in range(n + 1): + self.assertEqual(perm(n, k), + factorial(n) // factorial(n - k)) + + # Test for Pascal's identity + for n in range(1, 100): + for k in range(1, n): + self.assertEqual(perm(n, k), perm(n - 1, k - 1) * k + perm(n - 1, k)) + + # Test corner cases + for n in range(1, 100): + self.assertEqual(perm(n, 0), 1) + self.assertEqual(perm(n, 1), n) + self.assertEqual(perm(n, n), factorial(n)) + + # Raises TypeError if any argument is non-integer or argument count is + # not 2 + self.assertRaises(TypeError, perm, 10, 1.0) + self.assertRaises(TypeError, perm, 10, decimal.Decimal(1.0)) + self.assertRaises(TypeError, perm, 10, "1") + self.assertRaises(TypeError, perm, 10.0, 1) + self.assertRaises(TypeError, perm, decimal.Decimal(10.0), 1) + self.assertRaises(TypeError, perm, "10", 1) + + self.assertRaises(TypeError, perm, 10) + self.assertRaises(TypeError, perm, 10, 1, 3) + self.assertRaises(TypeError, perm) + + # Raises Value error if not k or n are negative numbers + self.assertRaises(ValueError, perm, -1, 1) + self.assertRaises(ValueError, perm, -2**1000, 1) + self.assertRaises(ValueError, perm, 1, -1) + self.assertRaises(ValueError, perm, 1, -2**1000) + + # Raises value error if k is greater than n + self.assertRaises(ValueError, perm, 1, 2) + self.assertRaises(ValueError, perm, 1, 2**1000) + + n = 2**1000 + self.assertEqual(perm(n, 0), 1) + self.assertEqual(perm(n, 1), n) + self.assertEqual(perm(n, 2), n * (n-1)) + self.assertRaises((OverflowError, MemoryError), perm, n, n) + + for n, k in (True, True), (True, False), (False, False): + self.assertEqual(perm(n, k), 1) + self.assertIs(type(perm(n, k)), int) + self.assertEqual(perm(IntSubclass(5), IntSubclass(2)), 20) + self.assertEqual(perm(MyIndexable(5), MyIndexable(2)), 20) + for k in range(3): + self.assertIs(type(perm(IntSubclass(5), IntSubclass(k))), int) + self.assertIs(type(perm(MyIndexable(5), MyIndexable(k))), int) + def testComb(self): comb = math.comb factorial = math.factorial @@ -1925,8 +1986,11 @@ def testComb(self): for n, k in (True, True), (True, False), (False, False): self.assertEqual(comb(n, k), 1) self.assertIs(type(comb(n, k)), int) + self.assertEqual(comb(IntSubclass(5), IntSubclass(2)), 10) self.assertEqual(comb(MyIndexable(5), MyIndexable(2)), 10) - self.assertIs(type(comb(MyIndexable(5), MyIndexable(2))), int) + for k in range(3): + self.assertIs(type(comb(IntSubclass(5), IntSubclass(k))), int) + self.assertIs(type(comb(MyIndexable(5), MyIndexable(k))), int) def test_main(): diff --git a/Misc/NEWS.d/next/Library/2019-06-01-22-54-03.bpo-37128.oGXBWN.rst b/Misc/NEWS.d/next/Library/2019-06-01-22-54-03.bpo-37128.oGXBWN.rst new file mode 100644 index 000000000000..f1b825890231 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-06-01-22-54-03.bpo-37128.oGXBWN.rst @@ -0,0 +1 @@ +Added :func:`math.perm`. diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index 92ec4bec9bf1..0efe5cc409ce 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -638,6 +638,41 @@ math_prod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k return return_value; } +PyDoc_STRVAR(math_perm__doc__, +"perm($module, n, k, /)\n" +"--\n" +"\n" +"Number of ways to choose k items from n items without repetition and with order.\n" +"\n" +"It is mathematically equal to the expression n! / (n - k)!.\n" +"\n" +"Raises TypeError if the arguments are not integers.\n" +"Raises ValueError if the arguments are negative or if k > n."); + +#define MATH_PERM_METHODDEF \ + {"perm", (PyCFunction)(void(*)(void))math_perm, METH_FASTCALL, math_perm__doc__}, + +static PyObject * +math_perm_impl(PyObject *module, PyObject *n, PyObject *k); + +static PyObject * +math_perm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *n; + PyObject *k; + + if (!_PyArg_CheckPositional("perm", nargs, 2, 2)) { + goto exit; + } + n = args[0]; + k = args[1]; + return_value = math_perm_impl(module, n, k); + +exit: + return return_value; +} + PyDoc_STRVAR(math_comb__doc__, "comb($module, n, k, /)\n" "--\n" @@ -674,4 +709,4 @@ math_comb(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=6709521e5e1d90ec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a82b0e705b6d0ec0 input=a9049054013a1b77]*/ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index bea4607b9be1..6e1099321c54 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -2998,6 +2998,120 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) } +/*[clinic input] +math.perm + + n: object + k: object + / + +Number of ways to choose k items from n items without repetition and with order. + +It is mathematically equal to the expression n! / (n - k)!. + +Raises TypeError if the arguments are not integers. +Raises ValueError if the arguments are negative or if k > n. +[clinic start generated code]*/ + +static PyObject * +math_perm_impl(PyObject *module, PyObject *n, PyObject *k) +/*[clinic end generated code: output=e021a25469653e23 input=f71ee4f6ff26be24]*/ +{ + PyObject *result = NULL, *factor = NULL; + int overflow, cmp; + long long i, factors; + + n = PyNumber_Index(n); + if (n == NULL) { + return NULL; + } + if (!PyLong_CheckExact(n)) { + Py_SETREF(n, _PyLong_Copy((PyLongObject *)n)); + if (n == NULL) { + return NULL; + } + } + k = PyNumber_Index(k); + if (k == NULL) { + Py_DECREF(n); + return NULL; + } + if (!PyLong_CheckExact(k)) { + Py_SETREF(k, _PyLong_Copy((PyLongObject *)k)); + if (k == NULL) { + Py_DECREF(n); + return NULL; + } + } + + if (Py_SIZE(n) < 0) { + PyErr_SetString(PyExc_ValueError, + "n must be a non-negative integer"); + goto error; + } + cmp = PyObject_RichCompareBool(n, k, Py_LT); + if (cmp != 0) { + if (cmp > 0) { + PyErr_SetString(PyExc_ValueError, + "k must be an integer less than or equal to n"); + } + goto error; + } + + factors = PyLong_AsLongLongAndOverflow(k, &overflow); + if (overflow > 0) { + PyErr_Format(PyExc_OverflowError, + "k must not exceed %lld", + LLONG_MAX); + goto error; + } + else if (overflow < 0 || factors < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "k must be a non-negative integer"); + } + goto error; + } + + if (factors == 0) { + result = PyLong_FromLong(1); + goto done; + } + + result = n; + Py_INCREF(result); + if (factors == 1) { + goto done; + } + + factor = n; + Py_INCREF(factor); + for (i = 1; i < factors; ++i) { + Py_SETREF(factor, PyNumber_Subtract(factor, _PyLong_One)); + if (factor == NULL) { + goto error; + } + Py_SETREF(result, PyNumber_Multiply(result, factor)); + if (result == NULL) { + goto error; + } + } + Py_DECREF(factor); + +done: + Py_DECREF(n); + Py_DECREF(k); + return result; + +error: + Py_XDECREF(factor); + Py_XDECREF(result); + Py_DECREF(n); + Py_DECREF(k); + return NULL; +} + + /*[clinic input] math.comb @@ -3028,11 +3142,24 @@ math_comb_impl(PyObject *module, PyObject *n, PyObject *k) if (n == NULL) { return NULL; } + if (!PyLong_CheckExact(n)) { + Py_SETREF(n, _PyLong_Copy((PyLongObject *)n)); + if (n == NULL) { + return NULL; + } + } k = PyNumber_Index(k); if (k == NULL) { Py_DECREF(n); return NULL; } + if (!PyLong_CheckExact(k)) { + Py_SETREF(k, _PyLong_Copy((PyLongObject *)k)); + if (k == NULL) { + Py_DECREF(n); + return NULL; + } + } if (Py_SIZE(n) < 0) { PyErr_SetString(PyExc_ValueError, @@ -3050,7 +3177,7 @@ math_comb_impl(PyObject *module, PyObject *n, PyObject *k) "k must be an integer less than or equal to n"); goto error; } - cmp = PyObject_RichCompareBool(k, temp, Py_GT); + cmp = PyObject_RichCompareBool(temp, k, Py_LT); if (cmp > 0) { Py_SETREF(k, temp); } @@ -3174,6 +3301,7 @@ static PyMethodDef math_methods[] = { {"tanh", math_tanh, METH_O, math_tanh_doc}, MATH_TRUNC_METHODDEF MATH_PROD_METHODDEF + MATH_PERM_METHODDEF MATH_COMB_METHODDEF {NULL, NULL} /* sentinel */ };

1 0

Add more tests for preserving identity in marshal. (GH-13736)
by Serhiy Storchaka 02 Jun '19

02 Jun '19

https://github.com/python/cpython/commit/d71f3170ac9c850f6d1f9bffaa628dc473… commit: d71f3170ac9c850f6d1f9bffaa628dc473df5e75 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-06-02T09:03:59+03:00 summary: Add more tests for preserving identity in marshal. (GH-13736) files: M Lib/test/test_marshal.py diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index a3bd350c77b9..ace1593999d4 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -383,10 +383,7 @@ def CollectObjectIDs(ids, obj): return len(ids) class InstancingTestCase(unittest.TestCase, HelperMixin): - intobj = 123321 - floatobj = 1.2345 - strobj = "abcde"*3 - dictobj = {"hello":floatobj, "goodbye":floatobj, floatobj:"hello"} + keys = (123, 1.2345, 'abc', (123, 'abc'), frozenset({123, 'abc'})) def helper3(self, rsample, recursive=False, simple=False): #we have two instances @@ -394,11 +391,12 @@ def helper3(self, rsample, recursive=False, simple=False): n0 = CollectObjectIDs(set(), sample) - s3 = marshal.dumps(sample, 3) - n3 = CollectObjectIDs(set(), marshal.loads(s3)) + for v in range(3, marshal.version + 1): + s3 = marshal.dumps(sample, v) + n3 = CollectObjectIDs(set(), marshal.loads(s3)) - #same number of instances generated - self.assertEqual(n3, n0) + #same number of instances generated + self.assertEqual(n3, n0) if not recursive: #can compare with version 2 @@ -414,20 +412,54 @@ def helper3(self, rsample, recursive=False, simple=False): self.assertGreaterEqual(len(s2), len(s3)) def testInt(self): - self.helper(self.intobj) - self.helper3(self.intobj, simple=True) + intobj = 123321 + self.helper(intobj) + self.helper3(intobj, simple=True) def testFloat(self): - self.helper(self.floatobj) - self.helper3(self.floatobj) + floatobj = 1.2345 + self.helper(floatobj) + self.helper3(floatobj) def testStr(self): - self.helper(self.strobj) - self.helper3(self.strobj) + strobj = "abcde"*3 + self.helper(strobj) + self.helper3(strobj) + + def testBytes(self): + bytesobj = b"abcde"*3 + self.helper(bytesobj) + self.helper3(bytesobj) + + def testList(self): + for obj in self.keys: + listobj = [obj, obj] + self.helper(listobj) + self.helper3(listobj) + + def testTuple(self): + for obj in self.keys: + tupleobj = (obj, obj) + self.helper(tupleobj) + self.helper3(tupleobj) + + def testSet(self): + for obj in self.keys: + setobj = {(obj, 1), (obj, 2)} + self.helper(setobj) + self.helper3(setobj) + + def testFrozenSet(self): + for obj in self.keys: + frozensetobj = frozenset({(obj, 1), (obj, 2)}) + self.helper(frozensetobj) + self.helper3(frozensetobj) def testDict(self): - self.helper(self.dictobj) - self.helper3(self.dictobj) + for obj in self.keys: + dictobj = {"hello": obj, "goodbye": obj, obj: "hello"} + self.helper(dictobj) + self.helper3(dictobj) def testModule(self): with open(__file__, "rb") as f: @@ -438,10 +470,11 @@ def testModule(self): self.helper3(code) def testRecursion(self): - d = dict(self.dictobj) + obj = 1.2345 + d = {"hello": obj, "goodbye": obj, obj: "hello"} d["self"] = d self.helper3(d, recursive=True) - l = [self.dictobj] + l = [obj, obj] l.append(l) self.helper3(l, recursive=True)

1 0