Mailman 3 October 2017 - Python-checkins

bpo-31158: Fix nondeterministic read in test_pty (#3808) (#3853)
by Victor Stinner Oct. 2, 2017

Oct. 2, 2017

https://github.com/python/cpython/commit/20cbc1d29facead32c2da06c81a09af0e0… commit: 20cbc1d29facead32c2da06c81a09af0e057015c branch: 2.7 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-10-02T02:58:09-07:00 summary: bpo-31158: Fix nondeterministic read in test_pty (#3808) (#3853) (cherry picked from commit e6f62f69f07892b993910ff03c9db3ffa5cb9ca5) files: M Lib/test/test_pty.py diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index bec38c45456..f623aa09620 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -11,6 +11,7 @@ import select import signal import socket +import io # readline import unittest TEST_STRING_1 = "I wish to buy a fish license.\n" @@ -24,6 +25,16 @@ def debug(msg): pass +# Note that os.read() is nondeterministic so we need to be very careful +# to make the test suite deterministic. A normal call to os.read() may +# give us less than expected. +# +# Beware, on my Linux system, if I put 'foo\n' into a terminal fd, I get +# back 'foo\r\n' at the other end. The behavior depends on the termios +# setting. The newline translation may be OS-specific. To make the +# test suite deterministic and OS-independent, the functions _readline +# and normalize_output can be used. + def normalize_output(data): # Some operating systems do conversions on newline. We could possibly # fix that by doing the appropriate termios.tcsetattr()s. I couldn't @@ -45,6 +56,12 @@ def normalize_output(data): return data +def _readline(fd): + """Read one line. May block forever if no newline is read.""" + reader = io.FileIO(fd, mode='rb', closefd=False) + return reader.readline() + + # Marginal testing of pty suite. Cannot do extensive 'do or fail' testing # because pty code is not too portable. @@ -97,14 +114,14 @@ def test_basic(self): debug("Writing to slave_fd") os.write(slave_fd, TEST_STRING_1) - s1 = os.read(master_fd, 1024) + s1 = _readline(master_fd) self.assertEqual('I wish to buy a fish license.\n', normalize_output(s1)) debug("Writing chunked output") os.write(slave_fd, TEST_STRING_2[:5]) os.write(slave_fd, TEST_STRING_2[5:]) - s2 = os.read(master_fd, 1024) + s2 = _readline(master_fd) self.assertEqual('For my pet fish, Eric.\n', normalize_output(s2)) os.close(slave_fd)

1 0

[3.6] bpo-31158: Fix nondeterministic read in test_pty (GH-3808) (GH-3852)
by Victor Stinner Oct. 2, 2017

Oct. 2, 2017

https://github.com/python/cpython/commit/66fb5ef3bb9e36187a0e5052dfd9989944… commit: 66fb5ef3bb9e36187a0e5052dfd99899447df671 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Victor Stinner <victor.stinner(a)gmail.com> date: 2017-10-02T02:57:56-07:00 summary: [3.6] bpo-31158: Fix nondeterministic read in test_pty (GH-3808) (GH-3852) (cherry picked from commit e6f62f69f07892b993910ff03c9db3ffa5cb9ca5) files: M Lib/test/test_pty.py diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index 15f88e4fcd7..45ebea51a05 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -10,6 +10,7 @@ import select import signal import socket +import io # readline import unittest TEST_STRING_1 = b"I wish to buy a fish license.\n" @@ -23,6 +24,16 @@ def debug(msg): pass +# Note that os.read() is nondeterministic so we need to be very careful +# to make the test suite deterministic. A normal call to os.read() may +# give us less than expected. +# +# Beware, on my Linux system, if I put 'foo\n' into a terminal fd, I get +# back 'foo\r\n' at the other end. The behavior depends on the termios +# setting. The newline translation may be OS-specific. To make the +# test suite deterministic and OS-independent, the functions _readline +# and normalize_output can be used. + def normalize_output(data): # Some operating systems do conversions on newline. We could possibly # fix that by doing the appropriate termios.tcsetattr()s. I couldn't @@ -44,6 +55,12 @@ def normalize_output(data): return data +def _readline(fd): + """Read one line. May block forever if no newline is read.""" + reader = io.FileIO(fd, mode='rb', closefd=False) + return reader.readline() + + # Marginal testing of pty suite. Cannot do extensive 'do or fail' testing # because pty code is not too portable. @@ -98,14 +115,14 @@ def test_basic(self): debug("Writing to slave_fd") os.write(slave_fd, TEST_STRING_1) - s1 = os.read(master_fd, 1024) + s1 = _readline(master_fd) self.assertEqual(b'I wish to buy a fish license.\n', normalize_output(s1)) debug("Writing chunked output") os.write(slave_fd, TEST_STRING_2[:5]) os.write(slave_fd, TEST_STRING_2[5:]) - s2 = os.read(master_fd, 1024) + s2 = _readline(master_fd) self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2)) os.close(slave_fd)

1 0

bpo-31158: Fix nondeterministic read in test_pty (#3808)
by Victor Stinner Oct. 2, 2017

Oct. 2, 2017

https://github.com/python/cpython/commit/e6f62f69f07892b993910ff03c9db3ffa5… commit: e6f62f69f07892b993910ff03c9db3ffa5cb9ca5 branch: master author: Cornelius Diekmann <c.diekmann(a)googlemail.com> committer: Victor Stinner <victor.stinner(a)gmail.com> date: 2017-10-02T02:39:55-07:00 summary: bpo-31158: Fix nondeterministic read in test_pty (#3808) * bpo-31158: Fix nondeterministic read in test_pty * Reuse existing readline implementation from io. Thx to @pitrou * Updated comment Ideally, this commit is fixuped into the previous commit. Since there is already a comment on github, I won't rebase. files: M Lib/test/test_pty.py diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index f283e1930b7..3b448569a2f 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -10,6 +10,7 @@ import select import signal import socket +import io # readline import unittest TEST_STRING_1 = b"I wish to buy a fish license.\n" @@ -23,6 +24,16 @@ def debug(msg): pass +# Note that os.read() is nondeterministic so we need to be very careful +# to make the test suite deterministic. A normal call to os.read() may +# give us less than expected. +# +# Beware, on my Linux system, if I put 'foo\n' into a terminal fd, I get +# back 'foo\r\n' at the other end. The behavior depends on the termios +# setting. The newline translation may be OS-specific. To make the +# test suite deterministic and OS-independent, the functions _readline +# and normalize_output can be used. + def normalize_output(data): # Some operating systems do conversions on newline. We could possibly fix # that by doing the appropriate termios.tcsetattr()s. I couldn't figure out @@ -43,6 +54,12 @@ def normalize_output(data): return data +def _readline(fd): + """Read one line. May block forever if no newline is read.""" + reader = io.FileIO(fd, mode='rb', closefd=False) + return reader.readline() + + # Marginal testing of pty suite. Cannot do extensive 'do or fail' testing # because pty code is not too portable. @@ -94,14 +111,14 @@ def test_basic(self): debug("Writing to slave_fd") os.write(slave_fd, TEST_STRING_1) - s1 = os.read(master_fd, 1024) + s1 = _readline(master_fd) self.assertEqual(b'I wish to buy a fish license.\n', normalize_output(s1)) debug("Writing chunked output") os.write(slave_fd, TEST_STRING_2[:5]) os.write(slave_fd, TEST_STRING_2[5:]) - s2 = os.read(master_fd, 1024) + s2 = _readline(master_fd) self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2)) os.close(slave_fd)

1 0

Daily reference leaks (4243df51fe43): sum=11
by solipsis＠pitrou.net Oct. 2, 2017

Oct. 2, 2017

results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 0, 7] memory blocks, sum=7 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [1, -2, 1] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/refloggVoJq7', '--timeout', '7200']

1 0

bpo-31659: Use simple slicing to format PEM cert (GH-3849)
by INADA Naoki Oct. 2, 2017

Oct. 2, 2017

https://github.com/python/cpython/commit/b75a228af8c0553aef44e4e03763af90fb… commit: b75a228af8c0553aef44e4e03763af90fbc8737f branch: master author: INADA Naoki <methane(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2017-10-02T16:33:42+09:00 summary: bpo-31659: Use simple slicing to format PEM cert (GH-3849) DER_cert_to_PEM_cert() used textwrap.fill() to format PEM. But it's library to wrap lines on word boundary, while PEM is base64 encoded string. Additionally, importing textwrap is little slow. files: M Lib/ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index 24f24b17bf1..75caae0c440 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -91,7 +91,6 @@ """ import ipaddress -import textwrap import re import sys import os @@ -1201,9 +1200,10 @@ def DER_cert_to_PEM_cert(der_cert_bytes): PEM version of it as a string.""" f = str(base64.standard_b64encode(der_cert_bytes), 'ASCII', 'strict') - return (PEM_HEADER + '\n' + - textwrap.fill(f, 64) + '\n' + - PEM_FOOTER + '\n') + ss = [PEM_HEADER] + ss += [f[i:i+64] for i in range(0, len(f), 64)] + ss.append(PEM_FOOTER + '\n') + return '\n'.join(ss) def PEM_cert_to_DER_cert(pem_cert_string): """Takes a certificate in ASCII PEM format and returns the

1 0

remove comment about long-gone SGI modules (#3850)
by Benjamin Peterson Oct. 2, 2017

Oct. 2, 2017

https://github.com/python/cpython/commit/edc05c5d88c1ce853b3d04b34679ed7d69… commit: edc05c5d88c1ce853b3d04b34679ed7d69c2142a branch: master author: Benjamin Peterson <benjamin(a)python.org> committer: GitHub <noreply(a)github.com> date: 2017-10-01T23:37:57-07:00 summary: remove comment about long-gone SGI modules (#3850) files: M setup.py diff --git a/setup.py b/setup.py index 0b536c8ca32..11c4ec67f43 100644 --- a/setup.py +++ b/setup.py @@ -615,8 +615,6 @@ def detect_modules(self): math_libs = self.detect_math_libs() - # XXX Omitted modules: gl, pure, dl, SGI-specific modules - # # The following modules are all pretty straightforward, and compile # on pretty much any POSIXish platform.

1 0

[3.6] IDLE: make filetypes a tuple constant. (GH-3847) (#3848)
by Terry Jan Reedy Oct. 1, 2017

Oct. 1, 2017

https://github.com/python/cpython/commit/c0418160457970c4654ad8d910cd0ad21a… commit: c0418160457970c4654ad8d910cd0ad21a02f3f7 branch: 3.6 author: Terry Jan Reedy <tjreedy(a)udel.edu> committer: GitHub <noreply(a)github.com> date: 2017-10-01T19:37:19-04:00 summary: [3.6] IDLE: make filetypes a tuple constant. (GH-3847) (#3848) Save a bit of code, plus non-tuples get converted anyway to tuples by tkinter for the tk call. (cherry picked from commit 5961e7c156f90c7f9444ae95b9d3e55114ca2169) files: M Lib/idlelib/iomenu.py diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index 3414c7b3aff..f9b6907b40c 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -487,11 +487,11 @@ def print_window(self, event): opendialog = None savedialog = None - filetypes = [ + filetypes = ( ("Python files", "*.py *.pyw", "TEXT"), ("Text files", "*.txt", "TEXT"), ("All files", "*"), - ] + ) defaultextension = '.py' if sys.platform == 'darwin' else ''

1 0

IDLE: make filetypes a tuple constant. (#3847)
by Terry Jan Reedy Oct. 1, 2017

Oct. 1, 2017

https://github.com/python/cpython/commit/5961e7c156f90c7f9444ae95b9d3e55114… commit: 5961e7c156f90c7f9444ae95b9d3e55114ca2169 branch: master author: Terry Jan Reedy <tjreedy(a)udel.edu> committer: GitHub <noreply(a)github.com> date: 2017-10-01T19:01:27-04:00 summary: IDLE: make filetypes a tuple constant. (#3847) Save a bit of code, plus non-tuples get converted anyway to tuples by tkinter for the tk call. files: M Lib/idlelib/iomenu.py diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index 3414c7b3aff..f9b6907b40c 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -487,11 +487,11 @@ def print_window(self, event): opendialog = None savedialog = None - filetypes = [ + filetypes = ( ("Python files", "*.py *.pyw", "TEXT"), ("Text files", "*.txt", "TEXT"), ("All files", "*"), - ] + ) defaultextension = '.py' if sys.platform == 'darwin' else ''

1 0

Daily reference leaks (4243df51fe43): sum=3
by solipsis＠pitrou.net Oct. 1, 2017

Oct. 1, 2017

results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [7, -7, 1] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_spawn leaked [0, 0, -2] memory blocks, sum=-2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogrq2FrR', '--timeout', '7200']

1 0

bpo-31336: Speed up type creation. (#3279)
by Serhiy Storchaka Oct. 1, 2017

Oct. 1, 2017

https://github.com/python/cpython/commit/2102c789035ccacbac4362589402ac68ba… commit: 2102c789035ccacbac4362589402ac68baa2cd29 branch: master author: scoder <stefan_ml(a)behnel.de> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2017-10-01T11:37:47+03:00 summary: bpo-31336: Speed up type creation. (#3279) Speed up class creation by 10-20% by reducing the overhead in the necessary special method lookups. files: A Misc/NEWS.d/next/Core and Builtins/2017-09-13-12-04-23.bpo-31336.gi2ahY.rst M Objects/typeobject.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-13-12-04-23.bpo-31336.gi2ahY.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-13-12-04-23.bpo-31336.gi2ahY.rst new file mode 100644 index 00000000000..e62b065af17 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-13-12-04-23.bpo-31336.gi2ahY.rst @@ -0,0 +1,2 @@ +Speed up class creation by 10-20% by reducing the overhead in the +necessary special method lookups. Patch by Stefan Behnel. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 5e0d81f6859..e72460bfbb3 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2367,35 +2367,39 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) &bases, &PyDict_Type, &orig_dict)) return NULL; - /* Determine the proper metatype to deal with this: */ - winner = _PyType_CalculateMetaclass(metatype, bases); - if (winner == NULL) { - return NULL; - } - - if (winner != metatype) { - if (winner->tp_new != type_new) /* Pass it to the winner */ - return winner->tp_new(winner, args, kwds); - metatype = winner; - } - /* Adjust for empty tuple bases */ nbases = PyTuple_GET_SIZE(bases); if (nbases == 0) { - bases = PyTuple_Pack(1, &PyBaseObject_Type); + base = &PyBaseObject_Type; + bases = PyTuple_Pack(1, base); if (bases == NULL) - goto error; + return NULL; nbases = 1; } - else - Py_INCREF(bases); + else { + /* Search the bases for the proper metatype to deal with this: */ + winner = _PyType_CalculateMetaclass(metatype, bases); + if (winner == NULL) { + return NULL; + } - /* Calculate best base, and check that all bases are type objects */ - base = best_base(bases); - if (base == NULL) { - goto error; + if (winner != metatype) { + if (winner->tp_new != type_new) /* Pass it to the winner */ + return winner->tp_new(winner, args, kwds); + metatype = winner; + } + + /* Calculate best base, and check that all bases are type objects */ + base = best_base(bases); + if (base == NULL) { + return NULL; + } + + Py_INCREF(bases); } + /* Use "goto error" from this point on as we now own the reference to "bases". */ + dict = PyDict_Copy(orig_dict); if (dict == NULL) goto error; @@ -2945,25 +2949,23 @@ PyType_GetSlot(PyTypeObject *type, int slot) return *(void**)(((char*)type) + slotoffsets[slot]); } -/* Internal API to look for a name through the MRO. - This returns a borrowed reference, and doesn't set an exception! */ -PyObject * -_PyType_Lookup(PyTypeObject *type, PyObject *name) +/* Internal API to look for a name through the MRO, bypassing the method cache. + This returns a borrowed reference, and might set an exception. + 'error' is set to: -1: error with exception; 1: error without exception; 0: ok */ +static PyObject * +find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) { Py_ssize_t i, n; PyObject *mro, *res, *base, *dict; - unsigned int h; + Py_hash_t hash; - if (MCACHE_CACHEABLE_NAME(name) && - PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { - /* fast path */ - h = MCACHE_HASH_METHOD(type, name); - if (method_cache[h].version == type->tp_version_tag && - method_cache[h].name == name) { -#if MCACHE_STATS - method_cache_hits++; -#endif - return method_cache[h].value; + if (!PyUnicode_CheckExact(name) || + (hash = ((PyASCIIObject *) name)->hash) == -1) + { + hash = PyObject_Hash(name); + if (hash == -1) { + *error = -1; + return NULL; } } @@ -2971,28 +2973,22 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) mro = type->tp_mro; if (mro == NULL) { - if ((type->tp_flags & Py_TPFLAGS_READYING) == 0 && - PyType_Ready(type) < 0) { - /* It's not ideal to clear the error condition, - but this function is documented as not setting - an exception, and I don't want to change that. - When PyType_Ready() can't proceed, it won't - set the "ready" flag, so future attempts to ready - the same type will call it again -- hopefully - in a context that propagates the exception out. - */ - PyErr_Clear(); - return NULL; + if ((type->tp_flags & Py_TPFLAGS_READYING) == 0) { + if (PyType_Ready(type) < 0) { + *error = -1; + return NULL; + } + mro = type->tp_mro; } - mro = type->tp_mro; if (mro == NULL) { + *error = 1; return NULL; } } res = NULL; - /* keep a strong reference to mro because type->tp_mro can be replaced - during PyDict_GetItem(dict, name) */ + /* Keep a strong reference to mro because type->tp_mro can be replaced + during dict lookup, e.g. when comparing to non-string keys. */ Py_INCREF(mro); assert(PyTuple_Check(mro)); n = PyTuple_GET_SIZE(mro); @@ -3001,11 +2997,61 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) assert(PyType_Check(base)); dict = ((PyTypeObject *)base)->tp_dict; assert(dict && PyDict_Check(dict)); - res = PyDict_GetItem(dict, name); + res = _PyDict_GetItem_KnownHash(dict, name, hash); if (res != NULL) break; + if (PyErr_Occurred()) { + *error = -1; + goto done; + } } + *error = 0; +done: Py_DECREF(mro); + return res; +} + +/* Internal API to look for a name through the MRO. + This returns a borrowed reference, and doesn't set an exception! */ +PyObject * +_PyType_Lookup(PyTypeObject *type, PyObject *name) +{ + PyObject *res; + int error; + unsigned int h; + + if (MCACHE_CACHEABLE_NAME(name) && + PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { + /* fast path */ + h = MCACHE_HASH_METHOD(type, name); + if (method_cache[h].version == type->tp_version_tag && + method_cache[h].name == name) { +#if MCACHE_STATS + method_cache_hits++; +#endif + return method_cache[h].value; + } + } + + /* We may end up clearing live exceptions below, so make sure it's ours. */ + assert(!PyErr_Occurred()); + + res = find_name_in_mro(type, name, &error); + /* Only put NULL results into cache if there was no error. */ + if (error) { + /* It's not ideal to clear the error condition, + but this function is documented as not setting + an exception, and I don't want to change that. + E.g., when PyType_Ready() can't proceed, it won't + set the "ready" flag, so future attempts to ready + the same type will call it again -- hopefully + in a context that propagates the exception out. + */ + if (error == -1) { + PyErr_Clear(); + } + return NULL; + } if (MCACHE_CACHEABLE_NAME(name) && assign_version_tag(type)) { h = MCACHE_HASH_METHOD(type, name); @@ -6965,6 +7011,7 @@ update_one_slot(PyTypeObject *type, slotdef *p) void *generic = NULL, *specific = NULL; int use_generic = 0; int offset = p->offset; + int error; void **ptr = slotptr(type, offset); if (ptr == NULL) { @@ -6973,9 +7020,18 @@ update_one_slot(PyTypeObject *type, slotdef *p) } while (p->offset == offset); return p; } + /* We may end up clearing live exceptions below, so make sure it's ours. */ + assert(!PyErr_Occurred()); do { - descr = _PyType_Lookup(type, p->name_strobj); + /* Use faster uncached lookup as we won't get any cache hits during type setup. */ + descr = find_name_in_mro(type, p->name_strobj, &error); if (descr == NULL) { + if (error == -1) { + /* It is unlikely by not impossible that there has been an exception + during lookup. Since this function originally expected no errors, + we ignore them here in order to keep up the interface. */ + PyErr_Clear(); + } if (ptr == (void**)&type->tp_iternext) { specific = (void *)_PyObject_NextNotImplemented; }

1 0