Mailman 3 March 2015 - Python-checkins

cpython: Issue #10395: Added os.path.commonpath(). Implemented in posixpath and ntpath.
by serhiy.storchaka March 31, 2015

March 31, 2015

https://hg.python.org/cpython/rev/ec6c812fbc1f changeset: 95326:ec6c812fbc1f user: Serhiy Storchaka <storchaka(a)gmail.com> date: Tue Mar 31 15:31:53 2015 +0300 summary: Issue #10395: Added os.path.commonpath(). Implemented in posixpath and ntpath. Based on patch by Rafik Draoui. files: Doc/library/os.path.rst | 19 ++++++- Doc/whatsnew/3.5.rst | 7 ++ Lib/ntpath.py | 63 +++++++++++++++++++++++- Lib/posixpath.py | 45 ++++++++++… [View More]++++++- Lib/test/test_ntpath.py | 69 ++++++++++++++++++++++++++ Lib/test/test_posixpath.py | 54 ++++++++++++++++++++ Misc/NEWS | 3 + 7 files changed, 255 insertions(+), 5 deletions(-) diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -66,11 +66,24 @@ empty string (``''``). +.. function:: commonpath(paths) + + Return the longest common sub-path of each pathname in the sequence + *paths*. Raise ValueError if *paths* contains both absolute and relative + pathnames, or if *paths* is empty. Unlike :func:`commonprefix`, this + returns a valid path. + + Availability: Unix, Windows + + .. versionadded:: 3.5 + + .. function:: commonprefix(list) - Return the longest path prefix (taken character-by-character) that is a prefix - of all paths in *list*. If *list* is empty, return the empty string (``''``). - Note that this may return invalid paths because it works a character at a time. + Return the longest path prefix (taken character-by-character) that is a + prefix of all paths in *list*. If *list* is empty, return the empty string + (``''``). Note that this may return invalid paths because it works a + character at a time. To obtain a valid path, see :func:`commonpath`. .. function:: dirname(path) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -370,6 +370,13 @@ * :class:`os.stat_result` now has a :attr:`~os.stat_result.st_file_attributes` attribute on Windows. (Contributed by Ben Hoyt in :issue:`21719`.) +os.path +------- + +* New :func:`~os.path.commonpath` function that extracts common path prefix. + Unlike the :func:`~os.path.commonprefix` function, it always returns a valid + patch. (Contributed by Rafik Draoui and Serhiy Storchaka in :issue:`10395`.) + pickle ------ diff --git a/Lib/ntpath.py b/Lib/ntpath.py --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -17,7 +17,7 @@ "ismount", "expanduser","expandvars","normpath","abspath", "splitunc","curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", - "samefile", "sameopenfile", "samestat",] + "samefile", "sameopenfile", "samestat", "commonpath"] # strings representing various path-related bits and pieces # These are primarily for export; internally, they are hardcoded. @@ -589,6 +589,67 @@ raise +# Return the longest common sub-path of the sequence of paths given as input. +# The function is case-insensitive and 'separator-insensitive', i.e. if the +# only difference between two paths is the use of '\' versus '/' as separator, +# they are deemed to be equal. +# +# However, the returned path will have the standard '\' separator (even if the +# given paths had the alternative '/' separator) and will have the case of the +# first path given in the sequence. Additionally, any trailing separator is +# stripped from the returned path. + +def commonpath(paths): + """Given a sequence of path names, returns the longest common sub-path.""" + + if not paths: + raise ValueError('commonpath() arg is an empty sequence') + + if isinstance(paths[0], bytes): + sep = b'\\' + altsep = b'/' + curdir = b'.' + else: + sep = '\\' + altsep = '/' + curdir = '.' + + try: + drivesplits = [splitdrive(p.replace(altsep, sep).lower()) for p in paths] + split_paths = [p.split(sep) for d, p in drivesplits] + + try: + isabs, = set(p[:1] == sep for d, p in drivesplits) + except ValueError: + raise ValueError("Can't mix absolute and relative paths") from None + + # Check that all drive letters or UNC paths match. The check is made only + # now otherwise type errors for mixing strings and bytes would not be + # caught. + if len(set(d for d, p in drivesplits)) != 1: + raise ValueError("Paths don't have the same drive") + + drive, path = splitdrive(paths[0].replace(altsep, sep)) + common = path.split(sep) + common = [c for c in common if c and c != curdir] + + split_paths = [[c for c in s if c and c != curdir] for s in split_paths] + s1 = min(split_paths) + s2 = max(split_paths) + for i, c in enumerate(s1): + if c != s2[i]: + common = common[:i] + break + else: + common = common[:len(s1)] + + prefix = drive + sep if isabs else drive + return prefix + sep.join(common) + except (TypeError, AttributeError): + genericpath._check_arg_types('commonpath', *paths) + raise + + # determine if two files are in fact the same file try: # GetFinalPathNameByHandle is available starting with Windows 6.0. diff --git a/Lib/posixpath.py b/Lib/posixpath.py --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -22,7 +22,8 @@ "ismount", "expanduser","expandvars","normpath","abspath", "samefile","sameopenfile","samestat", "curdir","pardir","sep","pathsep","defpath","altsep","extsep", - "devnull","realpath","supports_unicode_filenames","relpath"] + "devnull","realpath","supports_unicode_filenames","relpath", + "commonpath"] # Strings representing various path-related bits and pieces. # These are primarily for export; internally, they are hardcoded. @@ -455,3 +456,45 @@ except (TypeError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise + + +# Return the longest common sub-path of the sequence of paths given as input. +# The paths are not normalized before comparing them (this is the +# responsibility of the caller). Any trailing separator is stripped from the +# returned path. + +def commonpath(paths): + """Given a sequence of path names, returns the longest common sub-path.""" + + if not paths: + raise ValueError('commonpath() arg is an empty sequence') + + if isinstance(paths[0], bytes): + sep = b'/' + curdir = b'.' + else: + sep = '/' + curdir = '.' + + try: + split_paths = [path.split(sep) for path in paths] + + try: + isabs, = set(p[:1] == sep for p in paths) + except ValueError: + raise ValueError("Can't mix absolute and relative paths") from None + + split_paths = [[c for c in s if c and c != curdir] for s in split_paths] + s1 = min(split_paths) + s2 = max(split_paths) + common = s1 + for i, c in enumerate(s1): + if c != s2[i]: + common = s1[:i] + break + + prefix = sep if isabs else sep[:0] + return prefix + sep.join(common) + except (TypeError, AttributeError): + genericpath._check_arg_types('commonpath', *paths) + raise diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -330,6 +330,75 @@ tester('ntpath.relpath("/a/b", "/a/b")', '.') tester('ntpath.relpath("c:/foo", "C:/FOO")', '.') + def test_commonpath(self): + def check(paths, expected): + tester(('ntpath.commonpath(%r)' % paths).replace('\\\\', '\\'), + expected) + def check_error(exc, paths): + self.assertRaises(exc, ntpath.commonpath, paths) + self.assertRaises(exc, ntpath.commonpath, + [os.fsencode(p) for p in paths]) + + self.assertRaises(ValueError, ntpath.commonpath, []) + check_error(ValueError, ['C:\\Program Files', 'Program Files']) + check_error(ValueError, ['C:\\Program Files', 'C:Program Files']) + check_error(ValueError, ['\\Program Files', 'Program Files']) + check_error(ValueError, ['Program Files', 'C:\\Program Files']) + check(['C:\\Program Files'], 'C:\\Program Files') + check(['C:\\Program Files', 'C:\\Program Files'], 'C:\\Program Files') + check(['C:\\Program Files\\', 'C:\\Program Files'], + 'C:\\Program Files') + check(['C:\\Program Files\\', 'C:\\Program Files\\'], + 'C:\\Program Files') + check(['C:\\\\Program Files', 'C:\\Program Files\\\\'], + 'C:\\Program Files') + check(['C:\\.\\Program Files', 'C:\\Program Files\\.'], + 'C:\\Program Files') + check(['C:\\', 'C:\\bin'], 'C:\\') + check(['C:\\Program Files', 'C:\\bin'], 'C:\\') + check(['C:\\Program Files', 'C:\\Program Files\\Bar'], + 'C:\\Program Files') + check(['C:\\Program Files\\Foo', 'C:\\Program Files\\Bar'], + 'C:\\Program Files') + check(['C:\\Program Files', 'C:\\Projects'], 'C:\\') + check(['C:\\Program Files\\', 'C:\\Projects'], 'C:\\') + + check(['C:\\Program Files\\Foo', 'C:/Program Files/Bar'], + 'C:\\Program Files') + check(['C:\\Program Files\\Foo', 'c:/program files/bar'], + 'C:\\Program Files') + check(['c:/program files/bar', 'C:\\Program Files\\Foo'], + 'c:\\program files') + + check_error(ValueError, ['C:\\Program Files', 'D:\\Program Files']) + + check(['spam'], 'spam') + check(['spam', 'spam'], 'spam') + check(['spam', 'alot'], '') + check(['and\\jam', 'and\\spam'], 'and') + check(['and\\\\jam', 'and\\spam\\\\'], 'and') + check(['and\\.\\jam', '.\\and\\spam'], 'and') + check(['and\\jam', 'and\\spam', 'alot'], '') + check(['and\\jam', 'and\\spam', 'and'], 'and') + check(['C:and\\jam', 'C:and\\spam'], 'C:and') + + check([''], '') + check(['', 'spam\\alot'], '') + check_error(ValueError, ['', '\\spam\\alot']) + + self.assertRaises(TypeError, ntpath.commonpath, + [b'C:\\Program Files', 'C:\\Program Files\\Foo']) + self.assertRaises(TypeError, ntpath.commonpath, + [b'C:\\Program Files', 'Program Files\\Foo']) + self.assertRaises(TypeError, ntpath.commonpath, + [b'Program Files', 'C:\\Program Files\\Foo']) + self.assertRaises(TypeError, ntpath.commonpath, + ['C:\\Program Files', b'C:\\Program Files\\Foo']) + self.assertRaises(TypeError, ntpath.commonpath, + ['C:\\Program Files', b'Program Files\\Foo']) + self.assertRaises(TypeError, ntpath.commonpath, + ['Program Files', b'C:\\Program Files\\Foo']) + def test_sameopenfile(self): with TemporaryFile() as tf1, TemporaryFile() as tf2: # Make sure the same file is really the same diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -522,6 +522,60 @@ finally: os.getcwdb = real_getcwdb + def test_commonpath(self): + def check(paths, expected): + self.assertEqual(posixpath.commonpath(paths), expected) + self.assertEqual(posixpath.commonpath([os.fsencode(p) for p in paths]), + os.fsencode(expected)) + def check_error(exc, paths): + self.assertRaises(exc, posixpath.commonpath, paths) + self.assertRaises(exc, posixpath.commonpath, + [os.fsencode(p) for p in paths]) + + self.assertRaises(ValueError, posixpath.commonpath, []) + check_error(ValueError, ['/usr', 'usr']) + check_error(ValueError, ['usr', '/usr']) + + check(['/usr/local'], '/usr/local') + check(['/usr/local', '/usr/local'], '/usr/local') + check(['/usr/local/', '/usr/local'], '/usr/local') + check(['/usr/local/', '/usr/local/'], '/usr/local') + check(['/usr//local', '//usr/local'], '/usr/local') + check(['/usr/./local', '/./usr/local'], '/usr/local') + check(['/', '/dev'], '/') + check(['/usr', '/dev'], '/') + check(['/usr/lib/', '/usr/lib/python3'], '/usr/lib') + check(['/usr/lib/', '/usr/lib64/'], '/usr') + + check(['/usr/lib', '/usr/lib64'], '/usr') + check(['/usr/lib/', '/usr/lib64'], '/usr') + + check(['spam'], 'spam') + check(['spam', 'spam'], 'spam') + check(['spam', 'alot'], '') + check(['and/jam', 'and/spam'], 'and') + check(['and//jam', 'and/spam//'], 'and') + check(['and/./jam', './and/spam'], 'and') + check(['and/jam', 'and/spam', 'alot'], '') + check(['and/jam', 'and/spam', 'and'], 'and') + + check([''], '') + check(['', 'spam/alot'], '') + check_error(ValueError, ['', '/spam/alot']) + + self.assertRaises(TypeError, posixpath.commonpath, + [b'/usr/lib/', '/usr/lib/python3']) + self.assertRaises(TypeError, posixpath.commonpath, + [b'/usr/lib/', 'usr/lib/python3']) + self.assertRaises(TypeError, posixpath.commonpath, + [b'usr/lib/', '/usr/lib/python3']) + self.assertRaises(TypeError, posixpath.commonpath, + ['/usr/lib/', b'/usr/lib/python3']) + self.assertRaises(TypeError, posixpath.commonpath, + ['/usr/lib/', b'usr/lib/python3']) + self.assertRaises(TypeError, posixpath.commonpath, + ['usr/lib/', b'/usr/lib/python3']) + class PosixCommonTest(test_genericpath.CommonTest, unittest.TestCase): pathmodule = posixpath diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Library ------- +- Issue #10395: Added os.path.commonpath(). Implemented in posixpath and ntpath. + Based on patch by Rafik Draoui. + - Issue #23611: Serializing more "lookupable" objects (such as unbound methods or nested classes) now are supported with pickle protocols < 4. -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython: Issue #23618: Fix internal_connect_select()
by victor.stinner March 31, 2015

March 31, 2015

https://hg.python.org/cpython/rev/47b2d1ff9743 changeset: 95325:47b2d1ff9743 user: Victor Stinner <victor.stinner(a)gmail.com> date: Tue Mar 31 14:24:47 2015 +0200 summary: Issue #23618: Fix internal_connect_select() files: Modules/socketmodule.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -680,7 +680,7 @@ static int … [View More]

1 0

cpython: Issue #23618: internal_connect_select() now waits also for error events
by victor.stinner March 31, 2015

March 31, 2015

https://hg.python.org/cpython/rev/c027d6468667 changeset: 95324:c027d6468667 user: Victor Stinner <victor.stinner(a)gmail.com> date: Tue Mar 31 13:56:29 2015 +0200 summary: Issue #23618: internal_connect_select() now waits also for error events files: Modules/socketmodule.c | 21 +++++++++++++++------ 1 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c --- a/Modules/socketmodule.c +++ b/Modules/… [View More]socketmodule.c @@ -592,14 +592,15 @@ } static int -internal_select_impl(PySocketSockObject *s, int writing, _PyTime_t interval) +internal_select_impl(PySocketSockObject *s, int writing, _PyTime_t interval, + int error) { int n; #ifdef HAVE_POLL struct pollfd pollfd; _PyTime_t ms; #else - fd_set fds; + fd_set fds, efds; struct timeval tv; #endif @@ -608,6 +609,9 @@ assert(PyGILState_Check()); #endif + /* Error condition is for output only */ + assert(!(error && !writing)); + /* Nothing to do unless we're in timeout mode (not non-blocking) */ if (s->sock_timeout <= 0) return 0; @@ -625,6 +629,8 @@ #ifdef HAVE_POLL pollfd.fd = s->sock_fd; pollfd.events = writing ? POLLOUT : POLLIN; + if (error) + pollfd.events |= POLLERR; /* s->sock_timeout is in seconds, timeout in ms */ ms = _PyTime_AsMilliseconds(interval, _PyTime_ROUND_CEILING); @@ -638,15 +644,18 @@ FD_ZERO(&fds); FD_SET(s->sock_fd, &fds); + FD_ZERO(&efds); + if (error) + FD_SET(s->sock_fd, &efds); /* See if the socket is ready */ Py_BEGIN_ALLOW_THREADS; if (writing) n = select(Py_SAFE_DOWNCAST(s->sock_fd+1, SOCKET_T, int), - NULL, &fds, NULL, &tv); + NULL, &fds, &efds, &tv); else n = select(Py_SAFE_DOWNCAST(s->sock_fd+1, SOCKET_T, int), - &fds, NULL, NULL, &tv); + &fds, NULL, &efds, &tv); Py_END_ALLOW_THREADS; #endif @@ -665,13 +674,13 @@ static int internal_select(PySocketSockObject *s, int writing, _PyTime_t interval) { - return internal_select_impl(s, writing, interval); + return internal_select_impl(s, writing, interval, 0); } static int internal_connect_select(PySocketSockObject *s) { - return internal_select(s, 1, s->sock_timeout); + return internal_select(s, 1, s->sock_timeout, 1); } /* -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython: Issue #23618: Refactor internal_select() to prepare socket.connect() for EINTR
by victor.stinner March 31, 2015

March 31, 2015

https://hg.python.org/cpython/rev/fa5542660b17 changeset: 95323:fa5542660b17 user: Victor Stinner <victor.stinner(a)gmail.com> date: Tue Mar 31 13:50:44 2015 +0200 summary: Issue #23618: Refactor internal_select() to prepare socket.connect() for EINTR files: Modules/socketmodule.c | 36 +++++++++++++++++++---------- 1 files changed, 23 insertions(+), 13 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c --- a/Modules/socketmodule.c +++ b/… [View More]Modules/socketmodule.c @@ -591,19 +591,13 @@ return 1; } -/* Do a select()/poll() on the socket, if necessary (sock_timeout > 0). - The argument writing indicates the direction. - This does not raise an exception; we'll let our caller do that - after they've reacquired the interpreter lock. - Returns 1 on timeout, -1 on error, 0 otherwise. */ static int -internal_select(PySocketSockObject *s, int writing, _PyTime_t interval) +internal_select_impl(PySocketSockObject *s, int writing, _PyTime_t interval) { int n; #ifdef HAVE_POLL struct pollfd pollfd; - _PyTime_t timeout; - int timeout_int; + _PyTime_t ms; #else fd_set fds; struct timeval tv; @@ -633,12 +627,11 @@ pollfd.events = writing ? POLLOUT : POLLIN; /* s->sock_timeout is in seconds, timeout in ms */ - timeout = _PyTime_AsMilliseconds(interval, _PyTime_ROUND_CEILING); - assert(timeout <= INT_MAX); - timeout_int = (int)timeout; + ms = _PyTime_AsMilliseconds(interval, _PyTime_ROUND_CEILING); + assert(ms <= INT_MAX); Py_BEGIN_ALLOW_THREADS; - n = poll(&pollfd, 1, timeout_int); + n = poll(&pollfd, 1, (int)ms); Py_END_ALLOW_THREADS; #else _PyTime_AsTimeval_noraise(interval, &tv, _PyTime_ROUND_CEILING); @@ -664,6 +657,23 @@ return 0; } +/* Do a select()/poll() on the socket, if necessary (sock_timeout > 0). + The argument writing indicates the direction. + This does not raise an exception; we'll let our caller do that + after they've reacquired the interpreter lock. + Returns 1 on timeout, -1 on error, 0 otherwise. */ +static int +internal_select(PySocketSockObject *s, int writing, _PyTime_t interval) +{ + return internal_select_impl(s, writing, interval); +} + +static int +internal_connect_select(PySocketSockObject *s) +{ + return internal_select(s, 1, s->sock_timeout); +} + /* Two macros for automatic retry of select() in case of false positives (for example, select() could indicate a socket is ready for reading @@ -2492,7 +2502,7 @@ if (s->sock_timeout > 0 && res < 0 && errno == EINPROGRESS && IS_SELECTABLE(s)) { - timeout = internal_select(s, 1, s->sock_timeout); + timeout = internal_connect_select(s); if (timeout == 0) { /* Bug #1019808: in case of an EINPROGRESS, -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython: Issue #23618: Refactor the _socket module
by victor.stinner March 31, 2015

March 31, 2015

https://hg.python.org/cpython/rev/e8246baad0f6 changeset: 95322:e8246baad0f6 user: Victor Stinner <victor.stinner(a)gmail.com> date: Tue Mar 31 13:15:31 2015 +0200 summary: Issue #23618: Refactor the _socket module * Inline internal_select() function * Rename internal_select_ex() internal_select() files: Modules/socketmodule.c | 28 +++++++++++----------------- 1 files changed, 11 insertions(+), 17 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/… [View More]socketmodule.c --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -597,7 +597,7 @@ after they've reacquired the interpreter lock. Returns 1 on timeout, -1 on error, 0 otherwise. */ static int -internal_select_ex(PySocketSockObject *s, int writing, _PyTime_t interval) +internal_select(PySocketSockObject *s, int writing, _PyTime_t interval) { int n; #ifdef HAVE_POLL @@ -664,12 +664,6 @@ return 0; } -static int -internal_select(PySocketSockObject *s, int writing) -{ - return internal_select_ex(s, writing, s->sock_timeout); -} - /* Two macros for automatic retry of select() in case of false positives (for example, select() could indicate a socket is ready for reading @@ -678,7 +672,7 @@ BEGIN_SELECT_LOOP(s) - timeout = internal_select_ex(s, 0, interval); + timeout = internal_select(s, 0, interval); if (!timeout) { Py_BEGIN_ALLOW_THREADS @@ -2075,7 +2069,7 @@ BEGIN_SELECT_LOOP(s) do { - timeout = internal_select_ex(s, 0, interval); + timeout = internal_select(s, 0, interval); if (!timeout) { Py_BEGIN_ALLOW_THREADS @@ -2498,7 +2492,7 @@ if (s->sock_timeout > 0 && res < 0 && errno == EINPROGRESS && IS_SELECTABLE(s)) { - timeout = internal_select(s, 1); + timeout = internal_select(s, 1, s->sock_timeout); if (timeout == 0) { /* Bug #1019808: in case of an EINPROGRESS, @@ -2731,7 +2725,7 @@ BEGIN_SELECT_LOOP(s) do { - timeout = internal_select_ex(s, 0, interval); + timeout = internal_select(s, 0, interval); if (!timeout) { Py_BEGIN_ALLOW_THREADS @@ -2907,7 +2901,7 @@ BEGIN_SELECT_LOOP(s) do { memset(&addrbuf, 0, addrlen); - timeout = internal_select_ex(s, 0, interval); + timeout = internal_select(s, 0, interval); if (!timeout) { Py_BEGIN_ALLOW_THREADS @@ -3114,7 +3108,7 @@ msg.msg_iovlen = iovlen; msg.msg_control = controlbuf; msg.msg_controllen = controllen; - timeout = internal_select_ex(s, 0, interval); + timeout = internal_select(s, 0, interval); if (timeout == 1) { PyErr_SetString(socket_timeout, "timed out"); @@ -3407,7 +3401,7 @@ BEGIN_SELECT_LOOP(s) do { - timeout = internal_select_ex(s, 1, interval); + timeout = internal_select(s, 1, interval); if (!timeout) { Py_BEGIN_ALLOW_THREADS @@ -3464,7 +3458,7 @@ } do { - timeout = internal_select(s, 1); + timeout = internal_select(s, 1, s->sock_timeout); n = -1; if (!timeout) { @@ -3554,7 +3548,7 @@ BEGIN_SELECT_LOOP(s) do { - timeout = internal_select_ex(s, 1, interval); + timeout = internal_select(s, 1, interval); if (!timeout) { Py_BEGIN_ALLOW_THREADS @@ -3770,7 +3764,7 @@ BEGIN_SELECT_LOOP(s) do { - timeout = internal_select_ex(s, 1, interval); + timeout = internal_select(s, 1, interval); if (!timeout) { Py_BEGIN_ALLOW_THREADS; -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython: What's New in Python 3.5, PEP 475: mention modified signal functions
by victor.stinner March 31, 2015

March 31, 2015

https://hg.python.org/cpython/rev/5f1b7ab3517b changeset: 95321:5f1b7ab3517b user: Victor Stinner <victor.stinner(a)gmail.com> date: Tue Mar 31 12:19:15 2015 +0200 summary: What's New in Python 3.5, PEP 475: mention modified signal functions files: Doc/library/signal.rst | 4 ++-- Doc/whatsnew/3.5.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst --- a/Doc/library/signal.rst +++ b/Doc/… [View More]

1 0

cpython: Issue #23611: Serializing more "lookupable" objects (such as unbound methods
by serhiy.storchaka March 31, 2015

March 31, 2015

https://hg.python.org/cpython/rev/ca12465418bd changeset: 95320:ca12465418bd user: Serhiy Storchaka <storchaka(a)gmail.com> date: Tue Mar 31 14:07:24 2015 +0300 summary: Issue #23611: Serializing more "lookupable" objects (such as unbound methods or nested classes) now are supported with pickle protocols < 4. files: Doc/whatsnew/3.5.rst | 7 + Lib/pickle.py | 33 ++++--- Lib/test/pickletester.py | 19 +++- Misc/NEWS | 3 + … [View More] Modules/_pickle.c | 120 ++++++++++++++++---------- 5 files changed, 115 insertions(+), 67 deletions(-) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -370,6 +370,13 @@ * :class:`os.stat_result` now has a :attr:`~os.stat_result.st_file_attributes` attribute on Windows. (Contributed by Ben Hoyt in :issue:`21719`.) +pickle +------ + +* Serializing more "lookupable" objects (such as unbound methods or nested + classes) now are supported with pickle protocols < 4. + (Contributed by Serhiy Storchaka in :issue:`23611`.) + re -- diff --git a/Lib/pickle.py b/Lib/pickle.py --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -258,24 +258,20 @@ # Tools used for pickling. -def _getattribute(obj, name, allow_qualname=False): - dotted_path = name.split(".") - if not allow_qualname and len(dotted_path) > 1: - raise AttributeError("Can't get qualified attribute {!r} on {!r}; " + - "use protocols >= 4 to enable support" - .format(name, obj)) - for subpath in dotted_path: +def _getattribute(obj, name): + for subpath in name.split('.'): if subpath == '<locals>': raise AttributeError("Can't get local attribute {!r} on {!r}" .format(name, obj)) try: + parent = obj obj = getattr(obj, subpath) except AttributeError: raise AttributeError("Can't get attribute {!r} on {!r}" .format(name, obj)) - return obj + return obj, parent -def whichmodule(obj, name, allow_qualname=False): +def whichmodule(obj, name): """Find the module an object belong to.""" module_name = getattr(obj, '__module__', None) if module_name is not None: @@ -286,7 +282,7 @@ if module_name == '__main__' or module is None: continue try: - if _getattribute(module, name, allow_qualname) is obj: + if _getattribute(module, name)[0] is obj: return module_name except AttributeError: pass @@ -899,16 +895,16 @@ write = self.write memo = self.memo - if name is None and self.proto >= 4: + if name is None: name = getattr(obj, '__qualname__', None) if name is None: name = obj.__name__ - module_name = whichmodule(obj, name, allow_qualname=self.proto >= 4) + module_name = whichmodule(obj, name) try: __import__(module_name, level=0) module = sys.modules[module_name] - obj2 = _getattribute(module, name, allow_qualname=self.proto >= 4) + obj2, parent = _getattribute(module, name) except (ImportError, KeyError, AttributeError): raise PicklingError( "Can't pickle %r: it's not found as %s.%s" % @@ -930,11 +926,16 @@ else: write(EXT4 + pack("<i", code)) return + lastname = name.rpartition('.')[2] + if parent is module: + name = lastname # Non-ASCII identifiers are supported only with protocols >= 3. if self.proto >= 4: self.save(module_name) self.save(name) write(STACK_GLOBAL) + elif parent is not module: + self.save_reduce(getattr, (parent, lastname)) elif self.proto >= 3: write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + bytes(name, "utf-8") + b'\n') @@ -1373,8 +1374,10 @@ elif module in _compat_pickle.IMPORT_MAPPING: module = _compat_pickle.IMPORT_MAPPING[module] __import__(module, level=0) - return _getattribute(sys.modules[module], name, - allow_qualname=self.proto >= 4) + if self.proto >= 4: + return _getattribute(sys.modules[module], name)[0] + else: + return getattr(sys.modules[module], name) def load_reduce(self): stack = self.stack diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1602,13 +1602,24 @@ class B: class C: pass - - for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): for obj in [Nested.A, Nested.A.B, Nested.A.B.C]: with self.subTest(proto=proto, obj=obj): unpickled = self.loads(self.dumps(obj, proto)) self.assertIs(obj, unpickled) + def test_recursive_nested_names(self): + global Recursive + class Recursive: + pass + Recursive.mod = sys.modules[Recursive.__module__] + Recursive.__qualname__ = 'Recursive.mod.Recursive' + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(Recursive, proto)) + self.assertIs(unpickled, Recursive) + del Recursive.mod # break reference loop + def test_py_methods(self): global PyMethodsTest class PyMethodsTest: @@ -1647,7 +1658,7 @@ (PyMethodsTest.biscuits, PyMethodsTest), (PyMethodsTest.Nested.pie, PyMethodsTest.Nested) ) - for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): for method in py_methods: with self.subTest(proto=proto, method=method): unpickled = self.loads(self.dumps(method, proto)) @@ -1687,7 +1698,7 @@ (Subclass.Nested("sweet").count, ("e",)), (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")), ) - for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): for method, args in c_methods: with self.subTest(proto=proto, method=method): unpickled = self.loads(self.dumps(method, proto)) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Library ------- +- Issue #23611: Serializing more "lookupable" objects (such as unbound methods + or nested classes) now are supported with pickle protocols < 4. + - Issue #13583: sqlite3.Row now supports slice indexing. - Issue #18473: Fixed 2to3 and 3to2 compatible pickle mappings. Fixed diff --git a/Modules/_pickle.c b/Modules/_pickle.c --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -152,6 +152,8 @@ /* codecs.encode, used for saving bytes in older protocols */ PyObject *codecs_encode; + /* builtins.getattr, used for saving nested names with protocol < 4 */ + PyObject *getattr; } PickleState; /* Forward declaration of the _pickle module definition. */ @@ -188,16 +190,26 @@ Py_CLEAR(st->name_mapping_3to2); Py_CLEAR(st->import_mapping_3to2); Py_CLEAR(st->codecs_encode); + Py_CLEAR(st->getattr); } /* Initialize the given pickle module state. */ static int _Pickle_InitState(PickleState *st) { + PyObject *builtins; PyObject *copyreg = NULL; PyObject *compat_pickle = NULL; PyObject *codecs = NULL; + builtins = PyEval_GetBuiltins(); + if (builtins == NULL) + goto error; + st->getattr = PyDict_GetItemString(builtins, "getattr"); + if (st->getattr == NULL) + goto error; + Py_INCREF(st->getattr); + copyreg = PyImport_ImportModule("copyreg"); if (!copyreg) goto error; @@ -1535,7 +1547,7 @@ } static PyObject * -get_dotted_path(PyObject *obj, PyObject *name, int allow_qualname) { +get_dotted_path(PyObject *obj, PyObject *name) { _Py_static_string(PyId_dot, "."); _Py_static_string(PyId_locals, "<locals>"); PyObject *dotted_path; @@ -1546,20 +1558,6 @@ return NULL; n = PyList_GET_SIZE(dotted_path); assert(n >= 1); - if (!allow_qualname && n > 1) { - if (obj == NULL) - PyErr_Format(PyExc_AttributeError, - "Can't pickle qualified object %R; " - "use protocols >= 4 to enable support", - name); - else - PyErr_Format(PyExc_AttributeError, - "Can't pickle qualified attribute %R on %R; " - "use protocols >= 4 to enable support", - name, obj); - Py_DECREF(dotted_path); - return NULL; - } for (i = 0; i < n; i++) { PyObject *subpath = PyList_GET_ITEM(dotted_path, i); PyObject *result = PyUnicode_RichCompare( @@ -1582,22 +1580,28 @@ } static PyObject * -get_deep_attribute(PyObject *obj, PyObject *names) +get_deep_attribute(PyObject *obj, PyObject *names, PyObject **pparent) { Py_ssize_t i, n; + PyObject *parent = NULL; assert(PyList_CheckExact(names)); Py_INCREF(obj); n = PyList_GET_SIZE(names); for (i = 0; i < n; i++) { PyObject *name = PyList_GET_ITEM(names, i); - PyObject *tmp; - tmp = PyObject_GetAttr(obj, name); - Py_DECREF(obj); - if (tmp == NULL) + Py_XDECREF(parent); + parent = obj; + obj = PyObject_GetAttr(parent, name); + if (obj == NULL) { + Py_DECREF(parent); return NULL; - obj = tmp; - } + } + } + if (pparent != NULL) + *pparent = parent; + else + Py_XDECREF(parent); return obj; } @@ -1617,18 +1621,22 @@ { PyObject *dotted_path, *attr; - dotted_path = get_dotted_path(obj, name, allow_qualname); - if (dotted_path == NULL) - return NULL; - attr = get_deep_attribute(obj, dotted_path); - Py_DECREF(dotted_path); + if (allow_qualname) { + dotted_path = get_dotted_path(obj, name); + if (dotted_path == NULL) + return NULL; + attr = get_deep_attribute(obj, dotted_path, NULL); + Py_DECREF(dotted_path); + } + else + attr = PyObject_GetAttr(obj, name); if (attr == NULL) reformat_attribute_error(obj, name); return attr; } static PyObject * -whichmodule(PyObject *global, PyObject *global_name, int allow_qualname) +whichmodule(PyObject *global, PyObject *dotted_path) { PyObject *module_name; PyObject *modules_dict; @@ -1637,7 +1645,6 @@ _Py_IDENTIFIER(__module__); _Py_IDENTIFIER(modules); _Py_IDENTIFIER(__main__); - PyObject *dotted_path; module_name = _PyObject_GetAttrId(global, &PyId___module__); @@ -1663,10 +1670,6 @@ return NULL; } - dotted_path = get_dotted_path(NULL, global_name, allow_qualname); - if (dotted_path == NULL) - return NULL; - i = 0; while (PyDict_Next(modules_dict, &i, &module_name, &module)) { PyObject *candidate; @@ -1676,19 +1679,16 @@ if (module == Py_None) continue; - candidate = get_deep_attribute(module, dotted_path); + candidate = get_deep_attribute(module, dotted_path, NULL); if (candidate == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - Py_DECREF(dotted_path); + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return NULL; - } PyErr_Clear(); continue; } if (candidate == global) { Py_INCREF(module_name); - Py_DECREF(dotted_path); Py_DECREF(candidate); return module_name; } @@ -1698,7 +1698,6 @@ /* If no module is found, use __main__. */ module_name = _PyUnicode_FromId(&PyId___main__); Py_INCREF(module_name); - Py_DECREF(dotted_path); return module_name; } @@ -3105,6 +3104,9 @@ PyObject *global_name = NULL; PyObject *module_name = NULL; PyObject *module = NULL; + PyObject *parent = NULL; + PyObject *dotted_path = NULL; + PyObject *lastname = NULL; PyObject *cls; PickleState *st = _Pickle_GetGlobalState(); int status = 0; @@ -3118,13 +3120,11 @@ global_name = name; } else { - if (self->proto >= 4) { - global_name = _PyObject_GetAttrId(obj, &PyId___qualname__); - if (global_name == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) - goto error; - PyErr_Clear(); - } + global_name = _PyObject_GetAttrId(obj, &PyId___qualname__); + if (global_name == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) + goto error; + PyErr_Clear(); } if (global_name == NULL) { global_name = _PyObject_GetAttrId(obj, &PyId___name__); @@ -3133,7 +3133,10 @@ } } - module_name = whichmodule(obj, global_name, self->proto >= 4); + dotted_path = get_dotted_path(module, global_name); + if (dotted_path == NULL) + goto error; + module_name = whichmodule(obj, dotted_path); if (module_name == NULL) goto error; @@ -3152,7 +3155,10 @@ obj, module_name); goto error; } - cls = getattribute(module, global_name, self->proto >= 4); + lastname = PyList_GET_ITEM(dotted_path, PyList_GET_SIZE(dotted_path)-1); + Py_INCREF(lastname); + cls = get_deep_attribute(module, dotted_path, &parent); + Py_CLEAR(dotted_path); if (cls == NULL) { PyErr_Format(st->PicklingError, "Can't pickle %R: attribute lookup %S on %S failed", @@ -3239,6 +3245,11 @@ } else { gen_global: + if (parent == module) { + Py_INCREF(lastname); + Py_DECREF(global_name); + global_name = lastname; + } if (self->proto >= 4) { const char stack_global_op = STACK_GLOBAL; @@ -3250,6 +3261,15 @@ if (_Pickler_Write(self, &stack_global_op, 1) < 0) goto error; } + else if (parent != module) { + PickleState *st = _Pickle_GetGlobalState(); + PyObject *reduce_value = Py_BuildValue("(O(OO))", + st->getattr, parent, lastname); + status = save_reduce(self, reduce_value, NULL); + Py_DECREF(reduce_value); + if (status < 0) + goto error; + } else { /* Generate a normal global opcode if we are using a pickle protocol < 4, or if the object is not registered in the @@ -3328,6 +3348,9 @@ Py_XDECREF(module_name); Py_XDECREF(global_name); Py_XDECREF(module); + Py_XDECREF(parent); + Py_XDECREF(dotted_path); + Py_XDECREF(lastname); return status; } @@ -7150,6 +7173,7 @@ Py_VISIT(st->name_mapping_3to2); Py_VISIT(st->import_mapping_3to2); Py_VISIT(st->codecs_encode); + Py_VISIT(st->getattr); return 0; } -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython: Issue #13583: sqlite3.Row now supports slice indexing.
by serhiy.storchaka March 31, 2015

March 31, 2015

https://hg.python.org/cpython/rev/e47f520eb756 changeset: 95319:e47f520eb756 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Tue Mar 31 13:33:11 2015 +0300 summary: Issue #13583: sqlite3.Row now supports slice indexing. Tests by Jessica McKellar. files: Doc/library/sqlite3.rst | 3 +++ Lib/sqlite3/test/factory.py | 18 ++++++++++++++++++ Misc/NEWS | 2 ++ Modules/_sqlite/row.c | 3 +-- 4 files changed, 24 insertions(+), 2 … [View More]deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -649,6 +649,9 @@ This method returns a list of column names. Immediately after a query, it is the first member of each tuple in :attr:`Cursor.description`. + .. versionchanged:: 3.5 + Added support of slicing. + Let's assume we initialize a table as in the example given above:: conn = sqlite3.connect(":memory:") diff --git a/Lib/sqlite3/test/factory.py b/Lib/sqlite3/test/factory.py --- a/Lib/sqlite3/test/factory.py +++ b/Lib/sqlite3/test/factory.py @@ -111,6 +111,24 @@ with self.assertRaises(IndexError): row[2**1000] + def CheckSqliteRowSlice(self): + # A sqlite.Row can be sliced like a list. + self.con.row_factory = sqlite.Row + row = self.con.execute("select 1, 2, 3, 4").fetchone() + self.assertEqual(row[0:0], ()) + self.assertEqual(row[0:1], (1,)) + self.assertEqual(row[1:3], (2, 3)) + self.assertEqual(row[3:1], ()) + # Explicit bounds are optional. + self.assertEqual(row[1:], (2, 3, 4)) + self.assertEqual(row[:3], (1, 2, 3)) + # Slices can use negative indices. + self.assertEqual(row[-2:-1], (3,)) + self.assertEqual(row[-2:], (3, 4)) + # Slicing supports steps. + self.assertEqual(row[0:4:2], (1, 3)) + self.assertEqual(row[3:0:-2], (4, 2)) + def CheckSqliteRowIter(self): """Checks if the row object is iterable""" self.con.row_factory = sqlite.Row diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,8 @@ Library ------- +- Issue #13583: sqlite3.Row now supports slice indexing. + - Issue #18473: Fixed 2to3 and 3to2 compatible pickle mappings. Fixed ambigious reverse mappings. Added many new mappings. Import mapping is no longer applied to modules already mapped with full name mapping. diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c --- a/Modules/_sqlite/row.c +++ b/Modules/_sqlite/row.c @@ -142,8 +142,7 @@ return NULL; } else if (PySlice_Check(idx)) { - PyErr_SetString(PyExc_ValueError, "slices not implemented, yet"); - return NULL; + return PyObject_GetItem(self->data, idx); } else { PyErr_SetString(PyExc_IndexError, "Index must be int or string"); -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython (3.4): Issue #18473: Fixed 2to3 and 3to2 compatible pickle mappings.
by serhiy.storchaka March 31, 2015

March 31, 2015

https://hg.python.org/cpython/rev/8d86dfe53b97 changeset: 95316:8d86dfe53b97 branch: 3.4 parent: 95309:28b778b0a132 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Tue Mar 31 13:12:37 2015 +0300 summary: Issue #18473: Fixed 2to3 and 3to2 compatible pickle mappings. Fixed ambigious reverse mappings. Added many new mappings. Import mapping is no longer applied to modules already mapped with full name mapping. Added tests for compatible pickling and … [View More]unpickling and for consistency of _compat_pickle mappings. files: Lib/_compat_pickle.py | 121 +++++++++++++++++++-- Lib/pickle.py | 4 +- Lib/test/pickletester.py | 49 ++++++++- Lib/test/test_pickle.py | 152 ++++++++++++++++++++++++++- Misc/NEWS | 4 + Modules/_pickle.c | 26 ++-- 6 files changed, 326 insertions(+), 30 deletions(-) diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -6,18 +6,13 @@ # lib2to3 and use the mapping defined there, because lib2to3 uses pickle. # Thus, this could cause the module to be imported recursively. IMPORT_MAPPING = { - 'StringIO': 'io', - 'cStringIO': 'io', - 'cPickle': 'pickle', '__builtin__' : 'builtins', 'copy_reg': 'copyreg', 'Queue': 'queue', 'SocketServer': 'socketserver', 'ConfigParser': 'configparser', 'repr': 'reprlib', - 'FileDialog': 'tkinter.filedialog', 'tkFileDialog': 'tkinter.filedialog', - 'SimpleDialog': 'tkinter.simpledialog', 'tkSimpleDialog': 'tkinter.simpledialog', 'tkColorChooser': 'tkinter.colorchooser', 'tkCommonDialog': 'tkinter.commondialog', @@ -39,7 +34,6 @@ 'dbm': 'dbm.ndbm', 'gdbm': 'dbm.gnu', 'xmlrpclib': 'xmlrpc.client', - 'DocXMLRPCServer': 'xmlrpc.server', 'SimpleXMLRPCServer': 'xmlrpc.server', 'httplib': 'http.client', 'htmlentitydefs' : 'html.entities', @@ -47,16 +41,13 @@ 'Cookie': 'http.cookies', 'cookielib': 'http.cookiejar', 'BaseHTTPServer': 'http.server', - 'SimpleHTTPServer': 'http.server', - 'CGIHTTPServer': 'http.server', 'test.test_support': 'test.support', 'commands': 'subprocess', - 'UserString' : 'collections', - 'UserList' : 'collections', 'urlparse' : 'urllib.parse', 'robotparser' : 'urllib.robotparser', - 'whichdb': 'dbm', - 'anydbm': 'dbm' + 'urllib2': 'urllib.request', + 'anydbm': 'dbm', + '_abcoll' : 'collections.abc', } @@ -68,12 +59,35 @@ ('__builtin__', 'reduce'): ('functools', 'reduce'), ('__builtin__', 'intern'): ('sys', 'intern'), ('__builtin__', 'unichr'): ('builtins', 'chr'), - ('__builtin__', 'basestring'): ('builtins', 'str'), + ('__builtin__', 'unicode'): ('builtins', 'str'), ('__builtin__', 'long'): ('builtins', 'int'), ('itertools', 'izip'): ('builtins', 'zip'), ('itertools', 'imap'): ('builtins', 'map'), ('itertools', 'ifilter'): ('builtins', 'filter'), ('itertools', 'ifilterfalse'): ('itertools', 'filterfalse'), + ('itertools', 'izip_longest'): ('itertools', 'zip_longest'), + ('UserDict', 'IterableUserDict'): ('collections', 'UserDict'), + ('UserList', 'UserList'): ('collections', 'UserList'), + ('UserString', 'UserString'): ('collections', 'UserString'), + ('whichdb', 'whichdb'): ('dbm', 'whichdb'), + ('_socket', 'fromfd'): ('socket', 'fromfd'), + ('_multiprocessing', 'Connection'): ('multiprocessing.connection', 'Connection'), + ('multiprocessing.process', 'Process'): ('multiprocessing.context', 'Process'), + ('multiprocessing.forking', 'Popen'): ('multiprocessing.popen_fork', 'Popen'), + ('urllib', 'ContentTooShortError'): ('urllib.error', 'ContentTooShortError'), + ('urllib', 'getproxies'): ('urllib.request', 'getproxies'), + ('urllib', 'pathname2url'): ('urllib.request', 'pathname2url'), + ('urllib', 'quote_plus'): ('urllib.parse', 'quote_plus'), + ('urllib', 'quote'): ('urllib.parse', 'quote'), + ('urllib', 'unquote_plus'): ('urllib.parse', 'unquote_plus'), + ('urllib', 'unquote'): ('urllib.parse', 'unquote'), + ('urllib', 'url2pathname'): ('urllib.request', 'url2pathname'), + ('urllib', 'urlcleanup'): ('urllib.request', 'urlcleanup'), + ('urllib', 'urlencode'): ('urllib.parse', 'urlencode'), + ('urllib', 'urlopen'): ('urllib.request', 'urlopen'), + ('urllib', 'urlretrieve'): ('urllib.request', 'urlretrieve'), + ('urllib2', 'HTTPError'): ('urllib.error', 'HTTPError'), + ('urllib2', 'URLError'): ('urllib.error', 'URLError'), } PYTHON2_EXCEPTIONS = ( @@ -130,8 +144,87 @@ for excname in PYTHON2_EXCEPTIONS: NAME_MAPPING[("exceptions", excname)] = ("builtins", excname) -NAME_MAPPING[("exceptions", "StandardError")] = ("builtins", "Exception") +MULTIPROCESSING_EXCEPTIONS = ( + 'AuthenticationError', + 'BufferTooShort', + 'ProcessError', + 'TimeoutError', +) + +for excname in MULTIPROCESSING_EXCEPTIONS: + NAME_MAPPING[("multiprocessing", excname)] = ("multiprocessing.context", excname) # Same, but for 3.x to 2.x REVERSE_IMPORT_MAPPING = dict((v, k) for (k, v) in IMPORT_MAPPING.items()) +assert len(REVERSE_IMPORT_MAPPING) == len(IMPORT_MAPPING) REVERSE_NAME_MAPPING = dict((v, k) for (k, v) in NAME_MAPPING.items()) +assert len(REVERSE_NAME_MAPPING) == len(NAME_MAPPING) + +# Non-mutual mappings. + +IMPORT_MAPPING.update({ + 'cPickle': 'pickle', + '_elementtree': 'xml.etree.ElementTree', + 'FileDialog': 'tkinter.filedialog', + 'SimpleDialog': 'tkinter.simpledialog', + 'DocXMLRPCServer': 'xmlrpc.server', + 'SimpleHTTPServer': 'http.server', + 'CGIHTTPServer': 'http.server', +}) + +REVERSE_IMPORT_MAPPING.update({ + '_bz2': 'bz2', + '_dbm': 'dbm', + '_functools': 'functools', + '_gdbm': 'gdbm', + '_pickle': 'pickle', +}) + +NAME_MAPPING.update({ + ('__builtin__', 'basestring'): ('builtins', 'str'), + ('exceptions', 'StandardError'): ('builtins', 'Exception'), + ('UserDict', 'UserDict'): ('collections', 'UserDict'), + ('socket', '_socketobject'): ('socket', 'SocketType'), +}) + +REVERSE_NAME_MAPPING.update({ + ('_functools', 'reduce'): ('__builtin__', 'reduce'), + ('tkinter.filedialog', 'FileDialog'): ('FileDialog', 'FileDialog'), + ('tkinter.filedialog', 'LoadFileDialog'): ('FileDialog', 'LoadFileDialog'), + ('tkinter.filedialog', 'SaveFileDialog'): ('FileDialog', 'SaveFileDialog'), + ('tkinter.simpledialog', 'SimpleDialog'): ('SimpleDialog', 'SimpleDialog'), + ('xmlrpc.server', 'ServerHTMLDoc'): ('DocXMLRPCServer', 'ServerHTMLDoc'), + ('xmlrpc.server', 'XMLRPCDocGenerator'): + ('DocXMLRPCServer', 'XMLRPCDocGenerator'), + ('xmlrpc.server', 'DocXMLRPCRequestHandler'): + ('DocXMLRPCServer', 'DocXMLRPCRequestHandler'), + ('xmlrpc.server', 'DocXMLRPCServer'): + ('DocXMLRPCServer', 'DocXMLRPCServer'), + ('xmlrpc.server', 'DocCGIXMLRPCRequestHandler'): + ('DocXMLRPCServer', 'DocCGIXMLRPCRequestHandler'), + ('http.server', 'SimpleHTTPRequestHandler'): + ('SimpleHTTPServer', 'SimpleHTTPRequestHandler'), + ('http.server', 'CGIHTTPRequestHandler'): + ('CGIHTTPServer', 'CGIHTTPRequestHandler'), + ('_socket', 'socket'): ('socket', '_socketobject'), +}) + +PYTHON3_OSERROR_EXCEPTIONS = ( + 'BrokenPipeError', + 'ChildProcessError', + 'ConnectionAbortedError', + 'ConnectionError', + 'ConnectionRefusedError', + 'ConnectionResetError', + 'FileExistsError', + 'FileNotFoundError', + 'InterruptedError', + 'IsADirectoryError', + 'NotADirectoryError', + 'PermissionError', + 'ProcessLookupError', + 'TimeoutError', +) + +for excname in PYTHON3_OSERROR_EXCEPTIONS: + REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'OSError') diff --git a/Lib/pickle.py b/Lib/pickle.py --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -944,7 +944,7 @@ r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING if (module_name, name) in r_name_mapping: module_name, name = r_name_mapping[(module_name, name)] - if module_name in r_import_mapping: + elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: write(GLOBAL + bytes(module_name, "ascii") + b'\n' + @@ -1370,7 +1370,7 @@ if self.proto < 3 and self.fix_imports: if (module, name) in _compat_pickle.NAME_MAPPING: module, name = _compat_pickle.NAME_MAPPING[(module, name)] - if module in _compat_pickle.IMPORT_MAPPING: + elif module in _compat_pickle.IMPORT_MAPPING: module = _compat_pickle.IMPORT_MAPPING[module] __import__(module, level=0) return _getattribute(sys.modules[module], name, diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1,8 +1,10 @@ +import collections import copyreg +import dbm import io +import functools import pickle import pickletools -import random import struct import sys import unittest @@ -1645,6 +1647,51 @@ unpickled = self.loads(self.dumps(method, proto)) self.assertEqual(method(*args), unpickled(*args)) + def test_compat_pickle(self): + tests = [ + (range(1, 7), '__builtin__', 'xrange'), + (map(int, '123'), 'itertools', 'imap'), + (functools.reduce, '__builtin__', 'reduce'), + (dbm.whichdb, 'whichdb', 'whichdb'), + (Exception(), 'exceptions', 'Exception'), + (collections.UserDict(), 'UserDict', 'IterableUserDict'), + (collections.UserList(), 'UserList', 'UserList'), + (collections.defaultdict(), 'collections', 'defaultdict'), + ] + for val, mod, name in tests: + for proto in range(3): + with self.subTest(type=type(val), proto=proto): + pickled = self.dumps(val, proto) + self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled) + self.assertIs(type(self.loads(pickled)), type(val)) + + def test_compat_unpickle(self): + # xrange(1, 7) + pickled = b'\x80\x02c__builtin__\nxrange\nK\x01K\x07K\x01\x87R.' + unpickled = self.loads(pickled) + self.assertIs(type(unpickled), range) + self.assertEqual(unpickled, range(1, 7)) + self.assertEqual(list(unpickled), [1, 2, 3, 4, 5, 6]) + # reduce + pickled = b'\x80\x02c__builtin__\nreduce\n.' + self.assertIs(self.loads(pickled), functools.reduce) + # whichdb.whichdb + pickled = b'\x80\x02cwhichdb\nwhichdb\n.' + self.assertIs(self.loads(pickled), dbm.whichdb) + # Exception(), StandardError() + for name in (b'Exception', b'StandardError'): + pickled = (b'\x80\x02cexceptions\n' + name + b'\nU\x03ugh\x85R.') + unpickled = self.loads(pickled) + self.assertIs(type(unpickled), Exception) + self.assertEqual(str(unpickled), 'ugh') + # UserDict.UserDict({1: 2}), UserDict.IterableUserDict({1: 2}) + for name in (b'UserDict', b'IterableUserDict'): + pickled = (b'\x80\x02(cUserDict\n' + name + + b'\no}U\x04data}K\x01K\x02ssb.') + unpickled = self.loads(pickled) + self.assertIs(type(unpickled), collections.UserDict) + self.assertEqual(unpickled, collections.UserDict({1: 2})) + class BigmemPickleTests(unittest.TestCase): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -1,3 +1,6 @@ +from _compat_pickle import (IMPORT_MAPPING, REVERSE_IMPORT_MAPPING, + NAME_MAPPING, REVERSE_NAME_MAPPING) +import builtins import pickle import io import collections @@ -207,9 +210,156 @@ check(u, stdsize + 32 * P + 2 + 1) +ALT_IMPORT_MAPPING = { + ('_elementtree', 'xml.etree.ElementTree'), + ('cPickle', 'pickle'), +} + +ALT_NAME_MAPPING = { + ('__builtin__', 'basestring', 'builtins', 'str'), + ('exceptions', 'StandardError', 'builtins', 'Exception'), + ('UserDict', 'UserDict', 'collections', 'UserDict'), + ('socket', '_socketobject', 'socket', 'SocketType'), +} + +def mapping(module, name): + if (module, name) in NAME_MAPPING: + module, name = NAME_MAPPING[(module, name)] + elif module in IMPORT_MAPPING: + module = IMPORT_MAPPING[module] + return module, name + +def reverse_mapping(module, name): + if (module, name) in REVERSE_NAME_MAPPING: + module, name = REVERSE_NAME_MAPPING[(module, name)] + elif module in REVERSE_IMPORT_MAPPING: + module = REVERSE_IMPORT_MAPPING[module] + return module, name + +def getmodule(module): + try: + return sys.modules[module] + except KeyError: + __import__(module) + return sys.modules[module] + +def getattribute(module, name): + obj = getmodule(module) + for n in name.split('.'): + obj = getattr(obj, n) + return obj + +def get_exceptions(mod): + for name in dir(mod): + attr = getattr(mod, name) + if isinstance(attr, type) and issubclass(attr, BaseException): + yield name, attr + +class CompatPickleTests(unittest.TestCase): + def test_import(self): + modules = set(IMPORT_MAPPING.values()) + modules |= set(REVERSE_IMPORT_MAPPING) + modules |= {module for module, name in REVERSE_NAME_MAPPING} + modules |= {module for module, name in NAME_MAPPING.values()} + for module in modules: + try: + getmodule(module) + except ImportError as exc: + if support.verbose: + print(exc) + + def test_import_mapping(self): + for module3, module2 in REVERSE_IMPORT_MAPPING.items(): + with self.subTest((module3, module2)): + try: + getmodule(module3) + except ImportError as exc: + if support.verbose: + print(exc) + if module3[:1] != '_': + self.assertIn(module2, IMPORT_MAPPING) + self.assertEqual(IMPORT_MAPPING[module2], module3) + + def test_name_mapping(self): + for (module3, name3), (module2, name2) in REVERSE_NAME_MAPPING.items(): + with self.subTest(((module3, name3), (module2, name2))): + attr = getattribute(module3, name3) + if (module2, name2) == ('exceptions', 'OSError'): + self.assertTrue(issubclass(attr, OSError)) + else: + module, name = mapping(module2, name2) + if module3[:1] != '_': + self.assertEqual((module, name), (module3, name3)) + self.assertEqual(getattribute(module, name), attr) + + def test_reverse_import_mapping(self): + for module2, module3 in IMPORT_MAPPING.items(): + with self.subTest((module2, module3)): + try: + getmodule(module3) + except ImportError as exc: + if support.verbose: + print(exc) + if ((module2, module3) not in ALT_IMPORT_MAPPING and + REVERSE_IMPORT_MAPPING.get(module3, None) != module2): + for (m3, n3), (m2, n2) in REVERSE_NAME_MAPPING.items(): + if (module3, module2) == (m3, m2): + break + else: + self.fail('No reverse mapping from %r to %r' % + (module3, module2)) + module = REVERSE_IMPORT_MAPPING.get(module3, module3) + module = IMPORT_MAPPING.get(module, module) + self.assertEqual(module, module3) + + def test_reverse_name_mapping(self): + for (module2, name2), (module3, name3) in NAME_MAPPING.items(): + with self.subTest(((module2, name2), (module3, name3))): + attr = getattribute(module3, name3) + module, name = reverse_mapping(module3, name3) + if (module2, name2, module3, name3) not in ALT_NAME_MAPPING: + self.assertEqual((module, name), (module2, name2)) + module, name = mapping(module, name) + self.assertEqual((module, name), (module3, name3)) + + def test_exceptions(self): + self.assertEqual(mapping('exceptions', 'StandardError'), + ('builtins', 'Exception')) + self.assertEqual(mapping('exceptions', 'Exception'), + ('builtins', 'Exception')) + self.assertEqual(reverse_mapping('builtins', 'Exception'), + ('exceptions', 'Exception')) + self.assertEqual(mapping('exceptions', 'OSError'), + ('builtins', 'OSError')) + self.assertEqual(reverse_mapping('builtins', 'OSError'), + ('exceptions', 'OSError')) + + for name, exc in get_exceptions(builtins): + with self.subTest(name): + if exc in (BlockingIOError, ResourceWarning): + continue + if exc is not OSError and issubclass(exc, OSError): + self.assertEqual(reverse_mapping('builtins', name), + ('exceptions', 'OSError')) + else: + self.assertEqual(reverse_mapping('builtins', name), + ('exceptions', name)) + self.assertEqual(mapping('exceptions', name), + ('builtins', name)) + + import multiprocessing.context + for name, exc in get_exceptions(multiprocessing.context): + with self.subTest(name): + self.assertEqual(reverse_mapping('multiprocessing.context', name), + ('multiprocessing', name)) + self.assertEqual(mapping('multiprocessing', name), + ('multiprocessing.context', name)) + + def test_main(): tests = [PickleTests, PyPicklerTests, PyPersPicklerTests, - PyDispatchTableTests, PyChainDispatchTableTests] + PyDispatchTableTests, PyChainDispatchTableTests, + CompatPickleTests] if has_c_implementation: tests.extend([CPicklerTests, CPersPicklerTests, CDumpPickle_LoadPickle, DumpPickle_CLoadPickle, diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,10 @@ Library ------- +- Issue #18473: Fixed 2to3 and 3to2 compatible pickle mappings. Fixed + ambigious reverse mappings. Added many new mappings. Import mapping is no + longer applied to modules already mapped with full name mapping. + - Issue #23745: The new email header parser now handles duplicate MIME parameter names without error, similar to how get_param behaves. diff --git a/Modules/_pickle.c b/Modules/_pickle.c --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -3026,6 +3026,7 @@ Py_INCREF(fixed_global_name); *module_name = fixed_module_name; *global_name = fixed_global_name; + return 0; } else if (PyErr_Occurred()) { return -1; @@ -6278,20 +6279,21 @@ else if (PyErr_Occurred()) { return NULL; } - - /* Check if the module was renamed. */ - item = PyDict_GetItemWithError(st->import_mapping_2to3, module_name); - if (item) { - if (!PyUnicode_Check(item)) { - PyErr_Format(PyExc_RuntimeError, - "_compat_pickle.IMPORT_MAPPING values should be " - "strings, not %.200s", Py_TYPE(item)->tp_name); + else { + /* Check if the module was renamed. */ + item = PyDict_GetItemWithError(st->import_mapping_2to3, module_name); + if (item) { + if (!PyUnicode_Check(item)) { + PyErr_Format(PyExc_RuntimeError, + "_compat_pickle.IMPORT_MAPPING values should be " + "strings, not %.200s", Py_TYPE(item)->tp_name); + return NULL; + } + module_name = item; + } + else if (PyErr_Occurred()) { return NULL; } - module_name = item; - } - else if (PyErr_Occurred()) { - return NULL; } } -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython (merge default -> default): Merge heads
by serhiy.storchaka March 31, 2015

March 31, 2015

https://hg.python.org/cpython/rev/475461033bb2 changeset: 95318:475461033bb2 parent: 95317:5980e81219ed parent: 95315:76d297869859 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Tue Mar 31 13:18:35 2015 +0300 summary: Merge heads files: Doc/library/select.rst | 12 + Doc/library/selectors.rst | 6 + Doc/whatsnew/3.5.rst | 5 +- Lib/selectors.py | 13 +- Lib/test/eintrdata/eintr_tester.… [View More]py | 26 ++- Lib/test/test_selectors.py | 37 +++- Modules/selectmodule.c | 151 ++++++++++------ 7 files changed, 181 insertions(+), 69 deletions(-) diff --git a/Doc/library/select.rst b/Doc/library/select.rst --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -249,6 +249,12 @@ returning. If *timeout* is omitted, -1, or :const:`None`, the call will block until there is an event for this poll object. + .. versionchanged:: 3.5 + The function is now retried with a recomputed timeout when interrupted by + a signal, except if the signal handler raises an exception (see + :pep:`475` for the rationale), instead of raising + :exc:`InterruptedError`. + .. _epoll-objects: @@ -454,6 +460,12 @@ - max_events must be 0 or a positive integer - timeout in seconds (floats possible) + .. versionchanged:: 3.5 + The function is now retried with a recomputed timeout when interrupted by + a signal, except if the signal handler raises an exception (see + :pep:`475` for the rationale), instead of raising + :exc:`InterruptedError`. + .. _kevent-objects: diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -159,6 +159,12 @@ timeout has elapsed if the current process receives a signal: in this case, an empty list will be returned. + .. versionchanged:: 3.5 + The selector is now retried with a recomputed timeout when interrupted + by a signal if the signal handler did not raise an exception (see + :pep:`475` for the rationale), instead of returning an empty list + of events before the timeout. + .. method:: close() Close the selector. diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -619,9 +619,10 @@ instead of raising :exc:`InterruptedError` if the signal handler does not raise an exception: - - :func:`os.open`, :func:`open` + - :func:`open`, :func:`os.open`, :func:`io.open` - :func:`os.read`, :func:`os.write` - - :func:`select.select`, :func:`select.poll.poll`, :func:`select.epoll.poll` + - :func:`select.select`, :func:`select.poll.poll`, :func:`select.epoll.poll`, + :func:`select.kqueue.control`, :func:`select.devpoll.poll` - :func:`time.sleep` * Before Python 3.5, a :class:`datetime.time` object was considered to be false diff --git a/Lib/selectors.py b/Lib/selectors.py --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -479,11 +479,10 @@ # devpoll() has a resolution of 1 millisecond, round away from # zero to wait *at least* timeout seconds. timeout = math.ceil(timeout * 1e3) + + fd_event_list = self._devpoll.poll(timeout) + ready = [] - try: - fd_event_list = self._devpoll.poll(timeout) - except InterruptedError: - return ready for fd, event in fd_event_list: events = 0 if event & ~select.POLLIN: @@ -549,11 +548,9 @@ def select(self, timeout=None): timeout = None if timeout is None else max(timeout, 0) max_ev = len(self._fd_to_key) + kev_list = self._kqueue.control(None, max_ev, timeout) + ready = [] - try: - kev_list = self._kqueue.control(None, max_ev, timeout) - except InterruptedError: - return ready for kev in kev_list: fd = kev.ident flag = kev.filter diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -315,8 +315,8 @@ def test_select(self): t0 = time.monotonic() select.select([], [], [], self.sleep_time) + dt = time.monotonic() - t0 self.stop_alarm() - dt = time.monotonic() - t0 self.assertGreaterEqual(dt, self.sleep_time) @unittest.skipUnless(hasattr(select, 'poll'), 'need select.poll') @@ -325,8 +325,8 @@ t0 = time.monotonic() poller.poll(self.sleep_time * 1e3) + dt = time.monotonic() - t0 self.stop_alarm() - dt = time.monotonic() - t0 self.assertGreaterEqual(dt, self.sleep_time) @unittest.skipUnless(hasattr(select, 'epoll'), 'need select.epoll') @@ -336,8 +336,30 @@ t0 = time.monotonic() poller.poll(self.sleep_time) + dt = time.monotonic() - t0 self.stop_alarm() + self.assertGreaterEqual(dt, self.sleep_time) + + @unittest.skipUnless(hasattr(select, 'kqueue'), 'need select.kqueue') + def test_kqueue(self): + kqueue = select.kqueue() + self.addCleanup(kqueue.close) + + t0 = time.monotonic() + kqueue.control(None, 1, self.sleep_time) dt = time.monotonic() - t0 + self.stop_alarm() + self.assertGreaterEqual(dt, self.sleep_time) + + @unittest.skipUnless(hasattr(select, 'devpoll'), 'need select.devpoll') + def test_devpoll(self): + poller = select.devpoll() + self.addCleanup(poller.close) + + t0 = time.monotonic() + poller.poll(self.sleep_time * 1e3) + dt = time.monotonic() - t0 + self.stop_alarm() self.assertGreaterEqual(dt, self.sleep_time) diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -357,7 +357,35 @@ @unittest.skipUnless(hasattr(signal, "alarm"), "signal.alarm() required for this test") - def test_select_interrupt(self): + def test_select_interrupt_exc(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + class InterruptSelect(Exception): + pass + + def handler(*args): + raise InterruptSelect + + orig_alrm_handler = signal.signal(signal.SIGALRM, handler) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(signal.alarm, 0) + + signal.alarm(1) + + s.register(rd, selectors.EVENT_READ) + t = time() + # select() is interrupted by a signal which raises an exception + with self.assertRaises(InterruptSelect): + s.select(30) + # select() was interrupted before the timeout of 30 seconds + self.assertLess(time() - t, 5.0) + + @unittest.skipUnless(hasattr(signal, "alarm"), + "signal.alarm() required for this test") + def test_select_interrupt_noraise(self): s = self.SELECTOR() self.addCleanup(s.close) @@ -371,8 +399,11 @@ s.register(rd, selectors.EVENT_READ) t = time() - self.assertFalse(s.select(2)) - self.assertLess(time() - t, 2.5) + # select() is interrupted by a signal, but the signal handler doesn't + # raise an exception, so select() should by retries with a recomputed + # timeout + self.assertFalse(s.select(1.5)) + self.assertGreaterEqual(time() - t, 1.0) class ScalableSelectorMixIn: diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -876,40 +876,38 @@ devpoll_poll(devpollObject *self, PyObject *args) { struct dvpoll dvp; - PyObject *result_list = NULL, *tout = NULL; + PyObject *result_list = NULL, *timeout_obj = NULL; int poll_result, i; - long timeout; PyObject *value, *num1, *num2; + _PyTime_t timeout, ms, deadline = 0; if (self->fd_devpoll < 0) return devpoll_err_closed(); - if (!PyArg_UnpackTuple(args, "poll", 0, 1, &tout)) { + if (!PyArg_ParseTuple(args, "|O:poll", &timeout_obj)) { return NULL; } /* Check values for timeout */ - if (tout == NULL || tout == Py_None) + if (timeout_obj == NULL || timeout_obj == Py_None) { timeout = -1; - else if (!PyNumber_Check(tout)) { - PyErr_SetString(PyExc_TypeError, - "timeout must be an integer or None"); - return NULL; + ms = -1; } else { - tout = PyNumber_Long(tout); - if (!tout) + if (_PyTime_FromMillisecondsObject(&timeout, timeout_obj, + _PyTime_ROUND_CEILING) < 0) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_SetString(PyExc_TypeError, + "timeout must be an integer or None"); + } return NULL; - timeout = PyLong_AsLong(tout); - Py_DECREF(tout); - if (timeout == -1 && PyErr_Occurred()) + } + + ms = _PyTime_AsMilliseconds(timeout, _PyTime_ROUND_CEILING); + if (ms < -1 || ms > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "timeout is too large"); return NULL; - } - - if ((timeout < -1) || (timeout > INT_MAX)) { - PyErr_SetString(PyExc_OverflowError, - "timeout is out of range"); - return NULL; + } } if (devpoll_flush(self)) @@ -917,12 +915,36 @@ dvp.dp_fds = self->fds; dvp.dp_nfds = self->max_n_fds; - dvp.dp_timeout = timeout; + dvp.dp_timeout = (int)ms; - /* call devpoll() */ - Py_BEGIN_ALLOW_THREADS - poll_result = ioctl(self->fd_devpoll, DP_POLL, &dvp); - Py_END_ALLOW_THREADS + if (timeout >= 0) + deadline = _PyTime_GetMonotonicClock() + timeout; + + do { + /* call devpoll() */ + Py_BEGIN_ALLOW_THREADS + errno = 0; + poll_result = ioctl(self->fd_devpoll, DP_POLL, &dvp); + Py_END_ALLOW_THREADS + + if (errno != EINTR) + break; + + /* devpoll() was interrupted by a signal */ + if (PyErr_CheckSignals()) + return NULL; + + if (timeout >= 0) { + timeout = deadline - _PyTime_GetMonotonicClock(); + if (timeout < 0) { + poll_result = 0; + break; + } + ms = _PyTime_AsMilliseconds(timeout, _PyTime_ROUND_CEILING); + dvp.dp_timeout = (int)ms; + /* retry devpoll() with the recomputed timeout */ + } + } while (1); if (poll_result < 0) { PyErr_SetFromErrno(PyExc_IOError); @@ -930,28 +952,26 @@ } /* build the result list */ - result_list = PyList_New(poll_result); if (!result_list) return NULL; - else { - for (i = 0; i < poll_result; i++) { - num1 = PyLong_FromLong(self->fds[i].fd); - num2 = PyLong_FromLong(self->fds[i].revents); - if ((num1 == NULL) || (num2 == NULL)) { - Py_XDECREF(num1); - Py_XDECREF(num2); - goto error; - } - value = PyTuple_Pack(2, num1, num2); - Py_DECREF(num1); - Py_DECREF(num2); - if (value == NULL) - goto error; - if ((PyList_SetItem(result_list, i, value)) == -1) { - Py_DECREF(value); - goto error; - } + + for (i = 0; i < poll_result; i++) { + num1 = PyLong_FromLong(self->fds[i].fd); + num2 = PyLong_FromLong(self->fds[i].revents); + if ((num1 == NULL) || (num2 == NULL)) { + Py_XDECREF(num1); + Py_XDECREF(num2); + goto error; + } + value = PyTuple_Pack(2, num1, num2); + Py_DECREF(num1); + Py_DECREF(num2); + if (value == NULL) + goto error; + if ((PyList_SetItem(result_list, i, value)) == -1) { + Py_DECREF(value); + goto error; } } @@ -2083,8 +2103,9 @@ PyObject *result = NULL; struct kevent *evl = NULL; struct kevent *chl = NULL; - struct timespec timeout; + struct timespec timeoutspec; struct timespec *ptimeoutspec; + _PyTime_t timeout, deadline = 0; if (self->kqfd < 0) return kqueue_queue_err_closed(); @@ -2103,9 +2124,7 @@ ptimeoutspec = NULL; } else { - _PyTime_t ts; - - if (_PyTime_FromSecondsObject(&ts, + if (_PyTime_FromSecondsObject(&timeout, otimeout, _PyTime_ROUND_CEILING) < 0) { PyErr_Format(PyExc_TypeError, "timeout argument must be an number " @@ -2114,15 +2133,15 @@ return NULL; } - if (_PyTime_AsTimespec(ts, &timeout) == -1) + if (_PyTime_AsTimespec(timeout, &timeoutspec) == -1) return NULL; - if (timeout.tv_sec < 0) { + if (timeoutspec.tv_sec < 0) { PyErr_SetString(PyExc_ValueError, "timeout must be positive or None"); return NULL; } - ptimeoutspec = &timeout; + ptimeoutspec = &timeoutspec; } if (ch != NULL && ch != Py_None) { @@ -2167,10 +2186,34 @@ } } - Py_BEGIN_ALLOW_THREADS - gotevents = kevent(self->kqfd, chl, nchanges, - evl, nevents, ptimeoutspec); - Py_END_ALLOW_THREADS + if (ptimeoutspec) + deadline = _PyTime_GetMonotonicClock() + timeout; + + do { + Py_BEGIN_ALLOW_THREADS + errno = 0; + gotevents = kevent(self->kqfd, chl, nchanges, + evl, nevents, ptimeoutspec); + Py_END_ALLOW_THREADS + + if (errno != EINTR) + break; + + /* kevent() was interrupted by a signal */ + if (PyErr_CheckSignals()) + goto error; + + if (ptimeoutspec) { + timeout = deadline - _PyTime_GetMonotonicClock(); + if (timeout < 0) { + gotevents = 0; + break; + } + if (_PyTime_AsTimespec(timeout, &timeoutspec) == -1) + goto error; + /* retry kevent() with the recomputed timeout */ + } + } while (1); if (gotevents == -1) { PyErr_SetFromErrno(PyExc_OSError); -- Repository URL: https://hg.python.org/cpython [View Less]

1 0