Mailman 3 January 2015 - Python-checkins

cpython (3.4): Issue #22668: Remove endianness assumption in test.
by stefan.krah Jan. 29, 2015

Jan. 29, 2015

https://hg.python.org/cpython/rev/9a4af12dcc9d changeset: 94375:9a4af12dcc9d branch: 3.4 parent: 94372:e9c1fca50b46 user: Stefan Krah <skrah(a)bytereef.org> date: Thu Jan 29 17:33:31 2015 +0100 summary: Issue #22668: Remove endianness assumption in test. files: Lib/test/test_memoryview.py | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -361,18 +361,20 @@ self.assertEqual(list(reversed(m)), list(m[::-1])) def test_issue22668(self): - m = memoryview(bytes(range(8))) + a = array.array('H', [256, 256, 256, 256]) + x = memoryview(a) + m = x.cast('B') b = m.cast('H') c = b[0:2] d = memoryview(b) del b - + self.assertEqual(c[0], 256) self.assertEqual(d[0], 256) self.assertEqual(c.format, "H") self.assertEqual(d.format, "H") - + _ = m.cast('I') self.assertEqual(c[0], 256) self.assertEqual(d[0], 256) -- Repository URL: https://hg.python.org/cpython

1 0

cpython: Whitespace.
by stefan.krah Jan. 29, 2015

Jan. 29, 2015

https://hg.python.org/cpython/rev/7bb40e624387 changeset: 94374:7bb40e624387 user: Stefan Krah <skrah(a)bytereef.org> date: Thu Jan 29 14:33:37 2015 +0100 summary: Whitespace. files: Lib/test/test_memoryview.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -367,12 +367,12 @@ d = memoryview(b) del b - + self.assertEqual(c[0], 256) self.assertEqual(d[0], 256) self.assertEqual(c.format, "H") self.assertEqual(d.format, "H") - + _ = m.cast('I') self.assertEqual(c[0], 256) self.assertEqual(d[0], 256) -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.4): Issue #22668: Ensure that format strings survive slicing after casting.
by stefan.krah Jan. 29, 2015

Jan. 29, 2015

https://hg.python.org/cpython/rev/e9c1fca50b46 changeset: 94372:e9c1fca50b46 branch: 3.4 parent: 94370:c4fd6df9aea6 user: Stefan Krah <skrah(a)bytereef.org> date: Thu Jan 29 14:27:23 2015 +0100 summary: Issue #22668: Ensure that format strings survive slicing after casting. files: Include/memoryobject.h | 4 +- Lib/test/test_memoryview.py | 19 ++++++++ Objects/memoryobject.c | 56 +++++++++++++++++++++++- 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/Include/memoryobject.h b/Include/memoryobject.h --- a/Include/memoryobject.h +++ b/Include/memoryobject.h @@ -45,7 +45,7 @@ } _PyManagedBufferObject; -/* static storage used for casting between formats */ +/* deprecated, removed in 3.5 */ #define _Py_MEMORYVIEW_MAX_FORMAT 3 /* must be >= 3 */ /* memoryview state flags */ @@ -62,7 +62,7 @@ int flags; /* state flags */ Py_ssize_t exports; /* number of buffer re-exports */ Py_buffer view; /* private copy of the exporter's view */ - char format[_Py_MEMORYVIEW_MAX_FORMAT]; /* used for casting */ + char format[_Py_MEMORYVIEW_MAX_FORMAT]; /* deprecated, removed in 3.5 */ PyObject *weakreflist; Py_ssize_t ob_array[1]; /* shape, strides, suboffsets */ } PyMemoryViewObject; diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -360,6 +360,25 @@ self.assertEqual(list(reversed(m)), aslist) self.assertEqual(list(reversed(m)), list(m[::-1])) + def test_issue22668(self): + m = memoryview(bytes(range(8))) + b = m.cast('H') + c = b[0:2] + d = memoryview(b) + + del b + + self.assertEqual(c[0], 256) + self.assertEqual(d[0], 256) + self.assertEqual(c.format, "H") + self.assertEqual(d.format, "H") + + _ = m.cast('I') + self.assertEqual(c[0], 256) + self.assertEqual(d[0], 256) + self.assertEqual(c.format, "H") + self.assertEqual(d.format, "H") + # Variations on source objects for the buffer: bytes-like objects, then arrays # with itemsize > 1. diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1135,6 +1135,51 @@ return -1; } +Py_LOCAL_INLINE(char *) +get_native_fmtstr(const char *fmt) +{ + int at = 0; + + if (fmt[0] == '@') { + at = 1; + fmt++; + } + if (fmt[0] == '\0' || fmt[1] != '\0') { + return NULL; + } + +#define RETURN(s) do { return at ? "@" s : s; } while (0) + + switch (fmt[0]) { + case 'c': RETURN("c"); + case 'b': RETURN("b"); + case 'B': RETURN("B"); + case 'h': RETURN("h"); + case 'H': RETURN("H"); + case 'i': RETURN("i"); + case 'I': RETURN("I"); + case 'l': RETURN("l"); + case 'L': RETURN("L"); + #ifdef HAVE_LONG_LONG + case 'q': RETURN("q"); + case 'Q': RETURN("Q"); + #endif + case 'n': RETURN("n"); + case 'N': RETURN("N"); + case 'f': RETURN("f"); + case 'd': RETURN("d"); + #ifdef HAVE_C99_BOOL + case '?': RETURN("?"); + #else + case '?': RETURN("?"); + #endif + case 'P': RETURN("P"); + } + + return NULL; +} + + /* Cast a memoryview's data type to 'format'. The input array must be C-contiguous. At least one of input-format, output-format must have byte size. The output array is 1-D, with the same byte length as the @@ -1184,10 +1229,13 @@ goto out; } - strncpy(mv->format, PyBytes_AS_STRING(asciifmt), - _Py_MEMORYVIEW_MAX_FORMAT); - mv->format[_Py_MEMORYVIEW_MAX_FORMAT-1] = '\0'; - view->format = mv->format; + view->format = get_native_fmtstr(PyBytes_AS_STRING(asciifmt)); + if (view->format == NULL) { + /* NOT_REACHED: get_native_fmtchar() already validates the format. */ + PyErr_SetString(PyExc_RuntimeError, + "memoryview: internal error"); + goto out; + } view->itemsize = itemsize; view->ndim = 1; -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.4 -> default): Closes #22668: Merge from 3.4.
by stefan.krah Jan. 29, 2015

Jan. 29, 2015

https://hg.python.org/cpython/rev/37112bd3dfb3 changeset: 94373:37112bd3dfb3 parent: 94371:497c9e57a54e parent: 94372:e9c1fca50b46 user: Stefan Krah <skrah(a)bytereef.org> date: Thu Jan 29 14:29:51 2015 +0100 summary: Closes #22668: Merge from 3.4. files: Doc/whatsnew/3.5.rst | 6 ++ Include/memoryobject.h | 4 - Lib/test/test_memoryview.py | 19 ++++++++ Lib/test/test_sys.py | 2 +- Objects/memoryobject.c | 56 +++++++++++++++++++++++- 5 files changed, 78 insertions(+), 9 deletions(-) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -485,6 +485,12 @@ Changes in the C API -------------------- +* The undocumented :c:member:`~PyMemoryViewObject.format` member of the + (non-public) :c:type:`PyMemoryViewObject` structure has been removed. + + All extensions relying on the relevant parts in ``memoryobject.h`` + must be rebuilt. + * The :c:type:`PyMemAllocator` structure was renamed to :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. diff --git a/Include/memoryobject.h b/Include/memoryobject.h --- a/Include/memoryobject.h +++ b/Include/memoryobject.h @@ -45,9 +45,6 @@ } _PyManagedBufferObject; -/* static storage used for casting between formats */ -#define _Py_MEMORYVIEW_MAX_FORMAT 3 /* must be >= 3 */ - /* memoryview state flags */ #define _Py_MEMORYVIEW_RELEASED 0x001 /* access to master buffer blocked */ #define _Py_MEMORYVIEW_C 0x002 /* C-contiguous layout */ @@ -62,7 +59,6 @@ int flags; /* state flags */ Py_ssize_t exports; /* number of buffer re-exports */ Py_buffer view; /* private copy of the exporter's view */ - char format[_Py_MEMORYVIEW_MAX_FORMAT]; /* used for casting */ PyObject *weakreflist; Py_ssize_t ob_array[1]; /* shape, strides, suboffsets */ } PyMemoryViewObject; diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -360,6 +360,25 @@ self.assertEqual(list(reversed(m)), aslist) self.assertEqual(list(reversed(m)), list(m[::-1])) + def test_issue22668(self): + m = memoryview(bytes(range(8))) + b = m.cast('H') + c = b[0:2] + d = memoryview(b) + + del b + + self.assertEqual(c[0], 256) + self.assertEqual(d[0], 256) + self.assertEqual(c.format, "H") + self.assertEqual(d.format, "H") + + _ = m.cast('I') + self.assertEqual(c[0], 256) + self.assertEqual(d[0], 256) + self.assertEqual(c.format, "H") + self.assertEqual(d.format, "H") + # Variations on source objects for the buffer: bytes-like objects, then arrays # with itemsize > 1. diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -965,7 +965,7 @@ check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit) check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # memoryview - check(memoryview(b''), size('Pnin 2P2n2i5P 3cPn')) + check(memoryview(b''), size('Pnin 2P2n2i5P Pn')) # module check(unittest, size('PnPPP')) # None diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1132,6 +1132,51 @@ return -1; } +Py_LOCAL_INLINE(char *) +get_native_fmtstr(const char *fmt) +{ + int at = 0; + + if (fmt[0] == '@') { + at = 1; + fmt++; + } + if (fmt[0] == '\0' || fmt[1] != '\0') { + return NULL; + } + +#define RETURN(s) do { return at ? "@" s : s; } while (0) + + switch (fmt[0]) { + case 'c': RETURN("c"); + case 'b': RETURN("b"); + case 'B': RETURN("B"); + case 'h': RETURN("h"); + case 'H': RETURN("H"); + case 'i': RETURN("i"); + case 'I': RETURN("I"); + case 'l': RETURN("l"); + case 'L': RETURN("L"); + #ifdef HAVE_LONG_LONG + case 'q': RETURN("q"); + case 'Q': RETURN("Q"); + #endif + case 'n': RETURN("n"); + case 'N': RETURN("N"); + case 'f': RETURN("f"); + case 'd': RETURN("d"); + #ifdef HAVE_C99_BOOL + case '?': RETURN("?"); + #else + case '?': RETURN("?"); + #endif + case 'P': RETURN("P"); + } + + return NULL; +} + + /* Cast a memoryview's data type to 'format'. The input array must be C-contiguous. At least one of input-format, output-format must have byte size. The output array is 1-D, with the same byte length as the @@ -1181,10 +1226,13 @@ goto out; } - strncpy(mv->format, PyBytes_AS_STRING(asciifmt), - _Py_MEMORYVIEW_MAX_FORMAT); - mv->format[_Py_MEMORYVIEW_MAX_FORMAT-1] = '\0'; - view->format = mv->format; + view->format = get_native_fmtstr(PyBytes_AS_STRING(asciifmt)); + if (view->format == NULL) { + /* NOT_REACHED: get_native_fmtchar() already validates the format. */ + PyErr_SetString(PyExc_RuntimeError, + "memoryview: internal error"); + goto out; + } view->itemsize = itemsize; view->ndim = 1; -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.4): asyncio: sync with Tulip
by victor.stinner Jan. 29, 2015

Jan. 29, 2015

https://hg.python.org/cpython/rev/c4fd6df9aea6 changeset: 94370:c4fd6df9aea6 branch: 3.4 parent: 94368:4fe1303bf531 user: Victor Stinner <victor.stinner(a)gmail.com> date: Thu Jan 29 14:15:19 2015 +0100 summary: asyncio: sync with Tulip * Cleanup gather(): use cancelled() method instead of using private Future attribute * Fix _UnixReadPipeTransport and _UnixWritePipeTransport. Only start reading when connection_made() has been called. * Issue #23333: Fix BaseSelectorEventLoop._accept_connection(). Close the transport on error. In debug mode, log errors using call_exception_handler() files: Lib/asyncio/selector_events.py | 44 +++++++++- Lib/asyncio/tasks.py | 2 +- Lib/asyncio/unix_events.py | 17 ++- Lib/test/test_asyncio/test_events.py | 37 ++++++--- Lib/test/test_asyncio/test_unix_events.py | 31 +++---- 5 files changed, 86 insertions(+), 45 deletions(-) diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -22,6 +22,7 @@ from . import selectors from . import transports from . import sslproto +from .coroutines import coroutine from .log import logger @@ -181,16 +182,47 @@ else: raise # The event loop will catch, log and ignore it. else: + extra = {'peername': addr} + accept = self._accept_connection2(protocol_factory, conn, extra, + sslcontext, server) + self.create_task(accept) + + @coroutine + def _accept_connection2(self, protocol_factory, conn, extra, + sslcontext=None, server=None): + protocol = None + transport = None + try: protocol = protocol_factory() + waiter = futures.Future(loop=self) if sslcontext: - self._make_ssl_transport( - conn, protocol, sslcontext, - server_side=True, extra={'peername': addr}, server=server) + transport = self._make_ssl_transport( + conn, protocol, sslcontext, waiter=waiter, + server_side=True, extra=extra, server=server) else: - self._make_socket_transport( - conn, protocol , extra={'peername': addr}, + transport = self._make_socket_transport( + conn, protocol, waiter=waiter, extra=extra, server=server) - # It's now up to the protocol to handle the connection. + + try: + yield from waiter + except: + transport.close() + raise + + # It's now up to the protocol to handle the connection. + except Exception as exc: + if self.get_debug(): + context = { + 'message': ('Error on transport creation ' + 'for incoming connection'), + 'exception': exc, + } + if protocol is not None: + context['protocol'] = protocol + if transport is not None: + context['transport'] = transport + self.call_exception_handler(context) def add_reader(self, fd, callback, *args): """Add a reader callback.""" diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -592,7 +592,7 @@ fut.exception() return - if fut._state == futures._CANCELLED: + if fut.cancelled(): res = futures.CancelledError() if not return_exceptions: outer.set_exception(res) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -298,8 +298,10 @@ _set_nonblocking(self._fileno) self._protocol = protocol self._closing = False - self._loop.add_reader(self._fileno, self._read_ready) self._loop.call_soon(self._protocol.connection_made, self) + # only start reading when connection_made() has been called + self._loop.call_soon(self._loop.add_reader, + self._fileno, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called self._loop.call_soon(waiter._set_result_unless_cancelled, None) @@ -401,13 +403,16 @@ self._conn_lost = 0 self._closing = False # Set when close() or write_eof() called. - # On AIX, the reader trick only works for sockets. - # On other platforms it works for pipes and sockets. - # (Exception: OS X 10.4? Issue #19294.) + self._loop.call_soon(self._protocol.connection_made, self) + + # On AIX, the reader trick (to be notified when the read end of the + # socket is closed) only works for sockets. On other platforms it + # works for pipes and sockets. (Exception: OS X 10.4? Issue #19294.) if is_socket or not sys.platform.startswith("aix"): - self._loop.add_reader(self._fileno, self._read_ready) + # only start reading when connection_made() has been called + self._loop.call_soon(self._loop.add_reader, + self._fileno, self._read_ready) - self._loop.call_soon(self._protocol.connection_made, self) if waiter is not None: # only wake up the waiter when connection_made() has been called self._loop.call_soon(waiter._set_result_unless_cancelled, None) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -886,13 +886,18 @@ if hasattr(sslcontext_client, 'check_hostname'): sslcontext_client.check_hostname = True + # no CA loaded f_c = self.loop.create_connection(MyProto, host, port, ssl=sslcontext_client) - with test_utils.disable_logger(): - with self.assertRaisesRegex(ssl.SSLError, - 'certificate verify failed '): - self.loop.run_until_complete(f_c) + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex(ssl.SSLError, + 'certificate verify failed '): + self.loop.run_until_complete(f_c) + + # execute the loop to log the connection error + test_utils.run_briefly(self.loop) # close connection self.assertIsNone(proto.transport) @@ -919,15 +924,20 @@ f_c = self.loop.create_unix_connection(MyProto, path, ssl=sslcontext_client, server_hostname='invalid') - with test_utils.disable_logger(): - with self.assertRaisesRegex(ssl.SSLError, - 'certificate verify failed '): - self.loop.run_until_complete(f_c) + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex(ssl.SSLError, + 'certificate verify failed '): + self.loop.run_until_complete(f_c) + + # execute the loop to log the connection error + test_utils.run_briefly(self.loop) # close connection self.assertIsNone(proto.transport) server.close() + def test_legacy_create_unix_server_ssl_verify_failed(self): with test_utils.force_legacy_ssl_support(): self.test_create_unix_server_ssl_verify_failed() @@ -949,11 +959,12 @@ # incorrect server_hostname f_c = self.loop.create_connection(MyProto, host, port, ssl=sslcontext_client) - with test_utils.disable_logger(): - with self.assertRaisesRegex( - ssl.CertificateError, - "hostname '127.0.0.1' doesn't match 'localhost'"): - self.loop.run_until_complete(f_c) + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex( + ssl.CertificateError, + "hostname '127.0.0.1' doesn't match 'localhost'"): + self.loop.run_until_complete(f_c) # close connection proto.transport.close() diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -350,16 +350,13 @@ return transport def test_ctor(self): - tr = self.read_pipe_transport() + waiter = asyncio.Future(loop=self.loop) + tr = self.read_pipe_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.protocol.connection_made.assert_called_with(tr) self.loop.assert_reader(5, tr._read_ready) - test_utils.run_briefly(self.loop) - self.protocol.connection_made.assert_called_with(tr) - - def test_ctor_with_waiter(self): - fut = asyncio.Future(loop=self.loop) - tr = self.read_pipe_transport(waiter=fut) - test_utils.run_briefly(self.loop) - self.assertIsNone(fut.result()) + self.assertIsNone(waiter.result()) @mock.patch('os.read') def test__read_ready(self, m_read): @@ -502,17 +499,13 @@ return transport def test_ctor(self): - tr = self.write_pipe_transport() + waiter = asyncio.Future(loop=self.loop) + tr = self.write_pipe_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.protocol.connection_made.assert_called_with(tr) self.loop.assert_reader(5, tr._read_ready) - test_utils.run_briefly(self.loop) - self.protocol.connection_made.assert_called_with(tr) - - def test_ctor_with_waiter(self): - fut = asyncio.Future(loop=self.loop) - tr = self.write_pipe_transport(waiter=fut) - self.loop.assert_reader(5, tr._read_ready) - test_utils.run_briefly(self.loop) - self.assertEqual(None, fut.result()) + self.assertEqual(None, waiter.result()) def test_can_write_eof(self): tr = self.write_pipe_transport() -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.4 -> default): Merge 3.4 (asyncio)
by victor.stinner Jan. 29, 2015

Jan. 29, 2015

https://hg.python.org/cpython/rev/497c9e57a54e changeset: 94371:497c9e57a54e parent: 94369:647e6176d7d7 parent: 94370:c4fd6df9aea6 user: Victor Stinner <victor.stinner(a)gmail.com> date: Thu Jan 29 14:15:42 2015 +0100 summary: Merge 3.4 (asyncio) files: Lib/asyncio/selector_events.py | 44 +++++++++- Lib/asyncio/tasks.py | 2 +- Lib/asyncio/unix_events.py | 17 ++- Lib/test/test_asyncio/test_events.py | 37 ++++++--- Lib/test/test_asyncio/test_unix_events.py | 31 +++---- 5 files changed, 86 insertions(+), 45 deletions(-) diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -22,6 +22,7 @@ from . import selectors from . import transports from . import sslproto +from .coroutines import coroutine from .log import logger @@ -181,16 +182,47 @@ else: raise # The event loop will catch, log and ignore it. else: + extra = {'peername': addr} + accept = self._accept_connection2(protocol_factory, conn, extra, + sslcontext, server) + self.create_task(accept) + + @coroutine + def _accept_connection2(self, protocol_factory, conn, extra, + sslcontext=None, server=None): + protocol = None + transport = None + try: protocol = protocol_factory() + waiter = futures.Future(loop=self) if sslcontext: - self._make_ssl_transport( - conn, protocol, sslcontext, - server_side=True, extra={'peername': addr}, server=server) + transport = self._make_ssl_transport( + conn, protocol, sslcontext, waiter=waiter, + server_side=True, extra=extra, server=server) else: - self._make_socket_transport( - conn, protocol , extra={'peername': addr}, + transport = self._make_socket_transport( + conn, protocol, waiter=waiter, extra=extra, server=server) - # It's now up to the protocol to handle the connection. + + try: + yield from waiter + except: + transport.close() + raise + + # It's now up to the protocol to handle the connection. + except Exception as exc: + if self.get_debug(): + context = { + 'message': ('Error on transport creation ' + 'for incoming connection'), + 'exception': exc, + } + if protocol is not None: + context['protocol'] = protocol + if transport is not None: + context['transport'] = transport + self.call_exception_handler(context) def add_reader(self, fd, callback, *args): """Add a reader callback.""" diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -592,7 +592,7 @@ fut.exception() return - if fut._state == futures._CANCELLED: + if fut.cancelled(): res = futures.CancelledError() if not return_exceptions: outer.set_exception(res) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -298,8 +298,10 @@ _set_nonblocking(self._fileno) self._protocol = protocol self._closing = False - self._loop.add_reader(self._fileno, self._read_ready) self._loop.call_soon(self._protocol.connection_made, self) + # only start reading when connection_made() has been called + self._loop.call_soon(self._loop.add_reader, + self._fileno, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called self._loop.call_soon(waiter._set_result_unless_cancelled, None) @@ -401,13 +403,16 @@ self._conn_lost = 0 self._closing = False # Set when close() or write_eof() called. - # On AIX, the reader trick only works for sockets. - # On other platforms it works for pipes and sockets. - # (Exception: OS X 10.4? Issue #19294.) + self._loop.call_soon(self._protocol.connection_made, self) + + # On AIX, the reader trick (to be notified when the read end of the + # socket is closed) only works for sockets. On other platforms it + # works for pipes and sockets. (Exception: OS X 10.4? Issue #19294.) if is_socket or not sys.platform.startswith("aix"): - self._loop.add_reader(self._fileno, self._read_ready) + # only start reading when connection_made() has been called + self._loop.call_soon(self._loop.add_reader, + self._fileno, self._read_ready) - self._loop.call_soon(self._protocol.connection_made, self) if waiter is not None: # only wake up the waiter when connection_made() has been called self._loop.call_soon(waiter._set_result_unless_cancelled, None) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -886,13 +886,18 @@ if hasattr(sslcontext_client, 'check_hostname'): sslcontext_client.check_hostname = True + # no CA loaded f_c = self.loop.create_connection(MyProto, host, port, ssl=sslcontext_client) - with test_utils.disable_logger(): - with self.assertRaisesRegex(ssl.SSLError, - 'certificate verify failed '): - self.loop.run_until_complete(f_c) + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex(ssl.SSLError, + 'certificate verify failed '): + self.loop.run_until_complete(f_c) + + # execute the loop to log the connection error + test_utils.run_briefly(self.loop) # close connection self.assertIsNone(proto.transport) @@ -919,15 +924,20 @@ f_c = self.loop.create_unix_connection(MyProto, path, ssl=sslcontext_client, server_hostname='invalid') - with test_utils.disable_logger(): - with self.assertRaisesRegex(ssl.SSLError, - 'certificate verify failed '): - self.loop.run_until_complete(f_c) + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex(ssl.SSLError, + 'certificate verify failed '): + self.loop.run_until_complete(f_c) + + # execute the loop to log the connection error + test_utils.run_briefly(self.loop) # close connection self.assertIsNone(proto.transport) server.close() + def test_legacy_create_unix_server_ssl_verify_failed(self): with test_utils.force_legacy_ssl_support(): self.test_create_unix_server_ssl_verify_failed() @@ -949,11 +959,12 @@ # incorrect server_hostname f_c = self.loop.create_connection(MyProto, host, port, ssl=sslcontext_client) - with test_utils.disable_logger(): - with self.assertRaisesRegex( - ssl.CertificateError, - "hostname '127.0.0.1' doesn't match 'localhost'"): - self.loop.run_until_complete(f_c) + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex( + ssl.CertificateError, + "hostname '127.0.0.1' doesn't match 'localhost'"): + self.loop.run_until_complete(f_c) # close connection proto.transport.close() diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -350,16 +350,13 @@ return transport def test_ctor(self): - tr = self.read_pipe_transport() + waiter = asyncio.Future(loop=self.loop) + tr = self.read_pipe_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.protocol.connection_made.assert_called_with(tr) self.loop.assert_reader(5, tr._read_ready) - test_utils.run_briefly(self.loop) - self.protocol.connection_made.assert_called_with(tr) - - def test_ctor_with_waiter(self): - fut = asyncio.Future(loop=self.loop) - tr = self.read_pipe_transport(waiter=fut) - test_utils.run_briefly(self.loop) - self.assertIsNone(fut.result()) + self.assertIsNone(waiter.result()) @mock.patch('os.read') def test__read_ready(self, m_read): @@ -502,17 +499,13 @@ return transport def test_ctor(self): - tr = self.write_pipe_transport() + waiter = asyncio.Future(loop=self.loop) + tr = self.write_pipe_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.protocol.connection_made.assert_called_with(tr) self.loop.assert_reader(5, tr._read_ready) - test_utils.run_briefly(self.loop) - self.protocol.connection_made.assert_called_with(tr) - - def test_ctor_with_waiter(self): - fut = asyncio.Future(loop=self.loop) - tr = self.write_pipe_transport(waiter=fut) - self.loop.assert_reader(5, tr._read_ready) - test_utils.run_briefly(self.loop) - self.assertEqual(None, fut.result()) + self.assertEqual(None, waiter.result()) def test_can_write_eof(self): tr = self.write_pipe_transport() -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.4): asyncio doc: document Protocol state machine
by victor.stinner Jan. 29, 2015

Jan. 29, 2015

https://hg.python.org/cpython/rev/4fe1303bf531 changeset: 94368:4fe1303bf531 branch: 3.4 parent: 94366:65ee4dec4f1d user: Victor Stinner <victor.stinner(a)gmail.com> date: Thu Jan 29 13:33:15 2015 +0100 summary: asyncio doc: document Protocol state machine files: Doc/library/asyncio-protocol.rst | 8 ++++++++ Lib/asyncio/protocols.py | 5 +++++ 2 files changed, 13 insertions(+), 0 deletions(-) diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -374,6 +374,14 @@ a connection. However, :meth:`eof_received` is called at most once and, if called, :meth:`data_received` won't be called after it. +State machine: + + start -> :meth:`~BaseProtocol.connection_made` + [-> :meth:`~Protocol.data_received` \*] + [-> :meth:`~Protocol.eof_received` ?] + -> :meth:`~BaseProtocol.connection_lost` -> end + + Datagram protocols ------------------ diff --git a/Lib/asyncio/protocols.py b/Lib/asyncio/protocols.py --- a/Lib/asyncio/protocols.py +++ b/Lib/asyncio/protocols.py @@ -78,6 +78,11 @@ State machine of calls: start -> CM [-> DR*] [-> ER?] -> CL -> end + + * CM: connection_made() + * DR: data_received() + * ER: eof_received() + * CL: connection_lost() """ def data_received(self, data): -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.4 -> default): Merge 3.4 (asyncio doc)
by victor.stinner Jan. 29, 2015

Jan. 29, 2015

https://hg.python.org/cpython/rev/647e6176d7d7 changeset: 94369:647e6176d7d7 parent: 94367:25028b0e1183 parent: 94368:4fe1303bf531 user: Victor Stinner <victor.stinner(a)gmail.com> date: Thu Jan 29 13:33:28 2015 +0100 summary: Merge 3.4 (asyncio doc) files: Doc/library/asyncio-protocol.rst | 8 ++++++++ Lib/asyncio/protocols.py | 5 +++++ 2 files changed, 13 insertions(+), 0 deletions(-) diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -374,6 +374,14 @@ a connection. However, :meth:`eof_received` is called at most once and, if called, :meth:`data_received` won't be called after it. +State machine: + + start -> :meth:`~BaseProtocol.connection_made` + [-> :meth:`~Protocol.data_received` \*] + [-> :meth:`~Protocol.eof_received` ?] + -> :meth:`~BaseProtocol.connection_lost` -> end + + Datagram protocols ------------------ diff --git a/Lib/asyncio/protocols.py b/Lib/asyncio/protocols.py --- a/Lib/asyncio/protocols.py +++ b/Lib/asyncio/protocols.py @@ -78,6 +78,11 @@ State machine of calls: start -> CM [-> DR*] [-> ER?] -> CL -> end + + * CM: connection_made() + * DR: data_received() + * ER: eof_received() + * CL: connection_lost() """ def data_received(self, data): -- Repository URL: https://hg.python.org/cpython

1 0

Daily reference leaks (25028b0e1183): sum=9
by solipsis＠pitrou.net Jan. 29, 2015

Jan. 29, 2015

results for 25028b0e1183 on branch "default" -------------------------------------------- test_asyncio leaked [3, 0, 3] memory blocks, sum=6 test_functools leaked [0, 0, 3] memory blocks, sum=3 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogCIJoGx', '-x']

1 0

cpython (3.4): asyncio: sync with Tulip
by victor.stinner Jan. 29, 2015

Jan. 29, 2015

https://hg.python.org/cpython/rev/65ee4dec4f1d changeset: 94366:65ee4dec4f1d branch: 3.4 parent: 94364:44f90017eee3 user: Victor Stinner <victor.stinner(a)gmail.com> date: Thu Jan 29 02:56:05 2015 +0100 summary: asyncio: sync with Tulip * _SelectorTransport constructor: extra parameter is now optional * Fix _SelectorDatagramTransport constructor. Only start reading after connection_made() has been called. * Fix _SelectorSslTransport.close(). Don't call protocol.connection_lost() if protocol.connection_made() was not called yet: if the SSL handshake failed or is still in progress. The close() method can be called if the creation of the connection is cancelled, by a timeout for example. files: Lib/asyncio/selector_events.py | 13 ++++++-- Lib/test/test_asyncio/test_selector_events.py | 15 +++++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -467,7 +467,7 @@ _buffer_factory = bytearray # Constructs initial value for self._buffer. - def __init__(self, loop, sock, protocol, extra, server=None): + def __init__(self, loop, sock, protocol, extra=None, server=None): super().__init__(extra, loop) self._extra['socket'] = sock self._extra['sockname'] = sock.getsockname() @@ -479,6 +479,7 @@ self._sock = sock self._sock_fd = sock.fileno() self._protocol = protocol + self._protocol_connected = True self._server = server self._buffer = self._buffer_factory() self._conn_lost = 0 # Set when call to connection_lost scheduled. @@ -555,7 +556,8 @@ def _call_connection_lost(self, exc): try: - self._protocol.connection_lost(exc) + if self._protocol_connected: + self._protocol.connection_lost(exc) finally: self._sock.close() self._sock = None @@ -718,6 +720,8 @@ sslsock = sslcontext.wrap_socket(rawsock, **wrap_kwargs) super().__init__(loop, sslsock, protocol, extra, server) + # the protocol connection is only made after the SSL handshake + self._protocol_connected = False self._server_hostname = server_hostname self._waiter = waiter @@ -797,6 +801,7 @@ self._read_wants_write = False self._write_wants_read = False self._loop.add_reader(self._sock_fd, self._read_ready) + self._protocol_connected = True self._loop.call_soon(self._protocol.connection_made, self) # only wake up the waiter when connection_made() has been called self._loop.call_soon(self._wakeup_waiter) @@ -928,8 +933,10 @@ waiter=None, extra=None): super().__init__(loop, sock, protocol, extra) self._address = address - self._loop.add_reader(self._sock_fd, self._read_ready) self._loop.call_soon(self._protocol.connection_made, self) + # only start reading when connection_made() has been called + self._loop.call_soon(self._loop.add_reader, + self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called self._loop.call_soon(waiter._set_result_unless_cancelled, None) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -1427,7 +1427,7 @@ self.assertFalse(tr.can_write_eof()) self.assertRaises(NotImplementedError, tr.write_eof) - def test_close(self): + def check_close(self): tr = self._make_one() tr.close() @@ -1439,6 +1439,19 @@ self.assertEqual(tr._conn_lost, 1) self.assertEqual(1, self.loop.remove_reader_count[1]) + test_utils.run_briefly(self.loop) + + def test_close(self): + self.check_close() + self.assertTrue(self.protocol.connection_made.called) + self.assertTrue(self.protocol.connection_lost.called) + + def test_close_not_connected(self): + self.sslsock.do_handshake.side_effect = ssl.SSLWantReadError + self.check_close() + self.assertFalse(self.protocol.connection_made.called) + self.assertFalse(self.protocol.connection_lost.called) + @unittest.skipIf(ssl is None, 'No SSL support') def test_server_hostname(self): self.ssl_transport(server_hostname='localhost') -- Repository URL: https://hg.python.org/cpython

1 0