Mailman 3 December 2017 - Python-checkins

bpo-32418: Add get_loop() method on Server, AbstractServer classes (#4997)
by Andrew Svetlov 30 Dec '17

30 Dec '17

https://github.com/python/cpython/commit/1634fc289a13b0e1fdad4694d5cae7dab5… commit: 1634fc289a13b0e1fdad4694d5cae7dab55f7186 branch: master author: Srinivas Reddy Thatiparthy (శ్రీనివాస్ రెడ్డి తాటిపర్తి) <srinivasreddy(a)users.noreply.github.com> committer: Andrew Svetlov <andrew.svetlov(a)gmail.com> date: 2017-12-30T17:09:32+02:00 summary: bpo-32418: Add get_loop() method on Server, AbstractServer classes (#4997) * Add abstract get_loop() method to Server, AbstractServer classes. * Add test cases for get_loop() method in Server, AbstractServer classes * Add documentation for get_loop() method files: A Misc/NEWS.d/next/Documentation/2017-12-24-17-29-37.bpo-32418.eZe-ID.rst M Doc/library/asyncio-eventloop.rst M Lib/asyncio/base_events.py M Lib/asyncio/events.py M Lib/test/test_asyncio/test_events.py diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 33b86d6f033..4fbbcd86cf4 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -913,6 +913,12 @@ Server The server is closed asynchronously, use the :meth:`wait_closed` coroutine to wait until the server is closed. + .. method:: get_loop() + + Gives the event loop associated with the server object. + + .. versionadded:: 3.7 + .. coroutinemethod:: wait_closed() Wait until the :meth:`close` method completes. diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 00831b39853..ab002319c18 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -185,6 +185,9 @@ def close(self): if self._active_count == 0: self._wakeup() + def get_loop(self): + return self._loop + def _wakeup(self): waiters = self._waiters self._waiters = None diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 9496d5c765f..731a0c5e802 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -155,6 +155,10 @@ def close(self): """Coroutine to wait until service is closed.""" return NotImplemented + def get_loop(self): + """ Get the event loop the Server object is attached to.""" + return NotImplemented + class AbstractEventLoop: """Abstract event loop.""" diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index da2e036648b..e8320564c74 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2826,6 +2826,19 @@ class TestCGetEventLoop(GetEventLoopTestsMixin, unittest.TestCase): get_running_loop_impl = events._c_get_running_loop get_event_loop_impl = events._c_get_event_loop +class TestServer(unittest.TestCase): + + def test_get_loop(self): + loop = asyncio.new_event_loop() + proto = MyProto(loop) + server = loop.run_until_complete(loop.create_server(lambda: proto, '0.0.0.0', 0)) + self.assertEqual(server.get_loop(), loop) + loop.close() + +class TestAbstractServer(unittest.TestCase): + + def test_get_loop(self): + self.assertEqual(events.AbstractServer().get_loop(), NotImplemented) if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Documentation/2017-12-24-17-29-37.bpo-32418.eZe-ID.rst b/Misc/NEWS.d/next/Documentation/2017-12-24-17-29-37.bpo-32418.eZe-ID.rst new file mode 100644 index 00000000000..9441b74ef84 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2017-12-24-17-29-37.bpo-32418.eZe-ID.rst @@ -0,0 +1 @@ +Add get_loop() method to Server and AbstractServer classes.

1 0

Fix typo (#5049)
by Andrew Svetlov 30 Dec '17

30 Dec '17

https://github.com/python/cpython/commit/fc35932afdad91f87b49a4854d43332674… commit: fc35932afdad91f87b49a4854d4333267494c6c1 branch: master author: Andrew Svetlov <andrew.svetlov(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-12-30T15:40:27+02:00 summary: Fix typo (#5049) files: M Modules/_asynciomodule.c diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 5ec4ad182ab..22ce32c593c 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2265,7 +2265,7 @@ _asyncio_Task_set_exception(TaskObj *self, PyObject *exception) /*[clinic end generated code: output=bc377fc28067303d input=9a8f65c83dcf893a]*/ { PyErr_SetString(PyExc_RuntimeError, - "Task doed not support set_exception operation"); + "Task does not support set_exception operation"); return NULL; }

1 0

Daily reference leaks (4243df51fe43): sum=1
by solipsis＠pitrou.net 30 Dec '17

30 Dec '17

results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [0, 0, 3] memory blocks, sum=3 test_collections leaked [-7, 1, 0] memory blocks, sum=-6 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [0, 0, -2] memory blocks, sum=-2 test_multiprocessing_forkserver leaked [2, 0, 0] memory blocks, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflog4uE42t', '--timeout', '7200']

1 0

bpo-32439: Clean up the code for compiling comparison expressions. (#5029)
by Serhiy Storchaka 30 Dec '17

30 Dec '17

https://github.com/python/cpython/commit/02b9ef27752ff4873c592ac3afe7e3410f… commit: 02b9ef27752ff4873c592ac3afe7e3410f715984 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-12-30T09:47:42+02:00 summary: bpo-32439: Clean up the code for compiling comparison expressions. (#5029) files: M Python/compile.c diff --git a/Python/compile.c b/Python/compile.c index e5745d16e30..cd039168eba 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3420,35 +3420,32 @@ static int compiler_compare(struct compiler *c, expr_ty e) { Py_ssize_t i, n; - basicblock *cleanup = NULL; - /* XXX the logic can be cleaned up for 1 or multiple comparisons */ VISIT(c, expr, e->v.Compare.left); - n = asdl_seq_LEN(e->v.Compare.ops); - assert(n > 0); - if (n > 1) { - cleanup = compiler_new_block(c); + assert(asdl_seq_LEN(e->v.Compare.ops) > 0); + n = asdl_seq_LEN(e->v.Compare.ops) - 1; + if (n == 0) { + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, 0)); + ADDOP_I(c, COMPARE_OP, + cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, 0)))); + } + else { + basicblock *cleanup = compiler_new_block(c); if (cleanup == NULL) return 0; - VISIT(c, expr, - (expr_ty)asdl_seq_GET(e->v.Compare.comparators, 0)); - } - for (i = 1; i < n; i++) { - ADDOP(c, DUP_TOP); - ADDOP(c, ROT_THREE); - ADDOP_I(c, COMPARE_OP, - cmpop((cmpop_ty)(asdl_seq_GET( - e->v.Compare.ops, i - 1)))); - ADDOP_JABS(c, JUMP_IF_FALSE_OR_POP, cleanup); - NEXT_BLOCK(c); - if (i < (n - 1)) + for (i = 0; i < n; i++) { VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i)); - } - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n - 1)); - ADDOP_I(c, COMPARE_OP, - cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, n - 1)))); - if (n > 1) { + ADDOP(c, DUP_TOP); + ADDOP(c, ROT_THREE); + ADDOP_I(c, COMPARE_OP, + cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, i)))); + ADDOP_JABS(c, JUMP_IF_FALSE_OR_POP, cleanup); + NEXT_BLOCK(c); + } + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n)); + ADDOP_I(c, COMPARE_OP, + cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, n)))); basicblock *end = compiler_new_block(c); if (end == NULL) return 0;

1 0

bpo-23749: Implement loop.start_tls() (#5039)
by Yury Selivanov 30 Dec '17

30 Dec '17

https://github.com/python/cpython/commit/f111b3dcb414093a4efb9d74b69925e535… commit: f111b3dcb414093a4efb9d74b69925e535ddc470 branch: master author: Yury Selivanov <yury(a)magic.io> committer: GitHub <noreply(a)github.com> date: 2017-12-30T00:35:36-05:00 summary: bpo-23749: Implement loop.start_tls() (#5039) files: A Lib/test/test_asyncio/functional.py A Misc/NEWS.d/next/Library/2017-12-29-00-44-42.bpo-23749.QL1Cxd.rst M Doc/library/asyncio-eventloop.rst M Lib/asyncio/base_events.py M Lib/asyncio/events.py M Lib/asyncio/proactor_events.py M Lib/asyncio/selector_events.py M Lib/test/test_asyncio/test_events.py M Lib/test/test_asyncio/test_sslproto.py M Lib/test/test_asyncio/utils.py diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 5dd258df312..33b86d6f033 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -537,6 +537,38 @@ Creating listening connections .. versionadded:: 3.5.3 +TLS Upgrade +----------- + +.. coroutinemethod:: AbstractEventLoop.start_tls(transport, protocol, sslcontext, \*, server_side=False, server_hostname=None, ssl_handshake_timeout=None) + + Upgrades an existing connection to TLS. + + Returns a new transport instance, that the *protocol* must start using + immediately after the *await*. The *transport* instance passed to + the *start_tls* method should never be used again. + + Parameters: + + * *transport* and *protocol* instances that methods like + :meth:`~AbstractEventLoop.create_server` and + :meth:`~AbstractEventLoop.create_connection` return. + + * *sslcontext*: a configured instance of :class:`~ssl.SSLContext`. + + * *server_side* pass ``True`` when a server-side connection is being + upgraded (like the one created by :meth:`~AbstractEventLoop.create_server`). + + * *server_hostname*: sets or overrides the host name that the target + server's certificate will be matched against. + + * *ssl_handshake_timeout* is (for an SSL connection) the time in seconds to + wait for the SSL handshake to complete before aborting the connection. + ``10.0`` seconds if ``None`` (default). + + .. versionadded:: 3.7 + + Watch file descriptors ---------------------- diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 96cc4f02588..00831b39853 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -29,9 +29,15 @@ import warnings import weakref +try: + import ssl +except ImportError: # pragma: no cover + ssl = None + from . import coroutines from . import events from . import futures +from . import sslproto from . import tasks from .log import logger @@ -279,7 +285,8 @@ def _make_ssl_transport( self, rawsock, protocol, sslcontext, waiter=None, *, server_side=False, server_hostname=None, extra=None, server=None, - ssl_handshake_timeout=None): + ssl_handshake_timeout=None, + call_connection_made=True): """Create SSL transport.""" raise NotImplementedError @@ -795,6 +802,42 @@ def _getaddrinfo_debug(self, host, port, family, type, proto, flags): return transport, protocol + async def start_tls(self, transport, protocol, sslcontext, *, + server_side=False, + server_hostname=None, + ssl_handshake_timeout=None): + """Upgrade transport to TLS. + + Return a new transport that *protocol* should start using + immediately. + """ + if ssl is None: + raise RuntimeError('Python ssl module is not available') + + if not isinstance(sslcontext, ssl.SSLContext): + raise TypeError( + f'sslcontext is expected to be an instance of ssl.SSLContext, ' + f'got {sslcontext!r}') + + if not getattr(transport, '_start_tls_compatible', False): + raise TypeError( + f'transport {self!r} is not supported by start_tls()') + + waiter = self.create_future() + ssl_protocol = sslproto.SSLProtocol( + self, protocol, sslcontext, waiter, + server_side, server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout, + call_connection_made=False) + + transport.set_protocol(ssl_protocol) + self.call_soon(ssl_protocol.connection_made, transport) + if not transport.is_reading(): + self.call_soon(transport.resume_reading) + + await waiter + return ssl_protocol._app_transport + async def create_datagram_endpoint(self, protocol_factory, local_addr=None, remote_addr=None, *, family=0, proto=0, flags=0, diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 3a5dbadbb10..9496d5c765f 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -305,6 +305,17 @@ def set_default_executor(self, executor): """ raise NotImplementedError + async def start_tls(self, transport, protocol, sslcontext, *, + server_side=False, + server_hostname=None, + ssl_handshake_timeout=None): + """Upgrade a transport to TLS. + + Return a new transport that *protocol* should start using + immediately. + """ + raise NotImplementedError + async def create_unix_connection( self, protocol_factory, path=None, *, ssl=None, sock=None, diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 2661cddef73..ab1285b7999 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -223,6 +223,8 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport, transports.WriteTransport): """Transport for write pipes.""" + _start_tls_compatible = True + def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): raise TypeError( diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 1e4bd83a1b1..5692e38486a 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -694,6 +694,8 @@ def get_write_buffer_size(self): class _SelectorSocketTransport(_SelectorTransport): + _start_tls_compatible = True + def __init__(self, loop, sock, protocol, waiter=None, extra=None, server=None): super().__init__(loop, sock, protocol, extra, server) diff --git a/Lib/test/test_asyncio/functional.py b/Lib/test/test_asyncio/functional.py new file mode 100644 index 00000000000..5fd174b6f43 --- /dev/null +++ b/Lib/test/test_asyncio/functional.py @@ -0,0 +1,279 @@ +import asyncio +import asyncio.events +import contextlib +import os +import pprint +import select +import socket +import ssl +import tempfile +import threading + + +class FunctionalTestCaseMixin: + + def new_loop(self): + return asyncio.new_event_loop() + + def run_loop_briefly(self, *, delay=0.01): + self.loop.run_until_complete(asyncio.sleep(delay, loop=self.loop)) + + def loop_exception_handler(self, loop, context): + self.__unhandled_exceptions.append(context) + self.loop.default_exception_handler(context) + + def setUp(self): + self.loop = self.new_loop() + asyncio.set_event_loop(None) + + self.loop.set_exception_handler(self.loop_exception_handler) + self.__unhandled_exceptions = [] + + # Disable `_get_running_loop`. + self._old_get_running_loop = asyncio.events._get_running_loop + asyncio.events._get_running_loop = lambda: None + + def tearDown(self): + try: + self.loop.close() + + if self.__unhandled_exceptions: + print('Unexpected calls to loop.call_exception_handler():') + pprint.pprint(self.__unhandled_exceptions) + self.fail('unexpected calls to loop.call_exception_handler()') + + finally: + asyncio.events._get_running_loop = self._old_get_running_loop + asyncio.set_event_loop(None) + self.loop = None + + def tcp_server(self, server_prog, *, + family=socket.AF_INET, + addr=None, + timeout=5, + backlog=1, + max_clients=10): + + if addr is None: + if hasattr(socket, 'AF_UNIX') and family == socket.AF_UNIX: + with tempfile.NamedTemporaryFile() as tmp: + addr = tmp.name + else: + addr = ('127.0.0.1', 0) + + sock = socket.socket(family, socket.SOCK_STREAM) + + if timeout is None: + raise RuntimeError('timeout is required') + if timeout <= 0: + raise RuntimeError('only blocking sockets are supported') + sock.settimeout(timeout) + + try: + sock.bind(addr) + sock.listen(backlog) + except OSError as ex: + sock.close() + raise ex + + return TestThreadedServer( + self, sock, server_prog, timeout, max_clients) + + def tcp_client(self, client_prog, + family=socket.AF_INET, + timeout=10): + + sock = socket.socket(family, socket.SOCK_STREAM) + + if timeout is None: + raise RuntimeError('timeout is required') + if timeout <= 0: + raise RuntimeError('only blocking sockets are supported') + sock.settimeout(timeout) + + return TestThreadedClient( + self, sock, client_prog, timeout) + + def unix_server(self, *args, **kwargs): + if not hasattr(socket, 'AF_UNIX'): + raise NotImplementedError + return self.tcp_server(*args, family=socket.AF_UNIX, **kwargs) + + def unix_client(self, *args, **kwargs): + if not hasattr(socket, 'AF_UNIX'): + raise NotImplementedError + return self.tcp_client(*args, family=socket.AF_UNIX, **kwargs) + + @contextlib.contextmanager + def unix_sock_name(self): + with tempfile.TemporaryDirectory() as td: + fn = os.path.join(td, 'sock') + try: + yield fn + finally: + try: + os.unlink(fn) + except OSError: + pass + + def _abort_socket_test(self, ex): + try: + self.loop.stop() + finally: + self.fail(ex) + + +############################################################################## +# Socket Testing Utilities +############################################################################## + + +class TestSocketWrapper: + + def __init__(self, sock): + self.__sock = sock + + def recv_all(self, n): + buf = b'' + while len(buf) < n: + data = self.recv(n - len(buf)) + if data == b'': + raise ConnectionAbortedError + buf += data + return buf + + def start_tls(self, ssl_context, *, + server_side=False, + server_hostname=None): + + assert isinstance(ssl_context, ssl.SSLContext) + + ssl_sock = ssl_context.wrap_socket( + self.__sock, server_side=server_side, + server_hostname=server_hostname, + do_handshake_on_connect=False) + + ssl_sock.do_handshake() + + self.__sock.close() + self.__sock = ssl_sock + + def __getattr__(self, name): + return getattr(self.__sock, name) + + def __repr__(self): + return '<{} {!r}>'.format(type(self).__name__, self.__sock) + + +class SocketThread(threading.Thread): + + def stop(self): + self._active = False + self.join() + + def __enter__(self): + self.start() + return self + + def __exit__(self, *exc): + self.stop() + + +class TestThreadedClient(SocketThread): + + def __init__(self, test, sock, prog, timeout): + threading.Thread.__init__(self, None, None, 'test-client') + self.daemon = True + + self._timeout = timeout + self._sock = sock + self._active = True + self._prog = prog + self._test = test + + def run(self): + try: + self._prog(TestSocketWrapper(self._sock)) + except Exception as ex: + self._test._abort_socket_test(ex) + + +class TestThreadedServer(SocketThread): + + def __init__(self, test, sock, prog, timeout, max_clients): + threading.Thread.__init__(self, None, None, 'test-server') + self.daemon = True + + self._clients = 0 + self._finished_clients = 0 + self._max_clients = max_clients + self._timeout = timeout + self._sock = sock + self._active = True + + self._prog = prog + + self._s1, self._s2 = socket.socketpair() + self._s1.setblocking(False) + + self._test = test + + def stop(self): + try: + if self._s2 and self._s2.fileno() != -1: + try: + self._s2.send(b'stop') + except OSError: + pass + finally: + super().stop() + + def run(self): + try: + with self._sock: + self._sock.setblocking(0) + self._run() + finally: + self._s1.close() + self._s2.close() + + def _run(self): + while self._active: + if self._clients >= self._max_clients: + return + + r, w, x = select.select( + [self._sock, self._s1], [], [], self._timeout) + + if self._s1 in r: + return + + if self._sock in r: + try: + conn, addr = self._sock.accept() + except BlockingIOError: + continue + except socket.timeout: + if not self._active: + return + else: + raise + else: + self._clients += 1 + conn.settimeout(self._timeout) + try: + with conn: + self._handle_client(conn) + except Exception as ex: + self._active = False + try: + raise + finally: + self._test._abort_socket_test(ex) + + def _handle_client(self, sock): + self._prog(TestSocketWrapper(sock)) + + @property + def addr(self): + return self._sock.getsockname() diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 79e8d79e6b1..da2e036648b 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -31,21 +31,7 @@ from asyncio import proactor_events from asyncio import selector_events from test.test_asyncio import utils as test_utils -try: - from test import support -except ImportError: - from asyncio import test_support as support - - -def data_file(filename): - if hasattr(support, 'TEST_HOME_DIR'): - fullname = os.path.join(support.TEST_HOME_DIR, filename) - if os.path.isfile(fullname): - return fullname - fullname = os.path.join(os.path.dirname(__file__), filename) - if os.path.isfile(fullname): - return fullname - raise FileNotFoundError(filename) +from test import support def osx_tiger(): @@ -80,23 +66,6 @@ def __await__(self): pass -ONLYCERT = data_file('ssl_cert.pem') -ONLYKEY = data_file('ssl_key.pem') -SIGNED_CERTFILE = data_file('keycert3.pem') -SIGNING_CA = data_file('pycacert.pem') -PEERCERT = {'serialNumber': 'B09264B1F2DA21D1', - 'version': 1, - 'subject': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'issuer': ((('countryName', 'XY'),), - (('organizationName', 'Python Software Foundation CA'),), - (('commonName', 'our-ca-server'),)), - 'notAfter': 'Nov 13 19:47:07 2022 GMT', - 'notBefore': 'Jan 4 19:47:07 2013 GMT'} - - class MyBaseProto(asyncio.Protocol): connected = None done = None @@ -853,16 +822,8 @@ def test_ssl_connect_accepted_socket(self): 'SSL not supported with proactor event loops before Python 3.5' ) - server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - server_context.load_cert_chain(ONLYCERT, ONLYKEY) - if hasattr(server_context, 'check_hostname'): - server_context.check_hostname = False - server_context.verify_mode = ssl.CERT_NONE - - client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - if hasattr(server_context, 'check_hostname'): - client_context.check_hostname = False - client_context.verify_mode = ssl.CERT_NONE + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() self.test_connect_accepted_socket(server_context, client_context) @@ -1048,7 +1009,7 @@ def _make_ssl_unix_server(self, factory, certfile, keyfile=None): def test_create_server_ssl(self): proto = MyProto(loop=self.loop) server, host, port = self._make_ssl_server( - lambda: proto, ONLYCERT, ONLYKEY) + lambda: proto, test_utils.ONLYCERT, test_utils.ONLYKEY) f_c = self.loop.create_connection(MyBaseProto, host, port, ssl=test_utils.dummy_ssl_context()) @@ -1081,7 +1042,7 @@ def test_create_server_ssl(self): def test_create_unix_server_ssl(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( - lambda: proto, ONLYCERT, ONLYKEY) + lambda: proto, test_utils.ONLYCERT, test_utils.ONLYKEY) f_c = self.loop.create_unix_connection( MyBaseProto, path, ssl=test_utils.dummy_ssl_context(), @@ -1111,7 +1072,7 @@ def test_create_unix_server_ssl(self): def test_create_server_ssl_verify_failed(self): proto = MyProto(loop=self.loop) server, host, port = self._make_ssl_server( - lambda: proto, SIGNED_CERTFILE) + lambda: proto, test_utils.SIGNED_CERTFILE) sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) sslcontext_client.options |= ssl.OP_NO_SSLv2 @@ -1141,7 +1102,7 @@ def test_create_server_ssl_verify_failed(self): def test_create_unix_server_ssl_verify_failed(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( - lambda: proto, SIGNED_CERTFILE) + lambda: proto, test_utils.SIGNED_CERTFILE) sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) sslcontext_client.options |= ssl.OP_NO_SSLv2 @@ -1170,13 +1131,13 @@ def test_create_unix_server_ssl_verify_failed(self): def test_create_server_ssl_match_failed(self): proto = MyProto(loop=self.loop) server, host, port = self._make_ssl_server( - lambda: proto, SIGNED_CERTFILE) + lambda: proto, test_utils.SIGNED_CERTFILE) sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) sslcontext_client.options |= ssl.OP_NO_SSLv2 sslcontext_client.verify_mode = ssl.CERT_REQUIRED sslcontext_client.load_verify_locations( - cafile=SIGNING_CA) + cafile=test_utils.SIGNING_CA) if hasattr(sslcontext_client, 'check_hostname'): sslcontext_client.check_hostname = True @@ -1199,12 +1160,12 @@ def test_create_server_ssl_match_failed(self): def test_create_unix_server_ssl_verified(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( - lambda: proto, SIGNED_CERTFILE) + lambda: proto, test_utils.SIGNED_CERTFILE) sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) sslcontext_client.options |= ssl.OP_NO_SSLv2 sslcontext_client.verify_mode = ssl.CERT_REQUIRED - sslcontext_client.load_verify_locations(cafile=SIGNING_CA) + sslcontext_client.load_verify_locations(cafile=test_utils.SIGNING_CA) if hasattr(sslcontext_client, 'check_hostname'): sslcontext_client.check_hostname = True @@ -1224,12 +1185,12 @@ def test_create_unix_server_ssl_verified(self): def test_create_server_ssl_verified(self): proto = MyProto(loop=self.loop) server, host, port = self._make_ssl_server( - lambda: proto, SIGNED_CERTFILE) + lambda: proto, test_utils.SIGNED_CERTFILE) sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) sslcontext_client.options |= ssl.OP_NO_SSLv2 sslcontext_client.verify_mode = ssl.CERT_REQUIRED - sslcontext_client.load_verify_locations(cafile=SIGNING_CA) + sslcontext_client.load_verify_locations(cafile=test_utils.SIGNING_CA) if hasattr(sslcontext_client, 'check_hostname'): sslcontext_client.check_hostname = True @@ -1241,7 +1202,7 @@ def test_create_server_ssl_verified(self): # extra info is available self.check_ssl_extra_info(client,peername=(host, port), - peercert=PEERCERT) + peercert=test_utils.PEERCERT) # close connection proto.transport.close() diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index a7498e85c25..886c5cf3626 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -13,6 +13,7 @@ from asyncio import sslproto from asyncio import tasks from test.test_asyncio import utils as test_utils +from test.test_asyncio import functional as func_tests @unittest.skipIf(ssl is None, 'No ssl module') @@ -158,5 +159,156 @@ def test_set_new_app_protocol(self): self.assertIs(ssl_proto._app_protocol, new_app_proto) +############################################################################## +# Start TLS Tests +############################################################################## + + +class BaseStartTLS(func_tests.FunctionalTestCaseMixin): + + def new_loop(self): + raise NotImplementedError + + def test_start_tls_client_1(self): + HELLO_MSG = b'1' * 1024 * 1024 * 5 + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + def serve(sock): + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.start_tls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr) + + tr.write(HELLO_MSG) + new_tr = await self.loop.start_tls(tr, proto, client_context) + + self.assertEqual(await on_data, b'O') + new_tr.write(HELLO_MSG) + await on_eof + + new_tr.close() + + with self.tcp_server(serve) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), loop=self.loop, timeout=10)) + + def test_start_tls_server_1(self): + HELLO_MSG = b'1' * 1024 * 1024 * 5 + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + def client(sock, addr): + sock.connect(addr) + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.start_tls(client_context) + sock.sendall(HELLO_MSG) + sock.close() + + class ServerProto(asyncio.Protocol): + def __init__(self, on_con, on_eof): + self.on_con = on_con + self.on_eof = on_eof + self.data = b'' + + def connection_made(self, tr): + self.on_con.set_result(tr) + + def data_received(self, data): + self.data += data + + def eof_received(self): + self.on_eof.set_result(1) + + async def main(): + tr = await on_con + tr.write(HELLO_MSG) + + self.assertEqual(proto.data, b'') + + new_tr = await self.loop.start_tls( + tr, proto, server_context, + server_side=True) + + await on_eof + self.assertEqual(proto.data, HELLO_MSG) + new_tr.close() + + server.close() + await server.wait_closed() + + on_con = self.loop.create_future() + on_eof = self.loop.create_future() + proto = ServerProto(on_con, on_eof) + + server = self.loop.run_until_complete( + self.loop.create_server( + lambda: proto, '127.0.0.1', 0)) + addr = server.sockets[0].getsockname() + + with self.tcp_client(lambda sock: client(sock, addr)): + self.loop.run_until_complete( + asyncio.wait_for(main(), loop=self.loop, timeout=10)) + + def test_start_tls_wrong_args(self): + async def main(): + with self.assertRaisesRegex(TypeError, 'SSLContext, got'): + await self.loop.start_tls(None, None, None) + + sslctx = test_utils.simple_server_sslcontext() + with self.assertRaisesRegex(TypeError, 'is not supported'): + await self.loop.start_tls(None, None, sslctx) + + self.loop.run_until_complete(main()) + + +(a)unittest.skipIf(ssl is None, 'No ssl module') +class SelectorStartTLS(BaseStartTLS, unittest.TestCase): + + def new_loop(self): + return asyncio.SelectorEventLoop() + + +(a)unittest.skipIf(ssl is None, 'No ssl module') +(a)unittest.skipUnless(hasattr(asyncio, 'ProactorEventLoop'), 'Windows only') +class ProactorStartTLS(BaseStartTLS, unittest.TestCase): + + def new_loop(self): + return asyncio.ProactorEventLoop() + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index eaafe3af8b8..a78e01988d8 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -35,6 +35,49 @@ from test import support +def data_file(filename): + if hasattr(support, 'TEST_HOME_DIR'): + fullname = os.path.join(support.TEST_HOME_DIR, filename) + if os.path.isfile(fullname): + return fullname + fullname = os.path.join(os.path.dirname(__file__), filename) + if os.path.isfile(fullname): + return fullname + raise FileNotFoundError(filename) + + +ONLYCERT = data_file('ssl_cert.pem') +ONLYKEY = data_file('ssl_key.pem') +SIGNED_CERTFILE = data_file('keycert3.pem') +SIGNING_CA = data_file('pycacert.pem') +PEERCERT = {'serialNumber': 'B09264B1F2DA21D1', + 'version': 1, + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'issuer': ((('countryName', 'XY'),), + (('organizationName', 'Python Software Foundation CA'),), + (('commonName', 'our-ca-server'),)), + 'notAfter': 'Nov 13 19:47:07 2022 GMT', + 'notBefore': 'Jan 4 19:47:07 2013 GMT'} + + +def simple_server_sslcontext(): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(ONLYCERT, ONLYKEY) + server_context.check_hostname = False + server_context.verify_mode = ssl.CERT_NONE + return server_context + + +def simple_client_sslcontext(): + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + return client_context + + def dummy_ssl_context(): if ssl is None: return None diff --git a/Misc/NEWS.d/next/Library/2017-12-29-00-44-42.bpo-23749.QL1Cxd.rst b/Misc/NEWS.d/next/Library/2017-12-29-00-44-42.bpo-23749.QL1Cxd.rst new file mode 100644 index 00000000000..d6de1fef901 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-29-00-44-42.bpo-23749.QL1Cxd.rst @@ -0,0 +1 @@ +asyncio: Implement loop.start_tls()

1 0

[65 flat] Results for Python (master branch) 2017-12-29
by lp_benchmark_robot＠intel.com 29 Dec '17

29 Dec '17

Results for project python/master, build date: 2017-12-29 03:03:56-08:00. - commit: e325608 - previous commit: 531dd7e - revision date: 2017-12-28 23:24:40-08:00 - environment: Broadwell-EP - cpu: Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz 2x22 cores, stepping 1, LLC 55 MB - mem: 128 GB - os: Ubuntu 16.04.2 LTS - kernel: 4.4.0-62-generic x86_64 GNU/Linux Baseline results were generated using release v3.6.0, with hash 5c4568a from 2016-12-22 23:38:47+00:00. +-----+------------------------+--------+------------+------------+------------+ | | |relative|change since|change since|current rev | | | benchmark|std_dev*| last run | baseline |run with PGO| +-----+------------------------+--------+------------+------------+------------+ | :-| | 2to3| 1.095% | +1.615% | +7.213% | +7.126% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method| 0.604% | -0.079% | +23.728% | +9.367% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_slots| 1.817% | +0.146% | +25.534% | +4.980% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_unknown| 1.046% | -0.314% | +22.935% | +7.907% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_simple| 3.059% | -0.160% | +8.631% | +9.215% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chameleon| 1.300% | +0.336% | +9.651% | +13.793% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chaos| 0.819% | +0.061% | +5.705% | +10.983% | +-----+------------------------+--------+------------+------------+------------+ | :-| | crypto_pyaes| 0.595% | +0.205% | -1.638% | +9.368% | +-----+------------------------+--------+------------+------------+------------+ | :-| | deltablue| 2.746% | +0.352% | +10.712% | +14.661% | +-----+------------------------+--------+------------+------------+------------+ | :-| | django_template| 1.297% | -0.742% | +12.110% | +15.067% | +-----+------------------------+--------+------------+------------+------------+ | :-| | dulwich_log| 1.247% | -0.021% | +3.764% | +7.901% | +-----+------------------------+--------+------------+------------+------------+ | :-| | fannkuch| 0.573% | -0.053% | +5.270% | +4.439% | +-----+------------------------+--------+------------+------------+------------+ | :-| | float| 0.727% | +0.894% | +1.952% | +6.033% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_text| 1.027% | -0.172% | +12.822% | +10.568% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_xml| 1.198% | -0.526% | +8.808% | +10.690% | +-----+------------------------+--------+------------+------------+------------+ | :-| | go| 0.885% | +0.227% | +6.307% | +9.599% | +-----+------------------------+--------+------------+------------+------------+ | :-| | hexiom| 0.512% | +0.114% | +10.244% | +10.741% | +-----+------------------------+--------+------------+------------+------------+ | :-| | html5lib| 3.204% | -0.632% | +6.773% | +9.246% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_dumps| 1.160% | +0.358% | +3.702% | +8.373% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_loads| 1.289% | +0.225% | +0.174% | +12.734% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_format| 1.390% | -0.886% | +6.188% | +12.669% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_silent| 2.664% | +0.678% | +46.781% | +13.070% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_simple| 1.848% | -0.707% | +6.494% | +14.858% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mako| 0.801% | +0.202% | +16.669% | +11.837% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mdp| 4.374% | +0.055% | +10.769% | +8.965% | +-----+------------------------+--------+------------+------------+------------+ | :-| | meteor_contest| 3.534% | +0.067% | +5.112% | +4.621% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nbody| 0.636% | +0.065% | -0.656% | -3.768% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nqueens| 1.900% | +0.136% | +4.355% | +7.598% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pathlib| 1.081% | -0.398% | +2.789% | +11.552% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle| 4.335% | -0.520% | +0.348% | +21.615% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_dict| 0.612% | -0.228% | -0.621% | +23.883% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_list| 0.821% | +0.019% | +2.598% | +21.242% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_pure_python| 3.639% | -0.561% | +10.241% | +10.595% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pidigits| 0.220% | -0.051% | +0.033% | +9.805% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup| 0.155% | -0.068% | +10.470% | +5.402% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup_no_site| 0.114% | +0.020% | +1.973% | +5.281% | +-----+------------------------+--------+------------+------------+------------+ | :-| | raytrace| 1.105% | +0.219% | +9.827% | +11.455% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_compile| 4.363% | +0.793% | +6.001% | +9.442% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_dna| 0.309% | +0.113% | -2.268% | +11.512% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_effbot| 2.999% | +0.056% | -6.304% | +7.167% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_v8| 1.479% | +0.110% | +4.443% | +9.675% | +-----+------------------------+--------+------------+------------+------------+ | :-| | richards| 1.680% | -0.477% | +7.892% | +14.757% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_fft| 0.804% | -0.185% | -2.485% | +3.068% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_lu| 1.714% | -0.215% | +26.984% | +7.575% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_monte_carlo| 1.887% | +0.508% | +4.502% | +2.725% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sor| 0.773% | +0.306% | +14.285% | +6.156% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sparse_mat_mult| 0.876% | -0.211% | -3.643% | -4.259% | +-----+------------------------+--------+------------+------------+------------+ | :-| | spectral_norm| 0.498% | -0.115% | +5.438% | +2.880% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_declarative| 1.236% | -0.059% | +6.540% | +5.889% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_imperative| 3.639% | -0.970% | +5.860% | +3.119% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlite_synth| 4.822% | +2.196% | +20.130% | +8.135% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_expand| 2.070% | -0.147% | +12.036% | +6.033% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_integrate| 2.065% | +0.172% | +9.077% | +6.338% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_str| 4.162% | -0.674% | +11.814% | +9.108% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_sum| 6.240% | +0.335% | +10.708% | +9.106% | +-----+------------------------+--------+------------+------------+------------+ | :-| | telco| 5.917% | +0.383% | +19.169% | +12.553% | +-----+------------------------+--------+------------+------------+------------+ | :-| | tornado_http| 1.010% | +0.086% | +6.121% | +6.395% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpack_sequence| 1.773% | +0.167% | +1.705% | +0.426% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle| 5.614% | +1.505% | +9.395% | +20.774% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_list| 5.356% | -0.049% | -6.843% | +18.757% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_pure_python| 1.450% | -0.255% | +7.954% | +6.904% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_generate| 1.302% | -0.390% | +1.857% | +12.728% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_iterparse| 2.484% | -0.391% | +2.550% | +8.039% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_parse| 3.415% | -0.326% | -10.958% | +15.123% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_process| 1.363% | -0.053% | +4.277% | +12.013% | +-----+------------------------+--------+------------+------------+------------+ * Relative Standard Deviation (Standard Deviation/Average) If this is not displayed properly please visit our results page here: http://languagesperformance.intel.com/65-flat-results-for-python-master-bra… Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0

return the new file descriptor from os.dup2 (closes bpo-32441) (#5041)
by Benjamin Peterson 29 Dec '17

29 Dec '17

https://github.com/python/cpython/commit/bbdb17d19bb1d5443ca4417254e014ad64… commit: bbdb17d19bb1d5443ca4417254e014ad64c04540 branch: master author: Benjamin Peterson <benjamin(a)python.org> committer: GitHub <noreply(a)github.com> date: 2017-12-29T13:13:06-08:00 summary: return the new file descriptor from os.dup2 (closes bpo-32441) (#5041) files: A Misc/NEWS.d/next/Library/2017-12-28-21-30-40.bpo-32441.LqlboJ.rst M Doc/library/os.rst M Lib/test/test_os.py M Modules/clinic/posixmodule.c.h M Modules/posixmodule.c diff --git a/Doc/library/os.rst b/Doc/library/os.rst index a24934ceb7b..ee08853087f 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -735,13 +735,17 @@ as internal buffering of data. .. function:: dup2(fd, fd2, inheritable=True) - Duplicate file descriptor *fd* to *fd2*, closing the latter first if necessary. - The file descriptor *fd2* is :ref:`inheritable <fd_inheritance>` by default, - or non-inheritable if *inheritable* is ``False``. + Duplicate file descriptor *fd* to *fd2*, closing the latter first if + necessary. Return *fd2*. The new file descriptor is :ref:`inheritable + <fd_inheritance>` by default or non-inheritable if *inheritable* + is ``False``. .. versionchanged:: 3.4 Add the optional *inheritable* parameter. + .. versionchanged:: 3.7 + Return *fd2* on success. Previously, ``None`` was always returned. + .. function:: fchmod(fd, mode) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index f235f801e03..83e214d91ea 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3079,19 +3079,15 @@ def test_dup2(self): # inheritable by default fd2 = os.open(__file__, os.O_RDONLY) - try: - os.dup2(fd, fd2) - self.assertEqual(os.get_inheritable(fd2), True) - finally: - os.close(fd2) + self.addCleanup(os.close, fd2) + self.assertEqual(os.dup2(fd, fd2), fd2) + self.assertTrue(os.get_inheritable(fd2)) # force non-inheritable fd3 = os.open(__file__, os.O_RDONLY) - try: - os.dup2(fd, fd3, inheritable=False) - self.assertEqual(os.get_inheritable(fd3), False) - finally: - os.close(fd3) + self.addCleanup(os.close, fd3) + self.assertEqual(os.dup2(fd, fd3, inheritable=False), fd3) + self.assertFalse(os.get_inheritable(fd3)) @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") def test_openpty(self): diff --git a/Misc/NEWS.d/next/Library/2017-12-28-21-30-40.bpo-32441.LqlboJ.rst b/Misc/NEWS.d/next/Library/2017-12-28-21-30-40.bpo-32441.LqlboJ.rst new file mode 100644 index 00000000000..a0fe4f3cc6c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-28-21-30-40.bpo-32441.LqlboJ.rst @@ -0,0 +1,2 @@ +Return the new file descriptor (i.e., the second argument) from ``os.dup2``. +Previously, ``None`` was always returned. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 9decf7a8d24..6f4c028e7ef 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -3486,7 +3486,7 @@ PyDoc_STRVAR(os_dup2__doc__, #define OS_DUP2_METHODDEF \ {"dup2", (PyCFunction)os_dup2, METH_FASTCALL|METH_KEYWORDS, os_dup2__doc__}, -static PyObject * +static int os_dup2_impl(PyObject *module, int fd, int fd2, int inheritable); static PyObject * @@ -3498,12 +3498,17 @@ os_dup2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn int fd; int fd2; int inheritable = 1; + int _return_value; if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, &fd, &fd2, &inheritable)) { goto exit; } - return_value = os_dup2_impl(module, fd, fd2, inheritable); + _return_value = os_dup2_impl(module, fd, fd2, inheritable); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); exit: return return_value; @@ -6405,4 +6410,4 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #ifndef OS_GETRANDOM_METHODDEF #define OS_GETRANDOM_METHODDEF #endif /* !defined(OS_GETRANDOM_METHODDEF) */ -/*[clinic end generated code: output=b6ade5f170d5a431 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6345053cd5992caf input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 38b6c80e6bd..47b79fcc798 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7770,7 +7770,7 @@ os_dup_impl(PyObject *module, int fd) /*[clinic input] -os.dup2 +os.dup2 -> int fd: int fd2: int inheritable: bool=True @@ -7778,9 +7778,9 @@ os.dup2 Duplicate file descriptor. [clinic start generated code]*/ -static PyObject * +static int os_dup2_impl(PyObject *module, int fd, int fd2, int inheritable) -/*[clinic end generated code: output=db832a2d872ccc5f input=76e96f511be0352f]*/ +/*[clinic end generated code: output=bc059d34a73404d1 input=c3cddda8922b038d]*/ { int res; #if defined(HAVE_DUP3) && \ @@ -7789,8 +7789,10 @@ os_dup2_impl(PyObject *module, int fd, int fd2, int inheritable) int dup3_works = -1; #endif - if (fd < 0 || fd2 < 0) - return posix_error(); + if (fd < 0 || fd2 < 0) { + posix_error(); + return -1; + } /* dup2() can fail with EINTR if the target FD is already open, because it * then has to be closed. See os_close_impl() for why we don't handle EINTR @@ -7802,13 +7804,16 @@ os_dup2_impl(PyObject *module, int fd, int fd2, int inheritable) res = dup2(fd, fd2); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS - if (res < 0) - return posix_error(); + if (res < 0) { + posix_error(); + return -1; + } + res = fd2; // msvcrt dup2 returns 0 on success. /* Character files like console cannot be make non-inheritable */ if (!inheritable && _Py_set_inheritable(fd2, 0, NULL) < 0) { close(fd2); - return NULL; + return -1; } #elif defined(HAVE_FCNTL_H) && defined(F_DUP2FD_CLOEXEC) @@ -7818,8 +7823,10 @@ os_dup2_impl(PyObject *module, int fd, int fd2, int inheritable) else res = dup2(fd, fd2); Py_END_ALLOW_THREADS - if (res < 0) - return posix_error(); + if (res < 0) { + posix_error(); + return -1; + } #else @@ -7831,8 +7838,10 @@ os_dup2_impl(PyObject *module, int fd, int fd2, int inheritable) if (res < 0) { if (dup3_works == -1) dup3_works = (errno != ENOSYS); - if (dup3_works) - return posix_error(); + if (dup3_works) { + posix_error(); + return -1; + } } } @@ -7842,12 +7851,14 @@ os_dup2_impl(PyObject *module, int fd, int fd2, int inheritable) Py_BEGIN_ALLOW_THREADS res = dup2(fd, fd2); Py_END_ALLOW_THREADS - if (res < 0) - return posix_error(); + if (res < 0) { + posix_error(); + return -1; + } if (!inheritable && _Py_set_inheritable(fd2, 0, NULL) < 0) { close(fd2); - return NULL; + return -1; } #ifdef HAVE_DUP3 } @@ -7855,7 +7866,7 @@ os_dup2_impl(PyObject *module, int fd, int fd2, int inheritable) #endif - Py_RETURN_NONE; + return res; }

1 0

bpo-18035: telnetlib: select.error doesn't have an errno attribute (#5044)
by Gregory P. Smith 29 Dec '17

29 Dec '17

https://github.com/python/cpython/commit/3ceaed0dce81fd881bbaf2dbdbe827d968… commit: 3ceaed0dce81fd881bbaf2dbdbe827d9681887da branch: 2.7 author: Segev Finer <segev208(a)gmail.com> committer: Gregory P. Smith <greg(a)krypto.org> date: 2017-12-29T12:44:04-08:00 summary: bpo-18035: telnetlib: select.error doesn't have an errno attribute (#5044) select.error doesn't have an errno attribute so access the errno by indexing instead. files: A Misc/NEWS.d/next/Library/2017-12-29-15-16-56.bpo-18035.c6rdCt.rst M Lib/telnetlib.py diff --git a/Lib/telnetlib.py b/Lib/telnetlib.py index 2eaa8e37098..d0246c0aea7 100644 --- a/Lib/telnetlib.py +++ b/Lib/telnetlib.py @@ -317,7 +317,7 @@ def _read_until_with_poll(self, match, timeout): ready = poller.poll(None if timeout is None else 1000 * call_timeout) except select.error as e: - if e.errno == errno.EINTR: + if e[0] == errno.EINTR: if timeout is not None: elapsed = time() - time_start call_timeout = timeout-elapsed @@ -688,7 +688,7 @@ def _expect_with_poll(self, expect_list, timeout=None): ready = poller.poll(None if timeout is None else 1000 * call_timeout) except select.error as e: - if e.errno == errno.EINTR: + if e[0] == errno.EINTR: if timeout is not None: elapsed = time() - time_start call_timeout = timeout-elapsed diff --git a/Misc/NEWS.d/next/Library/2017-12-29-15-16-56.bpo-18035.c6rdCt.rst b/Misc/NEWS.d/next/Library/2017-12-29-15-16-56.bpo-18035.c6rdCt.rst new file mode 100644 index 00000000000..7b6e0dc6b28 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-29-15-16-56.bpo-18035.c6rdCt.rst @@ -0,0 +1,2 @@ +``telnetlib``: ``select.error`` doesn't have an ``errno`` attribute. Patch +by Segev Finer.

1 0

bpo-32427: Expose dataclasses.MISSING object. (#5045)
by Eric V. Smith 29 Dec '17

29 Dec '17

https://github.com/python/cpython/commit/03220fdb26c0b6a50ce5ed1fdfbf232094… commit: 03220fdb26c0b6a50ce5ed1fdfbf232094b66db6 branch: master author: Eric V. Smith <ericvsmith(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2017-12-29T13:59:58-05:00 summary: bpo-32427: Expose dataclasses.MISSING object. (#5045) files: M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 7a725dfb520..eaaed63ef28 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -8,6 +8,7 @@ 'field', 'FrozenInstanceError', 'InitVar', + 'MISSING', # Helper functions. 'fields', @@ -29,11 +30,11 @@ def __repr__(self): return '<factory>' _HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() -# A sentinel object to detect if a parameter is supplied or not. -class _MISSING_FACTORY: - def __repr__(self): - return '<missing>' -_MISSING = _MISSING_FACTORY() +# A sentinel object to detect if a parameter is supplied or not. Use +# a class to give it a better repr. +class _MISSING_TYPE: + pass +MISSING = _MISSING_TYPE() # Since most per-field metadata will be unused, create an empty # read-only proxy that can be shared among all fields. @@ -114,7 +115,7 @@ def __repr__(self): # This function is used instead of exposing Field creation directly, # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. -def field(*, default=_MISSING, default_factory=_MISSING, init=True, repr=True, +def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None): """Return an object to identify dataclass fields. @@ -130,7 +131,7 @@ def field(*, default=_MISSING, default_factory=_MISSING, init=True, repr=True, It is an error to specify both default and default_factory. """ - if default is not _MISSING and default_factory is not _MISSING: + if default is not MISSING and default_factory is not MISSING: raise ValueError('cannot specify both default and default_factory') return Field(default, default_factory, init, repr, hash, compare, metadata) @@ -149,12 +150,12 @@ def _tuple_str(obj_name, fields): def _create_fn(name, args, body, globals=None, locals=None, - return_type=_MISSING): + return_type=MISSING): # Note that we mutate locals when exec() is called. Caller beware! if locals is None: locals = {} return_annotation = '' - if return_type is not _MISSING: + if return_type is not MISSING: locals['_return_type'] = return_type return_annotation = '->_return_type' args = ','.join(args) @@ -182,7 +183,7 @@ def _field_init(f, frozen, globals, self_name): # initialize this field. default_name = f'_dflt_{f.name}' - if f.default_factory is not _MISSING: + if f.default_factory is not MISSING: if f.init: # This field has a default factory. If a parameter is # given, use it. If not, call the factory. @@ -210,10 +211,10 @@ def _field_init(f, frozen, globals, self_name): else: # No default factory. if f.init: - if f.default is _MISSING: + if f.default is MISSING: # There's no default, just do an assignment. value = f.name - elif f.default is not _MISSING: + elif f.default is not MISSING: globals[default_name] = f.default value = f.name else: @@ -236,14 +237,14 @@ def _init_param(f): # For example, the equivalent of 'x:int=3' (except instead of 'int', # reference a variable set to int, and instead of '3', reference a # variable set to 3). - if f.default is _MISSING and f.default_factory is _MISSING: + if f.default is MISSING and f.default_factory is MISSING: # There's no default, and no default_factory, just # output the variable name and type. default = '' - elif f.default is not _MISSING: + elif f.default is not MISSING: # There's a default, this will be the name that's used to look it up. default = f'=_dflt_{f.name}' - elif f.default_factory is not _MISSING: + elif f.default_factory is not MISSING: # There's a factory function. Set a marker. default = '=_HAS_DEFAULT_FACTORY' return f'{f.name}:_type_{f.name}{default}' @@ -261,13 +262,13 @@ def _init_fn(fields, frozen, has_post_init, self_name): for f in fields: # Only consider fields in the __init__ call. if f.init: - if not (f.default is _MISSING and f.default_factory is _MISSING): + if not (f.default is MISSING and f.default_factory is MISSING): seen_default = True elif seen_default: raise TypeError(f'non-default argument {f.name!r} ' 'follows default argument') - globals = {'_MISSING': _MISSING, + globals = {'MISSING': MISSING, '_HAS_DEFAULT_FACTORY': _HAS_DEFAULT_FACTORY} body_lines = [] @@ -368,7 +369,7 @@ def _get_field(cls, a_name, a_type): # If the default value isn't derived from field, then it's # only a normal default value. Convert it to a Field(). - default = getattr(cls, a_name, _MISSING) + default = getattr(cls, a_name, MISSING) if isinstance(default, Field): f = default else: @@ -404,7 +405,7 @@ def _get_field(cls, a_name, a_type): # Special restrictions for ClassVar and InitVar. if f._field_type in (_FIELD_CLASSVAR, _FIELD_INITVAR): - if f.default_factory is not _MISSING: + if f.default_factory is not MISSING: raise TypeError(f'field {f.name} cannot have a ' 'default factory') # Should I check for other field settings? default_factory @@ -474,7 +475,7 @@ def _process_class(cls, repr, eq, order, hash, init, frozen): # with the real default. This is so that normal class # introspection sees a real default value, not a Field. if isinstance(getattr(cls, f.name, None), Field): - if f.default is _MISSING: + if f.default is MISSING: # If there's no default, delete the class attribute. # This happens if we specify field(repr=False), for # example (that is, we specified a field object, but diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 7fbea76ccd8..ed695639882 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -1,6 +1,6 @@ from dataclasses import ( dataclass, field, FrozenInstanceError, fields, asdict, astuple, - make_dataclass, replace, InitVar, Field + make_dataclass, replace, InitVar, Field, MISSING ) import pickle @@ -917,12 +917,12 @@ def validate_class(cls): param = next(params) self.assertEqual(param.name, 'k') self.assertIs (param.annotation, F) - # Don't test for the default, since it's set to _MISSING + # Don't test for the default, since it's set to MISSING self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_OR_KEYWORD) param = next(params) self.assertEqual(param.name, 'l') self.assertIs (param.annotation, float) - # Don't test for the default, since it's set to _MISSING + # Don't test for the default, since it's set to MISSING self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_OR_KEYWORD) self.assertRaises(StopIteration, next, params) @@ -948,6 +948,52 @@ class C: validate_class(C) + def test_missing_default(self): + # Test that MISSING works the same as a default not being + # specified. + @dataclass + class C: + x: int=field(default=MISSING) + with self.assertRaisesRegex(TypeError, + r'__init__\(\) missing 1 required ' + 'positional argument'): + C() + self.assertNotIn('x', C.__dict__) + + @dataclass + class D: + x: int + with self.assertRaisesRegex(TypeError, + r'__init__\(\) missing 1 required ' + 'positional argument'): + D() + self.assertNotIn('x', D.__dict__) + + def test_missing_default_factory(self): + # Test that MISSING works the same as a default factory not + # being specified (which is really the same as a default not + # being specified, too). + @dataclass + class C: + x: int=field(default_factory=MISSING) + with self.assertRaisesRegex(TypeError, + r'__init__\(\) missing 1 required ' + 'positional argument'): + C() + self.assertNotIn('x', C.__dict__) + + @dataclass + class D: + x: int=field(default=MISSING, default_factory=MISSING) + with self.assertRaisesRegex(TypeError, + r'__init__\(\) missing 1 required ' + 'positional argument'): + D() + self.assertNotIn('x', D.__dict__) + + def test_missing_repr(self): + self.assertIn('MISSING_TYPE object', repr(MISSING)) + def test_dont_include_other_annotations(self): @dataclass class C:

1 0

Daily reference leaks (4243df51fe43): sum=12
by solipsis＠pitrou.net 29 Dec '17

29 Dec '17

results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 7, 0] memory blocks, sum=7 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_spawn leaked [1, -2, 2] memory blocks, sum=1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogL0NAO5', '--timeout', '7200']

1 0