[Python-checkins] cpython (merge 3.5 -> 3.6): Merge 3.5 (issue #27906)
yury.selivanov
python-checkins at python.org
Thu Sep 15 14:15:37 EDT 2016
https://hg.python.org/cpython/rev/62948164ff94
changeset: 103831:62948164ff94
branch: 3.6
parent: 103828:b368d78432af
parent: 103830:1dcfafed3cb0
user: Yury Selivanov <yury at magic.io>
date: Thu Sep 15 14:14:48 2016 -0400
summary:
Merge 3.5 (issue #27906)
files:
Lib/asyncio/base_events.py | 2 +-
Lib/asyncio/proactor_events.py | 2 +-
Lib/asyncio/selector_events.py | 73 +++++----
Lib/test/test_asyncio/test_base_events.py | 2 +-
Lib/test/test_asyncio/test_selector_events.py | 14 +
Misc/NEWS | 3 +
6 files changed, 60 insertions(+), 36 deletions(-)
diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py
--- a/Lib/asyncio/base_events.py
+++ b/Lib/asyncio/base_events.py
@@ -1035,7 +1035,7 @@
for sock in sockets:
sock.listen(backlog)
sock.setblocking(False)
- self._start_serving(protocol_factory, sock, ssl, server)
+ self._start_serving(protocol_factory, sock, ssl, server, backlog)
if self._debug:
logger.info("%r is serving", server)
return server
diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py
--- a/Lib/asyncio/proactor_events.py
+++ b/Lib/asyncio/proactor_events.py
@@ -495,7 +495,7 @@
self._csock.send(b'\0')
def _start_serving(self, protocol_factory, sock,
- sslcontext=None, server=None):
+ sslcontext=None, server=None, backlog=100):
def loop(f=None):
try:
diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py
--- a/Lib/asyncio/selector_events.py
+++ b/Lib/asyncio/selector_events.py
@@ -162,43 +162,50 @@
exc_info=True)
def _start_serving(self, protocol_factory, sock,
- sslcontext=None, server=None):
+ sslcontext=None, server=None, backlog=100):
self.add_reader(sock.fileno(), self._accept_connection,
- protocol_factory, sock, sslcontext, server)
+ protocol_factory, sock, sslcontext, server, backlog)
def _accept_connection(self, protocol_factory, sock,
- sslcontext=None, server=None):
- try:
- conn, addr = sock.accept()
- if self._debug:
- logger.debug("%r got a new connection from %r: %r",
- server, addr, conn)
- conn.setblocking(False)
- except (BlockingIOError, InterruptedError, ConnectionAbortedError):
- pass # False alarm.
- except OSError as exc:
- # There's nowhere to send the error, so just log it.
- if exc.errno in (errno.EMFILE, errno.ENFILE,
- errno.ENOBUFS, errno.ENOMEM):
- # Some platforms (e.g. Linux keep reporting the FD as
- # ready, so we remove the read handler temporarily.
- # We'll try again in a while.
- self.call_exception_handler({
- 'message': 'socket.accept() out of system resource',
- 'exception': exc,
- 'socket': sock,
- })
- self.remove_reader(sock.fileno())
- self.call_later(constants.ACCEPT_RETRY_DELAY,
- self._start_serving,
- protocol_factory, sock, sslcontext, server)
+ sslcontext=None, server=None, backlog=100):
+ # This method is only called once for each event loop tick where the
+ # listening socket has triggered an EVENT_READ. There may be multiple
+ # connections waiting for an .accept() so it is called in a loop.
+ # See https://bugs.python.org/issue27906 for more details.
+ for _ in range(backlog):
+ try:
+ conn, addr = sock.accept()
+ if self._debug:
+ logger.debug("%r got a new connection from %r: %r",
+ server, addr, conn)
+ conn.setblocking(False)
+ except (BlockingIOError, InterruptedError, ConnectionAbortedError):
+ # Early exit because the socket accept buffer is empty.
+ return None
+ except OSError as exc:
+ # There's nowhere to send the error, so just log it.
+ if exc.errno in (errno.EMFILE, errno.ENFILE,
+ errno.ENOBUFS, errno.ENOMEM):
+ # Some platforms (e.g. Linux keep reporting the FD as
+ # ready, so we remove the read handler temporarily.
+ # We'll try again in a while.
+ self.call_exception_handler({
+ 'message': 'socket.accept() out of system resource',
+ 'exception': exc,
+ 'socket': sock,
+ })
+ self.remove_reader(sock.fileno())
+ self.call_later(constants.ACCEPT_RETRY_DELAY,
+ self._start_serving,
+ protocol_factory, sock, sslcontext, server,
+ backlog)
+ else:
+ raise # The event loop will catch, log and ignore it.
else:
- raise # The event loop will catch, log and ignore it.
- else:
- extra = {'peername': addr}
- accept = self._accept_connection2(protocol_factory, conn, extra,
- sslcontext, server)
- self.create_task(accept)
+ extra = {'peername': addr}
+ accept = self._accept_connection2(protocol_factory, conn, extra,
+ sslcontext, server)
+ self.create_task(accept)
@coroutine
def _accept_connection2(self, protocol_factory, conn, extra,
diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -1634,7 +1634,7 @@
self.loop.call_later.assert_called_with(constants.ACCEPT_RETRY_DELAY,
# self.loop._start_serving
mock.ANY,
- MyProto, sock, None, None)
+ MyProto, sock, None, None, mock.ANY)
def test_call_coroutine(self):
@asyncio.coroutine
diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py
--- a/Lib/test/test_asyncio/test_selector_events.py
+++ b/Lib/test/test_asyncio/test_selector_events.py
@@ -687,6 +687,20 @@
selectors.EVENT_WRITE)])
self.loop.remove_writer.assert_called_with(1)
+ def test_accept_connection_multiple(self):
+ sock = mock.Mock()
+ sock.accept.return_value = (mock.Mock(), mock.Mock())
+ backlog = 100
+ # Mock the coroutine generation for a connection to prevent
+ # warnings related to un-awaited coroutines.
+ mock_obj = mock.patch.object
+ with mock_obj(self.loop, '_accept_connection2') as accept2_mock:
+ accept2_mock.return_value = None
+ with mock_obj(self.loop, 'create_task') as task_mock:
+ task_mock.return_value = None
+ self.loop._accept_connection(mock.Mock(), sock, backlog=backlog)
+ self.assertEqual(sock.accept.call_count, backlog)
+
class SelectorTransportTests(test_utils.TestCase):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -418,6 +418,9 @@
- Issue #27456: asyncio: Set TCP_NODELAY by default.
+- Issue #27906: Fix socket accept exhaustion during high TCP traffic.
+ Patch by Kevin Conway.
+
IDLE
----
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list