Mailman 3 January 2014 - Python-checkins

cpython: asyncio: Refactor drain logic in streams.py to be reusable.
by guido.van.rossum 29 Jan '14

29 Jan '14

http://hg.python.org/cpython/rev/472376125c8f changeset: 88823:472376125c8f user: Guido van Rossum <guido(a)python.org> date: Wed Jan 29 14:24:45 2014 -0800 summary: asyncio: Refactor drain logic in streams.py to be reusable. files: Lib/asyncio/streams.py | 97 ++++++++++++++++++----------- 1 files changed, 61 insertions(+), 36 deletions(-) diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -94,8 +94,63 @@ return (yield from loop.create_server(factory, host, port, **kwds)) -class StreamReaderProtocol(protocols.Protocol): - """Trivial helper class to adapt between Protocol and StreamReader. +class FlowControlMixin(protocols.Protocol): + """Reusable flow control logic for StreamWriter.drain(). + + This implements the protocol methods pause_writing(), + resume_reading() and connection_lost(). If the subclass overrides + these it must call the super methods. + + StreamWriter.drain() must check for error conditions and then call + _make_drain_waiter(), which will return either () or a Future + depending on the paused state. + """ + + def __init__(self, loop=None): + self._loop = loop # May be None; we may never need it. + self._paused = False + self._drain_waiter = None + + def pause_writing(self): + assert not self._paused + self._paused = True + + def resume_writing(self): + assert self._paused + self._paused = False + waiter = self._drain_waiter + if waiter is not None: + self._drain_waiter = None + if not waiter.done(): + waiter.set_result(None) + + def connection_lost(self, exc): + # Wake up the writer if currently paused. + if not self._paused: + return + waiter = self._drain_waiter + if waiter is None: + return + self._drain_waiter = None + if waiter.done(): + return + if exc is None: + waiter.set_result(None) + else: + waiter.set_exception(exc) + + def _make_drain_waiter(self): + if not self._paused: + return () + waiter = self._drain_waiter + assert waiter is None or waiter.cancelled() + waiter = futures.Future(loop=self._loop) + self._drain_waiter = waiter + return waiter + + +class StreamReaderProtocol(FlowControlMixin, protocols.Protocol): + """Helper class to adapt between Protocol and StreamReader. (This is a helper class instead of making StreamReader itself a Protocol subclass, because the StreamReader has other potential @@ -104,12 +159,10 @@ """ def __init__(self, stream_reader, client_connected_cb=None, loop=None): + super().__init__(loop=loop) self._stream_reader = stream_reader self._stream_writer = None - self._drain_waiter = None - self._paused = False self._client_connected_cb = client_connected_cb - self._loop = loop # May be None; we may never need it. def connection_made(self, transport): self._stream_reader.set_transport(transport) @@ -127,16 +180,7 @@ self._stream_reader.feed_eof() else: self._stream_reader.set_exception(exc) - # Also wake up the writing side. - if self._paused: - waiter = self._drain_waiter - if waiter is not None: - self._drain_waiter = None - if not waiter.done(): - if exc is None: - waiter.set_result(None) - else: - waiter.set_exception(exc) + super().connection_lost(exc) def data_received(self, data): self._stream_reader.feed_data(data) @@ -144,19 +188,6 @@ def eof_received(self): self._stream_reader.feed_eof() - def pause_writing(self): - assert not self._paused - self._paused = True - - def resume_writing(self): - assert self._paused - self._paused = False - waiter = self._drain_waiter - if waiter is not None: - self._drain_waiter = None - if not waiter.done(): - waiter.set_result(None) - class StreamWriter: """Wraps a Transport. @@ -211,17 +242,11 @@ completed, which will happen when the buffer is (partially) drained and the protocol is resumed. """ - if self._reader._exception is not None: + if self._reader is not None and self._reader._exception is not None: raise self._reader._exception if self._transport._conn_lost: # Uses private variable. raise ConnectionResetError('Connection lost') - if not self._protocol._paused: - return () - waiter = self._protocol._drain_waiter - assert waiter is None or waiter.cancelled() - waiter = futures.Future(loop=self._loop) - self._protocol._drain_waiter = waiter - return waiter + return self._protocol._make_drain_waiter() class StreamReader: -- Repository URL: http://hg.python.org/cpython

1 0

cpython: asyncio: Get rid of _try_connected().
by guido.van.rossum 29 Jan '14

29 Jan '14

http://hg.python.org/cpython/rev/3fe80ecba874 changeset: 88822:3fe80ecba874 user: Victor Stinner <victor.stinner(a)gmail.com> date: Wed Jan 29 14:22:56 2014 -0800 summary: asyncio: Get rid of _try_connected(). files: Lib/asyncio/base_subprocess.py | 32 +++++++++------------ 1 files changed, 14 insertions(+), 18 deletions(-) diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -75,19 +75,27 @@ proc = self._proc loop = self._loop if proc.stdin is not None: - transp, proto = yield from loop.connect_write_pipe( + _, pipe = yield from loop.connect_write_pipe( lambda: WriteSubprocessPipeProto(self, STDIN), proc.stdin) + self._pipes[STDIN] = pipe if proc.stdout is not None: - transp, proto = yield from loop.connect_read_pipe( + _, pipe = yield from loop.connect_read_pipe( lambda: ReadSubprocessPipeProto(self, STDOUT), proc.stdout) + self._pipes[STDOUT] = pipe if proc.stderr is not None: - transp, proto = yield from loop.connect_read_pipe( + _, pipe = yield from loop.connect_read_pipe( lambda: ReadSubprocessPipeProto(self, STDERR), proc.stderr) - if not self._pipes: - self._try_connected() + self._pipes[STDERR] = pipe + + assert self._pending_calls is not None + + self._loop.call_soon(self._protocol.connection_made, self) + for callback, data in self._pending_calls: + self._loop.call_soon(callback, *data) + self._pending_calls = None def _call(self, cb, *data): if self._pending_calls is not None: @@ -95,14 +103,6 @@ else: self._loop.call_soon(cb, *data) - def _try_connected(self): - assert self._pending_calls is not None - if all(p is not None and p.connected for p in self._pipes.values()): - self._loop.call_soon(self._protocol.connection_made, self) - for callback, data in self._pending_calls: - self._loop.call_soon(callback, *data) - self._pending_calls = None - def _pipe_connection_lost(self, fd, exc): self._call(self._protocol.pipe_connection_lost, fd, exc) self._try_finish() @@ -136,19 +136,15 @@ class WriteSubprocessPipeProto(protocols.BaseProtocol): - pipe = None def __init__(self, proc, fd): self.proc = proc self.fd = fd - self.connected = False + self.pipe = None self.disconnected = False - proc._pipes[fd] = self def connection_made(self, transport): - self.connected = True self.pipe = transport - self.proc._try_connected() def connection_lost(self, exc): self.disconnected = True -- Repository URL: http://hg.python.org/cpython

1 0

cpython: asyncio: Add write flow control to unix pipes.
by guido.van.rossum 29 Jan '14

29 Jan '14

http://hg.python.org/cpython/rev/c71fa690254b changeset: 88821:c71fa690254b user: Guido van Rossum <guido(a)python.org> date: Wed Jan 29 13:20:39 2014 -0800 summary: asyncio: Add write flow control to unix pipes. files: Lib/asyncio/unix_events.py | 14 +++++++++++--- 1 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -246,7 +246,8 @@ self._loop = None -class _UnixWritePipeTransport(transports.WriteTransport): +class _UnixWritePipeTransport(selector_events._FlowControlMixin, + transports.WriteTransport): def __init__(self, loop, pipe, protocol, waiter=None, extra=None): super().__init__(extra) @@ -277,12 +278,17 @@ if waiter is not None: self._loop.call_soon(waiter.set_result, None) + def get_write_buffer_size(self): + return sum(len(data) for data in self._buffer) + def _read_ready(self): # Pipe was closed by peer. self._close() def write(self, data): - assert isinstance(data, bytes), repr(data) + assert isinstance(data, (bytes, bytearray, memoryview)), repr(data) + if isinstance(data, bytearray): + data = memoryview(data) if not data: return @@ -310,6 +316,7 @@ self._loop.add_writer(self._fileno, self._write_ready) self._buffer.append(data) + self._maybe_pause_protocol() def _write_ready(self): data = b''.join(self._buffer) @@ -329,7 +336,8 @@ else: if n == len(data): self._loop.remove_writer(self._fileno) - if self._closing: + self._maybe_resume_protocol() # May append to buffer. + if not self._buffer and self._closing: self._loop.remove_reader(self._fileno) self._call_connection_lost(None) return -- Repository URL: http://hg.python.org/cpython

1 0

cpython: asyncio: Refactoring: move write flow control to a subclass/mixin.
by guido.van.rossum 29 Jan '14

29 Jan '14

http://hg.python.org/cpython/rev/9f0cf41cf593 changeset: 88820:9f0cf41cf593 user: Guido van Rossum <guido(a)python.org> date: Wed Jan 29 13:15:59 2014 -0800 summary: asyncio: Refactoring: move write flow control to a subclass/mixin. files: Lib/asyncio/selector_events.py | 98 +++++++++++++-------- 1 files changed, 61 insertions(+), 37 deletions(-) diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -339,7 +339,67 @@ sock.close() -class _SelectorTransport(transports.Transport): +class _FlowControlMixin(transports.Transport): + """All the logic for (write) flow control in a mix-in base class. + + The subclass must implement get_write_buffer_size(). It must call + _maybe_pause_protocol() whenever the write buffer size increases, + and _maybe_resume_protocol() whenever it decreases. It may also + override set_write_buffer_limits() (e.g. to specify different + defaults). + + The subclass constructor must call super().__init__(extra). This + will call set_write_buffer_limits(). + + The user may call set_write_buffer_limits() and + get_write_buffer_size(), and their protocol's pause_writing() and + resume_writing() may be called. + """ + + def __init__(self, extra=None): + super().__init__(extra) + self._protocol_paused = False + self.set_write_buffer_limits() + + def _maybe_pause_protocol(self): + size = self.get_write_buffer_size() + if size <= self._high_water: + return + if not self._protocol_paused: + self._protocol_paused = True + try: + self._protocol.pause_writing() + except Exception: + logger.exception('pause_writing() failed') + + def _maybe_resume_protocol(self): + if (self._protocol_paused and + self.get_write_buffer_size() <= self._low_water): + self._protocol_paused = False + try: + self._protocol.resume_writing() + except Exception: + logger.exception('resume_writing() failed') + + def set_write_buffer_limits(self, high=None, low=None): + if high is None: + if low is None: + high = 64*1024 + else: + high = 4*low + if low is None: + low = high // 4 + if not high >= low >= 0: + raise ValueError('high (%r) must be >= low (%r) must be >= 0' % + (high, low)) + self._high_water = high + self._low_water = low + + def get_write_buffer_size(self): + raise NotImplementedError + + +class _SelectorTransport(_FlowControlMixin, transports.Transport): max_size = 256 * 1024 # Buffer size passed to recv(). @@ -362,8 +422,6 @@ self._buffer = self._buffer_factory() self._conn_lost = 0 # Set when call to connection_lost scheduled. self._closing = False # Set when close() called. - self._protocol_paused = False - self.set_write_buffer_limits() if self._server is not None: self._server.attach(self) @@ -410,40 +468,6 @@ server.detach(self) self._server = None - def _maybe_pause_protocol(self): - size = self.get_write_buffer_size() - if size <= self._high_water: - return - if not self._protocol_paused: - self._protocol_paused = True - try: - self._protocol.pause_writing() - except Exception: - logger.exception('pause_writing() failed') - - def _maybe_resume_protocol(self): - if (self._protocol_paused and - self.get_write_buffer_size() <= self._low_water): - self._protocol_paused = False - try: - self._protocol.resume_writing() - except Exception: - logger.exception('resume_writing() failed') - - def set_write_buffer_limits(self, high=None, low=None): - if high is None: - if low is None: - high = 64*1024 - else: - high = 4*low - if low is None: - low = high // 4 - if not high >= low >= 0: - raise ValueError('high (%r) must be >= low (%r) must be >= 0' % - (high, low)) - self._high_water = high - self._low_water = low - def get_write_buffer_size(self): return len(self._buffer) -- Repository URL: http://hg.python.org/cpython

1 0

cpython: asyncio: _fatal_error() of _UnixWritePipeTransport and
by guido.van.rossum 29 Jan '14

29 Jan '14

http://hg.python.org/cpython/rev/f21a0ad40734 changeset: 88819:f21a0ad40734 user: Victor Stinner <victor.stinner(a)gmail.com> date: Wed Jan 29 13:12:03 2014 -0800 summary: asyncio: _fatal_error() of _UnixWritePipeTransport and _ProactorBasePipeTransport shouldn't log BrokenPipeError nor ConnectionResetError. (Same behaviour as _SelectorTransport._fatal_error().) files: Lib/asyncio/proactor_events.py | 3 ++- Lib/asyncio/unix_events.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -54,7 +54,8 @@ self._read_fut.cancel() def _fatal_error(self, exc): - logger.exception('Fatal error for %s', self) + if not isinstance(exc, (BrokenPipeError, ConnectionResetError)): + logger.exception('Fatal error for %s', self) self._force_close(exc) def _force_close(self, exc): diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -363,7 +363,8 @@ def _fatal_error(self, exc): # should be called by exception handler only - logger.exception('Fatal error for %s', self) + if not isinstance(exc, (BrokenPipeError, ConnectionResetError)): + logger.exception('Fatal error for %s', self) self._close(exc) def _close(self, exc=None): -- Repository URL: http://hg.python.org/cpython

1 0

cpython: inspect.Signature.from_function: Use CO_VARARGS & CO_VARKEYWORDS constants
by yury.selivanov 29 Jan '14

29 Jan '14

http://hg.python.org/cpython/rev/1435ebd8b88e changeset: 88818:1435ebd8b88e parent: 88816:e508c2b8fd44 user: Yury Selivanov <yselivanov(a)sprymix.com> date: Wed Jan 29 16:50:40 2014 -0500 summary: inspect.Signature.from_function: Use CO_VARARGS & CO_VARKEYWORDS constants files: Lib/inspect.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2109,7 +2109,7 @@ default=defaults[offset])) # *args - if func_code.co_flags & 0x04: + if func_code.co_flags & CO_VARARGS: name = arg_names[pos_count + keyword_only_count] annotation = annotations.get(name, _empty) parameters.append(Parameter(name, annotation=annotation, @@ -2126,9 +2126,9 @@ kind=_KEYWORD_ONLY, default=default)) # **kwargs - if func_code.co_flags & 0x08: + if func_code.co_flags & CO_VARKEYWORDS: index = pos_count + keyword_only_count - if func_code.co_flags & 0x04: + if func_code.co_flags & CO_VARARGS: index += 1 name = arg_names[index] -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.7): Idlelib & buildbots: suppress py3 deprecation message even if enabled.
by terry.reedy 29 Jan '14

29 Jan '14

http://hg.python.org/cpython/rev/09716a4d6c52 changeset: 88817:09716a4d6c52 branch: 2.7 parent: 88808:3971e1b07af4 user: Terry Jan Reedy <tjreedy(a)udel.edu> date: Wed Jan 29 14:42:32 2014 -0500 summary: Idlelib & buildbots: suppress py3 deprecation message even if enabled. files: Lib/idlelib/idle_test/test_calltips.py | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py --- a/Lib/idlelib/idle_test/test_calltips.py +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -3,6 +3,7 @@ CTi = ct.CallTips() # needed for get_entity test in 2.7 import textwrap import types +import warnings default_tip = '' @@ -162,8 +163,11 @@ class Py2Test(unittest.TestCase): def test_paramtuple_float(self): - # 18539: (a,b) becomes '.0' in code object; change that but not float - def f((a,b), c=0.0): pass + # 18539: (a,b) becomes '.0' in code object; change that but not 0.0 + with warnings.catch_warnings(): + # Suppess message of py3 deprecation of parameter unpacking + warnings.simplefilter("ignore") + def f((a,b), c=0.0): pass self.assertEqual(signature(f), '(<tuple>, c=0.0)') if __name__ == '__main__': -- Repository URL: http://hg.python.org/cpython

1 0

cpython: inspect.signature: Make sure that if a callable object has '_patialmethod'
by yury.selivanov 29 Jan '14

29 Jan '14

http://hg.python.org/cpython/rev/e508c2b8fd44 changeset: 88816:e508c2b8fd44 user: Yury Selivanov <yselivanov(a)sprymix.com> date: Wed Jan 29 12:18:59 2014 -0500 summary: inspect.signature: Make sure that if a callable object has '_patialmethod' attribute, that attribute is an instance of 'functools.partialmethod'. files: Lib/inspect.py | 29 ++++++++++++++------------- Lib/test/test_inspect.py | 5 ++++ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1651,20 +1651,21 @@ except AttributeError: pass else: - # Unbound partialmethod (see functools.partialmethod) - # This means, that we need to calculate the signature - # as if it's a regular partial object, but taking into - # account that the first positional argument - # (usually `self`, or `cls`) will not be passed - # automatically (as for boundmethods) - - wrapped_sig = signature(partialmethod.func) - sig = _signature_get_partial(wrapped_sig, partialmethod, (None,)) - - first_wrapped_param = tuple(wrapped_sig.parameters.values())[0] - new_params = (first_wrapped_param,) + tuple(sig.parameters.values()) - - return sig.replace(parameters=new_params) + if isinstance(partialmethod, functools.partialmethod): + # Unbound partialmethod (see functools.partialmethod) + # This means, that we need to calculate the signature + # as if it's a regular partial object, but taking into + # account that the first positional argument + # (usually `self`, or `cls`) will not be passed + # automatically (as for boundmethods) + + wrapped_sig = signature(partialmethod.func) + sig = _signature_get_partial(wrapped_sig, partialmethod, (None,)) + + first_wrapped_param = tuple(wrapped_sig.parameters.values())[0] + new_params = (first_wrapped_param,) + tuple(sig.parameters.values()) + + return sig.replace(parameters=new_params) if _signature_is_builtin(obj): return Signature.from_builtin(obj) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -1983,6 +1983,11 @@ ('c', 1, ..., 'keyword_only')), 'spam')) + def test_signature_on_fake_partialmethod(self): + def foo(a): pass + foo._partialmethod = 'spam' + self.assertEqual(str(inspect.signature(foo)), '(a)') + def test_signature_on_decorated(self): import functools -- Repository URL: http://hg.python.org/cpython

1 0

cpython: inspect.Signature.bind: Update method signature to rule out possiblity
by yury.selivanov 29 Jan '14

29 Jan '14

http://hg.python.org/cpython/rev/379d9a193539 changeset: 88815:379d9a193539 user: Yury Selivanov <yselivanov(a)sprymix.com> date: Wed Jan 29 12:10:27 2014 -0500 summary: inspect.Signature.bind: Update method signature to rule out possiblity of name conflict between '__bind_self' and actual keyword argument to 'bind' or 'bind_partial'. files: Lib/inspect.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2470,19 +2470,19 @@ return self._bound_arguments_cls(self, arguments) - def bind(__bind_self, *args, **kwargs): + def bind(*args, **kwargs): '''Get a BoundArguments object, that maps the passed `args` and `kwargs` to the function's signature. Raises `TypeError` if the passed arguments can not be bound. ''' - return __bind_self._bind(args, kwargs) - - def bind_partial(__bind_self, *args, **kwargs): + return args[0]._bind(args[1:], kwargs) + + def bind_partial(*args, **kwargs): '''Get a BoundArguments object, that partially maps the passed `args` and `kwargs` to the function's signature. Raises `TypeError` if the passed arguments can not be bound. ''' - return __bind_self._bind(args, kwargs, partial=True) + return args[0]._bind(args[1:], kwargs, partial=True) def __str__(self): result = [] -- Repository URL: http://hg.python.org/cpython

1 0

cpython: inspect: Rename private helper function
by yury.selivanov 29 Jan '14

29 Jan '14

http://hg.python.org/cpython/rev/07808c511537 changeset: 88814:07808c511537 user: Yury Selivanov <yselivanov(a)sprymix.com> date: Wed Jan 29 12:05:40 2014 -0500 summary: inspect: Rename private helper function files: Lib/inspect.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1505,7 +1505,7 @@ types.BuiltinFunctionType) -def _get_user_defined_method(cls, method_name): +def _signature_get_user_defined_method(cls, method_name): try: meth = getattr(cls, method_name) except AttributeError: @@ -1682,17 +1682,17 @@ # First, let's see if it has an overloaded __call__ defined # in its metaclass - call = _get_user_defined_method(type(obj), '__call__') + call = _signature_get_user_defined_method(type(obj), '__call__') if call is not None: sig = signature(call) else: # Now we check if the 'obj' class has a '__new__' method - new = _get_user_defined_method(obj, '__new__') + new = _signature_get_user_defined_method(obj, '__new__') if new is not None: sig = signature(new) else: # Finally, we should have at least __init__ implemented - init = _get_user_defined_method(obj, '__init__') + init = _signature_get_user_defined_method(obj, '__init__') if init is not None: sig = signature(init) @@ -1711,7 +1711,7 @@ # We also check that the 'obj' is not an instance of # _WrapperDescriptor or _MethodWrapper to avoid # infinite recursion (and even potential segfault) - call = _get_user_defined_method(type(obj), '__call__') + call = _signature_get_user_defined_method(type(obj), '__call__') if call is not None: sig = signature(call) -- Repository URL: http://hg.python.org/cpython

1 0