[pypy-commit] pypy py3.3: merge py3k
pjenvey
noreply at buildbot.pypy.org
Thu Jul 17 00:07:09 CEST 2014
Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.3
Changeset: r72463:7536507ca6e5
Date: 2014-07-16 14:50 -0700
http://bitbucket.org/pypy/pypy/changeset/7536507ca6e5/
Log: merge py3k
diff --git a/lib-python/2.7/xml/sax/saxutils.py b/lib-python/2.7/xml/sax/saxutils.py
--- a/lib-python/2.7/xml/sax/saxutils.py
+++ b/lib-python/2.7/xml/sax/saxutils.py
@@ -98,13 +98,14 @@
except AttributeError:
pass
# wrap a binary writer with TextIOWrapper
- class UnbufferedTextIOWrapper(io.TextIOWrapper):
- def write(self, s):
- super(UnbufferedTextIOWrapper, self).write(s)
- self.flush()
- return UnbufferedTextIOWrapper(buffer, encoding=encoding,
+ return _UnbufferedTextIOWrapper(buffer, encoding=encoding,
errors='xmlcharrefreplace',
newline='\n')
+# PyPy: moved this class outside the function above
+class _UnbufferedTextIOWrapper(io.TextIOWrapper):
+ def write(self, s):
+ super(_UnbufferedTextIOWrapper, self).write(s)
+ self.flush()
class XMLGenerator(handler.ContentHandler):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1461,9 +1461,7 @@
return buf.as_str()
def str_or_None_w(self, w_obj):
- if self.is_w(w_obj, self.w_None):
- return None
- return self.str_w(w_obj)
+ return None if self.is_none(w_obj) else self.str_w(w_obj)
def str_w(self, w_obj):
"""
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1,18 +1,17 @@
-from __future__ import with_statement
-from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+import weakref
+from rpython.rlib import rpoll, rsocket
from rpython.rlib.rarithmetic import intmask
-from rpython.rlib import rpoll, rsocket
from rpython.rlib.ropenssl import *
from rpython.rlib.rposix import get_errno, set_errno
+from rpython.rtyper.lltypesystem import lltype, rffi
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.typedef import GetSetProperty, TypeDef
from pypy.module._socket import interp_socket
from pypy.module.exceptions import interp_exceptions
-import weakref
## user defined constants
@@ -303,19 +302,15 @@
Mix string into the OpenSSL PRNG state. entropy (a float) is a lower
bound on the entropy contained in string."""
-
- buf = rffi.str2charp(string)
- try:
+ with rffi.scoped_str2charp(string) as buf:
libssl_RAND_add(buf, len(string), entropy)
- finally:
- rffi.free_charp(buf)
def RAND_status(space):
"""RAND_status() -> 0 or 1
- Returns 1 if the OpenSSL PRNG has been seeded with enough data and 0 if not.
- It is necessary to seed the PRNG with RAND_add() on some platforms before
- using the ssl() function."""
+ Returns 1 if the OpenSSL PRNG has been seeded with enough data
+ and 0 if not. It is necessary to seed the PRNG with RAND_add()
+ on some platforms before using the ssl() function."""
res = libssl_RAND_status()
return space.wrap(res)
@@ -327,16 +322,12 @@
Queries the entropy gather daemon (EGD) on socket path. Returns number
of bytes read. Raises socket.sslerror if connection to EGD fails or
if it does provide enough data to seed PRNG."""
-
- socket_path = rffi.str2charp(path)
- try:
+ with rffi.scoped_str2charp(path) as socket_path:
bytes = libssl_RAND_egd(socket_path)
- finally:
- rffi.free_charp(socket_path)
if bytes == -1:
- msg = "EGD connection failed or EGD did not return"
- msg += " enough data to seed the PRNG"
- raise ssl_error(space, msg)
+ raise ssl_error(space,
+ "EGD connection failed or EGD did not return "
+ "enough data to seed the PRNG")
return space.wrap(bytes)
@@ -361,7 +352,7 @@
of bytes written."""
w_socket = self._get_socket(space)
- sockstate = check_socket_and_wait_for_timeout(space, w_socket, True)
+ sockstate = checkwait(space, w_socket, True)
if sockstate == SOCKET_HAS_TIMED_OUT:
raise ssl_error(space, "The write operation timed out")
elif sockstate == SOCKET_HAS_BEEN_CLOSED:
@@ -377,11 +368,9 @@
err = libssl_SSL_get_error(self.ssl, num_bytes)
if err == SSL_ERROR_WANT_READ:
- sockstate = check_socket_and_wait_for_timeout(
- space, w_socket, False)
+ sockstate = checkwait(space, w_socket, False)
elif err == SSL_ERROR_WANT_WRITE:
- sockstate = check_socket_and_wait_for_timeout(
- space, w_socket, True)
+ sockstate = checkwait(space, w_socket, True)
else:
sockstate = SOCKET_OPERATION_OK
@@ -421,19 +410,20 @@
count = libssl_SSL_pending(self.ssl)
if not count:
- sockstate = check_socket_and_wait_for_timeout(
- space, w_socket, False)
+ sockstate = checkwait(space, w_socket, False)
if sockstate == SOCKET_HAS_TIMED_OUT:
raise ssl_error(space, "The read operation timed out")
elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
- raise ssl_error(space, "Underlying socket too large for select().")
+ raise ssl_error(space,
+ "Underlying socket too large for select().")
elif sockstate == SOCKET_HAS_BEEN_CLOSED:
if libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN:
if space.is_none(w_buf):
return space.wrapbytes('')
else:
return space.wrap(0)
- raise ssl_error(space, "Socket closed without SSL shutdown handshake")
+ raise ssl_error(space,
+ "Socket closed without SSL shutdown handshake")
rwbuffer = None
if not space.is_none(w_buf):
@@ -450,11 +440,9 @@
err = libssl_SSL_get_error(self.ssl, count)
if err == SSL_ERROR_WANT_READ:
- sockstate = check_socket_and_wait_for_timeout(self.space,
- self.w_socket, False)
+ sockstate = checkwait(space, w_socket, False)
elif err == SSL_ERROR_WANT_WRITE:
- sockstate = check_socket_and_wait_for_timeout(self.space,
- self.w_socket, True)
+ sockstate = checkwait(space, w_socket, True)
elif (err == SSL_ERROR_ZERO_RETURN and
libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN):
if space.is_none(w_buf):
@@ -465,7 +453,7 @@
sockstate = SOCKET_OPERATION_OK
if sockstate == SOCKET_HAS_TIMED_OUT:
- raise ssl_error(self.space, "The read operation timed out")
+ raise ssl_error(space, "The read operation timed out")
elif sockstate == SOCKET_IS_NONBLOCKING:
break
@@ -475,13 +463,13 @@
break
if count <= 0:
- raise _ssl_seterror(self.space, self, count)
+ raise _ssl_seterror(space, self, count)
result = buf.str(count)
if rwbuffer is not None:
rwbuffer.setslice(0, result)
- return self.space.wrap(count)
+ return space.wrap(count)
else:
return space.wrapbytes(result)
@@ -508,11 +496,9 @@
err = libssl_SSL_get_error(self.ssl, ret)
# XXX PyErr_CheckSignals()
if err == SSL_ERROR_WANT_READ:
- sockstate = check_socket_and_wait_for_timeout(
- space, w_socket, False)
+ sockstate = checkwait(space, w_socket, False)
elif err == SSL_ERROR_WANT_WRITE:
- sockstate = check_socket_and_wait_for_timeout(
- space, w_socket, True)
+ sockstate = checkwait(space, w_socket, True)
else:
sockstate = SOCKET_OPERATION_OK
if sockstate == SOCKET_HAS_TIMED_OUT:
@@ -520,7 +506,8 @@
elif sockstate == SOCKET_HAS_BEEN_CLOSED:
raise ssl_error(space, "Underlying socket has been closed.")
elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
- raise ssl_error(space, "Underlying socket too large for select().")
+ raise ssl_error(space,
+ "Underlying socket too large for select().")
elif sockstate == SOCKET_IS_NONBLOCKING:
break
@@ -545,7 +532,6 @@
raise ssl_error(space, "Underlying socket has been closed")
zeros = 0
-
while True:
# Disable read-ahead so that unwrap can work correctly.
# Otherwise OpenSSL might read in too much data,
@@ -575,11 +561,9 @@
# Possibly retry shutdown until timeout or failure
ssl_err = libssl_SSL_get_error(self.ssl, ret)
if ssl_err == SSL_ERROR_WANT_READ:
- sockstate = check_socket_and_wait_for_timeout(
- space, w_socket, False)
+ sockstate = checkwait(space, w_socket, False)
elif ssl_err == SSL_ERROR_WANT_WRITE:
- sockstate = check_socket_and_wait_for_timeout(
- space, w_socket, True)
+ sockstate = checkwait(space, w_socket, True)
else:
break
@@ -589,7 +573,8 @@
else:
raise ssl_error(space, "The write operation timed out")
elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
- raise ssl_error(space, "Underlying socket too large for select().")
+ raise ssl_error(space,
+ "Underlying socket too large for select().")
elif sockstate != SOCKET_OPERATION_OK:
# Retain the SSL error code
break
@@ -607,35 +592,29 @@
return space.w_None
name = libssl_SSL_CIPHER_get_name(current)
- if name:
- w_name = space.wrap(rffi.charp2str(name))
- else:
- w_name = space.w_None
+ w_name = space.wrap(rffi.charp2str(name)) if name else space.w_None
proto = libssl_SSL_CIPHER_get_version(current)
- if proto:
- w_proto = space.wrap(rffi.charp2str(proto))
- else:
- w_proto = space.w_None
+ w_proto = space.wrap(rffi.charp2str(proto)) if proto else space.w_None
bits = libssl_SSL_CIPHER_get_bits(current,
lltype.nullptr(rffi.INTP.TO))
w_bits = space.newint(bits)
-
return space.newtuple([w_name, w_proto, w_bits])
@unwrap_spec(der=bool)
def peer_certificate(self, space, der=False):
"""peer_certificate([der=False]) -> certificate
- Returns the certificate for the peer. If no certificate was provided,
- returns None. If a certificate was provided, but not validated, returns
- an empty dictionary. Otherwise returns a dict containing information
- about the peer certificate.
+ Returns the certificate for the peer. If no certificate was
+ provided, returns None. If a certificate was provided, but not
+ validated, returns an empty dictionary. Otherwise returns a
+ dict containing information about the peer certificate.
- If the optional argument is True, returns a DER-encoded copy of the
- peer certificate, or None if no certificate was provided. This will
- return the certificate even if it wasn't validated."""
+ If the optional argument is True, returns a DER-encoded copy of
+ the peer certificate, or None if no certificate was provided.
+ This will return the certificate even if it wasn't validated.
+ """
if not self.peer_cert:
return space.w_None
@@ -648,8 +627,7 @@
raise _ssl_seterror(space, self, length)
try:
# this is actually an immutable bytes sequence
- return space.wrap(rffi.charpsize2str(buf_ptr[0],
- length))
+ return space.wrap(rffi.charpsize2str(buf_ptr[0], length))
finally:
libssl_OPENSSL_free(buf_ptr[0])
else:
@@ -795,15 +773,16 @@
name = libssl_sk_GENERAL_NAME_value(names, j)
gntype = intmask(name[0].c_type)
if gntype == GEN_DIRNAME:
- # we special-case DirName as a tuple of tuples of attributes
+ # we special-case DirName as a tuple of tuples of
+ # attributes
dirname = libssl_pypy_GENERAL_NAME_dirn(name)
w_t = space.newtuple([
space.wrap("DirName"),
_create_tuple_for_X509_NAME(space, dirname)
])
elif gntype in (GEN_EMAIL, GEN_DNS, GEN_URI):
- # GENERAL_NAME_print() doesn't handle NULL bytes in ASN1_string
- # correctly, CVE-2013-4238
+ # GENERAL_NAME_print() doesn't handle NULL bytes in
+ # ASN1_string correctly, CVE-2013-4238
if gntype == GEN_EMAIL:
v = space.wrap("email")
elif gntype == GEN_DNS:
@@ -877,17 +856,14 @@
sock_fd = space.int_w(space.call_method(w_sock, "fileno"))
w_timeout = space.call_method(w_sock, "gettimeout")
- if space.is_none(w_timeout):
- has_timeout = False
- else:
- has_timeout = True
+ has_timeout = not space.is_none(w_timeout)
ss.ssl = libssl_SSL_new(ctx) # new ssl struct
libssl_SSL_set_fd(ss.ssl, sock_fd) # set the socket for SSL
# The ACCEPT_MOVING_WRITE_BUFFER flag is necessary because the address
# of a str object may be changed by the garbage collector.
- libssl_SSL_set_mode(ss.ssl,
- SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER)
+ libssl_SSL_set_mode(
+ ss.ssl, SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER)
if server_hostname:
libssl_SSL_set_tlsext_host_name(ss.ssl, server_hostname);
@@ -907,7 +883,7 @@
ss.w_socket = weakref.ref(w_sock)
return ss
-def check_socket_and_wait_for_timeout(space, w_sock, writing):
+def checkwait(space, w_sock, writing):
"""If the socket has a timeout, do a select()/poll() on the socket.
The argument writing indicates the direction.
Returns one of the possibilities in the timeout_state enum (above)."""
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -667,11 +667,6 @@
space.wrap(nValues[0]),
space.wrap(l)])
-def str_or_None_w(space, w_obj):
- if space.is_w(w_obj, space.w_None):
- return None
- return space.str_w(w_obj)
-
def ConnectRegistry(space, w_machine, w_hkey):
"""key = ConnectRegistry(computer_name, key)
@@ -683,7 +678,7 @@
The return value is the handle of the opened key.
If the function fails, an EnvironmentError exception is raised."""
- machine = str_or_None_w(space, w_machine)
+ machine = space.str_or_None_w(w_machine)
hkey = hkey_w(w_hkey, space)
with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey:
ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1377,49 +1377,53 @@
def do_residual_call(self, funcbox, argboxes, descr, pc,
assembler_call=False,
assembler_call_jd=None):
- # First build allboxes: it may need some reordering from the
- # list provided in argboxes, depending on the order in which
- # the arguments are expected by the function
- #
- allboxes = self._build_allboxes(funcbox, argboxes, descr)
- effectinfo = descr.get_extra_info()
- if (assembler_call or
- effectinfo.check_forces_virtual_or_virtualizable()):
- # residual calls require attention to keep virtualizables in-sync
- self.metainterp.clear_exception()
- if effectinfo.oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL:
- resbox = self._do_jit_force_virtual(allboxes, descr, pc)
+ debug_start("jit-residual-call")
+ try:
+ # First build allboxes: it may need some reordering from the
+ # list provided in argboxes, depending on the order in which
+ # the arguments are expected by the function
+ #
+ allboxes = self._build_allboxes(funcbox, argboxes, descr)
+ effectinfo = descr.get_extra_info()
+ if (assembler_call or
+ effectinfo.check_forces_virtual_or_virtualizable()):
+ # residual calls require attention to keep virtualizables in-sync
+ self.metainterp.clear_exception()
+ if effectinfo.oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL:
+ resbox = self._do_jit_force_virtual(allboxes, descr, pc)
+ if resbox is not None:
+ return resbox
+ self.metainterp.vable_and_vrefs_before_residual_call()
+ resbox = self.metainterp.execute_and_record_varargs(
+ rop.CALL_MAY_FORCE, allboxes, descr=descr)
+ if effectinfo.is_call_release_gil():
+ self.metainterp.direct_call_release_gil()
+ self.metainterp.vrefs_after_residual_call()
+ vablebox = None
+ if assembler_call:
+ vablebox = self.metainterp.direct_assembler_call(
+ assembler_call_jd)
if resbox is not None:
- return resbox
- self.metainterp.vable_and_vrefs_before_residual_call()
- resbox = self.metainterp.execute_and_record_varargs(
- rop.CALL_MAY_FORCE, allboxes, descr=descr)
- if effectinfo.is_call_release_gil():
- self.metainterp.direct_call_release_gil()
- self.metainterp.vrefs_after_residual_call()
- vablebox = None
- if assembler_call:
- vablebox = self.metainterp.direct_assembler_call(
- assembler_call_jd)
- if resbox is not None:
- self.make_result_of_lastop(resbox)
- self.metainterp.vable_after_residual_call(funcbox)
- self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None)
- if vablebox is not None:
- self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None)
- self.metainterp.handle_possible_exception()
- # XXX refactor: direct_libffi_call() is a hack
- if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL:
- self.metainterp.direct_libffi_call()
- return resbox
- else:
- effect = effectinfo.extraeffect
- if effect == effectinfo.EF_LOOPINVARIANT:
- return self.execute_varargs(rop.CALL_LOOPINVARIANT, allboxes,
- descr, False, False)
- exc = effectinfo.check_can_raise()
- pure = effectinfo.check_is_elidable()
- return self.execute_varargs(rop.CALL, allboxes, descr, exc, pure)
+ self.make_result_of_lastop(resbox)
+ self.metainterp.vable_after_residual_call(funcbox)
+ self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None)
+ if vablebox is not None:
+ self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None)
+ self.metainterp.handle_possible_exception()
+ # XXX refactor: direct_libffi_call() is a hack
+ if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL:
+ self.metainterp.direct_libffi_call()
+ return resbox
+ else:
+ effect = effectinfo.extraeffect
+ if effect == effectinfo.EF_LOOPINVARIANT:
+ return self.execute_varargs(rop.CALL_LOOPINVARIANT, allboxes,
+ descr, False, False)
+ exc = effectinfo.check_can_raise()
+ pure = effectinfo.check_is_elidable()
+ return self.execute_varargs(rop.CALL, allboxes, descr, exc, pure)
+ finally:
+ debug_stop("jit-residual-call")
def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc):
if isinstance(condbox, ConstInt) and condbox.value == 0:
diff --git a/rpython/jit/tool/oparser_model.py b/rpython/jit/tool/oparser_model.py
--- a/rpython/jit/tool/oparser_model.py
+++ b/rpython/jit/tool/oparser_model.py
@@ -124,6 +124,15 @@
class ExtendedTreeLoop(model.TreeLoop):
+ def as_json(self):
+ return {
+ 'comment': self.comment,
+ 'name': self.name,
+ 'operations': [op.as_json() for op in self.operations],
+ 'inputargs': self.inputargs,
+ 'last_offset': self.last_offset
+ }
+
def getboxes(self):
def opboxes(operations):
for op in operations:
diff --git a/rpython/tool/jitlogparser/logparser2json.py b/rpython/tool/jitlogparser/logparser2json.py
new file mode 100755
--- /dev/null
+++ b/rpython/tool/jitlogparser/logparser2json.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+""" Convert logfile (from jit-log-opt and jit-backend) to json format.
+Usage:
+
+logparser2json.py <logfile.log> <outfile.json>
+"""
+
+import os
+import sys
+import json
+from rpython.tool.jitlogparser.parser import import_log, parse_log_counts
+from rpython.tool.logparser import extract_category
+from rpython.tool.jitlogparser.storage import LoopStorage
+
+def mangle_descr(descr):
+ if descr.startswith('TargetToken('):
+ return descr[len('TargetToken('):-1]
+ if descr.startswith('<Guard'):
+ return 'bridge-' + str(int(descr[len('<Guard0x'):-1], 16))
+ if descr.startswith('<Loop'):
+ return 'entry-' + descr[len('<Loop'):-1]
+ return descr.replace(" ", '-')
+
+def create_loop_dict(loops):
+ d = {}
+ for loop in loops:
+ d[mangle_descr(loop.descr)] = loop
+ return d
+
+def main(progname, logfilename, outfilename):
+ storage = LoopStorage(extrapath=os.path.dirname(progname))
+ log, loops = import_log(logfilename)
+ parse_log_counts(extract_category(log, 'jit-backend-count'), loops)
+ storage.loops = [loop for loop in loops
+ if not loop.descr.startswith('bridge')]
+ storage.loop_dict = create_loop_dict(loops)
+ json.dump([loop.force_asm().as_json() for loop in storage.loops],
+ open(outfilename, "w"), indent=4)
+
+if __name__ == '__main__':
+ if len(sys.argv) != 3:
+ print __doc__
+ sys.exit(1)
+ main(sys.argv[0], sys.argv[1], sys.argv[2])
diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -36,6 +36,20 @@
if self._is_guard:
self.guard_no = int(self.descr[len('<Guard0x'):-1], 16)
+ def as_json(self):
+ d = {
+ 'name': self.name,
+ 'args': self.args,
+ 'res': self.res,
+ }
+ if self.descr is not None:
+ d['descr'] = self.descr
+ if self.bridge is not None:
+ d['bridge'] = self.bridge.as_json()
+ if self.asm is not None:
+ d['asm'] = self.asm
+ return d
+
def setfailargs(self, failargs):
self.failargs = failargs
diff --git a/rpython/tool/jitlogparser/test/test_storage.py b/rpython/tool/jitlogparser/test/test_storage.py
--- a/rpython/tool/jitlogparser/test/test_storage.py
+++ b/rpython/tool/jitlogparser/test/test_storage.py
@@ -6,3 +6,4 @@
tmppath.join("x.py").write("def f(): pass") # one code
s = LoopStorage(str(tmppath))
assert s.load_code(str(tmppath.join('x.py'))) == s.load_code('x.py')
+
More information about the pypy-commit
mailing list