[pypy-commit] pypy rgc-mem-pressure: merge in default
justinpeel
noreply at buildbot.pypy.org
Mon Oct 17 04:25:09 CEST 2011
Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48106:c26127bd0a39
Date: 2011-10-16 20:24 -0600
http://bitbucket.org/pypy/pypy/changeset/c26127bd0a39/
Log: merge in default
diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+ (null)
+ |
+ | HTTPConnection()
+ v
+ Idle
+ |
+ | putrequest()
+ v
+ Request-started
+ |
+ | ( putheader() )* endheaders()
+ v
+ Request-sent
+ |
+ | response = getresponse()
+ v
+ Unread-response [Response-headers-read]
+ |\____________________
+ | |
+ | response.read() | putrequest()
+ v v
+ Idle Req-started-unread-response
+ ______/|
+ / |
+ response.read() | | ( putheader() )* endheaders()
+ v v
+ Request-started Req-sent-unread-response
+ |
+ | response.read()
+ v
+ Request-sent
+
+This diagram presents the following rules:
+ -- a second request may not be started until {response-headers-read}
+ -- a response [object] cannot be retrieved until {request-sent}
+ -- there is no differentiation between an unread response body and a
+ partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+ HTTPResponse class does not enforce this state machine, which
+ implies sophisticated clients may accelerate the request/response
+ pipeline. Caution should be taken, though: accelerating the states
+ beyond the above pattern may imply knowledge of the server's
+ connection-close behavior for certain requests. For example, it
+ is impossible to tell whether the server will close the connection
+ UNTIL the response headers have been read; this means that further
+ requests cannot be placed into the pipeline until it is known that
+ the server will NOT be closing the connection.
+
+Logical State __state __response
+------------- ------- ----------
+Idle _CS_IDLE None
+Request-started _CS_REQ_STARTED None
+Request-sent _CS_REQ_SENT None
+Unread-response _CS_IDLE <response_class>
+Req-started-unread-response _CS_REQ_STARTED <response_class>
+Req-sent-unread-response _CS_REQ_SENT <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+ if py3kwarning:
+ warnings.filterwarnings("ignore", ".*mimetools has been removed",
+ DeprecationWarning)
+ import mimetools
+
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+ "HTTPException", "NotConnected", "UnknownProtocol",
+ "UnknownTransferEncoding", "UnimplementedFileMode",
+ "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+ "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+ "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+ 100: 'Continue',
+ 101: 'Switching Protocols',
+
+ 200: 'OK',
+ 201: 'Created',
+ 202: 'Accepted',
+ 203: 'Non-Authoritative Information',
+ 204: 'No Content',
+ 205: 'Reset Content',
+ 206: 'Partial Content',
+
+ 300: 'Multiple Choices',
+ 301: 'Moved Permanently',
+ 302: 'Found',
+ 303: 'See Other',
+ 304: 'Not Modified',
+ 305: 'Use Proxy',
+ 306: '(Unused)',
+ 307: 'Temporary Redirect',
+
+ 400: 'Bad Request',
+ 401: 'Unauthorized',
+ 402: 'Payment Required',
+ 403: 'Forbidden',
+ 404: 'Not Found',
+ 405: 'Method Not Allowed',
+ 406: 'Not Acceptable',
+ 407: 'Proxy Authentication Required',
+ 408: 'Request Timeout',
+ 409: 'Conflict',
+ 410: 'Gone',
+ 411: 'Length Required',
+ 412: 'Precondition Failed',
+ 413: 'Request Entity Too Large',
+ 414: 'Request-URI Too Long',
+ 415: 'Unsupported Media Type',
+ 416: 'Requested Range Not Satisfiable',
+ 417: 'Expectation Failed',
+
+ 500: 'Internal Server Error',
+ 501: 'Not Implemented',
+ 502: 'Bad Gateway',
+ 503: 'Service Unavailable',
+ 504: 'Gateway Timeout',
+ 505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+ def addheader(self, key, value):
+ """Add header for field key handling repeats."""
+ prev = self.dict.get(key)
+ if prev is None:
+ self.dict[key] = value
+ else:
+ combined = ", ".join((prev, value))
+ self.dict[key] = combined
+
+ def addcontinue(self, key, more):
+ """Add more field data from a continuation line."""
+ prev = self.dict[key]
+ self.dict[key] = prev + "\n " + more
+
+ def readheaders(self):
+ """Read header lines.
+
+ Read header lines up to the entirely blank line that terminates them.
+ The (normally blank) line that ends the headers is skipped, but not
+ included in the returned list. If a non-header line ends the headers,
+ (which is an error), an attempt is made to backspace over it; it is
+ never included in the returned list.
+
+ The variable self.status is set to the empty string if all went well,
+ otherwise it is an error message. The variable self.headers is a
+ completely uninterpreted list of lines contained in the header (so
+ printing them will reproduce the header exactly as it appears in the
+ file).
+
+ If multiple header fields with the same name occur, they are combined
+ according to the rules in RFC 2616 sec 4.2:
+
+ Appending each subsequent field-value to the first, each separated
+ by a comma. The order in which header fields with the same field-name
+ are received is significant to the interpretation of the combined
+ field value.
+ """
+ # XXX The implementation overrides the readheaders() method of
+ # rfc822.Message. The base class design isn't amenable to
+ # customized behavior here so the method here is a copy of the
+ # base class code with a few small changes.
+
+ self.dict = {}
+ self.unixfrom = ''
+ self.headers = hlist = []
+ self.status = ''
+ headerseen = ""
+ firstline = 1
+ startofline = unread = tell = None
+ if hasattr(self.fp, 'unread'):
+ unread = self.fp.unread
+ elif self.seekable:
+ tell = self.fp.tell
+ while True:
+ if tell:
+ try:
+ startofline = tell()
+ except IOError:
+ startofline = tell = None
+ self.seekable = 0
+ line = self.fp.readline()
+ if not line:
+ self.status = 'EOF in headers'
+ break
+ # Skip unix From name time lines
+ if firstline and line.startswith('From '):
+ self.unixfrom = self.unixfrom + line
+ continue
+ firstline = 0
+ if headerseen and line[0] in ' \t':
+ # XXX Not sure if continuation lines are handled properly
+ # for http and/or for repeating headers
+ # It's a continuation line.
+ hlist.append(line)
+ self.addcontinue(headerseen, line.strip())
+ continue
+ elif self.iscomment(line):
+ # It's a comment. Ignore it.
+ continue
+ elif self.islast(line):
+ # Note! No pushback here! The delimiter line gets eaten.
+ break
+ headerseen = self.isheader(line)
+ if headerseen:
+ # It's a legal header line, save it.
+ hlist.append(line)
+ self.addheader(headerseen, line[len(headerseen)+1:].strip())
+ continue
+ else:
+ # It's not a header line; throw it back and stop here.
+ if not self.dict:
+ self.status = 'No headers'
+ else:
+ self.status = 'Non-header line where header expected'
+ # Try to undo the read.
+ if unread:
+ unread(line)
+ elif tell:
+ self.fp.seek(startofline)
+ else:
+ self.status = self.status + '; bad seek'
+ break
+
+class HTTPResponse:
+
+ # strict: If true, raise BadStatusLine if the status line can't be
+ # parsed as a valid HTTP/1.0 or 1.1 status line. By default it is
+ # false because it prevents clients from talking to HTTP/0.9
+ # servers. Note that a response with a sufficiently corrupted
+ # status line will look like an HTTP/0.9 response.
+
+ # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+ def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+ if buffering:
+ # The caller won't be using any sock.recv() calls, so buffering
+ # is fine and recommended for performance.
+ self.fp = sock.makefile('rb')
+ else:
+ # The buffer size is specified as zero, because the headers of
+ # the response are read with readline(). If the reads were
+ # buffered the readline() calls could consume some of the
+ # response, which make be read via a recv() on the underlying
+ # socket.
+ self.fp = sock.makefile('rb', 0)
+ self.debuglevel = debuglevel
+ self.strict = strict
+ self._method = method
+
+ self.msg = None
+
+ # from the Status-Line of the response
+ self.version = _UNKNOWN # HTTP-Version
+ self.status = _UNKNOWN # Status-Code
+ self.reason = _UNKNOWN # Reason-Phrase
+
+ self.chunked = _UNKNOWN # is "chunked" being used?
+ self.chunk_left = _UNKNOWN # bytes left to read in current chunk
+ self.length = _UNKNOWN # number of bytes left in response
+ self.will_close = _UNKNOWN # conn will close at end of response
+
+ def _read_status(self):
+ # Initialize with Simple-Response defaults
+ line = self.fp.readline()
+ if self.debuglevel > 0:
+ print "reply:", repr(line)
+ if not line:
+ # Presumably, the server closed the connection before
+ # sending a valid response.
+ raise BadStatusLine(line)
+ try:
+ [version, status, reason] = line.split(None, 2)
+ except ValueError:
+ try:
+ [version, status] = line.split(None, 1)
+ reason = ""
+ except ValueError:
+ # empty version will cause next test to fail and status
+ # will be treated as 0.9 response.
+ version = ""
+ if not version.startswith('HTTP/'):
+ if self.strict:
+ self.close()
+ raise BadStatusLine(line)
+ else:
+ # assume it's a Simple-Response from an 0.9 server
+ self.fp = LineAndFileWrapper(line, self.fp)
+ return "HTTP/0.9", 200, ""
+
+ # The status code is a three-digit number
+ try:
+ status = int(status)
+ if status < 100 or status > 999:
+ raise BadStatusLine(line)
+ except ValueError:
+ raise BadStatusLine(line)
+ return version, status, reason
+
+ def begin(self):
+ if self.msg is not None:
+ # we've already started reading the response
+ return
+
+ # read until we get a non-100 response
+ while True:
+ version, status, reason = self._read_status()
+ if status != CONTINUE:
+ break
+ # skip the header from the 100 response
+ while True:
+ skip = self.fp.readline().strip()
+ if not skip:
+ break
+ if self.debuglevel > 0:
+ print "header:", skip
+
+ self.status = status
+ self.reason = reason.strip()
+ if version == 'HTTP/1.0':
+ self.version = 10
+ elif version.startswith('HTTP/1.'):
+ self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1
+ elif version == 'HTTP/0.9':
+ self.version = 9
+ else:
+ raise UnknownProtocol(version)
+
+ if self.version == 9:
+ self.length = None
+ self.chunked = 0
+ self.will_close = 1
+ self.msg = HTTPMessage(StringIO())
+ return
+
+ self.msg = HTTPMessage(self.fp, 0)
+ if self.debuglevel > 0:
+ for hdr in self.msg.headers:
+ print "header:", hdr,
+
+ # don't let the msg keep an fp
+ self.msg.fp = None
+
+ # are we using the chunked-style of transfer encoding?
+ tr_enc = self.msg.getheader('transfer-encoding')
+ if tr_enc and tr_enc.lower() == "chunked":
+ self.chunked = 1
+ self.chunk_left = None
+ else:
+ self.chunked = 0
+
+ # will the connection close at the end of the response?
+ self.will_close = self._check_close()
+
+ # do we have a Content-Length?
+ # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+ length = self.msg.getheader('content-length')
+ if length and not self.chunked:
+ try:
+ self.length = int(length)
+ except ValueError:
+ self.length = None
+ else:
+ if self.length < 0: # ignore nonsensical negative lengths
+ self.length = None
+ else:
+ self.length = None
+
+ # does the body have a fixed length? (of zero)
+ if (status == NO_CONTENT or status == NOT_MODIFIED or
+ 100 <= status < 200 or # 1xx codes
+ self._method == 'HEAD'):
+ self.length = 0
+
+ # if the connection remains open, and we aren't using chunked, and
+ # a content-length was not provided, then assume that the connection
+ # WILL close.
+ if not self.will_close and \
+ not self.chunked and \
+ self.length is None:
+ self.will_close = 1
+
+ def _check_close(self):
+ conn = self.msg.getheader('connection')
+ if self.version == 11:
+ # An HTTP/1.1 proxy is assumed to stay open unless
+ # explicitly closed.
+ conn = self.msg.getheader('connection')
+ if conn and "close" in conn.lower():
+ return True
+ return False
+
+ # Some HTTP/1.0 implementations have support for persistent
+ # connections, using rules different than HTTP/1.1.
+
+ # For older HTTP, Keep-Alive indicates persistent connection.
+ if self.msg.getheader('keep-alive'):
+ return False
+
+ # At least Akamai returns a "Connection: Keep-Alive" header,
+ # which was supposed to be sent by the client.
+ if conn and "keep-alive" in conn.lower():
+ return False
+
+ # Proxy-Connection is a netscape hack.
+ pconn = self.msg.getheader('proxy-connection')
+ if pconn and "keep-alive" in pconn.lower():
+ return False
+
+ # otherwise, assume it will close
+ return True
+
+ def close(self):
+ if self.fp:
+ self.fp.close()
+ self.fp = None
+
+ def isclosed(self):
+ # NOTE: it is possible that we will not ever call self.close(). This
+ # case occurs when will_close is TRUE, length is None, and we
+ # read up to the last byte, but NOT past it.
+ #
+ # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+ # called, meaning self.isclosed() is meaningful.
+ return self.fp is None
+
+ # XXX It would be nice to have readline and __iter__ for this, too.
+
+ def read(self, amt=None):
+ if self.fp is None:
+ return ''
+
+ if self._method == 'HEAD':
+ self.close()
+ return ''
+
+ if self.chunked:
+ return self._read_chunked(amt)
+
+ if amt is None:
+ # unbounded read
+ if self.length is None:
+ s = self.fp.read()
+ else:
+ s = self._safe_read(self.length)
+ self.length = 0
+ self.close() # we read everything
+ return s
+
+ if self.length is not None:
+ if amt > self.length:
+ # clip the read to the "end of response"
+ amt = self.length
+
+ # we do not use _safe_read() here because this may be a .will_close
+ # connection, and the user is reading more bytes than will be provided
+ # (for example, reading in 1k chunks)
+ s = self.fp.read(amt)
+ if self.length is not None:
+ self.length -= len(s)
+ if not self.length:
+ self.close()
+ return s
+
+ def _read_chunked(self, amt):
+ assert self.chunked != _UNKNOWN
+ chunk_left = self.chunk_left
+ value = []
+ while True:
+ if chunk_left is None:
+ line = self.fp.readline()
+ i = line.find(';')
+ if i >= 0:
+ line = line[:i] # strip chunk-extensions
+ try:
+ chunk_left = int(line, 16)
+ except ValueError:
+ # close the connection as protocol synchronisation is
+ # probably lost
+ self.close()
+ raise IncompleteRead(''.join(value))
+ if chunk_left == 0:
+ break
+ if amt is None:
+ value.append(self._safe_read(chunk_left))
+ elif amt < chunk_left:
+ value.append(self._safe_read(amt))
+ self.chunk_left = chunk_left - amt
+ return ''.join(value)
+ elif amt == chunk_left:
+ value.append(self._safe_read(amt))
+ self._safe_read(2) # toss the CRLF at the end of the chunk
+ self.chunk_left = None
+ return ''.join(value)
+ else:
+ value.append(self._safe_read(chunk_left))
+ amt -= chunk_left
+
+ # we read the whole chunk, get another
+ self._safe_read(2) # toss the CRLF at the end of the chunk
+ chunk_left = None
+
+ # read and discard trailer up to the CRLF terminator
+ ### note: we shouldn't have any trailers!
+ while True:
+ line = self.fp.readline()
+ if not line:
+ # a vanishingly small number of sites EOF without
+ # sending the trailer
+ break
+ if line == '\r\n':
+ break
+
+ # we read everything; close the "file"
+ self.close()
+
+ return ''.join(value)
+
+ def _safe_read(self, amt):
+ """Read the number of bytes requested, compensating for partial reads.
+
+ Normally, we have a blocking socket, but a read() can be interrupted
+ by a signal (resulting in a partial read).
+
+ Note that we cannot distinguish between EOF and an interrupt when zero
+ bytes have been read. IncompleteRead() will be raised in this
+ situation.
+
+ This function should be used when <amt> bytes "should" be present for
+ reading. If the bytes are truly not available (due to EOF), then the
+ IncompleteRead exception can be used to detect the problem.
+ """
+ # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+ # return less than x bytes unless EOF is encountered. It now handles
+ # signal interruptions (socket.error EINTR) internally. This code
+ # never caught that exception anyways. It seems largely pointless.
+ # self.fp.read(amt) will work fine.
+ s = []
+ while amt > 0:
+ chunk = self.fp.read(min(amt, MAXAMOUNT))
+ if not chunk:
+ raise IncompleteRead(''.join(s), amt)
+ s.append(chunk)
+ amt -= len(chunk)
+ return ''.join(s)
+
+ def fileno(self):
+ return self.fp.fileno()
+
+ def getheader(self, name, default=None):
+ if self.msg is None:
+ raise ResponseNotReady()
+ return self.msg.getheader(name, default)
+
+ def getheaders(self):
+ """Return list of (header, value) tuples."""
+ if self.msg is None:
+ raise ResponseNotReady()
+ return self.msg.items()
+
+
+class HTTPConnection:
+
+ _http_vsn = 11
+ _http_vsn_str = 'HTTP/1.1'
+
+ response_class = HTTPResponse
+ default_port = HTTP_PORT
+ auto_open = 1
+ debuglevel = 0
+ strict = 0
+
+ def __init__(self, host, port=None, strict=None,
+ timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+ self.timeout = timeout
+ self.source_address = source_address
+ self.sock = None
+ self._buffer = []
+ self.__response = None
+ self.__state = _CS_IDLE
+ self._method = None
+ self._tunnel_host = None
+ self._tunnel_port = None
+ self._tunnel_headers = {}
+
+ self._set_hostport(host, port)
+ if strict is not None:
+ self.strict = strict
+
+ def set_tunnel(self, host, port=None, headers=None):
+ """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+ The headers argument should be a mapping of extra HTTP headers
+ to send with the CONNECT request.
+ """
+ self._tunnel_host = host
+ self._tunnel_port = port
+ if headers:
+ self._tunnel_headers = headers
+ else:
+ self._tunnel_headers.clear()
+
+ def _set_hostport(self, host, port):
+ if port is None:
+ i = host.rfind(':')
+ j = host.rfind(']') # ipv6 addresses have [...]
+ if i > j:
+ try:
+ port = int(host[i+1:])
+ except ValueError:
+ raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+ host = host[:i]
+ else:
+ port = self.default_port
+ if host and host[0] == '[' and host[-1] == ']':
+ host = host[1:-1]
+ self.host = host
+ self.port = port
+
+ def set_debuglevel(self, level):
+ self.debuglevel = level
+
+ def _tunnel(self):
+ self._set_hostport(self._tunnel_host, self._tunnel_port)
+ self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+ for header, value in self._tunnel_headers.iteritems():
+ self.send("%s: %s\r\n" % (header, value))
+ self.send("\r\n")
+ response = self.response_class(self.sock, strict = self.strict,
+ method = self._method)
+ (version, code, message) = response._read_status()
+
+ if code != 200:
+ self.close()
+ raise socket.error("Tunnel connection failed: %d %s" % (code,
+ message.strip()))
+ while True:
+ line = response.fp.readline()
+ if line == '\r\n': break
+
+
+ def connect(self):
+ """Connect to the host and port specified in __init__."""
+ self.sock = socket.create_connection((self.host,self.port),
+ self.timeout, self.source_address)
+
+ if self._tunnel_host:
+ self._tunnel()
+
+ def close(self):
+ """Close the connection to the HTTP server."""
+ if self.sock:
+ self.sock.close() # close it manually... there may be other refs
+ self.sock = None
+ if self.__response:
+ self.__response.close()
+ self.__response = None
+ self.__state = _CS_IDLE
+
+ def send(self, data):
+ """Send `data' to the server."""
+ if self.sock is None:
+ if self.auto_open:
+ self.connect()
+ else:
+ raise NotConnected()
+
+ if self.debuglevel > 0:
+ print "send:", repr(data)
+ blocksize = 8192
+ if hasattr(data,'read') and not isinstance(data, array):
+ if self.debuglevel > 0: print "sendIng a read()able"
+ datablock = data.read(blocksize)
+ while datablock:
+ self.sock.sendall(datablock)
+ datablock = data.read(blocksize)
+ else:
+ self.sock.sendall(data)
+
+ def _output(self, s):
+ """Add a line of output to the current request buffer.
+
+ Assumes that the line does *not* end with \\r\\n.
+ """
+ self._buffer.append(s)
+
+ def _send_output(self, message_body=None):
+ """Send the currently buffered request and clear the buffer.
+
+ Appends an extra \\r\\n to the buffer.
+ A message_body may be specified, to be appended to the request.
+ """
+ self._buffer.extend(("", ""))
+ msg = "\r\n".join(self._buffer)
+ del self._buffer[:]
+ # If msg and message_body are sent in a single send() call,
+ # it will avoid performance problems caused by the interaction
+ # between delayed ack and the Nagle algorithim.
+ if isinstance(message_body, str):
+ msg += message_body
+ message_body = None
+ self.send(msg)
+ if message_body is not None:
+ #message_body was not a string (i.e. it is a file) and
+ #we must run the risk of Nagle
+ self.send(message_body)
+
+ def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+ """Send a request to the server.
+
+ `method' specifies an HTTP request method, e.g. 'GET'.
+ `url' specifies the object being requested, e.g. '/index.html'.
+ `skip_host' if True does not add automatically a 'Host:' header
+ `skip_accept_encoding' if True does not add automatically an
+ 'Accept-Encoding:' header
+ """
+
+ # if a prior response has been completed, then forget about it.
+ if self.__response and self.__response.isclosed():
+ self.__response = None
+
+
+ # in certain cases, we cannot issue another request on this connection.
+ # this occurs when:
+ # 1) we are in the process of sending a request. (_CS_REQ_STARTED)
+ # 2) a response to a previous request has signalled that it is going
+ # to close the connection upon completion.
+ # 3) the headers for the previous response have not been read, thus
+ # we cannot determine whether point (2) is true. (_CS_REQ_SENT)
+ #
+ # if there is no prior response, then we can request at will.
+ #
+ # if point (2) is true, then we will have passed the socket to the
+ # response (effectively meaning, "there is no prior response"), and
+ # will open a new one when a new request is made.
+ #
+ # Note: if a prior response exists, then we *can* start a new request.
+ # We are not allowed to begin fetching the response to this new
+ # request, however, until that prior response is complete.
+ #
+ if self.__state == _CS_IDLE:
+ self.__state = _CS_REQ_STARTED
+ else:
+ raise CannotSendRequest()
+
+ # Save the method we use, we need it later in the response phase
+ self._method = method
+ if not url:
+ url = '/'
+ hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+ self._output(hdr)
+
+ if self._http_vsn == 11:
+ # Issue some standard headers for better HTTP/1.1 compliance
+
+ if not skip_host:
+ # this header is issued *only* for HTTP/1.1
+ # connections. more specifically, this means it is
+ # only issued when the client uses the new
+ # HTTPConnection() class. backwards-compat clients
+ # will be using HTTP/1.0 and those clients may be
+ # issuing this header themselves. we should NOT issue
+ # it twice; some web servers (such as Apache) barf
+ # when they see two Host: headers
+
+ # If we need a non-standard port,include it in the
+ # header. If the request is going through a proxy,
+ # but the host of the actual URL, not the host of the
+ # proxy.
+
+ netloc = ''
+ if url.startswith('http'):
+ nil, netloc, nil, nil, nil = urlsplit(url)
+
+ if netloc:
+ try:
+ netloc_enc = netloc.encode("ascii")
+ except UnicodeEncodeError:
+ netloc_enc = netloc.encode("idna")
+ self.putheader('Host', netloc_enc)
+ else:
+ try:
+ host_enc = self.host.encode("ascii")
+ except UnicodeEncodeError:
+ host_enc = self.host.encode("idna")
+ # Wrap the IPv6 Host Header with [] (RFC 2732)
+ if host_enc.find(':') >= 0:
+ host_enc = "[" + host_enc + "]"
+ if self.port == self.default_port:
+ self.putheader('Host', host_enc)
+ else:
+ self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+ # note: we are assuming that clients will not attempt to set these
+ # headers since *this* library must deal with the
+ # consequences. this also means that when the supporting
+ # libraries are updated to recognize other forms, then this
+ # code should be changed (removed or updated).
+
+ # we only want a Content-Encoding of "identity" since we don't
+ # support encodings such as x-gzip or x-deflate.
+ if not skip_accept_encoding:
+ self.putheader('Accept-Encoding', 'identity')
+
+ # we can accept "chunked" Transfer-Encodings, but no others
+ # NOTE: no TE header implies *only* "chunked"
+ #self.putheader('TE', 'chunked')
+
+ # if TE is supplied in the header, then it must appear in a
+ # Connection header.
+ #self.putheader('Connection', 'TE')
+
+ else:
+ # For HTTP/1.0, the server will assume "not chunked"
+ pass
+
+ def putheader(self, header, *values):
+ """Send a request header line to the server.
+
+ For example: h.putheader('Accept', 'text/html')
+ """
+ if self.__state != _CS_REQ_STARTED:
+ raise CannotSendHeader()
+
+ hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+ self._output(hdr)
+
+ def endheaders(self, message_body=None):
+ """Indicate that the last header line has been sent to the server.
+
+ This method sends the request to the server. The optional
+ message_body argument can be used to pass message body
+ associated with the request. The message body will be sent in
+ the same packet as the message headers if possible. The
+ message_body should be a string.
+ """
+ if self.__state == _CS_REQ_STARTED:
+ self.__state = _CS_REQ_SENT
+ else:
+ raise CannotSendHeader()
+ self._send_output(message_body)
+
+ def request(self, method, url, body=None, headers={}):
+ """Send a complete request to the server."""
+ self._send_request(method, url, body, headers)
+
+ def _set_content_length(self, body):
+ # Set the content-length based on the body.
+ thelen = None
+ try:
+ thelen = str(len(body))
+ except TypeError, te:
+ # If this is a file-like object, try to
+ # fstat its file descriptor
+ try:
+ thelen = str(os.fstat(body.fileno()).st_size)
+ except (AttributeError, OSError):
+ # Don't send a length if this failed
+ if self.debuglevel > 0: print "Cannot stat!!"
+
+ if thelen is not None:
+ self.putheader('Content-Length', thelen)
+
+ def _send_request(self, method, url, body, headers):
+ # Honor explicitly requested Host: and Accept-Encoding: headers.
+ header_names = dict.fromkeys([k.lower() for k in headers])
+ skips = {}
+ if 'host' in header_names:
+ skips['skip_host'] = 1
+ if 'accept-encoding' in header_names:
+ skips['skip_accept_encoding'] = 1
+
+ self.putrequest(method, url, **skips)
+
+ if body and ('content-length' not in header_names):
+ self._set_content_length(body)
+ for hdr, value in headers.iteritems():
+ self.putheader(hdr, value)
+ self.endheaders(body)
+
+ def getresponse(self, buffering=False):
+ "Get the response from the server."
+
+ # if a prior response has been completed, then forget about it.
+ if self.__response and self.__response.isclosed():
+ self.__response = None
+
+ #
+ # if a prior response exists, then it must be completed (otherwise, we
+ # cannot read this response's header to determine the connection-close
+ # behavior)
+ #
+ # note: if a prior response existed, but was connection-close, then the
+ # socket and response were made independent of this HTTPConnection
+ # object since a new request requires that we open a whole new
+ # connection
+ #
+ # this means the prior response had one of two states:
+ # 1) will_close: this connection was reset and the prior socket and
+ # response operate independently
+ # 2) persistent: the response was retained and we await its
+ # isclosed() status to become true.
+ #
+ if self.__state != _CS_REQ_SENT or self.__response:
+ raise ResponseNotReady()
+
+ args = (self.sock,)
+ kwds = {"strict":self.strict, "method":self._method}
+ if self.debuglevel > 0:
+ args += (self.debuglevel,)
+ if buffering:
+ #only add this keyword if non-default, for compatibility with
+ #other response_classes.
+ kwds["buffering"] = True;
+ response = self.response_class(*args, **kwds)
+
+ try:
+ response.begin()
+ except:
+ response.close()
+ raise
+ assert response.will_close != _UNKNOWN
+ self.__state = _CS_IDLE
+
+ if response.will_close:
+ # this effectively passes the connection to the response
+ self.close()
+ else:
+ # remember this, so we can tell when it is complete
+ self.__response = response
+
+ return response
+
+
+class HTTP:
+ "Compatibility class with httplib.py from 1.5."
+
+ _http_vsn = 10
+ _http_vsn_str = 'HTTP/1.0'
+
+ debuglevel = 0
+
+ _connection_class = HTTPConnection
+
+ def __init__(self, host='', port=None, strict=None):
+ "Provide a default host, since the superclass requires one."
+
+ # some joker passed 0 explicitly, meaning default port
+ if port == 0:
+ port = None
+
+ # Note that we may pass an empty string as the host; this will throw
+ # an error when we attempt to connect. Presumably, the client code
+ # will call connect before then, with a proper host.
+ self._setup(self._connection_class(host, port, strict))
+
+ def _setup(self, conn):
+ self._conn = conn
+
+ # set up delegation to flesh out interface
+ self.send = conn.send
+ self.putrequest = conn.putrequest
+ self.putheader = conn.putheader
+ self.endheaders = conn.endheaders
+ self.set_debuglevel = conn.set_debuglevel
+
+ conn._http_vsn = self._http_vsn
+ conn._http_vsn_str = self._http_vsn_str
+
+ self.file = None
+
+ def connect(self, host=None, port=None):
+ "Accept arguments to set the host/port, since the superclass doesn't."
+
+ if host is not None:
+ self._conn._set_hostport(host, port)
+ self._conn.connect()
+
+ def getfile(self):
+ "Provide a getfile, since the superclass' does not use this concept."
+ return self.file
+
+ def getreply(self, buffering=False):
+ """Compat definition since superclass does not define it.
+
+ Returns a tuple consisting of:
+ - server status code (e.g. '200' if all goes well)
+ - server "reason" corresponding to status code
+ - any RFC822 headers in the response from the server
+ """
+ try:
+ if not buffering:
+ response = self._conn.getresponse()
+ else:
+ #only add this keyword if non-default for compatibility
+ #with other connection classes
+ response = self._conn.getresponse(buffering)
+ except BadStatusLine, e:
+ ### hmm. if getresponse() ever closes the socket on a bad request,
+ ### then we are going to have problems with self.sock
+
+ ### should we keep this behavior? do people use it?
+ # keep the socket open (as a file), and return it
+ self.file = self._conn.sock.makefile('rb', 0)
+
+ # close our socket -- we want to restart after any protocol error
+ self.close()
+
+ self.headers = None
+ return -1, e.line, None
+
+ self.headers = response.msg
+ self.file = response.fp
+ return response.status, response.reason, response.msg
+
+ def close(self):
+ self._conn.close()
+
+ # note that self.file == response.fp, which gets closed by the
+ # superclass. just clear the object ref here.
+ ### hmm. messy. if status==-1, then self.file is owned by us.
+ ### well... we aren't explicitly closing, but losing this ref will
+ ### do it
+ self.file = None
+
+try:
+ import ssl
+except ImportError:
+ pass
+else:
+ class HTTPSConnection(HTTPConnection):
+ "This class allows communication via SSL."
+
+ default_port = HTTPS_PORT
+
+ def __init__(self, host, port=None, key_file=None, cert_file=None,
+ strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+ source_address=None):
+ HTTPConnection.__init__(self, host, port, strict, timeout,
+ source_address)
+ self.key_file = key_file
+ self.cert_file = cert_file
+
+ def connect(self):
+ "Connect to a host on a given (SSL) port."
+
+ sock = socket.create_connection((self.host, self.port),
+ self.timeout, self.source_address)
+ if self._tunnel_host:
+ self.sock = sock
+ self._tunnel()
+ self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+ __all__.append("HTTPSConnection")
+
+ class HTTPS(HTTP):
+ """Compatibility with 1.5 httplib interface
+
+ Python 1.5.2 did not have an HTTPS class, but it defined an
+ interface for sending http requests that is also useful for
+ https.
+ """
+
+ _connection_class = HTTPSConnection
+
+ def __init__(self, host='', port=None, key_file=None, cert_file=None,
+ strict=None):
+ # provide a default host, pass the X509 cert info
+
+ # urf. compensate for bad input.
+ if port == 0:
+ port = None
+ self._setup(self._connection_class(host, port, key_file,
+ cert_file, strict))
+
+ # we never actually use these for anything, but we keep them
+ # here for compatibility with post-1.5.2 CVS.
+ self.key_file = key_file
+ self.cert_file = cert_file
+
+
+ def FakeSocket (sock, sslobj):
+ warnings.warn("FakeSocket is deprecated, and won't be in 3.x. " +
+ "Use the result of ssl.wrap_socket() directly instead.",
+ DeprecationWarning, stacklevel=2)
+ return sslobj
+
+
+class HTTPException(Exception):
+ # Subclasses that define an __init__ must call Exception.__init__
+ # or define self.args. Otherwise, str() will fail.
+ pass
+
+class NotConnected(HTTPException):
+ pass
+
+class InvalidURL(HTTPException):
+ pass
+
+class UnknownProtocol(HTTPException):
+ def __init__(self, version):
+ self.args = version,
+ self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+ pass
+
+class UnimplementedFileMode(HTTPException):
+ pass
+
+class IncompleteRead(HTTPException):
+ def __init__(self, partial, expected=None):
+ self.args = partial,
+ self.partial = partial
+ self.expected = expected
+ def __repr__(self):
+ if self.expected is not None:
+ e = ', %i more expected' % self.expected
+ else:
+ e = ''
+ return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+ def __str__(self):
+ return repr(self)
+
+class ImproperConnectionState(HTTPException):
+ pass
+
+class CannotSendRequest(ImproperConnectionState):
+ pass
+
+class CannotSendHeader(ImproperConnectionState):
+ pass
+
+class ResponseNotReady(ImproperConnectionState):
+ pass
+
+class BadStatusLine(HTTPException):
+ def __init__(self, line):
+ if not line:
+ line = repr(line)
+ self.args = line,
+ self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+ """A limited file-like object for HTTP/0.9 responses."""
+
+ # The status-line parsing code calls readline(), which normally
+ # get the HTTP status line. For a 0.9 response, however, this is
+ # actually the first line of the body! Clients need to get a
+ # readable file object that contains that line.
+
+ def __init__(self, line, file):
+ self._line = line
+ self._file = file
+ self._line_consumed = 0
+ self._line_offset = 0
+ self._line_left = len(line)
+
+ def __getattr__(self, attr):
+ return getattr(self._file, attr)
+
+ def _done(self):
+ # called when the last byte is read from the line. After the
+ # call, all read methods are delegated to the underlying file
+ # object.
+ self._line_consumed = 1
+ self.read = self._file.read
+ self.readline = self._file.readline
+ self.readlines = self._file.readlines
+
+ def read(self, amt=None):
+ if self._line_consumed:
+ return self._file.read(amt)
+ assert self._line_left
+ if amt is None or amt > self._line_left:
+ s = self._line[self._line_offset:]
+ self._done()
+ if amt is None:
+ return s + self._file.read()
+ else:
+ return s + self._file.read(amt - len(s))
+ else:
+ assert amt <= self._line_left
+ i = self._line_offset
+ j = i + amt
+ s = self._line[i:j]
+ self._line_offset = j
+ self._line_left -= amt
+ if self._line_left == 0:
+ self._done()
+ return s
+
+ def readline(self):
+ if self._line_consumed:
+ return self._file.readline()
+ assert self._line_left
+ s = self._line[self._line_offset:]
+ self._done()
+ return s
+
+ def readlines(self, size=None):
+ if self._line_consumed:
+ return self._file.readlines(size)
+ assert self._line_left
+ L = [self._line[self._line_offset:]]
+ self._done()
+ if size is None:
+ return L + self._file.readlines()
+ else:
+ return L + self._file.readlines(size)
+
+def test():
+ """Test this module.
+
+ A hodge podge of tests collected here, because they have too many
+ external dependencies for the regular test suite.
+ """
+
+ import sys
+ import getopt
+ opts, args = getopt.getopt(sys.argv[1:], 'd')
+ dl = 0
+ for o, a in opts:
+ if o == '-d': dl = dl + 1
+ host = 'www.python.org'
+ selector = '/'
+ if args[0:]: host = args[0]
+ if args[1:]: selector = args[1]
+ h = HTTP()
+ h.set_debuglevel(dl)
+ h.connect(host)
+ h.putrequest('GET', selector)
+ h.endheaders()
+ status, reason, headers = h.getreply()
+ print 'status =', status
+ print 'reason =', reason
+ print "read", len(h.getfile().read())
+ print
+ if headers:
+ for header in headers.headers: print header.strip()
+ print
+
+ # minimal test that code to extract host from url works
+ class HTTP11(HTTP):
+ _http_vsn = 11
+ _http_vsn_str = 'HTTP/1.1'
+
+ h = HTTP11('www.python.org')
+ h.putrequest('GET', 'http://www.python.org/~jeremy/')
+ h.endheaders()
+ h.getreply()
+ h.close()
+
+ try:
+ import ssl
+ except ImportError:
+ pass
+ else:
+
+ for host, selector in (('sourceforge.net', '/projects/python'),
+ ):
+ print "https://%s%s" % (host, selector)
+ hs = HTTPS()
+ hs.set_debuglevel(dl)
+ hs.connect(host)
+ hs.putrequest('GET', selector)
+ hs.endheaders()
+ status, reason, headers = hs.getreply()
+ print 'status =', status
+ print 'reason =', reason
+ print "read", len(hs.getfile().read())
+ print
+ if headers:
+ for header in headers.headers: print header.strip()
+ print
+
+if __name__ == '__main__':
+ test()
diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
)
b = array.array(self.badtypecode())
- self.assertRaises(TypeError, "a + b")
-
- self.assertRaises(TypeError, "a + 'bad'")
+ with self.assertRaises(TypeError):
+ a + b
+ with self.assertRaises(TypeError):
+ a + 'bad'
def test_iadd(self):
a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
)
b = array.array(self.badtypecode())
- self.assertRaises(TypeError, "a += b")
-
- self.assertRaises(TypeError, "a += 'bad'")
+ with self.assertRaises(TypeError):
+ a += b
+ with self.assertRaises(TypeError):
+ a += 'bad'
def test_mul(self):
a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
array.array(self.typecode)
)
- self.assertRaises(TypeError, "a * 'bad'")
+ with self.assertRaises(TypeError):
+ a * 'bad'
def test_imul(self):
a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
a *= -1
self.assertEqual(a, array.array(self.typecode))
- self.assertRaises(TypeError, "a *= 'bad'")
+ with self.assertRaises(TypeError):
+ a *= 'bad'
def test_getitem(self):
a = array.array(self.typecode, self.example)
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
def getresponse(self):
return MockHTTPResponse(MockFile(), {}, 200, "OK")
+ def close(self):
+ pass
+
class MockHandler:
# useful for testing handler machinery
# see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1440 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below). It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work. Each Handler implements a particular protocol or
+option. The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL. For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns. The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib. pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back. One difference is that you can also pass
+a Request instance instead of URL. Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers. Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate. If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request. The
+state can be as simple as the URL. It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+ uri='https://mahler:8092/site-updates.py',
+ user='klem',
+ passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled? The client needs to know the HTTP error code. But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+ addinfourl, splitport, splittag,
+ splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+ global _opener
+ if _opener is None:
+ _opener = build_opener()
+ return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+ global _opener
+ _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden. we just want to be
+# subtypes.
+
+class URLError(IOError):
+ # URLError is a sub-type of IOError, but it doesn't share any of
+ # the implementation. need to override __init__ and __str__.
+ # It sets self.args for compatibility with other EnvironmentError
+ # subclasses, but args doesn't have the typical format with errno in
+ # slot 0 and strerror in slot 1. This may be better than nothing.
+ def __init__(self, reason):
+ self.args = reason,
+ self.reason = reason
+
+ def __str__(self):
+ return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+ """Raised when HTTP error occurs, but also acts like non-error return"""
+ __super_init = addinfourl.__init__
+
+ def __init__(self, url, code, msg, hdrs, fp):
+ self.code = code
+ self.msg = msg
+ self.hdrs = hdrs
+ self.fp = fp
+ self.filename = url
+ # The addinfourl classes depend on fp being a valid file
+ # object. In some cases, the HTTPError may not have a valid
+ # file object. If this happens, the simplest workaround is to
+ # not initialize the base classes.
+ if fp is not None:
+ self.__super_init(fp, hdrs, url, code)
+
+ def __str__(self):
+ return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+ """Return request-host, as defined by RFC 2965.
+
+ Variation from RFC: returned value is lowercased, for convenient
+ comparison.
+
+ """
+ url = request.get_full_url()
+ host = urlparse.urlparse(url)[1]
+ if host == "":
+ host = request.get_header("Host", "")
+
+ # remove port, if present
+ host = _cut_port_re.sub("", host, 1)
+ return host.lower()
+
+class Request:
+
+ def __init__(self, url, data=None, headers={},
+ origin_req_host=None, unverifiable=False):
+ # unwrap('<URL:type://host/path>') --> 'type://host/path'
+ self.__original = unwrap(url)
+ self.__original, fragment = splittag(self.__original)
+ self.type = None
+ # self.__r_type is what's left after doing the splittype
+ self.host = None
+ self.port = None
+ self._tunnel_host = None
+ self.data = data
+ self.headers = {}
+ for key, value in headers.items():
+ self.add_header(key, value)
+ self.unredirected_hdrs = {}
+ if origin_req_host is None:
+ origin_req_host = request_host(self)
+ self.origin_req_host = origin_req_host
+ self.unverifiable = unverifiable
+
+ def __getattr__(self, attr):
+ # XXX this is a fallback mechanism to guard against these
+ # methods getting called in a non-standard order. this may be
+ # too complicated and/or unnecessary.
+ # XXX should the __r_XXX attributes be public?
+ if attr[:12] == '_Request__r_':
+ name = attr[12:]
+ if hasattr(Request, 'get_' + name):
+ getattr(self, 'get_' + name)()
+ return getattr(self, attr)
+ raise AttributeError, attr
+
+ def get_method(self):
+ if self.has_data():
+ return "POST"
+ else:
+ return "GET"
+
+ # XXX these helper methods are lame
+
+ def add_data(self, data):
+ self.data = data
+
+ def has_data(self):
+ return self.data is not None
+
+ def get_data(self):
+ return self.data
+
+ def get_full_url(self):
+ return self.__original
+
+ def get_type(self):
+ if self.type is None:
+ self.type, self.__r_type = splittype(self.__original)
+ if self.type is None:
+ raise ValueError, "unknown url type: %s" % self.__original
+ return self.type
+
+ def get_host(self):
+ if self.host is None:
+ self.host, self.__r_host = splithost(self.__r_type)
+ if self.host:
+ self.host = unquote(self.host)
+ return self.host
+
+ def get_selector(self):
+ return self.__r_host
+
+ def set_proxy(self, host, type):
+ if self.type == 'https' and not self._tunnel_host:
+ self._tunnel_host = self.host
+ else:
+ self.type = type
+ self.__r_host = self.__original
+
+ self.host = host
+
+ def has_proxy(self):
+ return self.__r_host == self.__original
+
+ def get_origin_req_host(self):
+ return self.origin_req_host
+
+ def is_unverifiable(self):
+ return self.unverifiable
+
+ def add_header(self, key, val):
+ # useful for something like authentication
+ self.headers[key.capitalize()] = val
+
+ def add_unredirected_header(self, key, val):
+ # will not be added to a redirected request
+ self.unredirected_hdrs[key.capitalize()] = val
+
+ def has_header(self, header_name):
+ return (header_name in self.headers or
+ header_name in self.unredirected_hdrs)
+
+ def get_header(self, header_name, default=None):
+ return self.headers.get(
+ header_name,
+ self.unredirected_hdrs.get(header_name, default))
+
+ def header_items(self):
+ hdrs = self.unredirected_hdrs.copy()
+ hdrs.update(self.headers)
+ return hdrs.items()
+
+class OpenerDirector:
+ def __init__(self):
+ client_version = "Python-urllib/%s" % __version__
+ self.addheaders = [('User-agent', client_version)]
+ # manage the individual handlers
+ self.handlers = []
+ self.handle_open = {}
+ self.handle_error = {}
+ self.process_response = {}
+ self.process_request = {}
+
+ def add_handler(self, handler):
+ if not hasattr(handler, "add_parent"):
+ raise TypeError("expected BaseHandler instance, got %r" %
+ type(handler))
+
+ added = False
+ for meth in dir(handler):
+ if meth in ["redirect_request", "do_open", "proxy_open"]:
+ # oops, coincidental match
+ continue
+
+ i = meth.find("_")
+ protocol = meth[:i]
+ condition = meth[i+1:]
+
+ if condition.startswith("error"):
+ j = condition.find("_") + i + 1
+ kind = meth[j+1:]
+ try:
+ kind = int(kind)
+ except ValueError:
+ pass
+ lookup = self.handle_error.get(protocol, {})
+ self.handle_error[protocol] = lookup
+ elif condition == "open":
+ kind = protocol
+ lookup = self.handle_open
+ elif condition == "response":
+ kind = protocol
+ lookup = self.process_response
+ elif condition == "request":
+ kind = protocol
+ lookup = self.process_request
+ else:
+ continue
+
+ handlers = lookup.setdefault(kind, [])
+ if handlers:
+ bisect.insort(handlers, handler)
+ else:
+ handlers.append(handler)
+ added = True
+
+ if added:
+ # the handlers must work in an specific order, the order
+ # is specified in a Handler attribute
+ bisect.insort(self.handlers, handler)
+ handler.add_parent(self)
+
+ def close(self):
+ # Only exists for backwards compatibility.
+ pass
+
+ def _call_chain(self, chain, kind, meth_name, *args):
+ # Handlers raise an exception if no one else should try to handle
+ # the request, or return None if they can't but another handler
+ # could. Otherwise, they return the response.
+ handlers = chain.get(kind, ())
+ for handler in handlers:
+ func = getattr(handler, meth_name)
+
+ result = func(*args)
+ if result is not None:
+ return result
+
+ def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+ # accept a URL or a Request object
+ if isinstance(fullurl, basestring):
+ req = Request(fullurl, data)
+ else:
+ req = fullurl
+ if data is not None:
+ req.add_data(data)
+
+ req.timeout = timeout
+ protocol = req.get_type()
+
+ # pre-process request
+ meth_name = protocol+"_request"
+ for processor in self.process_request.get(protocol, []):
+ meth = getattr(processor, meth_name)
+ req = meth(req)
+
+ response = self._open(req, data)
+
+ # post-process response
+ meth_name = protocol+"_response"
+ for processor in self.process_response.get(protocol, []):
+ meth = getattr(processor, meth_name)
+ try:
+ response = meth(req, response)
+ except:
+ response.close()
+ raise
+
+ return response
+
+ def _open(self, req, data=None):
+ result = self._call_chain(self.handle_open, 'default',
+ 'default_open', req)
+ if result:
+ return result
+
+ protocol = req.get_type()
+ result = self._call_chain(self.handle_open, protocol, protocol +
+ '_open', req)
+ if result:
+ return result
+
+ return self._call_chain(self.handle_open, 'unknown',
+ 'unknown_open', req)
+
+ def error(self, proto, *args):
+ if proto in ('http', 'https'):
+ # XXX http[s] protocols are special-cased
+ dict = self.handle_error['http'] # https is not different than http
+ proto = args[2] # YUCK!
+ meth_name = 'http_error_%s' % proto
+ http_err = 1
+ orig_args = args
+ else:
+ dict = self.handle_error
+ meth_name = proto + '_error'
+ http_err = 0
+ args = (dict, proto, meth_name) + args
+ result = self._call_chain(*args)
+ if result:
+ return result
+
+ if http_err:
+ args = (dict, 'default', 'http_error_default') + orig_args
+ return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+ """Create an opener object from a list of handlers.
+
+ The opener will use several default handlers, including support
+ for HTTP, FTP and when applicable, HTTPS.
+
+ If any of the handlers passed as arguments are subclasses of the
+ default handlers, the default handlers will not be used.
+ """
+ import types
+ def isclass(obj):
+ return isinstance(obj, (types.ClassType, type))
+
+ opener = OpenerDirector()
+ default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+ HTTPDefaultErrorHandler, HTTPRedirectHandler,
+ FTPHandler, FileHandler, HTTPErrorProcessor]
+ if hasattr(httplib, 'HTTPS'):
+ default_classes.append(HTTPSHandler)
+ skip = set()
+ for klass in default_classes:
+ for check in handlers:
+ if isclass(check):
+ if issubclass(check, klass):
+ skip.add(klass)
+ elif isinstance(check, klass):
+ skip.add(klass)
+ for klass in skip:
+ default_classes.remove(klass)
+
+ for klass in default_classes:
+ opener.add_handler(klass())
+
+ for h in handlers:
+ if isclass(h):
+ h = h()
+ opener.add_handler(h)
+ return opener
+
+class BaseHandler:
+ handler_order = 500
+
+ def add_parent(self, parent):
+ self.parent = parent
+
+ def close(self):
+ # Only exists for backwards compatibility
+ pass
+
+ def __lt__(self, other):
+ if not hasattr(other, "handler_order"):
+ # Try to preserve the old behavior of having custom classes
+ # inserted after default ones (works only for custom user
+ # classes which are not aware of handler_order).
+ return True
+ return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+ """Process HTTP error responses."""
+ handler_order = 1000 # after all other processing
+
+ def http_response(self, request, response):
+ code, msg, hdrs = response.code, response.msg, response.info()
+
+ # According to RFC 2616, "2xx" code indicates that the client's
+ # request was successfully received, understood, and accepted.
+ if not (200 <= code < 300):
+ response = self.parent.error(
+ 'http', request, response, code, msg, hdrs)
+
+ return response
+
+ https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+ def http_error_default(self, req, fp, code, msg, hdrs):
+ raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+ # maximum number of redirections to any single URL
+ # this is needed because of the state that cookies introduce
+ max_repeats = 4
+ # maximum total number of redirections (regardless of URL) before
+ # assuming we're in a loop
+ max_redirections = 10
+
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ """Return a Request or None in response to a redirect.
+
+ This is called by the http_error_30x methods when a
+ redirection response is received. If a redirection should
+ take place, return a new Request to allow http_error_30x to
+ perform the redirect. Otherwise, raise HTTPError if no-one
+ else should try to handle this url. Return None if you can't
+ but another Handler might.
+ """
+ m = req.get_method()
+ if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+ or code in (301, 302, 303) and m == "POST"):
+ # Strictly (according to RFC 2616), 301 or 302 in response
+ # to a POST MUST NOT cause a redirection without confirmation
+ # from the user (of urllib2, in this case). In practice,
+ # essentially all clients do redirect in this case, so we
+ # do the same.
+ # be conciliant with URIs containing a space
+ newurl = newurl.replace(' ', '%20')
+ newheaders = dict((k,v) for k,v in req.headers.items()
+ if k.lower() not in ("content-length", "content-type")
+ )
+ return Request(newurl,
+ headers=newheaders,
+ origin_req_host=req.get_origin_req_host(),
+ unverifiable=True)
+ else:
+ raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+ # Implementation note: To avoid the server sending us into an
+ # infinite loop, the request object needs to track what URLs we
+ # have already seen. Do this by adding a handler-specific
+ # attribute to the Request object.
+ def http_error_302(self, req, fp, code, msg, headers):
+ # Some servers (incorrectly) return multiple Location headers
+ # (so probably same goes for URI). Use first header.
+ if 'location' in headers:
+ newurl = headers.getheaders('location')[0]
+ elif 'uri' in headers:
+ newurl = headers.getheaders('uri')[0]
+ else:
+ return
+
+ # fix a possible malformed URL
+ urlparts = urlparse.urlparse(newurl)
+ if not urlparts.path:
+ urlparts = list(urlparts)
+ urlparts[2] = "/"
+ newurl = urlparse.urlunparse(urlparts)
+
+ newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+ # XXX Probably want to forget about the state of the current
+ # request, although that might interact poorly with other
+ # handlers that also use handler-specific request attributes
+ new = self.redirect_request(req, fp, code, msg, headers, newurl)
+ if new is None:
+ return
+
+ # loop detection
+ # .redirect_dict has a key url if url was previously visited.
+ if hasattr(req, 'redirect_dict'):
+ visited = new.redirect_dict = req.redirect_dict
+ if (visited.get(newurl, 0) >= self.max_repeats or
+ len(visited) >= self.max_redirections):
+ raise HTTPError(req.get_full_url(), code,
+ self.inf_msg + msg, headers, fp)
+ else:
+ visited = new.redirect_dict = req.redirect_dict = {}
+ visited[newurl] = visited.get(newurl, 0) + 1
+
+ # Don't close the fp until we are sure that we won't use it
+ # with HTTPError.
+ fp.read()
+ fp.close()
+
+ return self.parent.open(new, timeout=req.timeout)
+
+ http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+ inf_msg = "The HTTP server returned a redirect error that would " \
+ "lead to an infinite loop.\n" \
+ "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+ """Return (scheme, user, password, host/port) given a URL or an authority.
+
+ If a URL is supplied, it must have an authority (host:port) component.
+ According to RFC 3986, having an authority component means the URL must
+ have two slashes after the scheme:
+
+ >>> _parse_proxy('file:/ftp.example.com/')
+ Traceback (most recent call last):
+ ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+ The first three items of the returned tuple may be None.
+
+ Examples of authority parsing:
+
+ >>> _parse_proxy('proxy.example.com')
+ (None, None, None, 'proxy.example.com')
+ >>> _parse_proxy('proxy.example.com:3128')
+ (None, None, None, 'proxy.example.com:3128')
+
+ The authority component may optionally include userinfo (assumed to be
+ username:password):
+
+ >>> _parse_proxy('joe:password at proxy.example.com')
+ (None, 'joe', 'password', 'proxy.example.com')
+ >>> _parse_proxy('joe:password at proxy.example.com:3128')
+ (None, 'joe', 'password', 'proxy.example.com:3128')
+
+ Same examples, but with URLs instead:
+
+ >>> _parse_proxy('http://proxy.example.com/')
+ ('http', None, None, 'proxy.example.com')
+ >>> _parse_proxy('http://proxy.example.com:3128/')
+ ('http', None, None, 'proxy.example.com:3128')
+ >>> _parse_proxy('http://joe:password@proxy.example.com/')
+ ('http', 'joe', 'password', 'proxy.example.com')
+ >>> _parse_proxy('http://joe:password@proxy.example.com:3128')
+ ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+ Everything after the authority is ignored:
+
+ >>> _parse_proxy('ftp://joe:password@proxy.example.com/rubbish:3128')
+ ('ftp', 'joe', 'password', 'proxy.example.com')
+
+ Test for no trailing '/' case:
+
+ >>> _parse_proxy('http://joe:password@proxy.example.com')
+ ('http', 'joe', 'password', 'proxy.example.com')
+
+ """
+ scheme, r_scheme = splittype(proxy)
+ if not r_scheme.startswith("/"):
+ # authority
+ scheme = None
+ authority = proxy
+ else:
+ # URL
+ if not r_scheme.startswith("//"):
+ raise ValueError("proxy URL with no authority: %r" % proxy)
+ # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+ # and 3.3.), path is empty or starts with '/'
+ end = r_scheme.find("/", 2)
+ if end == -1:
+ end = None
+ authority = r_scheme[2:end]
+ userinfo, hostport = splituser(authority)
+ if userinfo is not None:
+ user, password = splitpasswd(userinfo)
+ else:
+ user = password = None
+ return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+ # Proxies must be in front
+ handler_order = 100
+
+ def __init__(self, proxies=None):
+ if proxies is None:
+ proxies = getproxies()
+ assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+ self.proxies = proxies
+ for type, url in proxies.items():
+ setattr(self, '%s_open' % type,
+ lambda r, proxy=url, type=type, meth=self.proxy_open: \
+ meth(r, proxy, type))
+
+ def proxy_open(self, req, proxy, type):
+ orig_type = req.get_type()
+ proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+ if proxy_type is None:
+ proxy_type = orig_type
+
+ if req.host and proxy_bypass(req.host):
+ return None
+
+ if user and password:
+ user_pass = '%s:%s' % (unquote(user), unquote(password))
+ creds = base64.b64encode(user_pass).strip()
+ req.add_header('Proxy-authorization', 'Basic ' + creds)
+ hostport = unquote(hostport)
+ req.set_proxy(hostport, proxy_type)
+
+ if orig_type == proxy_type or orig_type == 'https':
+ # let other handlers take care of it
+ return None
+ else:
+ # need to start over, because the other handlers don't
+ # grok the proxy's URL type
+ # e.g. if we have a constructor arg proxies like so:
+ # {'http': 'ftp://proxy.example.com'}, we may end up turning
+ # a request for http://acme.example.com/a into one for
+ # ftp://proxy.example.com/a
+ return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+ def __init__(self):
+ self.passwd = {}
+
+ def add_password(self, realm, uri, user, passwd):
+ # uri could be a single URI or a sequence
+ if isinstance(uri, basestring):
+ uri = [uri]
+ if not realm in self.passwd:
+ self.passwd[realm] = {}
+ for default_port in True, False:
+ reduced_uri = tuple(
+ [self.reduce_uri(u, default_port) for u in uri])
+ self.passwd[realm][reduced_uri] = (user, passwd)
+
+ def find_user_password(self, realm, authuri):
+ domains = self.passwd.get(realm, {})
+ for default_port in True, False:
+ reduced_authuri = self.reduce_uri(authuri, default_port)
+ for uris, authinfo in domains.iteritems():
+ for uri in uris:
+ if self.is_suburi(uri, reduced_authuri):
+ return authinfo
+ return None, None
+
+ def reduce_uri(self, uri, default_port=True):
+ """Accept authority or URI and extract only the authority and path."""
+ # note HTTP URLs do not have a userinfo component
+ parts = urlparse.urlsplit(uri)
+ if parts[1]:
+ # URI
+ scheme = parts[0]
+ authority = parts[1]
+ path = parts[2] or '/'
+ else:
+ # host or host:port
+ scheme = None
+ authority = uri
+ path = '/'
+ host, port = splitport(authority)
+ if default_port and port is None and scheme is not None:
+ dport = {"http": 80,
+ "https": 443,
+ }.get(scheme)
+ if dport is not None:
+ authority = "%s:%d" % (host, dport)
+ return authority, path
+
+ def is_suburi(self, base, test):
+ """Check if test is below base in a URI tree
+
+ Both args must be URIs in reduced form.
+ """
+ if base == test:
+ return True
+ if base[0] != test[0]:
+ return False
+ common = posixpath.commonprefix((base[1], test[1]))
+ if len(common) == len(base[1]):
+ return True
+ return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+ def find_user_password(self, realm, authuri):
+ user, password = HTTPPasswordMgr.find_user_password(self, realm,
+ authuri)
+ if user is not None:
+ return user, password
+ return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+ # XXX this allows for multiple auth-schemes, but will stupidly pick
+ # the last one with a realm specified.
+
+ # allow for double- and single-quoted realm values
+ # (single quotes are a violation of the RFC, but appear in the wild)
+ rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+ 'realm=(["\'])(.*?)\\2', re.I)
+
+ # XXX could pre-emptively send auth info already accepted (RFC 2617,
+ # end of section 2, and section 1.2 immediately after "credentials"
+ # production).
+
+ def __init__(self, password_mgr=None):
+ if password_mgr is None:
+ password_mgr = HTTPPasswordMgr()
+ self.passwd = password_mgr
+ self.add_password = self.passwd.add_password
+ self.retried = 0
+
+ def reset_retry_count(self):
+ self.retried = 0
+
+ def http_error_auth_reqed(self, authreq, host, req, headers):
+ # host may be an authority (without userinfo) or a URL with an
+ # authority
+ # XXX could be multiple headers
+ authreq = headers.get(authreq, None)
+
+ if self.retried > 5:
+ # retry sending the username:password 5 times before failing.
+ raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+ headers, None)
+ else:
+ self.retried += 1
+
+ if authreq:
+ mo = AbstractBasicAuthHandler.rx.search(authreq)
+ if mo:
+ scheme, quote, realm = mo.groups()
+ if scheme.lower() == 'basic':
+ response = self.retry_http_basic_auth(host, req, realm)
+ if response and response.code != 401:
+ self.retried = 0
+ return response
+
+ def retry_http_basic_auth(self, host, req, realm):
+ user, pw = self.passwd.find_user_password(realm, host)
+ if pw is not None:
+ raw = "%s:%s" % (user, pw)
+ auth = 'Basic %s' % base64.b64encode(raw).strip()
+ if req.headers.get(self.auth_header, None) == auth:
+ return None
+ req.add_unredirected_header(self.auth_header, auth)
+ return self.parent.open(req, timeout=req.timeout)
+ else:
+ return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+ auth_header = 'Authorization'
+
+ def http_error_401(self, req, fp, code, msg, headers):
+ url = req.get_full_url()
+ response = self.http_error_auth_reqed('www-authenticate',
+ url, req, headers)
+ self.reset_retry_count()
+ return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+ auth_header = 'Proxy-authorization'
+
+ def http_error_407(self, req, fp, code, msg, headers):
+ # http_error_auth_reqed requires that there is no userinfo component in
+ # authority. Assume there isn't one, since urllib2 does not (and
+ # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+ # userinfo.
+ authority = req.get_host()
+ response = self.http_error_auth_reqed('proxy-authenticate',
+ authority, req, headers)
+ self.reset_retry_count()
+ return response
+
+
+def randombytes(n):
+ """Return n random bytes."""
+ # Use /dev/urandom if it is available. Fall back to random module
+ # if not. It might be worthwhile to extend this function to use
+ # other platform-specific mechanisms for getting random bytes.
+ if os.path.exists("/dev/urandom"):
+ f = open("/dev/urandom")
+ s = f.read(n)
+ f.close()
+ return s
+ else:
+ L = [chr(random.randrange(0, 256)) for i in range(n)]
+ return "".join(L)
+
+class AbstractDigestAuthHandler:
+ # Digest authentication is specified in RFC 2617.
+
+ # XXX The client does not inspect the Authentication-Info header
+ # in a successful response.
+
+ # XXX It should be possible to test this implementation against
+ # a mock server that just generates a static set of challenges.
+
+ # XXX qop="auth-int" supports is shaky
+
+ def __init__(self, passwd=None):
+ if passwd is None:
+ passwd = HTTPPasswordMgr()
+ self.passwd = passwd
+ self.add_password = self.passwd.add_password
+ self.retried = 0
+ self.nonce_count = 0
+ self.last_nonce = None
+
+ def reset_retry_count(self):
+ self.retried = 0
+
+ def http_error_auth_reqed(self, auth_header, host, req, headers):
+ authreq = headers.get(auth_header, None)
+ if self.retried > 5:
+ # Don't fail endlessly - if we failed once, we'll probably
+ # fail a second time. Hm. Unless the Password Manager is
+ # prompting for the information. Crap. This isn't great
+ # but it's better than the current 'repeat until recursion
+ # depth exceeded' approach <wink>
+ raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+ headers, None)
+ else:
+ self.retried += 1
+ if authreq:
+ scheme = authreq.split()[0]
+ if scheme.lower() == 'digest':
+ return self.retry_http_digest_auth(req, authreq)
+
+ def retry_http_digest_auth(self, req, auth):
+ token, challenge = auth.split(' ', 1)
+ chal = parse_keqv_list(parse_http_list(challenge))
+ auth = self.get_authorization(req, chal)
+ if auth:
+ auth_val = 'Digest %s' % auth
+ if req.headers.get(self.auth_header, None) == auth_val:
+ return None
+ req.add_unredirected_header(self.auth_header, auth_val)
+ resp = self.parent.open(req, timeout=req.timeout)
+ return resp
+
+ def get_cnonce(self, nonce):
+ # The cnonce-value is an opaque
+ # quoted string value provided by the client and used by both client
+ # and server to avoid chosen plaintext attacks, to provide mutual
+ # authentication, and to provide some message integrity protection.
+ # This isn't a fabulous effort, but it's probably Good Enough.
+ dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+ randombytes(8))).hexdigest()
+ return dig[:16]
+
+ def get_authorization(self, req, chal):
+ try:
+ realm = chal['realm']
+ nonce = chal['nonce']
+ qop = chal.get('qop')
+ algorithm = chal.get('algorithm', 'MD5')
+ # mod_digest doesn't send an opaque, even though it isn't
+ # supposed to be optional
+ opaque = chal.get('opaque', None)
+ except KeyError:
+ return None
+
+ H, KD = self.get_algorithm_impls(algorithm)
+ if H is None:
+ return None
+
+ user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+ if user is None:
+ return None
+
+ # XXX not implemented yet
+ if req.has_data():
+ entdig = self.get_entity_digest(req.get_data(), chal)
+ else:
+ entdig = None
+
+ A1 = "%s:%s:%s" % (user, realm, pw)
+ A2 = "%s:%s" % (req.get_method(),
+ # XXX selector: what about proxies and full urls
+ req.get_selector())
+ if qop == 'auth':
+ if nonce == self.last_nonce:
+ self.nonce_count += 1
+ else:
+ self.nonce_count = 1
+ self.last_nonce = nonce
+
+ ncvalue = '%08x' % self.nonce_count
+ cnonce = self.get_cnonce(nonce)
+ noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+ respdig = KD(H(A1), noncebit)
+ elif qop is None:
+ respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+ else:
+ # XXX handle auth-int.
+ raise URLError("qop '%s' is not supported." % qop)
+
+ # XXX should the partial digests be encoded too?
+
+ base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+ 'response="%s"' % (user, realm, nonce, req.get_selector(),
+ respdig)
+ if opaque:
+ base += ', opaque="%s"' % opaque
+ if entdig:
+ base += ', digest="%s"' % entdig
+ base += ', algorithm="%s"' % algorithm
+ if qop:
+ base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+ return base
+
+ def get_algorithm_impls(self, algorithm):
+ # algorithm should be case-insensitive according to RFC2617
+ algorithm = algorithm.upper()
+ # lambdas assume digest modules are imported at the top level
+ if algorithm == 'MD5':
+ H = lambda x: hashlib.md5(x).hexdigest()
+ elif algorithm == 'SHA':
+ H = lambda x: hashlib.sha1(x).hexdigest()
+ # XXX MD5-sess
+ KD = lambda s, d: H("%s:%s" % (s, d))
+ return H, KD
+
+ def get_entity_digest(self, data, chal):
+ # XXX not implemented yet
+ return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+ """An authentication protocol defined by RFC 2069
+
+ Digest authentication improves on basic authentication because it
+ does not transmit passwords in the clear.
+ """
+
+ auth_header = 'Authorization'
+ handler_order = 490 # before Basic auth
+
+ def http_error_401(self, req, fp, code, msg, headers):
+ host = urlparse.urlparse(req.get_full_url())[1]
+ retry = self.http_error_auth_reqed('www-authenticate',
+ host, req, headers)
+ self.reset_retry_count()
+ return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+ auth_header = 'Proxy-Authorization'
+ handler_order = 490 # before Basic auth
+
+ def http_error_407(self, req, fp, code, msg, headers):
+ host = req.get_host()
+ retry = self.http_error_auth_reqed('proxy-authenticate',
+ host, req, headers)
+ self.reset_retry_count()
+ return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+ def __init__(self, debuglevel=0):
+ self._debuglevel = debuglevel
+
+ def set_http_debuglevel(self, level):
+ self._debuglevel = level
+
+ def do_request_(self, request):
+ host = request.get_host()
+ if not host:
+ raise URLError('no host given')
+
+ if request.has_data(): # POST
+ data = request.get_data()
+ if not request.has_header('Content-type'):
+ request.add_unredirected_header(
+ 'Content-type',
+ 'application/x-www-form-urlencoded')
+ if not request.has_header('Content-length'):
+ request.add_unredirected_header(
+ 'Content-length', '%d' % len(data))
+
+ sel_host = host
+ if request.has_proxy():
+ scheme, sel = splittype(request.get_selector())
+ sel_host, sel_path = splithost(sel)
+
+ if not request.has_header('Host'):
+ request.add_unredirected_header('Host', sel_host)
+ for name, value in self.parent.addheaders:
+ name = name.capitalize()
+ if not request.has_header(name):
+ request.add_unredirected_header(name, value)
+
+ return request
+
+ def do_open(self, http_class, req):
+ """Return an addinfourl object for the request, using http_class.
+
+ http_class must implement the HTTPConnection API from httplib.
+ The addinfourl return value is a file-like object. It also
+ has methods and attributes including:
+ - info(): return a mimetools.Message object for the headers
+ - geturl(): return the original request URL
+ - code: HTTP status code
+ """
+ host = req.get_host()
+ if not host:
+ raise URLError('no host given')
+
+ h = http_class(host, timeout=req.timeout) # will parse host:port
+ h.set_debuglevel(self._debuglevel)
+
+ headers = dict(req.unredirected_hdrs)
+ headers.update(dict((k, v) for k, v in req.headers.items()
+ if k not in headers))
+
+ # We want to make an HTTP/1.1 request, but the addinfourl
+ # class isn't prepared to deal with a persistent connection.
+ # It will try to read all remaining data from the socket,
+ # which will block while the server waits for the next request.
+ # So make sure the connection gets closed after the (only)
+ # request.
+ headers["Connection"] = "close"
+ headers = dict(
+ (name.title(), val) for name, val in headers.items())
+
+ if req._tunnel_host:
+ tunnel_headers = {}
+ proxy_auth_hdr = "Proxy-Authorization"
+ if proxy_auth_hdr in headers:
+ tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+ # Proxy-Authorization should not be sent to origin
+ # server.
+ del headers[proxy_auth_hdr]
+ h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+ try:
+ h.request(req.get_method(), req.get_selector(), req.data, headers)
+ try:
+ r = h.getresponse(buffering=True)
+ except TypeError: #buffering kw not supported
+ r = h.getresponse()
+ except socket.error, err: # XXX what error?
+ h.close()
+ raise URLError(err)
+
+ # Pick apart the HTTPResponse object to get the addinfourl
+ # object initialized properly.
+
+ # Wrap the HTTPResponse object in socket's file object adapter
+ # for Windows. That adapter calls recv(), so delegate recv()
+ # to read(). This weird wrapping allows the returned object to
+ # have readline() and readlines() methods.
+
+ # XXX It might be better to extract the read buffering code
+ # out of socket._fileobject() and into a base class.
+
+ r.recv = r.read
+ fp = socket._fileobject(r, close=True)
+
+ resp = addinfourl(fp, r.msg, req.get_full_url())
+ resp.code = r.status
+ resp.msg = r.reason
+ return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+ def http_open(self, req):
+ return self.do_open(httplib.HTTPConnection, req)
+
+ http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+ class HTTPSHandler(AbstractHTTPHandler):
+
+ def https_open(self, req):
+ return self.do_open(httplib.HTTPSConnection, req)
+
+ https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+ def __init__(self, cookiejar=None):
+ import cookielib
+ if cookiejar is None:
+ cookiejar = cookielib.CookieJar()
+ self.cookiejar = cookiejar
+
+ def http_request(self, request):
+ self.cookiejar.add_cookie_header(request)
+ return request
+
+ def http_response(self, request, response):
+ self.cookiejar.extract_cookies(response, request)
+ return response
+
+ https_request = http_request
+ https_response = http_response
+
+class UnknownHandler(BaseHandler):
+ def unknown_open(self, req):
+ type = req.get_type()
+ raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+ """Parse list of key=value strings where keys are not duplicated."""
+ parsed = {}
+ for elt in l:
+ k, v = elt.split('=', 1)
+ if v[0] == '"' and v[-1] == '"':
+ v = v[1:-1]
+ parsed[k] = v
+ return parsed
+
+def parse_http_list(s):
+ """Parse lists as described by RFC 2068 Section 2.
+
+ In particular, parse comma-separated lists where the elements of
+ the list may include quoted-strings. A quoted-string could
+ contain a comma. A non-quoted string could have quotes in the
+ middle. Neither commas nor quotes count if they are escaped.
+ Only double-quotes count, not single-quotes.
+ """
+ res = []
+ part = ''
+
+ escape = quote = False
+ for cur in s:
+ if escape:
+ part += cur
+ escape = False
+ continue
+ if quote:
+ if cur == '\\':
+ escape = True
+ continue
+ elif cur == '"':
+ quote = False
+ part += cur
+ continue
+
+ if cur == ',':
+ res.append(part)
+ part = ''
+ continue
+
+ if cur == '"':
+ quote = True
+
+ part += cur
+
+ # append last part
+ if part:
+ res.append(part)
+
+ return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+ try:
+ return socket.gethostbyname(host)
+ except socket.gaierror:
+ return None
+
+class FileHandler(BaseHandler):
+ # Use local file or FTP depending on form of URL
+ def file_open(self, req):
+ url = req.get_selector()
+ if url[:2] == '//' and url[2:3] != '/' and (req.host and
+ req.host != 'localhost'):
+ req.type = 'ftp'
+ return self.parent.open(req)
+ else:
+ return self.open_local_file(req)
+
+ # names for the localhost
+ names = None
+ def get_names(self):
+ if FileHandler.names is None:
+ try:
+ FileHandler.names = tuple(
+ socket.gethostbyname_ex('localhost')[2] +
+ socket.gethostbyname_ex(socket.gethostname())[2])
+ except socket.gaierror:
+ FileHandler.names = (socket.gethostbyname('localhost'),)
+ return FileHandler.names
+
+ # not entirely sure what the rules are here
+ def open_local_file(self, req):
+ import email.utils
+ import mimetypes
+ host = req.get_host()
+ filename = req.get_selector()
+ localfile = url2pathname(filename)
+ try:
+ stats = os.stat(localfile)
+ size = stats.st_size
+ modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+ mtype = mimetypes.guess_type(filename)[0]
+ headers = mimetools.Message(StringIO(
+ 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+ (mtype or 'text/plain', size, modified)))
+ if host:
+ host, port = splitport(host)
+ if not host or \
+ (not port and _safe_gethostbyname(host) in self.get_names()):
+ if host:
+ origurl = 'file://' + host + filename
+ else:
+ origurl = 'file://' + filename
+ return addinfourl(open(localfile, 'rb'), headers, origurl)
+ except OSError, msg:
+ # urllib2 users shouldn't expect OSErrors coming from urlopen()
+ raise URLError(msg)
+ raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+ def ftp_open(self, req):
+ import ftplib
+ import mimetypes
+ host = req.get_host()
+ if not host:
+ raise URLError('ftp error: no host given')
+ host, port = splitport(host)
+ if port is None:
+ port = ftplib.FTP_PORT
+ else:
+ port = int(port)
+
+ # username/password handling
+ user, host = splituser(host)
+ if user:
+ user, passwd = splitpasswd(user)
+ else:
+ passwd = None
+ host = unquote(host)
+ user = user or ''
+ passwd = passwd or ''
+
+ try:
+ host = socket.gethostbyname(host)
+ except socket.error, msg:
+ raise URLError(msg)
+ path, attrs = splitattr(req.get_selector())
+ dirs = path.split('/')
+ dirs = map(unquote, dirs)
+ dirs, file = dirs[:-1], dirs[-1]
+ if dirs and not dirs[0]:
+ dirs = dirs[1:]
+ try:
+ fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+ type = file and 'I' or 'D'
+ for attr in attrs:
+ attr, value = splitvalue(attr)
+ if attr.lower() == 'type' and \
+ value in ('a', 'A', 'i', 'I', 'd', 'D'):
+ type = value.upper()
+ fp, retrlen = fw.retrfile(file, type)
+ headers = ""
+ mtype = mimetypes.guess_type(req.get_full_url())[0]
+ if mtype:
+ headers += "Content-type: %s\n" % mtype
+ if retrlen is not None and retrlen >= 0:
+ headers += "Content-length: %d\n" % retrlen
+ sf = StringIO(headers)
+ headers = mimetools.Message(sf)
+ return addinfourl(fp, headers, req.get_full_url())
+ except ftplib.all_errors, msg:
+ raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+ def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+ fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+## fw.ftp.set_debuglevel(1)
+ return fw
+
+class CacheFTPHandler(FTPHandler):
+ # XXX would be nice to have pluggable cache strategies
+ # XXX this stuff is definitely not thread safe
+ def __init__(self):
+ self.cache = {}
+ self.timeout = {}
+ self.soonest = 0
+ self.delay = 60
+ self.max_conns = 16
+
+ def setTimeout(self, t):
+ self.delay = t
+
+ def setMaxConns(self, m):
+ self.max_conns = m
+
+ def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+ key = user, host, port, '/'.join(dirs), timeout
+ if key in self.cache:
+ self.timeout[key] = time.time() + self.delay
+ else:
+ self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+ self.timeout[key] = time.time() + self.delay
+ self.check_cache()
+ return self.cache[key]
+
+ def check_cache(self):
+ # first check for old ones
+ t = time.time()
+ if self.soonest <= t:
+ for k, v in self.timeout.items():
+ if v < t:
+ self.cache[k].close()
+ del self.cache[k]
+ del self.timeout[k]
+ self.soonest = min(self.timeout.values())
+
+ # then check the size
+ if len(self.cache) == self.max_conns:
+ for k, v in self.timeout.items():
+ if v == self.soonest:
+ del self.cache[k]
+ del self.timeout[k]
+ break
+ self.soonest = min(self.timeout.values())
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
from ctypes_support import standard_c_lib as libc
from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
+from ctypes import Structure, c_int, c_long, byref, POINTER
from errno import EINVAL, EPERM
import _structseq
@@ -165,7 +165,6 @@
@builtinify
def getpagesize():
- pagesize = 0
if _getpagesize:
return _getpagesize()
else:
diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -276,8 +276,8 @@
# create the Attribute and do the generalization asked for
newattr = Attribute(attr, self.bookkeeper)
if s_value:
- if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
- import pdb; pdb.set_trace()
+ #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
+ # import pdb; pdb.set_trace()
newattr.s_value = s_value
# keep all subattributes' values
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -127,7 +127,7 @@
pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
ChoiceOption("name", "Object Space name",
- ["std", "flow", "thunk", "dump", "taint"],
+ ["std", "flow", "thunk", "dump"],
"std",
cmdline='--objspace -o'),
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -37,29 +37,6 @@
.. _`thunk object space docs`: objspace-proxies.html#thunk
.. _`interface section of the thunk object space docs`: objspace-proxies.html#thunk-interface
-.. broken:
-
- Taint Object Space Functionality
- ================================
-
- When the taint object space is used (choose with :config:`objspace.name`),
- the following names are put into ``__pypy__``:
-
- - ``taint``
- - ``is_tainted``
- - ``untaint``
- - ``taint_atomic``
- - ``_taint_debug``
- - ``_taint_look``
- - ``TaintError``
-
- Those are all described in the `interface section of the taint object space
- docs`_.
-
- For more detailed explanations and examples see the `taint object space docs`_.
-
- .. _`taint object space docs`: objspace-proxies.html#taint
- .. _`interface section of the taint object space docs`: objspace-proxies.html#taint-interface
Transparent Proxy Functionality
===============================
diff --git a/pypy/doc/config/objspace.name.txt b/pypy/doc/config/objspace.name.txt
--- a/pypy/doc/config/objspace.name.txt
+++ b/pypy/doc/config/objspace.name.txt
@@ -4,7 +4,6 @@
for normal usage):
* thunk_: The thunk object space adds lazy evaluation to PyPy.
- * taint_: The taint object space adds soft security features.
* dump_: Using this object spaces results in the dumpimp of all operations
to a log.
@@ -12,5 +11,4 @@
.. _`Object Space Proxies`: ../objspace-proxies.html
.. _`Standard Object Space`: ../objspace.html#standard-object-space
.. _thunk: ../objspace-proxies.html#thunk
-.. _taint: ../objspace-proxies.html#taint
.. _dump: ../objspace-proxies.html#dump
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -309,7 +309,6 @@
.. _`object space`: objspace.html
.. _FlowObjSpace: objspace.html#the-flow-object-space
.. _`trace object space`: objspace.html#the-trace-object-space
-.. _`taint object space`: objspace-proxies.html#taint
.. _`thunk object space`: objspace-proxies.html#thunk
.. _`transparent proxies`: objspace-proxies.html#tproxy
.. _`Differences between PyPy and CPython`: cpython_differences.html
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -129,297 +129,6 @@
function behaves lazily: all calls to it return a thunk object.
-.. broken right now:
-
- .. _taint:
-
- The Taint Object Space
- ======================
-
- Motivation
- ----------
-
- The Taint Object Space provides a form of security: "tainted objects",
- inspired by various sources, see [D12.1]_ for a more detailed discussion.
-
- The basic idea of this kind of security is not to protect against
- malicious code but to help with handling and boxing sensitive data.
- It covers two kinds of sensitive data: secret data which should not leak,
- and untrusted data coming from an external source and that must be
- validated before it is used.
-
- The idea is that, considering a large application that handles these
- kinds of sensitive data, there are typically only a small number of
- places that need to explicitly manipulate that sensitive data; all the
- other places merely pass it around, or do entirely unrelated things.
-
- Nevertheless, if a large application needs to be reviewed for security,
- it must be entirely carefully checked, because it is possible that a
- bug at some apparently unrelated place could lead to a leak of sensitive
- information in a way that an external attacker could exploit. For
- example, if any part of the application provides web services, an
- attacker might be able to issue unexpected requests with a regular web
- browser and deduce secret information from the details of the answers he
- gets. Another example is the common CGI attack where an attacker sends
- malformed inputs and causes the CGI script to do unintended things.
-
- An approach like that of the Taint Object Space allows the small parts
- of the program that manipulate sensitive data to be explicitly marked.
- The effect of this is that although these small parts still need a
- careful security review, the rest of the application no longer does,
- because even a bug would be unable to leak the information.
-
- We have implemented a simple two-level model: objects are either
- regular (untainted), or sensitive (tainted). Objects are marked as
- sensitive if they are secret or untrusted, and only declassified at
- carefully-checked positions (e.g. where the secret data is needed, or
- after the untrusted data has been fully validated).
-
- It would be simple to extend the code for more fine-grained scales of
- secrecy. For example it is typical in the literature to consider
- user-specified lattices of secrecy levels, corresponding to multiple
- "owners" that cannot access data belonging to another "owner" unless
- explicitly authorized to do so.
-
- Tainting and untainting
- -----------------------
-
- Start a py.py with the Taint Object Space and try the following example::
-
- $ py.py -o taint
- >>>> from __pypy__ import taint
- >>>> x = taint(6)
-
- # x is hidden from now on. We can pass it around and
- # even operate on it, but not inspect it. Taintness
- # is propagated to operation results.
-
- >>>> x
- TaintError
-
- >>>> if x > 5: y = 2 # see below
- TaintError
-
- >>>> y = x + 5 # ok
- >>>> lst = [x, y]
- >>>> z = lst.pop()
- >>>> t = type(z) # type() works too, tainted answer
- >>>> t
- TaintError
- >>>> u = t is int # even 'is' works
- >>>> u
- TaintError
-
- Notice that using a tainted boolean like ``x > 5`` in an ``if``
- statement is forbidden. This is because knowing which path is followed
- would give away a hint about ``x``; in the example above, if the
- statement ``if x > 5: y = 2`` was allowed to run, we would know
- something about the value of ``x`` by looking at the (untainted) value
- in the variable ``y``.
-
- Of course, there is a way to inspect tainted objects. The basic way is
- to explicitly "declassify" it with the ``untaint()`` function. In an
- application, the places that use ``untaint()`` are the places that need
- careful security review. To avoid unexpected objects showing up, the
- ``untaint()`` function must be called with the exact type of the object
- to declassify. It will raise ``TaintError`` if the type doesn't match::
-
- >>>> from __pypy__ import taint
- >>>> untaint(int, x)
- 6
- >>>> untaint(int, z)
- 11
- >>>> untaint(bool, x > 5)
- True
- >>>> untaint(int, x > 5)
- TaintError
-
-
- Taint Bombs
- -----------
-
- In this area, a common problem is what to do about failing operations.
- If an operation raises an exception when manipulating a tainted object,
- then the very presence of the exception can leak information about the
- tainted object itself. Consider::
-
- >>>> 5 / (x-6)
-
- By checking if this raises ``ZeroDivisionError`` or not, we would know
- if ``x`` was equal to 6 or not. The solution to this problem in the
- Taint Object Space is to introduce *Taint Bombs*. They are a kind of
- tainted object that doesn't contain a real object, but a pending
- exception. Taint Bombs are indistinguishable from normal tainted
- objects to unprivileged code. See::
-
- >>>> x = taint(6)
- >>>> i = 5 / (x-6) # no exception here
- >>>> j = i + 1 # nor here
- >>>> k = j + 5 # nor here
- >>>> untaint(int, k)
- TaintError
-
- In the above example, all of ``i``, ``j`` and ``k`` contain a Taint
- Bomb. Trying to untaint it raises an exception - a generic
- ``TaintError``. What we win is that the exception gives little away,
- and most importantly it occurs at the point where ``untaint()`` is
- called, not where the operation failed. This means that all calls to
- ``untaint()`` - but not the rest of the code - must be carefully
- reviewed for what occurs if they receive a Taint Bomb; they might catch
- the ``TaintError`` and give the user a generic message that something
- went wrong, if we are reasonably careful that the message or even its
- presence doesn't give information away. This might be a
- problem by itself, but there is no satisfying general solution here:
- it must be considered on a case-by-case basis. Again, what the
- Taint Object Space approach achieves is not solving these problems, but
- localizing them to well-defined small parts of the application - namely,
- around calls to ``untaint()``.
-
- The ``TaintError`` exception deliberately does not include any
- useful error messages, because they might give information away.
- Of course, this makes debugging quite a bit harder; a difficult
- problem to solve properly. So far we have implemented a way to peek in a Taint
- Box or Bomb, ``__pypy__._taint_look(x)``, and a "debug mode" that
- prints the exception as soon as a Bomb is created - both write
- information to the low-level stderr of the application, where we hope
- that it is unlikely to be seen by anyone but the application
- developer.
-
-
- Taint Atomic functions
- ----------------------
-
- Occasionally, a more complicated computation must be performed on a
- tainted object. This requires first untainting the object, performing the
- computations, and then carefully tainting the result again (including
- hiding all exceptions into Bombs).
-
- There is a built-in decorator that does this for you::
-
- >>>> @__pypy__.taint_atomic
- >>>> def myop(x, y):
- .... while x > 0:
- .... x -= y
- .... return x
- ....
- >>>> myop(42, 10)
- -8
- >>>> z = myop(taint(42), 10)
- >>>> z
- TaintError
- >>>> untaint(int, z)
- -8
-
- The decorator makes a whole function behave like a built-in operation.
- If no tainted argument is passed in, the function behaves normally. But
- if any of the arguments is tainted, it is automatically untainted - so
- the function body always sees untainted arguments - and the eventual
- result is tainted again (possibly in a Taint Bomb).
-
- It is important for the function marked as ``taint_atomic`` to have no
- visible side effects, as these could cause information leakage.
- This is currently not enforced, which means that all ``taint_atomic``
- functions have to be carefully reviewed for security (but not the
- callers of ``taint_atomic`` functions).
-
- A possible future extension would be to forbid side-effects on
- non-tainted objects from all ``taint_atomic`` functions.
-
- An example of usage: given a tainted object ``passwords_db`` that
- references a database of passwords, we can write a function
- that checks if a password is valid as follows::
-
- @taint_atomic
- def validate(passwords_db, username, password):
- assert type(passwords_db) is PasswordDatabase
- assert type(username) is str
- assert type(password) is str
- ...load username entry from passwords_db...
- return expected_password == password
-
- It returns a tainted boolean answer, or a Taint Bomb if something
- went wrong. A caller can do::
-
- ok = validate(passwords_db, 'john', '1234')
- ok = untaint(bool, ok)
-
- This can give three outcomes: ``True``, ``False``, or a ``TaintError``
- exception (with no information on it) if anything went wrong. If even
- this is considered giving too much information away, the ``False`` case
- can be made indistinguishable from the ``TaintError`` case (simply by
- raising an exception in ``validate()`` if the password is wrong).
-
- In the above example, the security results achieved are the following:
- as long as ``validate()`` does not leak information, no other part of
- the code can obtain more information about a passwords database than a
- Yes/No answer to a precise query.
-
- A possible extension of the ``taint_atomic`` decorator would be to check
- the argument types, as ``untaint()`` does, for the same reason: to
- prevent bugs where a function like ``validate()`` above is accidentally
- called with the wrong kind of tainted object, which would make it
- misbehave. For now, all ``taint_atomic`` functions should be
- conservative and carefully check all assumptions on their input
- arguments.
-
-
- .. _`taint-interface`:
-
- Interface
- ---------
-
- .. _`like a built-in operation`:
-
- The basic rule of the Tainted Object Space is that it introduces two new
- kinds of objects, Tainted Boxes and Tainted Bombs (which are not types
- in the Python sense). Each box internally contains a regular object;
- each bomb internally contains an exception object. An operation
- involving Tainted Boxes is performed on the objects contained in the
- boxes, and gives a Tainted Box or a Tainted Bomb as a result (such an
- operation does not let an exception be raised). An operation called
- with a Tainted Bomb argument immediately returns the same Tainted Bomb.
-
- In a PyPy running with (or translated with) the Taint Object Space,
- the ``__pypy__`` module exposes the following interface:
-
- * ``taint(obj)``
-
- Return a new Tainted Box wrapping ``obj``. Return ``obj`` itself
- if it is already tainted (a Box or a Bomb).
-
- * ``is_tainted(obj)``
-
- Check if ``obj`` is tainted (a Box or a Bomb).
-
- * ``untaint(type, obj)``
-
- Untaints ``obj`` if it is tainted. Raise ``TaintError`` if the type
- of the untainted object is not exactly ``type``, or if ``obj`` is a
- Bomb.
-
- * ``taint_atomic(func)``
-
- Return a wrapper function around the callable ``func``. The wrapper
- behaves `like a built-in operation`_ with respect to untainting the
- arguments, tainting the result, and returning a Bomb.
-
- * ``TaintError``
-
- Exception. On purpose, it provides no attribute or error message.
-
- * ``_taint_debug(level)``
-
- Set the debugging level to ``level`` (0=off). At level 1 or above,
- all Taint Bombs print a diagnostic message to stderr when they are
- created.
-
- * ``_taint_look(obj)``
-
- For debugging purposes: prints (to stderr) the type and address of
- the object in a Tainted Box, or prints the exception if ``obj`` is
- a Taint Bomb.
-
-
.. _dump:
The Dump Object Space
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -391,8 +391,11 @@
def decrement_ticker(self, by):
value = self._ticker
if self.has_bytecode_counter: # this 'if' is constant-folded
- value -= by
- self._ticker = value
+ if jit.isconstant(by) and by == 0:
+ pass # normally constant-folded too
+ else:
+ value -= by
+ self._ticker = value
return value
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
parenlev = parenlev - 1
if parenlev < 0:
raise TokenError("unmatched '%s'" % initial, line,
- lnum-1, 0, token_list)
+ lnum, start + 1, token_list)
if token in python_opmap:
punct = python_opmap[token]
else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
assert exc.lineno == 1
assert exc.offset == 5
assert exc.lastlineno == 5
+ exc = py.test.raises(SyntaxError, parse, "abc)").value
+ assert exc.msg == "unmatched ')'"
+ assert exc.lineno == 1
+ assert exc.offset == 4
def test_is(self):
self.parse("x is y")
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -165,6 +165,7 @@
'unicodegetitem' : (('ref', 'int'), 'int'),
'unicodesetitem' : (('ref', 'int', 'int'), 'int'),
'cast_ptr_to_int' : (('ref',), 'int'),
+ 'cast_int_to_ptr' : (('int',), 'ref'),
'debug_merge_point': (('ref', 'int'), None),
'force_token' : ((), 'int'),
'call_may_force' : (('int', 'varargs'), 'intorptr'),
@@ -875,9 +876,6 @@
def op_new_array(self, arraydescr, count):
return do_new_array(arraydescr.ofs, count)
- def op_cast_ptr_to_int(self, descr, ptr):
- return cast_to_int(ptr)
-
def op_force_token(self, descr):
opaque_frame = _to_opaque(self)
return llmemory.cast_ptr_to_adr(opaque_frame)
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
def freeing_block(self, start, stop):
pass
+ def record_constptrs(self, op, gcrefs_output_list):
+ for i in range(op.numargs()):
+ v = op.getarg(i)
+ if isinstance(v, ConstPtr) and bool(v.value):
+ p = v.value
+ rgc._make_sure_does_not_move(p)
+ gcrefs_output_list.append(p)
+
# ____________________________________________________________
class GcLLDescr_boehm(GcLLDescription):
@@ -141,6 +149,14 @@
get_funcptr_for_newstr = None
get_funcptr_for_newunicode = None
+ def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
+ # record all GCREFs too, because Boehm cannot see them and keep them
+ # alive if they end up as constants in the assembler
+ for op in operations:
+ self.record_constptrs(op, gcrefs_output_list)
+ return GcLLDescription.rewrite_assembler(self, cpu, operations,
+ gcrefs_output_list)
+
# ____________________________________________________________
# All code below is for the hybrid or minimark GC
@@ -757,14 +773,6 @@
funcptr(llmemory.cast_ptr_to_adr(gcref_struct),
llmemory.cast_ptr_to_adr(gcref_newptr))
- def record_constptrs(self, op, gcrefs_output_list):
- for i in range(op.numargs()):
- v = op.getarg(i)
- if isinstance(v, ConstPtr) and bool(v.value):
- p = v.value
- rgc._make_sure_does_not_move(p)
- gcrefs_output_list.append(p)
-
def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
# Perform two kinds of rewrites in parallel:
#
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1468,20 +1468,16 @@
return u''.join(u.chars)
- def test_casts(self):
- py.test.skip("xxx fix or kill")
- from pypy.rpython.lltypesystem import lltype, llmemory
- TP = lltype.GcStruct('x')
- x = lltype.malloc(TP)
- x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+ def test_cast_int_to_ptr(self):
+ res = self.execute_operation(rop.CAST_INT_TO_PTR,
+ [BoxInt(-17)], 'ref').value
+ assert lltype.cast_ptr_to_int(res) == -17
+
+ def test_cast_ptr_to_int(self):
+ x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
res = self.execute_operation(rop.CAST_PTR_TO_INT,
- [BoxPtr(x)], 'int').value
- expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
- assert rffi.get_real_int(res) == rffi.get_real_int(expected)
- res = self.execute_operation(rop.CAST_PTR_TO_INT,
- [ConstPtr(x)], 'int').value
- expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
- assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+ [BoxPtr(x)], 'int').value
+ assert res == -19
def test_ooops_non_gc(self):
x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2299,13 +2295,6 @@
#
cpu.bh_strsetitem(x, 4, ord('/'))
assert str.chars[4] == '/'
- #
-## x = cpu.bh_newstr(5)
-## y = cpu.bh_cast_ptr_to_int(x)
-## z = cpu.bh_cast_ptr_to_int(x)
-## y = rffi.get_real_int(y)
-## z = rffi.get_real_int(z)
-## assert type(y) == type(z) == int and y == z
def test_sorting_of_fields(self):
S = self.S
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1387,7 +1387,8 @@
def genop_same_as(self, op, arglocs, resloc):
self.mov(arglocs[0], resloc)
- #genop_cast_ptr_to_int = genop_same_as
+ genop_cast_ptr_to_int = genop_same_as
+ genop_cast_int_to_ptr = genop_same_as
def genop_int_mod(self, op, arglocs, resloc):
if IS_X86_32:
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1152,7 +1152,8 @@
self.possibly_free_var(op.getarg(0))
resloc = self.force_allocate_reg(op.result)
self.Perform(op, [argloc], resloc)
- #consider_cast_ptr_to_int = consider_same_as
+ consider_cast_ptr_to_int = consider_same_as
+ consider_cast_int_to_ptr = consider_same_as
def consider_strlen(self, op):
args = op.getarglist()
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -9,7 +9,12 @@
"""
import autopath
-import operator, sys, os, re, py, new
+import new
+import operator
+import py
+import re
+import sys
+import subprocess
from bisect import bisect_left
# don't use pypy.tool.udir here to avoid removing old usessions which
@@ -44,14 +49,16 @@
f = open(tmpfile, 'wb')
f.write(data)
f.close()
- g = os.popen(objdump % {
+ p = subprocess.Popen(objdump % {
'file': tmpfile,
'origin': originaddr,
'backend': objdump_backend_option[backend_name],
- }, 'r')
- result = g.readlines()
- g.close()
- lines = result[6:] # drop some objdump cruft
+ }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ assert not p.returncode, ('Encountered an error running objdump: %s' %
+ stderr)
+ # drop some objdump cruft
+ lines = stdout.splitlines()[6:]
return format_code_dump_with_labels(originaddr, lines, label_list)
def format_code_dump_with_labels(originaddr, lines, label_list):
@@ -85,8 +92,12 @@
#
print 'loading symbols from %s...' % (filename,)
symbols = {}
- g = os.popen(symbollister % filename, "r")
- for line in g:
+ p = subprocess.Popen(symbollister % filename, shell=True,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ assert not p.returncode, ('Encountered an error running nm: %s' %
+ stderr)
+ for line in stdout.splitlines():
match = re_symbolentry.match(line)
if match:
addr = long(match.group(1), 16)
@@ -94,7 +105,6 @@
if name.startswith('pypy_g_'):
name = '\xb7' + name[7:]
symbols[addr] = name
- g.close()
print '%d symbols found' % (len(symbols),)
return symbols
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -455,6 +455,23 @@
# the special return value None forces op.result to be considered
# equal to op.args[0]
return [op0, op1, None]
+ if (hints.get('promote_string') and
+ op.args[0].concretetype is not lltype.Void):
+ S = lltype.Ptr(rstr.STR)
+ assert op.args[0].concretetype == S
+ self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+ "str.eq_nonnull",
+ [S, S],
+ lltype.Signed,
+ EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+ descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+ EffectInfo.OS_STREQ_NONNULL)
+ # XXX this is fairly ugly way of creating a constant,
+ # however, callinfocollection has no better interface
+ c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+ op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
+ op.result)
+ return [SpaceOperation('-live-', [], None), op1, None]
else:
log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
@@ -783,8 +800,7 @@
def rewrite_op_cast_ptr_to_int(self, op):
if self._is_gc(op.args[0]):
- #return op
- raise NotImplementedError("cast_ptr_to_int")
+ return op
def rewrite_op_force_cast(self, op):
v_arg = op.args[0]
@@ -1526,6 +1542,10 @@
def rewrite_op_jit_force_virtual(self, op):
return self._do_builtin_call(op)
+ def rewrite_op_jit_is_virtual(self, op):
+ raise Exception, (
+ "'vref.virtual' should not be used from jit-visible code")
+
def rewrite_op_jit_force_virtualizable(self, op):
# this one is for virtualizables
vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -99,6 +99,12 @@
if i == oopspecindex:
return True
return False
+ def callinfo_for_oopspec(self, oopspecindex):
+ assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+ class c:
+ class adr:
+ ptr = 1
+ return ('calldescr', c)
class FakeBuiltinCallControl:
def __init__(self):
@@ -119,6 +125,7 @@
EI.OS_STR2UNICODE:([PSTR], PUNICODE),
EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
EI.OS_STR_SLICE: ([PSTR, INT, INT], PSTR),
+ EI.OS_STREQ_NONNULL: ([PSTR, PSTR], INT),
EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
EI.OS_UNI_SLICE: ([PUNICODE, INT, INT], PUNICODE),
EI.OS_UNI_EQUAL: ([PUNICODE, PUNICODE], lltype.Bool),
@@ -844,6 +851,21 @@
assert op1.args[2] == ListOfKind('ref', [v1, v2])
assert op1.result == v3
+def test_str_promote():
+ PSTR = lltype.Ptr(rstr.STR)
+ v1 = varoftype(PSTR)
+ v2 = varoftype(PSTR)
+ op = SpaceOperation('hint',
+ [v1, Constant({'promote_string': True}, lltype.Void)],
+ v2)
+ tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+ op0, op1, _ = tr.rewrite_operation(op)
+ assert op1.opname == 'str_guard_value'
+ assert op1.args[0] == v1
+ assert op1.args[2] == 'calldescr'
+ assert op1.result == v2
+ assert op0.opname == '-live-'
+
def test_unicode_concat():
# test that the oopspec is present and correctly transformed
PSTR = lltype.Ptr(rstr.UNICODE)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,7 +2,7 @@
from pypy.rlib.rtimer import read_timestamp
from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
from pypy.rlib.debug import make_sure_not_resized
from pypy.rpython.lltypesystem import lltype, llmemory, rclass
from pypy.rpython.lltypesystem.lloperation import llop
@@ -503,6 +503,15 @@
@arguments("r", returns="r")
def bhimpl_cast_opaque_ptr(a):
return a
+ @arguments("r", returns="i")
+ def bhimpl_cast_ptr_to_int(a):
+ i = lltype.cast_ptr_to_int(a)
+ ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+ return i
+ @arguments("i", returns="r")
+ def bhimpl_cast_int_to_ptr(i):
+ ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+ return lltype.cast_int_to_ptr(llmemory.GCREF, i)
@arguments("i", returns="i")
def bhimpl_int_copy(a):
@@ -523,6 +532,9 @@
@arguments("f")
def bhimpl_float_guard_value(a):
pass
+ @arguments("r", "i", "d", returns="r")
+ def bhimpl_str_guard_value(a, i, d):
+ return a
@arguments("self", "i")
def bhimpl_int_push(self, a):
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -54,9 +54,10 @@
if box in self.new_boxes:
self.new_boxes[box] = False
if box in self.dependencies:
- for dep in self.dependencies[box]:
+ deps = self.dependencies[box]
+ del self.dependencies[box]
+ for dep in deps:
self._escape(dep)
- del self.dependencies[box]
def clear_caches(self, opnum, descr, argboxes):
if opnum == rop.SETFIELD_GC:
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -9,6 +9,7 @@
from pypy.jit.metainterp.resoperation import ResOperation, rop
from pypy.jit.codewriter import heaptracker, longlong
+from pypy.rlib.objectmodel import compute_identity_hash
# ____________________________________________________________
@@ -104,7 +105,7 @@
getref._annspecialcase_ = 'specialize:arg(1)'
def _get_hash_(self):
- raise NotImplementedError
+ return compute_identity_hash(self)
def clonebox(self):
raise NotImplementedError
@@ -133,6 +134,9 @@
def _get_str(self):
raise NotImplementedError
+ def same_box(self, other):
+ return self is other
+
class AbstractDescr(AbstractValue):
__slots__ = ()
@@ -241,32 +245,15 @@
def constbox(self):
return self
+ def same_box(self, other):
+ return self.same_constant(other)
+
def same_constant(self, other):
raise NotImplementedError
def __repr__(self):
return 'Const(%s)' % self._getrepr_()
- def __eq__(self, other):
- "NOT_RPYTHON"
- # Remember that you should not compare Consts with '==' in RPython.
- # Consts have no special __hash__, in order to force different Consts
- # from being considered as different keys when stored in dicts
- # (as they always are after translation). Use a dict_equal_consts()
- # to get the other behavior (i.e. using this __eq__).
- if self.__class__ is not other.__class__:
- return False
- try:
- return self.value == other.value
- except TypeError:
- if (isinstance(self.value, Symbolic) and
- isinstance(other.value, Symbolic)):
- return self.value is other.value
- raise
-
- def __ne__(self, other):
- return not (self == other)
-
class ConstInt(Const):
type = INT
@@ -688,33 +675,6 @@
# ____________________________________________________________
-def dict_equal_consts():
- "NOT_RPYTHON"
- # Returns a dict in which Consts that compare as equal
- # are identified when used as keys.
- return r_dict(dc_eq, dc_hash)
-
-def dc_eq(c1, c2):
- return c1 == c2
-
-def dc_hash(c):
- "NOT_RPYTHON"
- # This is called during translation only. Avoid using identityhash(),
- # to avoid forcing a hash, at least on lltype objects.
- if not isinstance(c, Const):
- return hash(c)
- if isinstance(c.value, Symbolic):
- return id(c.value)
- try:
- if isinstance(c, ConstPtr):
- p = lltype.normalizeptr(c.value)
- if p is not None:
- return hash(p._obj)
- else:
- return 0
- return c._get_hash_()
- except lltype.DelayedPointer:
- return -2 # xxx risk of changing hash...
def make_hashable_int(i):
from pypy.rpython.lltypesystem.ll2ctypes import NotCtypesAllocatedStructure
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2329,7 +2329,7 @@
def _variables_equal(box, varname, strict):
if varname not in virtuals:
if strict:
- assert box == oparse.getvar(varname)
+ assert box.same_box(oparse.getvar(varname))
else:
assert box.value == oparse.getvar(varname).value
else:
diff --git a/pypy/jit/metainterp/optimizeopt/util.py b/pypy/jit/metainterp/optimizeopt/util.py
--- a/pypy/jit/metainterp/optimizeopt/util.py
+++ b/pypy/jit/metainterp/optimizeopt/util.py
@@ -90,14 +90,11 @@
for i in range(len(args1)):
arg1 = args1[i]
arg2 = args2[i]
- if isinstance(arg1, history.Const):
- if arg1.__class__ is not arg2.__class__:
+ if arg1 is None:
+ if arg2 is not None:
return False
- if not arg1.same_constant(arg2):
- return False
- else:
- if not arg1 is arg2:
- return False
+ elif not arg1.same_box(arg2):
+ return False
return True
def args_hash(args):
@@ -106,10 +103,8 @@
for arg in args:
if arg is None:
y = 17
- elif isinstance(arg, history.Const):
+ else:
y = arg._get_hash_()
- else:
- y = compute_identity_hash(arg)
res = intmask((1000003 * res) ^ y)
return res
@@ -145,9 +140,12 @@
for i in range(op1.numargs()):
x = op1.getarg(i)
y = op2.getarg(i)
- assert x == remap.get(y, y)
+ assert x.same_box(remap.get(y, y))
if op2.result in remap:
- assert op1.result == remap[op2.result]
+ if op2.result is None:
+ assert op1.result == remap[op2.result]
+ else:
+ assert op1.result.same_box(remap[op2.result])
else:
remap[op2.result] = op1.result
if op1.getopnum() != rop.JUMP: # xxx obscure
@@ -156,11 +154,20 @@
assert len(op1.getfailargs()) == len(op2.getfailargs())
if strict_fail_args:
for x, y in zip(op1.getfailargs(), op2.getfailargs()):
- assert x == remap.get(y, y)
+ if x is None:
+ assert remap.get(y, y) is None
+ else:
+ assert x.same_box(remap.get(y, y))
else:
fail_args1 = set(op1.getfailargs())
fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
- assert fail_args1 == fail_args2
+ for x in fail_args1:
+ for y in fail_args2:
+ if x.same_box(y):
+ fail_args2.remove(y)
+ break
+ else:
+ assert False
assert len(oplist1) == len(oplist2)
print '-'*totwidth
return True
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -587,10 +587,7 @@
return True
#
if v1.is_nonnull() and v2.is_nonnull():
- if l1box is not None and l2box is not None and (
- l1box == l2box or (isinstance(l1box, ConstInt) and
- isinstance(l2box, ConstInt) and
- l1box.value == l2box.value)):
+ if l1box is not None and l2box is not None and l1box.same_box(l2box):
do = EffectInfo.OS_STREQ_LENGTHOK
else:
do = EffectInfo.OS_STREQ_NONNULL
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -162,15 +162,18 @@
if registers[i] is oldbox:
registers[i] = newbox
if not we_are_translated():
- assert oldbox not in registers[count:]
+ for b in registers[count:]:
+ assert not oldbox.same_box(b)
+
def make_result_of_lastop(self, resultbox):
got_type = resultbox.type
- if not we_are_translated():
- typeof = {'i': history.INT,
- 'r': history.REF,
- 'f': history.FLOAT}
- assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
+ # XXX disabled for now, conflicts with str_guard_value
+ #if not we_are_translated():
+ # typeof = {'i': history.INT,
+ # 'r': history.REF,
+ # 'f': history.FLOAT}
+ # assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
target_index = ord(self.bytecode[self.pc-1])
if got_type == history.INT:
self.registers_i[target_index] = resultbox
@@ -219,6 +222,7 @@
'cast_float_to_int', 'cast_int_to_float',
'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
'float_neg', 'float_abs',
+ 'cast_ptr_to_int', 'cast_int_to_ptr',
]:
exec py.code.Source('''
@arguments("box")
@@ -895,6 +899,21 @@
def _opimpl_guard_value(self, orgpc, box):
self.implement_guard_value(orgpc, box)
+ @arguments("orgpc", "box", "box", "descr")
+ def opimpl_str_guard_value(self, orgpc, box, funcbox, descr):
+ if isinstance(box, Const):
+ return box # no promotion needed, already a Const
+ else:
+ constbox = box.constbox()
+ resbox = self.do_residual_call(funcbox, descr, [box, constbox])
+ promoted_box = resbox.constbox()
+ # This is GUARD_VALUE because GUARD_TRUE assumes the existance
+ # of a label when computing resumepc
+ self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
+ resumepc=orgpc)
+ self.metainterp.replace_box(box, constbox)
+ return constbox
+
opimpl_int_guard_value = _opimpl_guard_value
opimpl_ref_guard_value = _opimpl_guard_value
opimpl_float_guard_value = _opimpl_guard_value
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -431,6 +431,8 @@
'INT_IS_TRUE/1b',
'INT_NEG/1',
'INT_INVERT/1',
+ 'CAST_PTR_TO_INT/1',
+ 'CAST_INT_TO_PTR/1',
#
'SAME_AS/1', # gets a Const or a Box, turns it into another Box
#
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -13,7 +13,7 @@
from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
loop_invariant, elidable, promote, jit_debug, assert_green,
AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
- isconstant, isvirtual)
+ isconstant, isvirtual, promote_string)
from pypy.rlib.rarithmetic import ovfcheck
from pypy.rpython.lltypesystem import lltype, llmemory, rffi
from pypy.rpython.ootypesystem import ootype
@@ -3440,7 +3440,6 @@
class TestLLtype(BaseLLtypeTests, LLJitMixin):
def test_tagged(self):
- py.test.skip("implement me")
from pypy.rlib.objectmodel import UnboxedValue
class Base(object):
__slots__ = ()
@@ -3492,3 +3491,35 @@
pc += 1
return pc
res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+ def test_rerased(self):
+ from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+ eraseX, uneraseX = new_erasing_pair("X")
+ #
+ class X:
+ def __init__(self, a, b):
+ self.a = a
+ self.b = b
+ #
+ def f(i, j):
+ # 'j' should be 0 or 1, not other values
+ if j > 0:
+ e = eraseX(X(i, j))
+ else:
+ try:
+ e = erase_int(i)
+ except OverflowError:
+ return -42
+ if j & 1:
+ x = uneraseX(e)
+ return x.a - x.b
+ else:
+ return unerase_int(e)
+ #
+ x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+ assert x == -128
+ bigint = sys.maxint//2 + 1
+ x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+ assert x == -42
+ x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+ assert x == 999
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -355,6 +355,14 @@
assert not h.is_unescaped(box1)
assert not h.is_unescaped(box2)
+ def test_circular_virtuals(self):
+ h = HeapCache()
+ h.new(box1)
+ h.new(box2)
+ h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+ h.invalidate_caches(rop.SETFIELD_GC, None, [box2, box1])
+ h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1]) # does not crash
+
def test_unescaped_array(self):
h = HeapCache()
h.new_array(box1, lengthbox1)
@@ -362,4 +370,4 @@
h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
assert h.is_unescaped(box1)
h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
- assert not h.is_unescaped(box1)
\ No newline at end of file
+ assert not h.is_unescaped(box1)
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -180,22 +180,27 @@
reader.consume_boxes(info, bi, br, bf)
b1s = reader.liveboxes[0]
b2s = reader.liveboxes[1]
- assert bi == [b1s, ConstInt(111), b1s]
- assert br == [ConstPtr(gcrefnull), b2s]
+ assert_same(bi, [b1s, ConstInt(111), b1s])
+ assert_same(br, [ConstPtr(gcrefnull), b2s])
bi, br, bf = [None]*2, [None]*0, [None]*0
info = MyBlackholeInterp([lltype.Signed,
lltype.Signed]).get_current_position_info()
reader.consume_boxes(info, bi, br, bf)
- assert bi == [ConstInt(222), ConstInt(333)]
+ assert_same(bi, [ConstInt(222), ConstInt(333)])
bi, br, bf = [None]*2, [None]*1, [None]*0
info = MyBlackholeInterp([lltype.Signed, llmemory.GCREF,
lltype.Signed]).get_current_position_info()
reader.consume_boxes(info, bi, br, bf)
b3s = reader.liveboxes[2]
- assert bi == [b1s, b3s]
- assert br == [b2s]
+ assert_same(bi, [b1s, b3s])
+ assert_same(br, [b2s])
#
+def assert_same(list1, list2):
+ assert len(list1) == len(list2)
+ for b1, b2 in zip(list1, list2):
+ assert b1.same_box(b2)
+
def test_simple_read_tagged_ints():
storage = Storage()
storage.rd_consts = []
@@ -1023,11 +1028,11 @@
metainterp = MyMetaInterp()
reader = ResumeDataFakeReader(storage, newboxes, metainterp)
lst = reader.consume_boxes()
- assert lst == [b1t, b1t, b3t]
+ assert_same(lst, [b1t, b1t, b3t])
lst = reader.consume_boxes()
- assert lst == [ConstInt(2), ConstInt(3)]
+ assert_same(lst, [ConstInt(2), ConstInt(3)])
lst = reader.consume_boxes()
- assert lst == [b1t, ConstInt(1), b1t, b1t]
+ assert_same(lst, [b1t, ConstInt(1), b1t, b1t])
assert metainterp.trace == []
@@ -1044,11 +1049,11 @@
reader = ResumeDataFakeReader(storage, newboxes, metainterp)
lst = reader.consume_boxes()
c1t = ConstInt(111)
- assert lst == [c1t, b2t, b3t]
+ assert_same(lst, [c1t, b2t, b3t])
lst = reader.consume_boxes()
- assert lst == [ConstInt(2), ConstInt(3)]
+ assert_same(lst, [ConstInt(2), ConstInt(3)])
lst = reader.consume_boxes()
- assert lst == [c1t, ConstInt(1), c1t, b2t]
+ assert_same(lst, [c1t, ConstInt(1), c1t, b2t])
assert metainterp.trace == []
@@ -1114,9 +1119,11 @@
# check that we get the operations in 'expected', in a possibly different
# order.
assert len(trace) == len(expected)
- for x in trace:
- assert x in expected
- expected.remove(x)
+ with CompareableConsts():
+ for x in trace:
+ assert x in expected
+ expected.remove(x)
+
ptr = b2t.value._obj.container._as_ptr()
assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE)
assert ptr.value == 111
@@ -1126,6 +1133,18 @@
assert ptr2.parent.value == 33
assert ptr2.parent.next == ptr
+class CompareableConsts(object):
+ def __init__(self):
+ self.oldeq = None
+
+ def __enter__(self):
+ assert self.oldeq is None
+ self.oldeq = Const.__eq__
+ Const.__eq__ = Const.same_box
+
+ def __exit__(self, type, value, traceback):
+ Const.__eq__ = self.oldeq
+
def test_virtual_adder_make_varray():
b2s, b4s = [BoxPtr(), BoxInt(4)]
c1s = ConstInt(111)
@@ -1163,8 +1182,9 @@
(rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None,
LLtypeMixin.arraydescr),
]
- for x, y in zip(expected, trace):
- assert x == y
+ with CompareableConsts():
+ for x, y in zip(expected, trace):
+ assert x == y
#
ptr = b2t.value._obj.container._as_ptr()
assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed))
@@ -1207,8 +1227,9 @@
(rop.SETFIELD_GC, [b2t, c1s], None, LLtypeMixin.adescr),
(rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
]
- for x, y in zip(expected, trace):
- assert x == y
+ with CompareableConsts():
+ for x, y in zip(expected, trace):
+ assert x == y
#
ptr = b2t.value._obj.container._as_ptr()
assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S)
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -3,7 +3,8 @@
from pypy.jit.codewriter.policy import StopAtXPolicy
from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
from pypy.rlib.debug import debug_print
-from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
+from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted,\
+ promote_string
from pypy.rlib.rstring import StringBuilder
from pypy.rpython.ootypesystem import ootype
@@ -507,6 +508,18 @@
'jump': 1, 'int_is_true': 1,
'guard_not_invalidated': 1})
+ def test_promote_string(self):
+ driver = JitDriver(greens = [], reds = ['n'])
+
+ def f(n):
+ while n < 21:
+ driver.jit_merge_point(n=n)
+ promote_string(str(n % 3))
+ n += 1
+ return 0
+
+ self.meta_interp(f, [0])
+ self.check_loops(call=3 + 1) # one for int2str
#class TestOOtype(StringTests, OOJitMixin):
# CALL = "oosend"
diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py
--- a/pypy/jit/metainterp/test/test_virtualref.py
+++ b/pypy/jit/metainterp/test/test_virtualref.py
@@ -3,6 +3,7 @@
from pypy.rpython.llinterp import LLException
from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
from pypy.rlib.objectmodel import compute_unique_id
from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
from pypy.jit.metainterp.resoperation import rop
@@ -595,6 +596,65 @@
res = self.meta_interp(fn, [10])
assert res == 6
+ def test_is_virtual(self):
+ myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+ class X:
+ pass
+ @dont_look_inside
+ def residual(vref):
+ return vref.virtual
+ #
+ def f(n):
+ res1 = -42
+ while n > 0:
+ myjitdriver.jit_merge_point(n=n, res1=res1)
+ x = X()
+ vref = virtual_ref(x)
+ res1 = residual(vref)
+ virtual_ref_finish(vref, x)
+ n -= 1
+ return res1
+ #
+ res = self.meta_interp(f, [10])
+ assert res == 1
+
+ def test_is_not_virtual_none(self):
+ myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+ @dont_look_inside
+ def residual(vref):
+ return vref.virtual
+ #
+ def f(n):
+ res1 = -42
+ while n > 0:
+ myjitdriver.jit_merge_point(n=n, res1=res1)
+ res1 = residual(vref_None)
+ n -= 1
+ return res1
+ #
+ res = self.meta_interp(f, [10])
+ assert res == 0
+
+ def test_is_not_virtual_non_none(self):
+ myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+ class X:
+ pass
+ @dont_look_inside
+ def residual(vref):
+ return vref.virtual
+ #
+ def f(n):
+ res1 = -42
+ while n > 0:
+ myjitdriver.jit_merge_point(n=n, res1=res1)
+ x = X()
+ res1 = residual(non_virtual_ref(x))
+ n -= 1
+ return res1
+ #
+ res = self.meta_interp(f, [10])
+ assert res == 0
+
class TestLLtype(VRefTests, LLJitMixin):
pass
diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py
--- a/pypy/jit/metainterp/virtualref.py
+++ b/pypy/jit/metainterp/virtualref.py
@@ -39,6 +39,7 @@
def replace_force_virtual_with_call(self, graphs):
# similar to rvirtualizable2.replace_force_virtualizable_with_call().
c_force_virtual_ptr = None
+ c_is_virtual_ptr = None
force_virtual_count = 0
for graph in graphs:
for block in graph.iterblocks():
@@ -52,6 +53,13 @@
op.opname = 'direct_call'
op.args = [c_force_virtual_ptr, op.args[0]]
force_virtual_count += 1
+ #
+ if op.opname == 'jit_is_virtual':
+ if c_is_virtual_ptr is None:
+ c_is_virtual_ptr = self.get_is_virtual_fnptr()
+ #
+ op.opname = 'direct_call'
+ op.args = [c_is_virtual_ptr, op.args[0]]
#
if c_force_virtual_ptr is not None:
log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
@@ -129,6 +137,17 @@
force_virtual_if_necessary)
return inputconst(lltype.typeOf(funcptr), funcptr)
+ def get_is_virtual_fnptr(self):
+ #
+ def is_virtual(inst):
+ if not inst:
+ return False
+ return inst.typeptr == self.jit_virtual_ref_vtable
+ #
+ FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+ funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+ return inputconst(lltype.typeOf(funcptr), funcptr)
+
def force_virtual(self, inst):
vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
token = vref.virtual_token
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -20,6 +20,9 @@
'any' : 'app_functional.any',
'all' : 'app_functional.all',
'sum' : 'app_functional.sum',
+ 'map' : 'app_functional.map',
+ 'reduce' : 'app_functional.reduce',
+ 'filter' : 'app_functional.filter',
'vars' : 'app_inspect.vars',
'dir' : 'app_inspect.dir',
@@ -86,11 +89,8 @@
'enumerate' : 'functional.W_Enumerate',
'min' : 'functional.min',
'max' : 'functional.max',
- 'map' : 'functional.map',
'zip' : 'functional.zip',
- 'reduce' : 'functional.reduce',
'reversed' : 'functional.reversed',
- 'filter' : 'functional.filter',
'super' : 'descriptor.W_Super',
'staticmethod' : 'descriptor.StaticMethod',
'classmethod' : 'descriptor.ClassMethod',
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -48,4 +48,118 @@
# Very intentionally *not* +=, that would have different semantics if
# start was a mutable type, such as a list
last = last + x
- return last
\ No newline at end of file
+ return last
+
+def map(func, *collections):
+ """map(function, sequence[, sequence, ...]) -> list
+
+Return a list of the results of applying the function to the items of
+the argument sequence(s). If more than one sequence is given, the
+function is called with an argument list consisting of the corresponding
+item of each sequence, substituting None for missing values when not all
+sequences have the same length. If the function is None, return a list of
+the items of the sequence (or a list of tuples if more than one sequence)."""
+ if not collections:
+ raise TypeError("map() requires at least two arguments")
+ num_collections = len(collections)
+ none_func = func is None
+ if num_collections == 1:
+ if none_func:
+ return list(collections[0])
+ else:
+ # Special case for the really common case of a single collection,
+ # this can be eliminated if we could unroll that loop that creates
+ # `args` based on whether or not len(collections) was constant
+ result = []
+ for item in collections[0]:
+ result.append(func(item))
+ return result
+ result = []
+ # Pair of (iterator, has_finished)
+ iterators = [(iter(seq), False) for seq in collections]
+ while True:
+ cont = False
+ args = []
+ for idx, (iterator, has_finished) in enumerate(iterators):
+ val = None
+ if not has_finished:
+ try:
+ val = next(iterator)
+ except StopIteration:
+ iterators[idx] = (None, True)
+ else:
+ cont = True
+ args.append(val)
+ args = tuple(args)
+ if cont:
+ if none_func:
+ result.append(args)
+ else:
+ result.append(func(*args))
+ else:
+ return result
+
+sentinel = object()
+
+def reduce(func, sequence, initial=sentinel):
+ """reduce(function, sequence[, initial]) -> value
+
+Apply a function of two arguments cumulatively to the items of a sequence,
+from left to right, so as to reduce the sequence to a single value.
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+((((1+2)+3)+4)+5). If initial is present, it is placed before the items
+of the sequence in the calculation, and serves as a default when the
+sequence is empty."""
+ iterator = iter(sequence)
+ if initial is sentinel:
+ try:
+ initial = next(iterator)
+ except StopIteration:
+ raise TypeError("reduce() of empty sequence with no initial value")
+ result = initial
+ for item in iterator:
+ result = func(result, item)
+ return result
+
+def filter(func, seq):
+ """filter(function or None, sequence) -> list, tuple, or string
+
+Return those items of sequence for which function(item) is true. If
+function is None, return the items that are true. If sequence is a tuple
+or string, return the same type, else return a list."""
+ if func is None:
+ func = bool
+ if isinstance(seq, str):
+ return _filter_string(func, seq, str)
+ elif isinstance(seq, unicode):
+ return _filter_string(func, seq, unicode)
+ elif isinstance(seq, tuple):
+ return _filter_tuple(func, seq)
+ result = []
+ for item in seq:
+ if func(item):
+ result.append(item)
+ return result
+
+def _filter_string(func, string, str_type):
+ if func is bool and type(string) is str_type:
+ return string
+ result = []
+ for i in range(len(string)):
+ # You must call __getitem__ on the strings, simply iterating doesn't
+ # work :/
+ item = string[i]
+ if func(item):
+ if not isinstance(item, str_type):
+ raise TypeError("__getitem__ returned a non-string type")
+ result.append(item)
+ return str_type().join(result)
+
+def _filter_tuple(func, seq):
+ result = []
+ for i in range(len(seq)):
+ # Again, must call __getitem__, at least there are tests.
+ item = seq[i]
+ if func(item):
+ result.append(item)
+ return tuple(result)
\ No newline at end of file
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -84,8 +84,8 @@
raise OperationError(space.w_TypeError,
w('eval() arg 1 must be a string or code object'))
- caller = space.getexecutioncontext().gettopframe_nohidden()
if space.is_w(w_globals, space.w_None):
+ caller = space.getexecutioncontext().gettopframe_nohidden()
if caller is None:
w_globals = space.newdict()
if space.is_w(w_locals, space.w_None):
@@ -97,13 +97,6 @@
elif space.is_w(w_locals, space.w_None):
w_locals = w_globals
- try:
- space.getitem(w_globals, space.wrap('__builtins__'))
- except OperationError, e:
- if not e.match(space, space.w_KeyError):
- raise
- if caller is not None:
- w_builtin = space.builtin.pick_builtin(caller.w_globals)
- space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+ space.builtin.pick_builtin(w_globals)
return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -201,118 +201,6 @@
"""
return min_max(space, __args__, "min")
- at unwrap_spec(collections_w="args_w")
-def map(space, w_func, collections_w):
- """does 3 separate things, hence this enormous docstring.
- 1. if function is None, return a list of tuples, each with one
- item from each collection. If the collections have different
- lengths, shorter ones are padded with None.
-
- 2. if function is not None, and there is only one collection,
- apply function to every item in the collection and return a
- list of the results.
-
- 3. if function is not None, and there are several collections,
- repeatedly call the function with one argument from each
- collection. If the collections have different lengths,
- shorter ones are padded with None
- """
- if not collections_w:
- msg = "map() requires at least two arguments"
- raise OperationError(space.w_TypeError, space.wrap(msg))
- none_func = space.is_w(w_func, space.w_None)
- if len(collections_w) == 1:
- w_collection = collections_w[0]
- if none_func:
- result_w = space.unpackiterable(w_collection)
- else:
- result_w = map_single_collection(space, w_func, w_collection)
- else:
- result_w = map_multiple_collections(space, w_func, collections_w,
- none_func)
- return space.newlist(result_w)
-
-def map_single_collection(space, w_func, w_collection):
- """Special case for 'map(func, coll)', where 'func' is not None and there
- is only one 'coll' argument."""
- w_iter = space.iter(w_collection)
- # xxx special hacks for speed
- from pypy.interpreter import function, pycode
- if isinstance(w_func, function.Function):
- # xxx compatibility issue: what if func_code is modified in the
- # middle of running map()?? That's far too obscure for me to care...
- code = w_func.getcode()
- fast_natural_arity = code.fast_natural_arity
- if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL):
- assert isinstance(code, pycode.PyCode)
- return map_single_user_function(code, w_func, w_iter)
- # /xxx end of special hacks
- return map_single_other_callable(space, w_func, w_iter)
-
-def map_single_other_callable(space, w_func, w_iter):
- result_w = []
- while True:
- try:
- w_item = space.next(w_iter)
- except OperationError, e:
- if not e.match(space, space.w_StopIteration):
- raise
- break
- result_w.append(space.call_function(w_func, w_item))
- return result_w
-map_single_other_callable._dont_inline_ = True
-
-from pypy.rlib.jit import JitDriver
-mapjitdriver = JitDriver(greens = ['code'],
- reds = ['w_func', 'w_iter', 'result_w'])
-def map_single_user_function(code, w_func, w_iter):
- result_w = []
- while True:
- mapjitdriver.can_enter_jit(code=code, w_func=w_func,
- w_iter=w_iter, result_w=result_w)
- mapjitdriver.jit_merge_point(code=code, w_func=w_func,
- w_iter=w_iter, result_w=result_w)
- space = w_func.space
- try:
- w_item = space.next(w_iter)
- except OperationError, e:
- if not e.match(space, space.w_StopIteration):
- raise
- break
- new_frame = space.createframe(code, w_func.w_func_globals,
- w_func)
- new_frame.locals_stack_w[0] = w_item
- w_res = new_frame.run()
- result_w.append(w_res)
- return result_w
-
-def map_multiple_collections(space, w_func, collections_w, none_func):
- result_w = []
- iterators_w = [space.iter(w_seq) for w_seq in collections_w]
- num_iterators = len(iterators_w)
- while True:
- cont = False
- args_w = [space.w_None] * num_iterators
- for i in range(num_iterators):
- if iterators_w[i] is not None:
- try:
- args_w[i] = space.next(iterators_w[i])
- except OperationError, e:
- if not e.match(space, space.w_StopIteration):
- raise
- iterators_w[i] = None
- else:
- cont = True
- if not cont:
- break
- w_args = space.newtuple(args_w)
- if none_func:
- w_res = w_args
- else:
- w_res = space.call(w_func, w_args)
- result_w.append(w_res)
- return result_w
-
@unwrap_spec(sequences_w="args_w")
def zip(space, sequences_w):
"""Return a list of tuples, where the nth tuple contains every nth item of
@@ -334,90 +222,6 @@
return space.newlist(result_w)
result_w.append(space.newtuple(items_w))
-def reduce(space, w_func, w_sequence, w_initial=NoneNotWrapped):
- """ Apply function of two arguments cumulatively to the items of sequence,
- from left to right, so as to reduce the sequence to a single value.
- Optionally begin with an initial value.
- """
- w_iter = space.iter(w_sequence)
- if w_initial is None:
- try:
- w_initial = space.next(w_iter)
- except OperationError, e:
- if e.match(space, space.w_StopIteration):
- msg = "reduce() of empty sequence with no initial value"
- raise OperationError(space.w_TypeError, space.wrap(msg))
- raise
- w_result = w_initial
- while True:
- try:
- w_next = space.next(w_iter)
- except OperationError, e:
- if not e.match(space, space.w_StopIteration):
- raise
- break
- w_result = space.call_function(w_func, w_result, w_next)
- return w_result
-
-def filter(space, w_func, w_seq):
- """construct a list of those elements of collection for which function
- is True. If function is None, then return the items in the sequence
- which are True.
- """
- if space.is_true(space.isinstance(w_seq, space.w_str)):
- return _filter_string(space, w_func, w_seq, space.w_str)
- if space.is_true(space.isinstance(w_seq, space.w_unicode)):
- return _filter_string(space, w_func, w_seq, space.w_unicode)
- if space.is_true(space.isinstance(w_seq, space.w_tuple)):
- return _filter_tuple(space, w_func, w_seq)
- w_iter = space.iter(w_seq)
- result_w = []
- none_func = space.is_w(w_func, space.w_None)
- while True:
- try:
- w_next = space.next(w_iter)
- except OperationError, e:
- if not e.match(space, space.w_StopIteration):
- raise
- break
- if none_func:
- w_keep = w_next
- else:
- w_keep = space.call_function(w_func, w_next)
- if space.is_true(w_keep):
- result_w.append(w_next)
- return space.newlist(result_w)
-
-def _filter_tuple(space, w_func, w_tuple):
- none_func = space.is_w(w_func, space.w_None)
- length = space.len_w(w_tuple)
- result_w = []
- for i in range(length):
- w_item = space.getitem(w_tuple, space.wrap(i))
- if none_func:
- w_keep = w_item
- else:
- w_keep = space.call_function(w_func, w_item)
- if space.is_true(w_keep):
- result_w.append(w_item)
- return space.newtuple(result_w)
-
-def _filter_string(space, w_func, w_string, w_str_type):
- none_func = space.is_w(w_func, space.w_None)
- if none_func and space.is_w(space.type(w_string), w_str_type):
- return w_string
- length = space.len_w(w_string)
- result_w = []
- for i in range(length):
- w_item = space.getitem(w_string, space.wrap(i))
- if none_func or space.is_true(space.call_function(w_func, w_item)):
- if not space.is_true(space.isinstance(w_item, w_str_type)):
- msg = "__getitem__ returned a non-string type"
- raise OperationError(space.w_TypeError, space.wrap(msg))
- result_w.append(w_item)
- w_empty = space.call_function(w_str_type)
- return space.call_method(w_empty, "join", space.newlist(result_w))
-
class W_Enumerate(Wrappable):
def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -147,7 +147,7 @@
assert list(xrange(A())) == [0, 1, 2, 3, 4]
assert list(xrange(0, A())) == [0, 1, 2, 3, 4]
assert list(xrange(0, 10, A())) == [0, 5]
-
+
def test_xrange_float(self):
assert list(xrange(0.1, 2.0, 1.1)) == [0, 1]
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -19,6 +19,11 @@
# - normal: self.sthread != None, not is_empty_handle(self.h)
# - finished: self.sthread != None, is_empty_handle(self.h)
+ def __del__(self):
+ sthread = self.sthread
+ if sthread is not None and not sthread.is_empty_handle(self.h):
+ sthread.destroy(self.h)
+
def check_sthread(self):
ec = self.space.getexecutioncontext()
if ec.stacklet_thread is not self.sthread:
@@ -28,6 +33,8 @@
def descr_init(self, w_callable, __args__):
if self.sthread is not None:
raise geterror(self.space, "continulet already __init__ialized")
+ sthread = build_sthread(self.space)
+ workaround_disable_jit(sthread)
#
# hackish: build the frame "by hand", passing it the correct arguments
space = self.space
@@ -41,7 +48,6 @@
self.bottomframe = bottomframe
#
global_state.origin = self
- sthread = build_sthread(self.space)
self.sthread = sthread
h = sthread.new(new_stacklet_callback)
post_switch(sthread, h)
@@ -71,6 +77,7 @@
global_state.clear()
raise geterror(self.space, "continulet already finished")
self.check_sthread()
+ workaround_disable_jit(self.sthread)
#
global_state.origin = self
if to is None:
@@ -259,6 +266,16 @@
sthread = ec.stacklet_thread = SThread(space, ec)
return sthread
+def workaround_disable_jit(sthread):
+ # A bad workaround to kill the JIT anywhere in this thread.
+ # This forces all the frames. It's a bad workaround because
+ # it takes O(depth) time, and it will cause some "abort:
+ # vable escape" in the JIT. The goal is to prevent any frame
+ # from being still virtuals, because the JIT generates code
+ # to un-virtualizable them "on demand" by loading values based
+ # on FORCE_TOKEN, which is an address in the stack.
+ sthread.ec.force_all_frames()
+
# ____________________________________________________________
def permute(space, args_w):
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,5 @@
from __future__ import with_statement
-import py
+import py, os, errno
from pypy.conftest import gettestobjspace, option
@@ -257,6 +257,39 @@
assert self.temppath in g.getvalue()
+class AppTestNonblocking(object):
+ def setup_class(cls):
+ from pypy.module._file.interp_file import W_File
+
+ cls.old_read = os.read
+
+ if option.runappdirect:
+ py.test.skip("works with internals of _file impl on py.py")
+
+ state = [0]
+ def read(fd, n=None):
+ if fd != 42:
+ return cls.old_read(fd, n)
+ if state[0] == 0:
+ state[0] += 1
+ return "xyz"
+ if state[0] < 3:
+ state[0] += 1
+ raise OSError(errno.EAGAIN, "xyz")
+ return ''
+ os.read = read
+ stdin = W_File(cls.space)
+ stdin.file_fdopen(42, "r", 1)
+ stdin.name = '<stdin>'
+ cls.w_stream = stdin
+
+ def teardown_class(cls):
+ os.read = cls.old_read
+
+ def test_nonblocking_file(self):
+ res = self.stream.read()
+ assert res == 'xyz'
+
class AppTestConcurrency(object):
# these tests only really make sense on top of a translated pypy-c,
# because on top of py.py the inner calls to os.write() don't
@@ -374,24 +407,24 @@
with self.file(self.temppath, 'w') as f:
f.write('foo')
assert f.closed
-
+
with self.file(self.temppath, 'r') as f:
s = f.readline()
assert s == "foo"
assert f.closed
-
+
def test_subclass_with(self):
file = self.file
class C(file):
def __init__(self, *args, **kwargs):
self.subclass_closed = False
file.__init__(self, *args, **kwargs)
-
+
def close(self):
self.subclass_closed = True
file.close(self)
-
+
with C(self.temppath, 'w') as f:
pass
assert f.subclass_closed
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -161,9 +161,6 @@
@binop
def mul(self, v1, v2):
return v1 * v2
- @binop
- def div(self, v1, v2):
- return v1 / v2
@unaryop
def pos(self, v):
@@ -217,6 +214,14 @@
return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
@binop
+ def div(self, v1, v2):
+ try:
+ return v1 / v2
+ except ZeroDivisionError:
+ if v1 == v2 == 0.0:
+ return rfloat.NAN
+ return rfloat.copysign(rfloat.INFINITY, v1 * v2)
+ @binop
def mod(self, v1, v2):
return math.fmod(v1, v2)
@binop
@@ -295,6 +300,11 @@
return str(widen(self.unbox(item)))
@binop
+ def div(self, v1, v2):
+ if v2 == 0:
+ return 0
+ return v1 / v2
+ @binop
def mod(self, v1, v2):
return v1 % v2
@@ -426,23 +436,22 @@
pass
if LONG_BIT == 32:
- class W_LongDtype(W_Int32Dtype):
- pass
+ long_dtype = W_Int32Dtype
+ ulong_dtype = W_UInt32Dtype
+elif LONG_BIT == 64:
+ long_dtype = W_Int64Dtype
+ ulong_dtype = W_UInt64Dtype
+else:
+ assert False
- class W_ULongDtype(W_UInt32Dtype):
- pass
-else:
- class W_LongDtype(W_Int64Dtype):
- pass
+class W_LongDtype(long_dtype):
+ num = 7
+ aliases = ["l"]
+ applevel_types = ["int"]
- class W_ULongDtype(W_UInt64Dtype):
- pass
-
-W_LongDtype.num = 7
-W_LongDtype.aliases = ["l"]
-W_LongDtype.applevel_types = ["int"]
-W_ULongDtype.num = 8
-W_ULongDtype.aliases = ["L"]
+class W_ULongDtype(ulong_dtype):
+ num = 8
+ aliases = ["L"]
W_Float32Dtype = create_low_level_dtype(
num = 11, kind = FLOATINGLTR, name = "float32",
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -285,7 +285,9 @@
assert b[i] == i * 5
def test_div(self):
- from numpy import array, dtype
+ from math import isnan
+ from numpy import array, dtype, inf
+
a = array(range(1, 6))
b = a / a
for i in range(5):
@@ -297,6 +299,24 @@
for i in range(5):
assert b[i] == 1
+ a = array([-1, 0, 1])
+ b = array([0, 0, 0])
+ c = a / b
+ assert (c == [0, 0, 0]).all()
+
+ a = array([-1.0, 0.0, 1.0])
+ b = array([0.0, 0.0, 0.0])
+ c = a / b
+ assert c[0] == -inf
+ assert isnan(c[1])
+ assert c[2] == inf
+
+ b = array([-0.0, -0.0, -0.0])
+ c = a / b
+ assert c[0] == inf
+ assert isnan(c[1])
+ assert c[2] == -inf
+
def test_div_other(self):
from numpy import array
a = array(range(5))
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -420,12 +420,15 @@
import sys
if not hasattr(os, "fork"):
skip("Need fork() to test execv()")
+ try:
+ output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+ except UnicodeEncodeError:
+ skip("encoding not good enough")
pid = os.fork()
if pid == 0:
os.execv(u"/bin/sh", ["sh", "-c",
u"echo caf\xe9 \u1234 > onefile"])
os.waitpid(pid, 0)
- output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
assert open("onefile").read() == output
os.unlink("onefile")
@@ -445,13 +448,16 @@
import sys
if not hasattr(os, "fork"):
skip("Need fork() to test execve()")
+ try:
+ output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+ except UnicodeEncodeError:
+ skip("encoding not good enough")
pid = os.fork()
if pid == 0:
os.execve(u"/bin/sh", ["sh", "-c",
u"echo caf\xe9 \u1234 > onefile"],
{'ddd': 'xxx'})
os.waitpid(pid, 0)
- output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
assert open("onefile").read() == output
os.unlink("onefile")
pass # <- please, inspect.getsource(), don't crash
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,20 +137,15 @@
def jump_absolute(self, jumpto, _, ec=None):
if we_are_jitted():
- # Normally, the tick counter is decremented by 100 for every
- # Python opcode. Here, to better support JIT compilation of
- # small loops, we decrement it by a possibly smaller constant.
- # We get the maximum 100 when the (unoptimized) trace length
- # is at least 3200 (a bit randomly).
- trace_length = r_uint(current_trace_length())
- decr_by = trace_length // 32
- if decr_by < 1:
- decr_by = 1
- elif decr_by > 100: # also if current_trace_length() returned -1
- decr_by = 100
+ #
+ # assume that only threads are using the bytecode counter
+ decr_by = 0
+ if self.space.actionflag.has_bytecode_counter: # constant-folded
+ if self.space.threadlocals.gil_ready: # quasi-immutable field
+ decr_by = _get_adapted_tick_counter()
#
self.last_instr = intmask(jumpto)
- ec.bytecode_trace(self, intmask(decr_by))
+ ec.bytecode_trace(self, decr_by)
jumpto = r_uint(self.last_instr)
#
pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto,
@@ -158,6 +153,20 @@
is_being_profiled=self.is_being_profiled)
return jumpto
+def _get_adapted_tick_counter():
+ # Normally, the tick counter is decremented by 100 for every
+ # Python opcode. Here, to better support JIT compilation of
+ # small loops, we decrement it by a possibly smaller constant.
+ # We get the maximum 100 when the (unoptimized) trace length
+ # is at least 3200 (a bit randomly).
+ trace_length = r_uint(current_trace_length())
+ decr_by = trace_length // 32
+ if decr_by < 1:
+ decr_by = 1
+ elif decr_by > 100: # also if current_trace_length() returned -1
+ decr_by = 100
+ return intmask(decr_by)
+
PyCode__initialize = PyCode._initialize
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -225,6 +225,8 @@
# strip comment
if '#' in line:
line = line[:line.index('#')]
+ if line.strip() == 'guard_not_invalidated?':
+ return 'guard_not_invalidated', None, [], '...', False
# find the resvar, if any
if ' = ' in line:
resvar, _, line = line.partition(' = ')
@@ -249,7 +251,7 @@
descr = descr[len('descr='):]
else:
descr = None
- return opname, resvar, args, descr
+ return opname, resvar, args, descr, True
@classmethod
def preprocess_expected_src(cls, src):
@@ -258,13 +260,23 @@
# replaced with the corresponding operations, so that tests don't have
# to repeat it every time
ticker_check = """
+ guard_not_invalidated?
+ ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
+ ticker_cond0 = int_lt(ticker0, 0)
+ guard_false(ticker_cond0, descr=...)
+ """
+ src = src.replace('--TICK--', ticker_check)
+ #
+ # this is the ticker check generated if we have threads
+ thread_ticker_check = """
+ guard_not_invalidated?
ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
ticker1 = int_sub(ticker0, 1)
setfield_raw(ticker_address, ticker1, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
ticker_cond0 = int_lt(ticker1, 0)
guard_false(ticker_cond0, descr=...)
"""
- src = src.replace('--TICK--', ticker_check)
+ src = src.replace('--THREAD-TICK--', thread_ticker_check)
#
# this is the ticker check generated in PyFrame.handle_operation_error
exc_ticker_check = """
@@ -298,7 +310,7 @@
if not cond:
raise InvalidMatch(message, frame=sys._getframe(1))
- def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr)):
+ def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
self._assert(op.name == exp_opname, "operation mismatch")
self.match_var(op.res, exp_res)
if exp_args != ['...']:
@@ -341,7 +353,7 @@
what is after the '...'
"""
iter_exp_ops = iter(expected_ops)
- iter_ops = iter(self.ops)
+ iter_ops = RevertableIterator(self.ops)
for opindex, exp_op in enumerate(iter_exp_ops):
try:
if exp_op == '...':
@@ -360,6 +372,9 @@
break
self.match_op(op, exp_op)
except InvalidMatch, e:
+ if exp_op[4] is False: # optional operation
+ iter_ops.revert_one()
+ continue # try to match with the next exp_op
e.opindex = opindex
raise
#
@@ -398,3 +413,18 @@
else:
return True
+
+class RevertableIterator(object):
+ def __init__(self, sequence):
+ self.sequence = sequence
+ self.index = 0
+ def __iter__(self):
+ return self
+ def next(self):
+ index = self.index
+ if index == len(self.sequence):
+ raise StopIteration
+ self.index = index + 1
+ return self.sequence[index]
+ def revert_one(self):
+ self.index -= 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -145,15 +145,17 @@
def test_parse_op(self):
res = OpMatcher.parse_op(" a = int_add( b, 3 ) # foo")
- assert res == ("int_add", "a", ["b", "3"], None)
+ assert res == ("int_add", "a", ["b", "3"], None, True)
res = OpMatcher.parse_op("guard_true(a)")
- assert res == ("guard_true", None, ["a"], None)
+ assert res == ("guard_true", None, ["a"], None, True)
res = OpMatcher.parse_op("setfield_gc(p0, i0, descr=<foobar>)")
- assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>")
+ assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>", True)
res = OpMatcher.parse_op("i1 = getfield_gc(p0, descr=<foobar>)")
- assert res == ("getfield_gc", "i1", ["p0"], "<foobar>")
+ assert res == ("getfield_gc", "i1", ["p0"], "<foobar>", True)
res = OpMatcher.parse_op("p0 = force_token()")
- assert res == ("force_token", "p0", [], None)
+ assert res == ("force_token", "p0", [], None, True)
+ res = OpMatcher.parse_op("guard_not_invalidated?")
+ assert res == ("guard_not_invalidated", None, [], '...', False)
def test_exact_match(self):
loop = """
@@ -341,7 +343,7 @@
# this is the actual loop
'int_lt', 'guard_true', 'int_add',
# this is the signal checking stuff
- 'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+ 'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
'jump'
]
@@ -407,7 +409,7 @@
# this is the actual loop
'int_lt', 'guard_true', 'force_token', 'int_add',
# this is the signal checking stuff
- 'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+ 'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
'jump'
]
@@ -425,10 +427,9 @@
guard_true(i6, descr=...)
i8 = int_add(i4, 1)
# signal checking stuff
+ guard_not_invalidated(descr=...)
i10 = getfield_raw(37212896, descr=<.* pypysig_long_struct.c_value .*>)
- i12 = int_sub(i10, 1)
- setfield_raw(37212896, i12, descr=<.* pypysig_long_struct.c_value .*>)
- i14 = int_lt(i12, 0)
+ i14 = int_lt(i10, 0)
guard_false(i14, descr=...)
jump(p0, p1, p2, p3, i8, descr=...)
""")
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -329,4 +329,20 @@
guard_false(i28, descr=...)
i30 = int_lshift(i20, 24)
i31 = int_or(i26, i30)
- """ % {"32_bit_only": extra})
\ No newline at end of file
+ """ % {"32_bit_only": extra})
+
+ def test_eval(self):
+ def main():
+ i = 1
+ a = compile('x+x+x+x+x+x', 'eval', 'eval')
+ b = {'x': 7}
+ while i < 1000:
+ y = eval(a,b,b) # ID: eval
+ i += 1
+ return y
+
+ log = self.run(main)
+ assert log.result == 42
+ # the following assertion fails if the loop was cancelled due
+ # to "abort: vable escape"
+ assert len(log.loops_by_id("eval")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -156,4 +156,41 @@
i40 = int_sub(i4, 1)
--TICK--
jump(p0, p1, p2, p3, i40, i38, descr=<Loop0>)
- """)
\ No newline at end of file
+ """)
+
+ def test_getattr_promote(self):
+ def main(n):
+ class A(object):
+ def meth_a(self):
+ return 1
+ def meth_b(self):
+ return 2
+ a = A()
+
+ l = ['a', 'b']
+ s = 0
+ for i in range(n):
+ name = 'meth_' + l[i & 1]
+ meth = getattr(a, name) # ID: getattr
+ s += meth()
+ return s
+
+ log = self.run(main, [1000])
+ assert log.result == main(1000)
+ loops = log.loops_by_filename(self.filepath)
+ assert len(loops) == 2
+ for loop in loops:
+ loop.match_by_id('getattr','''
+ guard_not_invalidated(descr=...)
+ i32 = strlen(p31)
+ i34 = int_add(5, i32)
+ p35 = newstr(i34)
+ strsetitem(p35, 0, 109)
+ strsetitem(p35, 1, 101)
+ strsetitem(p35, 2, 116)
+ strsetitem(p35, 3, 104)
+ strsetitem(p35, 4, 95)
+ copystrcontent(p31, p35, 0, 5, i32)
+ i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<SignedCallDescr>)
+ guard_value(i49, 1, descr=<Guard8>)
+ ''')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -0,0 +1,28 @@
+from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+
+
+class TestThread(BaseTestPyPyC):
+ def test_simple(self):
+ def main(n):
+ import thread
+ def f():
+ i = 0
+ while i < n:
+ i += 1
+ done.release()
+
+ done = thread.allocate_lock()
+ done.acquire()
+ thread.start_new_thread(f, ())
+ done.acquire()
+ return 0
+ log = self.run(main, [500])
+ assert round(log.result, 6) == round(main(500), 6)
+ loop, = log.loops_by_filename(self.filepath)
+ assert loop.match("""
+ i2 = int_lt(i0, i1)
+ guard_true(i2, descr=...)
+ i3 = int_add(i0, 1)
+ --THREAD-TICK--
+ jump(..., descr=<Loop0>)
+ """)
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -109,8 +109,11 @@
p = pypysig_getaddr_occurred()
value = p.c_value
if self.has_bytecode_counter: # this 'if' is constant-folded
- value -= by
- p.c_value = value
+ if jit.isconstant(by) and by == 0:
+ pass # normally constant-folded too
+ else:
+ value -= by
+ p.c_value = value
return value
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -47,7 +47,7 @@
'pypy_initial_path' : 'state.pypy_initial_path',
'_getframe' : 'vm._getframe',
- '_current_frames' : 'vm._current_frames',
+ '_current_frames' : 'currentframes._current_frames',
'setrecursionlimit' : 'vm.setrecursionlimit',
'getrecursionlimit' : 'vm.getrecursionlimit',
'setcheckinterval' : 'vm.setcheckinterval',
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/currentframes.py
@@ -0,0 +1,78 @@
+"""
+Implementation of the 'sys._current_frames()' routine.
+"""
+from pypy.interpreter import gateway
+
+app = gateway.applevel('''
+"NOT_RPYTHON"
+import __builtin__
+
+class fake_code(object):
+ co_name = "?"
+ co_filename = "?"
+ co_firstlineno = 0
+
+class fake_frame(object):
+ f_back = None
+ f_builtins = __builtin__.__dict__
+ f_code = fake_code()
+ f_exc_traceback = None
+ f_exc_type = None
+ f_exc_value = None
+ f_globals = {}
+ f_lasti = -1
+ f_lineno = 0
+ f_locals = {}
+ f_restricted = False
+ f_trace = None
+
+ def __init__(self, f):
+ if f is not None:
+ for name in ["f_builtins", "f_code", "f_globals", "f_lasti",
+ "f_lineno"]:
+ setattr(self, name, getattr(f, name))
+''')
+
+def _current_frames(space):
+ """_current_frames() -> dictionary
+
+ Return a dictionary mapping each current thread T's thread id to T's
+ current stack "frame". Functions in the traceback module can build the
+ call stack given such a frame.
+
+ Note that in PyPy this returns fake frame objects, to avoid a runtime
+ penalty everywhere with the JIT. (So far these fake frames can be
+ completely uninformative depending on the JIT state; we could return
+ more with more efforts.)
+
+ This function should be used for specialized purposes only."""
+ w_result = space.newdict()
+ w_fake_frame = app.wget(space, "fake_frame")
+ w_fake_code = app.wget(space, "fake_code")
+ ecs = space.threadlocals.getallvalues()
+ for thread_ident, ec in ecs.items():
+ vref = ec.topframeref
+ frames = []
+ while not vref.virtual:
+ f = vref()
+ if f is None:
+ break
+ frames.append(f)
+ vref = f.f_backref
+ else:
+ frames.append(None)
+ #
+ w_topframe = space.wrap(None)
+ w_prevframe = None
+ for f in frames:
+ w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+ if w_prevframe is None:
+ w_topframe = w_nextframe
+ else:
+ space.setattr(w_prevframe, space.wrap('f_back'), w_nextframe)
+ w_prevframe = w_nextframe
+ #
+ space.setitem(w_result,
+ space.wrap(thread_ident),
+ w_topframe)
+ return w_result
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -556,7 +556,7 @@
return sys._current_frames()
frames = f()
assert frames.keys() == [0]
- assert frames[0].f_code.co_name == 'f'
+ assert frames[0].f_code.co_name in ('f', '?')
class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
def setup_class(cls):
@@ -568,23 +568,25 @@
import thread
thread_id = thread.get_ident()
- self.ready = False
def other_thread():
- self.ready = True
print "thread started"
- time.sleep(5)
+ lock2.release()
+ lock1.acquire()
+ lock1 = thread.allocate_lock()
+ lock2 = thread.allocate_lock()
+ lock1.acquire()
+ lock2.acquire()
thread.start_new_thread(other_thread, ())
def f():
- for i in range(100):
- if self.ready: break
- time.sleep(0.1)
+ lock2.acquire()
return sys._current_frames()
frames = f()
+ lock1.release()
thisframe = frames.pop(thread_id)
- assert thisframe.f_code.co_name == 'f'
+ assert thisframe.f_code.co_name in ('f', '?')
assert len(frames) == 1
_, other_frame = frames.popitem()
- assert other_frame.f_code.co_name == 'other_thread'
+ assert other_frame.f_code.co_name in ('other_thread', '?')
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -45,25 +45,6 @@
f.mark_as_escaped()
return space.wrap(f)
-def _current_frames(space):
- """_current_frames() -> dictionary
-
- Return a dictionary mapping each current thread T's thread id to T's
- current stack frame.
-
- This function should be used for specialized purposes only."""
- raise OperationError(space.w_NotImplementedError,
- space.wrap("XXX sys._current_frames() incompatible with the JIT"))
- w_result = space.newdict()
- ecs = space.threadlocals.getallvalues()
- for thread_ident, ec in ecs.items():
- f = ec.gettopframe_nohidden()
- f.mark_as_escaped()
- space.setitem(w_result,
- space.wrap(thread_ident),
- space.wrap(f))
- return w_result
-
def setrecursionlimit(space, w_new_limit):
"""setrecursionlimit() sets the maximum number of nested calls that
can occur before a RuntimeError is raised. On PyPy the limit is
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -17,6 +17,7 @@
class GILThreadLocals(OSThreadLocals):
"""A version of OSThreadLocals that enforces a GIL."""
gil_ready = False
+ _immutable_fields_ = ['gil_ready?']
def initialize(self, space):
# add the GIL-releasing callback as an action on the space
diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -8,9 +8,14 @@
def __init__(self):
self._valuedict = {} # {thread_ident: ExecutionContext()}
+ self._freeze_()
+
+ def _freeze_(self):
+ self._valuedict.clear()
self._mainthreadident = 0
self._mostrecentkey = 0 # fast minicaching for the common case
self._mostrecentvalue = None # fast minicaching for the common case
+ return False
def getvalue(self):
ident = thread.get_ident()
diff --git a/pypy/objspace/std/objecttype.py b/pypy/objspace/std/objecttype.py
--- a/pypy/objspace/std/objecttype.py
+++ b/pypy/objspace/std/objecttype.py
@@ -44,7 +44,6 @@
raise OperationError(space.w_TypeError,
space.wrap("__class__ assignment: only for heap types"))
w_oldcls = space.type(w_obj)
- # XXX taint space should raise a TaintError here if w_oldcls is tainted
assert isinstance(w_oldcls, W_TypeObject)
if w_oldcls.get_full_instance_layout() == w_newcls.get_full_instance_layout():
w_obj.setclass(space, w_newcls)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -161,57 +161,59 @@
def str_swapcase__String(space, w_self):
self = w_self._value
- res = [' '] * len(self)
+ builder = StringBuilder(len(self))
for i in range(len(self)):
ch = self[i]
if ch.isupper():
o = ord(ch) + 32
- res[i] = chr(o)
+ builder.append(chr(o))
elif ch.islower():
o = ord(ch) - 32
- res[i] = chr(o)
+ builder.append(chr(o))
else:
- res[i] = ch
+ builder.append(ch)
- return space.wrap("".join(res))
+ return space.wrap(builder.build())
def str_capitalize__String(space, w_self):
input = w_self._value
- buffer = [' '] * len(input)
+ builder = StringBuilder(len(input))
if len(input) > 0:
ch = input[0]
if ch.islower():
o = ord(ch) - 32
- buffer[0] = chr(o)
+ builder.append(chr(o))
else:
- buffer[0] = ch
+ builder.append(ch)
for i in range(1, len(input)):
ch = input[i]
if ch.isupper():
o = ord(ch) + 32
- buffer[i] = chr(o)
+ builder.append(chr(o))
else:
- buffer[i] = ch
+ builder.append(ch)
- return space.wrap("".join(buffer))
+ return space.wrap(builder.build())
def str_title__String(space, w_self):
input = w_self._value
- buffer = [' '] * len(input)
+ builder = StringBuilder(len(input))
prev_letter=' '
- for pos in range(0, len(input)):
+ for pos in range(len(input)):
ch = input[pos]
if not prev_letter.isalpha():
- buffer[pos] = _upper(ch)
+ ch = _upper(ch)
+ builder.append(ch)
else:
- buffer[pos] = _lower(ch)
+ ch = _lower(ch)
+ builder.append(ch)
- prev_letter = buffer[pos]
+ prev_letter = ch
- return space.wrap("".join(buffer))
+ return space.wrap(builder.build())
def str_split__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
maxsplit = space.int_w(w_maxsplit)
@@ -754,27 +756,21 @@
input = w_self._value
width = space.int_w(w_width)
- if len(input) >= width:
+ num_zeros = width - len(input)
+ if num_zeros <= 0:
# cannot return w_self, in case it is a subclass of str
return space.wrap(input)
- buf = [' '] * width
+ builder = StringBuilder(width)
if len(input) > 0 and (input[0] == '+' or input[0] == '-'):
- buf[0] = input[0]
+ builder.append(input[0])
start = 1
- middle = width - len(input) + 1
else:
start = 0
- middle = width - len(input)
- for i in range(start, middle):
- buf[i] = '0'
-
- for i in range(middle, width):
- buf[i] = input[start]
- start = start + 1
-
- return space.wrap("".join(buf))
+ builder.append_multiple_char('0', num_zeros)
+ builder.append_slice(input, start, len(input))
+ return space.wrap(builder.build())
def hash__String(space, w_str):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -10,7 +10,8 @@
from pypy.objspace.std import identitydict
from pypy.rlib.objectmodel import we_are_translated
from pypy.rlib.objectmodel import current_object_addr_as_int, compute_hash
-from pypy.rlib.jit import promote, elidable_promote, we_are_jitted
+from pypy.rlib.jit import promote, elidable_promote, we_are_jitted,\
+ promote_string
from pypy.rlib.jit import elidable, dont_look_inside, unroll_safe
from pypy.rlib.rarithmetic import intmask, r_uint
@@ -101,6 +102,7 @@
'instancetypedef',
'terminator',
'_version_tag?',
+ 'interplevel_cls',
]
# for config.objspace.std.getattributeshortcut
@@ -399,6 +401,7 @@
if version_tag is None:
tup = w_self._lookup_where(name)
return tup
+ name = promote_string(name)
w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
return w_class, unwrap_cell(space, w_value)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -417,54 +417,54 @@
input = w_self._value
if len(input) == 0:
return W_UnicodeObject.EMPTY
- result = [u'\0'] * len(input)
- result[0] = unichr(unicodedb.toupper(ord(input[0])))
+ builder = UnicodeBuilder(len(input))
+ builder.append(unichr(unicodedb.toupper(ord(input[0]))))
for i in range(1, len(input)):
- result[i] = unichr(unicodedb.tolower(ord(input[i])))
- return W_UnicodeObject(u''.join(result))
+ builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+ return W_UnicodeObject(builder.build())
def unicode_title__Unicode(space, w_self):
input = w_self._value
if len(input) == 0:
return w_self
- result = [u'\0'] * len(input)
+ builder = UnicodeBuilder(len(input))
previous_is_cased = False
for i in range(len(input)):
unichar = ord(input[i])
if previous_is_cased:
- result[i] = unichr(unicodedb.tolower(unichar))
+ builder.append(unichr(unicodedb.tolower(unichar)))
else:
- result[i] = unichr(unicodedb.totitle(unichar))
+ builder.append(unichr(unicodedb.totitle(unichar)))
previous_is_cased = unicodedb.iscased(unichar)
- return W_UnicodeObject(u''.join(result))
+ return W_UnicodeObject(builder.build())
def unicode_lower__Unicode(space, w_self):
input = w_self._value
- result = [u'\0'] * len(input)
+ builder = UnicodeBuilder(len(input))
for i in range(len(input)):
- result[i] = unichr(unicodedb.tolower(ord(input[i])))
- return W_UnicodeObject(u''.join(result))
+ builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+ return W_UnicodeObject(builder.build())
def unicode_upper__Unicode(space, w_self):
input = w_self._value
- result = [u'\0'] * len(input)
+ builder = UnicodeBuilder(len(input))
for i in range(len(input)):
- result[i] = unichr(unicodedb.toupper(ord(input[i])))
- return W_UnicodeObject(u''.join(result))
+ builder.append(unichr(unicodedb.toupper(ord(input[i]))))
+ return W_UnicodeObject(builder.build())
def unicode_swapcase__Unicode(space, w_self):
input = w_self._value
- result = [u'\0'] * len(input)
+ builder = UnicodeBuilder(len(input))
for i in range(len(input)):
unichar = ord(input[i])
if unicodedb.islower(unichar):
- result[i] = unichr(unicodedb.toupper(unichar))
+ builder.append(unichr(unicodedb.toupper(unichar)))
elif unicodedb.isupper(unichar):
- result[i] = unichr(unicodedb.tolower(unichar))
+ builder.append(unichr(unicodedb.tolower(unichar)))
else:
- result[i] = input[i]
- return W_UnicodeObject(u''.join(result))
+ builder.append(input[i])
+ return W_UnicodeObject(builder.build())
def _normalize_index(length, index):
if index < 0:
diff --git a/pypy/objspace/taint.py b/pypy/objspace/taint.py
deleted file mode 100644
--- a/pypy/objspace/taint.py
+++ /dev/null
@@ -1,294 +0,0 @@
-"""
-Just an experiment.
-"""
-import os
-from pypy.objspace.std.objspace import StdObjSpace
-from pypy.objspace.proxy import patch_space_in_place
-from pypy.objspace.thunk import nb_forcing_args
-from pypy.interpreter.error import OperationError
-from pypy.interpreter import baseobjspace, gateway, executioncontext
-from pypy.interpreter.function import Method
-from pypy.interpreter.pyframe import PyFrame
-from pypy.tool.sourcetools import func_with_new_name
-from pypy.rlib.unroll import unrolling_iterable
-
-
-class W_Tainted(baseobjspace.W_Root):
- def __init__(self, w_obj):
- self.w_obj = w_obj
-
-## def getdict(self, space):
-## return taint(self.w_obj.getdict(space))
-
-## def getdictvalue(self, space, attr):
-## return taint(self.w_obj.getdictvalue(space, attr))
-
-## def setdictvalue(self, space, attr, w_value):
-## return self.w_obj.setdictvalue(space, attr, w_value)
-
-## ...
-
-class W_TaintBomb(baseobjspace.W_Root):
- filename = '?'
- codename = '?'
- codeline = 0
-
- def __init__(self, space, operr):
- self.space = space
- self.operr = operr
- self.record_debug_info()
-
- def record_debug_info(self):
- ec = self.space.getexecutioncontext()
- frame = ec.gettopframe_nohidden()
- if isinstance(frame, PyFrame): # and, in particular, frame != None
- self.filename = frame.pycode.co_filename
- self.codename = frame.pycode.co_name
- self.codeline = frame.get_last_lineno()
- if get_debug_level(self.space) > 0:
- self.debug_dump()
-
- def debug_dump(self):
- os.write(2, 'Taint Bomb from file "%s", line %d, in %s\n %s\n' % (
- self.filename, self.codeline, self.codename,
- self.operr.errorstr(self.space)))
-
- def explode(self):
- #msg = self.operr.errorstr(space)
- raise OperationError(self.space.w_TaintError, self.space.w_None)
-
-
-def taint(w_obj):
- """Return a tainted version of the argument."""
- if w_obj is None or isinstance(w_obj, W_Tainted):
- return w_obj
- else:
- return W_Tainted(w_obj)
-app_taint = gateway.interp2app(taint)
-
-def is_tainted(space, w_obj):
- """Return whether the argument is tainted."""
- res = isinstance(w_obj, W_Tainted) or isinstance(w_obj, W_TaintBomb)
- return space.wrap(res)
-app_is_tainted = gateway.interp2app(is_tainted)
-
-def untaint(space, w_expectedtype, w_obj):
- """untaint(expectedtype, tainted_obj) -> obj
-Untaint untainted_obj and return it. If the result is not of expectedtype,
-raise a type error."""
- if (isinstance(w_expectedtype, W_Tainted) or
- isinstance(w_expectedtype, W_TaintBomb)):
- raise OperationError(space.w_TypeError,
- space.wrap("untaint() arg 1 must be an untainted type"))
- if not space.is_true(space.isinstance(w_expectedtype, space.w_type)):
- raise OperationError(space.w_TypeError,
- space.wrap("untaint() arg 1 must be a type"))
- if isinstance(w_obj, W_Tainted):
- w_obj = w_obj.w_obj
- elif isinstance(w_obj, W_TaintBomb):
- w_obj.explode()
- #if isinstance(w_expectedtype, W_Tainted):
- # w_expectedtype = w_expectedtype.w_obj
- w_realtype = space.type(w_obj)
- if not space.is_w(w_realtype, w_expectedtype):
- #msg = "expected an object of type '%s'" % (
- # w_expectedtype.getname(space),)
- # #w_realtype.getname(space))
- raise OperationError(space.w_TaintError, space.w_None)
- return w_obj
-app_untaint = gateway.interp2app(untaint)
-
-# ____________________________________________________________
-
- at gateway.unwrap_spec(args_w='args_w')
-def taint_atomic_function(space, w_func, args_w):
- newargs_w = []
- tainted = False
- for w_arg in args_w:
- if isinstance(w_arg, W_Tainted):
- tainted = True
- w_arg = w_arg.w_obj
- elif isinstance(w_arg, W_TaintBomb):
- return w_arg
- newargs_w.append(w_arg)
- w_newargs = space.newtuple(newargs_w)
- try:
- w_res = space.call(w_func, w_newargs)
- except OperationError, operr:
- if not tainted:
- raise
- return W_TaintBomb(space, operr)
- if tainted:
- w_res = taint(w_res)
- return w_res
-
-app_taint_atomic_function = gateway.interp2app(taint_atomic_function)
-
-def taint_atomic(space, w_callable):
- """decorator to make a callable "taint-atomic": if the function is called
-with tainted arguments, those are untainted. The result of the function is
-tainted again. All exceptions that the callable raises are turned into
-taint bombs."""
- meth = Method(space, space.w_fn_taint_atomic_function,
- w_callable, space.type(w_callable))
- return space.wrap(meth)
-app_taint_atomic = gateway.interp2app(taint_atomic)
-
-# ____________________________________________________________
-
-executioncontext.ExecutionContext.taint_debug = 0
-
- at gateway.unwrap_spec(level=int)
-def taint_debug(space, level):
- """Set the debug level. If the debug level is greater than 0, the creation
-of taint bombs will print debug information. For debugging purposes
-only!"""
- space.getexecutioncontext().taint_debug = level
-app_taint_debug = gateway.interp2app(taint_debug)
-
-def taint_look(space, w_obj):
- """Print some info about the taintedness of an object. For debugging
-purposes only!"""
- if isinstance(w_obj, W_Tainted):
- info = space.type(w_obj.w_obj).getname(space)
- msg = space.str_w(w_obj.w_obj.getrepr(space, info))
- msg = 'Taint Box %s\n' % msg
- os.write(2, msg)
- elif isinstance(w_obj, W_TaintBomb):
- w_obj.debug_dump()
- else:
- os.write(2, 'not tainted\n')
-app_taint_look = gateway.interp2app(taint_look)
-
-def get_debug_level(space):
- return space.getexecutioncontext().taint_debug
-
-def debug_bomb(space, operr):
- ec = space.getexecutioncontext()
- filename = '?'
- codename = '?'
- codeline = 0
- frame = ec.gettopframe_nohidden()
- if isinstance(frame, PyFrame): # and, in particular, frame != None
- filename = frame.pycode.co_filename
- codename = frame.pycode.co_name
- codeline = frame.get_last_lineno()
- os.write(2, 'Taint Bomb in file "%s", line %d, in %s\n %s\n' % (
- filename, codeline, codename, operr.errorstr(space)))
-
-# ____________________________________________________________
-
-
-class TaintSpace(StdObjSpace):
-
- def __init__(self, *args, **kwds):
- StdObjSpace.__init__(self, *args, **kwds)
- w_dict = self.newdict()
- self.setitem(w_dict, self.wrap("__doc__"), self.wrap("""\
-Exception that is raised when an operation revealing information on a tainted
-object is performed."""))
- self.w_TaintError = self.call_function(
- self.w_type,
- self.wrap("TaintError"),
- self.newtuple([self.w_Exception]),
- w_dict
- )
- w___pypy__ = self.getbuiltinmodule("__pypy__")
- self.setattr(w___pypy__, self.wrap('taint'),
- self.wrap(app_taint))
- self.setattr(w___pypy__, self.wrap('is_tainted'),
- self.wrap(app_is_tainted))
- self.setattr(w___pypy__, self.wrap('untaint'),
- self.wrap(app_untaint))
- self.w_fn_taint_atomic_function = self.wrap(app_taint_atomic_function)
- self.setattr(w___pypy__, self.wrap('taint_atomic'),
- self.wrap(app_taint_atomic))
- self.setattr(w___pypy__, self.wrap('TaintError'),
- self.w_TaintError)
- self.setattr(w___pypy__, self.wrap('_taint_debug'),
- self.wrap(app_taint_debug))
- self.setattr(w___pypy__, self.wrap('_taint_look'),
- self.wrap(app_taint_look))
- patch_space_in_place(self, 'taint', proxymaker)
-
- # XXX may leak info, perfomance hit, what about taint bombs?
- from pypy.objspace.std.typeobject import W_TypeObject
-
- def taint_lookup(w_obj, name):
- if isinstance(w_obj, W_Tainted):
- w_obj = w_obj.w_obj
- w_type = self.type(w_obj)
- assert isinstance(w_type, W_TypeObject)
- return w_type.lookup(name)
-
- def taint_lookup_in_type_where(w_obj, name):
- if isinstance(w_obj, W_Tainted):
- w_type = w_obj.w_obj
- else:
- w_type = w_obj
- assert isinstance(w_type, W_TypeObject)
- return w_type.lookup_where(name)
-
- self.lookup = taint_lookup
- self.lookup_in_type_where = taint_lookup_in_type_where
-
-
-Space = TaintSpace
-
-
-def tainted_error(space, name):
- #msg = "operation '%s' forbidden on tainted object" % (name,)
- raise OperationError(space.w_TaintError, space.w_None)# space.wrap(msg))
-
-
-RegularMethods = dict.fromkeys(
- [name for name, _, _, _ in baseobjspace.ObjSpace.MethodTable])
-
-TaintResultIrregularMethods = dict.fromkeys(
- ['wrap', 'call_args'] +
- [name for name in baseobjspace.ObjSpace.IrregularOpTable
- if name.startswith('new')])
-
-def proxymaker(space, name, parentfn):
- arity = nb_forcing_args[name]
- indices = unrolling_iterable(range(arity))
- if name in RegularMethods:
-
- def proxy(*args_w):
- newargs_w = ()
- tainted = False
- for i in indices:
- w_arg = args_w[i]
- if isinstance(w_arg, W_Tainted):
- tainted = True
- w_arg = w_arg.w_obj
- elif isinstance(w_arg, W_TaintBomb):
- return w_arg
- newargs_w += (w_arg,)
- newargs_w += args_w[arity:]
- try:
- w_res = parentfn(*newargs_w)
- except OperationError, operr:
- if not tainted:
- raise
- return W_TaintBomb(space, operr)
- if tainted:
- w_res = taint(w_res)
- return w_res
-
- elif arity == 0:
- return None
-
- else:
-
- def proxy(*args_w):
- for i in indices:
- w_arg = args_w[i]
- if isinstance(w_arg, W_Tainted):
- tainted_error(space, name)
- elif isinstance(w_arg, W_TaintBomb):
- w_arg.explode()
- return parentfn(*args_w)
-
- proxy = func_with_new_name(proxy, '%s_proxy' % name)
- return proxy
diff --git a/pypy/objspace/test/test_taintobjspace.py b/pypy/objspace/test/test_taintobjspace.py
deleted file mode 100644
--- a/pypy/objspace/test/test_taintobjspace.py
+++ /dev/null
@@ -1,77 +0,0 @@
-from pypy.conftest import gettestobjspace
-
-class AppTest_Taint:
-
- def setup_class(cls):
- cls.space = gettestobjspace('taint')
-
- def test_simple(self):
- from __pypy__ import taint, untaint, TaintError
- x = taint(6)
- x = x * 7
- raises(TaintError, "if x: y = 1")
- t = type(x)
- raises(TaintError, "if t is int: y = 1")
- assert untaint(int, x) == 42
- raises(TaintError, "untaint(float, x)")
-
- def test_bomb(self):
- from __pypy__ import taint, untaint, TaintError
- x = taint(6)
- x = x / 0
- raises(TaintError, "if x: y = 1")
- t = type(x)
- raises(TaintError, "if t is int: y = 1")
- raises(TaintError, "untaint(int, x)")
- raises(TaintError, "untaint(float, x)")
-
- def test_taint_atomic(self):
- from __pypy__ import taint, untaint, TaintError, taint_atomic
- x = taint(6)
- x *= 7
-
- def dummy(x):
- if x > 40:
- return 5
- else:
- return 3
- dummy = taint_atomic(dummy)
-
- y = dummy(x)
- raises(TaintError, "if y == 3: z = 1")
- assert untaint(int, y) == 5
-
- def test_taint_atomic_exception(self):
- from __pypy__ import taint, untaint, TaintError, taint_atomic
- x = taint(6)
- x *= 7
-
- def dummy(x):
- if x + "world" == "hello world":
- return 5
- else:
- return 3
- dummy = taint_atomic(dummy)
-
- y = dummy(x)
- raises(TaintError, "if y == 3: z = 1")
- raises(TaintError, "untaint(int, y)")
-
- def test_taint_atomic_incoming_bomb(self):
- from __pypy__ import taint, untaint, TaintError, taint_atomic
- x = taint(6)
- x /= 0
- lst = []
-
- def dummy(x):
- lst.append("running!")
- if x > 40:
- return 5
- else:
- return 3
- dummy = taint_atomic(dummy)
-
- y = dummy(x)
- raises(TaintError, "if y == 3: z = 1")
- assert lst == []
- raises(TaintError, "untaint(int, y)")
diff --git a/pypy/rlib/_jit_vref.py b/pypy/rlib/_jit_vref.py
--- a/pypy/rlib/_jit_vref.py
+++ b/pypy/rlib/_jit_vref.py
@@ -25,6 +25,10 @@
def simple_call(self):
return self.s_instance
+ def getattr(self, s_attr):
+ assert s_attr.const == 'virtual'
+ return annmodel.s_Bool
+
def rtyper_makerepr(self, rtyper):
if rtyper.type_system.name == 'lltypesystem':
return vrefrepr
@@ -61,6 +65,13 @@
" prebuilt virtual_ref")
return lltype.nullptr(OBJECTPTR.TO)
+ def rtype_getattr(self, hop):
+ s_attr = hop.args_s[1]
+ assert s_attr.const == 'virtual'
+ v = hop.inputarg(self, arg=0)
+ hop.exception_cannot_occur()
+ return hop.genop('jit_is_virtual', [v], resulttype = lltype.Bool)
+
from pypy.rpython.ootypesystem.rclass import OBJECT
class OOVRefRepr(VRefRepr):
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -38,6 +38,7 @@
possible arguments are:
* promote - promote the argument from a variable into a constant
+ * promote_string - same, but promote string by *value*
* access_directly - directly access a virtualizable, as a structure
and don't treat it as a virtualizable
* fresh_virtualizable - means that virtualizable was just allocated.
@@ -51,6 +52,9 @@
def promote(x):
return hint(x, promote=True)
+def promote_string(x):
+ return hint(x, promote_string=True)
+
def dont_look_inside(func):
""" Make sure the JIT does not trace inside decorated function
(it becomes a call instead)
@@ -313,6 +317,12 @@
raise InvalidVirtualRef
return self._x
+ @property
+ def virtual(self):
+ """A property that is True if the vref contains a virtual that would
+ be forced by the '()' operator."""
+ return self._state == 'non-forced'
+
def _finish(self):
if self._state == 'non-forced':
self._state = 'invalid'
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -218,15 +218,13 @@
[v_value] = hop.inputargs(lltype.Signed)
c_one = hop.inputconst(lltype.Signed, 1)
hop.exception_is_here()
- v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+ v2 = hop.genop('int_add_ovf', [v_value, v_value],
resulttype = lltype.Signed)
v2p1 = hop.genop('int_add', [v2, c_one],
resulttype = lltype.Signed)
v_instance = hop.genop('cast_int_to_ptr', [v2p1],
resulttype=self.lowleveltype)
- v = hop.genop('cast_opaque_ptr', [v_instance],
- resulttype=self.lowleveltype)
- return v
+ return v_instance
def convert_const(self, value):
if value._identity is _identity_for_ints:
@@ -266,10 +264,10 @@
return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
def rtype_erase_int(self, hop):
- hop.exception_is_here()
[v_value] = hop.inputargs(lltype.Signed)
c_one = hop.inputconst(lltype.Signed, 1)
- v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+ hop.exception_is_here()
+ v2 = hop.genop('int_add_ovf', [v_value, v_value],
resulttype = lltype.Signed)
v2p1 = hop.genop('int_add', [v2, c_one],
resulttype = lltype.Signed)
diff --git a/pypy/rlib/streamio.py b/pypy/rlib/streamio.py
--- a/pypy/rlib/streamio.py
+++ b/pypy/rlib/streamio.py
@@ -37,7 +37,7 @@
# return value of tell(), but not as argument to read().
#
-import os, sys
+import os, sys, errno
from pypy.rlib.objectmodel import specialize, we_are_translated
from pypy.rlib.rarithmetic import r_longlong, intmask
from pypy.rlib import rposix
@@ -587,12 +587,22 @@
def readall(self):
pos = self.pos
assert pos >= 0
- chunks = [self.buf[pos:]]
+ if self.buf:
+ chunks = [self.buf[pos:]]
+ else:
+ chunks = []
self.buf = ""
self.pos = 0
bufsize = self.bufsize
while 1:
- data = self.do_read(bufsize)
+ try:
+ data = self.do_read(bufsize)
+ except OSError, o:
+ if o.errno != errno.EAGAIN:
+ raise
+ if not chunks:
+ raise
+ break
if not data:
break
chunks.append(data)
diff --git a/pypy/rlib/test/test__jit_vref.py b/pypy/rlib/test/test__jit_vref.py
--- a/pypy/rlib/test/test__jit_vref.py
+++ b/pypy/rlib/test/test__jit_vref.py
@@ -27,10 +27,13 @@
x1 = X()
vref = virtual_ref(x1)
assert vref._state == 'non-forced'
+ assert vref.virtual is True
assert vref() is x1
assert vref._state == 'forced'
+ assert vref.virtual is False
virtual_ref_finish(vref, x1)
assert vref._state == 'forced'
+ assert vref.virtual is False
assert vref() is x1
def test_direct_invalid():
@@ -135,6 +138,13 @@
x = self.interpret(f, [])
assert x == 42
+ def test_rtype_virtualattr(self):
+ def f():
+ vref = virtual_ref(X())
+ return vref.virtual
+ x = self.interpret(f, [])
+ assert x is False
+
class TestLLtype(BaseTestVRef, LLRtypeMixin):
OBJECTTYPE = OBJECTPTR
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -139,12 +139,11 @@
def test_isconstant(self):
def f(n):
- assert n >= 0
assert isconstant(n) is False
l = []
l.append(n)
return len(l)
- res = self.interpret(f, [234])
+ res = self.interpret(f, [-234])
assert res == 1
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -658,6 +658,8 @@
if T == llmemory.GCREF:
if isinstance(llobj._obj, _llgcopaque):
return ctypes.c_void_p(llobj._obj.intval)
+ if isinstance(llobj._obj, int): # tagged pointer
+ return ctypes.c_void_p(llobj._obj)
container = llobj._obj.container
T = lltype.Ptr(lltype.typeOf(container))
# otherwise it came from integer and we want a c_void_p with
@@ -1268,6 +1270,7 @@
class _llgcopaque(lltype._container):
_TYPE = llmemory.GCREF.TO
_name = "_llgcopaque"
+ _read_directly_intval = True # for _ptr._cast_to_int()
def __init__(self, void_p):
if isinstance(void_p, (int, long)):
@@ -1288,6 +1291,9 @@
return False
return force_cast(lltype.Signed, other._as_ptr()) == self.intval
+ def __hash__(self):
+ return self.intval
+
def __ne__(self, other):
return not self == other
@@ -1296,11 +1302,6 @@
return _opaque_objs[self.intval // 2]
return force_cast(PTRTYPE, self.intval)
-## def _cast_to_int(self):
-## return self.intval
-
-## def _cast_to_adr(self):
-## return _lladdress(self.intval)
def cast_adr_to_int(addr):
if isinstance(addr, llmemory.fakeaddress):
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -498,6 +498,8 @@
def _cast_to_int(self, symbolic=False):
if self:
+ if isinstance(self.ptr._obj0, int): # tagged integer
+ return self.ptr._obj0
if symbolic:
return AddressAsInt(self)
else:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -428,6 +428,7 @@
'jit_marker': LLOp(),
'jit_force_virtualizable':LLOp(canrun=True),
'jit_force_virtual': LLOp(canrun=True),
+ 'jit_is_virtual': LLOp(canrun=True),
'jit_force_quasi_immutable': LLOp(canrun=True),
'get_exception_addr': LLOp(),
'get_exc_value_addr': LLOp(),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1360,6 +1360,8 @@
obj = normalizeptr(self, check)._getobj(check)
if isinstance(obj, int):
return obj # special case for cast_int_to_ptr() results put into opaques
+ if getattr(obj, '_read_directly_intval', False):
+ return obj.intval # special case for _llgcopaque
result = intmask(obj._getid())
# assume that id() returns an addressish value which is
# not zero and aligned to at least a multiple of 4
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -538,6 +538,9 @@
def op_jit_force_virtual(x):
return x
+def op_jit_is_virtual(x):
+ return False
+
def op_jit_force_quasi_immutable(*args):
pass
diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py
--- a/pypy/rpython/lltypesystem/rtagged.py
+++ b/pypy/rpython/lltypesystem/rtagged.py
@@ -43,7 +43,7 @@
v_value = hop.inputarg(lltype.Signed, arg=1)
c_one = hop.inputconst(lltype.Signed, 1)
hop.exception_is_here()
- v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+ v2 = hop.genop('int_add_ovf', [v_value, v_value],
resulttype = lltype.Signed)
v2p1 = hop.genop('int_add', [v2, c_one],
resulttype = lltype.Signed)
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -1123,6 +1123,9 @@
#assert lltype.cast_ptr_to_int(ref1) == intval
+ x = rffi.cast(llmemory.GCREF, -17)
+ assert lltype.cast_ptr_to_int(x) == -17
+
def test_ptr_truth(self):
abc = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Void)), 0)
assert not abc
diff --git a/pypy/rpython/rclass.py b/pypy/rpython/rclass.py
--- a/pypy/rpython/rclass.py
+++ b/pypy/rpython/rclass.py
@@ -191,6 +191,10 @@
"class %r inherits from its parent _immutable_=True, "
"so it should also declare _immutable_=True" % (
self.classdef,))
+ if loc.classdict.get('_immutable_').value is not True:
+ raise TyperError(
+ "class %r: _immutable_ = something else than True" % (
+ self.classdef,))
hints = hints.copy()
hints['immutable'] = True
self.immutable_field_set = set() # unless overwritten below
diff --git a/pypy/tool/logparser.py b/pypy/tool/logparser.py
--- a/pypy/tool/logparser.py
+++ b/pypy/tool/logparser.py
@@ -298,6 +298,8 @@
image.paste(textpercent, (t1x, 5), textpercent)
image.paste(textlabel, (t2x, 5), textlabel)
images.append(image)
+ if not images:
+ return None
return combine(images, spacing=0, border=1, horizontal=False)
def get_timesummary_single_image(totaltimes, totaltime0, componentdict,
@@ -333,6 +335,8 @@
del totaltimes[None]
img2 = render_histogram(totaltimes, totaltime0, {},
width, summarybarheight)
+ if img2 is None:
+ return img1
return combine([img1, img2], spacing=spacing, horizontal=True)
# ----------
diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -833,7 +833,7 @@
return 'INSTRUMENT_COUNT(%s);' % counter_label
def OP_IS_EARLY_CONSTANT(self, op):
- return self.expr(op.result) + ' = 0;' # Allways false
+ return '%s = 0; /* IS_EARLY_CONSTANT */' % (self.expr(op.result),)
def OP_JIT_MARKER(self, op):
return '/* JIT_MARKER %s */' % op
@@ -845,6 +845,9 @@
return '%s = %s; /* JIT_FORCE_VIRTUAL */' % (self.expr(op.result),
self.expr(op.args[0]))
+ def OP_JIT_IS_VIRTUAL(self, op):
+ return '%s = 0; /* JIT_IS_VIRTUAL */' % (self.expr(op.result),)
+
def OP_JIT_FORCE_QUASI_IMMUTABLE(self, op):
return '/* JIT_FORCE_QUASI_IMMUTABLE %s */' % op
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -8,7 +8,7 @@
from pypy.translator.tool.cbuild import ExternalCompilationInfo
from pypy.rpython.lltypesystem import lltype
from pypy.tool.udir import udir
-from pypy.tool import isolate
+from pypy.tool import isolate, runsubprocess
from pypy.translator.c.support import log, c_string_constant
from pypy.rpython.typesystem import getfunctionptr
from pypy.translator.c import gc
@@ -563,14 +563,19 @@
else:
mk.definition('PYPY_MAIN_FUNCTION', "main")
- if (py.path.local.sysfind('python') or
- py.path.local.sysfind('python.exe')):
- python = 'python '
- elif sys.platform == 'win32':
+ if sys.platform == 'win32':
python = sys.executable.replace('\\', '/') + ' '
else:
python = sys.executable + ' '
+ # Is there a command 'python' that runs python 2.5-2.7?
+ # If there is, then we can use it instead of sys.executable
+ returncode, stdout, stderr = runsubprocess.run_subprocess(
+ "python", "-V")
+ if (stdout.startswith('Python 2.') or
+ stderr.startswith('Python 2.')):
+ python = 'python '
+
if self.translator.platform.name == 'msvc':
lblofiles = []
for cfile in mk.cfiles:
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -245,7 +245,7 @@
if (pending_acquires <= 0)
return 0;
InterlockedIncrement(&pending_acquires);
- PulseEvent(&cond_gil);
+ PulseEvent(cond_gil);
/* hack: the three following lines do a pthread_cond_wait(), and
normally specifying a timeout of INFINITE would be fine. But the
@@ -253,7 +253,7 @@
(small) risk that PulseEvent misses the WaitForSingleObject().
In this case the process will just sleep a few milliseconds. */
LeaveCriticalSection(&mutex_gil);
- WaitForSingleObject(&cond_gil, 15);
+ WaitForSingleObject(cond_gil, 15);
EnterCriticalSection(&mutex_gil);
InterlockedDecrement(&pending_acquires);
@@ -263,7 +263,7 @@
void RPyGilRelease(void)
{
LeaveCriticalSection(&mutex_gil);
- PulseEvent(&cond_gil);
+ PulseEvent(cond_gil);
}
void RPyGilAcquire(void)
diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -144,7 +144,7 @@
print ' --jit off turn off the JIT'
def print_version(*args):
- print "Python", sys.version
+ print >> sys.stderr, "Python", sys.version
raise SystemExit
def set_jit_option(options, jitparam, *args):
diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -24,15 +24,14 @@
return self._pkg_config("libffi", "--libs-only-L",
['/usr/lib/libffi'])
+ def library_dirs_for_libffi_a(self):
+ # places where we need to look for libffi.a
+ return self.library_dirs_for_libffi() + ['/usr/lib']
+
class Linux(BaseLinux):
shared_only = () # it seems that on 32-bit linux, compiling with -fPIC
# gives assembler that asmgcc is not happy about.
- def library_dirs_for_libffi_a(self):
- # places where we need to look for libffi.a
- return self.library_dirs_for_libffi() + ['/usr/lib']
-
-
class Linux64(BaseLinux):
pass
More information about the pypy-commit
mailing list