[pypy-commit] pypy numpypy-axisops: merge with default
mattip
noreply at buildbot.pypy.org
Tue Jan 3 23:04:51 CET 2012
Author: mattip
Branch: numpypy-axisops
Changeset: r51003:20bacb9bb86a
Date: 2012-01-03 23:36 +0200
http://bitbucket.org/pypy/pypy/changeset/20bacb9bb86a/
Log: merge with default
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -27,7 +27,7 @@
DEALINGS IN THE SOFTWARE.
-PyPy Copyright holders 2003-2011
+PyPy Copyright holders 2003-2012
-----------------------------------
Except when otherwise stated (look for LICENSE files or information at
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -420,8 +420,8 @@
debug._log = None
#
assert ops_offset is looptoken._x86_ops_offset
- # getfield_raw/int_add/setfield_raw + ops + None
- assert len(ops_offset) == 3 + len(operations) + 1
+ # 2*(getfield_raw/int_add/setfield_raw) + ops + None
+ assert len(ops_offset) == 2*3 + len(operations) + 1
assert (ops_offset[operations[0]] <=
ops_offset[operations[1]] <=
ops_offset[operations[2]] <=
diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py
--- a/pypy/jit/metainterp/compile.py
+++ b/pypy/jit/metainterp/compile.py
@@ -169,10 +169,10 @@
loop.original_jitcell_token = jitcell_token
for label in all_target_tokens:
assert isinstance(label, TargetToken)
- label.original_jitcell_token = jitcell_token
if label.virtual_state and label.short_preamble:
metainterp_sd.logger_ops.log_short_preamble([], label.short_preamble)
jitcell_token.target_tokens = all_target_tokens
+ propagate_original_jitcell_token(loop)
send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
record_loop_or_bridge(metainterp_sd, loop)
return all_target_tokens[0]
@@ -240,11 +240,11 @@
for box in loop.inputargs:
assert isinstance(box, Box)
- target_token = loop.operations[-1].getdescr()
+ target_token = loop.operations[-1].getdescr()
resumekey.compile_and_attach(metainterp, loop)
+
target_token = label.getdescr()
assert isinstance(target_token, TargetToken)
- target_token.original_jitcell_token = loop.original_jitcell_token
record_loop_or_bridge(metainterp_sd, loop)
return target_token
@@ -281,6 +281,15 @@
assert i == len(inputargs)
loop.operations = extra_ops + loop.operations
+def propagate_original_jitcell_token(trace):
+ for op in trace.operations:
+ if op.getopnum() == rop.LABEL:
+ token = op.getdescr()
+ assert isinstance(token, TargetToken)
+ assert token.original_jitcell_token is None
+ token.original_jitcell_token = trace.original_jitcell_token
+
+
def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type):
vinfo = jitdriver_sd.virtualizable_info
if vinfo is not None:
@@ -554,6 +563,7 @@
inputargs = metainterp.history.inputargs
if not we_are_translated():
self._debug_suboperations = new_loop.operations
+ propagate_original_jitcell_token(new_loop)
send_bridge_to_backend(metainterp.jitdriver_sd, metainterp.staticdata,
self, inputargs, new_loop.operations,
new_loop.original_jitcell_token)
@@ -740,6 +750,7 @@
jitdriver_sd = metainterp.jitdriver_sd
redargs = new_loop.inputargs
new_loop.original_jitcell_token = jitcell_token = make_jitcell_token(jitdriver_sd)
+ propagate_original_jitcell_token(new_loop)
send_loop_to_backend(self.original_greenkey, metainterp.jitdriver_sd,
metainterp_sd, new_loop, "entry bridge")
# send the new_loop to warmspot.py, to be called directly the next time
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -386,6 +386,17 @@
"""
self.optimize_loop(ops, expected)
+ def test_virtual_as_field_of_forced_box(self):
+ ops = """
+ [p0]
+ pv1 = new_with_vtable(ConstClass(node_vtable))
+ label(pv1, p0)
+ pv2 = new_with_vtable(ConstClass(node_vtable))
+ setfield_gc(pv2, pv1, descr=valuedescr)
+ jump(pv1, pv2)
+ """
+ with raises(InvalidLoop):
+ self.optimize_loop(ops, ops)
class OptRenameStrlen(Optimization):
def propagate_forward(self, op):
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -409,7 +409,13 @@
if self.level == LEVEL_CONSTANT:
return
assert 0 <= self.position_in_notvirtuals
- boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
+ if optimizer:
+ box = value.force_box(optimizer)
+ else:
+ if value.is_virtual():
+ raise BadVirtualState
+ box = value.get_key_box()
+ boxes[self.position_in_notvirtuals] = box
def _enum(self, virtual_state):
if self.level == LEVEL_CONSTANT:
@@ -471,8 +477,14 @@
optimizer = optimizer.optearlyforce
assert len(values) == len(self.state)
inputargs = [None] * len(self.notvirtuals)
+
+ # We try twice. The first time around we allow boxes to be forced
+ # which might change the virtual state if the box appear in more
+ # than one place among the inputargs.
for i in range(len(values)):
self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
+ for i in range(len(values)):
+ self.state[i].enum_forced_boxes(inputargs, values[i], None)
if keyboxes:
for i in range(len(values)):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -976,10 +976,13 @@
self.verify_green_args(jitdriver_sd, greenboxes)
self.debug_merge_point(jitdriver_sd, jdindex, self.metainterp.in_recursion,
greenboxes)
-
+
if self.metainterp.seen_loop_header_for_jdindex < 0:
- if not jitdriver_sd.no_loop_header or not any_operation:
+ if not any_operation:
return
+ if self.metainterp.in_recursion or not self.metainterp.get_procedure_token(greenboxes, True):
+ if not jitdriver_sd.no_loop_header:
+ return
# automatically add a loop_header if there is none
self.metainterp.seen_loop_header_for_jdindex = jdindex
#
@@ -2053,9 +2056,15 @@
from pypy.jit.metainterp.resoperation import opname
raise NotImplementedError(opname[opnum])
- def get_procedure_token(self, greenkey):
+ def get_procedure_token(self, greenkey, with_compiled_targets=False):
cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey)
- return cell.get_procedure_token()
+ token = cell.get_procedure_token()
+ if with_compiled_targets:
+ if not token:
+ return None
+ if not token.target_tokens:
+ return None
+ return token
def compile_loop(self, original_boxes, live_arg_boxes, start, resume_at_jump_descr):
num_green_args = self.jitdriver_sd.num_green_args
@@ -2088,11 +2097,9 @@
def compile_trace(self, live_arg_boxes, resume_at_jump_descr):
num_green_args = self.jitdriver_sd.num_green_args
greenkey = live_arg_boxes[:num_green_args]
- target_jitcell_token = self.get_procedure_token(greenkey)
+ target_jitcell_token = self.get_procedure_token(greenkey, True)
if not target_jitcell_token:
return
- if not target_jitcell_token.target_tokens:
- return
self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None,
descr=target_jitcell_token)
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2697,7 +2697,7 @@
# bridge back to the preamble of the first loop is produced. A guard in
# this bridge is later traced resulting in a failed attempt of retracing
# the second loop.
- self.check_trace_count(8)
+ self.check_trace_count(9)
# FIXME: Add a gloabl retrace counter and test that we are not trying more than 5 times.
diff --git a/pypy/jit/metainterp/test/test_loop.py b/pypy/jit/metainterp/test/test_loop.py
--- a/pypy/jit/metainterp/test/test_loop.py
+++ b/pypy/jit/metainterp/test/test_loop.py
@@ -756,7 +756,7 @@
res = self.meta_interp(interpret, [1])
assert res == interpret(1)
# XXX it's unsure how many loops should be there
- self.check_trace_count(3)
+ self.check_trace_count(2)
def test_path_with_operations_not_from_start(self):
jitdriver = JitDriver(greens = ['k'], reds = ['n', 'z'])
diff --git a/pypy/jit/metainterp/test/test_virtual.py b/pypy/jit/metainterp/test/test_virtual.py
--- a/pypy/jit/metainterp/test/test_virtual.py
+++ b/pypy/jit/metainterp/test/test_virtual.py
@@ -880,7 +880,7 @@
elif op == 'j':
j = Int(0)
elif op == '+':
- sa += i.val * j.val
+ sa += (i.val + 2) * (j.val + 2)
elif op == 'a':
i = Int(i.val + 1)
elif op == 'b':
@@ -902,6 +902,7 @@
assert res == f(10)
self.check_aborted_count(0)
self.check_target_token_count(3)
+ self.check_resops(int_mul=2)
def test_nested_loops_bridge(self):
class Int(object):
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -384,6 +384,9 @@
def descr_get_dtype(self, space):
return space.wrap(self.find_dtype())
+ def descr_get_ndim(self, space):
+ return space.wrap(len(self.shape))
+
@jit.unroll_safe
def descr_get_shape(self, space):
return space.newtuple([space.wrap(i) for i in self.shape])
@@ -413,7 +416,7 @@
def descr_repr(self, space):
res = StringBuilder()
res.append("array(")
- concrete = self.get_concrete()
+ concrete = self.get_concrete_or_scalar()
dtype = concrete.find_dtype()
if not concrete.size:
res.append('[]')
@@ -426,8 +429,9 @@
else:
concrete.to_str(space, 1, res, indent=' ')
if (dtype is not interp_dtype.get_dtype_cache(space).w_float64dtype and
- dtype is not interp_dtype.get_dtype_cache(space).w_int64dtype) or \
- not self.size:
+ not (dtype.kind == interp_dtype.SIGNEDLTR and
+ dtype.itemtype.get_element_size() == rffi.sizeof(lltype.Signed)) or
+ not self.size):
res.append(", dtype=" + dtype.name)
res.append(")")
return space.wrap(res.build())
@@ -910,78 +914,80 @@
each line will begin with indent.
'''
size = self.size
+ ccomma = ',' * comma
+ ncomma = ',' * (1 - comma)
+ dtype = self.find_dtype()
if size < 1:
builder.append('[]')
return
+ elif size == 1:
+ builder.append(dtype.itemtype.str_format(self.getitem(0)))
+ return
if size > 1000:
# Once this goes True it does not go back to False for recursive
# calls
use_ellipsis = True
- dtype = self.find_dtype()
ndims = len(self.shape)
i = 0
- start = True
builder.append('[')
if ndims > 1:
if use_ellipsis:
- for i in range(3):
- if start:
- start = False
- else:
- builder.append(',' * comma + '\n')
- if ndims == 3:
+ for i in range(min(3, self.shape[0])):
+ if i > 0:
+ builder.append(ccomma + '\n')
+ if ndims >= 3:
builder.append('\n' + indent)
else:
builder.append(indent)
view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
view.to_str(space, comma, builder, indent=indent + ' ',
use_ellipsis=use_ellipsis)
- builder.append('\n' + indent + '..., ')
- i = self.shape[0] - 3
+ if i < self.shape[0] - 1:
+ builder.append(ccomma +'\n' + indent + '...' + ncomma)
+ i = self.shape[0] - 3
+ else:
+ i += 1
while i < self.shape[0]:
- if start:
- start = False
- else:
- builder.append(',' * comma + '\n')
- if ndims == 3:
+ if i > 0:
+ builder.append(ccomma + '\n')
+ if ndims >= 3:
builder.append('\n' + indent)
else:
builder.append(indent)
+ # create_slice requires len(chunks) > 1 in order to reduce
+ # shape
view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
view.to_str(space, comma, builder, indent=indent + ' ',
use_ellipsis=use_ellipsis)
i += 1
elif ndims == 1:
- spacer = ',' * comma + ' '
+ spacer = ccomma + ' '
item = self.start
# An iterator would be a nicer way to walk along the 1d array, but
# how do I reset it if printing ellipsis? iterators have no
# "set_offset()"
i = 0
if use_ellipsis:
- for i in range(3):
- if start:
- start = False
- else:
+ for i in range(min(3, self.shape[0])):
+ if i > 0:
builder.append(spacer)
builder.append(dtype.itemtype.str_format(self.getitem(item)))
item += self.strides[0]
- # Add a comma only if comma is False - this prevents adding two
- # commas
- builder.append(spacer + '...' + ',' * (1 - comma))
- # Ugly, but can this be done with an iterator?
- item = self.start + self.backstrides[0] - 2 * self.strides[0]
- i = self.shape[0] - 3
+ if i < self.shape[0] - 1:
+ # Add a comma only if comma is False - this prevents adding
+ # two commas
+ builder.append(spacer + '...' + ncomma)
+ # Ugly, but can this be done with an iterator?
+ item = self.start + self.backstrides[0] - 2 * self.strides[0]
+ i = self.shape[0] - 3
+ else:
+ i += 1
while i < self.shape[0]:
- if start:
- start = False
- else:
+ if i > 0:
builder.append(spacer)
builder.append(dtype.itemtype.str_format(self.getitem(item)))
item += self.strides[0]
i += 1
- else:
- builder.append('[')
builder.append(']')
@jit.unroll_safe
@@ -1254,6 +1260,7 @@
shape = GetSetProperty(BaseArray.descr_get_shape,
BaseArray.descr_set_shape),
size = GetSetProperty(BaseArray.descr_get_size),
+ ndim = GetSetProperty(BaseArray.descr_get_ndim),
T = GetSetProperty(BaseArray.descr_get_transpose),
flat = GetSetProperty(BaseArray.descr_get_flatiter),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -158,6 +158,7 @@
assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None
assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2]
+
class AppTestNumArray(BaseNumpyAppTest):
def test_ndarray(self):
from numpypy import ndarray, array, dtype
@@ -179,6 +180,20 @@
ar = array(range(5))
assert type(ar) is type(ar + ar)
+ def test_ndim(self):
+ from numpypy import array
+ x = array(0.2)
+ assert x.ndim == 0
+ x = array([1, 2])
+ assert x.ndim == 1
+ x = array([[1, 2], [3, 4]])
+ assert x.ndim == 2
+ x = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+ assert x.ndim == 3
+ # numpy actually raises an AttributeError, but numpypy raises an
+ # TypeError
+ raises(TypeError, 'x.ndim = 3')
+
def test_init(self):
from numpypy import zeros
a = zeros(15)
@@ -1241,6 +1256,7 @@
assert isinstance(i['data'][0], int)
raises(TypeError, getattr, array(3), '__array_interface__')
+
class AppTestSupport(BaseNumpyAppTest):
def setup_class(cls):
import struct
@@ -1354,6 +1370,7 @@
class AppTestRepr(BaseNumpyAppTest):
def test_repr(self):
from numpypy import array, zeros
+ int_size = array(5).dtype.itemsize
a = array(range(5), float)
assert repr(a) == "array([0.0, 1.0, 2.0, 3.0, 4.0])"
a = array([], float)
@@ -1361,14 +1378,26 @@
a = zeros(1001)
assert repr(a) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])"
a = array(range(5), long)
- assert repr(a) == "array([0, 1, 2, 3, 4])"
+ if a.dtype.itemsize == int_size:
+ assert repr(a) == "array([0, 1, 2, 3, 4])"
+ else:
+ assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)"
+ a = array(range(5), 'int32')
+ if a.dtype.itemsize == int_size:
+ assert repr(a) == "array([0, 1, 2, 3, 4])"
+ else:
+ assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)"
a = array([], long)
assert repr(a) == "array([], dtype=int64)"
a = array([True, False, True, False], "?")
assert repr(a) == "array([True, False, True, False], dtype=bool)"
+ a = zeros([])
+ assert repr(a) == "array(0.0)"
+ a = array(0.2)
+ assert repr(a) == "array(0.2)"
def test_repr_multi(self):
- from numpypy import array, zeros
+ from numpypy import arange, zeros
a = zeros((3, 4))
assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0],
@@ -1381,6 +1410,16 @@
[[0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0]]])'''
+ a = arange(1002).reshape((2, 501))
+ assert repr(a) == '''array([[0, 1, 2, ..., 498, 499, 500],
+ [501, 502, 503, ..., 999, 1000, 1001]])'''
+ assert repr(a.T) == '''array([[0, 501],
+ [1, 502],
+ [2, 503],
+ ...,
+ [498, 999],
+ [499, 1000],
+ [500, 1001]])'''
def test_repr_slice(self):
from numpypy import array, zeros
@@ -1424,7 +1463,7 @@
a = zeros((400, 400), dtype=int)
assert str(a) == "[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n" \
- " [0 0 0 ..., 0 0 0]\n ..., \n [0 0 0 ..., 0 0 0]\n" \
+ " [0 0 0 ..., 0 0 0]\n ...,\n [0 0 0 ..., 0 0 0]\n" \
" [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]"
a = zeros((2, 2, 2))
r = str(a)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -8,10 +8,12 @@
from pypy.tool import logparser
from pypy.jit.tool.jitoutput import parse_prof
from pypy.module.pypyjit.test_pypy_c.model import (Log, find_ids_range,
- find_ids, TraceWithIds,
+ find_ids,
OpMatcher, InvalidMatch)
class BaseTestPyPyC(object):
+ log_string = 'jit-log-opt,jit-log-noopt,jit-log-virtualstate,jit-summary'
+
def setup_class(cls):
if '__pypy__' not in sys.builtin_module_names:
py.test.skip("must run this test with pypy")
@@ -52,8 +54,7 @@
cmdline += ['--jit', ','.join(jitcmdline)]
cmdline.append(str(self.filepath))
#
- print cmdline, logfile
- env={'PYPYLOG': 'jit-log-opt,jit-log-noopt,jit-log-virtualstate,jit-summary:' + str(logfile)}
+ env={'PYPYLOG': self.log_string + ':' + str(logfile)}
pipe = subprocess.Popen(cmdline,
env=env,
stdout=subprocess.PIPE,
diff --git a/pypy/module/pypyjit/test_pypy_c/test__ffi.py b/pypy/module/pypyjit/test_pypy_c/test__ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test__ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test__ffi.py
@@ -98,7 +98,8 @@
end = time.time()
return end - start
#
- log = self.run(main, [get_libc_name(), 200], threshold=150)
+ log = self.run(main, [get_libc_name(), 200], threshold=150,
+ import_site=True)
assert 1 <= log.result <= 1.5 # at most 0.5 seconds of overhead
loops = log.loops_by_id('sleep')
assert len(loops) == 1 # make sure that we actually JITted the loop
@@ -121,7 +122,7 @@
return fabs._ptr.getaddr(), x
libm_name = get_libm_name(sys.platform)
- log = self.run(main, [libm_name])
+ log = self.run(main, [libm_name], import_site=True)
fabs_addr, res = log.result
assert res == -4.0
loop, = log.loops_by_filename(self.filepath)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -15,7 +15,7 @@
i += letters[i % len(letters)] == uletters[i % len(letters)]
return i
- log = self.run(main, [300])
+ log = self.run(main, [300], import_site=True)
assert log.result == 300
loop, = log.loops_by_filename(self.filepath)
assert loop.match("""
@@ -55,7 +55,7 @@
i += int(long(string.digits[i % len(string.digits)], 16))
return i
- log = self.run(main, [1100])
+ log = self.run(main, [1100], import_site=True)
assert log.result == main(1100)
loop, = log.loops_by_filename(self.filepath)
assert loop.match("""
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -66,11 +66,11 @@
return None
copyright_str = """
-Copyright 2003-2011 PyPy development team.
+Copyright 2003-2012 PyPy development team.
All Rights Reserved.
For further information, see <http://pypy.org>
-Portions Copyright (c) 2001-2008 Python Software Foundation.
+Portions Copyright (c) 2001-2012 Python Software Foundation.
All Rights Reserved.
Portions Copyright (c) 2000 BeOpen.com.
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -528,6 +528,9 @@
set_param(driver, name1, int(value))
except ValueError:
raise
+ break
+ else:
+ raise ValueError
set_user_param._annspecialcase_ = 'specialize:arg(0)'
diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -32,6 +32,9 @@
def getres(self):
return self.res
+ def getdescr(self):
+ return self.descr
+
def is_guard(self):
return self._is_guard
@@ -182,7 +185,10 @@
return self.code.map[self.bytecode_no]
def getlineno(self):
- return self.getopcode().lineno
+ code = self.getopcode()
+ if code is None:
+ return None
+ return code.lineno
lineno = property(getlineno)
def getline_starts_here(self):
diff --git a/pypy/tool/jitlogparser/storage.py b/pypy/tool/jitlogparser/storage.py
--- a/pypy/tool/jitlogparser/storage.py
+++ b/pypy/tool/jitlogparser/storage.py
@@ -6,7 +6,6 @@
import py
import os
from lib_pypy.disassembler import dis
-from pypy.tool.jitlogparser.parser import Function
from pypy.tool.jitlogparser.module_finder import gather_all_code_objs
class LoopStorage(object):
diff --git a/pypy/translator/sandbox/pypy_interact.py b/pypy/translator/sandbox/pypy_interact.py
--- a/pypy/translator/sandbox/pypy_interact.py
+++ b/pypy/translator/sandbox/pypy_interact.py
@@ -13,7 +13,8 @@
ATM this only works with PyPy translated with Boehm or
the semispace or generation GCs.
--timeout=N limit execution time to N (real-time) seconds.
- --log=FILE log all user input into the FILE
+ --log=FILE log all user input into the FILE.
+ --verbose log all proxied system calls.
Note that you can get readline-like behavior with a tool like 'ledit',
provided you use enough -u options:
@@ -30,15 +31,15 @@
LIB_ROOT = os.path.dirname(os.path.dirname(pypy.__file__))
class PyPySandboxedProc(VirtualizedSandboxedProc, SimpleIOSandboxedProc):
- debug = True
argv0 = '/bin/pypy-c'
virtual_cwd = '/tmp'
virtual_env = {}
virtual_console_isatty = True
- def __init__(self, executable, arguments, tmpdir=None):
+ def __init__(self, executable, arguments, tmpdir=None, debug=True):
self.executable = executable = os.path.abspath(executable)
self.tmpdir = tmpdir
+ self.debug = debug
super(PyPySandboxedProc, self).__init__([self.argv0] + arguments,
executable=executable)
@@ -68,12 +69,13 @@
if __name__ == '__main__':
from getopt import getopt # and not gnu_getopt!
- options, arguments = getopt(sys.argv[1:], 't:h',
+ options, arguments = getopt(sys.argv[1:], 't:hv',
['tmp=', 'heapsize=', 'timeout=', 'log=',
- 'help'])
+ 'verbose', 'help'])
tmpdir = None
timeout = None
logfile = None
+ debug = False
extraoptions = []
def help():
@@ -105,6 +107,8 @@
timeout = int(value)
elif option == '--log':
logfile = value
+ elif option in ['-v', '--verbose']:
+ debug = True
elif option in ['-h', '--help']:
help()
else:
@@ -114,7 +118,7 @@
help()
sandproc = PyPySandboxedProc(arguments[0], extraoptions + arguments[1:],
- tmpdir=tmpdir)
+ tmpdir=tmpdir, debug=debug)
if timeout is not None:
sandproc.settimeout(timeout, interrupt_main=True)
if logfile is not None:
diff --git a/pypy/translator/sandbox/sandlib.py b/pypy/translator/sandbox/sandlib.py
--- a/pypy/translator/sandbox/sandlib.py
+++ b/pypy/translator/sandbox/sandlib.py
@@ -4,25 +4,29 @@
for the outer process, which can run CPython or PyPy.
"""
-import py
import sys, os, posixpath, errno, stat, time
-from pypy.tool.ansi_print import AnsiLog
import subprocess
from pypy.tool.killsubprocess import killsubprocess
from pypy.translator.sandbox.vfs import UID, GID
+import py
-class MyAnsiLog(AnsiLog):
- KW_TO_COLOR = {
- 'call': ((34,), False),
- 'result': ((34,), False),
- 'exception': ((34,), False),
- 'vpath': ((35,), False),
- 'timeout': ((1, 31), True),
- }
+def create_log():
+ """Make and return a log for the sandbox to use, if needed."""
+ # These imports are local to avoid importing pypy if we don't need to.
+ from pypy.tool.ansi_print import AnsiLog
-log = py.log.Producer("sandlib")
-py.log.setconsumer("sandlib", MyAnsiLog())
+ class MyAnsiLog(AnsiLog):
+ KW_TO_COLOR = {
+ 'call': ((34,), False),
+ 'result': ((34,), False),
+ 'exception': ((34,), False),
+ 'vpath': ((35,), False),
+ 'timeout': ((1, 31), True),
+ }
+ log = py.log.Producer("sandlib")
+ py.log.setconsumer("sandlib", MyAnsiLog())
+ return log
# Note: we use lib_pypy/marshal.py instead of the built-in marshal
# for two reasons. The built-in module could be made to segfault
@@ -127,6 +131,7 @@
for the external functions xxx that you want to support.
"""
debug = False
+ log = None
os_level_sandboxing = False # Linux only: /proc/PID/seccomp
def __init__(self, args, executable=None):
@@ -143,6 +148,9 @@
self.currenttimeout = None
self.currentlyidlefrom = None
+ if self.debug:
+ self.log = create_log()
+
def withlock(self, function, *args, **kwds):
lock = self.popenlock
if lock is not None:
@@ -170,7 +178,8 @@
if delay <= 0.0:
break # expired!
time.sleep(min(delay*1.001, 1))
- log.timeout("timeout!")
+ if self.log:
+ self.log.timeout("timeout!")
self.kill()
#if interrupt_main:
# if hasattr(os, 'kill'):
@@ -247,22 +256,22 @@
args = read_message(child_stdout)
except EOFError, e:
break
- if self.debug and not self.is_spam(fnname, *args):
- log.call('%s(%s)' % (fnname,
+ if self.log and not self.is_spam(fnname, *args):
+ self.log.call('%s(%s)' % (fnname,
', '.join([shortrepr(x) for x in args])))
try:
answer, resulttype = self.handle_message(fnname, *args)
except Exception, e:
tb = sys.exc_info()[2]
write_exception(child_stdin, e, tb)
- if self.debug:
+ if self.log:
if str(e):
- log.exception('%s: %s' % (e.__class__.__name__, e))
+ self.log.exception('%s: %s' % (e.__class__.__name__, e))
else:
- log.exception('%s' % (e.__class__.__name__,))
+ self.log.exception('%s' % (e.__class__.__name__,))
else:
- if self.debug and not self.is_spam(fnname, *args):
- log.result(shortrepr(answer))
+ if self.log and not self.is_spam(fnname, *args):
+ self.log.result(shortrepr(answer))
try:
write_message(child_stdin, 0) # error code - 0 for ok
write_message(child_stdin, answer, resulttype)
@@ -441,7 +450,8 @@
node = dirnode.join(name)
else:
node = dirnode
- log.vpath('%r => %r' % (vpath, node))
+ if self.log:
+ self.log.vpath('%r => %r' % (vpath, node))
return node
def do_ll_os__ll_os_stat(self, vpathname):
More information about the pypy-commit
mailing list