[pypy-commit] pypy dict-strategies: merge default
cfbolz
noreply at buildbot.pypy.org
Wed Jun 22 15:41:27 CEST 2011
Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: dict-strategies
Changeset: r45056:f7f8cbe664d0
Date: 2011-06-22 14:54 +0200
http://bitbucket.org/pypy/pypy/changeset/f7f8cbe664d0/
Log: merge default
diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
#
-__version__ = '2.0.3'
+__version__ = '2.1.0.dev4'
diff --git a/_pytest/assertion.py b/_pytest/assertion.py
deleted file mode 100644
--- a/_pytest/assertion.py
+++ /dev/null
@@ -1,177 +0,0 @@
-"""
-support for presented detailed information in failing assertions.
-"""
-import py
-import sys
-from _pytest.monkeypatch import monkeypatch
-
-def pytest_addoption(parser):
- group = parser.getgroup("debugconfig")
- group._addoption('--no-assert', action="store_true", default=False,
- dest="noassert",
- help="disable python assert expression reinterpretation."),
-
-def pytest_configure(config):
- # The _reprcompare attribute on the py.code module is used by
- # py._code._assertionnew to detect this plugin was loaded and in
- # turn call the hooks defined here as part of the
- # DebugInterpreter.
- m = monkeypatch()
- config._cleanup.append(m.undo)
- warn_about_missing_assertion()
- if not config.getvalue("noassert") and not config.getvalue("nomagic"):
- def callbinrepr(op, left, right):
- hook_result = config.hook.pytest_assertrepr_compare(
- config=config, op=op, left=left, right=right)
- for new_expl in hook_result:
- if new_expl:
- return '\n~'.join(new_expl)
- m.setattr(py.builtin.builtins,
- 'AssertionError', py.code._AssertionError)
- m.setattr(py.code, '_reprcompare', callbinrepr)
-
-def warn_about_missing_assertion():
- try:
- assert False
- except AssertionError:
- pass
- else:
- sys.stderr.write("WARNING: failing tests may report as passing because "
- "assertions are turned off! (are you using python -O?)\n")
-
-# Provide basestring in python3
-try:
- basestring = basestring
-except NameError:
- basestring = str
-
-
-def pytest_assertrepr_compare(op, left, right):
- """return specialised explanations for some operators/operands"""
- width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op
- left_repr = py.io.saferepr(left, maxsize=int(width/2))
- right_repr = py.io.saferepr(right, maxsize=width-len(left_repr))
- summary = '%s %s %s' % (left_repr, op, right_repr)
-
- issequence = lambda x: isinstance(x, (list, tuple))
- istext = lambda x: isinstance(x, basestring)
- isdict = lambda x: isinstance(x, dict)
- isset = lambda x: isinstance(x, set)
-
- explanation = None
- try:
- if op == '==':
- if istext(left) and istext(right):
- explanation = _diff_text(left, right)
- elif issequence(left) and issequence(right):
- explanation = _compare_eq_sequence(left, right)
- elif isset(left) and isset(right):
- explanation = _compare_eq_set(left, right)
- elif isdict(left) and isdict(right):
- explanation = _diff_text(py.std.pprint.pformat(left),
- py.std.pprint.pformat(right))
- elif op == 'not in':
- if istext(left) and istext(right):
- explanation = _notin_text(left, right)
- except py.builtin._sysex:
- raise
- except:
- excinfo = py.code.ExceptionInfo()
- explanation = ['(pytest_assertion plugin: representation of '
- 'details failed. Probably an object has a faulty __repr__.)',
- str(excinfo)
- ]
-
-
- if not explanation:
- return None
-
- # Don't include pageloads of data, should be configurable
- if len(''.join(explanation)) > 80*8:
- explanation = ['Detailed information too verbose, truncated']
-
- return [summary] + explanation
-
-
-def _diff_text(left, right):
- """Return the explanation for the diff between text
-
- This will skip leading and trailing characters which are
- identical to keep the diff minimal.
- """
- explanation = []
- i = 0 # just in case left or right has zero length
- for i in range(min(len(left), len(right))):
- if left[i] != right[i]:
- break
- if i > 42:
- i -= 10 # Provide some context
- explanation = ['Skipping %s identical '
- 'leading characters in diff' % i]
- left = left[i:]
- right = right[i:]
- if len(left) == len(right):
- for i in range(len(left)):
- if left[-i] != right[-i]:
- break
- if i > 42:
- i -= 10 # Provide some context
- explanation += ['Skipping %s identical '
- 'trailing characters in diff' % i]
- left = left[:-i]
- right = right[:-i]
- explanation += [line.strip('\n')
- for line in py.std.difflib.ndiff(left.splitlines(),
- right.splitlines())]
- return explanation
-
-
-def _compare_eq_sequence(left, right):
- explanation = []
- for i in range(min(len(left), len(right))):
- if left[i] != right[i]:
- explanation += ['At index %s diff: %r != %r' %
- (i, left[i], right[i])]
- break
- if len(left) > len(right):
- explanation += ['Left contains more items, '
- 'first extra item: %s' % py.io.saferepr(left[len(right)],)]
- elif len(left) < len(right):
- explanation += ['Right contains more items, '
- 'first extra item: %s' % py.io.saferepr(right[len(left)],)]
- return explanation # + _diff_text(py.std.pprint.pformat(left),
- # py.std.pprint.pformat(right))
-
-
-def _compare_eq_set(left, right):
- explanation = []
- diff_left = left - right
- diff_right = right - left
- if diff_left:
- explanation.append('Extra items in the left set:')
- for item in diff_left:
- explanation.append(py.io.saferepr(item))
- if diff_right:
- explanation.append('Extra items in the right set:')
- for item in diff_right:
- explanation.append(py.io.saferepr(item))
- return explanation
-
-
-def _notin_text(term, text):
- index = text.find(term)
- head = text[:index]
- tail = text[index+len(term):]
- correct_text = head + tail
- diff = _diff_text(correct_text, text)
- newdiff = ['%s is contained here:' % py.io.saferepr(term, maxsize=42)]
- for line in diff:
- if line.startswith('Skipping'):
- continue
- if line.startswith('- '):
- continue
- if line.startswith('+ '):
- newdiff.append(' ' + line[2:])
- else:
- newdiff.append(line)
- return newdiff
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/__init__.py
@@ -0,0 +1,128 @@
+"""
+support for presenting detailed information in failing assertions.
+"""
+import py
+import imp
+import marshal
+import struct
+import sys
+import pytest
+from _pytest.monkeypatch import monkeypatch
+from _pytest.assertion import reinterpret, util
+
+try:
+ from _pytest.assertion.rewrite import rewrite_asserts
+except ImportError:
+ rewrite_asserts = None
+else:
+ import ast
+
+def pytest_addoption(parser):
+ group = parser.getgroup("debugconfig")
+ group.addoption('--assertmode', action="store", dest="assertmode",
+ choices=("on", "old", "off", "default"), default="default",
+ metavar="on|old|off",
+ help="""control assertion debugging tools.
+'off' performs no assertion debugging.
+'old' reinterprets the expressions in asserts to glean information.
+'on' (the default) rewrites the assert statements in test modules to provide
+sub-expression results.""")
+ group.addoption('--no-assert', action="store_true", default=False,
+ dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+ group.addoption('--nomagic', action="store_true", default=False,
+ dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+
+class AssertionState:
+ """State for the assertion plugin."""
+
+ def __init__(self, config, mode):
+ self.mode = mode
+ self.trace = config.trace.root.get("assertion")
+
+def pytest_configure(config):
+ warn_about_missing_assertion()
+ mode = config.getvalue("assertmode")
+ if config.getvalue("noassert") or config.getvalue("nomagic"):
+ if mode not in ("off", "default"):
+ raise pytest.UsageError("assertion options conflict")
+ mode = "off"
+ elif mode == "default":
+ mode = "on"
+ if mode != "off":
+ def callbinrepr(op, left, right):
+ hook_result = config.hook.pytest_assertrepr_compare(
+ config=config, op=op, left=left, right=right)
+ for new_expl in hook_result:
+ if new_expl:
+ return '\n~'.join(new_expl)
+ m = monkeypatch()
+ config._cleanup.append(m.undo)
+ m.setattr(py.builtin.builtins, 'AssertionError',
+ reinterpret.AssertionError)
+ m.setattr(util, '_reprcompare', callbinrepr)
+ if mode == "on" and rewrite_asserts is None:
+ mode = "old"
+ config._assertstate = AssertionState(config, mode)
+ config._assertstate.trace("configured with mode set to %r" % (mode,))
+
+def _write_pyc(co, source_path):
+ if hasattr(imp, "cache_from_source"):
+ # Handle PEP 3147 pycs.
+ pyc = py.path.local(imp.cache_from_source(str(source_path)))
+ pyc.ensure()
+ else:
+ pyc = source_path + "c"
+ mtime = int(source_path.mtime())
+ fp = pyc.open("wb")
+ try:
+ fp.write(imp.get_magic())
+ fp.write(struct.pack("<l", mtime))
+ marshal.dump(co, fp)
+ finally:
+ fp.close()
+ return pyc
+
+def before_module_import(mod):
+ if mod.config._assertstate.mode != "on":
+ return
+ # Some deep magic: load the source, rewrite the asserts, and write a
+ # fake pyc, so that it'll be loaded when the module is imported.
+ source = mod.fspath.read()
+ try:
+ tree = ast.parse(source)
+ except SyntaxError:
+ # Let this pop up again in the real import.
+ mod.config._assertstate.trace("failed to parse: %r" % (mod.fspath,))
+ return
+ rewrite_asserts(tree)
+ try:
+ co = compile(tree, str(mod.fspath), "exec")
+ except SyntaxError:
+ # It's possible that this error is from some bug in the assertion
+ # rewriting, but I don't know of a fast way to tell.
+ mod.config._assertstate.trace("failed to compile: %r" % (mod.fspath,))
+ return
+ mod._pyc = _write_pyc(co, mod.fspath)
+ mod.config._assertstate.trace("wrote pyc: %r" % (mod._pyc,))
+
+def after_module_import(mod):
+ if not hasattr(mod, "_pyc"):
+ return
+ state = mod.config._assertstate
+ try:
+ mod._pyc.remove()
+ except py.error.ENOENT:
+ state.trace("couldn't find pyc: %r" % (mod._pyc,))
+ else:
+ state.trace("removed pyc: %r" % (mod._pyc,))
+
+def warn_about_missing_assertion():
+ try:
+ assert False
+ except AssertionError:
+ pass
+ else:
+ sys.stderr.write("WARNING: failing tests may report as passing because "
+ "assertions are turned off! (are you using python -O?)\n")
+
+pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/newinterpret.py
@@ -0,0 +1,333 @@
+"""
+Find intermediate evalutation results in assert statements through builtin AST.
+This should replace oldinterpret.py eventually.
+"""
+
+import sys
+import ast
+
+import py
+from _pytest.assertion import util
+from _pytest.assertion.reinterpret import BuiltinAssertionError
+
+
+if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
+ # See http://bugs.jython.org/issue1497
+ _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
+ "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
+ "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
+ "List", "Tuple")
+ _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
+ "AugAssign", "Print", "For", "While", "If", "With", "Raise",
+ "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
+ "Exec", "Global", "Expr", "Pass", "Break", "Continue")
+ _expr_nodes = set(getattr(ast, name) for name in _exprs)
+ _stmt_nodes = set(getattr(ast, name) for name in _stmts)
+ def _is_ast_expr(node):
+ return node.__class__ in _expr_nodes
+ def _is_ast_stmt(node):
+ return node.__class__ in _stmt_nodes
+else:
+ def _is_ast_expr(node):
+ return isinstance(node, ast.expr)
+ def _is_ast_stmt(node):
+ return isinstance(node, ast.stmt)
+
+
+class Failure(Exception):
+ """Error found while interpreting AST."""
+
+ def __init__(self, explanation=""):
+ self.cause = sys.exc_info()
+ self.explanation = explanation
+
+
+def interpret(source, frame, should_fail=False):
+ mod = ast.parse(source)
+ visitor = DebugInterpreter(frame)
+ try:
+ visitor.visit(mod)
+ except Failure:
+ failure = sys.exc_info()[1]
+ return getfailure(failure)
+ if should_fail:
+ return ("(assertion failed, but when it was re-run for "
+ "printing intermediate values, it did not fail. Suggestions: "
+ "compute assert expression before the assert or use --no-assert)")
+
+def run(offending_line, frame=None):
+ if frame is None:
+ frame = py.code.Frame(sys._getframe(1))
+ return interpret(offending_line, frame)
+
+def getfailure(e):
+ explanation = util.format_explanation(e.explanation)
+ value = e.cause[1]
+ if str(value):
+ lines = explanation.split('\n')
+ lines[0] += " << %s" % (value,)
+ explanation = '\n'.join(lines)
+ text = "%s: %s" % (e.cause[0].__name__, explanation)
+ if text.startswith('AssertionError: assert '):
+ text = text[16:]
+ return text
+
+operator_map = {
+ ast.BitOr : "|",
+ ast.BitXor : "^",
+ ast.BitAnd : "&",
+ ast.LShift : "<<",
+ ast.RShift : ">>",
+ ast.Add : "+",
+ ast.Sub : "-",
+ ast.Mult : "*",
+ ast.Div : "/",
+ ast.FloorDiv : "//",
+ ast.Mod : "%",
+ ast.Eq : "==",
+ ast.NotEq : "!=",
+ ast.Lt : "<",
+ ast.LtE : "<=",
+ ast.Gt : ">",
+ ast.GtE : ">=",
+ ast.Pow : "**",
+ ast.Is : "is",
+ ast.IsNot : "is not",
+ ast.In : "in",
+ ast.NotIn : "not in"
+}
+
+unary_map = {
+ ast.Not : "not %s",
+ ast.Invert : "~%s",
+ ast.USub : "-%s",
+ ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+ """Interpret AST nodes to gleam useful debugging information. """
+
+ def __init__(self, frame):
+ self.frame = frame
+
+ def generic_visit(self, node):
+ # Fallback when we don't have a special implementation.
+ if _is_ast_expr(node):
+ mod = ast.Expression(node)
+ co = self._compile(mod)
+ try:
+ result = self.frame.eval(co)
+ except Exception:
+ raise Failure()
+ explanation = self.frame.repr(result)
+ return explanation, result
+ elif _is_ast_stmt(node):
+ mod = ast.Module([node])
+ co = self._compile(mod, "exec")
+ try:
+ self.frame.exec_(co)
+ except Exception:
+ raise Failure()
+ return None, None
+ else:
+ raise AssertionError("can't handle %s" %(node,))
+
+ def _compile(self, source, mode="eval"):
+ return compile(source, "<assertion interpretation>", mode)
+
+ def visit_Expr(self, expr):
+ return self.visit(expr.value)
+
+ def visit_Module(self, mod):
+ for stmt in mod.body:
+ self.visit(stmt)
+
+ def visit_Name(self, name):
+ explanation, result = self.generic_visit(name)
+ # See if the name is local.
+ source = "%r in locals() is not globals()" % (name.id,)
+ co = self._compile(source)
+ try:
+ local = self.frame.eval(co)
+ except Exception:
+ # have to assume it isn't
+ local = None
+ if local is None or not self.frame.is_true(local):
+ return name.id, result
+ return explanation, result
+
+ def visit_Compare(self, comp):
+ left = comp.left
+ left_explanation, left_result = self.visit(left)
+ for op, next_op in zip(comp.ops, comp.comparators):
+ next_explanation, next_result = self.visit(next_op)
+ op_symbol = operator_map[op.__class__]
+ explanation = "%s %s %s" % (left_explanation, op_symbol,
+ next_explanation)
+ source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+ co = self._compile(source)
+ try:
+ result = self.frame.eval(co, __exprinfo_left=left_result,
+ __exprinfo_right=next_result)
+ except Exception:
+ raise Failure(explanation)
+ try:
+ if not self.frame.is_true(result):
+ break
+ except KeyboardInterrupt:
+ raise
+ except:
+ break
+ left_explanation, left_result = next_explanation, next_result
+
+ if util._reprcompare is not None:
+ res = util._reprcompare(op_symbol, left_result, next_result)
+ if res:
+ explanation = res
+ return explanation, result
+
+ def visit_BoolOp(self, boolop):
+ is_or = isinstance(boolop.op, ast.Or)
+ explanations = []
+ for operand in boolop.values:
+ explanation, result = self.visit(operand)
+ explanations.append(explanation)
+ if result == is_or:
+ break
+ name = is_or and " or " or " and "
+ explanation = "(" + name.join(explanations) + ")"
+ return explanation, result
+
+ def visit_UnaryOp(self, unary):
+ pattern = unary_map[unary.op.__class__]
+ operand_explanation, operand_result = self.visit(unary.operand)
+ explanation = pattern % (operand_explanation,)
+ co = self._compile(pattern % ("__exprinfo_expr",))
+ try:
+ result = self.frame.eval(co, __exprinfo_expr=operand_result)
+ except Exception:
+ raise Failure(explanation)
+ return explanation, result
+
+ def visit_BinOp(self, binop):
+ left_explanation, left_result = self.visit(binop.left)
+ right_explanation, right_result = self.visit(binop.right)
+ symbol = operator_map[binop.op.__class__]
+ explanation = "(%s %s %s)" % (left_explanation, symbol,
+ right_explanation)
+ source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+ co = self._compile(source)
+ try:
+ result = self.frame.eval(co, __exprinfo_left=left_result,
+ __exprinfo_right=right_result)
+ except Exception:
+ raise Failure(explanation)
+ return explanation, result
+
+ def visit_Call(self, call):
+ func_explanation, func = self.visit(call.func)
+ arg_explanations = []
+ ns = {"__exprinfo_func" : func}
+ arguments = []
+ for arg in call.args:
+ arg_explanation, arg_result = self.visit(arg)
+ arg_name = "__exprinfo_%s" % (len(ns),)
+ ns[arg_name] = arg_result
+ arguments.append(arg_name)
+ arg_explanations.append(arg_explanation)
+ for keyword in call.keywords:
+ arg_explanation, arg_result = self.visit(keyword.value)
+ arg_name = "__exprinfo_%s" % (len(ns),)
+ ns[arg_name] = arg_result
+ keyword_source = "%s=%%s" % (keyword.arg)
+ arguments.append(keyword_source % (arg_name,))
+ arg_explanations.append(keyword_source % (arg_explanation,))
+ if call.starargs:
+ arg_explanation, arg_result = self.visit(call.starargs)
+ arg_name = "__exprinfo_star"
+ ns[arg_name] = arg_result
+ arguments.append("*%s" % (arg_name,))
+ arg_explanations.append("*%s" % (arg_explanation,))
+ if call.kwargs:
+ arg_explanation, arg_result = self.visit(call.kwargs)
+ arg_name = "__exprinfo_kwds"
+ ns[arg_name] = arg_result
+ arguments.append("**%s" % (arg_name,))
+ arg_explanations.append("**%s" % (arg_explanation,))
+ args_explained = ", ".join(arg_explanations)
+ explanation = "%s(%s)" % (func_explanation, args_explained)
+ args = ", ".join(arguments)
+ source = "__exprinfo_func(%s)" % (args,)
+ co = self._compile(source)
+ try:
+ result = self.frame.eval(co, **ns)
+ except Exception:
+ raise Failure(explanation)
+ pattern = "%s\n{%s = %s\n}"
+ rep = self.frame.repr(result)
+ explanation = pattern % (rep, rep, explanation)
+ return explanation, result
+
+ def _is_builtin_name(self, name):
+ pattern = "%r not in globals() and %r not in locals()"
+ source = pattern % (name.id, name.id)
+ co = self._compile(source)
+ try:
+ return self.frame.eval(co)
+ except Exception:
+ return False
+
+ def visit_Attribute(self, attr):
+ if not isinstance(attr.ctx, ast.Load):
+ return self.generic_visit(attr)
+ source_explanation, source_result = self.visit(attr.value)
+ explanation = "%s.%s" % (source_explanation, attr.attr)
+ source = "__exprinfo_expr.%s" % (attr.attr,)
+ co = self._compile(source)
+ try:
+ result = self.frame.eval(co, __exprinfo_expr=source_result)
+ except Exception:
+ raise Failure(explanation)
+ explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+ self.frame.repr(result),
+ source_explanation, attr.attr)
+ # Check if the attr is from an instance.
+ source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+ source = source % (attr.attr,)
+ co = self._compile(source)
+ try:
+ from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+ except Exception:
+ from_instance = None
+ if from_instance is None or self.frame.is_true(from_instance):
+ rep = self.frame.repr(result)
+ pattern = "%s\n{%s = %s\n}"
+ explanation = pattern % (rep, rep, explanation)
+ return explanation, result
+
+ def visit_Assert(self, assrt):
+ test_explanation, test_result = self.visit(assrt.test)
+ explanation = "assert %s" % (test_explanation,)
+ if not self.frame.is_true(test_result):
+ try:
+ raise BuiltinAssertionError
+ except Exception:
+ raise Failure(explanation)
+ return explanation, test_result
+
+ def visit_Assign(self, assign):
+ value_explanation, value_result = self.visit(assign.value)
+ explanation = "... = %s" % (value_explanation,)
+ name = ast.Name("__exprinfo_expr", ast.Load(),
+ lineno=assign.value.lineno,
+ col_offset=assign.value.col_offset)
+ new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+ col_offset=assign.col_offset)
+ mod = ast.Module([new_assign])
+ co = self._compile(mod, "exec")
+ try:
+ self.frame.exec_(co, __exprinfo_expr=value_result)
+ except Exception:
+ raise Failure(explanation)
+ return explanation, value_result
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/oldinterpret.py
@@ -0,0 +1,552 @@
+import py
+import sys, inspect
+from compiler import parse, ast, pycodegen
+from _pytest.assertion.util import format_explanation
+from _pytest.assertion.reinterpret import BuiltinAssertionError
+
+passthroughex = py.builtin._sysex
+
+class Failure:
+ def __init__(self, node):
+ self.exc, self.value, self.tb = sys.exc_info()
+ self.node = node
+
+class View(object):
+ """View base class.
+
+ If C is a subclass of View, then C(x) creates a proxy object around
+ the object x. The actual class of the proxy is not C in general,
+ but a *subclass* of C determined by the rules below. To avoid confusion
+ we call view class the class of the proxy (a subclass of C, so of View)
+ and object class the class of x.
+
+ Attributes and methods not found in the proxy are automatically read on x.
+ Other operations like setting attributes are performed on the proxy, as
+ determined by its view class. The object x is available from the proxy
+ as its __obj__ attribute.
+
+ The view class selection is determined by the __view__ tuples and the
+ optional __viewkey__ method. By default, the selected view class is the
+ most specific subclass of C whose __view__ mentions the class of x.
+ If no such subclass is found, the search proceeds with the parent
+ object classes. For example, C(True) will first look for a subclass
+ of C with __view__ = (..., bool, ...) and only if it doesn't find any
+ look for one with __view__ = (..., int, ...), and then ..., object,...
+ If everything fails the class C itself is considered to be the default.
+
+ Alternatively, the view class selection can be driven by another aspect
+ of the object x, instead of the class of x, by overriding __viewkey__.
+ See last example at the end of this module.
+ """
+
+ _viewcache = {}
+ __view__ = ()
+
+ def __new__(rootclass, obj, *args, **kwds):
+ self = object.__new__(rootclass)
+ self.__obj__ = obj
+ self.__rootclass__ = rootclass
+ key = self.__viewkey__()
+ try:
+ self.__class__ = self._viewcache[key]
+ except KeyError:
+ self.__class__ = self._selectsubclass(key)
+ return self
+
+ def __getattr__(self, attr):
+ # attributes not found in the normal hierarchy rooted on View
+ # are looked up in the object's real class
+ return getattr(self.__obj__, attr)
+
+ def __viewkey__(self):
+ return self.__obj__.__class__
+
+ def __matchkey__(self, key, subclasses):
+ if inspect.isclass(key):
+ keys = inspect.getmro(key)
+ else:
+ keys = [key]
+ for key in keys:
+ result = [C for C in subclasses if key in C.__view__]
+ if result:
+ return result
+ return []
+
+ def _selectsubclass(self, key):
+ subclasses = list(enumsubclasses(self.__rootclass__))
+ for C in subclasses:
+ if not isinstance(C.__view__, tuple):
+ C.__view__ = (C.__view__,)
+ choices = self.__matchkey__(key, subclasses)
+ if not choices:
+ return self.__rootclass__
+ elif len(choices) == 1:
+ return choices[0]
+ else:
+ # combine the multiple choices
+ return type('?', tuple(choices), {})
+
+ def __repr__(self):
+ return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
+
+
+def enumsubclasses(cls):
+ for subcls in cls.__subclasses__():
+ for subsubclass in enumsubclasses(subcls):
+ yield subsubclass
+ yield cls
+
+
+class Interpretable(View):
+ """A parse tree node with a few extra methods."""
+ explanation = None
+
+ def is_builtin(self, frame):
+ return False
+
+ def eval(self, frame):
+ # fall-back for unknown expression nodes
+ try:
+ expr = ast.Expression(self.__obj__)
+ expr.filename = '<eval>'
+ self.__obj__.filename = '<eval>'
+ co = pycodegen.ExpressionCodeGenerator(expr).getCode()
+ result = frame.eval(co)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+ self.result = result
+ self.explanation = self.explanation or frame.repr(self.result)
+
+ def run(self, frame):
+ # fall-back for unknown statement nodes
+ try:
+ expr = ast.Module(None, ast.Stmt([self.__obj__]))
+ expr.filename = '<run>'
+ co = pycodegen.ModuleCodeGenerator(expr).getCode()
+ frame.exec_(co)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+
+ def nice_explanation(self):
+ return format_explanation(self.explanation)
+
+
+class Name(Interpretable):
+ __view__ = ast.Name
+
+ def is_local(self, frame):
+ source = '%r in locals() is not globals()' % self.name
+ try:
+ return frame.is_true(frame.eval(source))
+ except passthroughex:
+ raise
+ except:
+ return False
+
+ def is_global(self, frame):
+ source = '%r in globals()' % self.name
+ try:
+ return frame.is_true(frame.eval(source))
+ except passthroughex:
+ raise
+ except:
+ return False
+
+ def is_builtin(self, frame):
+ source = '%r not in locals() and %r not in globals()' % (
+ self.name, self.name)
+ try:
+ return frame.is_true(frame.eval(source))
+ except passthroughex:
+ raise
+ except:
+ return False
+
+ def eval(self, frame):
+ super(Name, self).eval(frame)
+ if not self.is_local(frame):
+ self.explanation = self.name
+
+class Compare(Interpretable):
+ __view__ = ast.Compare
+
+ def eval(self, frame):
+ expr = Interpretable(self.expr)
+ expr.eval(frame)
+ for operation, expr2 in self.ops:
+ if hasattr(self, 'result'):
+ # shortcutting in chained expressions
+ if not frame.is_true(self.result):
+ break
+ expr2 = Interpretable(expr2)
+ expr2.eval(frame)
+ self.explanation = "%s %s %s" % (
+ expr.explanation, operation, expr2.explanation)
+ source = "__exprinfo_left %s __exprinfo_right" % operation
+ try:
+ self.result = frame.eval(source,
+ __exprinfo_left=expr.result,
+ __exprinfo_right=expr2.result)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+ expr = expr2
+
+class And(Interpretable):
+ __view__ = ast.And
+
+ def eval(self, frame):
+ explanations = []
+ for expr in self.nodes:
+ expr = Interpretable(expr)
+ expr.eval(frame)
+ explanations.append(expr.explanation)
+ self.result = expr.result
+ if not frame.is_true(expr.result):
+ break
+ self.explanation = '(' + ' and '.join(explanations) + ')'
+
+class Or(Interpretable):
+ __view__ = ast.Or
+
+ def eval(self, frame):
+ explanations = []
+ for expr in self.nodes:
+ expr = Interpretable(expr)
+ expr.eval(frame)
+ explanations.append(expr.explanation)
+ self.result = expr.result
+ if frame.is_true(expr.result):
+ break
+ self.explanation = '(' + ' or '.join(explanations) + ')'
+
+
+# == Unary operations ==
+keepalive = []
+for astclass, astpattern in {
+ ast.Not : 'not __exprinfo_expr',
+ ast.Invert : '(~__exprinfo_expr)',
+ }.items():
+
+ class UnaryArith(Interpretable):
+ __view__ = astclass
+
+ def eval(self, frame, astpattern=astpattern):
+ expr = Interpretable(self.expr)
+ expr.eval(frame)
+ self.explanation = astpattern.replace('__exprinfo_expr',
+ expr.explanation)
+ try:
+ self.result = frame.eval(astpattern,
+ __exprinfo_expr=expr.result)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+
+ keepalive.append(UnaryArith)
+
+# == Binary operations ==
+for astclass, astpattern in {
+ ast.Add : '(__exprinfo_left + __exprinfo_right)',
+ ast.Sub : '(__exprinfo_left - __exprinfo_right)',
+ ast.Mul : '(__exprinfo_left * __exprinfo_right)',
+ ast.Div : '(__exprinfo_left / __exprinfo_right)',
+ ast.Mod : '(__exprinfo_left % __exprinfo_right)',
+ ast.Power : '(__exprinfo_left ** __exprinfo_right)',
+ }.items():
+
+ class BinaryArith(Interpretable):
+ __view__ = astclass
+
+ def eval(self, frame, astpattern=astpattern):
+ left = Interpretable(self.left)
+ left.eval(frame)
+ right = Interpretable(self.right)
+ right.eval(frame)
+ self.explanation = (astpattern
+ .replace('__exprinfo_left', left .explanation)
+ .replace('__exprinfo_right', right.explanation))
+ try:
+ self.result = frame.eval(astpattern,
+ __exprinfo_left=left.result,
+ __exprinfo_right=right.result)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+
+ keepalive.append(BinaryArith)
+
+
+class CallFunc(Interpretable):
+ __view__ = ast.CallFunc
+
+ def is_bool(self, frame):
+ source = 'isinstance(__exprinfo_value, bool)'
+ try:
+ return frame.is_true(frame.eval(source,
+ __exprinfo_value=self.result))
+ except passthroughex:
+ raise
+ except:
+ return False
+
+ def eval(self, frame):
+ node = Interpretable(self.node)
+ node.eval(frame)
+ explanations = []
+ vars = {'__exprinfo_fn': node.result}
+ source = '__exprinfo_fn('
+ for a in self.args:
+ if isinstance(a, ast.Keyword):
+ keyword = a.name
+ a = a.expr
+ else:
+ keyword = None
+ a = Interpretable(a)
+ a.eval(frame)
+ argname = '__exprinfo_%d' % len(vars)
+ vars[argname] = a.result
+ if keyword is None:
+ source += argname + ','
+ explanations.append(a.explanation)
+ else:
+ source += '%s=%s,' % (keyword, argname)
+ explanations.append('%s=%s' % (keyword, a.explanation))
+ if self.star_args:
+ star_args = Interpretable(self.star_args)
+ star_args.eval(frame)
+ argname = '__exprinfo_star'
+ vars[argname] = star_args.result
+ source += '*' + argname + ','
+ explanations.append('*' + star_args.explanation)
+ if self.dstar_args:
+ dstar_args = Interpretable(self.dstar_args)
+ dstar_args.eval(frame)
+ argname = '__exprinfo_kwds'
+ vars[argname] = dstar_args.result
+ source += '**' + argname + ','
+ explanations.append('**' + dstar_args.explanation)
+ self.explanation = "%s(%s)" % (
+ node.explanation, ', '.join(explanations))
+ if source.endswith(','):
+ source = source[:-1]
+ source += ')'
+ try:
+ self.result = frame.eval(source, **vars)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+ if not node.is_builtin(frame) or not self.is_bool(frame):
+ r = frame.repr(self.result)
+ self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+class Getattr(Interpretable):
+ __view__ = ast.Getattr
+
+ def eval(self, frame):
+ expr = Interpretable(self.expr)
+ expr.eval(frame)
+ source = '__exprinfo_expr.%s' % self.attrname
+ try:
+ self.result = frame.eval(source, __exprinfo_expr=expr.result)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+ self.explanation = '%s.%s' % (expr.explanation, self.attrname)
+ # if the attribute comes from the instance, its value is interesting
+ source = ('hasattr(__exprinfo_expr, "__dict__") and '
+ '%r in __exprinfo_expr.__dict__' % self.attrname)
+ try:
+ from_instance = frame.is_true(
+ frame.eval(source, __exprinfo_expr=expr.result))
+ except passthroughex:
+ raise
+ except:
+ from_instance = True
+ if from_instance:
+ r = frame.repr(self.result)
+ self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+# == Re-interpretation of full statements ==
+
+class Assert(Interpretable):
+ __view__ = ast.Assert
+
+ def run(self, frame):
+ test = Interpretable(self.test)
+ test.eval(frame)
+ # print the result as 'assert <explanation>'
+ self.result = test.result
+ self.explanation = 'assert ' + test.explanation
+ if not frame.is_true(test.result):
+ try:
+ raise BuiltinAssertionError
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+
+class Assign(Interpretable):
+ __view__ = ast.Assign
+
+ def run(self, frame):
+ expr = Interpretable(self.expr)
+ expr.eval(frame)
+ self.result = expr.result
+ self.explanation = '... = ' + expr.explanation
+ # fall-back-run the rest of the assignment
+ ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
+ mod = ast.Module(None, ast.Stmt([ass]))
+ mod.filename = '<run>'
+ co = pycodegen.ModuleCodeGenerator(mod).getCode()
+ try:
+ frame.exec_(co, __exprinfo_expr=expr.result)
+ except passthroughex:
+ raise
+ except:
+ raise Failure(self)
+
+class Discard(Interpretable):
+ __view__ = ast.Discard
+
+ def run(self, frame):
+ expr = Interpretable(self.expr)
+ expr.eval(frame)
+ self.result = expr.result
+ self.explanation = expr.explanation
+
+class Stmt(Interpretable):
+ __view__ = ast.Stmt
+
+ def run(self, frame):
+ for stmt in self.nodes:
+ stmt = Interpretable(stmt)
+ stmt.run(frame)
+
+
+def report_failure(e):
+ explanation = e.node.nice_explanation()
+ if explanation:
+ explanation = ", in: " + explanation
+ else:
+ explanation = ""
+ sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
+
+def check(s, frame=None):
+ if frame is None:
+ frame = sys._getframe(1)
+ frame = py.code.Frame(frame)
+ expr = parse(s, 'eval')
+ assert isinstance(expr, ast.Expression)
+ node = Interpretable(expr.node)
+ try:
+ node.eval(frame)
+ except passthroughex:
+ raise
+ except Failure:
+ e = sys.exc_info()[1]
+ report_failure(e)
+ else:
+ if not frame.is_true(node.result):
+ sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
+
+
+###########################################################
+# API / Entry points
+# #########################################################
+
+def interpret(source, frame, should_fail=False):
+ module = Interpretable(parse(source, 'exec').node)
+ #print "got module", module
+ if isinstance(frame, py.std.types.FrameType):
+ frame = py.code.Frame(frame)
+ try:
+ module.run(frame)
+ except Failure:
+ e = sys.exc_info()[1]
+ return getfailure(e)
+ except passthroughex:
+ raise
+ except:
+ import traceback
+ traceback.print_exc()
+ if should_fail:
+ return ("(assertion failed, but when it was re-run for "
+ "printing intermediate values, it did not fail. Suggestions: "
+ "compute assert expression before the assert or use --nomagic)")
+ else:
+ return None
+
+def getmsg(excinfo):
+ if isinstance(excinfo, tuple):
+ excinfo = py.code.ExceptionInfo(excinfo)
+ #frame, line = gettbline(tb)
+ #frame = py.code.Frame(frame)
+ #return interpret(line, frame)
+
+ tb = excinfo.traceback[-1]
+ source = str(tb.statement).strip()
+ x = interpret(source, tb.frame, should_fail=True)
+ if not isinstance(x, str):
+ raise TypeError("interpret returned non-string %r" % (x,))
+ return x
+
+def getfailure(e):
+ explanation = e.node.nice_explanation()
+ if str(e.value):
+ lines = explanation.split('\n')
+ lines[0] += " << %s" % (e.value,)
+ explanation = '\n'.join(lines)
+ text = "%s: %s" % (e.exc.__name__, explanation)
+ if text.startswith('AssertionError: assert '):
+ text = text[16:]
+ return text
+
+def run(s, frame=None):
+ if frame is None:
+ frame = sys._getframe(1)
+ frame = py.code.Frame(frame)
+ module = Interpretable(parse(s, 'exec').node)
+ try:
+ module.run(frame)
+ except Failure:
+ e = sys.exc_info()[1]
+ report_failure(e)
+
+
+if __name__ == '__main__':
+ # example:
+ def f():
+ return 5
+ def g():
+ return 3
+ def h(x):
+ return 'never'
+ check("f() * g() == 5")
+ check("not f()")
+ check("not (f() and g() or 0)")
+ check("f() == g()")
+ i = 4
+ check("i == f()")
+ check("len(f()) == 0")
+ check("isinstance(2+3+4, float)")
+
+ run("x = i")
+ check("x == 5")
+
+ run("assert not f(), 'oops'")
+ run("a, b, c = 1, 2")
+ run("a, b, c = f()")
+
+ check("max([f(),g()]) == 4")
+ check("'hello'[g()] == 'h'")
+ run("'guk%d' % h(f())")
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/reinterpret.py
@@ -0,0 +1,48 @@
+import sys
+import py
+
+BuiltinAssertionError = py.builtin.builtins.AssertionError
+
+class AssertionError(BuiltinAssertionError):
+ def __init__(self, *args):
+ BuiltinAssertionError.__init__(self, *args)
+ if args:
+ try:
+ self.msg = str(args[0])
+ except py.builtin._sysex:
+ raise
+ except:
+ self.msg = "<[broken __repr__] %s at %0xd>" %(
+ args[0].__class__, id(args[0]))
+ else:
+ f = py.code.Frame(sys._getframe(1))
+ try:
+ source = f.code.fullsource
+ if source is not None:
+ try:
+ source = source.getstatement(f.lineno, assertion=True)
+ except IndexError:
+ source = None
+ else:
+ source = str(source.deindent()).strip()
+ except py.error.ENOENT:
+ source = None
+ # this can also occur during reinterpretation, when the
+ # co_filename is set to "<run>".
+ if source:
+ self.msg = reinterpret(source, f, should_fail=True)
+ else:
+ self.msg = "<could not determine information>"
+ if not self.args:
+ self.args = (self.msg,)
+
+if sys.version_info > (3, 0):
+ AssertionError.__module__ = "builtins"
+ reinterpret_old = "old reinterpretation not available for py3"
+else:
+ from _pytest.assertion.oldinterpret import interpret as reinterpret_old
+if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
+ from _pytest.assertion.newinterpret import interpret as reinterpret
+else:
+ reinterpret = reinterpret_old
+
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/rewrite.py
@@ -0,0 +1,340 @@
+"""Rewrite assertion AST to produce nice error messages"""
+
+import ast
+import collections
+import itertools
+import sys
+
+import py
+from _pytest.assertion import util
+
+
+def rewrite_asserts(mod):
+ """Rewrite the assert statements in mod."""
+ AssertionRewriter().run(mod)
+
+
+_saferepr = py.io.saferepr
+from _pytest.assertion.util import format_explanation as _format_explanation
+
+def _format_boolop(operands, explanations, is_or):
+ show_explanations = []
+ for operand, expl in zip(operands, explanations):
+ show_explanations.append(expl)
+ if operand == is_or:
+ break
+ return "(" + (is_or and " or " or " and ").join(show_explanations) + ")"
+
+def _call_reprcompare(ops, results, expls, each_obj):
+ for i, res, expl in zip(range(len(ops)), results, expls):
+ try:
+ done = not res
+ except Exception:
+ done = True
+ if done:
+ break
+ if util._reprcompare is not None:
+ custom = util._reprcompare(ops[i], each_obj[i], each_obj[i + 1])
+ if custom is not None:
+ return custom
+ return expl
+
+
+unary_map = {
+ ast.Not : "not %s",
+ ast.Invert : "~%s",
+ ast.USub : "-%s",
+ ast.UAdd : "+%s"
+}
+
+binop_map = {
+ ast.BitOr : "|",
+ ast.BitXor : "^",
+ ast.BitAnd : "&",
+ ast.LShift : "<<",
+ ast.RShift : ">>",
+ ast.Add : "+",
+ ast.Sub : "-",
+ ast.Mult : "*",
+ ast.Div : "/",
+ ast.FloorDiv : "//",
+ ast.Mod : "%",
+ ast.Eq : "==",
+ ast.NotEq : "!=",
+ ast.Lt : "<",
+ ast.LtE : "<=",
+ ast.Gt : ">",
+ ast.GtE : ">=",
+ ast.Pow : "**",
+ ast.Is : "is",
+ ast.IsNot : "is not",
+ ast.In : "in",
+ ast.NotIn : "not in"
+}
+
+
+def set_location(node, lineno, col_offset):
+ """Set node location information recursively."""
+ def _fix(node, lineno, col_offset):
+ if "lineno" in node._attributes:
+ node.lineno = lineno
+ if "col_offset" in node._attributes:
+ node.col_offset = col_offset
+ for child in ast.iter_child_nodes(node):
+ _fix(child, lineno, col_offset)
+ _fix(node, lineno, col_offset)
+ return node
+
+
+class AssertionRewriter(ast.NodeVisitor):
+
+ def run(self, mod):
+ """Find all assert statements in *mod* and rewrite them."""
+ if not mod.body:
+ # Nothing to do.
+ return
+ # Insert some special imports at the top of the module but after any
+ # docstrings and __future__ imports.
+ aliases = [ast.alias(py.builtin.builtins.__name__, "@py_builtins"),
+ ast.alias("_pytest.assertion.rewrite", "@pytest_ar")]
+ expect_docstring = True
+ pos = 0
+ lineno = 0
+ for item in mod.body:
+ if (expect_docstring and isinstance(item, ast.Expr) and
+ isinstance(item.value, ast.Str)):
+ doc = item.value.s
+ if "PYTEST_DONT_REWRITE" in doc:
+ # The module has disabled assertion rewriting.
+ return
+ lineno += len(doc) - 1
+ expect_docstring = False
+ elif (not isinstance(item, ast.ImportFrom) or item.level > 0 and
+ item.identifier != "__future__"):
+ lineno = item.lineno
+ break
+ pos += 1
+ imports = [ast.Import([alias], lineno=lineno, col_offset=0)
+ for alias in aliases]
+ mod.body[pos:pos] = imports
+ # Collect asserts.
+ nodes = collections.deque([mod])
+ while nodes:
+ node = nodes.popleft()
+ for name, field in ast.iter_fields(node):
+ if isinstance(field, list):
+ new = []
+ for i, child in enumerate(field):
+ if isinstance(child, ast.Assert):
+ # Transform assert.
+ new.extend(self.visit(child))
+ else:
+ new.append(child)
+ if isinstance(child, ast.AST):
+ nodes.append(child)
+ setattr(node, name, new)
+ elif (isinstance(field, ast.AST) and
+ # Don't recurse into expressions as they can't contain
+ # asserts.
+ not isinstance(field, ast.expr)):
+ nodes.append(field)
+
+ def variable(self):
+ """Get a new variable."""
+ # Use a character invalid in python identifiers to avoid clashing.
+ name = "@py_assert" + str(next(self.variable_counter))
+ self.variables.add(name)
+ return name
+
+ def assign(self, expr):
+ """Give *expr* a name."""
+ name = self.variable()
+ self.statements.append(ast.Assign([ast.Name(name, ast.Store())], expr))
+ return ast.Name(name, ast.Load())
+
+ def display(self, expr):
+ """Call py.io.saferepr on the expression."""
+ return self.helper("saferepr", expr)
+
+ def helper(self, name, *args):
+ """Call a helper in this module."""
+ py_name = ast.Name("@pytest_ar", ast.Load())
+ attr = ast.Attribute(py_name, "_" + name, ast.Load())
+ return ast.Call(attr, list(args), [], None, None)
+
+ def builtin(self, name):
+ """Return the builtin called *name*."""
+ builtin_name = ast.Name("@py_builtins", ast.Load())
+ return ast.Attribute(builtin_name, name, ast.Load())
+
+ def explanation_param(self, expr):
+ specifier = "py" + str(next(self.variable_counter))
+ self.explanation_specifiers[specifier] = expr
+ return "%(" + specifier + ")s"
+
+ def push_format_context(self):
+ self.explanation_specifiers = {}
+ self.stack.append(self.explanation_specifiers)
+
+ def pop_format_context(self, expl_expr):
+ current = self.stack.pop()
+ if self.stack:
+ self.explanation_specifiers = self.stack[-1]
+ keys = [ast.Str(key) for key in current.keys()]
+ format_dict = ast.Dict(keys, list(current.values()))
+ form = ast.BinOp(expl_expr, ast.Mod(), format_dict)
+ name = "@py_format" + str(next(self.variable_counter))
+ self.on_failure.append(ast.Assign([ast.Name(name, ast.Store())], form))
+ return ast.Name(name, ast.Load())
+
+ def generic_visit(self, node):
+ """Handle expressions we don't have custom code for."""
+ assert isinstance(node, ast.expr)
+ res = self.assign(node)
+ return res, self.explanation_param(self.display(res))
+
+ def visit_Assert(self, assert_):
+ if assert_.msg:
+ # There's already a message. Don't mess with it.
+ return [assert_]
+ self.statements = []
+ self.variables = set()
+ self.variable_counter = itertools.count()
+ self.stack = []
+ self.on_failure = []
+ self.push_format_context()
+ # Rewrite assert into a bunch of statements.
+ top_condition, explanation = self.visit(assert_.test)
+ # Create failure message.
+ body = self.on_failure
+ negation = ast.UnaryOp(ast.Not(), top_condition)
+ self.statements.append(ast.If(negation, body, []))
+ explanation = "assert " + explanation
+ template = ast.Str(explanation)
+ msg = self.pop_format_context(template)
+ fmt = self.helper("format_explanation", msg)
+ err_name = ast.Name("AssertionError", ast.Load())
+ exc = ast.Call(err_name, [fmt], [], None, None)
+ if sys.version_info[0] >= 3:
+ raise_ = ast.Raise(exc, None)
+ else:
+ raise_ = ast.Raise(exc, None, None)
+ body.append(raise_)
+ # Delete temporary variables.
+ names = [ast.Name(name, ast.Del()) for name in self.variables]
+ if names:
+ delete = ast.Delete(names)
+ self.statements.append(delete)
+ # Fix line numbers.
+ for stmt in self.statements:
+ set_location(stmt, assert_.lineno, assert_.col_offset)
+ return self.statements
+
+ def visit_Name(self, name):
+ # Check if the name is local or not.
+ locs = ast.Call(self.builtin("locals"), [], [], None, None)
+ globs = ast.Call(self.builtin("globals"), [], [], None, None)
+ ops = [ast.In(), ast.IsNot()]
+ test = ast.Compare(ast.Str(name.id), ops, [locs, globs])
+ expr = ast.IfExp(test, self.display(name), ast.Str(name.id))
+ return name, self.explanation_param(expr)
+
+ def visit_BoolOp(self, boolop):
+ operands = []
+ explanations = []
+ self.push_format_context()
+ for operand in boolop.values:
+ res, explanation = self.visit(operand)
+ operands.append(res)
+ explanations.append(explanation)
+ expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
+ is_or = ast.Num(isinstance(boolop.op, ast.Or))
+ expl_template = self.helper("format_boolop",
+ ast.Tuple(operands, ast.Load()), expls,
+ is_or)
+ expl = self.pop_format_context(expl_template)
+ res = self.assign(ast.BoolOp(boolop.op, operands))
+ return res, self.explanation_param(expl)
+
+ def visit_UnaryOp(self, unary):
+ pattern = unary_map[unary.op.__class__]
+ operand_res, operand_expl = self.visit(unary.operand)
+ res = self.assign(ast.UnaryOp(unary.op, operand_res))
+ return res, pattern % (operand_expl,)
+
+ def visit_BinOp(self, binop):
+ symbol = binop_map[binop.op.__class__]
+ left_expr, left_expl = self.visit(binop.left)
+ right_expr, right_expl = self.visit(binop.right)
+ explanation = "(%s %s %s)" % (left_expl, symbol, right_expl)
+ res = self.assign(ast.BinOp(left_expr, binop.op, right_expr))
+ return res, explanation
+
+ def visit_Call(self, call):
+ new_func, func_expl = self.visit(call.func)
+ arg_expls = []
+ new_args = []
+ new_kwargs = []
+ new_star = new_kwarg = None
+ for arg in call.args:
+ res, expl = self.visit(arg)
+ new_args.append(res)
+ arg_expls.append(expl)
+ for keyword in call.keywords:
+ res, expl = self.visit(keyword.value)
+ new_kwargs.append(ast.keyword(keyword.arg, res))
+ arg_expls.append(keyword.arg + "=" + expl)
+ if call.starargs:
+ new_star, expl = self.visit(call.starargs)
+ arg_expls.append("*" + expl)
+ if call.kwargs:
+ new_kwarg, expl = self.visit(call.kwarg)
+ arg_expls.append("**" + expl)
+ expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
+ new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
+ res = self.assign(new_call)
+ res_expl = self.explanation_param(self.display(res))
+ outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl)
+ return res, outer_expl
+
+ def visit_Attribute(self, attr):
+ if not isinstance(attr.ctx, ast.Load):
+ return self.generic_visit(attr)
+ value, value_expl = self.visit(attr.value)
+ res = self.assign(ast.Attribute(value, attr.attr, ast.Load()))
+ res_expl = self.explanation_param(self.display(res))
+ pat = "%s\n{%s = %s.%s\n}"
+ expl = pat % (res_expl, res_expl, value_expl, attr.attr)
+ return res, expl
+
+ def visit_Compare(self, comp):
+ self.push_format_context()
+ left_res, left_expl = self.visit(comp.left)
+ res_variables = [self.variable() for i in range(len(comp.ops))]
+ load_names = [ast.Name(v, ast.Load()) for v in res_variables]
+ store_names = [ast.Name(v, ast.Store()) for v in res_variables]
+ it = zip(range(len(comp.ops)), comp.ops, comp.comparators)
+ expls = []
+ syms = []
+ results = [left_res]
+ for i, op, next_operand in it:
+ next_res, next_expl = self.visit(next_operand)
+ results.append(next_res)
+ sym = binop_map[op.__class__]
+ syms.append(ast.Str(sym))
+ expl = "%s %s %s" % (left_expl, sym, next_expl)
+ expls.append(ast.Str(expl))
+ res_expr = ast.Compare(left_res, [op], [next_res])
+ self.statements.append(ast.Assign([store_names[i]], res_expr))
+ left_res, left_expl = next_res, next_expl
+ # Use py.code._reprcompare if that's available.
+ expl_call = self.helper("call_reprcompare",
+ ast.Tuple(syms, ast.Load()),
+ ast.Tuple(load_names, ast.Load()),
+ ast.Tuple(expls, ast.Load()),
+ ast.Tuple(results, ast.Load()))
+ if len(comp.ops) > 1:
+ res = ast.BoolOp(ast.And(), load_names)
+ else:
+ res = load_names[0]
+ return res, self.explanation_param(self.pop_format_context(expl_call))
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/util.py
@@ -0,0 +1,213 @@
+"""Utilities for assertion debugging"""
+
+import py
+
+
+# The _reprcompare attribute on the util module is used by the new assertion
+# interpretation code and assertion rewriter to detect this plugin was
+# loaded and in turn call the hooks defined here as part of the
+# DebugInterpreter.
+_reprcompare = None
+
+def format_explanation(explanation):
+ """This formats an explanation
+
+ Normally all embedded newlines are escaped, however there are
+ three exceptions: \n{, \n} and \n~. The first two are intended
+ cover nested explanations, see function and attribute explanations
+ for examples (.visit_Call(), visit_Attribute()). The last one is
+ for when one explanation needs to span multiple lines, e.g. when
+ displaying diffs.
+ """
+ # simplify 'assert False where False = ...'
+ where = 0
+ while True:
+ start = where = explanation.find("False\n{False = ", where)
+ if where == -1:
+ break
+ level = 0
+ for i, c in enumerate(explanation[start:]):
+ if c == "{":
+ level += 1
+ elif c == "}":
+ level -= 1
+ if not level:
+ break
+ else:
+ raise AssertionError("unbalanced braces: %r" % (explanation,))
+ end = start + i
+ where = end
+ if explanation[end - 1] == '\n':
+ explanation = (explanation[:start] + explanation[start+15:end-1] +
+ explanation[end+1:])
+ where -= 17
+ raw_lines = (explanation or '').split('\n')
+ # escape newlines not followed by {, } and ~
+ lines = [raw_lines[0]]
+ for l in raw_lines[1:]:
+ if l.startswith('{') or l.startswith('}') or l.startswith('~'):
+ lines.append(l)
+ else:
+ lines[-1] += '\\n' + l
+
+ result = lines[:1]
+ stack = [0]
+ stackcnt = [0]
+ for line in lines[1:]:
+ if line.startswith('{'):
+ if stackcnt[-1]:
+ s = 'and '
+ else:
+ s = 'where '
+ stack.append(len(result))
+ stackcnt[-1] += 1
+ stackcnt.append(0)
+ result.append(' +' + ' '*(len(stack)-1) + s + line[1:])
+ elif line.startswith('}'):
+ assert line.startswith('}')
+ stack.pop()
+ stackcnt.pop()
+ result[stack[-1]] += line[1:]
+ else:
+ assert line.startswith('~')
+ result.append(' '*len(stack) + line[1:])
+ assert len(stack) == 1
+ return '\n'.join(result)
+
+
+# Provide basestring in python3
+try:
+ basestring = basestring
+except NameError:
+ basestring = str
+
+
+def assertrepr_compare(op, left, right):
+ """return specialised explanations for some operators/operands"""
+ width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op
+ left_repr = py.io.saferepr(left, maxsize=int(width/2))
+ right_repr = py.io.saferepr(right, maxsize=width-len(left_repr))
+ summary = '%s %s %s' % (left_repr, op, right_repr)
+
+ issequence = lambda x: isinstance(x, (list, tuple))
+ istext = lambda x: isinstance(x, basestring)
+ isdict = lambda x: isinstance(x, dict)
+ isset = lambda x: isinstance(x, set)
+
+ explanation = None
+ try:
+ if op == '==':
+ if istext(left) and istext(right):
+ explanation = _diff_text(left, right)
+ elif issequence(left) and issequence(right):
+ explanation = _compare_eq_sequence(left, right)
+ elif isset(left) and isset(right):
+ explanation = _compare_eq_set(left, right)
+ elif isdict(left) and isdict(right):
+ explanation = _diff_text(py.std.pprint.pformat(left),
+ py.std.pprint.pformat(right))
+ elif op == 'not in':
+ if istext(left) and istext(right):
+ explanation = _notin_text(left, right)
+ except py.builtin._sysex:
+ raise
+ except:
+ excinfo = py.code.ExceptionInfo()
+ explanation = ['(pytest_assertion plugin: representation of '
+ 'details failed. Probably an object has a faulty __repr__.)',
+ str(excinfo)
+ ]
+
+
+ if not explanation:
+ return None
+
+ # Don't include pageloads of data, should be configurable
+ if len(''.join(explanation)) > 80*8:
+ explanation = ['Detailed information too verbose, truncated']
+
+ return [summary] + explanation
+
+
+def _diff_text(left, right):
+ """Return the explanation for the diff between text
+
+ This will skip leading and trailing characters which are
+ identical to keep the diff minimal.
+ """
+ explanation = []
+ i = 0 # just in case left or right has zero length
+ for i in range(min(len(left), len(right))):
+ if left[i] != right[i]:
+ break
+ if i > 42:
+ i -= 10 # Provide some context
+ explanation = ['Skipping %s identical '
+ 'leading characters in diff' % i]
+ left = left[i:]
+ right = right[i:]
+ if len(left) == len(right):
+ for i in range(len(left)):
+ if left[-i] != right[-i]:
+ break
+ if i > 42:
+ i -= 10 # Provide some context
+ explanation += ['Skipping %s identical '
+ 'trailing characters in diff' % i]
+ left = left[:-i]
+ right = right[:-i]
+ explanation += [line.strip('\n')
+ for line in py.std.difflib.ndiff(left.splitlines(),
+ right.splitlines())]
+ return explanation
+
+
+def _compare_eq_sequence(left, right):
+ explanation = []
+ for i in range(min(len(left), len(right))):
+ if left[i] != right[i]:
+ explanation += ['At index %s diff: %r != %r' %
+ (i, left[i], right[i])]
+ break
+ if len(left) > len(right):
+ explanation += ['Left contains more items, '
+ 'first extra item: %s' % py.io.saferepr(left[len(right)],)]
+ elif len(left) < len(right):
+ explanation += ['Right contains more items, '
+ 'first extra item: %s' % py.io.saferepr(right[len(left)],)]
+ return explanation # + _diff_text(py.std.pprint.pformat(left),
+ # py.std.pprint.pformat(right))
+
+
+def _compare_eq_set(left, right):
+ explanation = []
+ diff_left = left - right
+ diff_right = right - left
+ if diff_left:
+ explanation.append('Extra items in the left set:')
+ for item in diff_left:
+ explanation.append(py.io.saferepr(item))
+ if diff_right:
+ explanation.append('Extra items in the right set:')
+ for item in diff_right:
+ explanation.append(py.io.saferepr(item))
+ return explanation
+
+
+def _notin_text(term, text):
+ index = text.find(term)
+ head = text[:index]
+ tail = text[index+len(term):]
+ correct_text = head + tail
+ diff = _diff_text(correct_text, text)
+ newdiff = ['%s is contained here:' % py.io.saferepr(term, maxsize=42)]
+ for line in diff:
+ if line.startswith('Skipping'):
+ continue
+ if line.startswith('- '):
+ continue
+ if line.startswith('+ '):
+ newdiff.append(' ' + line[2:])
+ else:
+ newdiff.append(line)
+ return newdiff
diff --git a/_pytest/doctest.py b/_pytest/doctest.py
--- a/_pytest/doctest.py
+++ b/_pytest/doctest.py
@@ -59,7 +59,7 @@
inner_excinfo = py.code.ExceptionInfo(excinfo.value.exc_info)
lines += ["UNEXPECTED EXCEPTION: %s" %
repr(inner_excinfo.value)]
-
+ lines += py.std.traceback.format_exception(*excinfo.value.exc_info)
return ReprFailDoctest(reprlocation, lines)
else:
return super(DoctestItem, self).repr_failure(excinfo)
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -16,9 +16,6 @@
group.addoption('--traceconfig',
action="store_true", dest="traceconfig", default=False,
help="trace considerations of conftest.py files."),
- group._addoption('--nomagic',
- action="store_true", dest="nomagic", default=False,
- help="don't reinterpret asserts, no traceback cutting. ")
group.addoption('--debug',
action="store_true", dest="debug", default=False,
help="generate and show internal debugging information.")
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -65,7 +65,8 @@
class LogXML(object):
def __init__(self, logfile, prefix):
- self.logfile = logfile
+ logfile = os.path.expanduser(os.path.expandvars(logfile))
+ self.logfile = os.path.normpath(logfile)
self.prefix = prefix
self.test_logs = []
self.passed = self.skipped = 0
@@ -76,7 +77,7 @@
names = report.nodeid.split("::")
names[0] = names[0].replace("/", '.')
names = tuple(names)
- d = {'time': self._durations.pop(names, "0")}
+ d = {'time': self._durations.pop(report.nodeid, "0")}
names = [x.replace(".py", "") for x in names if x != "()"]
classnames = names[:-1]
if self.prefix:
@@ -170,12 +171,11 @@
self.append_skipped(report)
def pytest_runtest_call(self, item, __multicall__):
- names = tuple(item.listnames())
start = time.time()
try:
return __multicall__.execute()
finally:
- self._durations[names] = time.time() - start
+ self._durations[item.nodeid] = time.time() - start
def pytest_collectreport(self, report):
if not report.passed:
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -46,23 +46,25 @@
def pytest_namespace():
- return dict(collect=dict(Item=Item, Collector=Collector, File=File))
+ collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
+ return dict(collect=collect)
def pytest_configure(config):
py.test.config = config # compatibiltiy
if config.option.exitfirst:
config.option.maxfail = 1
-def pytest_cmdline_main(config):
- """ default command line protocol for initialization, session,
- running tests and reporting. """
+def wrap_session(config, doit):
+ """Skeleton command line program"""
session = Session(config)
session.exitstatus = EXIT_OK
+ initstate = 0
try:
config.pluginmanager.do_configure(config)
+ initstate = 1
config.hook.pytest_sessionstart(session=session)
- config.hook.pytest_collection(session=session)
- config.hook.pytest_runtestloop(session=session)
+ initstate = 2
+ doit(config, session)
except pytest.UsageError:
raise
except KeyboardInterrupt:
@@ -77,18 +79,24 @@
sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
if not session.exitstatus and session._testsfailed:
session.exitstatus = EXIT_TESTSFAILED
- config.hook.pytest_sessionfinish(session=session,
- exitstatus=session.exitstatus)
- config.pluginmanager.do_unconfigure(config)
+ if initstate >= 2:
+ config.hook.pytest_sessionfinish(session=session,
+ exitstatus=session.exitstatus)
+ if initstate >= 1:
+ config.pluginmanager.do_unconfigure(config)
return session.exitstatus
+def pytest_cmdline_main(config):
+ return wrap_session(config, _main)
+
+def _main(config, session):
+ """ default command line protocol for initialization, session,
+ running tests and reporting. """
+ config.hook.pytest_collection(session=session)
+ config.hook.pytest_runtestloop(session=session)
+
def pytest_collection(session):
- session.perform_collect()
- hook = session.config.hook
- hook.pytest_collection_modifyitems(session=session,
- config=session.config, items=session.items)
- hook.pytest_collection_finish(session=session)
- return True
+ return session.perform_collect()
def pytest_runtestloop(session):
if session.config.option.collectonly:
@@ -374,6 +382,16 @@
return HookProxy(fspath, self.config)
def perform_collect(self, args=None, genitems=True):
+ hook = self.config.hook
+ try:
+ items = self._perform_collect(args, genitems)
+ hook.pytest_collection_modifyitems(session=self,
+ config=self.config, items=items)
+ finally:
+ hook.pytest_collection_finish(session=self)
+ return items
+
+ def _perform_collect(self, args, genitems):
if args is None:
args = self.config.args
self.trace("perform_collect", self, args)
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -153,7 +153,7 @@
def __repr__(self):
return "<MarkInfo %r args=%r kwargs=%r>" % (
- self._name, self.args, self.kwargs)
+ self.name, self.args, self.kwargs)
def pytest_itemcollected(item):
if not isinstance(item, pytest.Function):
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -6,7 +6,7 @@
import inspect
import time
from fnmatch import fnmatch
-from _pytest.main import Session
+from _pytest.main import Session, EXIT_OK
from py.builtin import print_
from _pytest.core import HookRelay
@@ -292,13 +292,19 @@
assert '::' not in str(arg)
p = py.path.local(arg)
x = session.fspath.bestrelpath(p)
- return session.perform_collect([x], genitems=False)[0]
+ config.hook.pytest_sessionstart(session=session)
+ res = session.perform_collect([x], genitems=False)[0]
+ config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
+ return res
def getpathnode(self, path):
- config = self.parseconfig(path)
+ config = self.parseconfigure(path)
session = Session(config)
x = session.fspath.bestrelpath(path)
- return session.perform_collect([x], genitems=False)[0]
+ config.hook.pytest_sessionstart(session=session)
+ res = session.perform_collect([x], genitems=False)[0]
+ config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
+ return res
def genitems(self, colitems):
session = colitems[0].session
@@ -312,7 +318,9 @@
config = self.parseconfigure(*args)
rec = self.getreportrecorder(config)
session = Session(config)
+ config.hook.pytest_sessionstart(session=session)
session.perform_collect()
+ config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
return session.items, rec
def runitem(self, source):
@@ -382,6 +390,8 @@
c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
keep=0, rootdir=self.tmpdir, lock_timeout=None)
c.parse(args)
+ c.pluginmanager.do_configure(c)
+ self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
return c
finally:
py.test.config = oldconfig
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -226,8 +226,13 @@
def _importtestmodule(self):
# we assume we are only called once per module
+ from _pytest import assertion
+ assertion.before_module_import(self)
try:
- mod = self.fspath.pyimport(ensuresyspath=True)
+ try:
+ mod = self.fspath.pyimport(ensuresyspath=True)
+ finally:
+ assertion.after_module_import(self)
except SyntaxError:
excinfo = py.code.ExceptionInfo()
raise self.CollectError(excinfo.getrepr(style="short"))
@@ -374,7 +379,7 @@
# test generators are seen as collectors but they also
# invoke setup/teardown on popular request
# (induced by the common "test_*" naming shared with normal tests)
- self.config._setupstate.prepare(self)
+ self.session._setupstate.prepare(self)
# see FunctionMixin.setup and test_setupstate_is_preserved_134
self._preservedparent = self.parent.obj
l = []
@@ -721,7 +726,7 @@
def _addfinalizer(self, finalizer, scope):
colitem = self._getscopeitem(scope)
- self.config._setupstate.addfinalizer(
+ self._pyfuncitem.session._setupstate.addfinalizer(
finalizer=finalizer, colitem=colitem)
def __repr__(self):
@@ -742,8 +747,10 @@
raise self.LookupError(msg)
def showfuncargs(config):
- from _pytest.main import Session
- session = Session(config)
+ from _pytest.main import wrap_session
+ return wrap_session(config, _showfuncargs_main)
+
+def _showfuncargs_main(config, session):
session.perform_collect()
if session.items:
plugins = session.items[0].getplugins()
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -14,17 +14,15 @@
#
# pytest plugin hooks
-# XXX move to pytest_sessionstart and fix py.test owns tests
-def pytest_configure(config):
- config._setupstate = SetupState()
+def pytest_sessionstart(session):
+ session._setupstate = SetupState()
def pytest_sessionfinish(session, exitstatus):
- if hasattr(session.config, '_setupstate'):
- hook = session.config.hook
- rep = hook.pytest__teardown_final(session=session)
- if rep:
- hook.pytest__teardown_final_logerror(session=session, report=rep)
- session.exitstatus = 1
+ hook = session.config.hook
+ rep = hook.pytest__teardown_final(session=session)
+ if rep:
+ hook.pytest__teardown_final_logerror(session=session, report=rep)
+ session.exitstatus = 1
class NodeInfo:
def __init__(self, location):
@@ -46,16 +44,16 @@
return reports
def pytest_runtest_setup(item):
- item.config._setupstate.prepare(item)
+ item.session._setupstate.prepare(item)
def pytest_runtest_call(item):
item.runtest()
def pytest_runtest_teardown(item):
- item.config._setupstate.teardown_exact(item)
+ item.session._setupstate.teardown_exact(item)
def pytest__teardown_final(session):
- call = CallInfo(session.config._setupstate.teardown_all, when="teardown")
+ call = CallInfo(session._setupstate.teardown_all, when="teardown")
if call.excinfo:
ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
call.excinfo.traceback = ntraceback.filter()
diff --git a/lib-python/modified-2.7/test/test_extcall.py b/lib-python/modified-2.7/test/test_extcall.py
--- a/lib-python/modified-2.7/test/test_extcall.py
+++ b/lib-python/modified-2.7/test/test_extcall.py
@@ -299,7 +299,7 @@
def f(a):
return a
self.assertEqual(f(**{u'a': 4}), 4)
- self.assertRaises(TypeError, lambda: f(**{u'stören': 4}))
+ self.assertRaises(TypeError, f, **{u'stören': 4})
self.assertRaises(TypeError, f, **{u'someLongString':2})
try:
f(a=4, **{u'a': 4})
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -275,7 +275,8 @@
return unicode(x, 'utf-8')
class Connection(object):
- def __init__(self, database, isolation_level="", detect_types=0, timeout=None, cached_statements=None, factory=None):
+ def __init__(self, database, timeout=5.0, detect_types=0, isolation_level="",
+ check_same_thread=True, factory=None, cached_statements=100):
self.db = c_void_p()
if sqlite.sqlite3_open(database, byref(self.db)) != SQLITE_OK:
raise OperationalError("Could not open database")
@@ -308,7 +309,8 @@
self._aggregates = {}
self.aggregate_instances = {}
self._collations = {}
- self.thread_ident = thread_get_ident()
+ if check_same_thread:
+ self.thread_ident = thread_get_ident()
def _get_exception(self, error_code = None):
if error_code is None:
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -1422,12 +1422,17 @@
converter = _time.localtime
else:
converter = _time.gmtime
- if 1 - (t % 1.0) < 0.000001:
- t = float(int(t)) + 1
- if t < 0:
- t -= 1
+ if t < 0.0:
+ us = int(round(((-t) % 1.0) * 1000000))
+ if us > 0:
+ us = 1000000 - us
+ t -= 1.0
+ else:
+ us = int(round((t % 1.0) * 1000000))
+ if us == 1000000:
+ us = 0
+ t += 1.0
y, m, d, hh, mm, ss, weekday, jday, dst = converter(t)
- us = int((t % 1.0) * 1000000)
ss = min(ss, 59) # clamp out leap seconds if the platform has them
result = cls(y, m, d, hh, mm, ss, us, tz)
if tz is not None:
diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -46,4 +46,42 @@
e = get_errno()
raise IOError(e, errno.errorcode[e])
+# Console I/O routines
+
+kbhit = _c._kbhit
+kbhit.argtypes = []
+kbhit.restype = ctypes.c_int
+
+getch = _c._getch
+getch.argtypes = []
+getch.restype = ctypes.c_char
+
+getwch = _c._getwch
+getwch.argtypes = []
+getwch.restype = ctypes.c_wchar
+
+getche = _c._getche
+getche.argtypes = []
+getche.restype = ctypes.c_char
+
+getwche = _c._getwche
+getwche.argtypes = []
+getwche.restype = ctypes.c_wchar
+
+putch = _c._putch
+putch.argtypes = [ctypes.c_char]
+putch.restype = None
+
+putwch = _c._putwch
+putwch.argtypes = [ctypes.c_wchar]
+putwch.restype = None
+
+ungetch = _c._ungetch
+ungetch.argtypes = [ctypes.c_char]
+ungetch.restype = None
+
+ungetwch = _c._ungetwch
+ungetwch.argtypes = [ctypes.c_wchar]
+ungetwch.restype = None
+
del ctypes
diff --git a/lib_pypy/pypy_test/test_datetime.py b/lib_pypy/pypy_test/test_datetime.py
--- a/lib_pypy/pypy_test/test_datetime.py
+++ b/lib_pypy/pypy_test/test_datetime.py
@@ -32,4 +32,28 @@
assert datetime.datetime.utcfromtimestamp(a).microsecond == 0
assert datetime.datetime.utcfromtimestamp(a).second == 1
-
+def test_more_datetime_rounding():
+ # this test verified on top of CPython 2.7 (using a plain
+ # "import datetime" above)
+ expected_results = {
+ -1000.0: 'datetime.datetime(1970, 1, 1, 0, 43, 20)',
+ -999.9999996: 'datetime.datetime(1970, 1, 1, 0, 43, 20)',
+ -999.4: 'datetime.datetime(1970, 1, 1, 0, 43, 20, 600000)',
+ -999.0000004: 'datetime.datetime(1970, 1, 1, 0, 43, 21)',
+ -1.0: 'datetime.datetime(1970, 1, 1, 0, 59, 59)',
+ -0.9999996: 'datetime.datetime(1970, 1, 1, 0, 59, 59)',
+ -0.4: 'datetime.datetime(1970, 1, 1, 0, 59, 59, 600000)',
+ -0.0000004: 'datetime.datetime(1970, 1, 1, 1, 0)',
+ 0.0: 'datetime.datetime(1970, 1, 1, 1, 0)',
+ 0.0000004: 'datetime.datetime(1970, 1, 1, 1, 0)',
+ 0.4: 'datetime.datetime(1970, 1, 1, 1, 0, 0, 400000)',
+ 0.9999996: 'datetime.datetime(1970, 1, 1, 1, 0, 1)',
+ 1000.0: 'datetime.datetime(1970, 1, 1, 1, 16, 40)',
+ 1000.0000004: 'datetime.datetime(1970, 1, 1, 1, 16, 40)',
+ 1000.4: 'datetime.datetime(1970, 1, 1, 1, 16, 40, 400000)',
+ 1000.9999996: 'datetime.datetime(1970, 1, 1, 1, 16, 41)',
+ 1293843661.191: 'datetime.datetime(2011, 1, 1, 2, 1, 1, 191000)',
+ }
+ for t in sorted(expected_results):
+ dt = datetime.datetime.fromtimestamp(t)
+ assert repr(dt) == expected_results[t]
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
from ctypes_support import standard_c_lib as libc
from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof
+from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
from errno import EINVAL, EPERM
import _structseq
@@ -25,6 +25,8 @@
_setrlimit = libc.setrlimit
try:
_getpagesize = libc.getpagesize
+ _getpagesize.argtypes = ()
+ _getpagesize.restype = c_int
except AttributeError:
from os import sysconf
_getpagesize = None
@@ -61,6 +63,10 @@
("ru_nivcsw", c_long),
)
+_getrusage.argtypes = (c_int, POINTER(_struct_rusage))
+_getrusage.restype = c_int
+
+
class struct_rusage:
__metaclass__ = _structseq.structseqtype
@@ -94,6 +100,12 @@
("rlim_max", rlim_t),
)
+_getrlimit.argtypes = (c_int, POINTER(rlimit))
+_getrlimit.restype = c_int
+_setrlimit.argtypes = (c_int, POINTER(rlimit))
+_setrlimit.restype = c_int
+
+
@builtinify
def getrusage(who):
ru = _struct_rusage()
diff --git a/py/__init__.py b/py/__init__.py
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -8,7 +8,7 @@
(c) Holger Krekel and others, 2004-2010
"""
-__version__ = '1.4.3'
+__version__ = '1.4.4.dev1'
from py import _apipkg
@@ -70,10 +70,6 @@
'getrawcode' : '._code.code:getrawcode',
'patch_builtins' : '._code.code:patch_builtins',
'unpatch_builtins' : '._code.code:unpatch_builtins',
- '_AssertionError' : '._code.assertion:AssertionError',
- '_reinterpret_old' : '._code.assertion:reinterpret_old',
- '_reinterpret' : '._code.assertion:reinterpret',
- '_reprcompare' : '._code.assertion:_reprcompare',
},
# backports and additions of builtins
diff --git a/py/_code/_assertionnew.py b/py/_code/_assertionnew.py
deleted file mode 100644
--- a/py/_code/_assertionnew.py
+++ /dev/null
@@ -1,339 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace _assertionold.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from py._code.assertion import _format_explanation, BuiltinAssertionError
-
-
-if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
- # See http://bugs.jython.org/issue1497
- _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
- "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
- "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
- "List", "Tuple")
- _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
- "AugAssign", "Print", "For", "While", "If", "With", "Raise",
- "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
- "Exec", "Global", "Expr", "Pass", "Break", "Continue")
- _expr_nodes = set(getattr(ast, name) for name in _exprs)
- _stmt_nodes = set(getattr(ast, name) for name in _stmts)
- def _is_ast_expr(node):
- return node.__class__ in _expr_nodes
- def _is_ast_stmt(node):
- return node.__class__ in _stmt_nodes
-else:
- def _is_ast_expr(node):
- return isinstance(node, ast.expr)
- def _is_ast_stmt(node):
- return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
- """Error found while interpreting AST."""
-
- def __init__(self, explanation=""):
- self.cause = sys.exc_info()
- self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
- mod = ast.parse(source)
- visitor = DebugInterpreter(frame)
- try:
- visitor.visit(mod)
- except Failure:
- failure = sys.exc_info()[1]
- return getfailure(failure)
- if should_fail:
- return ("(assertion failed, but when it was re-run for "
- "printing intermediate values, it did not fail. Suggestions: "
- "compute assert expression before the assert or use --no-assert)")
-
-def run(offending_line, frame=None):
- if frame is None:
- frame = py.code.Frame(sys._getframe(1))
- return interpret(offending_line, frame)
-
-def getfailure(failure):
- explanation = _format_explanation(failure.explanation)
- value = failure.cause[1]
- if str(value):
- lines = explanation.splitlines()
- if not lines:
- lines.append("")
- lines[0] += " << %s" % (value,)
- explanation = "\n".join(lines)
- text = "%s: %s" % (failure.cause[0].__name__, explanation)
- if text.startswith("AssertionError: assert "):
- text = text[16:]
- return text
-
-
-operator_map = {
- ast.BitOr : "|",
- ast.BitXor : "^",
- ast.BitAnd : "&",
- ast.LShift : "<<",
- ast.RShift : ">>",
- ast.Add : "+",
- ast.Sub : "-",
- ast.Mult : "*",
- ast.Div : "/",
- ast.FloorDiv : "//",
- ast.Mod : "%",
- ast.Eq : "==",
- ast.NotEq : "!=",
- ast.Lt : "<",
- ast.LtE : "<=",
- ast.Gt : ">",
- ast.GtE : ">=",
- ast.Pow : "**",
- ast.Is : "is",
- ast.IsNot : "is not",
- ast.In : "in",
- ast.NotIn : "not in"
-}
-
-unary_map = {
- ast.Not : "not %s",
- ast.Invert : "~%s",
- ast.USub : "-%s",
- ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
- """Interpret AST nodes to gleam useful debugging information. """
-
- def __init__(self, frame):
- self.frame = frame
-
- def generic_visit(self, node):
- # Fallback when we don't have a special implementation.
- if _is_ast_expr(node):
- mod = ast.Expression(node)
- co = self._compile(mod)
- try:
- result = self.frame.eval(co)
- except Exception:
- raise Failure()
- explanation = self.frame.repr(result)
- return explanation, result
- elif _is_ast_stmt(node):
- mod = ast.Module([node])
- co = self._compile(mod, "exec")
- try:
- self.frame.exec_(co)
- except Exception:
- raise Failure()
- return None, None
- else:
- raise AssertionError("can't handle %s" %(node,))
-
- def _compile(self, source, mode="eval"):
- return compile(source, "<assertion interpretation>", mode)
-
- def visit_Expr(self, expr):
- return self.visit(expr.value)
-
- def visit_Module(self, mod):
- for stmt in mod.body:
- self.visit(stmt)
-
- def visit_Name(self, name):
- explanation, result = self.generic_visit(name)
- # See if the name is local.
- source = "%r in locals() is not globals()" % (name.id,)
- co = self._compile(source)
- try:
- local = self.frame.eval(co)
- except Exception:
- # have to assume it isn't
- local = False
- if not local:
- return name.id, result
- return explanation, result
-
- def visit_Compare(self, comp):
- left = comp.left
- left_explanation, left_result = self.visit(left)
- for op, next_op in zip(comp.ops, comp.comparators):
- next_explanation, next_result = self.visit(next_op)
- op_symbol = operator_map[op.__class__]
- explanation = "%s %s %s" % (left_explanation, op_symbol,
- next_explanation)
- source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
- co = self._compile(source)
- try:
- result = self.frame.eval(co, __exprinfo_left=left_result,
- __exprinfo_right=next_result)
- except Exception:
- raise Failure(explanation)
- try:
- if not result:
- break
- except KeyboardInterrupt:
- raise
- except:
- break
- left_explanation, left_result = next_explanation, next_result
-
- rcomp = py.code._reprcompare
- if rcomp:
- res = rcomp(op_symbol, left_result, next_result)
- if res:
- explanation = res
- return explanation, result
-
- def visit_BoolOp(self, boolop):
- is_or = isinstance(boolop.op, ast.Or)
- explanations = []
- for operand in boolop.values:
- explanation, result = self.visit(operand)
- explanations.append(explanation)
- if result == is_or:
- break
- name = is_or and " or " or " and "
- explanation = "(" + name.join(explanations) + ")"
- return explanation, result
-
- def visit_UnaryOp(self, unary):
- pattern = unary_map[unary.op.__class__]
- operand_explanation, operand_result = self.visit(unary.operand)
- explanation = pattern % (operand_explanation,)
- co = self._compile(pattern % ("__exprinfo_expr",))
- try:
- result = self.frame.eval(co, __exprinfo_expr=operand_result)
- except Exception:
- raise Failure(explanation)
- return explanation, result
-
- def visit_BinOp(self, binop):
- left_explanation, left_result = self.visit(binop.left)
- right_explanation, right_result = self.visit(binop.right)
- symbol = operator_map[binop.op.__class__]
- explanation = "(%s %s %s)" % (left_explanation, symbol,
- right_explanation)
- source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
- co = self._compile(source)
- try:
- result = self.frame.eval(co, __exprinfo_left=left_result,
- __exprinfo_right=right_result)
- except Exception:
- raise Failure(explanation)
- return explanation, result
-
- def visit_Call(self, call):
- func_explanation, func = self.visit(call.func)
- arg_explanations = []
- ns = {"__exprinfo_func" : func}
- arguments = []
- for arg in call.args:
- arg_explanation, arg_result = self.visit(arg)
- arg_name = "__exprinfo_%s" % (len(ns),)
- ns[arg_name] = arg_result
- arguments.append(arg_name)
- arg_explanations.append(arg_explanation)
- for keyword in call.keywords:
- arg_explanation, arg_result = self.visit(keyword.value)
- arg_name = "__exprinfo_%s" % (len(ns),)
- ns[arg_name] = arg_result
- keyword_source = "%s=%%s" % (keyword.arg)
- arguments.append(keyword_source % (arg_name,))
- arg_explanations.append(keyword_source % (arg_explanation,))
- if call.starargs:
- arg_explanation, arg_result = self.visit(call.starargs)
- arg_name = "__exprinfo_star"
- ns[arg_name] = arg_result
- arguments.append("*%s" % (arg_name,))
- arg_explanations.append("*%s" % (arg_explanation,))
- if call.kwargs:
- arg_explanation, arg_result = self.visit(call.kwargs)
- arg_name = "__exprinfo_kwds"
- ns[arg_name] = arg_result
- arguments.append("**%s" % (arg_name,))
- arg_explanations.append("**%s" % (arg_explanation,))
- args_explained = ", ".join(arg_explanations)
- explanation = "%s(%s)" % (func_explanation, args_explained)
- args = ", ".join(arguments)
- source = "__exprinfo_func(%s)" % (args,)
- co = self._compile(source)
- try:
- result = self.frame.eval(co, **ns)
- except Exception:
- raise Failure(explanation)
- pattern = "%s\n{%s = %s\n}"
- rep = self.frame.repr(result)
- explanation = pattern % (rep, rep, explanation)
- return explanation, result
-
- def _is_builtin_name(self, name):
- pattern = "%r not in globals() and %r not in locals()"
- source = pattern % (name.id, name.id)
- co = self._compile(source)
- try:
- return self.frame.eval(co)
- except Exception:
- return False
-
- def visit_Attribute(self, attr):
- if not isinstance(attr.ctx, ast.Load):
- return self.generic_visit(attr)
- source_explanation, source_result = self.visit(attr.value)
- explanation = "%s.%s" % (source_explanation, attr.attr)
- source = "__exprinfo_expr.%s" % (attr.attr,)
- co = self._compile(source)
- try:
- result = self.frame.eval(co, __exprinfo_expr=source_result)
- except Exception:
- raise Failure(explanation)
- explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
- self.frame.repr(result),
- source_explanation, attr.attr)
- # Check if the attr is from an instance.
- source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
- source = source % (attr.attr,)
- co = self._compile(source)
- try:
- from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
- except Exception:
- from_instance = True
- if from_instance:
- rep = self.frame.repr(result)
- pattern = "%s\n{%s = %s\n}"
- explanation = pattern % (rep, rep, explanation)
- return explanation, result
-
- def visit_Assert(self, assrt):
- test_explanation, test_result = self.visit(assrt.test)
- if test_explanation.startswith("False\n{False =") and \
- test_explanation.endswith("\n"):
- test_explanation = test_explanation[15:-2]
- explanation = "assert %s" % (test_explanation,)
- if not test_result:
- try:
- raise BuiltinAssertionError
- except Exception:
- raise Failure(explanation)
- return explanation, test_result
-
- def visit_Assign(self, assign):
- value_explanation, value_result = self.visit(assign.value)
- explanation = "... = %s" % (value_explanation,)
- name = ast.Name("__exprinfo_expr", ast.Load(),
- lineno=assign.value.lineno,
- col_offset=assign.value.col_offset)
- new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
- col_offset=assign.col_offset)
- mod = ast.Module([new_assign])
- co = self._compile(mod, "exec")
- try:
- self.frame.exec_(co, __exprinfo_expr=value_result)
- except Exception:
- raise Failure(explanation)
- return explanation, value_result
diff --git a/py/_code/_assertionold.py b/py/_code/_assertionold.py
deleted file mode 100644
--- a/py/_code/_assertionold.py
+++ /dev/null
@@ -1,555 +0,0 @@
-import py
-import sys, inspect
-from compiler import parse, ast, pycodegen
-from py._code.assertion import BuiltinAssertionError, _format_explanation
-
-passthroughex = py.builtin._sysex
-
-class Failure:
- def __init__(self, node):
- self.exc, self.value, self.tb = sys.exc_info()
- self.node = node
-
-class View(object):
- """View base class.
-
- If C is a subclass of View, then C(x) creates a proxy object around
- the object x. The actual class of the proxy is not C in general,
- but a *subclass* of C determined by the rules below. To avoid confusion
- we call view class the class of the proxy (a subclass of C, so of View)
- and object class the class of x.
-
- Attributes and methods not found in the proxy are automatically read on x.
- Other operations like setting attributes are performed on the proxy, as
- determined by its view class. The object x is available from the proxy
- as its __obj__ attribute.
-
- The view class selection is determined by the __view__ tuples and the
- optional __viewkey__ method. By default, the selected view class is the
- most specific subclass of C whose __view__ mentions the class of x.
- If no such subclass is found, the search proceeds with the parent
- object classes. For example, C(True) will first look for a subclass
- of C with __view__ = (..., bool, ...) and only if it doesn't find any
- look for one with __view__ = (..., int, ...), and then ..., object,...
- If everything fails the class C itself is considered to be the default.
-
- Alternatively, the view class selection can be driven by another aspect
- of the object x, instead of the class of x, by overriding __viewkey__.
- See last example at the end of this module.
- """
-
- _viewcache = {}
- __view__ = ()
-
- def __new__(rootclass, obj, *args, **kwds):
- self = object.__new__(rootclass)
- self.__obj__ = obj
- self.__rootclass__ = rootclass
- key = self.__viewkey__()
- try:
- self.__class__ = self._viewcache[key]
- except KeyError:
- self.__class__ = self._selectsubclass(key)
- return self
-
- def __getattr__(self, attr):
- # attributes not found in the normal hierarchy rooted on View
- # are looked up in the object's real class
- return getattr(self.__obj__, attr)
-
- def __viewkey__(self):
- return self.__obj__.__class__
-
- def __matchkey__(self, key, subclasses):
- if inspect.isclass(key):
- keys = inspect.getmro(key)
- else:
- keys = [key]
- for key in keys:
- result = [C for C in subclasses if key in C.__view__]
- if result:
- return result
- return []
-
- def _selectsubclass(self, key):
- subclasses = list(enumsubclasses(self.__rootclass__))
- for C in subclasses:
- if not isinstance(C.__view__, tuple):
- C.__view__ = (C.__view__,)
- choices = self.__matchkey__(key, subclasses)
- if not choices:
- return self.__rootclass__
- elif len(choices) == 1:
- return choices[0]
- else:
- # combine the multiple choices
- return type('?', tuple(choices), {})
-
- def __repr__(self):
- return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
-
-
-def enumsubclasses(cls):
- for subcls in cls.__subclasses__():
- for subsubclass in enumsubclasses(subcls):
- yield subsubclass
- yield cls
-
-
-class Interpretable(View):
- """A parse tree node with a few extra methods."""
- explanation = None
-
- def is_builtin(self, frame):
- return False
-
- def eval(self, frame):
- # fall-back for unknown expression nodes
- try:
- expr = ast.Expression(self.__obj__)
- expr.filename = '<eval>'
- self.__obj__.filename = '<eval>'
- co = pycodegen.ExpressionCodeGenerator(expr).getCode()
- result = frame.eval(co)
- except passthroughex:
- raise
- except:
- raise Failure(self)
- self.result = result
- self.explanation = self.explanation or frame.repr(self.result)
-
- def run(self, frame):
- # fall-back for unknown statement nodes
- try:
- expr = ast.Module(None, ast.Stmt([self.__obj__]))
- expr.filename = '<run>'
- co = pycodegen.ModuleCodeGenerator(expr).getCode()
- frame.exec_(co)
- except passthroughex:
- raise
- except:
- raise Failure(self)
-
- def nice_explanation(self):
- return _format_explanation(self.explanation)
-
-
-class Name(Interpretable):
- __view__ = ast.Name
-
- def is_local(self, frame):
- source = '%r in locals() is not globals()' % self.name
- try:
- return frame.is_true(frame.eval(source))
- except passthroughex:
- raise
- except:
- return False
-
- def is_global(self, frame):
- source = '%r in globals()' % self.name
- try:
- return frame.is_true(frame.eval(source))
- except passthroughex:
- raise
- except:
- return False
-
- def is_builtin(self, frame):
- source = '%r not in locals() and %r not in globals()' % (
- self.name, self.name)
- try:
- return frame.is_true(frame.eval(source))
- except passthroughex:
- raise
- except:
- return False
-
- def eval(self, frame):
- super(Name, self).eval(frame)
- if not self.is_local(frame):
- self.explanation = self.name
-
-class Compare(Interpretable):
- __view__ = ast.Compare
-
- def eval(self, frame):
- expr = Interpretable(self.expr)
- expr.eval(frame)
- for operation, expr2 in self.ops:
- if hasattr(self, 'result'):
- # shortcutting in chained expressions
- if not frame.is_true(self.result):
- break
- expr2 = Interpretable(expr2)
- expr2.eval(frame)
- self.explanation = "%s %s %s" % (
- expr.explanation, operation, expr2.explanation)
- source = "__exprinfo_left %s __exprinfo_right" % operation
- try:
- self.result = frame.eval(source,
- __exprinfo_left=expr.result,
- __exprinfo_right=expr2.result)
- except passthroughex:
- raise
- except:
- raise Failure(self)
- expr = expr2
-
-class And(Interpretable):
- __view__ = ast.And
-
- def eval(self, frame):
- explanations = []
- for expr in self.nodes:
- expr = Interpretable(expr)
- expr.eval(frame)
- explanations.append(expr.explanation)
- self.result = expr.result
- if not frame.is_true(expr.result):
- break
- self.explanation = '(' + ' and '.join(explanations) + ')'
-
-class Or(Interpretable):
- __view__ = ast.Or
-
- def eval(self, frame):
- explanations = []
- for expr in self.nodes:
- expr = Interpretable(expr)
- expr.eval(frame)
- explanations.append(expr.explanation)
- self.result = expr.result
- if frame.is_true(expr.result):
- break
- self.explanation = '(' + ' or '.join(explanations) + ')'
-
-
-# == Unary operations ==
-keepalive = []
-for astclass, astpattern in {
- ast.Not : 'not __exprinfo_expr',
- ast.Invert : '(~__exprinfo_expr)',
- }.items():
-
- class UnaryArith(Interpretable):
- __view__ = astclass
-
- def eval(self, frame, astpattern=astpattern):
- expr = Interpretable(self.expr)
- expr.eval(frame)
- self.explanation = astpattern.replace('__exprinfo_expr',
- expr.explanation)
- try:
- self.result = frame.eval(astpattern,
- __exprinfo_expr=expr.result)
- except passthroughex:
- raise
- except:
- raise Failure(self)
-
- keepalive.append(UnaryArith)
-
-# == Binary operations ==
-for astclass, astpattern in {
- ast.Add : '(__exprinfo_left + __exprinfo_right)',
- ast.Sub : '(__exprinfo_left - __exprinfo_right)',
- ast.Mul : '(__exprinfo_left * __exprinfo_right)',
- ast.Div : '(__exprinfo_left / __exprinfo_right)',
- ast.Mod : '(__exprinfo_left % __exprinfo_right)',
- ast.Power : '(__exprinfo_left ** __exprinfo_right)',
- }.items():
-
- class BinaryArith(Interpretable):
- __view__ = astclass
-
- def eval(self, frame, astpattern=astpattern):
- left = Interpretable(self.left)
- left.eval(frame)
- right = Interpretable(self.right)
- right.eval(frame)
- self.explanation = (astpattern
- .replace('__exprinfo_left', left .explanation)
- .replace('__exprinfo_right', right.explanation))
- try:
- self.result = frame.eval(astpattern,
- __exprinfo_left=left.result,
- __exprinfo_right=right.result)
- except passthroughex:
- raise
- except:
- raise Failure(self)
-
- keepalive.append(BinaryArith)
-
-
-class CallFunc(Interpretable):
- __view__ = ast.CallFunc
-
- def is_bool(self, frame):
- source = 'isinstance(__exprinfo_value, bool)'
- try:
- return frame.is_true(frame.eval(source,
- __exprinfo_value=self.result))
- except passthroughex:
- raise
- except:
- return False
-
- def eval(self, frame):
- node = Interpretable(self.node)
- node.eval(frame)
- explanations = []
- vars = {'__exprinfo_fn': node.result}
- source = '__exprinfo_fn('
- for a in self.args:
- if isinstance(a, ast.Keyword):
- keyword = a.name
- a = a.expr
- else:
- keyword = None
- a = Interpretable(a)
- a.eval(frame)
- argname = '__exprinfo_%d' % len(vars)
- vars[argname] = a.result
- if keyword is None:
- source += argname + ','
- explanations.append(a.explanation)
- else:
- source += '%s=%s,' % (keyword, argname)
- explanations.append('%s=%s' % (keyword, a.explanation))
- if self.star_args:
- star_args = Interpretable(self.star_args)
- star_args.eval(frame)
- argname = '__exprinfo_star'
- vars[argname] = star_args.result
- source += '*' + argname + ','
- explanations.append('*' + star_args.explanation)
- if self.dstar_args:
- dstar_args = Interpretable(self.dstar_args)
- dstar_args.eval(frame)
- argname = '__exprinfo_kwds'
- vars[argname] = dstar_args.result
- source += '**' + argname + ','
- explanations.append('**' + dstar_args.explanation)
- self.explanation = "%s(%s)" % (
- node.explanation, ', '.join(explanations))
- if source.endswith(','):
- source = source[:-1]
- source += ')'
- try:
- self.result = frame.eval(source, **vars)
- except passthroughex:
- raise
- except:
- raise Failure(self)
- if not node.is_builtin(frame) or not self.is_bool(frame):
- r = frame.repr(self.result)
- self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
-
-class Getattr(Interpretable):
- __view__ = ast.Getattr
-
- def eval(self, frame):
- expr = Interpretable(self.expr)
- expr.eval(frame)
- source = '__exprinfo_expr.%s' % self.attrname
- try:
- self.result = frame.eval(source, __exprinfo_expr=expr.result)
- except passthroughex:
- raise
- except:
- raise Failure(self)
- self.explanation = '%s.%s' % (expr.explanation, self.attrname)
- # if the attribute comes from the instance, its value is interesting
- source = ('hasattr(__exprinfo_expr, "__dict__") and '
- '%r in __exprinfo_expr.__dict__' % self.attrname)
- try:
- from_instance = frame.is_true(
- frame.eval(source, __exprinfo_expr=expr.result))
- except passthroughex:
- raise
- except:
- from_instance = True
- if from_instance:
- r = frame.repr(self.result)
- self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
-
-# == Re-interpretation of full statements ==
-
-class Assert(Interpretable):
- __view__ = ast.Assert
-
- def run(self, frame):
- test = Interpretable(self.test)
- test.eval(frame)
- # simplify 'assert False where False = ...'
- if (test.explanation.startswith('False\n{False = ') and
- test.explanation.endswith('\n}')):
- test.explanation = test.explanation[15:-2]
- # print the result as 'assert <explanation>'
- self.result = test.result
- self.explanation = 'assert ' + test.explanation
- if not frame.is_true(test.result):
- try:
- raise BuiltinAssertionError
- except passthroughex:
- raise
- except:
- raise Failure(self)
-
-class Assign(Interpretable):
- __view__ = ast.Assign
-
- def run(self, frame):
- expr = Interpretable(self.expr)
- expr.eval(frame)
- self.result = expr.result
- self.explanation = '... = ' + expr.explanation
- # fall-back-run the rest of the assignment
- ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
- mod = ast.Module(None, ast.Stmt([ass]))
- mod.filename = '<run>'
- co = pycodegen.ModuleCodeGenerator(mod).getCode()
- try:
- frame.exec_(co, __exprinfo_expr=expr.result)
- except passthroughex:
- raise
- except:
- raise Failure(self)
-
-class Discard(Interpretable):
- __view__ = ast.Discard
-
- def run(self, frame):
- expr = Interpretable(self.expr)
- expr.eval(frame)
- self.result = expr.result
- self.explanation = expr.explanation
-
-class Stmt(Interpretable):
- __view__ = ast.Stmt
-
- def run(self, frame):
- for stmt in self.nodes:
- stmt = Interpretable(stmt)
- stmt.run(frame)
-
-
-def report_failure(e):
- explanation = e.node.nice_explanation()
- if explanation:
- explanation = ", in: " + explanation
- else:
- explanation = ""
- sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
-
-def check(s, frame=None):
- if frame is None:
- frame = sys._getframe(1)
- frame = py.code.Frame(frame)
- expr = parse(s, 'eval')
- assert isinstance(expr, ast.Expression)
- node = Interpretable(expr.node)
- try:
- node.eval(frame)
- except passthroughex:
- raise
- except Failure:
- e = sys.exc_info()[1]
- report_failure(e)
- else:
- if not frame.is_true(node.result):
- sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
-
-
-###########################################################
-# API / Entry points
-# #########################################################
-
-def interpret(source, frame, should_fail=False):
- module = Interpretable(parse(source, 'exec').node)
- #print "got module", module
- if isinstance(frame, py.std.types.FrameType):
- frame = py.code.Frame(frame)
- try:
- module.run(frame)
- except Failure:
- e = sys.exc_info()[1]
- return getfailure(e)
- except passthroughex:
- raise
- except:
- import traceback
- traceback.print_exc()
- if should_fail:
- return ("(assertion failed, but when it was re-run for "
- "printing intermediate values, it did not fail. Suggestions: "
- "compute assert expression before the assert or use --nomagic)")
- else:
- return None
-
-def getmsg(excinfo):
- if isinstance(excinfo, tuple):
- excinfo = py.code.ExceptionInfo(excinfo)
- #frame, line = gettbline(tb)
- #frame = py.code.Frame(frame)
- #return interpret(line, frame)
-
- tb = excinfo.traceback[-1]
- source = str(tb.statement).strip()
- x = interpret(source, tb.frame, should_fail=True)
- if not isinstance(x, str):
- raise TypeError("interpret returned non-string %r" % (x,))
- return x
-
-def getfailure(e):
- explanation = e.node.nice_explanation()
- if str(e.value):
- lines = explanation.split('\n')
- lines[0] += " << %s" % (e.value,)
- explanation = '\n'.join(lines)
- text = "%s: %s" % (e.exc.__name__, explanation)
- if text.startswith('AssertionError: assert '):
- text = text[16:]
- return text
-
-def run(s, frame=None):
- if frame is None:
- frame = sys._getframe(1)
- frame = py.code.Frame(frame)
- module = Interpretable(parse(s, 'exec').node)
- try:
- module.run(frame)
- except Failure:
- e = sys.exc_info()[1]
- report_failure(e)
-
-
-if __name__ == '__main__':
- # example:
- def f():
- return 5
- def g():
- return 3
- def h(x):
- return 'never'
- check("f() * g() == 5")
- check("not f()")
- check("not (f() and g() or 0)")
- check("f() == g()")
- i = 4
- check("i == f()")
- check("len(f()) == 0")
- check("isinstance(2+3+4, float)")
-
- run("x = i")
- check("x == 5")
-
- run("assert not f(), 'oops'")
- run("a, b, c = 1, 2")
- run("a, b, c = f()")
-
- check("max([f(),g()]) == 4")
- check("'hello'[g()] == 'h'")
- run("'guk%d' % h(f())")
diff --git a/py/_code/assertion.py b/py/_code/assertion.py
deleted file mode 100644
--- a/py/_code/assertion.py
+++ /dev/null
@@ -1,94 +0,0 @@
-import sys
-import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
-
-_reprcompare = None # if set, will be called by assert reinterp for comparison ops
-
-def _format_explanation(explanation):
- """This formats an explanation
-
- Normally all embedded newlines are escaped, however there are
- three exceptions: \n{, \n} and \n~. The first two are intended
- cover nested explanations, see function and attribute explanations
- for examples (.visit_Call(), visit_Attribute()). The last one is
- for when one explanation needs to span multiple lines, e.g. when
- displaying diffs.
- """
- raw_lines = (explanation or '').split('\n')
- # escape newlines not followed by {, } and ~
- lines = [raw_lines[0]]
- for l in raw_lines[1:]:
- if l.startswith('{') or l.startswith('}') or l.startswith('~'):
- lines.append(l)
- else:
- lines[-1] += '\\n' + l
-
- result = lines[:1]
- stack = [0]
- stackcnt = [0]
- for line in lines[1:]:
- if line.startswith('{'):
- if stackcnt[-1]:
- s = 'and '
- else:
- s = 'where '
- stack.append(len(result))
- stackcnt[-1] += 1
- stackcnt.append(0)
- result.append(' +' + ' '*(len(stack)-1) + s + line[1:])
- elif line.startswith('}'):
- assert line.startswith('}')
- stack.pop()
- stackcnt.pop()
- result[stack[-1]] += line[1:]
- else:
- assert line.startswith('~')
- result.append(' '*len(stack) + line[1:])
- assert len(stack) == 1
- return '\n'.join(result)
-
-
-class AssertionError(BuiltinAssertionError):
- def __init__(self, *args):
- BuiltinAssertionError.__init__(self, *args)
- if args:
- try:
- self.msg = str(args[0])
- except py.builtin._sysex:
- raise
- except:
- self.msg = "<[broken __repr__] %s at %0xd>" %(
- args[0].__class__, id(args[0]))
- else:
- f = py.code.Frame(sys._getframe(1))
- try:
- source = f.code.fullsource
- if source is not None:
- try:
- source = source.getstatement(f.lineno, assertion=True)
- except IndexError:
- source = None
- else:
- source = str(source.deindent()).strip()
- except py.error.ENOENT:
- source = None
- # this can also occur during reinterpretation, when the
- # co_filename is set to "<run>".
- if source:
- self.msg = reinterpret(source, f, should_fail=True)
- else:
- self.msg = "<could not determine information>"
- if not self.args:
- self.args = (self.msg,)
-
-if sys.version_info > (3, 0):
- AssertionError.__module__ = "builtins"
- reinterpret_old = "old reinterpretation not available for py3"
-else:
- from py._code._assertionold import interpret as reinterpret_old
-if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
- from py._code._assertionnew import interpret as reinterpret
-else:
- reinterpret = reinterpret_old
-
diff --git a/py/_code/code.py b/py/_code/code.py
--- a/py/_code/code.py
+++ b/py/_code/code.py
@@ -145,17 +145,6 @@
return self.frame.f_locals
locals = property(getlocals, None, None, "locals of underlaying frame")
- def reinterpret(self):
- """Reinterpret the failing statement and returns a detailed information
- about what operations are performed."""
- if self.exprinfo is None:
- source = str(self.statement).strip()
- x = py.code._reinterpret(source, self.frame, should_fail=True)
- if not isinstance(x, str):
- raise TypeError("interpret returned non-string %r" % (x,))
- self.exprinfo = x
- return self.exprinfo
-
def getfirstlinesource(self):
# on Jython this firstlineno can be -1 apparently
return max(self.frame.code.firstlineno, 0)
@@ -310,7 +299,7 @@
# ExceptionInfo-like classes may have different attributes.
if tup is None:
tup = sys.exc_info()
- if exprinfo is None and isinstance(tup[1], py.code._AssertionError):
+ if exprinfo is None and isinstance(tup[1], AssertionError):
exprinfo = getattr(tup[1], 'msg', None)
if exprinfo is None:
exprinfo = str(tup[1])
@@ -690,22 +679,15 @@
oldbuiltins = {}
-def patch_builtins(assertion=True, compile=True):
- """ put compile and AssertionError builtins to Python's builtins. """
- if assertion:
- from py._code import assertion
- l = oldbuiltins.setdefault('AssertionError', [])
- l.append(py.builtin.builtins.AssertionError)
- py.builtin.builtins.AssertionError = assertion.AssertionError
+def patch_builtins(compile=True):
+ """ put compile builtins to Python's builtins. """
if compile:
l = oldbuiltins.setdefault('compile', [])
l.append(py.builtin.builtins.compile)
py.builtin.builtins.compile = py.code.compile
-def unpatch_builtins(assertion=True, compile=True):
+def unpatch_builtins(compile=True):
""" remove compile and AssertionError builtins from Python builtins. """
- if assertion:
- py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
if compile:
py.builtin.builtins.compile = oldbuiltins['compile'].pop()
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -84,6 +84,7 @@
"_rawffi": [("objspace.usemodules.struct", True)],
"cpyext": [("translation.secondaryentrypoints", "cpyext"),
("translation.shared", sys.platform == "win32")],
+ "_ffi": [("translation.jit_ffi", True)],
}
module_import_dependencies = {
diff --git a/pypy/config/test/test_pypyoption.py b/pypy/config/test/test_pypyoption.py
--- a/pypy/config/test/test_pypyoption.py
+++ b/pypy/config/test/test_pypyoption.py
@@ -73,3 +73,7 @@
fn = prefix + "." + path + ".txt"
yield check_file_exists, fn
+def test__ffi_opt():
+ config = get_pypy_config(translating=True)
+ config.objspace.usemodules._ffi = True
+ assert config.translation.jit_ffi
diff --git a/pypy/config/translationoption.py b/pypy/config/translationoption.py
--- a/pypy/config/translationoption.py
+++ b/pypy/config/translationoption.py
@@ -117,6 +117,8 @@
ChoiceOption("jit_profiler", "integrate profiler support into the JIT",
["off", "oprofile"],
default="off"),
+ # jit_ffi is automatically turned on by withmod-_ffi (which is enabled by default)
+ BoolOption("jit_ffi", "optimize libffi calls", default=False, cmdline=None),
# misc
BoolOption("verbose", "Print extra information", default=False),
diff --git a/pypy/doc/garbage_collection.rst b/pypy/doc/garbage_collection.rst
--- a/pypy/doc/garbage_collection.rst
+++ b/pypy/doc/garbage_collection.rst
@@ -212,90 +212,4 @@
becomes free garbage, to be collected at the next major collection.
-Minimark GC
------------
-
-This is a simplification and rewrite of the ideas from the Hybrid GC.
-It uses a nursery for the young objects, and mark-and-sweep for the old
-objects. This is a moving GC, but objects may only move once (from
-the nursery to the old stage).
-
-The main difference with the Hybrid GC is that the mark-and-sweep
-objects (the "old stage") are directly handled by the GC's custom
-allocator, instead of being handled by malloc() calls. The gain is that
-it is then possible, during a major collection, to walk through all old
-generation objects without needing to store a list of pointers to them.
-So as a first approximation, when compared to the Hybrid GC, the
-Minimark GC saves one word of memory per old object.
-
-There are a number of environment variables that can be tweaked to
-influence the GC. (Their default value should be ok for most usages.)
-You can read more about them at the start of
-`pypy/rpython/memory/gc/minimark.py`_.
-
-In more details:
-
-- The small newly malloced objects are allocated in the nursery (case 1).
- All objects living in the nursery are "young".
-
-- The big objects are always handled directly by the system malloc().
- But the big newly malloced objects are still "young" when they are
- allocated (case 2), even though they don't live in the nursery.
-
-- When the nursery is full, we do a minor collection, i.e. we find
- which "young" objects are still alive (from cases 1 and 2). The
- "young" flag is then removed. The surviving case 1 objects are moved
- to the old stage. The dying case 2 objects are immediately freed.
-
-- The old stage is an area of memory containing old (small) objects. It
- is handled by `pypy/rpython/memory/gc/minimarkpage.py`_. It is organized
- as "arenas" of 256KB or 512KB, subdivided into "pages" of 4KB or 8KB.
- Each page can either be free, or contain small objects of all the same
- size. Furthermore at any point in time each object location can be
- either allocated or freed. The basic design comes from ``obmalloc.c``
- from CPython (which itself comes from the same source as the Linux
- system malloc()).
-
-- New objects are added to the old stage at every minor collection.
- Immediately after a minor collection, when we reach some threshold, we
- trigger a major collection. This is the mark-and-sweep step. It walks
- over *all* objects (mark), and then frees some fraction of them (sweep).
- This means that the only time when we want to free objects is while
- walking over all of them; we never ask to free an object given just its
- address. This allows some simplifications and memory savings when
- compared to ``obmalloc.c``.
-
-- As with all generational collectors, this GC needs a write barrier to
- record which old objects have a reference to young objects.
-
-- Additionally, we found out that it is useful to handle the case of
- big arrays specially: when we allocate a big array (with the system
- malloc()), we reserve a small number of bytes before. When the array
- grows old, we use the extra bytes as a set of bits. Each bit
- represents 128 entries in the array. Whenever the write barrier is
- called to record a reference from the Nth entry of the array to some
- young object, we set the bit number ``(N/128)`` to 1. This can
- considerably speed up minor collections, because we then only have to
- scan 128 entries of the array instead of all of them.
-
-- As usual, we need special care about weak references, and objects with
- finalizers. Weak references are allocated in the nursery, and if they
- survive they move to the old stage, as usual for all objects; the
- difference is that the reference they contain must either follow the
- object, or be set to NULL if the object dies. And the objects with
- finalizers, considered rare enough, are immediately allocated old to
- simplify the design. In particular their ``__del__`` method can only
- be called just after a major collection.
-
-- The objects move once only, so we can use a trick to implement id()
- and hash(). If the object is not in the nursery, it won't move any
- more, so its id() and hash() are the object's address, cast to an
- integer. If the object is in the nursery, and we ask for its id()
- or its hash(), then we pre-reserve a location in the old stage, and
- return the address of that location. If the object survives the
- next minor collection, we move it there, and so its id() and hash()
- are preserved. If the object dies then the pre-reserved location
- becomes free garbage, to be collected at the next major collection.
-
-
.. include:: _ref.txt
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -91,7 +91,7 @@
Remove the GIL
--------------
-This is a major task that requiers lots of thinking. However, few subprojects
+This is a major task that requires lots of thinking. However, few subprojects
can be potentially specified, unless a better plan can be thought out:
* A thread-aware garbage collector
@@ -124,6 +124,25 @@
for our needs. It's possible that this has changed, reviving the LLVM backend
(or writing new from scratch) for static compilation would be a good project.
+(On the other hand, just generating C code and using clang might be enough.
+The issue with that is the so-called "asmgcc GC root finder", which has tons
+of issues of this own. In my opinion (arigo), it would be definitely a
+better project to try to optimize the alternative, the "shadowstack" GC root
+finder, which is nicely portable. So far it gives a pypy that is around
+7% slower.)
+
+Embedding PyPy
+----------------------------------------
+
+Being able to embed PyPy, say with its own limited C API, would be
+useful. But here is the most interesting variant, straight from
+EuroPython live discussion :-) We can have a generic "libpypy.so" that
+can be used as a placeholder dynamic library, and when it gets loaded,
+it runs a .py module that installs (via ctypes) the interface it wants
+exported. This would give us a one-size-fits-all generic .so file to be
+imported by any application that wants to load .so files :-)
+
+
.. _`issue tracker`: http://bugs.pypy.org
.. _`mailing list`: http://mail.python.org/mailman/listinfo/pypy-dev
.. _`jitviewer`: http://bitbucket.org/pypy/jitviewer
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -90,15 +90,18 @@
### Construction ###
def __init__(self, space, args_w, keywords=None, keywords_w=None,
- w_stararg=None, w_starstararg=None):
+ w_stararg=None, w_starstararg=None, keyword_names_w=None):
self.space = space
assert isinstance(args_w, list)
self.arguments_w = args_w
self.keywords = keywords
self.keywords_w = keywords_w
+ self.keyword_names_w = keyword_names_w # matches the tail of .keywords
if keywords is not None:
assert keywords_w is not None
assert len(keywords_w) == len(keywords)
+ assert (keyword_names_w is None or
+ len(keyword_names_w) <= len(keywords))
make_sure_not_resized(self.keywords)
make_sure_not_resized(self.keywords_w)
@@ -132,7 +135,8 @@
def replace_arguments(self, args_w):
"Return a new Arguments with a args_w as positional arguments."
- return Arguments(self.space, args_w, self.keywords, self.keywords_w)
+ return Arguments(self.space, args_w, self.keywords, self.keywords_w,
+ keyword_names_w = self.keyword_names_w)
def prepend(self, w_firstarg):
"Return a new Arguments with a new argument inserted first."
@@ -201,15 +205,16 @@
space.w_TypeError,
space.wrap("keywords must be strings"))
if e.match(space, space.w_UnicodeEncodeError):
- raise OperationError(
- space.w_TypeError,
- space.wrap("keyword cannot be encoded to ascii"))
- raise
- if self.keywords and key in self.keywords:
- raise operationerrfmt(self.space.w_TypeError,
- "got multiple values "
- "for keyword argument "
- "'%s'", key)
+ # Allow this to pass through
+ key = None
+ else:
+ raise
+ else:
+ if self.keywords and key in self.keywords:
+ raise operationerrfmt(self.space.w_TypeError,
+ "got multiple values "
+ "for keyword argument "
+ "'%s'", key)
keywords[i] = key
keywords_w[i] = space.getitem(w_starstararg, w_key)
i += 1
@@ -219,6 +224,7 @@
else:
self.keywords = self.keywords + keywords
self.keywords_w = self.keywords_w + keywords_w
+ self.keyword_names_w = keys_w
def fixedunpack(self, argcount):
"""The simplest argument parsing: get the 'argcount' arguments,
@@ -339,6 +345,10 @@
used_keywords = [False] * num_kwds
for i in range(num_kwds):
name = keywords[i]
+ # If name was not encoded as a string, it could be None. In that
+ # case, it's definitely not going to be in the signature.
+ if name is None:
+ continue
j = signature.find_argname(name)
if j < 0:
continue
@@ -374,17 +384,26 @@
if has_kwarg:
w_kwds = self.space.newdict()
if num_remainingkwds:
+ #
+ limit = len(keywords)
+ if self.keyword_names_w is not None:
+ limit -= len(self.keyword_names_w)
for i in range(len(keywords)):
if not used_keywords[i]:
- key = keywords[i]
- self.space.setitem(w_kwds, self.space.wrap(key), keywords_w[i])
+ if i < limit:
+ w_key = self.space.wrap(keywords[i])
+ else:
+ w_key = self.keyword_names_w[i - limit]
+ self.space.setitem(w_kwds, w_key, keywords_w[i])
+ #
scope_w[co_argcount + has_vararg] = w_kwds
elif num_remainingkwds:
if co_argcount == 0:
raise ArgErrCount(avail, num_kwds,
co_argcount, has_vararg, has_kwarg,
defaults_w, missing)
- raise ArgErrUnknownKwds(num_remainingkwds, keywords, used_keywords)
+ raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
+ used_keywords, self.keyword_names_w)
if missing:
raise ArgErrCount(avail, num_kwds,
@@ -443,9 +462,15 @@
w_args = space.newtuple(self.arguments_w)
w_kwds = space.newdict()
if self.keywords is not None:
+ limit = len(self.keywords)
+ if self.keyword_names_w is not None:
+ limit -= len(self.keyword_names_w)
for i in range(len(self.keywords)):
- space.setitem(w_kwds, space.wrap(self.keywords[i]),
- self.keywords_w[i])
+ if i < limit:
+ w_key = space.wrap(self.keywords[i])
+ else:
+ w_key = self.keyword_names_w[i - limit]
+ space.setitem(w_kwds, w_key, self.keywords_w[i])
return w_args, w_kwds
class ArgumentsForTranslation(Arguments):
@@ -666,14 +691,33 @@
class ArgErrUnknownKwds(ArgErr):
- def __init__(self, num_remainingkwds, keywords, used_keywords):
- self.kwd_name = ''
+ def __init__(self, space, num_remainingkwds, keywords, used_keywords,
+ keyword_names_w):
+ name = ''
self.num_kwds = num_remainingkwds
if num_remainingkwds == 1:
for i in range(len(keywords)):
if not used_keywords[i]:
- self.kwd_name = keywords[i]
+ name = keywords[i]
+ if name is None:
+ # We'll assume it's unicode. Encode it.
+ # Careful, I *think* it should not be possible to
+ # get an IndexError here but you never know.
+ try:
+ if keyword_names_w is None:
+ raise IndexError
+ # note: negative-based indexing from the end
+ w_name = keyword_names_w[i - len(keywords)]
+ except IndexError:
+ name = '?'
+ else:
+ w_enc = space.wrap(space.sys.defaultencoding)
+ w_err = space.wrap("replace")
+ w_name = space.call_method(w_name, "encode", w_enc,
+ w_err)
+ name = space.str_w(w_name)
break
+ self.kwd_name = name
def getmsg(self, fnname):
if self.num_kwds == 1:
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -55,7 +55,7 @@
co_expr = compile(evalexpr, '<evalexpr>', 'eval')
space = self.space
pyco_expr = PyCode._from_code(space, co_expr)
- w_res = pyco_expr.exec_host_bytecode(space, w_dict, w_dict)
+ w_res = pyco_expr.exec_host_bytecode(w_dict, w_dict)
res = space.str_w(space.repr(w_res))
if not isinstance(expected, float):
assert res == repr(expected)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -986,10 +986,7 @@
compiler = self.createcompiler()
expression = compiler.compile(expression, '?', 'eval', 0,
hidden_applevel=hidden_applevel)
- if isinstance(expression, types.CodeType):
- # XXX only used by appsupport
- expression = PyCode._from_code(self, expression)
- if not isinstance(expression, PyCode):
+ else:
raise TypeError, 'space.eval(): expected a string, code or PyCode object'
return expression.exec_code(self, w_globals, w_locals)
@@ -1004,9 +1001,6 @@
compiler = self.createcompiler()
statement = compiler.compile(statement, filename, 'exec', 0,
hidden_applevel=hidden_applevel)
- if isinstance(statement, types.CodeType):
- # XXX only used by appsupport
- statement = PyCode._from_code(self, statement)
if not isinstance(statement, PyCode):
raise TypeError, 'space.exec_(): expected a string, code or PyCode object'
w_key = self.wrap('__builtins__')
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -95,7 +95,7 @@
if self.co_flags & CO_VARKEYWORDS:
argcount += 1
# Cell vars could shadow already-set arguments.
- # astcompiler.pyassem used to be clever about the order of
+ # The compiler used to be clever about the order of
# the variables in both co_varnames and co_cellvars, but
# it no longer is for the sake of simplicity. Moreover
# code objects loaded from CPython don't necessarily follow
@@ -256,7 +256,7 @@
tuple(self.co_freevars),
tuple(self.co_cellvars) )
- def exec_host_bytecode(self, w_dict, w_globals, w_locals):
+ def exec_host_bytecode(self, w_globals, w_locals):
from pypy.interpreter.pyframe import CPythonFrame
frame = CPythonFrame(self.space, self, w_globals, None)
frame.setdictscope(w_locals)
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
import py
from pypy.interpreter.argument import (Arguments, ArgumentsForTranslation,
ArgErr, ArgErrUnknownKwds, ArgErrMultipleValues, ArgErrCount, rawshape,
@@ -126,6 +127,7 @@
w_AttributeError = AttributeError
w_UnicodeEncodeError = UnicodeEncodeError
w_dict = dict
+ w_str = str
class TestArgumentsNormal(object):
@@ -485,26 +487,6 @@
args._match_signature(None, l, Signature(['abc']))
assert len(l) == 1
assert l[0] == space.wrap(5)
- #
- def str_w(w):
- try:
- return str(w)
- except UnicodeEncodeError:
- raise OperationError(space.w_UnicodeEncodeError,
- space.wrap("oups"))
- space.str_w = str_w
- w_starstar = space.wrap({u'\u1234': 5})
- err = py.test.raises(OperationError, Arguments,
- space, [], w_starstararg=w_starstar)
- # Check that we get a TypeError. On CPython it is because of
- # "no argument called '?'". On PyPy we get a TypeError too, but
- # earlier: "keyword cannot be encoded to ascii". The
- # difference, besides the error message, is only apparent if the
- # receiver also takes a **arg. Then CPython passes the
- # non-ascii unicode unmodified, whereas PyPy complains. We will
- # not care until someone has a use case for that.
- assert not err.value.match(space, space.w_UnicodeEncodeError)
- assert err.value.match(space, space.w_TypeError)
class TestErrorHandling(object):
def test_missing_args(self):
@@ -559,13 +541,26 @@
assert 0, "did not raise"
def test_unknown_keywords(self):
- err = ArgErrUnknownKwds(1, ['a', 'b'], [True, False])
+ space = DummySpace()
+ err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [True, False], None)
s = err.getmsg('foo')
assert s == "foo() got an unexpected keyword argument 'b'"
- err = ArgErrUnknownKwds(2, ['a', 'b', 'c'], [True, False, False])
+ err = ArgErrUnknownKwds(space, 2, ['a', 'b', 'c'],
+ [True, False, False], None)
s = err.getmsg('foo')
assert s == "foo() got 2 unexpected keyword arguments"
+ def test_unknown_unicode_keyword(self):
+ class DummySpaceUnicode(DummySpace):
+ class sys:
+ defaultencoding = 'utf-8'
+ space = DummySpaceUnicode()
+ err = ArgErrUnknownKwds(space, 1, ['a', None, 'b', 'c'],
+ [True, False, True, True],
+ [unichr(0x1234), u'b', u'c'])
+ s = err.getmsg('foo')
+ assert s == "foo() got an unexpected keyword argument '\xe1\x88\xb4'"
+
def test_multiple_values(self):
err = ArgErrMultipleValues('bla')
s = err.getmsg('foo')
@@ -592,6 +587,14 @@
exc = raises(TypeError, (lambda a, b, **kw: 0), a=1)
assert exc.value.message == "<lambda>() takes exactly 2 non-keyword arguments (0 given)"
+ def test_unicode_keywords(self):
+ def f(**kwargs):
+ assert kwargs[u"美"] == 42
+ f(**{u"美" : 42})
+ def f(x): pass
+ e = raises(TypeError, "f(**{u'ü' : 19})")
+ assert "?" in str(e.value)
+
def make_arguments_for_translation(space, args_w, keywords_w={},
w_stararg=None, w_starstararg=None):
return ArgumentsForTranslation(space, args_w, keywords_w.keys(),
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -136,6 +136,7 @@
'call' : (('ref', 'varargs'), 'intorptr'),
'call_assembler' : (('varargs',), 'intorptr'),
'cond_call_gc_wb' : (('ptr', 'ptr'), None),
+ 'cond_call_gc_wb_array': (('ptr', 'int', 'ptr'), None),
'oosend' : (('varargs',), 'intorptr'),
'oosend_pure' : (('varargs',), 'intorptr'),
'guard_true' : (('bool',), None),
@@ -857,6 +858,9 @@
def op_cond_call_gc_wb(self, descr, a, b):
py.test.skip("cond_call_gc_wb not supported")
+ def op_cond_call_gc_wb_array(self, descr, a, b, c):
+ py.test.skip("cond_call_gc_wb_array not supported")
+
def op_oosend(self, descr, obj, *args):
raise NotImplementedError("oosend for lltype backend??")
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -1,5 +1,6 @@
import py
from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass
+from pypy.rpython.lltypesystem.lloperation import llop
from pypy.jit.backend.llsupport import symbolic, support
from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr
from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat
@@ -45,6 +46,8 @@
size = 0 # help translation
is_immutable = False
+ tid = llop.combine_ushort(lltype.Signed, 0, 0)
+
def __init__(self, size, count_fields_if_immut=-1):
self.size = size
self.count_fields_if_immut = count_fields_if_immut
@@ -149,6 +152,7 @@
class BaseArrayDescr(AbstractDescr):
_clsname = ''
+ tid = llop.combine_ushort(lltype.Signed, 0, 0)
def get_base_size(self, translate_support_code):
basesize, _, _ = symbolic.get_array_token(_A, translate_support_code)
@@ -263,6 +267,9 @@
def __repr__(self):
res = '%s(%s)' % (self.__class__.__name__, self.arg_classes)
+ extraeffect = getattr(self.extrainfo, 'extraeffect', None)
+ if extraeffect is not None:
+ res += ' EF=%r' % extraeffect
oopspecindex = getattr(self.extrainfo, 'oopspecindex', 0)
if oopspecindex:
from pypy.jit.codewriter.effectinfo import EffectInfo
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -476,6 +476,7 @@
return cpu.cast_adr_to_int(funcaddr)
def get_write_barrier_from_array_fn(self, cpu):
+ # returns a function with arguments [array, index, newvalue]
llop1 = self.llop1
funcptr = llop1.get_write_barrier_from_array_failing_case(
self.WB_ARRAY_FUNCPTR)
@@ -552,7 +553,7 @@
self.WB_FUNCPTR = lltype.Ptr(lltype.FuncType(
[llmemory.Address, llmemory.Address], lltype.Void))
self.WB_ARRAY_FUNCPTR = lltype.Ptr(lltype.FuncType(
- [llmemory.Address, lltype.Signed], lltype.Void))
+ [llmemory.Address, lltype.Signed, llmemory.Address], lltype.Void))
self.write_barrier_descr = WriteBarrierDescr(self)
#
def malloc_array(itemsize, tid, num_elem):
@@ -763,10 +764,8 @@
newops.append(op)
return newops
- def _gen_write_barrier(self, newops, v_base, v_value_or_index):
- # NB. the 2nd argument of COND_CALL_GC_WB is either a pointer
- # (regular case), or an index (case of write_barrier_from_array)
- args = [v_base, v_value_or_index]
+ def _gen_write_barrier(self, newops, v_base, v_value):
+ args = [v_base, v_value]
newops.append(ResOperation(rop.COND_CALL_GC_WB, args, None,
descr=self.write_barrier_descr))
@@ -780,7 +779,10 @@
length = known_lengths.get(v_base, LARGE)
if length >= LARGE:
# unknown or too big: produce a write_barrier_from_array
- self._gen_write_barrier(newops, v_base, v_index)
+ args = [v_base, v_index, v_value]
+ newops.append(ResOperation(rop.COND_CALL_GC_WB_ARRAY, args,
+ None,
+ descr=self.write_barrier_descr))
return
# fall-back case: produce a write_barrier
self._gen_write_barrier(newops, v_base, v_value)
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -9,7 +9,7 @@
from pypy.jit.metainterp.resoperation import get_deep_immutable_oplist
from pypy.jit.tool.oparser import parse
from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
-from pypy.jit.metainterp.test.test_optimizeopt import equaloplists
+from pypy.jit.metainterp.optimizeopt.util import equaloplists
def test_boehm():
gc_ll_descr = GcLLDescr_boehm(None, None, None)
@@ -553,12 +553,15 @@
del operations[:2]
assert len(operations) == 2
#
- assert operations[0].getopnum() == rop.COND_CALL_GC_WB
- assert operations[0].getarg(0) == v_base
if isinstance(v_new_length, ConstInt) and v_new_length.value < 130:
+ assert operations[0].getopnum() == rop.COND_CALL_GC_WB
+ assert operations[0].getarg(0) == v_base
assert operations[0].getarg(1) == v_value
else:
+ assert operations[0].getopnum() == rop.COND_CALL_GC_WB_ARRAY
+ assert operations[0].getarg(0) == v_base
assert operations[0].getarg(1) == v_index
+ assert operations[0].getarg(2) == v_value
assert operations[0].result is None
#
assert operations[1].getopnum() == rop.SETARRAYITEM_RAW
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1694,12 +1694,13 @@
assert record == []
def test_cond_call_gc_wb_array(self):
- def func_void(a, b):
- record.append((a, b))
+ def func_void(a, b, c):
+ record.append((a, b, c))
record = []
#
S = lltype.GcStruct('S', ('tid', lltype.Signed))
- FUNC = self.FuncType([lltype.Ptr(S), lltype.Signed], lltype.Void)
+ FUNC = self.FuncType([lltype.Ptr(S), lltype.Signed, lltype.Ptr(S)],
+ lltype.Void)
func_ptr = llhelper(lltype.Ptr(FUNC), func_void)
funcbox = self.get_funcbox(self.cpu, func_ptr)
class WriteBarrierDescr(AbstractDescr):
@@ -1719,11 +1720,11 @@
s.tid = value
sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
del record[:]
- self.execute_operation(rop.COND_CALL_GC_WB,
- [BoxPtr(sgcref), ConstInt(123)],
- 'void', descr=WriteBarrierDescr())
+ self.execute_operation(rop.COND_CALL_GC_WB_ARRAY,
+ [BoxPtr(sgcref), ConstInt(123), BoxPtr(sgcref)],
+ 'void', descr=WriteBarrierDescr())
if cond:
- assert record == [(s, 123)]
+ assert record == [(s, 123, s)]
else:
assert record == []
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -2223,15 +2223,26 @@
def genop_discard_cond_call_gc_wb(self, op, arglocs):
# Write code equivalent to write_barrier() in the GC: it checks
# a flag in the object at arglocs[0], and if set, it calls the
- # function remember_young_pointer() from the GC. The two arguments
- # to the call are in arglocs[:2]. The rest, arglocs[2:], contains
+ # function remember_young_pointer() from the GC. The arguments
+ # to the call are in arglocs[:N]. The rest, arglocs[N:], contains
# registers that need to be saved and restored across the call.
- # If op.getarg(1) is a int, it is an array index and we must call
- # instead remember_young_pointer_from_array().
+ # N is either 2 (regular write barrier) or 3 (array write barrier).
descr = op.getdescr()
if we_are_translated():
cls = self.cpu.gc_ll_descr.has_write_barrier_class()
assert cls is not None and isinstance(descr, cls)
+ #
+ opnum = op.getopnum()
+ if opnum == rop.COND_CALL_GC_WB:
+ N = 2
+ func = descr.get_write_barrier_fn(self.cpu)
+ elif opnum == rop.COND_CALL_GC_WB_ARRAY:
+ N = 3
+ func = descr.get_write_barrier_from_array_fn(self.cpu)
+ assert func != 0
+ else:
+ raise AssertionError(opnum)
+ #
loc_base = arglocs[0]
self.mc.TEST8(addr_add_const(loc_base, descr.jit_wb_if_flag_byteofs),
imm(descr.jit_wb_if_flag_singlebyte))
@@ -2242,29 +2253,27 @@
if IS_X86_32:
limit = -1 # push all arglocs on the stack
elif IS_X86_64:
- limit = 1 # push only arglocs[2:] on the stack
+ limit = N - 1 # push only arglocs[N:] on the stack
for i in range(len(arglocs)-1, limit, -1):
loc = arglocs[i]
if isinstance(loc, RegLoc):
self.mc.PUSH_r(loc.value)
else:
- assert not IS_X86_64 # there should only be regs in arglocs[2:]
+ assert not IS_X86_64 # there should only be regs in arglocs[N:]
self.mc.PUSH_i32(loc.getint())
if IS_X86_64:
# We clobber these registers to pass the arguments, but that's
# okay, because consider_cond_call_gc_wb makes sure that any
# caller-save registers with values in them are present in
- # arglocs[2:] too, so they are saved on the stack above and
+ # arglocs[N:] too, so they are saved on the stack above and
# restored below.
- remap_frame_layout(self, arglocs[:2], [edi, esi],
+ if N == 2:
+ callargs = [edi, esi]
+ else:
+ callargs = [edi, esi, edx]
+ remap_frame_layout(self, arglocs[:N], callargs,
X86_64_SCRATCH_REG)
-
- if op.getarg(1).type == INT:
- func = descr.get_write_barrier_from_array_fn(self.cpu)
- assert func != 0
- else:
- func = descr.get_write_barrier_fn(self.cpu)
-
+ #
# misaligned stack in the call, but it's ok because the write barrier
# is not going to call anything more. Also, this assumes that the
# write barrier does not touch the xmm registers. (Slightly delicate
@@ -2273,8 +2282,8 @@
# be done properly)
self.mc.CALL(imm(func))
if IS_X86_32:
- self.mc.ADD_ri(esp.value, 2*WORD)
- for i in range(2, len(arglocs)):
+ self.mc.ADD_ri(esp.value, N*WORD)
+ for i in range(N, len(arglocs)):
loc = arglocs[i]
assert isinstance(loc, RegLoc)
self.mc.POP_r(loc.value)
@@ -2283,6 +2292,8 @@
assert 0 < offset <= 127
self.mc.overwrite(jz_location-1, chr(offset))
+ genop_discard_cond_call_gc_wb_array = genop_discard_cond_call_gc_wb
+
def genop_force_token(self, op, arglocs, resloc):
# RegAlloc.consider_force_token ensures this:
assert isinstance(resloc, RegLoc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -884,12 +884,12 @@
def consider_cond_call_gc_wb(self, op):
assert op.result is None
args = op.getarglist()
- loc_newvalue_or_index= self.rm.make_sure_var_in_reg(op.getarg(1), args)
- # ^^^ we force loc_newvalue_or_index in a reg (unless it's a Const),
- # because it will be needed anyway by the following setfield_gc.
- # It avoids loading it twice from the memory.
- loc_base = self.rm.make_sure_var_in_reg(op.getarg(0), args)
- arglocs = [loc_base, loc_newvalue_or_index]
+ N = len(args)
+ # we force all arguments in a reg (unless they are Consts),
+ # because it will be needed anyway by the following setfield_gc
+ # or setarrayitem_gc. It avoids loading it twice from the memory.
+ arglocs = [self.rm.make_sure_var_in_reg(op.getarg(i), args)
+ for i in range(N)]
# add eax, ecx and edx as extra "arguments" to ensure they are
# saved and restored. Fish in self.rm to know which of these
# registers really need to be saved (a bit of a hack). Moreover,
@@ -903,6 +903,8 @@
self.PerformDiscard(op, arglocs)
self.rm.possibly_free_vars_for_op(op)
+ consider_cond_call_gc_wb_array = consider_cond_call_gc_wb
+
def fastpath_malloc_fixedsize(self, op, descr):
assert isinstance(descr, BaseSizeDescr)
self._do_fastpath_malloc(op, descr.size, descr.tid)
diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py
--- a/pypy/jit/metainterp/compile.py
+++ b/pypy/jit/metainterp/compile.py
@@ -14,7 +14,7 @@
from pypy.jit.metainterp.history import BoxPtr, BoxObj, BoxFloat, Const
from pypy.jit.metainterp import history
from pypy.jit.metainterp.typesystem import llhelper, oohelper
-from pypy.jit.metainterp.optimizeutil import InvalidLoop
+from pypy.jit.metainterp.optimize import InvalidLoop
from pypy.jit.metainterp.resume import NUMBERING
from pypy.jit.codewriter import heaptracker, longlong
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -316,6 +316,7 @@
if value in (rop.FORCE_TOKEN,
rop.CALL_ASSEMBLER,
rop.COND_CALL_GC_WB,
+ rop.COND_CALL_GC_WB_ARRAY,
rop.DEBUG_MERGE_POINT,
rop.JIT_DEBUG,
rop.SETARRAYITEM_RAW,
diff --git a/pypy/jit/metainterp/logger.py b/pypy/jit/metainterp/logger.py
--- a/pypy/jit/metainterp/logger.py
+++ b/pypy/jit/metainterp/logger.py
@@ -103,6 +103,7 @@
if op.getopnum() == rop.DEBUG_MERGE_POINT:
jd_sd = self.metainterp_sd.jitdrivers_sd[op.getarg(0).getint()]
s = jd_sd.warmstate.get_location_str(op.getarglist()[2:])
+ s = s.replace(',', '.') # we use comma for argument splitting
return "debug_merge_point(%d, '%s')" % (op.getarg(1).getint(), s)
if ops_offset is None:
offset = -1
diff --git a/pypy/jit/metainterp/optimize.py b/pypy/jit/metainterp/optimize.py
--- a/pypy/jit/metainterp/optimize.py
+++ b/pypy/jit/metainterp/optimize.py
@@ -1,9 +1,20 @@
from pypy.rlib.debug import debug_start, debug_stop
+from pypy.jit.metainterp.jitexc import JitException
+
+class InvalidLoop(JitException):
+ """Raised when the optimize*.py detect that the loop that
+ we are trying to build cannot possibly make sense as a
+ long-running loop (e.g. it cannot run 2 complete iterations)."""
+
+class RetraceLoop(JitException):
+ """ Raised when inlining a short preamble resulted in an
+ InvalidLoop. This means the optimized loop is too specialized
+ to be useful here, so we trace it again and produced a second
+ copy specialized in some different way.
+ """
# ____________________________________________________________
-from pypy.jit.metainterp.optimizeopt import optimize_loop_1, optimize_bridge_1
-
def optimize_loop(metainterp_sd, old_loop_tokens, loop, enable_opts):
debug_start("jit-optimize")
try:
@@ -13,6 +24,7 @@
debug_stop("jit-optimize")
def _optimize_loop(metainterp_sd, old_loop_tokens, loop, enable_opts):
+ from pypy.jit.metainterp.optimizeopt import optimize_loop_1
cpu = metainterp_sd.cpu
loop.logops = metainterp_sd.logger_noopt.log_loop(loop.inputargs,
loop.operations)
@@ -36,6 +48,7 @@
def _optimize_bridge(metainterp_sd, old_loop_tokens, bridge, enable_opts,
inline_short_preamble, retraced=False):
+ from pypy.jit.metainterp.optimizeopt import optimize_bridge_1
cpu = metainterp_sd.cpu
bridge.logops = metainterp_sd.logger_noopt.log_loop(bridge.inputargs,
bridge.operations)
diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -15,7 +15,7 @@
('virtualize', OptVirtualize),
('string', OptString),
('heap', OptHeap),
- ('ffi', OptFfiCall),
+ ('ffi', None),
('unroll', None)]
# no direct instantiation of unroll
unroll_all_opts = unrolling_iterable(ALL_OPTS)
@@ -25,10 +25,9 @@
ALL_OPTS_NAMES = ':'.join([name for name, _ in ALL_OPTS])
PARAMETERS['enable_opts'] = ALL_OPTS_NAMES
-def optimize_loop_1(metainterp_sd, loop, enable_opts,
+def build_opt_chain(metainterp_sd, enable_opts,
inline_short_preamble=True, retraced=False):
- """Optimize loop.operations to remove internal overheadish operations.
- """
+ config = metainterp_sd.config
optimizations = []
unroll = 'unroll' in enable_opts
for name, opt in unroll_all_opts:
@@ -40,6 +39,11 @@
# FIXME: Workaround to disable string optimisation
# during preamble but to keep it during the loop
optimizations.append(o)
+ elif name == 'ffi' and config.translation.jit_ffi:
+ # we cannot put the class directly in the unrolling_iterable,
+ # because we do not want it to be seen at all (to avoid to
+ # introduce a dependency on libffi in case we do not need it)
+ optimizations.append(OptFfiCall())
if ('rewrite' not in enable_opts or 'virtualize' not in enable_opts
or 'heap' not in enable_opts):
@@ -48,6 +52,17 @@
if inline_short_preamble:
optimizations = [OptInlineShortPreamble(retraced)] + optimizations
+ return optimizations, unroll
+
+
+def optimize_loop_1(metainterp_sd, loop, enable_opts,
+ inline_short_preamble=True, retraced=False):
+ """Optimize loop.operations to remove internal overheadish operations.
+ """
+
+ optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts,
+ inline_short_preamble, retraced)
+
if unroll:
optimize_unroll(metainterp_sd, loop, optimizations)
else:
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -4,7 +4,7 @@
from pypy.rlib.debug import debug_start, debug_stop, debug_print, have_debug_prints
from pypy.jit.codewriter.effectinfo import EffectInfo
from pypy.jit.metainterp.resoperation import rop, ResOperation
-from pypy.jit.metainterp.optimizeutil import _findall
+from pypy.jit.metainterp.optimizeopt.util import _findall
from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
from pypy.jit.backend.llsupport.ffisupport import UnsupportedKind
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -1,5 +1,5 @@
import os
-from pypy.jit.metainterp.optimizeutil import _findall
+from pypy.jit.metainterp.optimizeopt.util import _findall
from pypy.jit.metainterp.resoperation import rop, ResOperation
from pypy.rlib.objectmodel import we_are_translated
from pypy.jit.metainterp.jitexc import JitException
@@ -112,7 +112,7 @@
class OptHeap(Optimization):
"""Cache repeated heap accesses"""
-
+
def __init__(self):
# cached fields: {descr: CachedField}
self.cached_fields = {}
@@ -129,7 +129,7 @@
self.force_all_lazy_setfields()
else:
assert 0 # was: new.lazy_setfields = self.lazy_setfields
-
+
for descr, d in self.cached_fields.items():
new.cached_fields[descr] = d.get_reconstructed(optimizer, valuemap)
diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -1,5 +1,5 @@
from pypy.jit.metainterp.optimizeopt.optimizer import Optimization, CONST_1, CONST_0
-from pypy.jit.metainterp.optimizeutil import _findall
+from pypy.jit.metainterp.optimizeopt.util import _findall
from pypy.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded, \
IntLowerBound, IntUpperBound
from pypy.jit.metainterp.history import Const, ConstInt
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -4,9 +4,9 @@
from pypy.jit.metainterp.resoperation import rop, ResOperation
from pypy.jit.metainterp import jitprof
from pypy.jit.metainterp.executor import execute_nonspec
-from pypy.jit.metainterp.optimizeutil import _findall, sort_descrs
-from pypy.jit.metainterp.optimizeutil import descrlist_dict
-from pypy.jit.metainterp.optimizeutil import InvalidLoop, args_dict
+from pypy.jit.metainterp.optimizeopt.util import _findall, sort_descrs
+from pypy.jit.metainterp.optimizeopt.util import descrlist_dict, args_dict
+from pypy.jit.metainterp.optimize import InvalidLoop
from pypy.jit.metainterp import resume, compile
from pypy.jit.metainterp.typesystem import llhelper, oohelper
from pypy.rpython.lltypesystem import lltype
@@ -141,6 +141,9 @@
# meaning it has been forced.
return self.box is None
+ def is_forced_virtual(self):
+ return False
+
def getfield(self, ofs, default):
raise NotImplementedError
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -1,7 +1,7 @@
from pypy.jit.metainterp.optimizeopt.optimizer import *
from pypy.jit.metainterp.resoperation import opboolinvers, opboolreflex
from pypy.jit.metainterp.history import ConstInt
-from pypy.jit.metainterp.optimizeutil import _findall
+from pypy.jit.metainterp.optimizeopt.util import _findall
from pypy.jit.metainterp.resoperation import rop, ResOperation
from pypy.jit.codewriter.effectinfo import EffectInfo
from pypy.jit.metainterp.optimizeopt.intutils import IntBound
@@ -184,6 +184,32 @@
else:
self.emit_operation(op)
+ def optimize_FLOAT_MUL(self, op):
+ arg1 = op.getarg(0)
+ arg2 = op.getarg(1)
+
+ # Constant fold f0 * 1.0 and turn f0 * -1.0 into a FLOAT_NEG, these
+ # work in all cases, including NaN and inf
+ for lhs, rhs in [(arg1, arg2), (arg2, arg1)]:
+ v1 = self.getvalue(lhs)
+ v2 = self.getvalue(rhs)
+
+ if v1.is_constant():
+ if v1.box.getfloat() == 1.0:
+ self.make_equal_to(op.result, v2)
+ return
+ elif v1.box.getfloat() == -1.0:
+ self.emit_operation(ResOperation(
+ rop.FLOAT_NEG, [rhs], op.result
+ ))
+ return
+ self.emit_operation(op)
+
+ def optimize_FLOAT_NEG(self, op):
+ v1 = op.getarg(0)
+ self.emit_operation(op)
+ self.pure(rop.FLOAT_NEG, [op.result], v1)
+
def optimize_CALL_PURE(self, op):
arg_consts = []
for i in range(op.numargs()):
@@ -193,7 +219,7 @@
break
arg_consts.append(const)
else:
- # all constant arguments: check if we already know the reslut
+ # all constant arguments: check if we already know the result
try:
result = self.optimizer.call_pure_results[arg_consts]
except KeyError:
diff --git a/pypy/jit/metainterp/optimizeopt/simplify.py b/pypy/jit/metainterp/optimizeopt/simplify.py
--- a/pypy/jit/metainterp/optimizeopt/simplify.py
+++ b/pypy/jit/metainterp/optimizeopt/simplify.py
@@ -1,7 +1,7 @@
from pypy.jit.metainterp.resoperation import ResOperation, rop
from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
-from pypy.jit.metainterp.optimizeutil import _findall
+from pypy.jit.metainterp.optimizeopt.util import _findall
class OptSimplify(Optimization):
def optimize_CALL_PURE(self, op):
diff --git a/pypy/jit/metainterp/optimizeopt/string.py b/pypy/jit/metainterp/optimizeopt/string.py
--- a/pypy/jit/metainterp/optimizeopt/string.py
+++ b/pypy/jit/metainterp/optimizeopt/string.py
@@ -8,7 +8,7 @@
from pypy.jit.metainterp.optimizeopt import optimizer, virtualize
from pypy.jit.metainterp.optimizeopt.optimizer import CONST_0, CONST_1
from pypy.jit.metainterp.optimizeopt.optimizer import llhelper
-from pypy.jit.metainterp.optimizeutil import _findall
+from pypy.jit.metainterp.optimizeopt.util import _findall
from pypy.jit.codewriter.effectinfo import EffectInfo
from pypy.jit.codewriter import heaptracker
from pypy.rlib.unroll import unrolling_iterable
diff --git a/pypy/jit/metainterp/optimizeopt/test/__init__.py b/pypy/jit/metainterp/optimizeopt/test/__init__.py
new file mode 100644
diff --git a/pypy/jit/metainterp/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
rename from pypy/jit/metainterp/test/test_optimizebasic.py
rename to pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -1,40 +1,15 @@
import py
from pypy.rlib.objectmodel import instantiate
-from pypy.jit.metainterp.test.test_optimizeutil import (LLtypeMixin,
- #OOtypeMixin,
- BaseTest)
+from pypy.jit.metainterp.optimizeopt.test.test_util import (
+ LLtypeMixin, BaseTest, FakeMetaInterpStaticData)
from pypy.jit.metainterp.test.test_compile import FakeLogger
import pypy.jit.metainterp.optimizeopt.optimizer as optimizeopt
import pypy.jit.metainterp.optimizeopt.virtualize as virtualize
-from pypy.jit.metainterp.optimizeutil import InvalidLoop
+from pypy.jit.metainterp.optimize import InvalidLoop
from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt
-from pypy.jit.metainterp.jitprof import EmptyProfiler
from pypy.jit.metainterp import executor, compile, resume, history
from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
-from pypy.jit.tool.oparser import pure_parse
-from pypy.jit.metainterp.optimizeutil import args_dict
-
-##class FakeFrame(object):
-## parent_resumedata_snapshot = None
-## parent_resumedata_frame_info_list = None
-
-## def __init__(self, code="", pc=0):
-## self.jitcode = code
-## self.pc = pc
-
-class Fake(object):
- failargs_limit = 1000
- storedebug = None
-
-class FakeMetaInterpStaticData(object):
-
- def __init__(self, cpu):
- self.cpu = cpu
- self.profiler = EmptyProfiler()
- self.options = Fake()
- self.globaldata = Fake()
- self.logger_ops = FakeLogger()
- self.logger_noopt = FakeLogger()
+
def test_store_final_boxes_in_guard():
from pypy.jit.metainterp.compile import ResumeGuardDescr
@@ -104,7 +79,7 @@
assert vinfo3 is vinfo4
def test_descrlist_dict():
- from pypy.jit.metainterp import optimizeutil
+ from pypy.jit.metainterp.optimizeopt import util as optimizeutil
h1 = optimizeutil.descrlist_hash([])
h2 = optimizeutil.descrlist_hash([LLtypeMixin.valuedescr])
h3 = optimizeutil.descrlist_hash(
@@ -133,160 +108,21 @@
# ____________________________________________________________
-def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={},
- text_right=None):
- # try to use the full width of the terminal to display the list
- # unfortunately, does not work with the default capture method of py.test
- # (which is fd), you you need to use either -s or --capture=sys, else you
- # get the standard 80 columns width
- totwidth = py.io.get_terminal_width()
- width = totwidth / 2 - 1
- print ' Comparing lists '.center(totwidth, '-')
- text_right = text_right or 'expected'
- print '%s| %s' % ('optimized'.center(width), text_right.center(width))
- for op1, op2 in zip(oplist1, oplist2):
- txt1 = str(op1)
- txt2 = str(op2)
- while txt1 or txt2:
- print '%s| %s' % (txt1[:width].ljust(width), txt2[:width])
- txt1 = txt1[width:]
- txt2 = txt2[width:]
- assert op1.getopnum() == op2.getopnum()
- assert op1.numargs() == op2.numargs()
- for i in range(op1.numargs()):
- x = op1.getarg(i)
- y = op2.getarg(i)
- assert x == remap.get(y, y)
- if op2.result in remap:
- assert op1.result == remap[op2.result]
- else:
- remap[op2.result] = op1.result
- if op1.getopnum() != rop.JUMP: # xxx obscure
- assert op1.getdescr() == op2.getdescr()
- if op1.getfailargs() or op2.getfailargs():
- assert len(op1.getfailargs()) == len(op2.getfailargs())
- if strict_fail_args:
- for x, y in zip(op1.getfailargs(), op2.getfailargs()):
- assert x == remap.get(y, y)
- else:
- fail_args1 = set(op1.getfailargs())
- fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
- assert fail_args1 == fail_args2
- assert len(oplist1) == len(oplist2)
- print '-'*totwidth
- return True
-
-def test_equaloplists():
- ops = """
- [i0]
- i1 = int_add(i0, 1)
- i2 = int_add(i1, 1)
- guard_true(i1) [i2]
- jump(i1)
- """
- namespace = {}
- loop1 = pure_parse(ops, namespace=namespace)
- loop2 = pure_parse(ops, namespace=namespace)
- loop3 = pure_parse(ops.replace("i2 = int_add", "i2 = int_sub"),
- namespace=namespace)
- assert equaloplists(loop1.operations, loop2.operations)
- py.test.raises(AssertionError,
- "equaloplists(loop1.operations, loop3.operations)")
-
-def test_equaloplists_fail_args():
- ops = """
- [i0]
- i1 = int_add(i0, 1)
- i2 = int_add(i1, 1)
- guard_true(i1) [i2, i1]
- jump(i1)
- """
- namespace = {}
- loop1 = pure_parse(ops, namespace=namespace)
- loop2 = pure_parse(ops.replace("[i2, i1]", "[i1, i2]"),
- namespace=namespace)
- py.test.raises(AssertionError,
- "equaloplists(loop1.operations, loop2.operations)")
- assert equaloplists(loop1.operations, loop2.operations,
- strict_fail_args=False)
- loop3 = pure_parse(ops.replace("[i2, i1]", "[i2, i0]"),
- namespace=namespace)
- py.test.raises(AssertionError,
- "equaloplists(loop1.operations, loop3.operations)")
-
-# ____________________________________________________________
-
-class Storage(compile.ResumeGuardDescr):
- "for tests."
- def __init__(self, metainterp_sd=None, original_greenkey=None):
- self.metainterp_sd = metainterp_sd
- self.original_greenkey = original_greenkey
- def store_final_boxes(self, op, boxes):
- op.setfailargs(boxes)
- def __eq__(self, other):
- return type(self) is type(other) # xxx obscure
-
-def _sortboxes(boxes):
- _kind2count = {history.INT: 1, history.REF: 2, history.FLOAT: 3}
- return sorted(boxes, key=lambda box: _kind2count[box.type])
class BaseTestBasic(BaseTest):
- def invent_fail_descr(self, fail_args):
- if fail_args is None:
- return None
- descr = Storage()
- descr.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
- descr.rd_snapshot = resume.Snapshot(None, _sortboxes(fail_args))
- return descr
-
- def assert_equal(self, optimized, expected):
- assert len(optimized.inputargs) == len(expected.inputargs)
- remap = {}
- for box1, box2 in zip(optimized.inputargs, expected.inputargs):
- assert box1.__class__ == box2.__class__
- remap[box2] = box1
- assert equaloplists(optimized.operations,
- expected.operations, False, remap)
+ enable_opts = "intbounds:rewrite:virtualize:string:heap"
def optimize_loop(self, ops, optops, call_pure_results=None):
+
loop = self.parse(ops)
- #
- self.loop = loop
- loop.call_pure_results = args_dict()
- if call_pure_results is not None:
- for k, v in call_pure_results.items():
- loop.call_pure_results[list(k)] = v
- metainterp_sd = FakeMetaInterpStaticData(self.cpu)
- if hasattr(self, 'vrefinfo'):
- metainterp_sd.virtualref_info = self.vrefinfo
- if hasattr(self, 'callinfocollection'):
- metainterp_sd.callinfocollection = self.callinfocollection
- #
- # XXX list the exact optimizations that are needed for each test
- from pypy.jit.metainterp.optimizeopt import (OptIntBounds,
- OptRewrite,
- OptVirtualize,
- OptString,
- OptHeap,
- Optimizer)
- from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
-
- optimizations = [OptIntBounds(),
- OptRewrite(),
- OptVirtualize(),
- OptString(),
- OptHeap(),
- OptFfiCall(),
- ]
- optimizer = Optimizer(metainterp_sd, loop, optimizations)
- optimizer.propagate_all_forward()
- #
expected = self.parse(optops)
+ self._do_optimize_loop(loop, call_pure_results)
print '\n'.join([str(o) for o in loop.operations])
self.assert_equal(loop, expected)
+
class BaseTestOptimizeBasic(BaseTestBasic):
def test_simple(self):
@@ -2290,6 +2126,81 @@
"""
self.optimize_loop(ops, expected)
+ def test_fold_constant_partial_ops_float(self):
+ ops = """
+ [f0]
+ f1 = float_mul(f0, 1.0)
+ f2 = escape(f1)
+ jump(f2)
+ """
+ expected = """
+ [f0]
+ f2 = escape(f0)
+ jump(f2)
+ """
+ self.optimize_loop(ops, expected)
+
+ ops = """
+ [f0]
+ f1 = float_mul(1.0, f0)
+ f2 = escape(f1)
+ jump(f2)
+ """
+ expected = """
+ [f0]
+ f2 = escape(f0)
+ jump(f2)
+ """
+ self.optimize_loop(ops, expected)
+
+
+ ops = """
+ [f0]
+ f1 = float_mul(f0, -1.0)
+ f2 = escape(f1)
+ jump(f2)
+ """
+ expected = """
+ [f0]
+ f1 = float_neg(f0)
+ f2 = escape(f1)
+ jump(f2)
+ """
+ self.optimize_loop(ops, expected)
+
+ ops = """
+ [f0]
+ f1 = float_mul(-1.0, f0)
+ f2 = escape(f1)
+ jump(f2)
+ """
+ expected = """
+ [f0]
+ f1 = float_neg(f0)
+ f2 = escape(f1)
+ jump(f2)
+ """
+ self.optimize_loop(ops, expected)
+
+ def test_fold_repeated_float_neg(self):
+ ops = """
+ [f0]
+ f1 = float_neg(f0)
+ f2 = float_neg(f1)
+ f3 = float_neg(f2)
+ f4 = float_neg(f3)
+ escape(f4)
+ jump(f4)
+ """
+ expected = """
+ [f0]
+ # The backend removes this dead op.
+ f1 = float_neg(f0)
+ escape(f0)
+ jump(f0)
+ """
+ self.optimize_loop(ops, expected)
+
# ----------
def make_fail_descr(self):
diff --git a/pypy/jit/metainterp/test/test_optimizefficall.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
rename from pypy/jit/metainterp/test/test_optimizefficall.py
rename to pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
--- a/pypy/jit/metainterp/test/test_optimizefficall.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
@@ -2,8 +2,8 @@
from pypy.rlib.libffi import Func, types
from pypy.jit.metainterp.history import AbstractDescr
from pypy.jit.codewriter.effectinfo import EffectInfo
-from pypy.jit.metainterp.test.test_optimizebasic import BaseTestBasic
-from pypy.jit.metainterp.test.test_optimizebasic import LLtypeMixin
+from pypy.jit.metainterp.optimizeopt.test.test_optimizebasic import BaseTestBasic
+from pypy.jit.metainterp.optimizeopt.test.test_optimizebasic import LLtypeMixin
class MyCallDescr(AbstractDescr):
"""
@@ -32,7 +32,8 @@
class TestFfiCall(BaseTestBasic, LLtypeMixin):
- jit_ffi = True
+
+ enable_opts = "intbounds:rewrite:virtualize:string:heap:ffi"
class namespace:
cpu = LLtypeMixin.cpu
diff --git a/pypy/jit/metainterp/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
rename from pypy/jit/metainterp/test/test_optimizeopt.py
rename to pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1,202 +1,88 @@
import py
from pypy.rlib.objectmodel import instantiate
-from pypy.jit.metainterp.test.test_optimizeutil import (LLtypeMixin,
- #OOtypeMixin,
- BaseTest)
+from pypy.jit.metainterp.optimizeopt.test.test_util import (
+ LLtypeMixin, BaseTest, Storage, _sortboxes)
import pypy.jit.metainterp.optimizeopt.optimizer as optimizeopt
import pypy.jit.metainterp.optimizeopt.virtualize as virtualize
-from pypy.jit.metainterp.optimizeopt import optimize_loop_1, ALL_OPTS_DICT
-from pypy.jit.metainterp.optimizeutil import InvalidLoop
+from pypy.jit.metainterp.optimizeopt import optimize_loop_1, ALL_OPTS_DICT, build_opt_chain
+from pypy.jit.metainterp.optimize import InvalidLoop
from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt
from pypy.jit.metainterp.history import TreeLoop, LoopToken
from pypy.jit.metainterp.jitprof import EmptyProfiler
from pypy.jit.metainterp import executor, compile, resume, history
from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
from pypy.jit.tool.oparser import pure_parse
-from pypy.jit.metainterp.test.test_optimizebasic import equaloplists
-from pypy.jit.metainterp.optimizeutil import args_dict
-
-class Fake(object):
- failargs_limit = 1000
- storedebug = None
-
-class FakeMetaInterpStaticData(object):
-
- def __init__(self, cpu, jit_ffi=False):
- self.cpu = cpu
- self.profiler = EmptyProfiler()
- self.options = Fake()
- self.globaldata = Fake()
- self.jit_ffi = jit_ffi
-
-def test_store_final_boxes_in_guard():
- from pypy.jit.metainterp.compile import ResumeGuardDescr
- from pypy.jit.metainterp.resume import tag, TAGBOX
- b0 = BoxInt()
- b1 = BoxInt()
- opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu),
- None)
- fdescr = ResumeGuardDescr()
- op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr)
- # setup rd data
- fi0 = resume.FrameInfo(None, "code0", 11)
- fdescr.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33)
- snapshot0 = resume.Snapshot(None, [b0])
- fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1])
+from pypy.jit.metainterp.optimizeopt.util import args_dict
+from pypy.jit.metainterp.optimizeopt.test.test_optimizebasic import FakeMetaInterpStaticData
+from pypy.config.pypyoption import get_pypy_config
+
+
+def test_build_opt_chain():
+ def check(chain, expected_names):
+ names = [opt.__class__.__name__ for opt in chain]
+ assert names == expected_names
#
- opt.store_final_boxes_in_guard(op)
- if op.getfailargs() == [b0, b1]:
- assert list(fdescr.rd_numb.nums) == [tag(1, TAGBOX)]
- assert list(fdescr.rd_numb.prev.nums) == [tag(0, TAGBOX)]
- else:
- assert op.getfailargs() == [b1, b0]
- assert list(fdescr.rd_numb.nums) == [tag(0, TAGBOX)]
- assert list(fdescr.rd_numb.prev.nums) == [tag(1, TAGBOX)]
- assert fdescr.rd_virtuals is None
- assert fdescr.rd_consts == []
-
-def test_sharing_field_lists_of_virtual():
- class FakeOptimizer(object):
- class cpu(object):
- pass
- opt = FakeOptimizer()
- virt1 = virtualize.AbstractVirtualStructValue(opt, None)
- lst1 = virt1._get_field_descr_list()
- assert lst1 == []
- lst2 = virt1._get_field_descr_list()
- assert lst1 is lst2
- virt1.setfield(LLtypeMixin.valuedescr, optimizeopt.OptValue(None))
- lst3 = virt1._get_field_descr_list()
- assert lst3 == [LLtypeMixin.valuedescr]
- lst4 = virt1._get_field_descr_list()
- assert lst3 is lst4
-
- virt2 = virtualize.AbstractVirtualStructValue(opt, None)
- lst5 = virt2._get_field_descr_list()
- assert lst5 is lst1
- virt2.setfield(LLtypeMixin.valuedescr, optimizeopt.OptValue(None))
- lst6 = virt1._get_field_descr_list()
- assert lst6 is lst3
-
-def test_reuse_vinfo():
- class FakeVInfo(object):
- def set_content(self, fieldnums):
- self.fieldnums = fieldnums
- def equals(self, fieldnums):
- return self.fieldnums == fieldnums
- class FakeVirtualValue(virtualize.AbstractVirtualValue):
- def _make_virtual(self, *args):
- return FakeVInfo()
- v1 = FakeVirtualValue(None, None, None)
- vinfo1 = v1.make_virtual_info(None, [1, 2, 4])
- vinfo2 = v1.make_virtual_info(None, [1, 2, 4])
- assert vinfo1 is vinfo2
- vinfo3 = v1.make_virtual_info(None, [1, 2, 6])
- assert vinfo3 is not vinfo2
- vinfo4 = v1.make_virtual_info(None, [1, 2, 6])
- assert vinfo3 is vinfo4
-
-def test_descrlist_dict():
- from pypy.jit.metainterp import optimizeutil
- h1 = optimizeutil.descrlist_hash([])
- h2 = optimizeutil.descrlist_hash([LLtypeMixin.valuedescr])
- h3 = optimizeutil.descrlist_hash(
- [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr])
- assert h1 != h2
- assert h2 != h3
- assert optimizeutil.descrlist_eq([], [])
- assert not optimizeutil.descrlist_eq([], [LLtypeMixin.valuedescr])
- assert optimizeutil.descrlist_eq([LLtypeMixin.valuedescr],
- [LLtypeMixin.valuedescr])
- assert not optimizeutil.descrlist_eq([LLtypeMixin.valuedescr],
- [LLtypeMixin.nextdescr])
- assert optimizeutil.descrlist_eq([LLtypeMixin.valuedescr, LLtypeMixin.nextdescr],
- [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr])
- assert not optimizeutil.descrlist_eq([LLtypeMixin.nextdescr, LLtypeMixin.valuedescr],
- [LLtypeMixin.valuedescr, LLtypeMixin.nextdescr])
-
- # descrlist_eq should compare by identity of the descrs, not by the result
- # of sort_key
- class FakeDescr(object):
- def sort_key(self):
- return 1
-
- assert not optimizeutil.descrlist_eq([FakeDescr()], [FakeDescr()])
+ metainterp_sd = FakeMetaInterpStaticData(None)
+ chain, _ = build_opt_chain(metainterp_sd, "", inline_short_preamble=False)
+ check(chain, ["OptSimplify"])
+ #
+ chain, _ = build_opt_chain(metainterp_sd, "")
+ check(chain, ["OptInlineShortPreamble", "OptSimplify"])
+ #
+ chain, _ = build_opt_chain(metainterp_sd, "")
+ check(chain, ["OptInlineShortPreamble", "OptSimplify"])
+ #
+ chain, _ = build_opt_chain(metainterp_sd, "heap:intbounds")
+ check(chain, ["OptInlineShortPreamble", "OptIntBounds", "OptHeap", "OptSimplify"])
+ #
+ chain, unroll = build_opt_chain(metainterp_sd, "unroll")
+ check(chain, ["OptInlineShortPreamble", "OptSimplify"])
+ assert unroll
+ #
+ chain, _ = build_opt_chain(metainterp_sd, "aaa:bbb", inline_short_preamble=False)
+ check(chain, ["OptSimplify"])
+ #
+ chain, _ = build_opt_chain(metainterp_sd, "ffi", inline_short_preamble=False)
+ check(chain, ["OptFfiCall", "OptSimplify"])
+ #
+ metainterp_sd.config = get_pypy_config(translating=True)
+ assert not metainterp_sd.config.translation.jit_ffi
+ chain, _ = build_opt_chain(metainterp_sd, "ffi", inline_short_preamble=False)
+ check(chain, ["OptSimplify"])
+
# ____________________________________________________________
-class Storage(compile.ResumeGuardDescr):
- "for tests."
- def __init__(self, metainterp_sd=None, original_greenkey=None):
- self.metainterp_sd = metainterp_sd
- self.original_greenkey = original_greenkey
- def store_final_boxes(self, op, boxes):
- op.setfailargs(boxes)
- def __eq__(self, other):
- return type(self) is type(other) # xxx obscure
+
+
+class FakeDescr(compile.ResumeGuardDescr):
+ class rd_snapshot:
+ class prev:
+ prev = None
+ boxes = []
+ boxes = []
def clone_if_mutable(self):
- res = Storage(self.metainterp_sd, self.original_greenkey)
- self.copy_all_attributes_into(res)
- return res
-
-def _sortboxes(boxes):
- _kind2count = {history.INT: 1, history.REF: 2, history.FLOAT: 3}
- return sorted(boxes, key=lambda box: _kind2count[box.type])
-
-class BaseTestOptimizeOpt(BaseTest):
- jit_ffi = False
-
- def invent_fail_descr(self, fail_args):
- if fail_args is None:
- return None
- descr = Storage()
- descr.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
- descr.rd_snapshot = resume.Snapshot(None, _sortboxes(fail_args))
- return descr
-
- def assert_equal(self, optimized, expected, text_right=None):
- assert len(optimized.inputargs) == len(expected.inputargs)
- remap = {}
- for box1, box2 in zip(optimized.inputargs, expected.inputargs):
- assert box1.__class__ == box2.__class__
- remap[box2] = box1
- assert equaloplists(optimized.operations,
- expected.operations, False, remap, text_right)
-
- def optimize_loop(self, ops, optops, expected_preamble=None,
+ return self
+
+
+class BaseTestWithUnroll(BaseTest):
+
+ enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+
+ def optimize_loop(self, ops, expected, expected_preamble=None,
call_pure_results=None):
loop = self.parse(ops)
- if optops != "crash!":
- expected = self.parse(optops)
- else:
- expected = "crash!"
+ if expected != "crash!":
+ expected = self.parse(expected)
if expected_preamble:
expected_preamble = self.parse(expected_preamble)
- #
- self.loop = loop
- loop.call_pure_results = args_dict()
- if call_pure_results is not None:
- for k, v in call_pure_results.items():
- loop.call_pure_results[list(k)] = v
+
loop.preamble = TreeLoop('preamble')
loop.preamble.inputargs = loop.inputargs
loop.preamble.token = LoopToken()
- metainterp_sd = FakeMetaInterpStaticData(self.cpu, self.jit_ffi)
- if hasattr(self, 'vrefinfo'):
- metainterp_sd.virtualref_info = self.vrefinfo
- if hasattr(self, 'callinfocollection'):
- metainterp_sd.callinfocollection = self.callinfocollection
- class FakeDescr(compile.ResumeGuardDescr):
- class rd_snapshot:
- class prev:
- prev = None
- boxes = []
- boxes = []
- def clone_if_mutable(self):
- return self
loop.preamble.start_resumedescr = FakeDescr()
- optimize_loop_1(metainterp_sd, loop, ALL_OPTS_DICT)
#
-
+ self._do_optimize_loop(loop, call_pure_results)
+ #
print
print loop.preamble.inputargs
print '\n'.join([str(o) for o in loop.preamble.operations])
@@ -204,16 +90,14 @@
print loop.inputargs
print '\n'.join([str(o) for o in loop.operations])
print
-
assert expected != "crash!", "should have raised an exception"
self.assert_equal(loop, expected)
if expected_preamble:
self.assert_equal(loop.preamble, expected_preamble,
text_right='expected preamble')
-
return loop
-class OptimizeOptTest(BaseTestOptimizeOpt):
+class OptimizeOptTest(BaseTestWithUnroll):
def setup_method(self, meth=None):
class FailDescr(compile.ResumeGuardDescr):
@@ -5953,3 +5837,30 @@
jump(i3, i4)
"""
self.optimize_loop(ops, expected)
+
+ def test_forced_virtual_pure_getfield(self):
+ ops = """
+ [p0]
+ p1 = getfield_gc_pure(p0, descr=valuedescr)
+ jump(p1)
+ """
+ self.optimize_loop(ops, ops)
+
+ ops = """
+ [p0]
+ p1 = new_with_vtable(ConstClass(node_vtable))
+ setfield_gc(p1, p0, descr=valuedescr)
+ escape(p1)
+ p2 = getfield_gc_pure(p1, descr=valuedescr)
+ escape(p2)
+ jump(p0)
+ """
+ expected = """
+ [p0]
+ p1 = new_with_vtable(ConstClass(node_vtable))
+ setfield_gc(p1, p0, descr=valuedescr)
+ escape(p1)
+ escape(p0)
+ jump(p0)
+ """
+ self.optimize_loop(ops, expected)
\ No newline at end of file
diff --git a/pypy/jit/metainterp/test/test_optimizeutil.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
rename from pypy/jit/metainterp/test/test_optimizeutil.py
rename to pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/test/test_optimizeutil.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -9,11 +9,15 @@
from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr,
Const, TreeLoop, BoxObj,
ConstObj, AbstractDescr)
-from pypy.jit.metainterp.optimizeutil import sort_descrs, InvalidLoop
+from pypy.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
+from pypy.jit.metainterp.optimize import InvalidLoop
from pypy.jit.codewriter.effectinfo import EffectInfo
from pypy.jit.codewriter.heaptracker import register_known_gctype, adr2int
-from pypy.jit.tool.oparser import parse
+from pypy.jit.tool.oparser import parse, pure_parse
from pypy.jit.metainterp.quasiimmut import QuasiImmutDescr
+from pypy.jit.metainterp import compile, resume, history
+from pypy.jit.metainterp.jitprof import EmptyProfiler
+from pypy.config.pypyoption import get_pypy_config
def test_sort_descrs():
class PseudoDescr(AbstractDescr):
@@ -28,6 +32,44 @@
sort_descrs(lst2)
assert lst2 == lst
+def test_equaloplists():
+ ops = """
+ [i0]
+ i1 = int_add(i0, 1)
+ i2 = int_add(i1, 1)
+ guard_true(i1) [i2]
+ jump(i1)
+ """
+ namespace = {}
+ loop1 = pure_parse(ops, namespace=namespace)
+ loop2 = pure_parse(ops, namespace=namespace)
+ loop3 = pure_parse(ops.replace("i2 = int_add", "i2 = int_sub"),
+ namespace=namespace)
+ assert equaloplists(loop1.operations, loop2.operations)
+ py.test.raises(AssertionError,
+ "equaloplists(loop1.operations, loop3.operations)")
+
+def test_equaloplists_fail_args():
+ ops = """
+ [i0]
+ i1 = int_add(i0, 1)
+ i2 = int_add(i1, 1)
+ guard_true(i1) [i2, i1]
+ jump(i1)
+ """
+ namespace = {}
+ loop1 = pure_parse(ops, namespace=namespace)
+ loop2 = pure_parse(ops.replace("[i2, i1]", "[i1, i2]"),
+ namespace=namespace)
+ py.test.raises(AssertionError,
+ "equaloplists(loop1.operations, loop2.operations)")
+ assert equaloplists(loop1.operations, loop2.operations,
+ strict_fail_args=False)
+ loop3 = pure_parse(ops.replace("[i2, i1]", "[i2, i0]"),
+ namespace=namespace)
+ py.test.raises(AssertionError,
+ "equaloplists(loop1.operations, loop3.operations)")
+
# ____________________________________________________________
class LLtypeMixin(object):
@@ -256,8 +298,45 @@
## u_vtable_adr: cpu.typedescrof(U)}
## namespace = locals()
+# ____________________________________________________________
+
+
+
+class Fake(object):
+ failargs_limit = 1000
+ storedebug = None
+
+
+class FakeMetaInterpStaticData(object):
+
+ def __init__(self, cpu):
+ self.cpu = cpu
+ self.profiler = EmptyProfiler()
+ self.options = Fake()
+ self.globaldata = Fake()
+ self.config = get_pypy_config(translating=True)
+ self.config.translation.jit_ffi = True
+
+
+class Storage(compile.ResumeGuardDescr):
+ "for tests."
+ def __init__(self, metainterp_sd=None, original_greenkey=None):
+ self.metainterp_sd = metainterp_sd
+ self.original_greenkey = original_greenkey
+ def store_final_boxes(self, op, boxes):
+ op.setfailargs(boxes)
+ def __eq__(self, other):
+ return type(self) is type(other) # xxx obscure
+ def clone_if_mutable(self):
+ res = Storage(self.metainterp_sd, self.original_greenkey)
+ self.copy_all_attributes_into(res)
+ return res
+
+def _sortboxes(boxes):
+ _kind2count = {history.INT: 1, history.REF: 2, history.FLOAT: 3}
+ return sorted(boxes, key=lambda box: _kind2count[box.type])
+
class BaseTest(object):
- invent_fail_descr = None
def parse(self, s, boxkinds=None):
return parse(s, self.cpu, self.namespace,
@@ -265,5 +344,40 @@
boxkinds=boxkinds,
invent_fail_descr=self.invent_fail_descr)
+ def invent_fail_descr(self, model, fail_args):
+ if fail_args is None:
+ return None
+ descr = Storage()
+ descr.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
+ descr.rd_snapshot = resume.Snapshot(None, _sortboxes(fail_args))
+ return descr
+
+ def assert_equal(self, optimized, expected, text_right=None):
+ from pypy.jit.metainterp.optimizeopt.util import equaloplists
+ assert len(optimized.inputargs) == len(expected.inputargs)
+ remap = {}
+ for box1, box2 in zip(optimized.inputargs, expected.inputargs):
+ assert box1.__class__ == box2.__class__
+ remap[box2] = box1
+ assert equaloplists(optimized.operations,
+ expected.operations, False, remap, text_right)
+
+ def _do_optimize_loop(self, loop, call_pure_results):
+ from pypy.jit.metainterp.optimizeopt import optimize_loop_1
+ from pypy.jit.metainterp.optimizeopt.util import args_dict
+
+ self.loop = loop
+ loop.call_pure_results = args_dict()
+ if call_pure_results is not None:
+ for k, v in call_pure_results.items():
+ loop.call_pure_results[list(k)] = v
+ metainterp_sd = FakeMetaInterpStaticData(self.cpu)
+ if hasattr(self, 'vrefinfo'):
+ metainterp_sd.virtualref_info = self.vrefinfo
+ if hasattr(self, 'callinfocollection'):
+ metainterp_sd.callinfocollection = self.callinfocollection
+ #
+ optimize_loop_1(metainterp_sd, loop, self.enable_opts)
+
# ____________________________________________________________
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -5,7 +5,7 @@
from pypy.jit.metainterp.resume import Snapshot
from pypy.jit.metainterp.history import TreeLoop, LoopToken
from pypy.rlib.debug import debug_start, debug_stop, debug_print
-from pypy.jit.metainterp.optimizeutil import InvalidLoop, RetraceLoop
+from pypy.jit.metainterp.optimize import InvalidLoop, RetraceLoop
from pypy.jit.metainterp.jitexc import JitException
from pypy.jit.metainterp.history import make_hashable_int
from pypy.jit.codewriter.effectinfo import EffectInfo
diff --git a/pypy/jit/metainterp/optimizeutil.py b/pypy/jit/metainterp/optimizeopt/util.py
rename from pypy/jit/metainterp/optimizeutil.py
rename to pypy/jit/metainterp/optimizeopt/util.py
--- a/pypy/jit/metainterp/optimizeutil.py
+++ b/pypy/jit/metainterp/optimizeopt/util.py
@@ -1,21 +1,10 @@
+import py
from pypy.rlib.objectmodel import r_dict, compute_identity_hash
from pypy.rlib.rarithmetic import intmask
from pypy.rlib.unroll import unrolling_iterable
from pypy.jit.metainterp import resoperation, history
-from pypy.jit.metainterp.jitexc import JitException
from pypy.rlib.debug import make_sure_not_resized
-
-class InvalidLoop(JitException):
- """Raised when the optimize*.py detect that the loop that
- we are trying to build cannot possibly make sense as a
- long-running loop (e.g. it cannot run 2 complete iterations)."""
-
-class RetraceLoop(JitException):
- """ Raised when inlining a short preamble resulted in an
- InvalidLoop. This means the optimized loop is too specialized
- to be useful here, so we trace it again and produced a second
- copy specialized in some different way.
- """
+from pypy.jit.metainterp.resoperation import rop
# ____________________________________________________________
# Misc. utilities
@@ -113,3 +102,49 @@
def args_dict_box():
return r_dict(args_eq, args_hash)
+
+
+# ____________________________________________________________
+
+def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={},
+ text_right=None):
+ # try to use the full width of the terminal to display the list
+ # unfortunately, does not work with the default capture method of py.test
+ # (which is fd), you you need to use either -s or --capture=sys, else you
+ # get the standard 80 columns width
+ totwidth = py.io.get_terminal_width()
+ width = totwidth / 2 - 1
+ print ' Comparing lists '.center(totwidth, '-')
+ text_right = text_right or 'expected'
+ print '%s| %s' % ('optimized'.center(width), text_right.center(width))
+ for op1, op2 in zip(oplist1, oplist2):
+ txt1 = str(op1)
+ txt2 = str(op2)
+ while txt1 or txt2:
+ print '%s| %s' % (txt1[:width].ljust(width), txt2[:width])
+ txt1 = txt1[width:]
+ txt2 = txt2[width:]
+ assert op1.getopnum() == op2.getopnum()
+ assert op1.numargs() == op2.numargs()
+ for i in range(op1.numargs()):
+ x = op1.getarg(i)
+ y = op2.getarg(i)
+ assert x == remap.get(y, y)
+ if op2.result in remap:
+ assert op1.result == remap[op2.result]
+ else:
+ remap[op2.result] = op1.result
+ if op1.getopnum() != rop.JUMP: # xxx obscure
+ assert op1.getdescr() == op2.getdescr()
+ if op1.getfailargs() or op2.getfailargs():
+ assert len(op1.getfailargs()) == len(op2.getfailargs())
+ if strict_fail_args:
+ for x, y in zip(op1.getfailargs(), op2.getfailargs()):
+ assert x == remap.get(y, y)
+ else:
+ fail_args1 = set(op1.getfailargs())
+ fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
+ assert fail_args1 == fail_args2
+ assert len(oplist1) == len(oplist2)
+ print '-'*totwidth
+ return True
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -1,7 +1,7 @@
from pypy.jit.metainterp.history import Const, ConstInt, BoxInt
from pypy.jit.metainterp.resoperation import rop, ResOperation
-from pypy.jit.metainterp.optimizeutil import _findall, sort_descrs
-from pypy.jit.metainterp.optimizeutil import descrlist_dict
+from pypy.jit.metainterp.optimizeopt.util import _findall, sort_descrs
+from pypy.jit.metainterp.optimizeopt.util import descrlist_dict
from pypy.rlib.objectmodel import we_are_translated
from pypy.jit.metainterp.optimizeopt import optimizer
from pypy.jit.metainterp.executor import execute
@@ -20,6 +20,9 @@
self.source_op = source_op # the NEW_WITH_VTABLE/NEW_ARRAY operation
# that builds this box
+ def is_forced_virtual(self):
+ return self.box is not None
+
def get_key_box(self):
if self.box is None:
return self.keybox
@@ -120,7 +123,6 @@
op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
descr=ofs)
newoperations.append(op)
- self._fields = None
def _get_field_descr_list(self):
_cached_sorted_fields = self._cached_sorted_fields
@@ -351,7 +353,7 @@
if not self.optimizer.cpu.ts.CONST_NULL.same_constant(objbox):
seo(ResOperation(rop.SETFIELD_GC, op.getarglist(), None,
descr = vrefinfo.descr_forced))
-
+
# - set 'virtual_token' to TOKEN_NONE
args = [op.getarg(0), ConstInt(vrefinfo.TOKEN_NONE)]
seo(ResOperation(rop.SETFIELD_GC, args, None,
@@ -365,6 +367,14 @@
def optimize_GETFIELD_GC(self, op):
value = self.getvalue(op.getarg(0))
+ # If this is an immutable field (as indicated by op.is_always_pure())
+ # then it's safe to reuse the virtual's field, even if it has been
+ # forced, because it should never be written to again.
+ if value.is_forced_virtual() and op.is_always_pure():
+ fieldvalue = value.getfield(op.getdescr(), None)
+ if fieldvalue is not None:
+ self.make_equal_to(op.result, fieldvalue)
+ return
if value.is_virtual():
assert isinstance(value, AbstractVirtualValue)
fieldvalue = value.getfield(op.getdescr(), None)
@@ -382,6 +392,7 @@
def optimize_SETFIELD_GC(self, op):
value = self.getvalue(op.getarg(0))
+
if value.is_virtual():
fieldvalue = self.getvalue(op.getarg(1))
value.setfield(op.getdescr(), fieldvalue)
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -21,7 +21,8 @@
from pypy.rlib.objectmodel import specialize
from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr, MissingLiveness
from pypy.jit.codewriter import heaptracker, longlong
-from pypy.jit.metainterp.optimizeutil import RetraceLoop, args_dict_box, args_dict
+from pypy.jit.metainterp.optimizeopt.util import args_dict_box, args_dict
+from pypy.jit.metainterp.optimize import RetraceLoop
# ____________________________________________________________
@@ -1262,8 +1263,7 @@
logger_ops = None
def __init__(self, cpu, options,
- ProfilerClass=EmptyProfiler, warmrunnerdesc=None,
- jit_ffi=True):
+ ProfilerClass=EmptyProfiler, warmrunnerdesc=None):
self.cpu = cpu
self.stats = self.cpu.stats
self.options = options
@@ -1273,7 +1273,11 @@
self.profiler = ProfilerClass()
self.profiler.cpu = cpu
self.warmrunnerdesc = warmrunnerdesc
- self.jit_ffi = jit_ffi
+ if warmrunnerdesc:
+ self.config = warmrunnerdesc.translator.config
+ else:
+ from pypy.config.pypyoption import get_pypy_config
+ self.config = get_pypy_config(translating=True)
backendmodule = self.cpu.__module__
backendmodule = backendmodule.split('.')[-2]
@@ -1924,7 +1928,6 @@
self.history.inputargs = original_inputargs
self.history.operations.pop() # remove the JUMP
- # FIXME: Why is self.history.inputargs not restored?
def compile_bridge(self, live_arg_boxes):
num_green_args = self.jitdriver_sd.num_green_args
@@ -1960,6 +1963,8 @@
start_resumedescr, False)
self.history.operations.pop() # remove the JUMP
if loop_token is None:
+ self.history.inputargs = original_inputargs
+ self.history.operations = original_operations
return
if loop_token.short_preamble:
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -191,9 +191,15 @@
# of the operation. It must inherit from AbstractDescr. The
# backend provides it with cpu.fielddescrof(), cpu.arraydescrof(),
# cpu.calldescrof(), and cpu.typedescrof().
+ self._check_descr(descr)
+ self._descr = descr
+
+ def _check_descr(self, descr):
+ if not we_are_translated() and getattr(descr, 'I_am_a_descr', False):
+ return # needed for the mock case in oparser_model
from pypy.jit.metainterp.history import check_descr
check_descr(descr)
- self._descr = descr
+
class GuardResOp(ResOpWithDescr):
@@ -471,8 +477,8 @@
'STRSETITEM/3',
'UNICODESETITEM/3',
#'RUNTIMENEW/1', # ootype operation
- 'COND_CALL_GC_WB/2d', # [objptr, newvalue] or [arrayptr, index]
- # (for the write barrier, latter is in an array)
+ 'COND_CALL_GC_WB/2d', # [objptr, newvalue] (for the write barrier)
+ 'COND_CALL_GC_WB_ARRAY/3d', # [objptr, arrayindex, newvalue] (write barr.)
'DEBUG_MERGE_POINT/*', # debugging only
'JIT_DEBUG/*', # debugging only
'VIRTUAL_REF_FINISH/2', # removed before it's passed to the backend
diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py
--- a/pypy/jit/metainterp/resume.py
+++ b/pypy/jit/metainterp/resume.py
@@ -10,7 +10,7 @@
from pypy.rlib.objectmodel import we_are_translated, specialize
from pypy.rlib.debug import have_debug_prints, ll_assert
from pypy.rlib.debug import debug_start, debug_stop, debug_print
-from pypy.jit.metainterp.optimizeutil import InvalidLoop
+from pypy.jit.metainterp.optimize import InvalidLoop
# Logic to encode the chain of frames and the state of the boxes at a
# guard operation, and to decode it again. This is a bit advanced,
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -500,7 +500,7 @@
y -= x
return y
#
- res = self.meta_interp(f, [3, 6], repeat=7)
+ res = self.meta_interp(f, [3, 6], repeat=7, function_threshold=0)
assert res == 6 - 4 - 5
self.check_history(call=0) # because the trace starts in the middle
#
@@ -2230,6 +2230,72 @@
self.check_loops(getfield_gc_pure=0)
self.check_loops(getfield_gc_pure=2, everywhere=True)
+ def test_frame_finished_during_retrace(self):
+ class Base(object):
+ pass
+ class A(Base):
+ def __init__(self, a):
+ self.val = a
+ self.num = 1
+ def inc(self):
+ return A(self.val + 1)
+ class B(Base):
+ def __init__(self, a):
+ self.val = a
+ self.num = 1000
+ def inc(self):
+ return B(self.val + 1)
+ myjitdriver = JitDriver(greens = [], reds = ['sa', 'a'])
+ def f():
+ myjitdriver.set_param('threshold', 3)
+ myjitdriver.set_param('trace_eagerness', 2)
+ a = A(0)
+ sa = 0
+ while a.val < 8:
+ myjitdriver.jit_merge_point(a=a, sa=sa)
+ a = a.inc()
+ if a.val > 4:
+ a = B(a.val)
+ sa += a.num
+ return sa
+ res = self.meta_interp(f, [])
+ assert res == f()
+
+ def test_frame_finished_during_continued_retrace(self):
+ class Base(object):
+ pass
+ class A(Base):
+ def __init__(self, a):
+ self.val = a
+ self.num = 100
+ def inc(self):
+ return A(self.val + 1)
+ class B(Base):
+ def __init__(self, a):
+ self.val = a
+ self.num = 10000
+ def inc(self):
+ return B(self.val + 1)
+ myjitdriver = JitDriver(greens = [], reds = ['sa', 'b', 'a'])
+ def f(b):
+ myjitdriver.set_param('threshold', 6)
+ myjitdriver.set_param('trace_eagerness', 4)
+ a = A(0)
+ sa = 0
+ while a.val < 15:
+ myjitdriver.jit_merge_point(a=a, b=b, sa=sa)
+ a = a.inc()
+ if a.val > 8:
+ a = B(a.val)
+ if b == 1:
+ b = 2
+ else:
+ b = 1
+ sa += a.num + b
+ return sa
+ res = self.meta_interp(f, [1])
+ assert res == f(1)
+
class TestOOtype(BasicTests, OOJitMixin):
def test_oohash(self):
diff --git a/pypy/jit/metainterp/test/test_compile.py b/pypy/jit/metainterp/test/test_compile.py
--- a/pypy/jit/metainterp/test/test_compile.py
+++ b/pypy/jit/metainterp/test/test_compile.py
@@ -1,3 +1,4 @@
+from pypy.config.pypyoption import get_pypy_config
from pypy.jit.metainterp.history import LoopToken, ConstInt, History, Stats
from pypy.jit.metainterp.history import BoxInt, INT
from pypy.jit.metainterp.compile import insert_loop_token, compile_new_loop
@@ -5,7 +6,7 @@
from pypy.jit.metainterp.compile import ResumeGuardCountersInt
from pypy.jit.metainterp.compile import compile_tmp_callback
from pypy.jit.metainterp import jitprof, typesystem, compile
-from pypy.jit.metainterp.test.test_optimizeutil import LLtypeMixin
+from pypy.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
from pypy.jit.tool.oparser import parse
from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT
@@ -57,11 +58,11 @@
logger_noopt = FakeLogger()
logger_ops = FakeLogger()
+ config = get_pypy_config(translating=True)
stats = Stats()
profiler = jitprof.EmptyProfiler()
warmrunnerdesc = None
- jit_ffi = False
def log(self, msg, event_kind=None):
pass
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -130,6 +130,38 @@
assert res == 50
self.check_loops(int_mod=1)
+ def test_repeated_lookup(self):
+ myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
+ class Wrapper(object):
+ _immutable_fields_ = ["value"]
+ def __init__(self, value):
+ self.value = value
+ def eq_func(a, b):
+ return a.value == b.value
+ def hash_func(x):
+ return objectmodel.compute_hash(x.value)
+
+ def f(n):
+ d = None
+ while n > 0:
+ myjitdriver.jit_merge_point(n=n, d=d)
+ d = objectmodel.r_dict(eq_func, hash_func)
+ y = Wrapper(str(n))
+ d[y] = n - 1
+ n = d[y]
+ return d[Wrapper(str(n + 1))]
+
+ res = self.meta_interp(f, [100], listops=True)
+ assert res == f(50)
+ # XXX: ideally there would be 7 calls here, but repeated CALL_PURE with
+ # the same arguments are not folded, because we have conflicting
+ # definitions of pure, once strhash can be appropriately folded
+ # this should be decreased to seven.
+ self.check_loops({"call": 8, "guard_false": 1, "guard_no_exception": 5,
+ "guard_true": 1, "int_and": 1, "int_gt": 1,
+ "int_is_true": 1, "int_sub": 1, "jump": 1,
+ "new_with_vtable": 1, "setfield_gc": 1})
+
class TestOOtype(DictTests, OOJitMixin):
pass
diff --git a/pypy/jit/metainterp/test/test_jitdriver.py b/pypy/jit/metainterp/test/test_jitdriver.py
--- a/pypy/jit/metainterp/test/test_jitdriver.py
+++ b/pypy/jit/metainterp/test/test_jitdriver.py
@@ -113,6 +113,7 @@
return n
#
def loop2(g, r):
+ myjitdriver1.set_param('function_threshold', 0)
while r > 0:
myjitdriver2.can_enter_jit(g=g, r=r)
myjitdriver2.jit_merge_point(g=g, r=r)
diff --git a/pypy/jit/metainterp/test/test_logger.py b/pypy/jit/metainterp/test/test_logger.py
--- a/pypy/jit/metainterp/test/test_logger.py
+++ b/pypy/jit/metainterp/test/test_logger.py
@@ -4,7 +4,7 @@
from pypy.jit.metainterp import logger
from pypy.jit.metainterp.typesystem import llhelper
from StringIO import StringIO
-from pypy.jit.metainterp.test.test_optimizeopt import equaloplists
+from pypy.jit.metainterp.optimizeopt.util import equaloplists
from pypy.jit.metainterp.history import AbstractDescr, LoopToken, BasicFailDescr
from pypy.jit.backend.model import AbstractCPU
@@ -53,7 +53,7 @@
def make_metainterp_sd(self):
class FakeJitDriver(object):
class warmstate(object):
- get_location_str = staticmethod(lambda args: args[0]._get_str())
+ get_location_str = staticmethod(lambda args: "dupa")
class FakeMetaInterpSd:
cpu = AbstractCPU()
@@ -116,10 +116,10 @@
def test_debug_merge_point(self):
inp = '''
[]
- debug_merge_point(0, 0, "dupa")
+ debug_merge_point(0, 0)
'''
_, loop, oloop = self.reparse(inp, check_equal=False)
- assert loop.operations[0].getarg(2)._get_str() == "dupa"
+ assert loop.operations[0].getarg(1).getint() == 0
assert oloop.operations[0].getarg(1)._get_str() == "dupa"
def test_floats(self):
diff --git a/pypy/jit/metainterp/test/test_pyjitpl.py b/pypy/jit/metainterp/test/test_pyjitpl.py
--- a/pypy/jit/metainterp/test/test_pyjitpl.py
+++ b/pypy/jit/metainterp/test/test_pyjitpl.py
@@ -6,7 +6,7 @@
from pypy.jit.metainterp.history import BoxInt, ConstInt
from pypy.jit.metainterp.history import History
from pypy.jit.metainterp.resoperation import ResOperation, rop
-from pypy.jit.metainterp.test.test_optimizeopt import equaloplists
+from pypy.jit.metainterp.optimizeopt.util import equaloplists
from pypy.jit.codewriter.jitcode import JitCode
diff --git a/pypy/jit/metainterp/test/test_recursive.py b/pypy/jit/metainterp/test/test_recursive.py
--- a/pypy/jit/metainterp/test/test_recursive.py
+++ b/pypy/jit/metainterp/test/test_recursive.py
@@ -483,6 +483,7 @@
def main(inline):
myjitdriver.set_param("threshold", 10)
+ myjitdriver.set_param('function_threshold', 60)
if inline:
myjitdriver.set_param('inlining', True)
else:
@@ -1193,6 +1194,51 @@
i -= 1
self.meta_interp(portal, [0, 10], inline=True)
+ def test_trace_from_start_always(self):
+ from pypy.rlib.nonconst import NonConstant
+
+ driver = JitDriver(greens = ['c'], reds = ['i', 'v'])
+
+ def portal(c, i, v):
+ while i > 0:
+ driver.jit_merge_point(c=c, i=i, v=v)
+ portal(c, i - 1, v)
+ if v:
+ driver.can_enter_jit(c=c, i=i, v=v)
+ break
+
+ def main(c, i, set_param, v):
+ if set_param:
+ driver.set_param('function_threshold', 0)
+ portal(c, i, v)
+
+ self.meta_interp(main, [10, 10, False, False], inline=True)
+ self.check_tree_loop_count(1)
+ self.check_loop_count(0)
+ self.meta_interp(main, [3, 10, True, False], inline=True)
+ self.check_tree_loop_count(0)
+ self.check_loop_count(0)
+
+ def test_trace_from_start_does_not_prevent_inlining(self):
+ driver = JitDriver(greens = ['c', 'bc'], reds = ['i'])
+
+ def portal(bc, c, i):
+ while True:
+ driver.jit_merge_point(c=c, bc=bc, i=i)
+ if bc == 0:
+ portal(1, 8, 0)
+ c += 1
+ else:
+ return
+ if c == 10: # bc == 0
+ c = 0
+ if i >= 100:
+ return
+ driver.can_enter_jit(c=c, bc=bc, i=i)
+ i += 1
+
+ self.meta_interp(portal, [0, 0, 0], inline=True)
+ self.check_loops(call=0, call_may_force=0)
class TestLLtype(RecursiveTests, LLJitMixin):
pass
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -6,7 +6,7 @@
from pypy.jit.metainterp.resume import *
from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt
from pypy.jit.metainterp.history import ConstPtr, ConstFloat
-from pypy.jit.metainterp.test.test_optimizeutil import LLtypeMixin
+from pypy.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
from pypy.jit.metainterp import executor
from pypy.jit.codewriter import heaptracker, longlong
diff --git a/pypy/jit/metainterp/test/test_virtualizable.py b/pypy/jit/metainterp/test/test_virtualizable.py
--- a/pypy/jit/metainterp/test/test_virtualizable.py
+++ b/pypy/jit/metainterp/test/test_virtualizable.py
@@ -11,7 +11,7 @@
from pypy.rpython.rclass import FieldListAccessor
from pypy.jit.metainterp.warmspot import get_stats, get_translator
from pypy.jit.metainterp import history
-from pypy.jit.metainterp.test.test_optimizeutil import LLtypeMixin
+from pypy.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
def promote_virtualizable(*args):
pass
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -66,6 +66,7 @@
def jittify_and_run(interp, graph, args, repeat=1,
backendopt=False, trace_limit=sys.maxint,
inline=False, loop_longevity=0, retrace_limit=5,
+ function_threshold=4,
enable_opts=ALL_OPTS_NAMES, **kwds):
from pypy.config.config import ConfigError
translator = interp.typer.annotator.translator
@@ -77,9 +78,14 @@
translator.config.translation.list_comprehension_operations = True
except ConfigError:
pass
+ try:
+ translator.config.translation.jit_ffi = True
+ except ConfigError:
+ pass
warmrunnerdesc = WarmRunnerDesc(translator, backendopt=backendopt, **kwds)
for jd in warmrunnerdesc.jitdrivers_sd:
jd.warmstate.set_param_threshold(3) # for tests
+ jd.warmstate.set_param_function_threshold(function_threshold)
jd.warmstate.set_param_trace_eagerness(2) # for tests
jd.warmstate.set_param_trace_limit(trace_limit)
jd.warmstate.set_param_inlining(inline)
@@ -422,7 +428,7 @@
if self.translator.rtyper.type_system.name == 'lltypesystem':
def maybe_enter_jit(*args):
try:
- maybe_compile_and_run(*args)
+ maybe_compile_and_run(state.increment_threshold, *args)
except JitException:
raise # go through
except Exception, e:
@@ -430,15 +436,13 @@
maybe_enter_jit._always_inline_ = True
else:
def maybe_enter_jit(*args):
- maybe_compile_and_run(*args)
+ maybe_compile_and_run(state.increment_threshold, *args)
maybe_enter_jit._always_inline_ = True
jd._maybe_enter_jit_fn = maybe_enter_jit
- can_inline = state.can_inline_greenargs
num_green_args = jd.num_green_args
def maybe_enter_from_start(*args):
- if not can_inline(*args[:num_green_args]):
- maybe_compile_and_run(*args)
+ maybe_compile_and_run(state.increment_function_threshold, *args)
maybe_enter_from_start._always_inline_ = True
jd._maybe_enter_from_start_fn = maybe_enter_from_start
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -208,15 +208,20 @@
meth = getattr(self, 'set_param_' + name)
meth(default_value)
- def set_param_threshold(self, threshold):
+ def _compute_threshold(self, threshold):
if threshold <= 0:
- self.increment_threshold = 0 # never reach the THRESHOLD_LIMIT
- return
+ return 0 # never reach the THRESHOLD_LIMIT
if threshold < 2:
threshold = 2
- self.increment_threshold = (self.THRESHOLD_LIMIT // threshold) + 1
+ return (self.THRESHOLD_LIMIT // threshold) + 1
# the number is at least 1, and at most about half THRESHOLD_LIMIT
+ def set_param_threshold(self, threshold):
+ self.increment_threshold = self._compute_threshold(threshold)
+
+ def set_param_function_threshold(self, threshold):
+ self.increment_function_threshold = self._compute_threshold(threshold)
+
def set_param_trace_eagerness(self, value):
self.trace_eagerness = value
@@ -291,7 +296,7 @@
self.make_jitdriver_callbacks()
confirm_enter_jit = self.confirm_enter_jit
- def maybe_compile_and_run(*args):
+ def maybe_compile_and_run(threshold, *args):
"""Entry point to the JIT. Called at the point with the
can_enter_jit() hint.
"""
@@ -307,7 +312,7 @@
if cell.counter >= 0:
# update the profiling counter
- n = cell.counter + self.increment_threshold
+ n = cell.counter + threshold
if n <= self.THRESHOLD_LIMIT: # bound not reached
cell.counter = n
return
diff --git a/pypy/jit/tool/oparser.py b/pypy/jit/tool/oparser.py
--- a/pypy/jit/tool/oparser.py
+++ b/pypy/jit/tool/oparser.py
@@ -3,24 +3,15 @@
in a nicer fashion
"""
-from pypy.jit.metainterp.history import TreeLoop, BoxInt, ConstInt,\
- ConstObj, ConstPtr, Box, BasicFailDescr, BoxFloat, ConstFloat,\
- LoopToken, get_const_ptr_for_string, get_const_ptr_for_unicode
+from pypy.jit.tool.oparser_model import get_model
+
from pypy.jit.metainterp.resoperation import rop, ResOperation, \
ResOpWithDescr, N_aryOp, \
UnaryOp, PlainResOp
-from pypy.jit.metainterp.typesystem import llhelper
-from pypy.jit.codewriter.heaptracker import adr2int
-from pypy.jit.codewriter import longlong
-from pypy.rpython.lltypesystem import lltype, llmemory
-from pypy.rpython.ootypesystem import ootype
class ParseError(Exception):
pass
-class Boxes(object):
- pass
-
class ESCAPE_OP(N_aryOp, ResOpWithDescr):
OPNUM = -123
@@ -54,37 +45,15 @@
def clone(self):
return FORCE_SPILL(self.OPNUM, self.getarglist()[:])
-class ExtendedTreeLoop(TreeLoop):
- def getboxes(self):
- def opboxes(operations):
- for op in operations:
- yield op.result
- for box in op.getarglist():
- yield box
- def allboxes():
- for box in self.inputargs:
- yield box
- for box in opboxes(self.operations):
- yield box
-
- boxes = Boxes()
- for box in allboxes():
- if isinstance(box, Box):
- name = str(box)
- setattr(boxes, name, box)
- return boxes
-
- def setvalues(self, **kwds):
- boxes = self.getboxes()
- for name, value in kwds.iteritems():
- getattr(boxes, name).value = value
-
-def default_fail_descr(fail_args=None):
- return BasicFailDescr()
+def default_fail_descr(model, fail_args=None):
+ return model.BasicFailDescr()
class OpParser(object):
+
+ use_mock_model = False
+
def __init__(self, input, cpu, namespace, type_system, boxkinds,
invent_fail_descr=default_fail_descr,
nonstrict=False):
@@ -100,7 +69,8 @@
self._cache = {}
self.invent_fail_descr = invent_fail_descr
self.nonstrict = nonstrict
- self.looptoken = LoopToken()
+ self.model = get_model(self.use_mock_model)
+ self.looptoken = self.model.LoopToken()
def get_const(self, name, typ):
if self._consts is None:
@@ -108,16 +78,16 @@
obj = self._consts[name]
if self.type_system == 'lltype':
if typ == 'ptr':
- return ConstPtr(obj)
+ return self.model.ConstPtr(obj)
else:
assert typ == 'class'
- return ConstInt(adr2int(llmemory.cast_ptr_to_adr(obj)))
+ return self.model.ConstInt(self.model.ptr_to_int(obj))
else:
if typ == 'ptr':
- return ConstObj(obj)
+ return self.model.ConstObj(obj)
else:
assert typ == 'class'
- return ConstObj(ootype.cast_to_object(obj))
+ return self.model.ConstObj(ootype.cast_to_object(obj))
def get_descr(self, poss_descr):
if poss_descr.startswith('<'):
@@ -132,16 +102,16 @@
pass
if elem.startswith('i'):
# integer
- box = BoxInt()
- _box_counter_more_than(elem[1:])
+ box = self.model.BoxInt()
+ _box_counter_more_than(self.model, elem[1:])
elif elem.startswith('f'):
- box = BoxFloat()
- _box_counter_more_than(elem[1:])
+ box = self.model.BoxFloat()
+ _box_counter_more_than(self.model, elem[1:])
elif elem.startswith('p'):
# pointer
- ts = getattr(self.cpu, 'ts', llhelper)
+ ts = getattr(self.cpu, 'ts', self.model.llhelper)
box = ts.BoxRef()
- _box_counter_more_than(elem[1:])
+ _box_counter_more_than(self.model, elem[1:])
else:
for prefix, boxclass in self.boxkinds.iteritems():
if elem.startswith(prefix):
@@ -175,21 +145,21 @@
def getvar(self, arg):
if not arg:
- return ConstInt(0)
+ return self.model.ConstInt(0)
try:
- return ConstInt(int(arg))
+ return self.model.ConstInt(int(arg))
except ValueError:
if self.is_float(arg):
- return ConstFloat(longlong.getfloatstorage(float(arg)))
+ return self.model.ConstFloat(self.model.convert_to_floatstorage(arg))
if (arg.startswith('"') or arg.startswith("'") or
arg.startswith('s"')):
# XXX ootype
info = arg[1:].strip("'\"")
- return get_const_ptr_for_string(info)
+ return self.model.get_const_ptr_for_string(info)
if arg.startswith('u"'):
# XXX ootype
info = arg[1:].strip("'\"")
- return get_const_ptr_for_unicode(info)
+ return self.model.get_const_ptr_for_unicode(info)
if arg.startswith('ConstClass('):
name = arg[len('ConstClass('):-1]
return self.get_const(name, 'class')
@@ -197,9 +167,9 @@
return None
elif arg == 'NULL':
if self.type_system == 'lltype':
- return ConstPtr(ConstPtr.value)
+ return self.model.ConstPtr(self.model.ConstPtr.value)
else:
- return ConstObj(ConstObj.value)
+ return self.model.ConstObj(self.model.ConstObj.value)
elif arg.startswith('ConstPtr('):
name = arg[len('ConstPtr('):-1]
return self.get_const(name, 'ptr')
@@ -211,11 +181,8 @@
args = []
descr = None
if argspec.strip():
- if opname == 'debug_merge_point':
- allargs = argspec.split(',', 2)
- else:
- allargs = [arg for arg in argspec.split(",")
- if arg != '']
+ allargs = [arg for arg in argspec.split(",")
+ if arg != '']
poss_descr = allargs[-1].strip()
if poss_descr.startswith('descr='):
@@ -266,14 +233,14 @@
"Unknown var in fail_args: %s" % arg)
fail_args.append(fail_arg)
if descr is None and self.invent_fail_descr:
- descr = self.invent_fail_descr(fail_args)
+ descr = self.invent_fail_descr(self.model, fail_args)
if hasattr(descr, '_oparser_uses_descr_of_guard'):
descr._oparser_uses_descr_of_guard(self, fail_args)
else:
fail_args = None
if opnum == rop.FINISH:
if descr is None and self.invent_fail_descr:
- descr = self.invent_fail_descr()
+ descr = self.invent_fail_descr(self.model)
elif opnum == rop.JUMP:
if descr is None and self.invent_fail_descr:
descr = self.looptoken
@@ -338,7 +305,7 @@
num, ops, last_offset = self.parse_ops(base_indent, newlines, 0)
if num < len(newlines):
raise ParseError("unexpected dedent at line: %s" % newlines[num])
- loop = ExtendedTreeLoop("loop")
+ loop = self.model.ExtendedTreeLoop("loop")
loop.comment = first_comment
loop.token = self.looptoken
loop.operations = ops
@@ -394,7 +361,7 @@
def parse(input, cpu=None, namespace=None, type_system='lltype',
boxkinds=None, invent_fail_descr=default_fail_descr,
- no_namespace=False, nonstrict=False):
+ no_namespace=False, nonstrict=False, OpParser=OpParser):
if namespace is None and not no_namespace:
namespace = {}
return OpParser(input, cpu, namespace, type_system, boxkinds,
@@ -405,6 +372,6 @@
return parse(*args, **kwds)
-def _box_counter_more_than(s):
+def _box_counter_more_than(model, s):
if s.isdigit():
- Box._counter = max(Box._counter, int(s)+1)
+ model.Box._counter = max(model.Box._counter, int(s)+1)
diff --git a/pypy/jit/tool/oparser_model.py b/pypy/jit/tool/oparser_model.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/tool/oparser_model.py
@@ -0,0 +1,148 @@
+class Boxes(object):
+ pass
+
+def get_real_model():
+ class LoopModel(object):
+ from pypy.jit.metainterp.history import TreeLoop, LoopToken
+ from pypy.jit.metainterp.history import Box, BoxInt, BoxFloat
+ from pypy.jit.metainterp.history import ConstInt, ConstObj, ConstPtr, ConstFloat
+ from pypy.jit.metainterp.history import BasicFailDescr
+ from pypy.jit.metainterp.typesystem import llhelper
+
+ from pypy.jit.metainterp.history import get_const_ptr_for_string
+ from pypy.jit.metainterp.history import get_const_ptr_for_unicode
+ get_const_ptr_for_string = staticmethod(get_const_ptr_for_string)
+ get_const_ptr_for_unicode = staticmethod(get_const_ptr_for_unicode)
+
+ @staticmethod
+ def convert_to_floatstorage(arg):
+ from pypy.jit.codewriter import longlong
+ return longlong.getfloatstorage(float(arg))
+
+ @staticmethod
+ def ptr_to_int(obj):
+ from pypy.jit.codewriter.heaptracker import adr2int
+ from pypy.rpython.lltypesystem import llmemory
+ return adr2int(llmemory.cast_ptr_to_adr(obj))
+
+ @staticmethod
+ def ootype_cast_to_object(obj):
+ from pypy.rpython.ootypesystem import ootype
+ return ootype.cast_to_object(obj)
+
+ return LoopModel
+
+def get_mock_model():
+ class LoopModel(object):
+
+ class TreeLoop(object):
+ def __init__(self, name):
+ self.name = name
+
+ class LoopToken(object):
+ I_am_a_descr = True
+
+ class BasicFailDescr(object):
+ I_am_a_descr = True
+
+ class Box(object):
+ _counter = 0
+ type = 'b'
+
+ def __init__(self, value=0):
+ self.value = value
+
+ def __repr__(self):
+ result = str(self)
+ result += '(%s)' % self.value
+ return result
+
+ def __str__(self):
+ if not hasattr(self, '_str'):
+ self._str = '%s%d' % (self.type, Box._counter)
+ Box._counter += 1
+ return self._str
+
+ class BoxInt(Box):
+ type = 'i'
+
+ class BoxFloat(Box):
+ type = 'f'
+
+ class BoxRef(Box):
+ type = 'p'
+
+ class Const(object):
+ def __init__(self, value=None):
+ self.value = value
+
+ def _get_str(self):
+ return str(self.value)
+
+ class ConstInt(Const):
+ pass
+
+ class ConstPtr(Const):
+ pass
+
+ class ConstFloat(Const):
+ pass
+
+ @classmethod
+ def get_const_ptr_for_string(cls, s):
+ return cls.ConstPtr(s)
+
+ @classmethod
+ def get_const_ptr_for_unicode(cls, s):
+ return cls.ConstPtr(s)
+
+ @staticmethod
+ def convert_to_floatstorage(arg):
+ return float(arg)
+
+ @staticmethod
+ def ptr_to_int(obj):
+ return id(obj)
+
+ class llhelper(object):
+ pass
+
+ LoopModel.llhelper.BoxRef = LoopModel.BoxRef
+
+ return LoopModel
+
+
+def get_model(use_mock):
+ if use_mock:
+ model = get_mock_model()
+ else:
+ model = get_real_model()
+
+ class ExtendedTreeLoop(model.TreeLoop):
+
+ def getboxes(self):
+ def opboxes(operations):
+ for op in operations:
+ yield op.result
+ for box in op.getarglist():
+ yield box
+ def allboxes():
+ for box in self.inputargs:
+ yield box
+ for box in opboxes(self.operations):
+ yield box
+
+ boxes = Boxes()
+ for box in allboxes():
+ if isinstance(box, model.Box):
+ name = str(box)
+ setattr(boxes, name, box)
+ return boxes
+
+ def setvalues(self, **kwds):
+ boxes = self.getboxes()
+ for name, value in kwds.iteritems():
+ getattr(boxes, name).value = value
+
+ model.ExtendedTreeLoop = ExtendedTreeLoop
+ return model
diff --git a/pypy/jit/tool/test/test_oparser.py b/pypy/jit/tool/test/test_oparser.py
--- a/pypy/jit/tool/test/test_oparser.py
+++ b/pypy/jit/tool/test/test_oparser.py
@@ -1,227 +1,274 @@
import py
+import sys
from pypy.rpython.lltypesystem import lltype, llmemory
-from pypy.jit.tool.oparser import parse, ParseError
+from pypy.jit.tool.oparser import parse, OpParser
from pypy.jit.metainterp.resoperation import rop
-from pypy.jit.metainterp.history import AbstractDescr, BoxInt, LoopToken,\
- BoxFloat
+from pypy.jit.metainterp.history import AbstractDescr, BoxInt, LoopToken
-def test_basic_parse():
- x = """
- [i0, i1]
- # a comment
- i2 = int_add(i0, i1)
- i3 = int_sub(i2, 3) # another comment
- finish() # (tricky)
- """
- loop = parse(x)
- assert len(loop.operations) == 3
- assert [op.getopnum() for op in loop.operations] == [rop.INT_ADD, rop.INT_SUB,
- rop.FINISH]
- assert len(loop.inputargs) == 2
- assert loop.operations[-1].getdescr()
+class BaseTestOparser(object):
-def test_const_ptr_subops():
- x = """
- [p0]
- guard_class(p0, ConstClass(vtable)) []
- """
- S = lltype.Struct('S')
- vtable = lltype.nullptr(S)
- loop = parse(x, None, locals())
- assert len(loop.operations) == 1
- assert loop.operations[0].getdescr()
- assert loop.operations[0].getfailargs() == []
+ OpParser = None
-def test_descr():
- class Xyz(AbstractDescr):
- pass
-
- x = """
- [p0]
- i1 = getfield_gc(p0, descr=stuff)
- """
- stuff = Xyz()
- loop = parse(x, None, locals())
- assert loop.operations[0].getdescr() is stuff
+ def parse(self, *args, **kwds):
+ kwds['OpParser'] = self.OpParser
+ return parse(*args, **kwds)
-def test_after_fail():
- x = """
- [i0]
- guard_value(i0, 3) []
- i1 = int_add(1, 2)
- """
- loop = parse(x, None, {})
- assert len(loop.operations) == 2
+ def test_basic_parse(self):
+ x = """
+ [i0, i1]
+ # a comment
+ i2 = int_add(i0, i1)
+ i3 = int_sub(i2, 3) # another comment
+ finish() # (tricky)
+ """
+ loop = self.parse(x)
+ assert len(loop.operations) == 3
+ assert [op.getopnum() for op in loop.operations] == [rop.INT_ADD, rop.INT_SUB,
+ rop.FINISH]
+ assert len(loop.inputargs) == 2
+ assert loop.operations[-1].getdescr()
-def test_descr_setfield():
- class Xyz(AbstractDescr):
- pass
-
- x = """
- [p0]
- setfield_gc(p0, 3, descr=stuff)
- """
- stuff = Xyz()
- loop = parse(x, None, locals())
- assert loop.operations[0].getdescr() is stuff
+ def test_const_ptr_subops(self):
+ x = """
+ [p0]
+ guard_class(p0, ConstClass(vtable)) []
+ """
+ S = lltype.Struct('S')
+ vtable = lltype.nullptr(S)
+ loop = self.parse(x, None, locals())
+ assert len(loop.operations) == 1
+ assert loop.operations[0].getdescr()
+ assert loop.operations[0].getfailargs() == []
-def test_boxname():
- x = """
- [i42]
- i50 = int_add(i42, 1)
- """
- loop = parse(x, None, {})
- assert str(loop.inputargs[0]) == 'i42'
- assert str(loop.operations[0].result) == 'i50'
+ def test_descr(self):
+ class Xyz(AbstractDescr):
+ I_am_a_descr = True # for the mock case
-def test_getboxes():
- x = """
- [i0]
- i1 = int_add(i0, 10)
- """
- loop = parse(x, None, {})
- boxes = loop.getboxes()
- assert boxes.i0 is loop.inputargs[0]
- assert boxes.i1 is loop.operations[0].result
-
-def test_setvalues():
- x = """
- [i0]
- i1 = int_add(i0, 10)
- """
- loop = parse(x, None, {})
- loop.setvalues(i0=32, i1=42)
- assert loop.inputargs[0].value == 32
- assert loop.operations[0].result.value == 42
+ x = """
+ [p0]
+ i1 = getfield_gc(p0, descr=stuff)
+ """
+ stuff = Xyz()
+ loop = self.parse(x, None, locals())
+ assert loop.operations[0].getdescr() is stuff
-def test_boxkind():
- x = """
- [sum0]
- """
- loop = parse(x, None, {}, boxkinds={'sum': BoxInt})
- b = loop.getboxes()
- assert isinstance(b.sum0, BoxInt)
-
-def test_getvar_const_ptr():
- x = '''
- []
- call(ConstPtr(func_ptr))
+ def test_after_fail(self):
+ x = """
+ [i0]
+ guard_value(i0, 3) []
+ i1 = int_add(1, 2)
+ """
+ loop = self.parse(x, None, {})
+ assert len(loop.operations) == 2
+
+ def test_descr_setfield(self):
+ class Xyz(AbstractDescr):
+ I_am_a_descr = True # for the mock case
+
+ x = """
+ [p0]
+ setfield_gc(p0, 3, descr=stuff)
+ """
+ stuff = Xyz()
+ loop = self.parse(x, None, locals())
+ assert loop.operations[0].getdescr() is stuff
+
+ def test_boxname(self):
+ x = """
+ [i42]
+ i50 = int_add(i42, 1)
+ """
+ loop = self.parse(x, None, {})
+ assert str(loop.inputargs[0]) == 'i42'
+ assert str(loop.operations[0].result) == 'i50'
+
+ def test_getboxes(self):
+ x = """
+ [i0]
+ i1 = int_add(i0, 10)
+ """
+ loop = self.parse(x, None, {})
+ boxes = loop.getboxes()
+ assert boxes.i0 is loop.inputargs[0]
+ assert boxes.i1 is loop.operations[0].result
+
+ def test_setvalues(self):
+ x = """
+ [i0]
+ i1 = int_add(i0, 10)
+ """
+ loop = self.parse(x, None, {})
+ loop.setvalues(i0=32, i1=42)
+ assert loop.inputargs[0].value == 32
+ assert loop.operations[0].result.value == 42
+
+ def test_getvar_const_ptr(self):
+ x = '''
+ []
+ call(ConstPtr(func_ptr))
+ '''
+ TP = lltype.GcArray(lltype.Signed)
+ NULL = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(TP))
+ loop = self.parse(x, None, {'func_ptr' : NULL})
+ assert loop.operations[0].getarg(0).value == NULL
+
+ def test_jump_target(self):
+ x = '''
+ []
+ jump()
+ '''
+ loop = self.parse(x)
+ assert loop.operations[0].getdescr() is loop.token
+
+ def test_jump_target_other(self):
+ looptoken = LoopToken()
+ looptoken.I_am_a_descr = True # for the mock case
+ x = '''
+ []
+ jump(descr=looptoken)
+ '''
+ loop = self.parse(x, namespace=locals())
+ assert loop.operations[0].getdescr() is looptoken
+
+ def test_floats(self):
+ x = '''
+ [f0]
+ f1 = float_add(f0, 3.5)
+ '''
+ loop = self.parse(x)
+ box = loop.operations[0].getarg(0)
+ # we cannot use isinstance, because in case of mock the class will be
+ # constructed on the fly
+ assert box.__class__.__name__ == 'BoxFloat'
+
+ def test_debug_merge_point(self):
+ x = '''
+ []
+ debug_merge_point(0, "info")
+ debug_merge_point(0, 'info')
+ debug_merge_point(1, '<some ('other.')> info')
+ debug_merge_point(0, '(stuff) #1')
+ '''
+ loop = self.parse(x)
+ assert loop.operations[0].getarg(1)._get_str() == 'info'
+ assert loop.operations[1].getarg(1)._get_str() == 'info'
+ assert loop.operations[2].getarg(1)._get_str() == "<some ('other.')> info"
+ assert loop.operations[3].getarg(1)._get_str() == "(stuff) #1"
+
+
+ def test_descr_with_obj_print(self):
+ x = '''
+ [p0]
+ setfield_gc(p0, 1, descr=<SomeDescr>)
+ '''
+ loop = self.parse(x)
+ # assert did not explode
+
+ example_loop_log = '''\
+ # bridge out of Guard12, 6 ops
+ [i0, i1, i2]
+ i4 = int_add(i0, 2)
+ i6 = int_sub(i1, 1)
+ i8 = int_gt(i6, 3)
+ guard_true(i8, descr=<Guard15>) [i4, i6]
+ debug_merge_point('(no jitdriver.get_printable_location!)', 0)
+ jump(i6, i4, descr=<Loop0>)
'''
- TP = lltype.GcArray(lltype.Signed)
- NULL = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(TP))
- loop = parse(x, None, {'func_ptr' : NULL})
- assert loop.operations[0].getarg(0).value == NULL
-def test_jump_target():
- x = '''
- []
- jump()
- '''
- loop = parse(x)
- assert loop.operations[0].getdescr() is loop.token
+ def test_parse_no_namespace(self):
+ loop = self.parse(self.example_loop_log, no_namespace=True)
-def test_jump_target_other():
- looptoken = LoopToken()
- x = '''
- []
- jump(descr=looptoken)
- '''
- loop = parse(x, namespace=locals())
- assert loop.operations[0].getdescr() is looptoken
+ def test_attach_comment_to_loop(self):
+ loop = self.parse(self.example_loop_log, no_namespace=True)
+ assert loop.comment == ' # bridge out of Guard12, 6 ops'
-def test_floats():
- x = '''
- [f0]
- f1 = float_add(f0, 3.5)
- '''
- loop = parse(x)
- assert isinstance(loop.operations[0].getarg(0), BoxFloat)
-
-def test_debug_merge_point():
- x = '''
- []
- debug_merge_point(0, "info")
- debug_merge_point(0, 'info')
- debug_merge_point(1, '<some ('other,')> info')
- debug_merge_point(0, '(stuff) #1')
- '''
- loop = parse(x)
- assert loop.operations[0].getarg(1)._get_str() == 'info'
- assert loop.operations[1].getarg(1)._get_str() == 'info'
- assert loop.operations[2].getarg(1)._get_str() == "<some ('other,')> info"
- assert loop.operations[3].getarg(1)._get_str() == "(stuff) #1"
-
+ def test_parse_new_with_comma(self):
+ # this is generated by PYPYJITLOG, check that we can handle it
+ x = '''
+ []
+ p0 = new(, descr=<SizeDescr 12>)
+ '''
+ loop = self.parse(x)
+ assert loop.operations[0].getopname() == 'new'
-def test_descr_with_obj_print():
- x = '''
- [p0]
- setfield_gc(p0, 1, descr=<SomeDescr>)
- '''
- loop = parse(x)
- # assert did not explode
+ def test_no_fail_args(self):
+ x = '''
+ [i0]
+ guard_true(i0, descr=<Guard0>)
+ '''
+ loop = self.parse(x, nonstrict=True)
+ assert loop.operations[0].getfailargs() == []
-example_loop_log = '''\
-# bridge out of Guard12, 6 ops
-[i0, i1, i2]
-i4 = int_add(i0, 2)
-i6 = int_sub(i1, 1)
-i8 = int_gt(i6, 3)
-guard_true(i8, descr=<Guard15>) [i4, i6]
-debug_merge_point('(no jitdriver.get_printable_location!)', 0)
-jump(i6, i4, descr=<Loop0>)
-'''
+ def test_no_inputargs(self):
+ x = '''
+ i2 = int_add(i0, i1)
+ '''
+ loop = self.parse(x, nonstrict=True)
+ assert loop.inputargs == []
+ assert loop.operations[0].getopname() == 'int_add'
-def test_parse_no_namespace():
- loop = parse(example_loop_log, no_namespace=True)
+ def test_offsets(self):
+ x = """
+ [i0, i1]
+ +10: i2 = int_add(i0, i1)
+ i3 = int_add(i2, 3)
+ """
+ # +30: --end of the loop--
+ loop = self.parse(x)
+ assert loop.operations[0].offset == 10
+ assert not hasattr(loop.operations[1], 'offset')
-def test_attach_comment_to_loop():
- loop = parse(example_loop_log, no_namespace=True)
- assert loop.comment == '# bridge out of Guard12, 6 ops'
+ def test_last_offset(self):
+ x = """
+ [i0, i1]
+ +10: i2 = int_add(i0, i1)
+ i3 = int_add(i2, 3)
+ +30: --end of the loop--
+ """
+ loop = self.parse(x)
+ assert len(loop.operations) == 2
+ assert loop.last_offset == 30
-def test_parse_new_with_comma():
- # this is generated by PYPYJITLOG, check that we can handle it
- x = '''
- []
- p0 = new(, descr=<SizeDescr 12>)
- '''
- loop = parse(x)
- assert loop.operations[0].getopname() == 'new'
-def test_no_fail_args():
- x = '''
- [i0]
- guard_true(i0, descr=<Guard0>)
- '''
- loop = parse(x, nonstrict=True)
- assert loop.operations[0].getfailargs() == []
+class TestOpParser(BaseTestOparser):
-def test_no_inputargs():
- x = '''
- i2 = int_add(i0, i1)
- '''
- loop = parse(x, nonstrict=True)
- assert loop.inputargs == []
- assert loop.operations[0].getopname() == 'int_add'
+ OpParser = OpParser
-def test_offsets():
- x = """
- [i0, i1]
- +10: i2 = int_add(i0, i1)
- i3 = int_add(i2, 3)
- """
- # +30: --end of the loop--
- loop = parse(x)
- assert loop.operations[0].offset == 10
- assert not hasattr(loop.operations[1], 'offset')
+ def test_boxkind(self):
+ x = """
+ [sum0]
+ """
+ loop = self.parse(x, None, {}, boxkinds={'sum': BoxInt})
+ b = loop.getboxes()
+ assert isinstance(b.sum0, BoxInt)
-def test_last_offset():
- x = """
- [i0, i1]
- +10: i2 = int_add(i0, i1)
- i3 = int_add(i2, 3)
- +30: --end of the loop--
- """
- loop = parse(x)
- assert len(loop.operations) == 2
- assert loop.last_offset == 30
+
+class ForbiddenModule(object):
+ def __init__(self, name, old_mod):
+ self.name = name
+ self.old_mod = old_mod
+
+ def __getattr__(self, attr):
+ assert False, "You should not import module %s" % self.name
+
+
+class TestOpParserWithMock(BaseTestOparser):
+
+ class OpParser(OpParser):
+ use_mock_model = True
+
+ def setup_class(cls):
+ forbidden_mods = [
+ 'pypy.jit.metainterp.history',
+ 'pypy.rpython.lltypesystem.lltype',
+ ]
+ for modname in forbidden_mods:
+ if modname in sys.modules:
+ newmod = ForbiddenModule(modname, sys.modules[modname])
+ sys.modules[modname] = newmod
+
+ def teardown_class(cls):
+ for modname, mod in sys.modules.iteritems():
+ if isinstance(mod, ForbiddenModule):
+ sys.modules[modname] = mod.old_mod
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -1,5 +1,6 @@
import autopath
import sys
+from pypy import conftest
class AppTestBuiltinApp:
def setup_class(cls):
@@ -15,6 +16,15 @@
cls.w_sane_lookup = cls.space.wrap(True)
except KeyError:
cls.w_sane_lookup = cls.space.wrap(False)
+ # starting with CPython 2.6, when the stack is almost out, we
+ # can get a random error, instead of just a RuntimeError.
+ # For example if an object x has a __getattr__, we can get
+ # AttributeError if attempting to call x.__getattr__ runs out
+ # of stack. That's annoying, so we just work around it.
+ if conftest.option.runappdirect:
+ cls.w_safe_runtimerror = cls.space.wrap(True)
+ else:
+ cls.w_safe_runtimerror = cls.space.wrap(sys.version_info < (2, 6))
def test_bytes_alias(self):
assert bytes is str
@@ -399,6 +409,8 @@
def test_cmp_cyclic(self):
if not self.sane_lookup:
skip("underlying Python implementation has insane dict lookup")
+ if not self.safe_runtimerror:
+ skip("underlying Python may raise random exceptions on stack ovf")
a = []; a.append(a)
b = []; b.append(b)
from UserList import UserList
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -176,7 +176,7 @@
except KeyError:
raise operationerrfmt(space.w_AttributeError,
"No symbol %s found in library %s", name, self.name)
-
+
elif (_MS_WINDOWS and
space.is_true(space.isinstance(w_name, space.w_int))):
ordinal = space.int_w(w_name)
@@ -261,7 +261,7 @@
def descr_size_alignment(self, space, n=1):
return space.newtuple([space.wrap(self.size * n),
space.wrap(self.alignment)])
-
+
class W_DataInstance(Wrappable):
def __init__(self, space, size, address=r_uint(0)):
@@ -427,7 +427,7 @@
if not (argletter in TYPEMAP_PTR_LETTERS and
letter in TYPEMAP_PTR_LETTERS):
msg = "Argument %d should be typecode %s, got %s"
- raise operationerrfmt(space.w_TypeError, msg,
+ raise operationerrfmt(space.w_TypeError, msg,
i+1, argletter, letter)
args_ll.append(arg.ll_buffer)
# XXX we could avoid the intermediate list args_ll
@@ -480,17 +480,25 @@
alignment = _create_new_accessor('alignment', 'c_alignment')
@unwrap_spec(address=r_uint, maxlength=int)
-def charp2string(space, address, maxlength=sys.maxint):
+def charp2string(space, address, maxlength=-1):
if address == 0:
return space.w_None
- s = rffi.charp2strn(rffi.cast(rffi.CCHARP, address), maxlength)
+ charp_addr = rffi.cast(rffi.CCHARP, address)
+ if maxlength == -1:
+ s = rffi.charp2str(charp_addr)
+ else:
+ s = rffi.charp2strn(charp_addr, maxlength)
return space.wrap(s)
@unwrap_spec(address=r_uint, maxlength=int)
-def wcharp2unicode(space, address, maxlength=sys.maxint):
+def wcharp2unicode(space, address, maxlength=-1):
if address == 0:
return space.w_None
- s = rffi.wcharp2unicoden(rffi.cast(rffi.CWCHARP, address), maxlength)
+ wcharp_addr = rffi.cast(rffi.CWCHARP, address)
+ if maxlength == -1:
+ s = rffi.wcharp2unicode(wcharp_addr)
+ else:
+ s = rffi.wcharp2unicoden(wcharp_addr, maxlength)
return space.wrap(s)
@unwrap_spec(address=r_uint, maxlength=int)
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
from pypy.rpython.lltypesystem import rffi, lltype
from pypy.interpreter.error import OperationError
from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -526,15 +526,7 @@
def array_tostring__Array(space, self):
cbuf = self.charbuf()
- s = ''.join([cbuf[i] for i in xrange(self.len * mytype.bytes)])
- return self.space.wrap(s)
-##
-## s = ''
-## i = 0
-## while i < self.len * mytype.bytes:
-## s += cbuf[i]
-## i += 1
-## return self.space.wrap(s)
+ return self.space.wrap(rffi.charpsize2str(cbuf, self.len * mytype.bytes))
def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n):
if not isinstance(w_f, W_File):
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -622,7 +622,13 @@
try:
if find_info:
w_mod = load_module(space, w_modulename, find_info)
- w_mod = space.getitem(space.sys.get("modules"), w_modulename)
+ try:
+ w_mod = space.getitem(space.sys.get("modules"),
+ w_modulename)
+ except OperationError, oe:
+ if not oe.match(space, space.w_KeyError):
+ raise
+ raise OperationError(space.w_ImportError, w_modulename)
if w_parent is not None:
space.setattr(w_parent, space.wrap(partname), w_mod)
return w_mod
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -37,6 +37,7 @@
ambig = "imamodule = 1",
test_reload = "def test():\n raise ValueError\n",
infinite_reload = "import infinite_reload; reload(infinite_reload)",
+ del_sys_module = "import sys\ndel sys.modules['del_sys_module']\n",
)
root.ensure("notapackage", dir=1) # empty, no __init__.py
setuppkg("pkg",
@@ -562,6 +563,14 @@
except ImportError:
pass
+ def test_del_from_sys_modules(self):
+ try:
+ import del_sys_module
+ except ImportError:
+ pass # ok
+ else:
+ assert False, 'should not work'
+
class TestAbi:
def test_abi_tag(self):
space1 = gettestobjspace(soabi='TEST')
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -8,8 +8,11 @@
interpleveldefs = {
'array': 'interp_numarray.SingleDimArray',
'zeros': 'interp_numarray.zeros',
+ 'empty': 'interp_numarray.zeros',
+ 'ones': 'interp_numarray.ones',
# ufuncs
+ 'abs': 'interp_ufuncs.absolute',
'absolute': 'interp_ufuncs.absolute',
'copysign': 'interp_ufuncs.copysign',
'exp': 'interp_ufuncs.exp',
@@ -20,4 +23,7 @@
'sign': 'interp_ufuncs.sign',
}
- appleveldefs = {}
+ appleveldefs = {
+ 'average': 'app_numpy.average',
+ 'mean': 'app_numpy.mean',
+ }
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -0,0 +1,11 @@
+import numpy
+
+def average(a):
+ # This implements a weighted average, for now we don't implement the
+ # weighting, just the average part!
+ return mean(a)
+
+def mean(a):
+ if not hasattr(a, "mean"):
+ a = numpy.array(a)
+ return a.mean()
\ No newline at end of file
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/compile.py
@@ -0,0 +1,49 @@
+
+""" This is a set of tools for standalone compiling of numpy expressions.
+It should not be imported by the module itself
+"""
+
+from pypy.module.micronumpy.interp_numarray import FloatWrapper, SingleDimArray
+
+class BogusBytecode(Exception):
+ pass
+
+def create_array(size):
+ a = SingleDimArray(size)
+ for i in range(size):
+ a.storage[i] = float(i % 10)
+ return a
+
+class TrivialSpace(object):
+ def wrap(self, x):
+ return x
+
+def numpy_compile(bytecode, array_size):
+ space = TrivialSpace()
+ stack = []
+ i = 0
+ for b in bytecode:
+ if b == 'a':
+ stack.append(create_array(array_size))
+ i += 1
+ elif b == 'f':
+ stack.append(FloatWrapper(1.2))
+ elif b == '+':
+ right = stack.pop()
+ stack.append(stack.pop().descr_add(space, right))
+ elif b == '-':
+ right = stack.pop()
+ stack.append(stack.pop().descr_sub(space, right))
+ elif b == '*':
+ right = stack.pop()
+ stack.append(stack.pop().descr_mul(space, right))
+ elif b == '/':
+ right = stack.pop()
+ stack.append(stack.pop().descr_div(space, right))
+ else:
+ print "Unknown opcode: %s" % b
+ raise BogusBytecode()
+ if len(stack) != 1:
+ print "Bogus bytecode, uneven stack length"
+ raise BogusBytecode()
+ return stack[0]
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,7 +1,7 @@
from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable
from pypy.interpreter.error import operationerrfmt
from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.typedef import TypeDef, GetSetProperty
from pypy.rlib import jit
from pypy.rpython.lltypesystem import lltype
from pypy.tool.sourcetools import func_with_new_name
@@ -44,9 +44,13 @@
self.invalidates = []
def invalidated(self):
+ if self.invalidates:
+ self._invalidated()
+
+ def _invalidated(self):
for arr in self.invalidates:
arr.force_if_needed()
- self.invalidates = []
+ del self.invalidates[:]
def _binop_impl(function):
signature = Signature()
@@ -80,18 +84,36 @@
def get_concrete(self):
raise NotImplementedError
+ def descr_get_shape(self, space):
+ return space.newtuple([self.descr_len(space)])
+
def descr_len(self, space):
return self.get_concrete().descr_len(space)
- @unwrap_spec(item=int)
- def descr_getitem(self, space, item):
- return self.get_concrete().descr_getitem(space, item)
+ def descr_getitem(self, space, w_idx):
+ # TODO: indexing by tuples
+ start, stop, step, slice_length = space.decode_index4(w_idx, self.find_size())
+ if step == 0:
+ # Single index
+ return space.wrap(self.get_concrete().getitem(start))
+ else:
+ # Slice
+ res = SingleDimSlice(start, stop, step, slice_length, self, self.signature.transition(SingleDimSlice.static_signature))
+ return space.wrap(res)
@unwrap_spec(item=int, value=float)
def descr_setitem(self, space, item, value):
self.invalidated()
return self.get_concrete().descr_setitem(space, item, value)
+ def descr_mean(self, space):
+ s = 0
+ concrete = self.get_concrete()
+ size = concrete.find_size()
+ for i in xrange(size):
+ s += concrete.getitem(i)
+ return space.wrap(s / size)
+
class FloatWrapper(BaseArray):
"""
@@ -119,6 +141,10 @@
self.forced_result = None
self.signature = signature
+ def _del_sources(self):
+ # Function for deleting references to source arrays, to allow garbage-collecting them
+ raise NotImplementedError
+
def compute(self):
i = 0
signature = self.signature
@@ -135,6 +161,7 @@
def force_if_needed(self):
if self.forced_result is None:
self.forced_result = self.compute()
+ self._del_sources()
def get_concrete(self):
self.force_if_needed()
@@ -145,6 +172,13 @@
return self.forced_result.eval(i)
return self._eval(i)
+ def find_size(self):
+ if self.forced_result is not None:
+ # The result has been computed and sources may be unavailable
+ return self.forced_result.find_size()
+ return self._find_size()
+
+
class Call1(VirtualArray):
_immutable_fields_ = ["function", "values"]
@@ -153,7 +187,10 @@
self.function = function
self.values = values
- def find_size(self):
+ def _del_sources(self):
+ self.values = None
+
+ def _find_size(self):
return self.values.find_size()
def _eval(self, i):
@@ -164,13 +201,18 @@
Intermediate class for performing binary operations.
"""
_immutable_fields_ = ["function", "left", "right"]
+
def __init__(self, function, left, right, signature):
VirtualArray.__init__(self, signature)
self.function = function
self.left = left
self.right = right
- def find_size(self):
+ def _del_sources(self):
+ self.left = None
+ self.right = None
+
+ def _find_size(self):
try:
return self.left.find_size()
except ValueError:
@@ -181,6 +223,58 @@
lhs, rhs = self.left.eval(i), self.right.eval(i)
return self.function(lhs, rhs)
+class ViewArray(BaseArray):
+ """
+ Class for representing views of arrays, they will reflect changes of parent
+ arrays. Example: slices
+ """
+ _immutable_fields_ = ["parent"]
+
+ def __init__(self, parent, signature):
+ BaseArray.__init__(self)
+ self.signature = signature
+ self.parent = parent
+ self.invalidates = parent.invalidates
+
+ def get_concrete(self):
+ # in fact, ViewArray never gets "concrete" as it never stores data.
+ # This implementation is needed for BaseArray getitem/setitem to work,
+ # can be refactored.
+ return self
+
+ def eval(self, i):
+ return self.parent.eval(self.calc_index(i))
+
+ def getitem(self, item):
+ return self.parent.getitem(self.calc_index(item))
+
+ @unwrap_spec(item=int, value=float)
+ def descr_setitem(self, space, item, value):
+ return self.parent.descr_setitem(space, self.calc_index(item), value)
+
+ def descr_len(self, space):
+ return space.wrap(self.find_size())
+
+ def calc_index(self, item):
+ raise NotImplementedError
+
+class SingleDimSlice(ViewArray):
+ _immutable_fields_ = ["start", "stop", "step", "size"]
+ static_signature = Signature()
+
+ def __init__(self, start, stop, step, slice_length, parent, signature):
+ ViewArray.__init__(self, parent, signature)
+ self.start = start
+ self.stop = stop
+ self.step = step
+ self.size = slice_length
+
+ def find_size(self):
+ return self.size
+
+ def calc_index(self, item):
+ return (self.start + item * self.step)
+
class SingleDimArray(BaseArray):
signature = Signature()
@@ -215,10 +309,8 @@
def descr_len(self, space):
return space.wrap(self.size)
- @unwrap_spec(item=int)
- def descr_getitem(self, space, item):
- item = self.getindex(space, item)
- return space.wrap(self.storage[item])
+ def getitem(self, item):
+ return self.storage[item]
@unwrap_spec(item=int, value=float)
def descr_setitem(self, space, item, value):
@@ -238,14 +330,23 @@
i += 1
return space.wrap(arr)
- at unwrap_spec(ObjSpace, int)
+ at unwrap_spec(size=int)
def zeros(space, size):
return space.wrap(SingleDimArray(size))
+ at unwrap_spec(size=int)
+def ones(space, size):
+ arr = SingleDimArray(size)
+ for i in xrange(size):
+ arr.storage[i] = 1.0
+ return space.wrap(arr)
BaseArray.typedef = TypeDef(
'numarray',
__new__ = interp2app(descr_new_numarray),
+
+ shape = GetSetProperty(BaseArray.descr_get_shape),
+
__len__ = interp2app(BaseArray.descr_len),
__getitem__ = interp2app(BaseArray.descr_getitem),
__setitem__ = interp2app(BaseArray.descr_setitem),
@@ -254,4 +355,6 @@
__sub__ = interp2app(BaseArray.descr_sub),
__mul__ = interp2app(BaseArray.descr_mul),
__div__ = interp2app(BaseArray.descr_div),
-)
\ No newline at end of file
+
+ mean = interp2app(BaseArray.descr_mean),
+)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -8,22 +8,24 @@
def ufunc(func):
signature = Signature()
- @unwrap_spec(array=BaseArray)
- def impl(space, array):
- w_res = Call1(func, array, array.signature.transition(signature))
- array.invalidates.append(w_res)
- return w_res
+ def impl(space, w_obj):
+ if isinstance(w_obj, BaseArray):
+ w_res = Call1(func, w_obj, w_obj.signature.transition(signature))
+ w_obj.invalidates.append(w_res)
+ return w_res
+ return space.wrap(func(space.float_w(w_obj)))
return func_with_new_name(impl, "%s_dispatcher" % func.__name__)
def ufunc2(func):
signature = Signature()
- @unwrap_spec(larray=BaseArray, rarray=BaseArray)
- def impl(space, larray, rarray):
- new_sig = larray.signature.transition(signature).transition(rarray.signature)
- w_res = Call2(func, larray, rarray, new_sig)
- larray.invalidates.append(w_res)
- rarray.invalidates.append(w_res)
- return w_res
+ def impl(space, w_lhs, w_rhs):
+ if isinstance(w_lhs, BaseArray) and isinstance(w_rhs, BaseArray):
+ new_sig = w_lhs.signature.transition(signature).transition(w_rhs.signature)
+ w_res = Call2(func, w_lhs, w_rhs, new_sig)
+ w_lhs.invalidates.append(w_res)
+ w_rhs.invalidates.append(w_res)
+ return w_res
+ return space.wrap(func(space.float_w(w_lhs), space.float_w(w_rhs)))
return func_with_new_name(impl, "%s_dispatcher" % func.__name__)
@ufunc
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -16,4 +16,14 @@
v3 = ar.descr_add(space, FloatWrapper(1.0))
assert v2.signature is v3.signature
v4 = ar.descr_add(space, ar)
- assert v1.signature is v4.signature
\ No newline at end of file
+ assert v1.signature is v4.signature
+
+ def test_slice_signature(self, space):
+ ar = SingleDimArray(10)
+ v1 = ar.descr_getitem(space, space.wrap(slice(1, 5, 1)))
+ v2 = ar.descr_getitem(space, space.wrap(slice(4, 6, 1)))
+ assert v1.signature is v2.signature
+
+ v3 = ar.descr_add(space, v1)
+ v4 = ar.descr_add(space, v2)
+ assert v3.signature is v4.signature
\ No newline at end of file
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -0,0 +1,13 @@
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+
+class AppTestNumPyModule(BaseNumpyAppTest):
+ def test_mean(self):
+ from numpy import array, mean
+ assert mean(array(range(5))) == 2.0
+ assert mean(range(5)) == 2.0
+
+ def test_average(self):
+ from numpy import array, average
+ assert average(range(10)) == 4.5
+ assert average(array(range(10))) == 4.5
\ No newline at end of file
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -18,6 +18,25 @@
a[13] = 5.3
assert a[13] == 5.3
+ def test_empty(self):
+ """
+ Test that empty() works.
+ """
+
+ from numpy import empty
+ a = empty(2)
+ a[1] = 1.0
+ assert a[1] == 1.0
+
+ def test_ones(self):
+ from numpy import ones
+ a = ones(3)
+ assert len(a) == 3
+ assert a[0] == 1
+ raises(IndexError, "a[3]")
+ a[2] = 4
+ assert a[2] == 4
+
def test_iterator_init(self):
from numpy import array
a = array(range(5))
@@ -46,6 +65,15 @@
assert len(a) == 5
assert len(a + a) == 5
+ def test_shape(self):
+ from numpy import array
+ a = array(range(5))
+ assert a.shape == (5,)
+ b = a + a
+ assert b.shape == (5,)
+ c = a[:3]
+ assert c.shape == (3,)
+
def test_add(self):
from numpy import array
a = array(range(5))
@@ -138,4 +166,51 @@
b = a + a
c = b + b
b[1] = 5
- assert c[1] == 4
\ No newline at end of file
+ assert c[1] == 4
+
+ def test_getslice(self):
+ from numpy import array
+ a = array(range(5))
+ s = a[1:5]
+ assert len(s) == 4
+ for i in range(4):
+ assert s[i] == a[i+1]
+
+ def test_getslice_step(self):
+ from numpy import array
+ a = array(range(10))
+ s = a[1:9:2]
+ assert len(s) == 4
+ for i in range(4):
+ assert s[i] == a[2*i+1]
+
+ def test_slice_update(self):
+ from numpy import array
+ a = array(range(5))
+ s = a[0:3]
+ s[1] = 10
+ assert a[1] == 10
+ a[2] = 20
+ assert s[2] == 20
+
+
+ def test_slice_invaidate(self):
+ # check that slice shares invalidation list with
+ from numpy import array
+ a = array(range(5))
+ s = a[0:2]
+ b = array([10,11])
+ c = s + b
+ a[0] = 100
+ assert c[0] == 10
+ assert c[1] == 12
+ d = s + b
+ a[1] = 101
+ assert d[0] == 110
+ assert d[1] == 12
+
+ def test_mean(self):
+ from numpy import array, mean
+ a = array(range(5))
+ assert a.mean() == 2.0
+ assert a[:4].mean() == 1.5
\ No newline at end of file
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -3,6 +3,13 @@
class AppTestUfuncs(BaseNumpyAppTest):
+ def test_single_item(self):
+ from numpy import negative, sign, minimum
+
+ assert negative(5.0) == -5.0
+ assert sign(-0.0) == 0.0
+ assert minimum(2.0, 3.0) == 2.0
+
def test_negative(self):
from numpy import array, negative
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,8 +1,9 @@
from pypy.jit.metainterp.test.support import LLJitMixin
+from pypy.rpython.test.test_llinterp import interpret
from pypy.module.micronumpy.interp_numarray import (SingleDimArray, Signature,
- FloatWrapper, Call1, Call2, add, mul)
+ FloatWrapper, Call1, Call2, SingleDimSlice, add, mul)
from pypy.module.micronumpy.interp_ufuncs import negative
-
+from pypy.module.micronumpy.compile import numpy_compile
class FakeSpace(object):
pass
@@ -91,4 +92,54 @@
self.meta_interp(f, [5], listops=True, backendopt=True)
# This is 3, not 2 because there is a bridge for the exit.
- self.check_loop_count(3)
\ No newline at end of file
+ self.check_loop_count(3)
+
+ def test_slice(self):
+ space = self.space
+
+ def f(i):
+ step = 3
+ ar = SingleDimArray(step*i)
+ s = SingleDimSlice(0, step*i, step, i, ar, ar.signature.transition(SingleDimSlice.static_signature))
+ v = Call2(add, s, s, Signature())
+ return v.get_concrete().storage[3]
+
+ result = self.meta_interp(f, [5], listops=True, backendopt=True)
+ self.check_loops({'int_mul': 1, 'getarrayitem_raw': 2, 'float_add': 1,
+ 'setarrayitem_raw': 1, 'int_add': 1,
+ 'int_lt': 1, 'guard_true': 1, 'jump': 1})
+ assert result == f(5)
+
+ def test_slice2(self):
+ space = self.space
+
+ def f(i):
+ step1 = 2
+ step2 = 3
+ ar = SingleDimArray(step2*i)
+ s1 = SingleDimSlice(0, step1*i, step1, i, ar, ar.signature.transition(SingleDimSlice.static_signature))
+ s2 = SingleDimSlice(0, step2*i, step2, i, ar, ar.signature.transition(SingleDimSlice.static_signature))
+ v = Call2(add, s1, s2, Signature())
+ return v.get_concrete().storage[3]
+
+ result = self.meta_interp(f, [5], listops=True, backendopt=True)
+ self.check_loops({'int_mul': 2, 'getarrayitem_raw': 2, 'float_add': 1,
+ 'setarrayitem_raw': 1, 'int_add': 1,
+ 'int_lt': 1, 'guard_true': 1, 'jump': 1})
+ assert result == f(5)
+
+class TestTranslation(object):
+ def test_compile(self):
+ x = numpy_compile('aa+f*f/a-', 10)
+ x = x.compute()
+ assert isinstance(x, SingleDimArray)
+ assert x.size == 10
+ assert x.storage[0] == 0
+ assert x.storage[1] == ((1 + 1) * 1.2) / 1.2 - 1
+
+ def test_translation(self):
+ # we import main to check if the target compiles
+ from pypy.translator.goal.targetnumpystandalone import main
+ from pypy.rpython.annlowlevel import llstr
+
+ interpret(main, [llstr('af+'), 100])
diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -4,6 +4,7 @@
This module exports a set of operators as functions. E.g. operator.add(x,y) is
equivalent to x+y.
'''
+from __pypy__ import builtinify
def countOf(a,b):
'countOf(a, b) -- Return the number of times b occurs in a.'
@@ -66,50 +67,56 @@
a[b:c] = d
__setslice__ = setslice
-class attrgetter(object):
- def __init__(self, attr, *attrs):
- self.attrs = (attr,) + attrs
+def attrgetter(attr, *attrs):
+ if attrs:
+ getters = [single_attr_getter(a) for a in (attr,) + attrs]
+ def getter(obj):
+ return tuple([getter(obj) for getter in getters])
+ else:
+ getter = single_attr_getter(attr)
+ return builtinify(getter)
- def _resolve_attr(self, obj, attr):
- last = 0
- while True:
- try:
- dot = attr.find(".", last)
- except AttributeError:
- raise TypeError
- if dot > 0:
- obj = getattr(obj, attr[last:dot])
- last = dot + 1
- else:
- return getattr(obj, attr[last:])
+def single_attr_getter(attr):
+ if not isinstance(attr, str):
+ if not isinstance(attr, unicode):
+ def _raise_typeerror(obj):
+ raise TypeError("argument must be a string, not %r" %
+ (type(attr).__name__,))
+ return _raise_typeerror
+ attr = attr.encode('ascii')
+ #
+ def make_getter(name, prevfn=None):
+ if prevfn is None:
+ def getter(obj):
+ return getattr(obj, name)
+ else:
+ def getter(obj):
+ return getattr(prevfn(obj), name)
+ return getter
+ #
+ last = 0
+ getter = None
+ while True:
+ dot = attr.find(".", last)
+ if dot < 0: break
+ getter = make_getter(attr[last:dot], getter)
+ last = dot + 1
+ return make_getter(attr[last:], getter)
- def __call__(self, obj):
- if len(self.attrs) == 1:
- return self._resolve_attr(obj, self.attrs[0])
- return tuple(self._resolve_attr(obj, attr) for attr in self.attrs)
-class itemgetter(object):
+def itemgetter(item, *items):
+ if items:
+ list_of_indices = [item] + list(items)
+ def getter(obj):
+ return tuple([obj[i] for i in list_of_indices])
+ else:
+ def getter(obj):
+ return obj[item]
+ return builtinify(getter)
- def __init__(self, item, *args):
- self.items = args
- self.item = item
- def __call__(self, obj):
- result = obj[self.item]
-
- if self.items:
- list = [result] + [obj[item] for item in self.items]
- return tuple(list)
-
- return result
-
-class methodcaller(object):
-
- def __init__(self, method_name, *args, **kwargs):
- self.method_name = method_name
- self.args = args
- self.kwargs = kwargs
-
- def __call__(self, obj):
- return getattr(obj, self.method_name)(*self.args, **self.kwargs)
+def methodcaller(method_name, *args, **kwargs):
+ def call(obj):
+ return getattr(obj, method_name)(*args, **kwargs)
+ return builtinify(call)
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -107,6 +107,9 @@
def tmpnam():
"""Return an absolute pathname of a file that did not exist at the
time the call is made."""
+ from warnings import warn
+ warn(RuntimeWarning("tmpnam is a potential security risk to your program"))
+
import tempfile
return tempfile.mktemp()
@@ -114,6 +117,9 @@
"""Return an absolute pathname of a file that did not exist at the
time the call is made. The directory and a prefix may be specified
as strings; they may be omitted or None if not needed."""
+ from warnings import warn
+ warn(RuntimeWarning("tempnam is a potential security risk to your program"))
+
import tempfile
return tempfile.mktemp('', prefix or 'tmp', dir)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -847,6 +847,21 @@
assert os.path.basename(s1).startswith(prefix or 'tmp')
assert os.path.basename(s2).startswith(prefix or 'tmp')
+ def test_tmpnam_warning(self):
+ import warnings, os
+ #
+ def f_tmpnam_warning(): os.tmpnam() # a single line
+ #
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter("always")
+ f_tmpnam_warning()
+ assert len(w) == 1
+ assert issubclass(w[-1].category, RuntimeWarning)
+ assert "potential security risk" in str(w[-1].message)
+ # check that the warning points to the call to os.tmpnam(),
+ # not to some code inside app_posix.py
+ assert w[-1].lineno == f_tmpnam_warning.func_code.co_firstlineno
+
class AppTestEnvironment(object):
def setup_class(cls):
diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -8,6 +8,7 @@
'set_param': 'interp_jit.set_param',
'residual_call': 'interp_jit.residual_call',
'set_compile_hook': 'interp_jit.set_compile_hook',
+ 'DebugMergePoint': 'interp_resop.W_DebugMergePoint',
}
def setup_after_space_initialization(self):
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -17,6 +17,8 @@
from opcode import opmap
from pypy.rlib.objectmodel import we_are_translated
from pypy.rlib.nonconst import NonConstant
+from pypy.jit.metainterp.resoperation import rop
+from pypy.module.pypyjit.interp_resop import debug_merge_point_from_boxes
PyFrame._virtualizable2_ = ['last_instr', 'pycode',
'valuestackdepth', 'valuestack_w[*]',
@@ -47,6 +49,16 @@
return (bytecode.co_flags & CO_GENERATOR) != 0
+def wrap_oplist(space, logops, operations):
+ list_w = []
+ for op in operations:
+ if op.getopnum() == rop.DEBUG_MERGE_POINT:
+ list_w.append(space.wrap(debug_merge_point_from_boxes(
+ op.getarglist())))
+ else:
+ list_w.append(space.wrap(logops.repr_of_resop(op)))
+ return list_w
+
class PyPyJitDriver(JitDriver):
reds = ['frame', 'ec']
greens = ['next_instr', 'is_being_profiled', 'pycode']
@@ -62,8 +74,7 @@
return
if space.is_true(cache.w_compile_hook):
logops = logger._make_log_operations()
- list_w = [space.wrap(logops.repr_of_resop(op))
- for op in operations]
+ list_w = wrap_oplist(space, logops, operations)
pycode = cast_base_ptr_to_instance(PyCode, ll_pycode)
cache.in_recursion = True
try:
@@ -85,8 +96,7 @@
return
if space.is_true(cache.w_compile_hook):
logops = logger._make_log_operations()
- list_w = [space.wrap(logops.repr_of_resop(op))
- for op in operations]
+ list_w = wrap_oplist(space, logops, operations)
cache.in_recursion = True
try:
space.call_function(cache.w_compile_hook,
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -0,0 +1,41 @@
+
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty
+from pypy.interpreter.baseobjspace import Wrappable, ObjSpace, W_Root
+from pypy.interpreter.gateway import unwrap_spec, interp2app
+from pypy.interpreter.pycode import PyCode
+from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
+from pypy.rpython.lltypesystem.rclass import OBJECT
+
+class W_DebugMergePoint(Wrappable):
+ """ A class representing debug_merge_point JIT operation
+ """
+
+ def __init__(self, mp_no, offset, pycode):
+ self.mp_no = mp_no
+ self.offset = offset
+ self.pycode = pycode
+
+ def descr_repr(self, space):
+ return space.wrap('DebugMergePoint()')
+
+ at unwrap_spec(mp_no=int, offset=int, pycode=PyCode)
+def new_debug_merge_point(space, w_tp, mp_no, offset, pycode):
+ return W_DebugMergePoint(mp_no, offset, pycode)
+
+def debug_merge_point_from_boxes(boxes):
+ mp_no = boxes[0].getint()
+ offset = boxes[2].getint()
+ llcode = lltype.cast_opaque_ptr(lltype.Ptr(OBJECT),
+ boxes[4].getref_base())
+ pycode = cast_base_ptr_to_instance(PyCode, llcode)
+ assert pycode is not None
+ return W_DebugMergePoint(mp_no, offset, pycode)
+
+W_DebugMergePoint.typedef = TypeDef(
+ 'DebugMergePoint',
+ __new__ = interp2app(new_debug_merge_point),
+ __doc__ = W_DebugMergePoint.__doc__,
+ __repr__ = interp2app(W_DebugMergePoint.descr_repr),
+ code = interp_attrproperty('pycode', W_DebugMergePoint),
+)
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -8,12 +8,13 @@
from pypy.jit.metainterp.logger import Logger
from pypy.rpython.annlowlevel import (cast_instance_to_base_ptr,
cast_base_ptr_to_instance)
+from pypy.rpython.lltypesystem import lltype, llmemory
from pypy.module.pypyjit.interp_jit import pypyjitdriver
from pypy.jit.tool.oparser import parse
from pypy.jit.metainterp.typesystem import llhelper
class MockSD(object):
- class cpu:
+ class cpu(object):
ts = llhelper
class AppTestJitHook(object):
@@ -27,14 +28,17 @@
pass
return f
""")
+ cls.w_f = w_f
ll_code = cast_instance_to_base_ptr(w_f.code)
+ code_gcref = lltype.cast_opaque_ptr(llmemory.GCREF, ll_code)
logger = Logger(MockSD())
oplist = parse("""
[i1, i2]
i3 = int_add(i1, i2)
+ debug_merge_point(0, 0, 0, 0, ConstPtr(ptr0))
guard_true(i3) []
- """).operations
+ """, namespace={'ptr0': code_gcref}).operations
def interp_on_compile():
pypyjitdriver.on_compile(logger, LoopToken(), oplist, 'loop',
@@ -63,7 +67,7 @@
assert all[0][0][0].co_name == 'f'
assert all[0][0][1] == 0
assert all[0][0][2] == False
- assert len(all[0][1]) == 2
+ assert len(all[0][1]) == 3
assert 'int_add' in all[0][1][0]
self.on_compile_bridge()
assert len(all) == 2
@@ -103,3 +107,20 @@
self.on_compile_bridge()
assert len(l) == 2 # and did not crash
+ def test_on_compile_types(self):
+ import pypyjit
+ l = []
+
+ def hook(*args):
+ l.append(args)
+
+ pypyjit.set_compile_hook(hook)
+ self.on_compile()
+ dmp = l[0][3][1]
+ assert isinstance(dmp, pypyjit.DebugMergePoint)
+ assert dmp.code is self.f.func_code
+
+ def test_creation(self):
+ import pypyjit
+ dmp = pypyjit.DebugMergePoint(0, 0, self.f.func_code)
+ assert dmp.code is self.f.func_code
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -2,6 +2,7 @@
import sys
import re
import os.path
+from _pytest.assertion import newinterpret
from pypy.tool.jitlogparser.parser import SimpleParser, Function, TraceForOpcode
from pypy.tool.jitlogparser.storage import LoopStorage
@@ -194,7 +195,7 @@
# transform self._assert(x, 'foo') into assert x, 'foo'
source = source.replace('self._assert(', 'assert ')
source = source[:-1] # remove the trailing ')'
- self.msg = py.code._reinterpret(source, f, should_fail=True)
+ self.msg = newinterpret.interpret(source, f, should_fail=True)
else:
self.msg = "<could not determine information>"
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -11,21 +11,14 @@
return 1 + rec(n-1)
#
# this loop is traced and then aborted, because the trace is too
- # long. But then "rec" is marked as "don't inline"
- i = 0
- j = 0
- while i < 20:
- i += 1
- j += rec(100)
- #
- # next time we try to trace "rec", instead of inlining we compile
- # it separately and generate a call_assembler
+ # long. But then "rec" is marked as "don't inline". Since we
+ # already traced function from the start (because of number),
+ # now we can inline it as call assembler
i = 0
j = 0
while i < 20:
i += 1
j += rec(100) # ID: call_rec
- a = 0
return j
#
log = self.run(fn, [], threshold=18)
@@ -38,6 +31,20 @@
...
""")
+ def test_fib(self):
+ def fib(n):
+ if n == 0 or n == 1:
+ return 1
+ return fib(n - 1) + fib(n - 2) # ID: call_rec
+
+ log = self.run(fib, [7], function_threshold=15)
+ loop, = log.loops_by_filename(self.filepath, is_entry_bridge='*')
+ #assert loop.match_by_id('call_rec', '''
+ #...
+ #p1 = call_assembler(..., descr=...)
+ #...
+ #''')
+
def test_simple_call(self):
src = """
OFFSET = 0
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -115,7 +115,6 @@
# ----------------------
loop, = log.loops_by_filename(self.filepath)
assert loop.match("""
- i8 = getfield_gc_pure(p5, descr=<SignedFieldDescr .*W_IntObject.inst_intval.*>)
i9 = int_lt(i8, i7)
guard_true(i9, descr=.*)
guard_not_invalidated(descr=.*)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -0,0 +1,42 @@
+from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+
+class TestString(BaseTestPyPyC):
+ def test_lookup_default_encoding(self):
+ def main(n):
+ import string
+ i = 0
+ letters = string.letters
+ uletters = unicode(string.letters)
+ while i < n:
+ i += letters[i % len(letters)] == uletters[i % len(letters)]
+ return i
+
+ log = self.run(main, [300])
+ assert log.result == 300
+ loop, = log.loops_by_filename(self.filepath)
+ assert loop.match("""
+ i14 = int_lt(i6, i9)
+ guard_true(i14, descr=<Guard42>)
+ i15 = int_mod(i6, i10)
+ i17 = int_rshift(i15, 63)
+ i18 = int_and(i10, i17)
+ i19 = int_add(i15, i18)
+ i21 = int_lt(i19, 0)
+ guard_false(i21, descr=<Guard43>)
+ i22 = int_ge(i19, i10)
+ guard_false(i22, descr=<Guard44>)
+ i23 = strgetitem(p11, i19)
+ i24 = int_ge(i19, i12)
+ guard_false(i24, descr=<Guard45>)
+ i25 = unicodegetitem(p13, i19)
+ guard_not_invalidated(descr=<Guard46>)
+ p27 = newstr(1)
+ strsetitem(p27, 0, i23)
+ p30 = call(ConstClass(ll_str2unicode__rpy_stringPtr), p27, descr=<GcPtrCallDescr>)
+ guard_no_exception(descr=<Guard47>)
+ i32 = call(ConstClass(_ll_2_str_eq_checknull_char__rpy_unicodePtr_UniChar), p30, i25, descr=<SignedCallDescr>)
+ guard_true(i32, descr=<Guard48>)
+ i34 = int_add(i6, 1)
+ --TICK--
+ jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=<Loop4>)
+ """)
\ No newline at end of file
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -7,6 +7,8 @@
class Module(MixedModule):
"""Sys Builtin Module. """
+ _immutable_fields_ = ["defaultencoding?"]
+
def __init__(self, space, w_name):
"""NOT_RPYTHON""" # because parent __init__ isn't
if space.config.translating:
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
new file mode 100644
--- /dev/null
+++ b/pypy/pytest.ini
@@ -0,0 +1,2 @@
+[pytest]
+addopts = --assertmode=old
\ No newline at end of file
diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py
--- a/pypy/rlib/clibffi.py
+++ b/pypy/rlib/clibffi.py
@@ -10,6 +10,7 @@
from pypy.rlib.rmmap import alloc
from pypy.rlib.rdynload import dlopen, dlclose, dlsym, dlsym_byordinal
from pypy.rlib.rdynload import DLOpenError, DLLHANDLE
+from pypy.rlib import jit
from pypy.tool.autopath import pypydir
from pypy.translator.tool.cbuild import ExternalCompilationInfo
from pypy.translator.platform import platform
@@ -18,6 +19,10 @@
import sys
import ctypes.util
+from pypy.tool.ansi_print import ansi_log
+log = py.log.Producer("libffi")
+py.log.setconsumer("libffi", ansi_log)
+
# maaaybe isinstance here would be better. Think
_MSVC = platform.name == "msvc"
_MINGW = platform.name == "mingw32"
@@ -67,12 +72,17 @@
result = os.path.join(dir, 'libffi.a')
if os.path.exists(result):
return result
- raise ImportError("'libffi.a' not found in %s" % (dirlist,))
+ log.WARNING("'libffi.a' not found in %s" % (dirlist,))
+ log.WARNING("trying to use the dynamic library instead...")
+ return None
+ path_libffi_a = None
if hasattr(platform, 'library_dirs_for_libffi_a'):
+ path_libffi_a = find_libffi_a()
+ if path_libffi_a is not None:
# platforms on which we want static linking
libraries = []
- link_files = [find_libffi_a()]
+ link_files = [path_libffi_a]
else:
# platforms on which we want dynamic linking
libraries = ['ffi']
@@ -261,6 +271,7 @@
elif _MSVC:
get_libc_handle = external('pypy_get_libc_handle', [], DLLHANDLE)
+ @jit.dont_look_inside
def get_libc_name():
return rwin32.GetModuleFileName(get_libc_handle())
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -273,7 +273,8 @@
class JitHintError(Exception):
"""Inconsistency in the JIT hints."""
-PARAMETERS = {'threshold': 1000,
+PARAMETERS = {'threshold': 1032, # just above 1024
+ 'function_threshold': 1617, # slightly more than one above
'trace_eagerness': 200,
'trace_limit': 12000,
'inlining': 0,
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -134,7 +134,8 @@
def external(name, argtypes, restype, **kw):
kw['compilation_info'] = eci
- eci.export_symbols += (name,)
+ if not kw.get('macro', False):
+ eci.export_symbols += (name,)
return rffi.llexternal(
name, argtypes, restype, **kw)
diff --git a/pypy/rlib/rrandom.py b/pypy/rlib/rrandom.py
--- a/pypy/rlib/rrandom.py
+++ b/pypy/rlib/rrandom.py
@@ -24,8 +24,7 @@
def __init__(self, seed=r_uint(0)):
self.state = [r_uint(0)] * N
self.index = 0
- if seed:
- self.init_genrand(seed)
+ self.init_genrand(seed)
def init_genrand(self, s):
mt = self.state
diff --git a/pypy/rlib/test/test_rrandom.py b/pypy/rlib/test/test_rrandom.py
--- a/pypy/rlib/test/test_rrandom.py
+++ b/pypy/rlib/test/test_rrandom.py
@@ -3,6 +3,12 @@
# the numbers were created by using CPython's _randommodule.c
+def test_init_from_zero():
+ rnd = Random(0)
+ assert rnd.state[:14] == [0, 1, 1812433255, 1900727105, 1208447044,
+ 2481403966, 4042607538, 337614300, 3232553940,
+ 1018809052, 3202401494, 1775180719, 3192392114, 594215549]
+
def test_init_from_seed():
rnd = Random(1000)
assert rnd.state[:14] == [1000, 4252021385, 1724402292, 571538732,
diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -139,10 +139,10 @@
source = py.code.Source("""
def call_external_function(%(argnames)s):
before = aroundstate.before
- after = aroundstate.after
if before: before()
# NB. it is essential that no exception checking occurs here!
res = funcptr(%(argnames)s)
+ after = aroundstate.after
if after: after()
return res
""" % locals())
@@ -253,7 +253,7 @@
if hasattr(callable, '_errorcode_'):
errorcode = callable._errorcode_
else:
- errorcode = TP.TO.RESULT._example()
+ errorcode = TP.TO.RESULT._defl()
callable_name = getattr(callable, '__name__', '?')
if callbackholder is not None:
callbackholder.callbacks[callable] = True
@@ -262,13 +262,9 @@
def wrapper(%s): # no *args - no GIL for mallocing the tuple
llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py
if aroundstate is not None:
- before = aroundstate.before
after = aroundstate.after
- else:
- before = None
- after = None
- if after:
- after()
+ if after:
+ after()
# from now on we hold the GIL
stackcounter.stacks_counter += 1
try:
@@ -282,8 +278,10 @@
traceback.print_exc()
result = errorcode
stackcounter.stacks_counter -= 1
- if before:
- before()
+ if aroundstate is not None:
+ before = aroundstate.before
+ if before:
+ before()
# here we don't hold the GIL any more. As in the wrapper() produced
# by llexternal, it is essential that no exception checking occurs
# after the call to before().
diff --git a/pypy/rpython/lltypesystem/rlist.py b/pypy/rpython/lltypesystem/rlist.py
--- a/pypy/rpython/lltypesystem/rlist.py
+++ b/pypy/rpython/lltypesystem/rlist.py
@@ -250,12 +250,11 @@
length = l.length
l.length = length + 1
l.ll_setitem_fast(length, newitem)
-ll_append_noresize.oopspec = 'list.append(l, newitem)'
def ll_both_none(lst1, lst2):
return not lst1 and not lst2
-
+
# ____________________________________________________________
#
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -323,6 +323,8 @@
return s
ll_str2unicode.oopspec = 'str.str2unicode(str)'
+ # it's pure but it does not look like it
+ @purefunction
def ll_strhash(s):
# unlike CPython, there is no reason to avoid to return -1
# but our malloc initializes the memory to zero, so we use zero as the
@@ -334,7 +336,6 @@
x = 29872897
s.hash = x
return x
- ll_strhash._pure_function_ = True # it's pure but it does not look like it
def ll_strfasthash(s):
return s.hash # assumes that the hash is already computed
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -927,7 +927,7 @@
def write_barrier_from_array(self, newvalue, addr_array, index):
if self.header(addr_array).tid & GCFLAG_NO_YOUNG_PTRS:
if self.card_page_indices > 0: # <- constant-folded
- self.remember_young_pointer_from_array(addr_array, index)
+ self.remember_young_pointer_from_array2(addr_array, index)
else:
self.remember_young_pointer(addr_array, newvalue)
@@ -976,7 +976,7 @@
def _init_writebarrier_with_card_marker(self):
DEBUG = self.DEBUG
- def remember_young_pointer_from_array(addr_array, index):
+ def remember_young_pointer_from_array2(addr_array, index):
# 'addr_array' is the address of the object in which we write,
# which must have an array part; 'index' is the index of the
# item that is (or contains) the pointer that we write.
@@ -1011,7 +1011,7 @@
#
# We set the flag (even if the newly written address does not
# actually point to the nursery, which seems to be ok -- actually
- # it seems more important that remember_young_pointer_from_array()
+ # it seems more important that remember_young_pointer_from_array2()
# does not take 3 arguments).
addr_byte.char[0] = chr(byte | bitmask)
#
@@ -1019,10 +1019,67 @@
self.old_objects_with_cards_set.append(addr_array)
objhdr.tid |= GCFLAG_CARDS_SET
- remember_young_pointer_from_array._dont_inline_ = True
+ remember_young_pointer_from_array2._dont_inline_ = True
assert self.card_page_indices > 0
- self.remember_young_pointer_from_array = (
- remember_young_pointer_from_array)
+ self.remember_young_pointer_from_array2 = (
+ remember_young_pointer_from_array2)
+
+ # xxx trying it out for the JIT: a 3-arguments version of the above
+ def remember_young_pointer_from_array3(addr_array, index, newvalue):
+ if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this
+ ll_assert(self.debug_is_old_object(addr_array),
+ "young array with GCFLAG_NO_YOUNG_PTRS")
+ objhdr = self.header(addr_array)
+ #
+ # a single check for the common case of neither GCFLAG_HAS_CARDS
+ # nor GCFLAG_NO_HEAP_PTRS
+ if objhdr.tid & (GCFLAG_HAS_CARDS | GCFLAG_NO_HEAP_PTRS) == 0:
+ # common case: fast path, jump to the end of the function
+ pass
+ elif objhdr.tid & GCFLAG_HAS_CARDS == 0:
+ # no cards, but GCFLAG_NO_HEAP_PTRS is set.
+ objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS
+ self.prebuilt_root_objects.append(addr_array)
+ # jump to the end of the function
+ else:
+ # case with cards.
+ #
+ # If the newly written address does not actually point to the
+ # nursery, leave now.
+ if not self.appears_to_be_young(newvalue):
+ return
+ #
+ # 'addr_array' is a raw_malloc'ed array with card markers
+ # in front. Compute the index of the bit to set:
+ bitindex = index >> self.card_page_shift
+ byteindex = bitindex >> 3
+ bitmask = 1 << (bitindex & 7)
+ #
+ # If the bit is already set, leave now.
+ size_gc_header = self.gcheaderbuilder.size_gc_header
+ addr_byte = addr_array - size_gc_header
+ addr_byte = llarena.getfakearenaaddress(addr_byte) + \
+ (~byteindex)
+ byte = ord(addr_byte.char[0])
+ if byte & bitmask:
+ return
+ addr_byte.char[0] = chr(byte | bitmask)
+ #
+ if objhdr.tid & GCFLAG_CARDS_SET == 0:
+ self.old_objects_with_cards_set.append(addr_array)
+ objhdr.tid |= GCFLAG_CARDS_SET
+ return
+ #
+ # Logic for the no-cards case, put here to minimize the number
+ # of checks done at the start of the function
+ if self.appears_to_be_young(newvalue):
+ self.old_objects_pointing_to_young.append(addr_array)
+ objhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS
+
+ remember_young_pointer_from_array3._dont_inline_ = True
+ assert self.card_page_indices > 0
+ self.remember_young_pointer_from_array3 = (
+ remember_young_pointer_from_array3)
def assume_young_pointers(self, addr_struct):
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -463,7 +463,7 @@
annmodel.SomeInteger()],
annmodel.s_None,
inline=True)
- func = getattr(gcdata.gc, 'remember_young_pointer_from_array',
+ func = getattr(gcdata.gc, 'remember_young_pointer_from_array3',
None)
if func is not None:
# func should not be a bound method, but a real function
@@ -471,7 +471,8 @@
self.write_barrier_from_array_failing_case_ptr = \
getfn(func,
[annmodel.SomeAddress(),
- annmodel.SomeInteger()],
+ annmodel.SomeInteger(),
+ annmodel.SomeAddress()],
annmodel.s_None)
self.statistics_ptr = getfn(GCClass.statistics.im_func,
[s_gc, annmodel.SomeInteger()],
diff --git a/pypy/tool/gcc_cache.py b/pypy/tool/gcc_cache.py
--- a/pypy/tool/gcc_cache.py
+++ b/pypy/tool/gcc_cache.py
@@ -39,7 +39,16 @@
data = ''
if not (data.startswith('True') or data.startswith('FAIL\n')):
try:
- platform.compile(c_files, eci)
+ _previous = platform.log_errors
+ try:
+ platform.log_errors = False
+ platform.compile(c_files, eci)
+ finally:
+ del platform.log_errors
+ # ^^^remove from the instance --- needed so that it can
+ # compare equal to another instance without it
+ if platform.log_errors != _previous:
+ platform.log_errors = _previous
data = 'True'
path.write(data)
except CompilationError, e:
diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -1,4 +1,5 @@
import re, sys
+
from pypy.jit.metainterp.resoperation import rop, opname
from pypy.jit.tool.oparser import OpParser
@@ -51,6 +52,7 @@
# factory method
Op = Op
+ use_mock_model = True
@classmethod
def parse_from_input(cls, input):
@@ -96,7 +98,7 @@
def __init__(self, operations, storage):
if operations[0].name == 'debug_merge_point':
self.inline_level = int(operations[0].args[0])
- m = re.search('<code object ([<>\w]+), file \'(.+?)\', line (\d+)> #(\d+) (\w+)',
+ m = re.search('<code object ([<>\w]+)\. file \'(.+?)\'\. line (\d+)> #(\d+) (\w+)',
operations[0].getarg(1))
if m is None:
# a non-code loop, like StrLiteralSearch or something
diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py
--- a/pypy/tool/jitlogparser/test/test_parser.py
+++ b/pypy/tool/jitlogparser/test/test_parser.py
@@ -38,10 +38,10 @@
def test_split():
ops = parse('''
[i0]
- debug_merge_point(0, "<code object stuff, file '/I/dont/exist.py', line 200> #10 ADD")
- debug_merge_point(0, "<code object stuff, file '/I/dont/exist.py', line 200> #11 SUB")
+ debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD")
+ debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 200> #11 SUB")
i1 = int_add(i0, 1)
- debug_merge_point(0, "<code object stuff, file '/I/dont/exist.py', line 200> #11 SUB")
+ debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 200> #11 SUB")
i2 = int_add(i1, 1)
''')
res = Function.from_operations(ops.operations, LoopStorage())
@@ -54,12 +54,12 @@
def test_inlined_call():
ops = parse("""
[]
- debug_merge_point(0, '<code object inlined_call, file 'source.py', line 12> #28 CALL_FUNCTION')
+ debug_merge_point(0, '<code object inlined_call. file 'source.py'. line 12> #28 CALL_FUNCTION')
i18 = getfield_gc(p0, descr=<BoolFieldDescr pypy.interpreter.pyframe.PyFrame.inst_is_being_profiled 89>)
- debug_merge_point(1, '<code object inner, file 'source.py', line 9> #0 LOAD_FAST')
- debug_merge_point(1, '<code object inner, file 'source.py', line 9> #3 LOAD_CONST')
- debug_merge_point(1, '<code object inner, file 'source.py', line 9> #7 RETURN_VALUE')
- debug_merge_point(0, '<code object inlined_call, file 'source.py', line 12> #31 STORE_FAST')
+ debug_merge_point(1, '<code object inner. file 'source.py'. line 9> #0 LOAD_FAST')
+ debug_merge_point(1, '<code object inner. file 'source.py'. line 9> #3 LOAD_CONST')
+ debug_merge_point(1, '<code object inner. file 'source.py'. line 9> #7 RETURN_VALUE')
+ debug_merge_point(0, '<code object inlined_call. file 'source.py'. line 12> #31 STORE_FAST')
""")
res = Function.from_operations(ops.operations, LoopStorage())
assert len(res.chunks) == 3 # two chunks + inlined call
@@ -72,10 +72,10 @@
def test_name():
ops = parse('''
[i0]
- debug_merge_point(0, "<code object stuff, file '/I/dont/exist.py', line 200> #10 ADD")
- debug_merge_point(0, "<code object stuff, file '/I/dont/exist.py', line 201> #11 SUB")
+ debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD")
+ debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 201> #11 SUB")
i1 = int_add(i0, 1)
- debug_merge_point(0, "<code object stuff, file '/I/dont/exist.py', line 202> #11 SUB")
+ debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 202> #11 SUB")
i2 = int_add(i1, 1)
''')
res = Function.from_operations(ops.operations, LoopStorage())
@@ -89,10 +89,10 @@
ops = parse('''
[i0]
i3 = int_add(i0, 1)
- debug_merge_point(0, "<code object stuff, file '/I/dont/exist.py', line 200> #10 ADD")
- debug_merge_point(0, "<code object stuff, file '/I/dont/exist.py', line 201> #11 SUB")
+ debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD")
+ debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 201> #11 SUB")
i1 = int_add(i0, 1)
- debug_merge_point(0, "<code object stuff, file '/I/dont/exist.py', line 202> #11 SUB")
+ debug_merge_point(0, "<code object stuff. file '/I/dont/exist.py'. line 202> #11 SUB")
i2 = int_add(i1, 1)
''')
res = Function.from_operations(ops.operations, LoopStorage())
@@ -102,10 +102,10 @@
fname = str(py.path.local(__file__).join('..', 'x.py'))
ops = parse('''
[i0, i1]
- debug_merge_point(0, "<code object f, file '%(fname)s', line 2> #0 LOAD_FAST")
- debug_merge_point(0, "<code object f, file '%(fname)s', line 2> #3 LOAD_FAST")
- debug_merge_point(0, "<code object f, file '%(fname)s', line 2> #6 BINARY_ADD")
- debug_merge_point(0, "<code object f, file '%(fname)s', line 2> #7 RETURN_VALUE")
+ debug_merge_point(0, "<code object f. file '%(fname)s'. line 2> #0 LOAD_FAST")
+ debug_merge_point(0, "<code object f. file '%(fname)s'. line 2> #3 LOAD_FAST")
+ debug_merge_point(0, "<code object f. file '%(fname)s'. line 2> #6 BINARY_ADD")
+ debug_merge_point(0, "<code object f. file '%(fname)s'. line 2> #7 RETURN_VALUE")
''' % locals())
res = Function.from_operations(ops.operations, LoopStorage())
assert res.chunks[1].lineno == 3
@@ -114,11 +114,11 @@
fname = str(py.path.local(__file__).join('..', 'x.py'))
ops = parse('''
[i0, i1]
- debug_merge_point(0, "<code object g, file '%(fname)s', line 5> #9 LOAD_FAST")
- debug_merge_point(0, "<code object g, file '%(fname)s', line 5> #12 LOAD_CONST")
- debug_merge_point(0, "<code object g, file '%(fname)s', line 5> #22 LOAD_CONST")
- debug_merge_point(0, "<code object g, file '%(fname)s', line 5> #28 LOAD_CONST")
- debug_merge_point(0, "<code object g, file '%(fname)s', line 5> #6 SETUP_LOOP")
+ debug_merge_point(0, "<code object g. file '%(fname)s'. line 5> #9 LOAD_FAST")
+ debug_merge_point(0, "<code object g. file '%(fname)s'. line 5> #12 LOAD_CONST")
+ debug_merge_point(0, "<code object g. file '%(fname)s'. line 5> #22 LOAD_CONST")
+ debug_merge_point(0, "<code object g. file '%(fname)s'. line 5> #28 LOAD_CONST")
+ debug_merge_point(0, "<code object g. file '%(fname)s'. line 5> #6 SETUP_LOOP")
''' % locals())
res = Function.from_operations(ops.operations, LoopStorage())
assert res.linerange == (7, 9)
@@ -128,7 +128,7 @@
fname = str(py.path.local(__file__).join('..', 'x.py'))
ops = parse("""
[p6, p1]
- debug_merge_point(0, '<code object h, file '%(fname)s', line 11> #17 FOR_ITER')
+ debug_merge_point(0, '<code object h. file '%(fname)s'. line 11> #17 FOR_ITER')
guard_class(p6, 144264192, descr=<Guard2>)
p12 = getfield_gc(p6, descr=<GcPtrFieldDescr pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_w_seq 12>)
""" % locals())
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -1,8 +1,13 @@
import autopath
import py
-from pypy.interpreter import gateway
+from pypy.interpreter import gateway, pycode
from pypy.interpreter.error import OperationError
+try:
+ from _pytest.assertion.newinterpret import interpret
+except ImportError:
+ from _pytest.assertion.oldinterpret import interpret
+
# ____________________________________________________________
class AppCode(object):
@@ -51,13 +56,11 @@
space = self.space
for key, w_value in vars.items():
space.setitem(self.w_locals, space.wrap(key), w_value)
- return space.eval(code, self.w_globals, self.w_locals)
-
- def exec_(self, code, **vars):
- space = self.space
- for key, w_value in vars.items():
- space.setitem(self.w_locals, space.wrap(key), w_value)
- space.exec_(code, self.w_globals, self.w_locals)
+ if isinstance(code, str):
+ return space.eval(code, self.w_globals, self.w_locals)
+ pyc = pycode.PyCode._from_code(space, code)
+ return pyc.exec_host_bytecode(self.w_globals, self.w_locals)
+ exec_ = eval
def repr(self, w_value):
return self.space.unwrap(self.space.repr(w_value))
@@ -163,8 +166,8 @@
except py.error.ENOENT:
source = None
from pypy import conftest
- if source and not py.test.config.option.nomagic:
- msg = py.code._reinterpret_old(source, runner, should_fail=True)
+ if source and py.test.config._assertstate.mode != "off":
+ msg = interpret(source, runner, should_fail=True)
space.setattr(w_self, space.wrap('args'),
space.newtuple([space.wrap(msg)]))
w_msg = space.wrap(msg)
diff --git a/pypy/tool/pytest/test/test_pytestsupport.py b/pypy/tool/pytest/test/test_pytestsupport.py
--- a/pypy/tool/pytest/test/test_pytestsupport.py
+++ b/pypy/tool/pytest/test/test_pytestsupport.py
@@ -4,7 +4,7 @@
from pypy.interpreter.pycode import PyCode
from pypy.interpreter.pyframe import PyFrame
from pypy.tool.pytest.appsupport import (AppFrame, build_pytest_assertion,
- AppExceptionInfo)
+ AppExceptionInfo, interpret)
import py
from pypy.tool.udir import udir
import os
@@ -22,8 +22,8 @@
co = PyCode._from_code(space, somefunc.func_code)
pyframe = PyFrame(space, co, space.newdict(), None)
runner = AppFrame(space, pyframe)
- py.code._reinterpret_old("f = lambda x: x+1", runner, should_fail=False)
- msg = py.code._reinterpret_old("assert isinstance(f(2), float)", runner)
+ interpret("f = lambda x: x+1", runner, should_fail=False)
+ msg = interpret("assert isinstance(f(2), float)", runner)
assert msg.startswith("assert isinstance(3, float)\n"
" + where 3 = ")
@@ -58,6 +58,12 @@
except AssertionError, e:
assert e.msg == "Failed"
+def app_test_comparison():
+ try:
+ assert 3 > 4
+ except AssertionError, e:
+ assert "3 > 4" in e.msg
+
def test_appexecinfo(space):
try:
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -3,9 +3,9 @@
It uses 'pypy/translator/goal/pypy-c' and parts of the rest of the working
copy. Usage:
- package.py root-pypy-dir [name-of-archive] [name-of-pypy-c]
+ package.py root-pypy-dir [name-of-archive] [name-of-pypy-c] [destination-for-tarball] [pypy-c-path]
-Usually you would do: package.py ../../.. pypy-VER-PLATFORM.
+Usually you would do: package.py ../../.. pypy-VER-PLATFORM
The output is found in the directory /tmp/usession-YOURNAME/build/.
"""
@@ -122,7 +122,10 @@
zf.close()
else:
archive = str(builddir.join(name + '.tar.bz2'))
- e = os.system('tar --owner=root --group=root --numeric-owner -cvjf ' + archive + " " + name)
+ if sys.platform == 'darwin':
+ e = os.system('tar --numeric-owner -cvjf ' + archive + " " + name)
+ else:
+ e = os.system('tar --owner=root --group=root --numeric-owner -cvjf ' + archive + " " + name)
if e:
raise OSError('"tar" returned exit status %r' % e)
finally:
diff --git a/pypy/tool/test/test_gcc_cache.py b/pypy/tool/test/test_gcc_cache.py
--- a/pypy/tool/test/test_gcc_cache.py
+++ b/pypy/tool/test/test_gcc_cache.py
@@ -1,11 +1,13 @@
-
+import sys
from pypy.tool.gcc_cache import *
from pypy.tool.udir import udir
-import md5
+import md5, cStringIO
from pypy.translator.tool.cbuild import ExternalCompilationInfo
+localudir = udir.join('test_gcc_cache').ensure(dir=1)
+
def test_gcc_exec():
- f = udir.join("x.c")
+ f = localudir.join("x.c")
f.write("""
#include <stdio.h>
#include <test_gcc_exec.h>
@@ -15,8 +17,8 @@
return 0;
}
""")
- dir1 = udir.join('test_gcc_exec_dir1').ensure(dir=1)
- dir2 = udir.join('test_gcc_exec_dir2').ensure(dir=1)
+ dir1 = localudir.join('test_gcc_exec_dir1').ensure(dir=1)
+ dir2 = localudir.join('test_gcc_exec_dir2').ensure(dir=1)
dir1.join('test_gcc_exec.h').write('#define ANSWER 3\n')
dir2.join('test_gcc_exec.h').write('#define ANSWER 42\n')
eci = ExternalCompilationInfo(include_dirs=[str(dir1)])
@@ -36,7 +38,7 @@
print '>>>'
def test_gcc_ask():
- f = udir.join("y.c")
+ f = localudir.join("y.c")
f.write("""
#include <stdio.h>
#include <test_gcc_ask.h>
@@ -46,8 +48,8 @@
return 0;
}
""")
- dir1 = udir.join('test_gcc_ask_dir1').ensure(dir=1)
- dir2 = udir.join('test_gcc_ask_dir2').ensure(dir=1)
+ dir1 = localudir.join('test_gcc_ask_dir1').ensure(dir=1)
+ dir2 = localudir.join('test_gcc_ask_dir2').ensure(dir=1)
dir1.join('test_gcc_ask.h').write('/* hello world */\n')
dir2.join('test_gcc_ask.h').write('#error boom\n')
eci = ExternalCompilationInfo(include_dirs=[str(dir1)])
@@ -63,3 +65,15 @@
print '<<<'
print err
print '>>>'
+
+def test_gcc_ask_doesnt_log_errors():
+ f = localudir.join('z.c')
+ f.write("""this file is not valid C code\n""")
+ eci = ExternalCompilationInfo()
+ oldstderr = sys.stderr
+ try:
+ sys.stderr = capture = cStringIO.StringIO()
+ py.test.raises(CompilationError, try_compile_cache, [f], eci)
+ finally:
+ sys.stderr = oldstderr
+ assert 'ERROR' not in capture.getvalue().upper()
diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -297,6 +297,13 @@
gc_startup_code = RefcountingGcPolicy.gc_startup_code.im_func
+ def compilation_info(self):
+ eci = BasicGcPolicy.compilation_info(self)
+ eci = eci.merge(ExternalCompilationInfo(
+ post_include_bits=['#define USING_NO_GC_AT_ALL'],
+ ))
+ return eci
+
class FrameworkGcPolicy(BasicGcPolicy):
transformerclass = framework.FrameworkGCTransformer
diff --git a/pypy/translator/c/gcc/test/elf/track12.s b/pypy/translator/c/gcc/test/elf/track12.s
new file mode 100644
--- /dev/null
+++ b/pypy/translator/c/gcc/test/elf/track12.s
@@ -0,0 +1,9 @@
+ .type pypy_f, @function
+pypy_f:
+ pushl 4(%esp)
+ call pypy_other
+ ;; expected {4(%esp) | %ebx, %esi, %edi, %ebp | (%esp)}
+ popl %eax
+ /* GCROOT %eax */
+ ret
+ .size pypy_f, .-pypy_f
diff --git a/pypy/translator/c/gcc/test/elf/track13.s b/pypy/translator/c/gcc/test/elf/track13.s
new file mode 100644
--- /dev/null
+++ b/pypy/translator/c/gcc/test/elf/track13.s
@@ -0,0 +1,9 @@
+ .type pypy_f, @function
+pypy_f:
+ call pypy_other
+ ;; expected {(%esp) | %ebx, %esi, %edi, %ebp | 8(%esp)}
+ pushl 8(%esp)
+ popl %eax
+ /* GCROOT %eax */
+ ret
+ .size pypy_f, .-pypy_f
diff --git a/pypy/translator/c/gcc/test/msvc/track_and_esp.s b/pypy/translator/c/gcc/test/msvc/track_and_esp.s
new file mode 100644
--- /dev/null
+++ b/pypy/translator/c/gcc/test/msvc/track_and_esp.s
@@ -0,0 +1,474 @@
+PUBLIC ??_C at _0BN@BIPHFGBC at pypy_g_ll_math_ll_math_frexp?$AA@ ; `string'
+PUBLIC _pypy_g_ll_math_ll_math_frexp
+; COMDAT ??_C at _0BN@BIPHFGBC at pypy_g_ll_math_ll_math_frexp?$AA@
+CONST SEGMENT
+??_C at _0BN@BIPHFGBC at pypy_g_ll_math_ll_math_frexp?$AA@ DB 'pypy_g_ll_math_l'
+ DB 'l_math_frexp', 00H ; `string'
+; Function compile flags: /Ogtpy
+CONST ENDS
+; COMDAT _pypy_g_ll_math_ll_math_frexp
+_TEXT SEGMENT
+_l_mantissa_0$ = -8 ; size = 8
+_l_v21638$ = -8 ; size = 8
+_l_x_14$ = 8 ; size = 8
+_pypy_g_ll_math_ll_math_frexp PROC ; COMDAT
+
+; 58245: struct pypy_tuple2_0 *pypy_g_ll_math_ll_math_frexp(double l_x_14) {
+
+ push ebp
+ mov ebp, esp
+ and esp, -64 ; ffffffc0H
+
+; 58246: long *l_exp_p_0; double l_mantissa_0; bool_t l_v21641;
+; 58247: bool_t l_v21643; bool_t l_v21644; bool_t l_v21646; bool_t l_v21647;
+; 58248: bool_t l_v21652; bool_t l_v21653; bool_t l_v21660; bool_t l_v21666;
+; 58249: bool_t l_v21670; bool_t l_v21674; bool_t l_v21676; double l_v21638;
+; 58250: long l_v21637; long l_v21649; long l_v21651; long l_v21677;
+; 58251: long l_v21678; struct pypy_exceptions_Exception0 *l_v21687;
+; 58252: struct pypy_header0 *l_v21654; struct pypy_object0 *l_v21682;
+; 58253: struct pypy_object0 *l_v21691; struct pypy_object_vtable0 *l_v21665;
+; 58254: struct pypy_object_vtable0 *l_v21669;
+; 58255: struct pypy_object_vtable0 *l_v21675;
+; 58256: struct pypy_object_vtable0 *l_v21683; struct pypy_tuple2_0 *l_v21640;
+; 58257: struct pypy_tuple2_0 *l_v21695; void* l_v21639; void* l_v21648;
+; 58258: void* l_v21650; void* l_v21656; void* l_v21658; void* l_v21659;
+; 58259: void* l_v21668; void* l_v21672; void* l_v21679; void* l_v21688;
+; 58260: void* l_v21696;
+; 58261: goto block0;
+; 58262:
+; 58263: block0:
+; 58264: l_v21641 = pypy_g_ll_math_ll_math_isnan(l_x_14);
+
+ fld QWORD PTR _l_x_14$[ebp]
+ sub esp, 52 ; 00000034H
+ push ebx
+ push esi
+ push edi
+ sub esp, 8
+ fstp QWORD PTR [esp]
+$block0$88239:
+ call _pypy_g_ll_math_ll_math_isnan
+
+; 58265: pypy_asm_gc_nocollect(pypy_g_ll_math_ll_math_isnan);
+; 58266: l_v21643 = l_v21641;
+; 58267: if (l_v21643) {
+; 58268: l_v21637 = 0L;
+; 58269: l_v21638 = l_x_14;
+
+ fld QWORD PTR _l_x_14$[ebp]
+ add esp, 8
+ test al, al
+
+; 58270: goto block3;
+
+ jne SHORT $LN10 at pypy_g_ll_@159
+
+; 58271: }
+; 58272: goto block1;
+; 58273:
+; 58274: block1:
+; 58275: l_v21644 = pypy_g_ll_math_ll_math_isinf(l_x_14);
+
+ sub esp, 8
+ fstp QWORD PTR [esp]
+$block1$88243:
+ call _pypy_g_ll_math_ll_math_isinf
+ add esp, 8
+
+; 58276: pypy_asm_gc_nocollect(pypy_g_ll_math_ll_math_isinf);
+; 58277: l_v21646 = l_v21644;
+; 58278: if (l_v21646) {
+
+ test al, al
+ je SHORT $block2$88245
+
+; 58279: l_v21637 = 0L;
+; 58280: l_v21638 = l_x_14;
+
+ fld QWORD PTR _l_x_14$[ebp]
+$LN10 at pypy_g_ll_@159:
+
+; 58288: goto block14;
+; 58289: }
+; 58290: l_v21637 = 0L;
+
+ xor edi, edi
+$LN30 at pypy_g_ll_@159:
+
+; 58291: l_v21638 = l_x_14;
+; 58292: goto block3;
+; 58293:
+; 58294: block3:
+; 58295: l_v21648 = (&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC)->ssgc_inst_free;
+
+ mov esi, DWORD PTR _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC+4
+ fstp QWORD PTR _l_v21638$[esp+64]
+
+; 58296: OP_RAW_MALLOC_USAGE((0 + ROUND_UP_FOR_ALLOCATION(sizeof(struct pypy_tuple2_0), sizeof(struct pypy_forwarding_stub0))), l_v21649);
+; 58297: l_v21650 = (&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC)->ssgc_inst_top_of_space;
+; 58298: OP_ADR_DELTA(l_v21650, l_v21648, l_v21651);
+
+ mov eax, DWORD PTR _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC+12
+ sub eax, esi
+
+; 58299: OP_INT_GT(l_v21649, l_v21651, l_v21652);
+
+ cmp eax, 24 ; 00000018H
+$block3$88242:
+
+; 58300: if (l_v21652) {
+
+ jge $block4$88260
+
+; 58334: l_v21695 = l_v21640;
+; 58335: goto block8;
+; 58336:
+; 58337: block8:
+; 58338: RPY_DEBUG_RETURN();
+; 58339: return l_v21695;
+; 58340:
+; 58341: block9:
+; 58342: PYPY_DEBUG_RECORD_TRACEBACK("ll_math_ll_math_frexp");
+; 58343: l_v21695 = ((struct pypy_tuple2_0 *) NULL);
+; 58344: goto block8;
+; 58345:
+; 58346: block10:
+; 58347: abort(); /* debug_llinterpcall should be unreachable */
+; 58348: l_v21665 = (&pypy_g_ExcData)->ed_exc_type;
+; 58349: l_v21666 = (l_v21665 == NULL);
+; 58350: if (!l_v21666) {
+; 58351: goto block11;
+; 58352: }
+; 58353: goto block5;
+; 58354:
+; 58355: block11:
+; 58356: PYPY_DEBUG_RECORD_TRACEBACK("ll_math_ll_math_frexp");
+; 58357: l_v21696 = NULL;
+; 58358: goto block6;
+; 58359:
+; 58360: block12:
+; 58361: l_v21668 = pypy_g_SemiSpaceGC_obtain_free_space((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), (0 + ROUND_UP_FOR_ALLOCATION(sizeof(struct pypy_tuple2_0), sizeof(struct pypy_forwarding_stub0))));
+
+ push 24 ; 00000018H
+ push OFFSET _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC
+$block12$88259:
+ call _pypy_g_SemiSpaceGC_obtain_free_space
+ ;; expected {4(%ebp) | 16(%esp), 12(%esp), 8(%esp), (%ebp) | }
+
+; 58362: l_v21669 = (&pypy_g_ExcData)->ed_exc_type;
+; 58363: l_v21670 = (l_v21669 == NULL);
+
+ xor ecx, ecx
+ add esp, 8
+ cmp DWORD PTR _pypy_g_ExcData, ecx
+
+; 58364: if (!l_v21670) {
+
+ je $LN5 at pypy_g_ll_@159
+
+; 58368: goto block4;
+; 58369:
+; 58370: block13:
+; 58371: PYPY_DEBUG_RECORD_TRACEBACK("ll_math_ll_math_frexp");
+
+ mov eax, DWORD PTR _pypydtcount
+ mov DWORD PTR _pypy_debug_tracebacks[eax*8], OFFSET ?loc@?N@??pypy_g_ll_math_ll_math_frexp@@9 at 9
+ mov DWORD PTR _pypy_debug_tracebacks[eax*8+4], ecx
+ inc eax
+ and eax, 8191 ; 00001fffH
+ mov DWORD PTR _pypy_debug_tracebacks[eax*8], OFFSET ?loc@?8??pypy_g_ll_math_ll_math_frexp@@9 at 9
+ mov DWORD PTR _pypy_debug_tracebacks[eax*8+4], ecx
+ inc eax
+ and eax, 8191 ; 00001fffH
+ mov DWORD PTR _pypydtcount, eax
+$block13$88313:
+$block9$88285:
+ xor eax, eax
+
+; 58423: goto block8;
+; 58424: }
+
+ pop edi
+ pop esi
+ pop ebx
+ mov esp, ebp
+ pop ebp
+ ret 0
+$block2$88245:
+
+; 58281: goto block3;
+; 58282: }
+; 58283: goto block2;
+; 58284:
+; 58285: block2:
+; 58286: OP_FLOAT_IS_TRUE(l_x_14, l_v21647);
+
+ fldz
+ fld QWORD PTR _l_x_14$[ebp]
+ fucom ST(1)
+ fnstsw ax
+ fstp ST(1)
+ test ah, 68 ; 00000044H
+
+; 58287: if (l_v21647) {
+
+ jnp $LN10 at pypy_g_ll_@159
+
+; 58372: l_v21696 = NULL;
+; 58373: goto block6;
+; 58374:
+; 58375: block14:
+; 58376: l_v21672 = pypy_g__ll_malloc_varsize_no_length__Signed_Signed_Sign(1L, (0 + 0), sizeof(long));
+
+ push 4
+ fstp ST(0)
+ push 0
+ push 1
+$block14$88247:
+ call _pypy_g__ll_malloc_varsize_no_length__Signed_Signed_Sign
+ ;; expected {4(%ebp) | 20(%esp), 16(%esp), 12(%esp), (%ebp) | }
+ mov esi, eax
+
+; 58377: OP_TRACK_ALLOC_START(l_v21672, /* nothing */);
+
+ push OFFSET ??_C at _0BN@BIPHFGBC at pypy_g_ll_math_ll_math_frexp?$AA@
+ push esi
+ call _pypy_debug_alloc_start
+ ;; expected {4(%ebp) | 28(%esp), 24(%esp), 20(%esp), (%ebp) | }
+ add esp, 20 ; 00000014H
+
+; 58378: l_exp_p_0 = (long *)l_v21672;
+; 58379: l_v21674 = (l_exp_p_0 != NULL);
+
+ test esi, esi
+
+; 58380: if (!l_v21674) {
+
+ jne SHORT $block15$88324
+
+; 58418: goto block8;
+; 58419:
+; 58420: block18:
+; 58421: PYPY_DEBUG_RECORD_TRACEBACK("ll_math_ll_math_frexp");
+
+ mov eax, DWORD PTR _pypydtcount
+ mov DWORD PTR _pypy_debug_tracebacks[eax*8], OFFSET ?loc@?BB@??pypy_g_ll_math_ll_math_frexp@@9 at 9
+ mov DWORD PTR _pypy_debug_tracebacks[eax*8+4], esi
+ inc eax
+ and eax, 8191 ; 00001fffH
+ mov DWORD PTR _pypydtcount, eax
+$block18$88323:
+
+; 58422: l_v21695 = ((struct pypy_tuple2_0 *) NULL);
+
+ xor eax, eax
+
+; 58423: goto block8;
+; 58424: }
+
+ pop edi
+ pop esi
+ pop ebx
+ mov esp, ebp
+ pop ebp
+ ret 0
+$block15$88324:
+
+; 58381: goto block18;
+; 58382: }
+; 58383: goto block15;
+; 58384:
+; 58385: block15:
+; 58386: l_mantissa_0 = pypy_g_frexp__Float_arrayPtr_star_2(l_x_14, l_exp_p_0);
+
+ fld QWORD PTR _l_x_14$[ebp]
+ push esi
+ sub esp, 8
+ fstp QWORD PTR [esp]
+ call _pypy_g_frexp__Float_arrayPtr_star_2
+ ;; expected {4(%ebp) | 20(%esp), 16(%esp), 12(%esp), (%ebp) | }
+
+; 58387: l_v21675 = (&pypy_g_ExcData)->ed_exc_type;
+; 58388: l_v21676 = (l_v21675 == NULL);
+
+ mov edi, DWORD PTR _pypy_g_ExcData
+ fstp QWORD PTR _l_mantissa_0$[esp+76]
+ add esp, 12 ; 0000000cH
+ test edi, edi
+
+; 58389: if (!l_v21676) {
+
+ je SHORT $block16$88328
+
+; 58403:
+; 58404: block17:
+; 58405: l_v21682 = (&pypy_g_ExcData)->ed_exc_value;
+; 58406: l_v21683 = (&pypy_g_ExcData)->ed_exc_type;
+; 58407: PYPY_DEBUG_CATCH_EXCEPTION("ll_math_ll_math_frexp", l_v21683, l_v21683 == (&pypy_g_py__code_assertion_AssertionError_vtable.ae_super.ae_super.se_super.e_super) || l_v21683 == (&pypy_g_exceptions_NotImplementedError_vtable.nie_super.re_super.se_super.e_super));
+
+ mov eax, DWORD PTR _pypydtcount
+ mov ebx, DWORD PTR _pypy_g_ExcData+4
+ mov DWORD PTR _pypy_debug_tracebacks[eax*8], OFFSET ?loc@?BA@??pypy_g_ll_math_ll_math_frexp@@9 at 9
+ mov DWORD PTR _pypy_debug_tracebacks[eax*8+4], edi
+ inc eax
+ and eax, 8191 ; 00001fffH
+$block17$88327:
+ mov DWORD PTR _pypydtcount, eax
+ cmp edi, OFFSET _pypy_g_py__code_assertion_AssertionError_vtable
+ je SHORT $LN1 at pypy_g_ll_@159
+ cmp edi, OFFSET _pypy_g_exceptions_NotImplementedError_vtable
+ jne SHORT $LN2 at pypy_g_ll_@159
+$LN1 at pypy_g_ll_@159:
+ call _pypy_debug_catch_fatal_exception
+$LN2 at pypy_g_ll_@159:
+
+; 58408: (&pypy_g_ExcData)->ed_exc_value = ((struct pypy_object0 *) NULL);
+
+ xor eax, eax
+
+; 58409: (&pypy_g_ExcData)->ed_exc_type = ((struct pypy_object_vtable0 *) NULL);
+; 58410: l_v21687 = (struct pypy_exceptions_Exception0 *)l_v21682;
+; 58411: l_v21688 = (void*)l_exp_p_0;
+; 58412: OP_TRACK_ALLOC_STOP(l_v21688, /* nothing */);
+
+ push esi
+ mov DWORD PTR _pypy_g_ExcData+4, eax
+ mov DWORD PTR _pypy_g_ExcData, eax
+ call _pypy_debug_alloc_stop
+ ;; expected {4(%ebp) | 12(%esp), 8(%esp), 4(%esp), (%ebp) | }
+
+; 58413: OP_RAW_FREE(l_v21688, /* nothing */);
+
+ push esi
+ call _PyObject_Free
+ ;; expected {4(%ebp) | 16(%esp), 12(%esp), 8(%esp), (%ebp) | }
+
+; 58414: l_v21691 = (struct pypy_object0 *)l_v21687;
+; 58415: pypy_g_RPyReRaiseException(l_v21683, l_v21691);
+
+ push ebx
+ push edi
+ call _pypy_g_RPyReRaiseException
+ add esp, 16 ; 00000010H
+
+; 58416: pypy_asm_gc_nocollect(pypy_g_RPyReRaiseException);
+; 58417: l_v21695 = ((struct pypy_tuple2_0 *) NULL);
+
+ xor eax, eax
+
+; 58423: goto block8;
+; 58424: }
+
+ pop edi
+ pop esi
+ pop ebx
+ mov esp, ebp
+ pop ebp
+ ret 0
+$block16$88328:
+
+; 58390: goto block17;
+; 58391: }
+; 58392: goto block16;
+; 58393:
+; 58394: block16:
+; 58395: l_v21677 = RPyBareItem(l_exp_p_0, 0L);
+; 58396: l_v21678 = (long)(l_v21677);
+
+ mov edi, DWORD PTR [esi]
+
+; 58397: l_v21679 = (void*)l_exp_p_0;
+; 58398: OP_TRACK_ALLOC_STOP(l_v21679, /* nothing */);
+
+ push esi
+ call _pypy_debug_alloc_stop
+ ;; expected {4(%ebp) | 12(%esp), 8(%esp), 4(%esp), (%ebp) | }
+
+; 58399: OP_RAW_FREE(l_v21679, /* nothing */);
+
+ push esi
+ call _PyObject_Free
+ ;; expected {4(%ebp) | 16(%esp), 12(%esp), 8(%esp), (%ebp) | }
+
+; 58400: l_v21637 = l_v21678;
+; 58401: l_v21638 = l_mantissa_0;
+
+ fld QWORD PTR _l_mantissa_0$[esp+72]
+ add esp, 8
+
+; 58402: goto block3;
+
+ jmp $LN30 at pypy_g_ll_@159
+$LN5 at pypy_g_ll_@159:
+
+; 58365: goto block13;
+; 58366: }
+; 58367: l_v21639 = l_v21668;
+
+ mov esi, eax
+$block4$88260:
+$block5$88263:
+
+; 58301: goto block12;
+; 58302: }
+; 58303: l_v21639 = l_v21648;
+; 58304: goto block4;
+; 58305:
+; 58306: block4:
+; 58307: OP_INT_IS_TRUE(RUNNING_ON_LLINTERP, l_v21653);
+; 58308: if (l_v21653) {
+; 58309: goto block10;
+; 58310: }
+; 58311: goto block5;
+; 58312:
+; 58313: block5:
+; 58314: l_v21654 = (struct pypy_header0 *)l_v21639;
+; 58315: RPyField(l_v21654, h_tid) = (GROUP_MEMBER_OFFSET(struct group_pypy_g_typeinfo_s, member20)+0L);
+
+ test esi, esi
+ jne SHORT $LN18 at pypy_g_ll_@159
+ call _RPyAbort
+$LN18 at pypy_g_ll_@159:
+
+; 58316: OP_ADR_ADD(l_v21639, (0 + ROUND_UP_FOR_ALLOCATION(sizeof(struct pypy_tuple2_0), sizeof(struct pypy_forwarding_stub0))), l_v21656);
+; 58317: (&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC)->ssgc_inst_free = l_v21656;
+; 58318: OP_ADR_ADD(l_v21639, 0, l_v21658);
+; 58319: l_v21659 = (void*)l_v21658;
+; 58320: l_v21696 = l_v21659;
+; 58321: goto block6;
+; 58322:
+; 58323: block6:
+; 58324: l_v21640 = (struct pypy_tuple2_0 *)l_v21696;
+; 58325: l_v21660 = (l_v21640 != NULL);
+; 58326: if (!l_v21660) {
+; 58327: goto block9;
+; 58328: }
+; 58329: goto block7;
+; 58330:
+; 58331: block7:
+; 58332: RPyField(l_v21640, t_item0) = l_v21638;
+
+ fld QWORD PTR _l_v21638$[esp+64]
+ mov DWORD PTR [esi], 81 ; 00000051H
+ lea ecx, DWORD PTR [esi+24]
+ mov DWORD PTR _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC+4, ecx
+ fstp QWORD PTR [esi+8]
+
+; 58333: RPyField(l_v21640, t_item1) = l_v21637;
+
+ mov DWORD PTR [esi+16], edi
+
+; 58423: goto block8;
+; 58424: }
+
+ pop edi
+ mov eax, esi
+ pop esi
+$block6$88281:
+$block8$88289:
+ pop ebx
+ mov esp, ebp
+ pop ebp
+ ret 0
+_pypy_g_ll_math_ll_math_frexp ENDP
+_TEXT ENDS
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -271,7 +271,8 @@
match = self.r_localvar_esp.match(localvar)
if match:
- if localvar == self.TOP_OF_STACK: # for pushl and popl, by
+ if localvar == self.TOP_OF_STACK_MINUS_WORD:
+ # for pushl and popl, by
hint = None # default ebp addressing is
else: # a bit nicer
hint = 'esp'
@@ -526,8 +527,9 @@
target = match.group("target")
if target == self.ESP:
# only for andl $-16, %esp used to align the stack in main().
- # main() should not be seen at all.
- raise AssertionError("instruction unexpected outside of main()")
+ # main() should not be seen at all. But on e.g. MSVC we see
+ # the instruction somewhere else too...
+ return InsnCannotFollowEsp()
else:
return self.binary_insn(line)
@@ -591,10 +593,12 @@
def _visit_push(self, line):
match = self.r_unaryinsn.match(line)
source = match.group(1)
- return [InsnStackAdjust(-self.WORD)] + self.insns_for_copy(source, self.TOP_OF_STACK)
+ return self.insns_for_copy(source, self.TOP_OF_STACK_MINUS_WORD) + \
+ [InsnStackAdjust(-self.WORD)]
def _visit_pop(self, target):
- return self.insns_for_copy(self.TOP_OF_STACK, target) + [InsnStackAdjust(+self.WORD)]
+ return [InsnStackAdjust(+self.WORD)] + \
+ self.insns_for_copy(self.TOP_OF_STACK_MINUS_WORD, target)
def _visit_prologue(self):
# for the prologue of functions that use %ebp as frame pointer
@@ -986,15 +990,15 @@
OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])'
LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)'
OFFSET_LABELS = 2**30
- TOP_OF_STACK = '0(%esp)'
+ TOP_OF_STACK_MINUS_WORD = '-4(%esp)'
r_functionstart = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$")
r_functionend = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$")
- LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]"
+ LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|-?\d*[(]%esp[)]"
LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]"
r_localvarnofp = re.compile(LOCALVAR)
r_localvarfp = re.compile(LOCALVARFP)
- r_localvar_esp = re.compile(r"(\d*)[(]%esp[)]")
+ r_localvar_esp = re.compile(r"(-?\d*)[(]%esp[)]")
r_localvar_ebp = re.compile(r"(-?\d*)[(]%ebp[)]")
r_rel_label = re.compile(r"(\d+):\s*$")
@@ -1047,7 +1051,7 @@
OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])'
LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)'
OFFSET_LABELS = 2**30
- TOP_OF_STACK = '0(%rsp)'
+ TOP_OF_STACK_MINUS_WORD = '-8(%rsp)'
r_functionstart = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$")
r_functionend = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$")
@@ -1143,7 +1147,7 @@
CALLEE_SAVE_REGISTERS = ['ebx', 'esi', 'edi', 'ebp']
REG2LOC = dict((_reg, LOC_REG | ((_i+1)<<2))
for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS))
- TOP_OF_STACK = 'DWORD PTR [esp]'
+ TOP_OF_STACK_MINUS_WORD = 'DWORD PTR [esp-4]'
OPERAND = r'(?:(:?WORD|DWORD|BYTE) PTR |OFFSET )?[_\w?:@$]*(?:[-+0-9]+)?(:?\[[-+*\w0-9]+\])?'
LABEL = r'([a-zA-Z_$@.][a-zA-Z0-9_$@.]*)'
@@ -1173,7 +1177,7 @@
r_gcroot_marker = re.compile(r"$1") # never matches
r_gcroot_marker_var = re.compile(r"DWORD PTR .+_constant_always_one_.+pypy_asm_gcroot")
r_gcnocollect_marker = re.compile(r"\spypy_asm_gc_nocollect\(("+OPERAND+")\);")
- r_bottom_marker = re.compile(r"; .+\tpypy_asm_stack_bottom\(\);")
+ r_bottom_marker = re.compile(r"; .+\spypy_asm_stack_bottom\(\);")
FUNCTIONS_NOT_RETURNING = {
'__exit': None,
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -570,7 +570,10 @@
mk.definition('ASMFILES', sfiles)
mk.definition('ASMLBLFILES', lblsfiles)
mk.definition('GCMAPFILES', gcmapfiles)
- mk.definition('DEBUGFLAGS', '-O2 -fomit-frame-pointer -g')
+ if sys.platform == 'win32':
+ mk.definition('DEBUGFLAGS', '/Zi')
+ else:
+ mk.definition('DEBUGFLAGS', '-O2 -fomit-frame-pointer -g')
if self.config.translation.shared:
mk.definition('PYPY_MAIN_FUNCTION', "pypy_main_startup")
@@ -623,7 +626,10 @@
mk.rule('.PRECIOUS', '%.s', "# don't remove .s files if Ctrl-C'ed")
else:
- mk.definition('DEBUGFLAGS', '-O1 -g')
+ if sys.platform == 'win32':
+ mk.definition('DEBUGFLAGS', '/Zi')
+ else:
+ mk.definition('DEBUGFLAGS', '-O1 -g')
mk.write()
#self.translator.platform,
# ,
@@ -900,8 +906,9 @@
print >> f, '}'
def commondefs(defines):
- from pypy.rlib.rarithmetic import LONG_BIT
+ from pypy.rlib.rarithmetic import LONG_BIT, LONGLONG_BIT
defines['PYPY_LONG_BIT'] = LONG_BIT
+ defines['PYPY_LONGLONG_BIT'] = LONGLONG_BIT
def add_extra_files(eci):
srcdir = py.path.local(autopath.pypydir).join('translator', 'c', 'src')
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -1031,7 +1031,7 @@
if (issubclass(value, BaseException) and
value.__module__ == 'exceptions'):
return 'PyExc_' + value.__name__
- if value is py.code._AssertionError:
+ if issubclass(value, AssertionError):
return 'PyExc_AssertionError'
if value is _StackOverflow:
return 'PyExc_RuntimeError'
diff --git a/pypy/translator/c/src/int.h b/pypy/translator/c/src/int.h
--- a/pypy/translator/c/src/int.h
+++ b/pypy/translator/c/src/int.h
@@ -73,15 +73,28 @@
/* NB. shifting has same limitations as C: the shift count must be
>= 0 and < LONG_BITS. */
-#define OP_INT_RSHIFT(x,y,r) r = Py_ARITHMETIC_RIGHT_SHIFT(long, x, y)
-#define OP_UINT_RSHIFT(x,y,r) r = (x) >> (y)
-#define OP_LLONG_RSHIFT(x,y,r) r = Py_ARITHMETIC_RIGHT_SHIFT(PY_LONG_LONG,x,y)
-#define OP_ULLONG_RSHIFT(x,y,r) r = (x) >> (y)
+#define CHECK_SHIFT_RANGE(y, bits) RPyAssert(y >= 0 && y < bits, \
+ "The shift count is outside of the supported range")
-#define OP_INT_LSHIFT(x,y,r) r = (x) << (y)
-#define OP_UINT_LSHIFT(x,y,r) r = (x) << (y)
-#define OP_LLONG_LSHIFT(x,y,r) r = (x) << (y)
-#define OP_ULLONG_LSHIFT(x,y,r) r = (x) << (y)
+
+#define OP_INT_RSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONG_BIT); \
+ r = Py_ARITHMETIC_RIGHT_SHIFT(long, x, (y))
+#define OP_UINT_RSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONG_BIT); \
+ r = (x) >> (y)
+#define OP_LLONG_RSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONGLONG_BIT); \
+ r = Py_ARITHMETIC_RIGHT_SHIFT(PY_LONG_LONG,x, (y))
+#define OP_ULLONG_RSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONGLONG_BIT); \
+ r = (x) >> (y)
+
+
+#define OP_INT_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONG_BIT); \
+ r = (x) << (y)
+#define OP_UINT_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONG_BIT); \
+ r = (x) << (y)
+#define OP_LLONG_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONGLONG_BIT); \
+ r = (x) << (y)
+#define OP_ULLONG_LSHIFT(x,y,r) CHECK_SHIFT_RANGE(y, PYPY_LONGLONG_BIT); \
+ r = (x) << (y)
#define OP_INT_LSHIFT_OVF(x,y,r) \
OP_INT_LSHIFT(x,y,r); \
diff --git a/pypy/translator/c/src/main.h b/pypy/translator/c/src/main.h
--- a/pypy/translator/c/src/main.h
+++ b/pypy/translator/c/src/main.h
@@ -79,6 +79,7 @@
fprintf(stderr, "Fatal error during initialization: %s\n", errmsg);
#endif
abort();
+ return 1;
}
int PYPY_MAIN_FUNCTION(int argc, char *argv[])
diff --git a/pypy/translator/c/src/mem.h b/pypy/translator/c/src/mem.h
--- a/pypy/translator/c/src/mem.h
+++ b/pypy/translator/c/src/mem.h
@@ -222,6 +222,15 @@
#endif /* USING_BOEHM_GC */
+
+#ifdef USING_NO_GC_AT_ALL
+#define OP_BOEHM_ZERO_MALLOC(size, r, restype, is_atomic, is_varsize) \
+ r = (restype) calloc(1, size);
+#define OP_BOEHM_DISAPPEARING_LINK(link, obj, r) /* nothing */
+#define OP_GC__DISABLE_FINALIZERS(r) /* nothing */
+#define OP_GC__ENABLE_FINALIZERS(r) /* nothing */
+#endif
+
/************************************************************/
/* weakref support */
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1117,6 +1117,7 @@
S = lltype.GcStruct('S', ('u', lltype.Ptr(U)))
A = lltype.GcArray(lltype.Ptr(S))
filename = self.filename_dump_typeids_z
+ open_flags = os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0)
def fn():
s = lltype.malloc(S)
@@ -1128,7 +1129,7 @@
#
p = rgc.get_typeids_z()
s = ''.join([p[i] for i in range(len(p))])
- fd = os.open(filename, os.O_WRONLY | os.O_CREAT, 0666)
+ fd = os.open(filename, open_flags, 0666)
os.write(fd, s)
os.close(fd)
return 0
@@ -1137,7 +1138,7 @@
def test_write_typeids_z(self):
self.run("write_typeids_z")
- f = open(self.filename_dump_typeids_z)
+ f = open(self.filename_dump_typeids_z, 'rb')
data_z = f.read()
f.close()
import zlib
diff --git a/pypy/translator/c/test/test_standalone.py b/pypy/translator/c/test/test_standalone.py
--- a/pypy/translator/c/test/test_standalone.py
+++ b/pypy/translator/c/test/test_standalone.py
@@ -596,6 +596,42 @@
# The traceback stops at f() because it's the first function that
# captures the AssertionError, which makes the program abort.
+ def test_int_lshift_too_large(self):
+ from pypy.rlib.rarithmetic import LONG_BIT, LONGLONG_BIT
+ def entry_point(argv):
+ a = int(argv[1])
+ b = int(argv[2])
+ print a << b
+ return 0
+
+ t, cbuilder = self.compile(entry_point, debug=True)
+ out = cbuilder.cmdexec("10 2", expect_crash=False)
+ assert out.strip() == str(10 << 2)
+ cases = [-4, LONG_BIT, LONGLONG_BIT]
+ for x in cases:
+ out, err = cbuilder.cmdexec("%s %s" % (1, x), expect_crash=True)
+ lines = err.strip()
+ assert 'The shift count is outside of the supported range' in lines
+
+ def test_llong_rshift_too_large(self):
+ from pypy.rlib.rarithmetic import LONG_BIT, LONGLONG_BIT
+ def entry_point(argv):
+ a = r_longlong(int(argv[1]))
+ b = r_longlong(int(argv[2]))
+ print a >> b
+ return 0
+
+ t, cbuilder = self.compile(entry_point, debug=True)
+ out = cbuilder.cmdexec("10 2", expect_crash=False)
+ assert out.strip() == str(10 >> 2)
+ out = cbuilder.cmdexec("%s %s" % (-42, LONGLONG_BIT - 1), expect_crash=False)
+ assert out.strip() == '-1'
+ cases = [-4, LONGLONG_BIT]
+ for x in cases:
+ out, err = cbuilder.cmdexec("%s %s" % (1, x), expect_crash=True)
+ lines = err.strip()
+ assert 'The shift count is outside of the supported range' in lines
+
def test_ll_assert_error_debug(self):
def entry_point(argv):
ll_assert(len(argv) != 1, "foobar")
diff --git a/pypy/translator/goal/targetnumpystandalone.py b/pypy/translator/goal/targetnumpystandalone.py
--- a/pypy/translator/goal/targetnumpystandalone.py
+++ b/pypy/translator/goal/targetnumpystandalone.py
@@ -10,46 +10,32 @@
"""
import time
-from pypy.module.micronumpy.numarray import SingleDimArray, Code, compute
+from pypy.module.micronumpy.compile import numpy_compile
from pypy.jit.codewriter.policy import JitPolicy
-
-def create_array(size):
- a = SingleDimArray(size)
- for i in range(size):
- a.storage[i] = float(i % 10)
- return a
+from pypy.rpython.annlowlevel import hlstr
def entry_point(argv):
if len(argv) != 3:
print __doc__
return 1
- bytecode = argv[1]
- for b in bytecode:
- if b not in 'alf':
- print "WRONG BYTECODE"
- print __doc__
- return 2
try:
size = int(argv[2])
except ValueError:
print "INVALID LITERAL FOR INT:", argv[2]
print __doc__
return 3
- no_arrays = bytecode.count('l')
- no_floats = bytecode.count('f')
- arrays = []
- floats = []
- for i in range(no_arrays):
- arrays.append(create_array(size))
- for i in range(no_floats):
- floats.append(float(i + 1))
- code = Code(bytecode, arrays, floats)
t0 = time.time()
- compute(code)
- print "bytecode:", bytecode, "size:", size
+ main(argv[0], size)
+ print "bytecode:", argv[0], "size:", size
print "took:", time.time() - t0
return 0
+def main(bc, size):
+ if not isinstance(bc, str):
+ bc = hlstr(bc) # for tests
+ a = numpy_compile(bc, size)
+ a = a.compute()
+
def target(*args):
return entry_point, None
diff --git a/pypy/translator/goal/translate.py b/pypy/translator/goal/translate.py
--- a/pypy/translator/goal/translate.py
+++ b/pypy/translator/goal/translate.py
@@ -103,6 +103,8 @@
specname = os.path.splitext(os.path.basename(targetspec))[0]
sys.path.insert(0, os.path.dirname(targetspec))
mod = __import__(specname)
+ if 'target' not in mod.__dict__:
+ raise Exception("file %r is not a valid targetxxx.py." % (targetspec,))
return mod.__dict__
def parse_options_and_load_target():
@@ -149,6 +151,9 @@
log.ERROR("Could not find target %r" % (arg, ))
sys.exit(1)
+ # apply the platform settings
+ set_platform(config)
+
targetspec = translateconfig.targetspec
targetspec_dic = load_target(targetspec)
@@ -164,9 +169,6 @@
existing_config=config,
translating=True)
- # apply the platform settings
- set_platform(config)
-
# apply the optimization level settings
set_opt_level(config, translateconfig.opt)
diff --git a/pypy/translator/platform/__init__.py b/pypy/translator/platform/__init__.py
--- a/pypy/translator/platform/__init__.py
+++ b/pypy/translator/platform/__init__.py
@@ -38,6 +38,7 @@
c_environ = None
relevant_environ = ()
+ log_errors = True
so_prefixes = ('',)
@@ -120,11 +121,12 @@
if returncode != 0:
errorfile = outname.new(ext='errors')
errorfile.write(stderr, 'wb')
- stderrlines = stderr.splitlines()
- for line in stderrlines:
- log.Error(line)
- # ^^^ don't use ERROR, because it might actually be fine.
- # Also, ERROR confuses lib-python/conftest.py.
+ if self.log_errors:
+ stderrlines = stderr.splitlines()
+ for line in stderrlines:
+ log.Error(line)
+ # ^^^ don't use ERROR, because it might actually be fine.
+ # Also, ERROR confuses lib-python/conftest.py.
raise CompilationError(stdout, stderr)
else:
for line in stderr.splitlines():
diff --git a/pypy/translator/platform/darwin.py b/pypy/translator/platform/darwin.py
--- a/pypy/translator/platform/darwin.py
+++ b/pypy/translator/platform/darwin.py
@@ -68,12 +68,10 @@
class Darwin_i386(Darwin):
name = "darwin_i386"
- link_flags = ('-arch', 'i386', '-mmacosx-version-min=10.4')
- cflags = ('-arch', 'i386', '-O3', '-fomit-frame-pointer',
- '-mmacosx-version-min=10.4')
+ link_flags = ('-arch', 'i386')
+ cflags = ('-arch', 'i386', '-O3', '-fomit-frame-pointer')
class Darwin_x86_64(Darwin):
name = "darwin_x86_64"
- link_flags = ('-arch', 'x86_64', '-mmacosx-version-min=10.4')
- cflags = ('-arch', 'x86_64', '-O3', '-fomit-frame-pointer',
- '-mmacosx-version-min=10.4')
+ link_flags = ('-arch', 'x86_64')
+ cflags = ('-arch', 'x86_64', '-O3', '-fomit-frame-pointer')
diff --git a/pytest.py b/pytest.py
old mode 100644
new mode 100755
--- a/pytest.py
+++ b/pytest.py
@@ -1,7 +1,6 @@
+#!/usr/bin/env python
"""
unit and functional testing with Python.
-(pypy version of startup script)
-see http://pytest.org for details.
"""
__all__ = ['main']
@@ -9,23 +8,6 @@
from _pytest import core as cmdline
from _pytest import __version__
-# This pytest.py script is located in the pypy source tree
-# which has a copy of pytest and py within its source tree.
-# If the environment also has an installed version of pytest/py
-# we are bound to get warnings so we disable them.
-# XXX eventually pytest and py should not be inlined shipped
-# with the pypy source code but become a requirement for installation.
-
-import warnings
-warnings.filterwarnings("ignore",
- "Module py was already imported", category=UserWarning)
-warnings.filterwarnings("ignore",
- "Module _pytest was already imported",
- category=UserWarning)
-warnings.filterwarnings("ignore",
- "Module pytest was already imported",
- category=UserWarning)
-
if __name__ == '__main__': # if run as a script or by 'python -m pytest'
raise SystemExit(main())
else:
More information about the pypy-commit
mailing list