[pypy-commit] pypy default: Merge refactor-wrapped-del: clean up and generalize the
arigo
noreply at buildbot.pypy.org
Wed Jul 13 14:16:57 CEST 2011
Author: Armin Rigo <arigo at tunes.org>
Branch:
Changeset: r45543:f9a24c80bd03
Date: 2011-07-13 14:16 +0200
http://bitbucket.org/pypy/pypy/changeset/f9a24c80bd03/
Log: Merge refactor-wrapped-del: clean up and generalize the way our
W_Root's __del__ works. They should not do too much, which is now
checked during translation, but they can call
self.enqueue_for_destruction() by passing them a callback, possibly
several times. The callbacks are invoked at a later safe point.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -130,6 +130,9 @@
raise operationerrfmt(space.w_TypeError,
"cannot create weak reference to '%s' object", typename)
+ def delweakref(self):
+ pass
+
def clear_all_weakrefs(self):
"""Call this at the beginning of interp-level __del__() methods
in subclasses. It ensures that weakrefs (if any) are cleared
@@ -143,29 +146,28 @@
# app-level, e.g. a user-defined __del__(), and this code
# tries to use weakrefs again, it won't reuse the broken
# (already-cleared) weakrefs from this lifeline.
- self.setweakref(lifeline.space, None)
+ self.delweakref()
lifeline.clear_all_weakrefs()
- __already_enqueued_for_destruction = False
+ __already_enqueued_for_destruction = ()
- def _enqueue_for_destruction(self, space, call_user_del=True):
+ def enqueue_for_destruction(self, space, callback, descrname):
"""Put the object in the destructor queue of the space.
- At a later, safe point in time, UserDelAction will use
- space.userdel() to call the object's app-level __del__ method.
+ At a later, safe point in time, UserDelAction will call
+ callback(self). If that raises OperationError, prints it
+ to stderr with the descrname string.
+
+ Note that 'callback' will usually need to start with:
+ assert isinstance(self, W_SpecificClass)
"""
# this function always resurect the object, so when
# running on top of CPython we must manually ensure that
# we enqueue it only once
if not we_are_translated():
- if self.__already_enqueued_for_destruction:
+ if callback in self.__already_enqueued_for_destruction:
return
- self.__already_enqueued_for_destruction = True
- self.clear_all_weakrefs()
- if call_user_del:
- space.user_del_action.register_dying_object(self)
-
- def _call_builtin_destructor(self):
- pass # method overridden in typedef.py
+ self.__already_enqueued_for_destruction += (callback,)
+ space.user_del_action.register_callback(self, callback, descrname)
# hooks that the mapdict implementations needs:
def _get_mapdict_map(self):
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -484,44 +484,31 @@
def __init__(self, space):
AsyncAction.__init__(self, space)
- self.dying_objects_w = []
- self.weakrefs_w = []
+ self.dying_objects = []
self.finalizers_lock_count = 0
- def register_dying_object(self, w_obj):
- self.dying_objects_w.append(w_obj)
- self.fire()
-
- def register_weakref_callback(self, w_ref):
- self.weakrefs_w.append(w_ref)
+ def register_callback(self, w_obj, callback, descrname):
+ self.dying_objects.append((w_obj, callback, descrname))
self.fire()
def perform(self, executioncontext, frame):
if self.finalizers_lock_count > 0:
return
- # Each call to perform() first grabs the self.dying_objects_w
+ # Each call to perform() first grabs the self.dying_objects
# and replaces it with an empty list. We do this to try to
# avoid too deep recursions of the kind of __del__ being called
# while in the middle of another __del__ call.
- pending_w = self.dying_objects_w
- self.dying_objects_w = []
+ pending = self.dying_objects
+ self.dying_objects = []
space = self.space
- for i in range(len(pending_w)):
- w_obj = pending_w[i]
- pending_w[i] = None
+ for i in range(len(pending)):
+ w_obj, callback, descrname = pending[i]
+ pending[i] = (None, None, None)
try:
- space.userdel(w_obj)
+ callback(w_obj)
except OperationError, e:
- e.write_unraisable(space, 'method __del__ of ', w_obj)
+ e.write_unraisable(space, descrname, w_obj)
e.clear(space) # break up reference cycles
- # finally, this calls the interp-level destructor for the
- # cases where there is both an app-level and a built-in __del__.
- w_obj._call_builtin_destructor()
- pending_w = self.weakrefs_w
- self.weakrefs_w = []
- for i in range(len(pending_w)):
- w_ref = pending_w[i]
- w_ref.activate_callback()
class FrameTraceAction(AsyncAction):
"""An action that calls the local trace functions (w_f_trace)."""
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -114,6 +114,7 @@
def descr_close(self):
"""x.close(arg) -> raise GeneratorExit inside generator."""
+ assert isinstance(self, GeneratorIterator)
space = self.space
try:
w_retval = self.throw(space.w_GeneratorExit, space.w_None,
@@ -141,22 +142,16 @@
code_name = self.pycode.co_name
return space.wrap(code_name)
- def descr__del__(self):
- """
- applevel __del__, which is called at a safe point after the
- interp-level __del__ enqueued the object for destruction
- """
- self.descr_close()
-
def __del__(self):
# Only bother enqueuing self to raise an exception if the frame is
# still not finished and finally or except blocks are present.
- must_call_close = False
if self.frame is not None:
block = self.frame.lastblock
while block is not None:
if not isinstance(block, LoopBlock):
- must_call_close = True
+ self.clear_all_weakrefs()
+ self.enqueue_for_destruction(self.space,
+ GeneratorIterator.descr_close,
+ "interrupting generator of ")
break
block = block.previous
- self._enqueue_for_destruction(self.space, must_call_close)
diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -1,3 +1,4 @@
+import gc
from pypy.interpreter import typedef
from pypy.tool.udir import udir
from pypy.interpreter.baseobjspace import Wrappable
@@ -180,6 +181,85 @@
assert err.value.message == "'some_type' objects are unhashable"
""")
+ def test_destructor(self):
+ space = self.space
+ class W_Level1(Wrappable):
+ def __init__(self, space1):
+ assert space1 is space
+ def __del__(self):
+ space.call_method(w_seen, 'append', space.wrap(1))
+ class W_Level2(Wrappable):
+ def __init__(self, space1):
+ assert space1 is space
+ def __del__(self):
+ self.enqueue_for_destruction(space, W_Level2.destructormeth,
+ 'FOO ')
+ def destructormeth(self):
+ space.call_method(w_seen, 'append', space.wrap(2))
+ W_Level1.typedef = typedef.TypeDef(
+ 'level1',
+ __new__ = typedef.generic_new_descr(W_Level1))
+ W_Level2.typedef = typedef.TypeDef(
+ 'level2',
+ __new__ = typedef.generic_new_descr(W_Level2))
+ #
+ w_seen = space.newlist([])
+ W_Level1(space)
+ gc.collect(); gc.collect()
+ assert space.unwrap(w_seen) == [1]
+ #
+ w_seen = space.newlist([])
+ W_Level2(space)
+ gc.collect(); gc.collect()
+ assert space.str_w(space.repr(w_seen)) == "[]" # not called yet
+ ec = space.getexecutioncontext()
+ self.space.user_del_action.perform(ec, None)
+ assert space.unwrap(w_seen) == [2]
+ #
+ w_seen = space.newlist([])
+ self.space.appexec([self.space.gettypeobject(W_Level1.typedef)],
+ """(level1):
+ class A3(level1):
+ pass
+ A3()
+ """)
+ gc.collect(); gc.collect()
+ assert space.unwrap(w_seen) == [1]
+ #
+ w_seen = space.newlist([])
+ self.space.appexec([self.space.gettypeobject(W_Level1.typedef),
+ w_seen],
+ """(level1, seen):
+ class A4(level1):
+ def __del__(self):
+ seen.append(4)
+ A4()
+ """)
+ gc.collect(); gc.collect()
+ assert space.unwrap(w_seen) == [4, 1]
+ #
+ w_seen = space.newlist([])
+ self.space.appexec([self.space.gettypeobject(W_Level2.typedef)],
+ """(level2):
+ class A5(level2):
+ pass
+ A5()
+ """)
+ gc.collect(); gc.collect()
+ assert space.unwrap(w_seen) == [2]
+ #
+ w_seen = space.newlist([])
+ self.space.appexec([self.space.gettypeobject(W_Level2.typedef),
+ w_seen],
+ """(level2, seen):
+ class A6(level2):
+ def __del__(self):
+ seen.append(6)
+ A6()
+ """)
+ gc.collect(); gc.collect()
+ assert space.unwrap(w_seen) == [6, 2]
+
class AppTestTypeDef:
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -228,21 +228,26 @@
return self._lifeline_
def setweakref(self, space, weakreflifeline):
self._lifeline_ = weakreflifeline
+ def delweakref(self):
+ self._lifeline_ = None
add(Proto)
if "del" in features:
+ parent_destructor = getattr(supercls, '__del__', None)
+ def call_parent_del(self):
+ assert isinstance(self, subcls)
+ parent_destructor(self)
+ def call_applevel_del(self):
+ assert isinstance(self, subcls)
+ self.space.userdel(self)
class Proto(object):
def __del__(self):
- self._enqueue_for_destruction(self.space)
- # if the base class needs its own interp-level __del__,
- # we override the _call_builtin_destructor() method to invoke it
- # after the app-level destructor.
- parent_destructor = getattr(supercls, '__del__', None)
- if parent_destructor is not None:
- def _call_builtin_destructor(self):
- parent_destructor(self)
- Proto._call_builtin_destructor = _call_builtin_destructor
-
+ self.clear_all_weakrefs()
+ self.enqueue_for_destruction(self.space, call_applevel_del,
+ 'method __del__ of ')
+ if parent_destructor is not None:
+ self.enqueue_for_destruction(self.space, call_parent_del,
+ 'internal destructor of ')
add(Proto)
if "slots" in features:
@@ -630,9 +635,12 @@
return self._lifeline_
def setweakref(self, space, weakreflifeline):
self._lifeline_ = weakreflifeline
+ def delweakref(self):
+ self._lifeline_ = None
cls._lifeline_ = None
cls.getweakref = getweakref
cls.setweakref = setweakref
+ cls.delweakref = delweakref
return weakref_descr
@@ -858,8 +866,6 @@
descrmismatch='close'),
__iter__ = interp2app(GeneratorIterator.descr__iter__,
descrmismatch='__iter__'),
- __del__ = interp2app(GeneratorIterator.descr__del__,
- descrmismatch='__del__'),
gi_running = interp_attrproperty('running', cls=GeneratorIterator),
gi_frame = GetSetProperty(GeneratorIterator.descr_gi_frame),
gi_code = GetSetProperty(GeneratorIterator.descr_gi_code),
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -43,11 +43,17 @@
# assume that the file and stream objects are only visible in the
# thread that runs __del__, so no race condition should be possible
self.clear_all_weakrefs()
+ if self.stream is not None:
+ self.enqueue_for_destruction(self.space, W_File.destructor,
+ 'close() method of ')
+
+ def destructor(self):
+ assert isinstance(self, W_File)
try:
self.direct_close()
except StreamErrors, e:
operr = wrap_streamerror(self.space, e, self.w_name)
- operr.write_unraisable(self.space, '__del__ of ', self)
+ raise operr
def fdopenstream(self, stream, fd, mode, w_name=None):
self.fd = fd
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -57,6 +57,11 @@
def __del__(self):
self.clear_all_weakrefs()
+ self.enqueue_for_destruction(self.space, W_IOBase.destructor,
+ 'internal __del__ of ')
+
+ def destructor(self):
+ assert isinstance(self, W_IOBase)
space = self.space
w_closed = space.findattr(self, space.wrap('closed'))
try:
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -10,7 +10,7 @@
class WeakrefLifeline(W_Root):
def __init__(self, space):
- self.space = space # this is here for W_Root.clear_all_weakrefs()
+ self.space = space
self.refs_weak = []
self.cached_weakref_index = -1
self.cached_proxy_index = -1
@@ -23,8 +23,10 @@
"""
for i in range(len(self.refs_weak) - 1, -1, -1):
w_ref = self.refs_weak[i]()
- if w_ref is not None:
- self.space.user_del_action.register_weakref_callback(w_ref)
+ if w_ref is not None and w_ref.w_callable is not None:
+ w_ref.enqueue_for_destruction(self.space,
+ W_WeakrefBase.activate_callback,
+ 'weakref callback of ')
def clear_all_weakrefs(self):
"""Clear all weakrefs. This is called when an app-level object has
@@ -118,11 +120,8 @@
self.w_obj_weak = dead_ref
def activate_callback(w_self):
- if not w_self.w_callable is None:
- try:
- w_self.space.call_function(w_self.w_callable, w_self)
- except OperationError, e:
- e.write_unraisable(w_self.space, 'weakref callback ', w_self.w_callable)
+ assert isinstance(w_self, W_WeakrefBase)
+ w_self.space.call_function(w_self.w_callable, w_self)
def descr__repr__(self, space):
w_obj = self.dereference()
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -431,12 +431,17 @@
return None
assert isinstance(lifeline, WeakrefLifeline)
return lifeline
+ getweakref._cannot_really_call_random_things_ = True
def setweakref(self, space, weakreflifeline):
from pypy.module._weakref.interp__weakref import WeakrefLifeline
- assert (isinstance(weakreflifeline, WeakrefLifeline) or
- weakreflifeline is None)
+ assert isinstance(weakreflifeline, WeakrefLifeline)
self._get_mapdict_map().write(self, ("weakref", SPECIAL), weakreflifeline)
+ setweakref._cannot_really_call_random_things_ = True
+
+ def delweakref(self):
+ self._get_mapdict_map().write(self, ("weakref", SPECIAL), None)
+ delweakref._cannot_really_call_random_things_ = True
class ObjectMixin(object):
_mixin_ = True
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -36,6 +36,8 @@
return self._lifeline_
def setweakref(self, space, weakreflifeline):
self._lifeline_ = weakreflifeline
+ def delweakref(self):
+ self._lifeline_ = None
class W_SetObject(W_BaseSetObject):
from pypy.objspace.std.settype import set_typedef as typedef
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -171,7 +171,7 @@
obj = c.instantiate()
assert obj.getweakref() is None
obj.setweakref(space, lifeline1)
- obj.setweakref(space, None)
+ obj.delweakref()
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -532,6 +532,8 @@
return self._lifeline_
def setweakref(self, space, weakreflifeline):
self._lifeline_ = weakreflifeline
+ def delweakref(self):
+ self._lifeline_ = None
# ____________________________________________________________
# Initialization of type objects
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -482,6 +482,13 @@
key[2:])
cache[key] = s_value
+ # add the attribute _dont_reach_me_in_del_ (see pypy.rpython.rclass)
+ try:
+ graph = self.bookkeeper.position_key[0]
+ graph.func._dont_reach_me_in_del_ = True
+ except (TypeError, AttributeError):
+ pass
+
return annmodel.s_None
def annotate_hooks(self, **kwds_s):
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -83,6 +83,9 @@
t, rtyper, fngraph = self.gengraph(fn, [int])
+ # added by compute_result_annotation()
+ assert fn._dont_reach_me_in_del_ == True
+
def getargs(func):
for graph in t.graphs:
if getattr(graph, 'func', None) is func:
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -400,6 +400,7 @@
assert len(s_func.descriptions) == 1
funcdesc, = s_func.descriptions
graph = funcdesc.getuniquegraph()
+ self.check_graph_of_del_does_not_call_too_much(graph)
FUNCTYPE = FuncType([Ptr(source_repr.object_type)], Void)
destrptr = functionptr(FUNCTYPE, graph.name,
graph=graph,
diff --git a/pypy/rpython/rclass.py b/pypy/rpython/rclass.py
--- a/pypy/rpython/rclass.py
+++ b/pypy/rpython/rclass.py
@@ -374,6 +374,43 @@
def can_ll_be_null(self, s_value):
return s_value.can_be_none()
+ def check_graph_of_del_does_not_call_too_much(self, graph):
+ # RPython-level __del__() methods should not do "too much".
+ # In the PyPy Python interpreter, they usually do simple things
+ # like file.__del__() closing the file descriptor; or if they
+ # want to do more like call an app-level __del__() method, they
+ # enqueue the object instead, and the actual call is done later.
+ #
+ # Here, as a quick way to check "not doing too much", we check
+ # that from no RPython-level __del__() method we can reach a
+ # JitDriver.
+ #
+ # XXX wrong complexity, but good enough because the set of
+ # reachable graphs should be small
+ callgraph = self.rtyper.annotator.translator.callgraph.values()
+ seen = {graph: None}
+ while True:
+ oldlength = len(seen)
+ for caller, callee in callgraph:
+ if caller in seen and callee not in seen:
+ func = getattr(callee, 'func', None)
+ if getattr(func, '_dont_reach_me_in_del_', False):
+ lst = [str(callee)]
+ g = caller
+ while g:
+ lst.append(str(g))
+ g = seen.get(g)
+ lst.append('')
+ raise TyperError("the RPython-level __del__() method "
+ "in %r calls:%s" % (
+ graph, '\n\t'.join(lst[::-1])))
+ if getattr(func, '_cannot_really_call_random_things_',
+ False):
+ continue
+ seen[callee] = caller
+ if len(seen) == oldlength:
+ break
+
# ____________________________________________________________
def rtype_new_instance(rtyper, classdef, llops, classcallhop=None):
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -7,6 +7,7 @@
from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
from pypy.rpython.rclass import IR_IMMUTABLE, IR_IMMUTABLE_ARRAY
from pypy.rpython.rclass import IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY
+from pypy.rpython.error import TyperError
from pypy.objspace.flow.model import summary
class EmptyBase(object):
@@ -1021,7 +1022,25 @@
assert typeOf(destrptra).TO.ARGS[0] != typeOf(destrptrb).TO.ARGS[0]
assert destrptra is not None
assert destrptrb is not None
-
+
+ def test_del_forbidden(self):
+ class A(object):
+ def __del__(self):
+ self.foo()
+ def foo(self):
+ self.bar()
+ def bar(self):
+ pass
+ bar._dont_reach_me_in_del_ = True
+ def f():
+ a = A()
+ a.foo()
+ a.bar()
+ t = TranslationContext()
+ t.buildannotator().build_types(f, [])
+ e = py.test.raises(TyperError, t.buildrtyper().specialize)
+ print e.value
+
def test_instance_repr(self):
from pypy.rlib.objectmodel import current_object_addr_as_int
class FooBar(object):
More information about the pypy-commit
mailing list