[pypy-commit] pypy default: (fijal, arigo reviewing) Merge lightweight-finalizers branch. This branch

fijal noreply at buildbot.pypy.org
Tue Oct 25 17:02:09 CEST 2011


Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48444:b3d0ebe10a5d
Date: 2011-10-25 17:01 +0200
http://bitbucket.org/pypy/pypy/changeset/b3d0ebe10a5d/

Log:	(fijal, arigo reviewing) Merge lightweight-finalizers branch. This
	branch adds a possitibility for a finalizer to be "lightweight".
	This means it won't resurrect an object or call arbitrary code. Such
	finalizers are detected automatically and can be dealt with more
	efficiently (such objects can live in the nursery for example).

diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -649,10 +649,13 @@
         def malloc_basic(size, tid):
             type_id = llop.extract_ushort(llgroup.HALFWORD, tid)
             has_finalizer = bool(tid & (1<<llgroup.HALFSHIFT))
+            has_light_finalizer = bool(tid & (1<<(llgroup.HALFSHIFT + 1)))
             check_typeid(type_id)
             res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
                                                   type_id, size,
-                                                  has_finalizer, False)
+                                                  has_finalizer,
+                                                  has_light_finalizer,
+                                                  False)
             # In case the operation above failed, we are returning NULL
             # from this function to assembler.  There is also an RPython
             # exception set, typically MemoryError; but it's easier and
@@ -723,7 +726,7 @@
             # also use it to allocate varsized objects.  The tid
             # and possibly the length are both set afterward.
             gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
-                                        0, size, False, False)
+                                        0, size, False, False, False)
             return rffi.cast(lltype.Signed, gcref)
         self.malloc_slowpath = malloc_slowpath
         self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -747,7 +750,9 @@
         type_id = self.layoutbuilder.get_type_id(S)
         assert not self.layoutbuilder.is_weakref_type(S)
         has_finalizer = bool(self.layoutbuilder.has_finalizer(S))
-        flags = int(has_finalizer) << llgroup.HALFSHIFT
+        has_light_finalizer = bool(self.layoutbuilder.has_light_finalizer(S))
+        flags = (int(has_finalizer) << llgroup.HALFSHIFT |
+                 int(has_light_finalizer) << (llgroup.HALFSHIFT + 1))
         descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags)
 
     def init_array_descr(self, A, descr):
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -247,11 +247,12 @@
         self.record = []
 
     def do_malloc_fixedsize_clear(self, RESTYPE, type_id, size,
-                                  has_finalizer, contains_weakptr):
+                                  has_finalizer, has_light_finalizer,
+                                  contains_weakptr):
         assert not contains_weakptr
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = int(has_finalizer) << 16
+        flags = (int(has_finalizer) << 16) | (int(has_light_finalizer) << 17)
         tid = llop.combine_ushort(lltype.Signed, type_id, flags)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -19,7 +19,7 @@
 class W_RSocket(Wrappable, RSocket):
     def __del__(self):
         self.clear_all_weakrefs()
-        self.close()
+        RSocket.__del__(self)
 
     def accept_w(self, space):
         """accept() -> (socket object, address info)
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -292,6 +292,9 @@
         elif _POSIX:
             self.closed = True
             if self.fd != -1:
+                # XXX this is buggy - raising in an RPython del is not a good
+                #     idea, we should swallow the exception or ignore the
+                #     underlaying close error code
                 os.close(self.fd)
                 self.fd = -1
             if self.size > 0:
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -56,6 +56,7 @@
 
 
 _FAMILIES = {}
+
 class Address(object):
     """The base class for RPython-level objects representing addresses.
     Fields:  addr    - a _c.sockaddr_ptr (memory owned by the Address instance)
@@ -77,9 +78,8 @@
         self.addrlen = addrlen
 
     def __del__(self):
-        addr = self.addr_p
-        if addr:
-            lltype.free(addr, flavor='raw')
+        if self.addr_p:
+            lltype.free(self.addr_p, flavor='raw')
 
     def setdata(self, addr, addrlen):
         # initialize self.addr and self.addrlen.  'addr' can be a different
@@ -613,7 +613,10 @@
         self.timeout = defaults.timeout
         
     def __del__(self):
-        self.close()
+        fd = self.fd
+        if fd != _c.INVALID_SOCKET:
+            self.fd = _c.INVALID_SOCKET
+            _c.socketclose(fd)
 
     if hasattr(_c, 'fcntl'):
         def _setblocking(self, block):
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -1,4 +1,5 @@
-from pypy.rpython.lltypesystem import lltype, llmemory, llarena
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.debug import ll_assert
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -62,6 +63,7 @@
     def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
                             is_gcarrayofgcptr,
                             getfinalizer,
+                            getlightfinalizer,
                             offsets_to_gc_pointers,
                             fixed_size, varsize_item_sizes,
                             varsize_offset_to_variable_part,
@@ -74,6 +76,7 @@
                             get_custom_trace,
                             fast_path_tracing):
         self.getfinalizer = getfinalizer
+        self.getlightfinalizer = getlightfinalizer
         self.is_varsize = is_varsize
         self.has_gcptr_in_varsize = has_gcptr_in_varsize
         self.is_gcarrayofgcptr = is_gcarrayofgcptr
@@ -139,6 +142,7 @@
 
         size = self.fixed_size(typeid)
         needs_finalizer = bool(self.getfinalizer(typeid))
+        finalizer_is_light = bool(self.getlightfinalizer(typeid))
         contains_weakptr = self.weakpointer_offset(typeid) >= 0
         assert not (needs_finalizer and contains_weakptr)
         if self.is_varsize(typeid):
@@ -158,6 +162,7 @@
             else:
                 malloc_fixedsize = self.malloc_fixedsize
             ref = malloc_fixedsize(typeid, size, needs_finalizer,
+                                   finalizer_is_light,
                                    contains_weakptr)
         # lots of cast and reverse-cast around...
         return llmemory.cast_ptr_to_adr(ref)
diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py
--- a/pypy/rpython/memory/gc/generation.py
+++ b/pypy/rpython/memory/gc/generation.py
@@ -167,7 +167,9 @@
         return self.nursery <= addr < self.nursery_top
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         if (has_finalizer or
             (raw_malloc_usage(size) > self.lb_young_fixedsize and
              raw_malloc_usage(size) > self.largest_young_fixedsize)):
@@ -179,6 +181,7 @@
             # "non-simple" case or object too big: don't use the nursery
             return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size,
                                                       has_finalizer,
+                                                      is_finalizer_light,
                                                       contains_weakptr)
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py
--- a/pypy/rpython/memory/gc/marksweep.py
+++ b/pypy/rpython/memory/gc/marksweep.py
@@ -93,7 +93,8 @@
         pass
 
     def malloc_fixedsize(self, typeid16, size,
-                         has_finalizer=False, contains_weakptr=False):
+                         has_finalizer=False, is_finalizer_light=False,
+                         contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
@@ -128,7 +129,9 @@
     malloc_fixedsize._dont_inline_ = True
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -290,6 +290,8 @@
         #
         # A list of all objects with finalizers (these are never young).
         self.objects_with_finalizers = self.AddressDeque()
+        self.young_objects_with_light_finalizers = self.AddressStack()
+        self.old_objects_with_light_finalizers = self.AddressStack()
         #
         # Two lists of the objects with weakrefs.  No weakref can be an
         # old object weakly pointing to a young object: indeed, weakrefs
@@ -457,14 +459,16 @@
 
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               needs_finalizer=False, contains_weakptr=False):
+                               needs_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
         #
         # If the object needs a finalizer, ask for a rawmalloc.
         # The following check should be constant-folded.
-        if needs_finalizer:
+        if needs_finalizer and not is_finalizer_light:
             ll_assert(not contains_weakptr,
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             obj = self.external_malloc(typeid, 0, can_make_young=False)
@@ -494,13 +498,14 @@
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
+            obj = result + size_gc_header
+            if is_finalizer_light:
+                self.young_objects_with_light_finalizers.append(obj)
             self.init_gc_object(result, typeid, flags=0)
             #
             # If it is a weakref, record it (check constant-folded).
             if contains_weakptr:
-                self.young_objects_with_weakrefs.append(result+size_gc_header)
-            #
-            obj = result + size_gc_header
+                self.young_objects_with_weakrefs.append(obj)
         #
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
@@ -1264,6 +1269,8 @@
         # weakrefs' targets.
         if self.young_objects_with_weakrefs.non_empty():
             self.invalidate_young_weakrefs()
+        if self.young_objects_with_light_finalizers.non_empty():
+            self.deal_with_young_objects_with_finalizers()
         #
         # Clear this mapping.
         if self.nursery_objects_shadows.length() > 0:
@@ -1584,6 +1591,9 @@
         # Weakref support: clear the weak pointers to dying objects
         if self.old_objects_with_weakrefs.non_empty():
             self.invalidate_old_weakrefs()
+        if self.old_objects_with_light_finalizers.non_empty():
+            self.deal_with_old_objects_with_finalizers()
+
         #
         # Walk all rawmalloced objects and free the ones that don't
         # have the GCFLAG_VISITED flag.
@@ -1649,8 +1659,7 @@
         if self.header(obj).tid & GCFLAG_VISITED:
             self.header(obj).tid &= ~GCFLAG_VISITED
             return False     # survives
-        else:
-            return True      # dies
+        return True      # dies
 
     def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~GCFLAG_VISITED
@@ -1829,6 +1838,39 @@
     # ----------
     # Finalizers
 
+    def deal_with_young_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        while self.young_objects_with_light_finalizers.non_empty():
+            obj = self.young_objects_with_light_finalizers.pop()
+            if not self.is_forwarded(obj):
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+
+    def deal_with_old_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.old_objects_with_light_finalizers.non_empty():
+            obj = self.old_objects_with_light_finalizers.pop()
+            if self.header(obj).tid & GCFLAG_VISITED:
+                # surviving
+                new_objects.append(obj)
+            else:
+                # dying
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+        self.old_objects_with_light_finalizers.delete()
+        self.old_objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self):
         # Walk over list of objects with finalizers.
         # If it is not surviving, add it to the list of to-be-called
@@ -1959,7 +2001,6 @@
             #
             self.old_objects_with_weakrefs.append(obj)
 
-
     def invalidate_old_weakrefs(self):
         """Called during a major collection."""
         # walk over list of objects that contain weakrefs
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -82,6 +82,7 @@
         self.free = self.tospace
         MovingGCBase.setup(self)
         self.objects_with_finalizers = self.AddressDeque()
+        self.objects_with_light_finalizers = self.AddressStack()
         self.objects_with_weakrefs = self.AddressStack()
 
     def _teardown(self):
@@ -93,7 +94,9 @@
     # because the spaces are filled with zeroes in advance.
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         result = self.free
@@ -102,7 +105,9 @@
         llarena.arena_reserve(result, totalsize)
         self.init_gc_object(result, typeid16)
         self.free = result + totalsize
-        if has_finalizer:
+        if is_finalizer_light:
+            self.objects_with_light_finalizers.append(result + size_gc_header)
+        elif has_finalizer:
             self.objects_with_finalizers.append(result + size_gc_header)
         if contains_weakptr:
             self.objects_with_weakrefs.append(result + size_gc_header)
@@ -263,6 +268,8 @@
         if self.run_finalizers.non_empty():
             self.update_run_finalizers()
         scan = self.scan_copied(scan)
+        if self.objects_with_light_finalizers.non_empty():
+            self.deal_with_objects_with_light_finalizers()
         if self.objects_with_finalizers.non_empty():
             scan = self.deal_with_objects_with_finalizers(scan)
         if self.objects_with_weakrefs.non_empty():
@@ -471,6 +478,23 @@
         # immortal objects always have GCFLAG_FORWARDED set;
         # see get_forwarding_address().
 
+    def deal_with_objects_with_light_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.objects_with_light_finalizers.non_empty():
+            obj = self.objects_with_light_finalizers.pop()
+            if self.surviving(obj):
+                new_objects.append(self.get_forwarding_address(obj))
+            else:
+                finalizer = self.getfinalizer(self.get_type_id(obj))
+                finalizer(obj, llmemory.NULL)
+        self.objects_with_light_finalizers.delete()
+        self.objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self, scan):
         # walk over list of objects with finalizers
         # if it is not copied, add it to the list of to-be-called finalizers
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -12,6 +12,7 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.translator.backendopt import graphanalyze
 from pypy.translator.backendopt.support import var_needsgc
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.annotation import model as annmodel
 from pypy.rpython import annlowlevel
 from pypy.rpython.rbuiltin import gen_cast
@@ -258,6 +259,7 @@
             [s_gc, s_typeid16,
              annmodel.SomeInteger(nonneg=True),
              annmodel.SomeBool(),
+             annmodel.SomeBool(),
              annmodel.SomeBool()], s_gcref,
             inline = False)
         if hasattr(GCClass, 'malloc_fixedsize'):
@@ -267,6 +269,7 @@
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
                  annmodel.SomeBool(),
+                 annmodel.SomeBool(),
                  annmodel.SomeBool()], s_gcref,
                 inline = False)
         else:
@@ -319,7 +322,7 @@
             raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality")
 
         # in some GCs we can inline the common case of
-        # malloc_fixedsize(typeid, size, True, False, False)
+        # malloc_fixedsize(typeid, size, False, False, False)
         if getattr(GCClass, 'inline_simple_malloc', False):
             # make a copy of this function so that it gets annotated
             # independently and the constants are folded inside
@@ -337,7 +340,7 @@
                 malloc_fast,
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
-                 s_False, s_False], s_gcref,
+                 s_False, s_False, s_False], s_gcref,
                 inline = True)
         else:
             self.malloc_fast_ptr = None
@@ -668,7 +671,13 @@
         kind_and_fptr = self.special_funcptr_for_type(TYPE)
         has_finalizer = (kind_and_fptr is not None and
                          kind_and_fptr[0] == "finalizer")
+        has_light_finalizer = (kind_and_fptr is not None and
+                               kind_and_fptr[0] == "light_finalizer")
+        if has_light_finalizer:
+            has_finalizer = True
         c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer)
+        c_has_light_finalizer = rmodel.inputconst(lltype.Bool,
+                                                  has_light_finalizer)
 
         if not op.opname.endswith('_varsize') and not flags.get('varsize'):
             #malloc_ptr = self.malloc_fixedsize_ptr
@@ -682,7 +691,8 @@
             else:
                 malloc_ptr = self.malloc_fixedsize_ptr
             args = [self.c_const_gc, c_type_id, c_size,
-                    c_has_finalizer, rmodel.inputconst(lltype.Bool, False)]
+                    c_has_finalizer, c_has_light_finalizer,
+                    rmodel.inputconst(lltype.Bool, False)]
         else:
             assert not c_has_finalizer.value
             info_varsize = self.layoutbuilder.get_info_varsize(type_id)
@@ -847,12 +857,13 @@
         # used by the JIT (see pypy.jit.backend.llsupport.gc)
         op = hop.spaceop
         [v_typeid, v_size,
-         v_has_finalizer, v_contains_weakptr] = op.args
+         v_has_finalizer, v_has_light_finalizer, v_contains_weakptr] = op.args
         livevars = self.push_roots(hop)
         hop.genop("direct_call",
                   [self.malloc_fixedsize_clear_ptr, self.c_const_gc,
                    v_typeid, v_size,
-                   v_has_finalizer, v_contains_weakptr],
+                   v_has_finalizer, v_has_light_finalizer,
+                   v_contains_weakptr],
                   resultvar=op.result)
         self.pop_roots(hop, livevars)
 
@@ -912,10 +923,10 @@
         info = self.layoutbuilder.get_info(type_id)
         c_size = rmodel.inputconst(lltype.Signed, info.fixedsize)
         malloc_ptr = self.malloc_fixedsize_ptr
-        c_has_finalizer = rmodel.inputconst(lltype.Bool, False)
+        c_false = rmodel.inputconst(lltype.Bool, False)
         c_has_weakptr = rmodel.inputconst(lltype.Bool, True)
         args = [self.c_const_gc, c_type_id, c_size,
-                c_has_finalizer, c_has_weakptr]
+                c_false, c_false, c_has_weakptr]
 
         # push and pop the current live variables *including* the argument
         # to the weakref_create operation, which must be kept alive and
@@ -1250,6 +1261,7 @@
             lltype2vtable = translator.rtyper.lltype2vtable
         else:
             lltype2vtable = None
+        self.translator = translator
         super(TransformerLayoutBuilder, self).__init__(GCClass, lltype2vtable)
 
     def has_finalizer(self, TYPE):
@@ -1257,6 +1269,10 @@
         return rtti is not None and getattr(rtti._obj, 'destructor_funcptr',
                                             None)
 
+    def has_light_finalizer(self, TYPE):
+        special = self.special_funcptr_for_type(TYPE)
+        return special is not None and special[0] == 'light_finalizer'
+
     def has_custom_trace(self, TYPE):
         rtti = get_rtti(TYPE)
         return rtti is not None and getattr(rtti._obj, 'custom_trace_funcptr',
@@ -1264,7 +1280,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         if not self.has_finalizer(TYPE):
-            return None
+            return None, False
         rtti = get_rtti(TYPE)
         destrptr = rtti._obj.destructor_funcptr
         DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
@@ -1276,7 +1292,9 @@
             return llmemory.NULL
         fptr = self.transformer.annotate_finalizer(ll_finalizer,
                 [llmemory.Address, llmemory.Address], llmemory.Address)
-        return fptr
+        g = destrptr._obj.graph
+        light = not FinalizerAnalyzer(self.translator).analyze_direct_call(g)
+        return fptr, light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         if not self.has_custom_trace(TYPE):
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -1,7 +1,6 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup
 from pypy.rpython.lltypesystem import rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import intmask
 from pypy.tool.identity_dict import identity_dict
@@ -85,6 +84,13 @@
         else:
             return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)
 
+    def q_light_finalizer(self, typeid):
+        typeinfo = self.get(typeid)
+        if typeinfo.infobits & T_HAS_LIGHTWEIGHT_FINALIZER:
+            return typeinfo.finalizer_or_customtrace
+        else:
+            return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)        
+
     def q_offsets_to_gc_pointers(self, typeid):
         return self.get(typeid).ofstoptrs
 
@@ -142,6 +148,7 @@
             self.q_has_gcptr_in_varsize,
             self.q_is_gcarrayofgcptr,
             self.q_finalizer,
+            self.q_light_finalizer,
             self.q_offsets_to_gc_pointers,
             self.q_fixed_size,
             self.q_varsize_item_sizes,
@@ -157,16 +164,17 @@
 
 
 # the lowest 16bits are used to store group member index
-T_MEMBER_INDEX         =   0xffff
-T_IS_VARSIZE           = 0x010000
-T_HAS_GCPTR_IN_VARSIZE = 0x020000
-T_IS_GCARRAY_OF_GCPTR  = 0x040000
-T_IS_WEAKREF           = 0x080000
-T_IS_RPYTHON_INSTANCE  = 0x100000    # the type is a subclass of OBJECT
-T_HAS_FINALIZER        = 0x200000
-T_HAS_CUSTOM_TRACE     = 0x400000
-T_KEY_MASK             = intmask(0xFF000000)
-T_KEY_VALUE            = intmask(0x5A000000)    # bug detection only
+T_MEMBER_INDEX              =   0xffff
+T_IS_VARSIZE                = 0x010000
+T_HAS_GCPTR_IN_VARSIZE      = 0x020000
+T_IS_GCARRAY_OF_GCPTR       = 0x040000
+T_IS_WEAKREF                = 0x080000
+T_IS_RPYTHON_INSTANCE       = 0x100000 # the type is a subclass of OBJECT
+T_HAS_FINALIZER             = 0x200000
+T_HAS_CUSTOM_TRACE          = 0x400000
+T_HAS_LIGHTWEIGHT_FINALIZER = 0x800000
+T_KEY_MASK                  = intmask(0xFF000000)
+T_KEY_VALUE                 = intmask(0x5A000000) # bug detection only
 
 def _check_valid_type_info(p):
     ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id")
@@ -194,6 +202,8 @@
         info.finalizer_or_customtrace = fptr
         if kind == "finalizer":
             infobits |= T_HAS_FINALIZER
+        elif kind == 'light_finalizer':
+            infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER
         elif kind == "custom_trace":
             infobits |= T_HAS_CUSTOM_TRACE
         else:
@@ -367,12 +377,15 @@
     def special_funcptr_for_type(self, TYPE):
         if TYPE in self._special_funcptrs:
             return self._special_funcptrs[TYPE]
-        fptr1 = self.make_finalizer_funcptr_for_type(TYPE)
+        fptr1, is_lightweight = self.make_finalizer_funcptr_for_type(TYPE)
         fptr2 = self.make_custom_trace_funcptr_for_type(TYPE)
         assert not (fptr1 and fptr2), (
             "type %r needs both a finalizer and a custom tracer" % (TYPE,))
         if fptr1:
-            kind_and_fptr = "finalizer", fptr1
+            if is_lightweight:
+                kind_and_fptr = "light_finalizer", fptr1
+            else:
+                kind_and_fptr = "finalizer", fptr1
         elif fptr2:
             kind_and_fptr = "custom_trace", fptr2
         else:
@@ -382,7 +395,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         # must be overridden for proper finalizer support
-        return None
+        return None, False
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         # must be overridden for proper custom tracer support
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -1,3 +1,4 @@
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.rpython.lltypesystem import lltype, llmemory, llheap
 from pypy.rpython import llinterp
 from pypy.rpython.annlowlevel import llhelper
@@ -196,9 +197,11 @@
             DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
             destrgraph = destrptr._obj.graph
         else:
-            return None
+            return None, False
 
         assert not type_contains_pyobjs(TYPE), "not implemented"
+        t = self.llinterp.typer.annotator.translator
+        light = not FinalizerAnalyzer(t).analyze_direct_call(destrgraph)
         def ll_finalizer(addr, dummy):
             assert dummy == llmemory.NULL
             try:
@@ -208,7 +211,7 @@
                 raise RuntimeError(
                     "a finalizer raised an exception, shouldn't happen")
             return llmemory.NULL
-        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer)
+        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer), light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         from pypy.rpython.memory.gctransform.support import get_rtti, \
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.memory.test import snippet
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.rpython.lltypesystem import lltype
-from pypy.rpython.lltypesystem.rstr import STR
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import compute_unique_id
@@ -57,7 +56,7 @@
                 while j < 20:
                     j += 1
                     a.append(j)
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -73,7 +72,7 @@
                 while j < 20:
                     j += 1
                     b.append((1, j, i))
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -129,7 +128,7 @@
         res = self.interpret(concat, [100])
         assert res == concat(100)
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
-        
+
     def test_finalizer(self):
         class B(object):
             pass
@@ -278,7 +277,7 @@
                                self.interpret, f, [])
 
     def test_weakref(self):
-        import weakref, gc
+        import weakref
         class A(object):
             pass
         def g():
@@ -299,7 +298,7 @@
         assert res
 
     def test_weakref_to_object_with_finalizer(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -338,7 +337,7 @@
         assert res
 
     def test_cycle_with_weakref_and_del(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -367,7 +366,7 @@
         assert res == 11
 
     def test_weakref_to_object_with_finalizer_ordering(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -616,7 +615,7 @@
                     assert not rgc.can_move(a)
                     return 1
                 return 0
-            except Exception, e:
+            except Exception:
                 return 2
 
         assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE)
@@ -647,8 +646,6 @@
         assert self.interpret(f, [bigsize, 0, flag]) == 0x62024241
 
     def test_tagged_simple(self):
-        from pypy.rlib.objectmodel import UnboxedValue
-
         class Unrelated(object):
             pass
 
@@ -689,8 +686,6 @@
         assert res == -897
 
     def test_tagged_id(self):
-        from pypy.rlib.objectmodel import UnboxedValue, compute_unique_id
-
         class Unrelated(object):
             pass
 
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -345,22 +345,22 @@
         b = B()
         b.nextid = 0
         b.num_deleted = 0
-        class A(object):
+        class AAA(object):
             def __init__(self):
                 self.id = b.nextid
                 b.nextid += 1
             def __del__(self):
                 b.num_deleted += 1
                 C()
-        class C(A):
+        class C(AAA):
             def __del__(self):
                 b.num_deleted += 1
         def f(x, y):
-            a = A()
+            a = AAA()
             i = 0
             while i < x:
                 i += 1
-                a = A()
+                a = AAA()
             llop.gc__collect(lltype.Void)
             llop.gc__collect(lltype.Void)
             return b.num_deleted
@@ -807,6 +807,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
@@ -843,6 +844,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -717,7 +717,7 @@
             raise TyperError("runtime type info function %r returns %r, "
                              "excepted Ptr(RuntimeTypeInfo)" % (func, s))
         funcptr = self.getcallable(graph)
-        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr)
+        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr, None)
 
 # register operations from annotation model
 RPythonTyper._registeroperations(annmodel)
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -3,7 +3,7 @@
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.lltypesystem.lltype import *
 from pypy.rpython.ootypesystem import ootype
-from pypy.rlib.rarithmetic import intmask, r_longlong
+from pypy.rlib.rarithmetic import r_longlong
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.rclass import IR_IMMUTABLE, IR_IMMUTABLE_ARRAY
 from pypy.rpython.rclass import IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY
@@ -972,10 +972,10 @@
         graph = graphof(t, f)
         TYPE = graph.startblock.operations[0].args[0].value
         RTTI = getRuntimeTypeInfo(TYPE)
-        queryptr = RTTI._obj.query_funcptr # should not raise
+        RTTI._obj.query_funcptr # should not raise
         destrptr = RTTI._obj.destructor_funcptr
         assert destrptr is not None
-    
+
     def test_del_inheritance(self):
         from pypy.rlib import rgc
         class State:
diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/finalizer.py
@@ -0,0 +1,34 @@
+
+from pypy.translator.backendopt import graphanalyze
+from pypy.rpython.lltypesystem import lltype
+
+class FinalizerAnalyzer(graphanalyze.BoolGraphAnalyzer):
+    """ Analyzer that determines whether a finalizer is lightweight enough
+    so it can be called without all the complicated logic in the garbage
+    collector. The set of operations here is restrictive for a good reason
+    - it's better to be safe. Specifically disallowed operations:
+
+    * anything that escapes self
+    * anything that can allocate
+    """
+    ok_operations = ['ptr_nonzero', 'ptr_eq', 'ptr_ne', 'free', 'same_as',
+                     'direct_ptradd', 'force_cast', 'track_alloc_stop',
+                     'raw_free']
+    
+    def analyze_simple_operation(self, op, graphinfo):
+        if op.opname in self.ok_operations:
+            return self.bottom_result()
+        if (op.opname.startswith('int_') or op.opname.startswith('float_')
+            or op.opname.startswith('cast_')):
+            return self.bottom_result()
+        if op.opname == 'setfield' or op.opname == 'bare_setfield':
+            TP = op.args[2].concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()
+        if op.opname == 'getfield':
+            TP = op.result.concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()            
+        return self.top_result()
diff --git a/pypy/translator/backendopt/test/test_finalizer.py b/pypy/translator/backendopt/test/test_finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/test/test_finalizer.py
@@ -0,0 +1,138 @@
+
+import py
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
+from pypy.translator.translator import TranslationContext, graphof
+from pypy.translator.backendopt.all import backend_optimizations
+from pypy.translator.unsimplify import varoftype
+from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.conftest import option
+
+
+class BaseFinalizerAnalyzerTests(object):
+    """ Below are typical destructors that we encounter in pypy
+    """
+
+    type_system = None
+    
+    def analyze(self, func, sig, func_to_analyze=None, backendopt=False):
+        if func_to_analyze is None:
+            func_to_analyze = func
+        t = TranslationContext()
+        t.buildannotator().build_types(func, sig)
+        t.buildrtyper(type_system=self.type_system).specialize()
+        if backendopt:
+            backend_optimizations(t)
+        if option.view:
+            t.view()
+        a = FinalizerAnalyzer(t)
+        fgraph = graphof(t, func_to_analyze)
+        result = a.analyze_direct_call(fgraph)
+        return result
+
+    def test_nothing(self):
+        def f():
+            pass
+        r = self.analyze(f, [])
+        assert not r
+
+def test_various_ops():
+    from pypy.objspace.flow.model import SpaceOperation, Constant
+
+    X = lltype.Ptr(lltype.GcStruct('X'))
+    Z = lltype.Ptr(lltype.Struct('Z'))
+    S = lltype.GcStruct('S', ('x', lltype.Signed),
+                        ('y', X),
+                        ('z', Z))
+    v1 = varoftype(lltype.Bool)
+    v2 = varoftype(lltype.Signed)
+    f = FinalizerAnalyzer(None)
+    r = f.analyze(SpaceOperation('cast_int_to_bool', [v2],
+                                                       v1))
+    assert not r
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Signed)
+    v3 = varoftype(X)
+    v4 = varoftype(Z)
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('x'),
+                                                          v2], None))
+    assert     f.analyze(SpaceOperation('bare_setfield', [v1, Constant('y'),
+                                                          v3], None))
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('z'),
+                                                          v4], None))
+    
+        
+class TestLLType(BaseFinalizerAnalyzerTests):
+    type_system = 'lltype'
+
+    def test_malloc(self):
+        S = lltype.GcStruct('S')
+        
+        def f():
+            return lltype.malloc(S)
+
+        r = self.analyze(f, [])
+        assert r
+
+    def test_raw_free_getfield(self):
+        S = lltype.Struct('S')
+        
+        class A(object):
+            def __init__(self):
+                self.x = lltype.malloc(S, flavor='raw')
+
+            def __del__(self):
+                if self.x:
+                    self.x = lltype.nullptr(S)
+                    lltype.free(self.x, flavor='raw')
+
+        def f():
+            return A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert not r
+
+    def test_c_call(self):
+        C = rffi.CArray(lltype.Signed)
+        c = rffi.llexternal('x', [lltype.Ptr(C)], lltype.Signed)
+
+        def g():
+            p = lltype.malloc(C, 3, flavor='raw')
+            f(p)
+        
+        def f(p):
+            c(rffi.ptradd(p, 0))
+            lltype.free(p, flavor='raw')
+
+        r = self.analyze(g, [], f, backendopt=True)
+        assert not r
+
+    def test_chain(self):
+        class B(object):
+            def __init__(self):
+                self.counter = 1
+        
+        class A(object):
+            def __init__(self):
+                self.x = B()
+
+            def __del__(self):
+                self.x.counter += 1
+
+        def f():
+            A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert r
+
+    def test_os_call(self):
+        py.test.skip("can allocate OSError, but also can raise, ignore for now")
+        import os
+        
+        def f(i):
+            os.close(i)
+
+        r = self.analyze(f, [int], backendopt=True)
+        assert not r
+
+class TestOOType(BaseFinalizerAnalyzerTests):
+    type_system = 'ootype'


More information about the pypy-commit mailing list