[pypy-svn] r27157 - in pypy/dist/pypy: annotation rpython translator translator/stackless translator/stackless/test

arigo at codespeak.net arigo at codespeak.net
Fri May 12 23:50:12 CEST 2006


Author: arigo
Date: Fri May 12 23:50:10 2006
New Revision: 27157

Added:
   pypy/dist/pypy/translator/stackless/test/test_clone.py   (contents, props changed)
Modified:
   pypy/dist/pypy/annotation/annrpython.py
   pypy/dist/pypy/rpython/rtyper.py
   pypy/dist/pypy/translator/stackless/code.py
   pypy/dist/pypy/translator/stackless/frame.py
   pypy/dist/pypy/translator/stackless/transform.py
   pypy/dist/pypy/translator/unsimplify.py
Log:
(pedronis, arigo)
The first, naive and simple test_clone.  As expected, the hard part is to
get it right with respect to the annotator and the rtyper and mixlevelhelpers.
This means a bug fix in copyvar() (!) and Yet Another Dict on the annotator.


Modified: pypy/dist/pypy/annotation/annrpython.py
==============================================================================
--- pypy/dist/pypy/annotation/annrpython.py	(original)
+++ pypy/dist/pypy/annotation/annrpython.py	Fri May 12 23:50:10 2006
@@ -34,6 +34,7 @@
         self.added_blocks = None # see processblock() below
         self.links_followed = {} # set of links that have ever been followed
         self.notify = {}        # {block: {positions-to-reflow-from-when-done}}
+        self.fixed_graphs = {}  # set of graphs not to annotate again
         # --- the following information is recorded for debugging only ---
         # --- and only if annotation.model.DEBUG is kept to True
         self.why_not_annotated = {} # {block: (exc_type, exc_value, traceback)}
@@ -333,7 +334,17 @@
             callpositions[callback] = True
 
         # generalize the function's input arguments
-        self.addpendingblock(graph, graph.startblock, inputcells, position_key)
+        if graph in self.fixed_graphs:
+            # special case for annotating/rtyping in several phases: calling
+            # a graph that has already been rtyped.  Safety-check the new
+            # annotations that are passed in, and don't annotate the old
+            # graph -- it's already low-level operations!
+            for a, s_newarg in zip(graph.getargs(), inputcells):
+                s_oldarg = self.binding(a)
+                assert s_oldarg.contains(s_newarg)
+        else:
+            self.addpendingblock(graph, graph.startblock, inputcells,
+                                 position_key)
 
         # get the (current) return value
         v = graph.getreturnvar()

Modified: pypy/dist/pypy/rpython/rtyper.py
==============================================================================
--- pypy/dist/pypy/rpython/rtyper.py	(original)
+++ pypy/dist/pypy/rpython/rtyper.py	Fri May 12 23:50:10 2006
@@ -289,6 +289,9 @@
         return LowLevelOpList(self, block)
 
     def specialize_block(self, block):
+        graph = self.annotator.annotated[block]
+        self.annotator.fixed_graphs[graph] = True
+
         # give the best possible types to the input args
         try:
             self.setup_block_entry(block)

Modified: pypy/dist/pypy/translator/stackless/code.py
==============================================================================
--- pypy/dist/pypy/translator/stackless/code.py	(original)
+++ pypy/dist/pypy/translator/stackless/code.py	Fri May 12 23:50:10 2006
@@ -4,6 +4,8 @@
 from pypy.translator.stackless import frame
 from pypy.translator.stackless.frame import STATE_HEADER, SAVED_REFERENCE
 
+EMPTY_STATE = frame.make_state_header_type('empty_state')
+
 # ____________________________________________________________
 
 SWITCH_STATE = frame.make_state_header_type('switch_state',
@@ -22,14 +24,17 @@
     elif global_state.restart_substate == 0:
         # STATE 0: we didn't do anything so far, but the stack is unwound
         global_state.restart_substate = -1
-        # grab the frame corresponding to ourself, and prepare it for
-        # the future switch() back, which will go to STATE 1 below
-        sourcestate = global_state.top
-        sourcestate.f_restart = INDEX_SWITCH + 1
+        # grab the frame corresponding to ourself
         # the 'targetstate' local is garbage here, it must be read back from
         # 's.c' where we saved it by the normal entry point above
-        s = lltype.cast_pointer(lltype.Ptr(SWITCH_STATE), sourcestate)
+        mystate = global_state.top
+        s = lltype.cast_pointer(lltype.Ptr(SWITCH_STATE), mystate)
         targetstate = lltype.cast_opaque_ptr(lltype.Ptr(STATE_HEADER), s.c)
+        # prepare a new saved state for the future switch() back,
+        # which will go to STATE 1 below
+        sourcestate = lltype.malloc(EMPTY_STATE).header
+        sourcestate.f_back = mystate.f_back
+        sourcestate.f_restart = INDEX_SWITCH + 1
         global_state.top = targetstate
         global_state.retval_ref = lltype.cast_opaque_ptr(SAVED_REFERENCE,
                                                          sourcestate)
@@ -45,7 +50,15 @@
                                # with the source state as return value
 ll_frame_switch.stackless_explicit = True
 
-INDEX_SWITCH = frame.RestartInfo.add_prebuilt(ll_frame_switch, 2)
+INDEX_SWITCH = frame.RestartInfo.add_prebuilt(ll_frame_switch,
+                                              [SWITCH_STATE, EMPTY_STATE])
+
+# ____________________________________________________________
+
+def ll_frame_clone(oldstate):
+    oldframe = lltype.cast_opaque_ptr(lltype.Ptr(STATE_HEADER), oldstate)
+    newframe = frame.ll_frame_reccopy(oldframe)
+    return lltype.cast_opaque_ptr(frame.OPAQUE_STATE_HEADER_PTR, newframe)
 
 # ____________________________________________________________
 
@@ -54,7 +67,7 @@
         # normal entry point for yield_current_frame_to_caller()
         # first unwind the stack
         u = UnwindException()
-        s = lltype.malloc(STATE_HEADER)
+        s = lltype.malloc(EMPTY_STATE).header
         s.f_restart = INDEX_YCFTC
         add_frame_state(u, s)
         raise u   # this goes to 'STATE 0' below
@@ -67,7 +80,7 @@
         caller_state = our_caller_state.f_back
         # when our immediate caller finishes (which is later, when the
         # tasklet finishes), then we will jump to 'STATE 1' below
-        endstate = lltype.malloc(STATE_HEADER)
+        endstate = lltype.malloc(EMPTY_STATE).header
         endstate.f_restart = INDEX_YCFTC + 1
         our_caller_state.f_back = endstate
         global_state.top = caller_state
@@ -97,7 +110,8 @@
 
 yield_current_frame_to_caller.stackless_explicit = True
 
-INDEX_YCFTC = frame.RestartInfo.add_prebuilt(yield_current_frame_to_caller, 2)
+INDEX_YCFTC = frame.RestartInfo.add_prebuilt(yield_current_frame_to_caller,
+                                             [EMPTY_STATE, EMPTY_STATE])
 
 # ____________________________________________________________
 
@@ -106,7 +120,7 @@
         # normal entry point for stack_frames_depth()
         # first unwind the stack
         u = UnwindException()
-        s = lltype.malloc(STATE_HEADER)
+        s = lltype.malloc(EMPTY_STATE).header
         s.f_restart = INDEX_DEPTH
         add_frame_state(u, s)
         raise u    # goes to STATE 0 below
@@ -122,7 +136,8 @@
         return depth
 stack_frames_depth.stackless_explicit = True
 
-INDEX_DEPTH = frame.RestartInfo.add_prebuilt(stack_frames_depth, 1)
+INDEX_DEPTH = frame.RestartInfo.add_prebuilt(stack_frames_depth,
+                                             [EMPTY_STATE])
 
 # ____________________________________________________________
 
@@ -131,7 +146,7 @@
         # normal entry point for stack_frames_depth()
         # first unwind the stack in the usual way
         u = UnwindException()
-        s = lltype.malloc(STATE_HEADER)
+        s = lltype.malloc(EMPTY_STATE).header
         s.f_restart = INDEX_UNWIND
         add_frame_state(u, s)
         raise u    # goes to STATE 0 below
@@ -141,7 +156,8 @@
         global_state.restart_substate = -1
 ll_stack_unwind.stackless_explicit = True
 
-INDEX_UNWIND = frame.RestartInfo.add_prebuilt(ll_stack_unwind, 1)
+INDEX_UNWIND = frame.RestartInfo.add_prebuilt(ll_stack_unwind,
+                                              [EMPTY_STATE])
 
 # ____________________________________________________________
 
@@ -200,8 +216,7 @@
     
     while True:
         back = pending.f_back
-        decoded = frame.decodestate(global_state.masterarray,
-                                    pending.f_restart)
+        decoded = frame.decodestate(pending.f_restart)
         (fn, global_state.restart_substate, retval_type) = decoded
         try:
             call_function(fn, retval_type)

Modified: pypy/dist/pypy/translator/stackless/frame.py
==============================================================================
--- pypy/dist/pypy/translator/stackless/frame.py	(original)
+++ pypy/dist/pypy/translator/stackless/frame.py	Fri May 12 23:50:10 2006
@@ -1,7 +1,11 @@
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython import extfunctable
 from pypy.rpython.typesystem import getfunctionptr
+from pypy.rpython.annlowlevel import annotate_lowlevel_helper
 from pypy.objspace.flow.model import FunctionGraph
+from pypy.tool.sourcetools import compile2
+from pypy.annotation import model as annmodel
+from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
 
 # ____________________________________________________________
 # generic data types
@@ -59,39 +63,49 @@
 
 
 def make_state_header_type(name, *fields):
-##    source = ['def ll_reccopy_%s(frame):' % name,
-##              '    if frame.f_back:'
-##              '        prev = frame.f_back.XXX',
-##              '    newframe = lltype.malloc(lltype.typeOf(FRAME))',
-##              '    newframe.']
-    
-##    copynames = [name for (name, _) in fields]
-##    copynames.append('header.restartstate')
-##    copynames.append('header.function')
-##    copynames.append('header.retval_type')
-##    for name in copynames:
-##        source.append('    newframe.%s = frame.%s' % (name, name))
-##    source.append('    return newframe')
-##    source.append('')
-##    miniglobals = {'lltype': lltype}
-##    exec compile2('\n'.join(source)) in miniglobals
-##    extras = {
-##        'adtmeths': {'reccopy': miniglobals['ll_frame_reccopy']}
-##        }
-    return lltype.GcStruct(name,
-                           ('header', STATE_HEADER),
-                           *fields)
+    fnname = 'll_reccopy_%s' % (name,)
+    source = ['def %s(frame):' % (fnname,),
+              '    frame = lltype.cast_pointer(lltype.Ptr(FRAME), frame)',
+              '    newframe = lltype.malloc(FRAME)',
+              '    if frame.header.f_back:',
+              '        newframe.header.f_back = ll_frame_reccopy(',
+              '                                     frame.header.f_back)',
+              '    newframe.header.f_restart = frame.header.f_restart']
+    for name, _ in fields:
+        source.append('    newframe.%s = frame.%s' % (name, name))
+    source.append('    return lltype.cast_pointer(lltype.Ptr(STATE_HEADER),')
+    source.append('                               newframe)')
+    source.append('')
+    miniglobals = {'lltype': lltype,
+                   'll_frame_reccopy': ll_frame_reccopy,
+                   'STATE_HEADER': STATE_HEADER,
+                   }
+    exec compile2('\n'.join(source)) in miniglobals
+    extras = {
+        'adtmeths': {'reccopy': miniglobals[fnname]}
+        }
+    FRAME = lltype.GcStruct(name,
+                            ('header', STATE_HEADER),
+                            *fields, **extras)
+    miniglobals['FRAME'] = FRAME
+    return FRAME
 
 # ____________________________________________________________
 # master array giving information about the restart points
 # (STATE_HEADER.frameinfo is an index into this array)
 
+RECCOPY_FUNC = lltype.FuncType([lltype.Ptr(STATE_HEADER)],
+                               lltype.Ptr(STATE_HEADER))
+
 FRAME_INFO = lltype.Struct('frame_info',
-                           ('fnaddr', llmemory.Address),
-                           ('info',   lltype.Signed))
+                           ('fnaddr',  llmemory.Address),
+                           ('info',    lltype.Signed),
+                           ('reccopy', lltype.Ptr(RECCOPY_FUNC)))
 FRAME_INFO_ARRAY = lltype.Array(FRAME_INFO)
 
-def decodestate(masterarray, index):
+def decodestate(index):
+    from pypy.translator.stackless.code import global_state
+    masterarray = global_state.masterarray
     finfo = masterarray[index]
     if finfo.fnaddr:
         restartstate = 0
@@ -103,22 +117,28 @@
             finfo.info)    # retval_type
 decodestate.stackless_explicit = True
 
+def ll_frame_reccopy(frame):
+    from pypy.translator.stackless.code import global_state
+    masterarray = global_state.masterarray
+    finfo = masterarray[frame.f_restart]
+    return finfo.reccopy(frame)
+
 
 class RestartInfo(object):
     __slots__ = ['func_or_graph',
                  'first_index',
-                 'nb_restart_states']
+                 'frame_types']
 
-    def __init__(self, func_or_graph, first_index, nb_restart_states):
+    def __init__(self, func_or_graph, first_index, frame_types):
         self.func_or_graph = func_or_graph
         self.first_index = first_index
-        self.nb_restart_states = nb_restart_states
+        self.frame_types = frame_types
 
     def compress(self, rtyper, masterarray):
-        if self.nb_restart_states > 0:
+        if self.frame_types:
+            bk = rtyper.annotator.bookkeeper
             graph = self.func_or_graph
             if not isinstance(graph, FunctionGraph):
-                bk = rtyper.annotator.bookkeeper
                 graph = bk.getdesc(graph).getuniquegraph()
             funcptr = getfunctionptr(graph)
             rettype = lltype.typeOf(funcptr).TO.RESULT
@@ -127,18 +147,26 @@
             finfo = masterarray[self.first_index]
             finfo.fnaddr = llmemory.cast_ptr_to_adr(funcptr)
             finfo.info = retval_type
-            for i in range(1, self.nb_restart_states):
+            for i in range(1, len(self.frame_types)):
                 finfo = masterarray[self.first_index+i]
                 finfo.info = i
+            for i in range(len(self.frame_types)):
+                reccopy = self.frame_types[i].reccopy
+                s_header = annmodel.SomePtr(lltype.Ptr(STATE_HEADER))
+                mixlevelannotator = MixLevelHelperAnnotator(rtyper)
+                fnptr = mixlevelannotator.delayedfunction(reccopy, [s_header],
+                                                          s_header)
+                mixlevelannotator.finish()
+                masterarray[self.first_index+i].reccopy = fnptr
 
     prebuilt = []
     prebuiltindex = 0
 
-    def add_prebuilt(cls, func, nb_restart_states):
+    def add_prebuilt(cls, func, frame_types):
         assert func.stackless_explicit    # did you forget this flag?
         n = cls.prebuiltindex
-        restart = cls(func, n, nb_restart_states)
+        restart = cls(func, n, frame_types)
         cls.prebuilt.append(restart)
-        cls.prebuiltindex += restart.nb_restart_states
+        cls.prebuiltindex += len(frame_types)
         return n
     add_prebuilt = classmethod(add_prebuilt)

Added: pypy/dist/pypy/translator/stackless/test/test_clone.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/translator/stackless/test/test_clone.py	Fri May 12 23:50:10 2006
@@ -0,0 +1,40 @@
+from pypy.translator.stackless.test.test_transform import \
+     llinterp_stackless_function, run_stackless_function
+from pypy.rpython import rstack
+
+
+def test_simple():
+    def g(lst):
+        lst.append(1)
+        parent = rstack.yield_current_frame_to_caller()
+        # compute a bit
+        lst.append(3)
+        # switch back for the fork
+        parent = parent.switch()
+        lst.append(6)   # we are here twice!
+        return parent
+
+    def f():
+        lst = []
+        c = g(lst)
+        lst.append(2)
+        c1 = c.switch()
+        lst.append(4)
+        c2 = c1.clone()      # clone() here
+        lst.append(5)
+        end1 = c1.switch()
+        lst.append(7)
+        end2 = c2.switch()
+        lst.append(8)
+        assert not end1
+        assert not end2
+        n = 0
+        for i in lst:
+            n = n*10 + i
+        return n
+
+    data = llinterp_stackless_function(f)
+    assert data == 123456768
+
+    res = run_stackless_function(f)
+    assert res == 123456768

Modified: pypy/dist/pypy/translator/stackless/transform.py
==============================================================================
--- pypy/dist/pypy/translator/stackless/transform.py	(original)
+++ pypy/dist/pypy/translator/stackless/transform.py	Fri May 12 23:50:10 2006
@@ -187,6 +187,9 @@
             ll_stackless.ll_stackless_switch:
                 mixlevelannotator.constfunc(
                     code.ll_frame_switch, [s_StatePtr], s_StatePtr),
+            ll_stackless.ll_stackless_clone:
+                mixlevelannotator.constfunc(
+                    code.ll_frame_clone, [s_StatePtr], s_StatePtr),
             ll_stack.ll_stack_unwind:
                 mixlevelannotator.constfunc(
                     code.ll_stack_unwind, [], annmodel.s_None),
@@ -535,8 +538,9 @@
         return llops
 
     def generate_restart_infos(self, graph):
+        frame_types = [rp.frame_state_type for rp in self.resume_points]
         restartinfo = frame.RestartInfo(graph, self.restartinfoindex,
-                                        len(self.resume_points))
+                                        frame_types)
         self.restartinfos.append(restartinfo)
         self.restartinfoindex += len(self.resume_points)
 

Modified: pypy/dist/pypy/translator/unsimplify.py
==============================================================================
--- pypy/dist/pypy/translator/unsimplify.py	(original)
+++ pypy/dist/pypy/translator/unsimplify.py	Fri May 12 23:50:10 2006
@@ -7,7 +7,7 @@
     if translator is not None:
         annotator = translator.annotator
         if annotator is not None and v in annotator.bindings:
-            annotator.bindings[newvar] = annotator.bindings[v]
+            annotator.setbinding(newvar, annotator.bindings[v])
     if hasattr(v, 'concretetype'):
         newvar.concretetype = v.concretetype
     return newvar



More information about the Pypy-commit mailing list