[pypy-svn] r68116 - in pypy/trunk/pypy/jit/metainterp: . test

pedronis at codespeak.net pedronis at codespeak.net
Thu Oct 1 19:06:03 CEST 2009


Author: pedronis
Date: Thu Oct  1 19:06:03 2009
New Revision: 68116

Modified:
   pypy/trunk/pypy/jit/metainterp/optimizeopt.py
   pypy/trunk/pypy/jit/metainterp/resume.py
   pypy/trunk/pypy/jit/metainterp/simple_optimize.py
   pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py
   pypy/trunk/pypy/jit/metainterp/test/test_resume.py
Log:
(cfbolz, pedronis) avoid flattening of the resume data first, tried to reduce loops and list construction,
we compute a numbering only once at the end now



Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/optimizeopt.py	(original)
+++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py	Thu Oct  1 19:06:03 2009
@@ -56,6 +56,11 @@
     def get_key_box(self):
         return self.box
 
+    def register_value(self, modifier):
+        box = self.get_key_box()   # may be a Const, too
+        if modifier.register_box(box):
+            self.get_args_for_fail(modifier)
+
     def get_args_for_fail(self, modifier):
         pass
 
@@ -499,18 +504,9 @@
     def store_final_boxes_in_guard(self, op):
         descr = op.descr
         assert isinstance(descr, compile.ResumeGuardDescr)
-        oldboxes = []
-        args = resume.flatten_resumedata(descr) # xxx expensive
-        for box in args:
-            if box in self.values:
-                box = self.values[box].get_key_box()   # may be a Const, too
-            oldboxes.append(box)
-        modifier = resume.ResumeDataVirtualAdder(descr, oldboxes)
-        for box in args:
-            if box in self.values:
-                value = self.values[box]
-                value.get_args_for_fail(modifier)
-        newboxes = modifier.finish()
+        modifier = resume.ResumeDataVirtualAdder(descr)
+        modifier.walk_snapshots(self.values)
+        newboxes = modifier.finish(self.values)
         descr.store_final_boxes(op, newboxes)
 
     def clean_fields_of_values(self, descr=None):

Modified: pypy/trunk/pypy/jit/metainterp/resume.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/resume.py	(original)
+++ pypy/trunk/pypy/jit/metainterp/resume.py	Thu Oct  1 19:06:03 2009
@@ -58,73 +58,39 @@
         snapshot = Snapshot(snapshot, virtualizable_boxes[:]) # xxx for now
     storage.rd_snapshot = snapshot
 
-_placeholder = (None, 0, 0)
-
-def flatten_resumedata(storage):
-    frame_info_list = storage.rd_frame_info_list
-    storage.rd_frame_info_list = None
-    j = frame_info_list.level
-    frame_infos = [_placeholder]*j
-    j -= 1
-    while True: # at least one
-        frame_infos[j] = (frame_info_list.jitcode, frame_info_list.pc,
-                          frame_info_list.exception_target)
-        frame_info_list = frame_info_list.prev
-        if frame_info_list is None:
-            break
-        j -= 1
-    storage.rd_frame_infos = frame_infos
-    memo = {}
-    consts = []
-    liveboxes = []
-    nums = []
-    snapshots = []
-    snapshot = storage.rd_snapshot
-    storage.rd_snapshot = None
-    while True:
-        snapshots.append(snapshot)
-        snapshot = snapshot.prev
-        if snapshot is None:
-            break    
-    n = len(snapshots)-1
-    while n >= 0:
-        boxes = snapshots[n].boxes
-        n -= 1
-        for box in boxes:
-            assert box is not None
-            if isinstance(box, Box):
-                try:
-                    num = memo[box]
-                except KeyError:
-                    num = len(liveboxes)
-                    liveboxes.append(box)
-                    memo[box] = num
-            else:
-                num = -2 - len(consts)
-                consts.append(box)
-            nums.append(num)
-        nums.append(-1)
-    nums = nums[:]
-    storage.rd_nums = nums
-    storage.rd_consts = consts[:]
-    storage.rd_virtuals = None
-    return liveboxes
  
 VIRTUAL_FLAG = int((sys.maxint+1) // 2)
 assert not (VIRTUAL_FLAG & (VIRTUAL_FLAG-1))    # a power of two
 
+_frame_info_placeholder = (None, 0, 0)
+
 class ResumeDataVirtualAdder(object):
 
-    def __init__(self, storage, liveboxes):
+    def __init__(self, storage):
         self.storage = storage
-        self.consts = storage.rd_consts[:]
-        assert storage.rd_virtuals is None
-        self.original_liveboxes = liveboxes
         self.liveboxes = {}
-        self._register_boxes(liveboxes)
+        self.consts = []
         self.virtuals = []
         self.vfieldboxes = []
 
+    def walk_snapshots(self, values):
+        nnums = 0
+        snapshot = self.storage.rd_snapshot
+        assert snapshot
+        while True: # at least one
+            boxes = snapshot.boxes
+            nnums += len(boxes)+1
+            for box in boxes:
+                if box in values:
+                    value = values[box]
+                    value.register_value(self)
+                else:
+                    self.register_box(box)
+            snapshot = snapshot.prev
+            if snapshot is None:
+                break
+        self.nnums = nnums
+
     def make_constant(self, box, const):
         # this part of the interface is not used so far by optimizeopt.py
         if self.liveboxes[box] == 0:
@@ -149,26 +115,60 @@
         self.vfieldboxes.append(fieldboxes)
         self._register_boxes(fieldboxes)
 
+    def register_box(self, box):
+        if isinstance(box, Box) and box not in self.liveboxes:
+            self.liveboxes[box] = 0
+            return True
+        return False
+                
     def _register_boxes(self, boxes):
         for box in boxes:
-            if isinstance(box, Box) and box not in self.liveboxes:
-                self.liveboxes[box] = 0
+            self.register_box(box)
 
     def is_virtual(self, virtualbox):
         return self.liveboxes[virtualbox] >= VIRTUAL_FLAG
 
-    def finish(self):
+    def _flatten_frame_info(self):
+        storage = self.storage        
+        frame_info_list = storage.rd_frame_info_list
+        storage.rd_frame_info_list = None
+        j = frame_info_list.level
+        frame_infos = [_frame_info_placeholder]*j
+        j -= 1
+        while True: # at least one
+            frame_infos[j] = (frame_info_list.jitcode, frame_info_list.pc,
+                              frame_info_list.exception_target)
+            frame_info_list = frame_info_list.prev
+            if frame_info_list is None:
+                break
+            j -= 1
+        storage.rd_frame_infos = frame_infos        
+
+    def finish(self, values):
+        self._flatten_frame_info()
         storage = self.storage
         liveboxes = []
         for box in self.liveboxes.iterkeys():
             if self.liveboxes[box] == 0:
                 self.liveboxes[box] = len(liveboxes)
                 liveboxes.append(box)
-        for i in range(len(storage.rd_nums)):
-            num = storage.rd_nums[i]
-            if num >= 0:
-                box = self.original_liveboxes[num]
-                storage.rd_nums[i] = self._getboxindex(box)
+        nums = storage.rd_nums = [0]*self.nnums
+        i = self.nnums-1
+        snapshot = self.storage.rd_snapshot
+        while True: # at least one
+            boxes = snapshot.boxes
+            nums[i] = -1
+            i -= 1
+            for j in range(len(boxes)-1, -1, -1):
+                box = boxes[j]
+                if box in values:
+                    box = values[box].get_key_box()
+                nums[i] = self._getboxindex(box)
+                i -= 1
+            snapshot = snapshot.prev
+            if snapshot is None:
+                break
+        storage.rd_virtuals = None
         if len(self.virtuals) > 0:
             storage.rd_virtuals = self.virtuals[:]
             for i in range(len(storage.rd_virtuals)):
@@ -177,6 +177,7 @@
                 vinfo.fieldnums = [self._getboxindex(box)
                                    for box in fieldboxes]
         storage.rd_consts = self.consts[:]
+        storage.rd_snapshot = None
         if debug:
             dump_storage(storage)
         return liveboxes

Modified: pypy/trunk/pypy/jit/metainterp/simple_optimize.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/simple_optimize.py	(original)
+++ pypy/trunk/pypy/jit/metainterp/simple_optimize.py	Thu Oct  1 19:06:03 2009
@@ -5,6 +5,8 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.metainterp import resume, compile
 
+EMPTY_VALUES = {}
+
 def optimize_loop(options, old_loops, loop, cpu=None):
     if old_loops:
         assert len(old_loops) == 1
@@ -19,8 +21,10 @@
             if op.is_guard():
                 descr = op.descr
                 assert isinstance(descr, compile.ResumeGuardDescr)
-                args = resume.flatten_resumedata(descr)
-                descr.store_final_boxes(op, args)
+                modifier = resume.ResumeDataVirtualAdder(descr)
+                modifier.walk_snapshots(EMPTY_VALUES)
+                newboxes = modifier.finish(EMPTY_VALUES)
+                descr.store_final_boxes(op, newboxes)
             newoperations.append(op)
         loop.operations = newoperations
         return None

Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py	(original)
+++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py	Thu Oct  1 19:06:03 2009
@@ -32,7 +32,6 @@
     op = ResOperation(rop.GUARD_TRUE, [], None, descr=fdescr)
     # setup rd data
     fi = [("code0", 1, 2), ("code1", 3, -1)]
-    fdescr.rd_virtuals = None
     fi0 = resume.FrameInfo(None, FakeFrame("code0", 1, 2))
     fdescr.rd_frame_info_list = resume.FrameInfo(fi0,
                                                  FakeFrame("code1", 3, -1))
@@ -139,7 +138,6 @@
         descr = Storage()
         descr.rd_frame_info_list = resume.FrameInfo(None, FakeFrame())
         descr.rd_snapshot = resume.Snapshot(None, fail_args)
-        descr.rd_virtuals = None
         return descr
 
     def assert_equal(self, optimized, expected):
@@ -1274,7 +1272,6 @@
                     fdescr.rd_frame_info_list = resume.FrameInfo(None,
                                                                  FakeFrame())
                     fdescr.rd_snapshot = resume.Snapshot(None, fail_args)
-                    fdescr.virtuals = None
                 self.oparse = oparse
         #
         fdescr = instantiate(FailDescr)

Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py
==============================================================================
--- pypy/trunk/pypy/jit/metainterp/test/test_resume.py	(original)
+++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py	Thu Oct  1 19:06:03 2009
@@ -9,21 +9,17 @@
 class Storage:
     pass
 
-def make_demo_storage():
+def test_simple_read():
     b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()]
     c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)]
     storage = Storage()
     storage.rd_frame_infos = []
-    storage.rd_virtuals = None
     storage.rd_consts = [c1, c2, c3]
     storage.rd_nums = [0, -2, 0, 1, -1,
                        -3, -4, -1,
                        0, 1, 2, -1
                        ]
-    return storage
-
-def test_simple():
-    storage = make_demo_storage()
+    storage.rd_virtuals = None
     b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()]
     assert b1s != b3s
     reader = ResumeDataReader(storage, [b1s, b2s, b3s])
@@ -40,7 +36,7 @@
     storage.rd_frame_infos = [(1, 2, 5), (3, 4, 7)]
     storage.rd_consts = []
     storage.rd_nums = []
-    storage.rd_virtuals = None
+    storage.rd_virtuals = None    
     #
     reader = ResumeDataReader(storage, [])
     assert reader.has_more_frame_infos()
@@ -95,40 +91,6 @@
     assert fi1.exception_target == 4
     assert fi1.level == 2
 
-def test_flatten_resumedata():
-    # temporary "expensive" mean to go from the new to the old world
-    b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()]
-    c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)]    
-
-    env = [b1, c1, b2, b1, c2]
-    snap = Snapshot(None, env)
-    frame = FakeFrame("JITCODE", 1, 2)    
-    fi = FrameInfo(None, frame)
-    env1 = [c3, b3, b1, c1]
-    snap1 = Snapshot(snap, env1)
-    frame1 = FakeFrame("JITCODE1", 3, 4)    
-    fi1 = FrameInfo(fi, frame1)
-
-    storage = Storage()
-    storage.rd_snapshot = snap1
-    storage.rd_frame_info_list = fi1
-
-    liveboxes = flatten_resumedata(storage)
-    assert storage.rd_snapshot is None
-    assert storage.rd_frame_info_list is None
-
-    assert storage.rd_frame_infos == [("JITCODE", 1, 2),
-                                      ("JITCODE1", 3, 4)]
-    assert storage.rd_virtuals is None
-    assert liveboxes == [b1, b2, b3]
-    assert storage.rd_consts == [c1, c2, c3, c1]
-    # check with reading
-    reader = ResumeDataReader(storage, liveboxes)
-    l = reader.consume_boxes()
-    assert l == env
-    l = reader.consume_boxes()
-    assert l == env1
-
 def test_capture_resumedata():
     b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()]
     c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)]    
@@ -198,6 +160,57 @@
 
 # ____________________________________________________________
 
+def test_walk_snapshots():
+    b1, b2, b3 = [BoxInt(), BoxInt(), BoxInt()]
+    c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)]    
+
+    env = [b1, c1, b2, b1, c2]
+    snap = Snapshot(None, env)
+    env1 = [c3, b3, b1, c1]
+    snap1 = Snapshot(snap, env1)
+
+    storage = Storage()
+    storage.rd_snapshot = snap1
+
+    modifier = ResumeDataVirtualAdder(storage)
+    modifier.walk_snapshots({})
+
+    assert modifier.liveboxes == {b1: 0, b2: 0, b3: 0}
+    assert modifier.nnums == len(env)+1+len(env1)+1
+
+    b1_2 = BoxInt()
+    class FakeValue(object):
+
+        def register_value(self, modifier):
+            modifier.register_box(b1_2)
+    val = FakeValue()
+
+    storage = Storage()
+    storage.rd_snapshot = snap1
+
+    modifier = ResumeDataVirtualAdder(storage)
+    modifier.walk_snapshots({b1: val, b2: val})    
+
+    assert modifier.liveboxes == {b1_2: 0, b3: 0}
+    assert modifier.nnums == len(env)+1+len(env1)+1
+
+def test_flatten_frame_info():
+    frame = FakeFrame("JITCODE", 1, 2)    
+    fi = FrameInfo(None, frame)
+    frame1 = FakeFrame("JITCODE1", 3, 4)    
+    fi1 = FrameInfo(fi, frame1)
+
+    storage = Storage()
+    storage.rd_frame_info_list = fi1
+
+    modifier = ResumeDataVirtualAdder(storage)
+    modifier._flatten_frame_info()
+    assert storage.rd_frame_info_list is None
+
+    assert storage.rd_frame_infos == [("JITCODE", 1, 2),
+                                      ("JITCODE1", 3, 4)]
+
+
 class MyMetaInterp:
     def __init__(self, cpu):
         self.cpu = cpu
@@ -221,15 +234,27 @@
     assert len(newboxes) == len(expected)
     return newboxes
 
+def make_storage(b1, b2, b3):
+    storage = Storage()
+    snapshot = Snapshot(None, [b1, ConstInt(1), b1, b2])
+    snapshot = Snapshot(snapshot, [ConstInt(2), ConstInt(3)])
+    snapshot = Snapshot(snapshot, [b1, b2, b3])    
+    storage.rd_snapshot = snapshot
+    # just to placate _flatten_frame_info
+    storage.rd_frame_info_list = FrameInfo(None, FakeFrame("", 0, -1))
+    return storage
+
 def test_virtual_adder_no_op():
-    storage = make_demo_storage()
     b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)]
-    modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s])
+    storage = make_storage(b1s, b2s, b3s)
+    modifier = ResumeDataVirtualAdder(storage)
+    modifier.walk_snapshots({})
     assert not modifier.is_virtual(b1s)
     assert not modifier.is_virtual(b2s)
     assert not modifier.is_virtual(b3s)
     # done
-    liveboxes = modifier.finish()
+    liveboxes = modifier.finish({})
+    assert storage.rd_snapshot is None
     b1t, b2t, b3t = [BoxInt(11), BoxPtr(demo55o), BoxInt(33)]
     newboxes = _resume_remap(liveboxes, [b1s, b2s, b3s], b1t, b2t, b3t)
     metainterp = MyMetaInterp(LLtypeMixin.cpu)
@@ -242,16 +267,46 @@
     assert lst == [b1t, b2t, b3t]
     assert metainterp.trace == []
 
+def test_virtual_adder_no_op_renaming():
+    b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)]
+    storage = make_storage(b1s, b2s, b3s)
+    modifier = ResumeDataVirtualAdder(storage)
+    b1_2 = BoxInt()
+    class FakeValue(object):
+
+        def register_value(self, modifier):
+            modifier.register_box(b1_2)
+
+        def get_key_box(self):
+            return b1_2
+
+    val = FakeValue()
+    values = {b1s: val, b2s: val}  
+    modifier.walk_snapshots(values)
+    assert not modifier.is_virtual(b1_2)
+    assert not modifier.is_virtual(b3s)
+    # done
+    liveboxes = modifier.finish(values)
+    assert storage.rd_snapshot is None
+    b1t, b3t = [BoxInt(11), BoxInt(33)]
+    newboxes = _resume_remap(liveboxes, [b1_2, b3s], b1t, b3t)
+    metainterp = MyMetaInterp(LLtypeMixin.cpu)
+    reader = ResumeDataReader(storage, newboxes, metainterp)
+    lst = reader.consume_boxes()
+    assert lst == [b1t, ConstInt(1), b1t, b1t]
+    lst = reader.consume_boxes()
+    assert lst == [ConstInt(2), ConstInt(3)]
+    lst = reader.consume_boxes()
+    assert lst == [b1t, b1t, b3t]
+    assert metainterp.trace == []    
 
 def test_virtual_adder_make_virtual():
-    storage = make_demo_storage()
     b1s, b2s, b3s, b4s, b5s = [BoxInt(1), BoxPtr(), BoxInt(3),
-                               BoxPtr(), BoxPtr()]
+                               BoxPtr(), BoxPtr()]  
     c1s = ConstInt(111)
-    modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s])
-    assert not modifier.is_virtual(b1s)
-    assert not modifier.is_virtual(b2s)
-    assert not modifier.is_virtual(b3s)
+    storage = make_storage(b1s, b2s, b3s)
+    modifier = ResumeDataVirtualAdder(storage)
+    modifier.walk_snapshots({})
     modifier.make_virtual(b2s,
                           ConstAddr(LLtypeMixin.node_vtable_adr,
                                     LLtypeMixin.cpu),
@@ -269,7 +324,7 @@
     assert     modifier.is_virtual(b4s)
     assert not modifier.is_virtual(b5s)
     # done
-    liveboxes = modifier.finish()
+    liveboxes = modifier.finish({})
     b1t, b3t, b5t = [BoxInt(11), BoxInt(33), BoxPtr(demo55o)]
     newboxes = _resume_remap(liveboxes, [b1s,
                                           #b2s -- virtual
@@ -315,22 +370,24 @@
 
 def test_virtual_adder_make_constant():
     for testnumber in [0, 1]:
-        storage = make_demo_storage()
         b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)]
         if testnumber == 0:
-            # I. making a constant with make_constant()
-            modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s])
+            # I. making a constant by directly specifying a constant in
+            #    the list of liveboxes
+            b1s = ConstInt(111)
+        storage = make_storage(b1s, b2s, b3s)
+        modifier = ResumeDataVirtualAdder(storage)
+        modifier.walk_snapshots({})
+
+        if testnumber == 1:
+            # II. making a constant with make_constant()
             modifier.make_constant(b1s, ConstInt(111))
             assert not modifier.is_virtual(b1s)
-        else:
-            # II. making a constant by directly specifying a constant in
-            #     the list of liveboxes
-            b1s = ConstInt(111)
-            modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s])
+
         assert not modifier.is_virtual(b2s)
         assert not modifier.is_virtual(b3s)
         # done
-        liveboxes = modifier.finish()
+        liveboxes = modifier.finish({})
         b2t, b3t = [BoxPtr(demo55o), BoxInt(33)]
         newboxes = _resume_remap(liveboxes, [b2s, b3s], b2t, b3t)
         metainterp = MyMetaInterp(LLtypeMixin.cpu)
@@ -346,13 +403,11 @@
 
 
 def test_virtual_adder_make_varray():
-    storage = make_demo_storage()
     b1s, b2s, b3s, b4s = [BoxInt(1), BoxPtr(), BoxInt(3), BoxInt(4)]
     c1s = ConstInt(111)
-    modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s])
-    assert not modifier.is_virtual(b1s)
-    assert not modifier.is_virtual(b2s)
-    assert not modifier.is_virtual(b3s)
+    storage = make_storage(b1s, b2s, b3s)
+    modifier = ResumeDataVirtualAdder(storage)
+    modifier.walk_snapshots({})    
     modifier.make_varray(b2s,
                          LLtypeMixin.arraydescr,
                          [b4s, c1s])   # new fields
@@ -361,7 +416,7 @@
     assert not modifier.is_virtual(b3s)
     assert not modifier.is_virtual(b4s)
     # done
-    liveboxes = modifier.finish()
+    liveboxes = modifier.finish({})
     b1t, b3t, b4t = [BoxInt(11), BoxInt(33), BoxInt(44)]
     newboxes = _resume_remap(liveboxes, [b1s,
                                           #b2s -- virtual
@@ -395,13 +450,11 @@
 
 
 def test_virtual_adder_make_vstruct():
-    storage = make_demo_storage()
     b1s, b2s, b3s, b4s = [BoxInt(1), BoxPtr(), BoxInt(3), BoxPtr()]
     c1s = ConstInt(111)
-    modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s])
-    assert not modifier.is_virtual(b1s)
-    assert not modifier.is_virtual(b2s)
-    assert not modifier.is_virtual(b3s)
+    storage = make_storage(b1s, b2s, b3s)
+    modifier = ResumeDataVirtualAdder(storage)
+    modifier.walk_snapshots({})
     modifier.make_vstruct(b2s,
                           LLtypeMixin.ssize,
                           [LLtypeMixin.adescr, LLtypeMixin.bdescr],
@@ -411,7 +464,7 @@
     assert not modifier.is_virtual(b3s)
     assert not modifier.is_virtual(b4s)
     # done
-    liveboxes = modifier.finish()
+    liveboxes = modifier.finish({})
     b1t, b3t, b4t = [BoxInt(11), BoxInt(33), BoxPtr()]
     newboxes = _resume_remap(liveboxes, [b1s,
                                           #b2s -- virtual



More information about the Pypy-commit mailing list