[pypy-svn] r47799 - in pypy/dist/pypy/lang/smalltalk: . test

akuhn at codespeak.net akuhn at codespeak.net
Wed Oct 24 10:42:39 CEST 2007


Author: akuhn
Date: Wed Oct 24 10:42:39 2007
New Revision: 47799

Modified:
   pypy/dist/pypy/lang/smalltalk/model.py
   pypy/dist/pypy/lang/smalltalk/squeakimage.py
   pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py
Log:
try to commit


Modified: pypy/dist/pypy/lang/smalltalk/model.py
==============================================================================
--- pypy/dist/pypy/lang/smalltalk/model.py	(original)
+++ pypy/dist/pypy/lang/smalltalk/model.py	Wed Oct 24 10:42:39 2007
@@ -70,6 +70,29 @@
         return len(self.words)    
 
 class W_CompiledMethod(W_Object):
+    """My instances are methods suitable for interpretation by the virtual machine.  This is the only class in the system whose instances intermix both indexable pointer fields and indexable integer fields.
+
+
+    The current format of a CompiledMethod is as follows:
+
+    	header (4 bytes)
+    	literals (4 bytes each)
+    	bytecodes  (variable)
+    	trailer (variable)
+
+    The header is a 30-bit integer with the following format:
+
+    (index 0)	9 bits:	main part of primitive number   (#primitive)
+    (index 9)	8 bits:	number of literals (#numLiterals)
+    (index 17)	1 bit:	whether a large frame size is needed (#frameSize)
+    (index 18)	6 bits:	number of temporary variables (#numTemps)
+    (index 24)	4 bits:	number of arguments to the method (#numArgs)
+    (index 28)	1 bit:	high-bit of primitive number (#primitive)
+    (index 29)	1 bit:	flag bit, ignored by the VM  (#flag)
+
+
+    The trailer has two variant formats.  In the first variant, the last byte is at least 252 and the last four bytes represent a source pointer into one of the sources files (see #sourcePointer).  In the second variant, the last byte is less than 252, and the last several bytes are a compressed version of the names of the method's temporary variables.  The number of bytes used for this purpose is the value of the last byte in the method.
+    """
     def __init__(self, w_class, size,
                  bytes="", argsize=0, tempsize=0, primitive=0):
         W_Object.__init__(self, w_class)

Modified: pypy/dist/pypy/lang/smalltalk/squeakimage.py
==============================================================================
--- pypy/dist/pypy/lang/smalltalk/squeakimage.py	(original)
+++ pypy/dist/pypy/lang/smalltalk/squeakimage.py	Wed Oct 24 10:42:39 2007
@@ -14,6 +14,7 @@
             inputfile.close()
         self.swap = False
         self.pos = 0
+        self.count = 0
 
     def peek(self):
         if self.pos >= len(self.data): 
@@ -28,12 +29,17 @@
     def next(self):
         integer = self.peek()
         self.pos += 4
+        self.count += 4
         return integer 
         
+    def reset_count(self):
+        self.count = 0    
+        
     def skipbytes(self, jump):
         assert jump > 0
         assert (self.pos + jump) <= len(self.data)
-        self.pos += jump    
+        self.pos += jump 
+        self.count += jump   
      
 def splitbits(integer, lengths):
     assert sum(lengths) <= 32
@@ -56,18 +62,17 @@
     def initialize(self):
         self.read_header()
         self.read_body()
-        self.init_specialobjectdumps()
-        self.init_compactclassdumps()
-        self.init_genericobjects()
-        
-    def init_genericobjects(self):
-        for dump in self.pointer2dump.itervalues():
-            dump.as_g_object(self)        
+        self.init_compactclassesarray()
+        self.init_g_objects()
+        
+    def init_g_objects(self):
+        for chunk in self.chunks.itervalues():
+            chunk.as_g_object(self)        
         
     def read_header(self):
         version = self.stream.next()
         if version != 0x1966: raise NotImplementedError
-        self.headersize = self.stream.next()
+        headersize = self.stream.next()
         self.endofmemory = self.stream.next()   
         self.oldbaseaddress = self.stream.next()   
         self.specialobjectspointer = self.stream.next()   
@@ -75,61 +80,56 @@
         savedwindowssize = self.stream.next()
         fullscreenflag = self.stream.next()
         extravmmemory = self.stream.next()
-        self.stream.skipbytes(self.headersize - (9 * 4))
+        self.stream.skipbytes(headersize - (9 * 4))
         
     def read_body(self):
-        dumps = []
-        self.pointer2dump = {}
-        while self.stream.pos <= self.endofmemory:
-            dump = self.read_object()
-            dumps.append(dump)
-            self.pointer2dump[dump.pos - self.headersize + self.oldbaseaddress] = dump
-        return dumps
-        
-    def init_specialobjectdumps(self):
-        dump = self.pointer2dump[self.specialobjectspointer]   
-        assert dump.size > 24 #and more
-        assert dump.format == 2
-        self.sodumps = [self.pointer2dump[pointer] for pointer in dump.data]
-        
-    def init_compactclassdumps(self):
-        dump = self.sodumps[COMPACT_CLASSES_ARRAY]  
-        assert len(dump.data) == 31
-        assert dump.format == 2
-        self.ccdumps = [self.pointer2dump[pointer] for pointer in dump.data]   
+        self.chunks = {}
+        self.stream.reset_count()
+        while self.stream.count < self.endofmemory:
+            chunk, pos = self.read_object()
+            self.chunks[pos + self.oldbaseaddress] = chunk
+        return self.chunks.values()
+        
+    def init_compactclassesarray(self):
+        special = self.chunks[self.specialobjectspointer]   
+        assert special.size > 24 #at least
+        assert special.format == 2
+        chunk = self.chunks[special.data[COMPACT_CLASSES_ARRAY]]
+        assert len(chunk.data) == 31
+        assert chunk.format == 2
+        self.compactclasses = [self.chunks[pointer] for pointer in chunk.data]   
         
     def init_actualobjects(self):
-        for dump in self.pointer2dump.itervalues():
-            dump.get_actual() # initialization            
+        for chunk in self.chunks.itervalues():
+            chunk.get_actual() # initialization            
 
     def read_object(self):
         kind = self.stream.peek() & 3 # 2 bits
         if kind == 0: # 00 bits 
-            dump = self.read_3wordobjectheader()
+            chunk, pos = self.read_3wordobjectheader()
         elif kind == 1: # 01 bits
-            dump = self.read_2wordobjectheader()
+            chunk, pos = self.read_2wordobjectheader()
         elif kind == 3: # 11 bits
-            dump = self.read_1wordobjectheader()
+            chunk, pos = self.read_1wordobjectheader()
         else: # 10 bits
             raise CorruptImageError("Unused block not allowed in image")
-        size = dump.size
-        dump.data = [self.stream.next() 
+        size = chunk.size
+        chunk.data = [self.stream.next() 
                      for _ in range(size - 1)] #size-1, excluding header   
-        return dump     
+        return chunk, pos     
         
     def read_1wordobjectheader(self):
         kind, size, format, classid, idhash = (
             splitbits(self.stream.next(), [2,6,4,5,12]))
         assert kind == 3
-        return ObjectDump(size, format, classid, idhash, self.stream.pos - 4,
-                          compact = True)
+        return ImageChunk(size, format, classid, idhash), self.stream.count - 4
 
     def read_2wordobjectheader(self):
         assert splitbits(self.stream.peek(), [2])[0] == 1 #kind
         classid = self.stream.next() - 1 # remove headertype to get pointer
         kind, size, format, _, idhash = splitbits(self.stream.next(), [2,6,4,5,12])
         assert kind == 1
-        return ObjectDump(size, format, classid, idhash, self.stream.pos - 4)
+        return ImageChunk(size, format, classid, idhash), self.stream.count - 4
 
     def read_3wordobjectheader(self):
         kind, size = splitbits(self.stream.next(), [2,30]) 
@@ -138,30 +138,56 @@
         classid = self.stream.next() - 0 # remove headertype to get pointer
         kind, _, format, _, idhash = splitbits(self.stream.next(), [2,6,4,5,12])
         assert kind == 0
-        return ObjectDump(size, format, classid, idhash, self.stream.pos - 4)
+        return ImageChunk(size, format, classid, idhash), self.stream.count - 4
 
 COMPACT_CLASSES_ARRAY = 28
 
+# ____________________________________________________________
+
 class GenericObject(object):
+    """ Intermediate representation of squeak objects. To establish all
+        pointers as object references, ImageReader creates instances of
+        GenericObject from the image chunks, and uses them as starting
+        point for the actual create of pypy.lang.smalltalk.model classes.
+        """
     def __init__(self):
         self.owner = None
         
     def isinitialized(self):
         return self.owner is not None     
     
-    def initialize(self, dump, reader):
+    def initialize_int(self, value, reader):
         self.owner = reader
-        self.size = dump.size
-        self.hash12 = dump.idhash 
-        self.format = dump.format
-        self.init_class(dump)
-        self.init_data(dump) 
-        
-    def init_class(self, dump):    
-        if dump.compact:
-            self.g_class = self.owner.ccdumps[dump.classid].as_g_object(self.owner)
+        self.value = value
+        self.size = -1
+    
+    def initialize(self, chunk, reader):
+        self.owner = reader
+        self.size = chunk.size
+        self.hash12 = chunk.hash12 
+        self.format = chunk.format
+        self.init_class(chunk)
+        self.init_data(chunk) 
+        self.w_object = None
+        
+    def init_class(self, chunk):    
+        if chunk.iscompact():
+            self.g_class = self.owner.compactclasses[chunk.classid].g_object
+        else:
+            self.g_class = self.owner.chunks[chunk.classid].g_object
+
+    def init_data(self, chunk):    
+        if not self.ispointers(): return
+        self.pointers = []
+        for pointer in chunk.data:
+            g_object = self.decode_pointer(pointer)
+            self.pointers.append(g_object)
+            
+    def decode_pointer(self, pointer):
+        if (pointer & 1) == 1:
+            return GenericObject().initialize_int(pointer >> 1, self.owner)
         else:
-            self.g_class = self.owner.pointer2dump[dump.classid].as_g_object(self.owner)
+            return self.owner.chunks[pointer].g_object
             
     def isbytes(self):
         return 8 <= self.format <= 11
@@ -170,54 +196,13 @@
         return self.format == 6
         
     def ispointers(self):
-        return self.format < 8 #TODO, what about compiled methods?             
-            
-    def init_data(self, dump):        
-        if not self.ispointers(): return
-        self.data = [self.owner.pointer2dump[p].as_g_object(self.owner)
-                     for p in dump.data]
+        return self.format < 5 #TODO, what about compiled methods?             
 
-class ObjectDump(object):
-    def __init__(self, size, format, classid, idhash, pos, compact = False):
-        self.pos = pos
-        self.size = size
-        self.format = format
-        self.classid = classid
-        self.idhash = idhash
-        self.data = None
-        self.classdescription = False
-        self.actual = None
-        self.compact = compact
-        self.g_object = GenericObject()
-    
-    def __eq__(self, other):
-        "(for testing)"
-        return (self.__class__ is other.__class__ and 
-                self.pos == other.pos and
-                self.format == other.format and
-                self.classid == other.classid and
-                self.idhash == other.idhash and
-                self.compact == other.compact)
-
-    def __ne__(self, other):
-        "(for testing)"
-        return not self == other
-        
-    def get_actual(self):
-        if self.actual is None: 
-            self.actual = self.create_actual()
-        return self.actual    
-        
-    def as_g_object(self, reader):
-        if self.g_object.isinitialized():
-            self.g_object.initialize(self, reader)
-        return self.g_object        
-        
-    def create_actual(self):
+    def init_w_object(self):
         from pypy.lang.smalltalk import model
-        if self.classdescription:
-            print self.format
-            return None
+        #if self.classdescription:
+        #    print self.format
+        #    return None
         if self.format == 0: # no instvars, non-indexed
             assert self.size == 0
             return model.W_PointersObject(size = 0)
@@ -244,5 +229,35 @@
         else:
             assert 0, "not reachable"                
         
+    
+
+class ImageChunk(object):
+    def __init__(self, size, format, classid, hash12):
+        self.size = size
+        self.format = format
+        self.classid = classid
+        self.hash12 = hash12
+        self.data = None
+        self.g_object = GenericObject()
+    
+    def __eq__(self, other):
+        "(for testing)"
+        return (self.__class__ is other.__class__ and 
+                self.format == other.format and
+                self.classid == other.classid and
+                self.hash12 == other.hash12 and
+                self.data == other.data)
+
+    def __ne__(self, other):
+        "(for testing)"
+        return not self == other
+        
+    def as_g_object(self, reader):
+        if not self.g_object.isinitialized():
+            self.g_object.initialize(self, reader)
+        return self.g_object  
+        
+    def iscompact(self):
+        return 0 < self.classid < 32                      
             
             

Modified: pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py
==============================================================================
--- pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py	(original)
+++ pypy/dist/pypy/lang/smalltalk/test/test_squeakimage.py	Wed Oct 24 10:42:39 2007
@@ -1,92 +1,90 @@
 import py
-import StringIO
 from pypy.lang.smalltalk import squeakimage
 
-filepath = py.magic.autopath().dirpath().dirpath().join('mini.image')
+# ----- helpers ----------------------------------------------
 
-def test_miniimageexists():
-    assert filepath.check(dir=False)
+def ints2str(*ints):
+    import struct
+    return struct.pack(">" + "i" * len(ints), *ints)
 
-def test_reader():
-    f = StringIO.StringIO('\x00\x00\x19\x66')
-    reader = squeakimage.Stream(f)
-    first = reader.peek()
-    assert first == 6502 
-    first = reader.next()
-    assert first == 6502 
-    py.test.raises(IndexError, lambda: reader.next())
-    
-def test_swap():
-    f = StringIO.StringIO('\x66\x19\x00\x00')
-    reader = squeakimage.Stream(f)
-    reader.swap = True
-    first = reader.next()
+def joinbits(values, lengths):
+    result = 0
+    for each, length in reversed(zip(values, lengths)):
+        result = result << length
+        result += each
+    return result   
+
+def imagereader_mock(string):
+    import StringIO
+    f = StringIO.StringIO(string)
+    stream = squeakimage.Stream(f)
+    return squeakimage.ImageReader(stream)
+
+
+# ----- tests ------------------------------------------------
+
+def test_stream():
+    stream = imagereader_mock('\x00\x00\x19\x66').stream
+    n = stream.peek()
+    assert n == 6502 
+    n = stream.next()
+    assert n == 6502 
+    py.test.raises(IndexError, lambda: stream.next())
+    
+def test_stream_swap():
+    stream = imagereader_mock('\x66\x19\x00\x00').stream
+    stream.swap = True
+    first = stream.next()
     assert first == 6502 
-    py.test.raises(IndexError, lambda: reader.next())
+    py.test.raises(IndexError, lambda: stream.next())
     
-def test_many():
-    f = StringIO.StringIO('\x00\x00\x19\x66' * 5)
-    reader = squeakimage.Stream(f)
+def test_stream_many():
+    stream = imagereader_mock('\x00\x00\x19\x66' * 5).stream
     for each in range(5):
-        first = reader.peek()
+        first = stream.peek()
         assert first == 6502 
-        value = reader.next()
+        value = stream.next()
         assert value == 6502 
-    py.test.raises(IndexError, lambda: reader.next())
+    py.test.raises(IndexError, lambda: stream.next())
     
-def test_skipbytes():
-    f = StringIO.StringIO('\xFF\xFF\xFF\x00\x00\x19\x66')     
-    reader = squeakimage.Stream(f)
-    reader.skipbytes(3)
-    value = reader.next()
+def test_stream_skipbytes():
+    stream = imagereader_mock('\xFF\xFF\xFF\x00\x00\x19\x66').stream
+    stream.skipbytes(3)
+    value = stream.next()
     assert value == 6502 
-    py.test.raises(IndexError, lambda: reader.next())
+    py.test.raises(IndexError, lambda: stream.next())
         
-def test_splitbits():
+def test_stream_count():
+    stream = imagereader_mock('\xFF' * 20).stream
+    stream.next()
+    stream.next()
+    stream.reset_count()
+    assert stream.count == 0
+    stream.next()        
+    assert stream.count == 4
+    stream.next()        
+    assert stream.count == 8
+    
+def test_simple_splitbits():
     assert ([1] * 4) == squeakimage.splitbits(0x01010101, [8,8,8,8])
     assert ([255] * 4) == squeakimage.splitbits(0xFfFfFfFf, [8,8,8,8])
+
+def test_fancy_splitbits():
     assert [4,3,2,1] == squeakimage.splitbits(0x01020304, [8,8,8,8])
     assert [1,3,7,15] == squeakimage.splitbits(0xFfFfFfFf, [1,2,3,4])
     
-def test_readheader():
-    reader = squeakimage.Stream(filepath.open())
-    ireader = squeakimage.ImageReader(reader)
-    ireader.read_header()
-    assert ireader.endofmemory == 0x93174
-    assert ireader.oldbaseaddress == 0x6649000
-    assert ireader.specialobjectspointer == 0x6668380
-    next = reader.next()
-    assert next != 0 #expects object header, which can not be 0x0 
-
-def test_readheader_and_body():
-    reader = squeakimage.Stream(filepath.open())
-    ireader = squeakimage.ImageReader(reader)
-    ireader.read_header()
-    objects = ireader.read_body()
-    assert len(objects) > 0
-    assert 15000 < len(objects) < 16000 
-
-def imagereader_mock(string):
-    f = StringIO.StringIO(string)
-    reader = squeakimage.Stream(f)
-    return squeakimage.ImageReader(reader)
-    
-def joinbits(values, lengths):
-    result = 0
-    for each, length in reversed(zip(values, lengths)):
-        result = result << length
-        result += each
-    return result   
+def test_format_splitbits():
+    x = 0xAA
+    assert [x & 3] == squeakimage.splitbits(x, [2])  
     
-def test_joinbits():
+def test_simple_joinbits():
     assert 0x01010101 == joinbits(([1] * 4), [8,8,8,8])
     assert 0xFfFfFfFf == joinbits([255] * 4, [8,8,8,8])
+    
+def test_fancy_joinbits():    
     assert 0x01020304 == joinbits([4,3,2,1], [8,8,8,8])
     assert 0x3Ff == joinbits([1,3,7,15], [1,2,3,4])
     
-def ints2str(*ints):
-    import struct
-    return struct.pack(">" + "i" * len(ints), *ints)
     
 def test_ints2str():
     assert "\x00\x00\x00\x02" == ints2str(2)       
@@ -99,47 +97,75 @@
 def test_1wordobjectheader():
     s = ints2str(joinbits([3, 1, 2, 3, 4], [2,6,4,5,12]))
     r = imagereader_mock(s)
-    assert squeakimage.ObjectDump(1, 2, 3, 4, 0, True) == r.read_1wordobjectheader()
+    assert (squeakimage.ImageChunk(1, 2, 3, 4), 0) == r.read_1wordobjectheader()
 
 def test_1wordobjectheader2():
     s = ints2str(joinbits([3, 1, 2, 3, 4], [2,6,4,5,12]))
     r = imagereader_mock(s * 3)
-    assert squeakimage.ObjectDump(1, 2, 3, 4, 0, True) == r.read_1wordobjectheader()
-    assert squeakimage.ObjectDump(1, 2, 3, 4, 4, True) == r.read_1wordobjectheader()
-    assert squeakimage.ObjectDump(1, 2, 3, 4, 8, True) == r.read_1wordobjectheader()
+    assert (squeakimage.ImageChunk(1, 2, 3, 4), 0) == r.read_1wordobjectheader()
+    assert (squeakimage.ImageChunk(1, 2, 3, 4), 4) == r.read_1wordobjectheader()
+    assert (squeakimage.ImageChunk(1, 2, 3, 4), 8) == r.read_1wordobjectheader()
 
 def test_2wordobjectheader():
     s = ints2str(4200 + 1, joinbits([1, 1, 2, 3, 4], [2,6,4,5,12]))
     r = imagereader_mock(s)
-    assert squeakimage.ObjectDump(1, 2, 4200, 4, 4) == r.read_2wordobjectheader()
+    assert (squeakimage.ImageChunk(1, 2, 4200, 4), 4) == r.read_2wordobjectheader()
 
 def test_3wordobjectheader():
     s = ints2str(1701 << 2, 4200 + 0, joinbits([0, 1, 2, 3, 4], [2,6,4,5,12]))
     r = imagereader_mock(s)
-    assert squeakimage.ObjectDump(1701, 2, 4200, 4, 8) == r.read_3wordobjectheader()
+    assert (squeakimage.ImageChunk(1701, 2, 4200, 4), 8) == r.read_3wordobjectheader()
     
 def test_read3wordheaderobject():
     size = 42
     s = ints2str(size << 2, 4200 + 0, joinbits([0, 1, 2, 3, 4], [2,6,4,5,12]))
     r = imagereader_mock(s + '\x00\x00\x19\x66' * (size - 1))
-    dump = r.read_object()
-    dump0 = squeakimage.ObjectDump(size, 2, 4200, 4, 8)
-    dump0.data = [6502] * (size - 1)
-    assert dump0 == dump
-    
-def test_smoketest0():
-    reader = squeakimage.Stream(filepath.open())
-    ireader = squeakimage.ImageReader(reader)
-    ireader.initialize()
-    
-    
-def test_smoketest():
-    reader = squeakimage.Stream(filepath.open())
-    ireader = squeakimage.ImageReader(reader)
-    ireader.read_header()
-    ireader.read_body()
-    ireader.init_specialobjectdumps()
-    ireader.init_compactclassdumps()
-    #ireader.init_actualobjects()
-           
+    chunk, pos = r.read_object()
+    chunk0 = squeakimage.ImageChunk(size, 2, 4200, 4)
+    chunk0.data = [6502] * (size - 1)
+    assert pos == 8
+    assert chunk0 == chunk
+    
+# ----- mini.image productline -------------------------------
+#       relying on order of methods
+
+mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image')
+
+def test_miniimageexists():
+    assert mini_image.check(dir=False)
+
+example = squeakimage.ImageReader(squeakimage.Stream(mini_image.open()))
+   
+def test_read_header():
+    example.read_header()
+    assert example.endofmemory == 0x93174
+    assert example.oldbaseaddress == 0x6649000
+    assert example.specialobjectspointer == 0x6668380
+    
+def test_read_all_header():    
+    next = example.stream.peek()
+    assert next != 0 #expects object header, which must not be 0x00000000 
+   
+def test_read_body():
+    objects = example.read_body()
+    assert len(objects) > 0
+    assert 15000 < len(objects) < 16000 
+    
+def test_read_all_body():    
+    py.test.raises(IndexError, lambda: example.stream.next())
+    
+def test_all_pointers_valid():
+    for each in example.chunks.itervalues():
+        if each.format < 5: 
+            for pointer in each.data:
+                if (pointer & 1) != 1:
+                    assert pointer in example.chunks   
+   
+def test_init_compactclasses():
+    example.init_compactclassesarray()
+    assert len(example.compactclasses) == 31
+    
+def test_init_g_objects():
+    example.init_g_objects()
+          
     
\ No newline at end of file



More information about the Pypy-commit mailing list