[pypy-svn] r46902 - in pypy/dist/pypy/module/struct: . test

arigo at codespeak.net arigo at codespeak.net
Wed Sep 26 17:21:34 CEST 2007


Author: arigo
Date: Wed Sep 26 17:21:33 2007
New Revision: 46902

Added:
   pypy/dist/pypy/module/struct/   (props changed)
   pypy/dist/pypy/module/struct/__init__.py   (contents, props changed)
   pypy/dist/pypy/module/struct/app_struct.py   (contents, props changed)
   pypy/dist/pypy/module/struct/error.py   (contents, props changed)
   pypy/dist/pypy/module/struct/formatiterator.py   (contents, props changed)
   pypy/dist/pypy/module/struct/interp_struct.py   (contents, props changed)
   pypy/dist/pypy/module/struct/nativefmttable.py   (contents, props changed)
   pypy/dist/pypy/module/struct/standardfmttable.py   (contents, props changed)
   pypy/dist/pypy/module/struct/test/   (props changed)
   pypy/dist/pypy/module/struct/test/test_struct.py   (contents, props changed)
Log:
Starting an interp-level struct format.

Probably trying to pull too many RPython tricks out of my hat in
order to get efficient code after translation.


Added: pypy/dist/pypy/module/struct/__init__.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/module/struct/__init__.py	Wed Sep 26 17:21:33 2007
@@ -0,0 +1,54 @@
+
+"""
+Mixed-module definition for the struct module.
+Note that there is also a pure Python implementation in pypy/lib/struct.py;
+the present mixed-module version of struct takes precedence if it is enabled.
+"""
+
+from pypy.interpreter.mixedmodule import MixedModule
+
+
+class Module(MixedModule):
+    """\
+Functions to convert between Python values and C structs.
+Python strings are used to hold the data representing the C struct
+and also as format strings to describe the layout of data in the C struct.
+
+The optional first format char indicates byte order, size and alignment:
+ @: native order, size & alignment (default)
+ =: native order, std. size & alignment
+ <: little-endian, std. size & alignment
+ >: big-endian, std. size & alignment
+ !: same as >
+
+The remaining chars indicate types of args and must match exactly;
+these can be preceded by a decimal repeat count:
+   x: pad byte (no data);
+   c:char;
+   b:signed byte;
+   B:unsigned byte;
+   h:short;
+   H:unsigned short;
+   i:int;
+   I:unsigned int;
+   l:long;
+   L:unsigned long;
+   q:long long;
+   Q:unsigned long long
+   f:float;
+   d:double.
+Special cases (preceding decimal count indicates length):
+   s:string (array of char); p: pascal string (with count byte).
+Special case (only available in native format):
+   P:an integer type that is wide enough to hold a pointer.
+Whitespace between formats is ignored.
+
+The variable struct.error is an exception raised on errors."""
+
+    interpleveldefs = {
+        'calcsize': 'interp_struct.calcsize',
+        }
+
+    appleveldefs = {
+        'error': 'app_struct.error',
+        }

Added: pypy/dist/pypy/module/struct/app_struct.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/module/struct/app_struct.py	Wed Sep 26 17:21:33 2007
@@ -0,0 +1,10 @@
+
+"""
+Application-level definitions for the zlib module.
+
+NOT_RPYTHON
+"""
+
+class error(Exception):
+    """Exception raised on various occasions; argument is a string
+    describing what is wrong."""

Added: pypy/dist/pypy/module/struct/error.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/module/struct/error.py	Wed Sep 26 17:21:33 2007
@@ -0,0 +1,9 @@
+
+class StructError(Exception):
+    "Interp-level error that gets mapped to an app-level struct.error."
+
+    def __init__(self, msg):
+        self.msg = msg
+
+    def __str__(self):
+        return self.msg

Added: pypy/dist/pypy/module/struct/formatiterator.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/module/struct/formatiterator.py	Wed Sep 26 17:21:33 2007
@@ -0,0 +1,94 @@
+
+from pypy.rlib.unroll import unrolling_iterable
+from pypy.rlib.rarithmetic import ovfcheck
+
+from pypy.module.struct.error import StructError
+from pypy.module.struct.standardfmttable import standard_fmttable
+from pypy.module.struct.nativefmttable import native_is_bigendian
+
+
+class FormatIterator(object):
+    """
+    An iterator-like object that follows format strings step by step.
+    It provides input to the packer/unpacker and accumulates their output.
+    The subclasses are specialized for either packing, unpacking, or
+    just computing the size.
+    """
+    _mixin_ = True
+
+    def __init__(self, fmt):
+        # decode the byte order, size and alignment based on the 1st char
+        native = True
+        bigendian = native_is_bigendian
+        index = 0
+        if len(fmt) > 0:
+            c = fmt[0]
+            index = 1
+            if c == '@':
+                pass
+            elif c == '=':
+                native = False
+            elif c == '<':
+                native = False
+                bigendian = False
+            elif c == '>' or c == '!':
+                native = False
+                bigendian = True
+            else:
+                index = 0
+        self.native = native
+        self.bigendian = bigendian
+
+        # immediately interpret the format string,
+        # calling self.operate() for each format unit
+        while index < len(fmt):
+            c = fmt[index]
+            index += 1
+            if c.isspace():
+                continue
+            if c.isdigit():
+                repetitions = ord(c) - ord('0')
+                while True:
+                    if index == len(fmt):
+                        raise StructError("incomplete struct format")
+                    c = fmt[index]
+                    index += 1
+                    if not c.isdigit():
+                        break
+                    repetitions = ovfcheck(repetitions * 10)
+                    repetitions = ovfcheck(repetitions + (ord(c) - ord('0')))
+                    # XXX catch OverflowError somewhere
+            else:
+                repetitions = 1
+
+            for fmtop in unroll_fmtops:
+                if c == fmtop.fmtchar:
+                    self.operate(fmtop, repetitions)
+                    break
+            else:
+                raise StructError("bad char in struct format")
+
+
+class CalcSizeFormatIterator(FormatIterator):
+    totalsize = 0
+
+    def operate(self, fmtop, repetitions):
+        if fmtop.size == 1:
+            size = repetitions  # skip the overflow-checked multiplication by 1
+        else:
+            size = ovfcheck(fmtop.size * repetitions)
+        self.totalsize = ovfcheck(self.totalsize + size)
+    operate._annspecialcase_ = 'specialize:argvalue(1)'
+
+
+class FmtOp(object):
+    def __init__(self, fmtchar, attrs):
+        self.fmtchar = fmtchar
+        self.__dict__.update(attrs)
+    def _freeze_(self):
+        return True
+
+_items = standard_fmttable.items()
+_items.sort()
+unroll_fmtops = unrolling_iterable([FmtOp(_key, _attrs)
+                                    for _key, _attrs in _items])

Added: pypy/dist/pypy/module/struct/interp_struct.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/module/struct/interp_struct.py	Wed Sep 26 17:21:33 2007
@@ -0,0 +1,8 @@
+from pypy.interpreter.gateway import ObjSpace
+from pypy.module.struct.formatiterator import CalcSizeFormatIterator
+
+
+def calcsize(space, format):
+    fmtiter = CalcSizeFormatIterator(format)
+    return space.wrap(fmtiter.totalsize)
+calcsize.unwrap_spec = [ObjSpace, str]

Added: pypy/dist/pypy/module/struct/nativefmttable.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/module/struct/nativefmttable.py	Wed Sep 26 17:21:33 2007
@@ -0,0 +1,3 @@
+import struct
+
+native_is_bigendian = struct.pack("=i", 1) == struct.pack(">i", 1)

Added: pypy/dist/pypy/module/struct/standardfmttable.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/module/struct/standardfmttable.py	Wed Sep 26 17:21:33 2007
@@ -0,0 +1,125 @@
+"""
+The format table for standard sizes and alignments.
+"""
+
+# Note: we follow Python 2.5 in being strict about the ranges of accepted
+# values when packing.
+
+import struct
+from pypy.module.struct.error import StructError
+from pypy.rlib.unroll import unrolling_iterable
+from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong
+
+# ____________________________________________________________
+
+def pack_pad(fmtiter):
+    fmtiter.result.append('\x00')
+
+def pack_char(fmtiter):
+    xxx
+
+def pack_string(fmtiter):
+    xxx
+
+def pack_pascal(fmtiter):
+    xxx
+
+def pack_float(fmtiter):
+    xxx
+
+# ____________________________________________________________
+
+native_int_size = struct.calcsize("l")
+
+def make_int_packer(size, signed):
+    if signed:
+        min = -(2 ** (size-1))
+        max = (2 ** (size-1)) - 1
+        if size <= native_int_size:
+            accept_method = 'accept_int_arg'
+        else:
+            accept_method = 'accept_longlong_arg'
+            min = r_longlong(min)
+            max = r_longlong(max)
+    else:
+        min = 0
+        max = (2 ** size) - 1
+        if size < native_int_size:
+            accept_method = 'accept_int_arg'
+        elif size == native_int_size:
+            accept_method = 'accept_uint_arg'
+            min = r_uint(min)
+            max = r_uint(max)
+        else:
+            accept_method = 'accept_ulonglong_arg'
+            min = r_ulonglong(min)
+            max = r_ulonglong(max)
+    if size > 1:
+        plural = "s"
+    else:
+        plural = ""
+    errormsg = "argument out of range for %d-byte%s integer format" % (size,
+                                                                       plural)
+    unroll_range_size = unrolling_iterable(range(size))
+
+    def pack_int(fmtiter):
+        method = getattr(fmtiter, accept_method)
+        value = method()
+        if value < min or value > max:
+            raise StructError(errormsg)
+        if fmtiter.bigendian:
+            for i in unroll_range_size:
+                x = (value >> (8*i)) & 0xff
+                fmtiter.result.append(chr(x))
+        else:
+            for i in unroll_range_size:
+                fmtiter.result.append(chr(value & 0xff))
+                value >>= 8
+
+    return pack_int
+
+# ____________________________________________________________
+
+def unpack_pad(fmtiter):
+    xxx
+
+def unpack_char(fmtiter):
+    xxx
+
+def unpack_string(fmtiter):
+    xxx
+
+def unpack_pascal(fmtiter):
+    xxx
+
+def unpack_float(fmtiter):
+    xxx
+
+# ____________________________________________________________
+
+def make_int_unpacker(size, signed):
+    return lambda fmtiter: xxx
+
+# ____________________________________________________________
+
+standard_fmttable = {
+    'x':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_pad, 'unpack' : unpack_pad},
+    'c':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_char, 'unpack' : unpack_char},
+    's':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_string, 'unpack' : unpack_string},
+    'p':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_pascal, 'unpack' : unpack_pascal},
+    'f':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float},
+    'd':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float},
+    }    
+
+for c, size in [('b', 1), ('h', 2), ('i', 4), ('q', 8)]:    # 'l' see below
+    standard_fmttable[c] = {'size': size,
+                            'alignment': 0,
+                            'pack': make_int_packer(size, True),
+                            'unpack': make_int_unpacker(size, True)}
+    standard_fmttable[c.upper()] = {'size': size,
+                                    'alignment': 0,
+                                    'pack': make_int_packer(size, False),
+                                    'unpack': make_int_unpacker(size, False)}
+
+standard_fmttable['l'] = standard_fmttable['i']
+standard_fmttable['L'] = standard_fmttable['I']

Added: pypy/dist/pypy/module/struct/test/test_struct.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/module/struct/test/test_struct.py	Wed Sep 26 17:21:33 2007
@@ -0,0 +1,73 @@
+"""
+Tests for the struct module implemented at interp-level in pypy/module/struct.
+"""
+
+import py
+from pypy.conftest import gettestobjspace
+
+
+class AppTestStruct(object):
+
+    def setup_class(cls):
+        """
+        Create a space with the struct module and import it for use by the
+        tests.
+        """
+        cls.space = gettestobjspace(usemodules=['struct'])
+        cls.w_struct = cls.space.call_function(
+            cls.space.builtin.get('__import__'),
+            cls.space.wrap('struct'))
+
+
+    def test_error(self):
+        """
+        struct.error should be an exception class.
+        """
+        assert issubclass(self.struct.error, Exception)
+
+
+    def test_calcsize_standard(self):
+        """
+        Check the standard size of the various format characters.
+        """
+        calcsize = self.struct.calcsize
+        assert calcsize('=') == 0
+        assert calcsize('<x') == 1
+        assert calcsize('>c') == 1
+        assert calcsize('!b') == 1
+        assert calcsize('=B') == 1
+        assert calcsize('<h') == 2
+        assert calcsize('>H') == 2
+        assert calcsize('!i') == 4
+        assert calcsize('=I') == 4
+        assert calcsize('<l') == 4
+        assert calcsize('>L') == 4
+        assert calcsize('!q') == 8
+        assert calcsize('=Q') == 8
+        assert calcsize('<f') == 4
+        assert calcsize('>d') == 8
+        assert calcsize('!13s') == 13
+        assert calcsize('=500p') == 500
+        # test with some repetitions and multiple format characters
+        assert calcsize('=bQ3i') == 1 + 8 + 3*4
+
+
+    def test_calcsize_native(self):
+        """
+        Check that the size of the various format characters is reasonable.
+        """
+        skip("in-progress")
+        calcsize = self.struct.calcsize
+        assert calcsize('') == 0
+        assert calcsize('x') == 1
+        assert calcsize('c') == 1
+        assert calcsize('b') == 1
+        assert calcsize('B') == 1
+        assert (2 <= calcsize('h') == calcsize('H')
+                  <  calcsize('i') == calcsize('I')
+                  <= calcsize('l') == calcsize('L')
+                  <= calcsize('q') == calcsize('Q'))
+        assert 4 <= calcsize('f') <= 8 <= calcsize('d')
+        assert calcsize('13s') == 13
+        assert calcsize('500p') == 500
+        assert 4 <= calcsize('P') <= 8



More information about the Pypy-commit mailing list