[pypy-svn] r45810 - in pypy/branch/pypy-more-rtti-inprogress/rlib: . test
arigo at codespeak.net
arigo at codespeak.net
Fri Aug 17 14:59:01 CEST 2007
Author: arigo
Date: Fri Aug 17 14:58:59 2007
New Revision: 45810
Modified:
pypy/branch/pypy-more-rtti-inprogress/rlib/rmarshal.py
pypy/branch/pypy-more-rtti-inprogress/rlib/test/test_rmarshal.py
Log:
- add support for more types
- make translatable.
Modified: pypy/branch/pypy-more-rtti-inprogress/rlib/rmarshal.py
==============================================================================
--- pypy/branch/pypy-more-rtti-inprogress/rlib/rmarshal.py (original)
+++ pypy/branch/pypy-more-rtti-inprogress/rlib/rmarshal.py Fri Aug 17 14:58:59 2007
@@ -6,8 +6,9 @@
from pypy.annotation.signature import annotation
from pypy.annotation.listdef import ListDef, TooLateForChange
from pypy.annotation.pairtype import pair, pairtype
-from pypy.rlib.rarithmetic import formatd
+from pypy.rlib.rarithmetic import formatd, r_longlong, intmask
from pypy.rlib.unroll import unrolling_iterable
+from pypy.rpython import controllerentry
class CannotMarshal(Exception):
pass
@@ -47,14 +48,17 @@
of type 'type'. It raises ValueError if the marshalled data is
invalid or contains an object of a different type.
"""
- loader = get_loader(type)
- # wrap the loader into a more convenient interface
+ loaditem = get_loader(type)
+ # wrap the loaditem into a more convenient interface
try:
- return _unmarshaller_cache[loader]
+ return _unmarshaller_cache[loaditem]
except KeyError:
def unmarshaller(buf):
- return loader(Loader(buf))
- _unmarshaller_cache[loader] = unmarshaller
+ loader = Loader(buf)
+ result = loaditem(loader)
+ loader.check_finished()
+ return result
+ _unmarshaller_cache[loaditem] = unmarshaller
return unmarshaller
get_unmarshaller._annspecialcase_ = 'specialize:memo'
_unmarshaller_cache = {}
@@ -64,27 +68,30 @@
# Dumpers and loaders
TYPE_NONE = 'N'
+TYPE_FALSE = 'F'
+TYPE_TRUE = 'T'
TYPE_INT = 'i'
+TYPE_INT64 = 'I'
TYPE_FLOAT = 'f'
-#TYPE_LONG = 'l'
TYPE_STRING = 's'
TYPE_TUPLE = '('
TYPE_LIST = '['
dumpers = []
loaders = []
-s_list_of_chars = ListDef(None, annmodel.SomeChar(),
- mutated=True, resized=True)
+s_list_of_chars = annmodel.SomeList(ListDef(None, annmodel.SomeChar(),
+ mutated=True, resized=True))
def add_dumper(s_obj, dumper):
dumpers.append((s_obj, dumper))
+ dumper.s_obj = s_obj
dumper._annenforceargs_ = [s_list_of_chars, s_obj]
def add_loader(s_obj, loader):
loaders.append((s_obj, loader))
def get_dumper_annotation(dumper):
- return dumper._annenforceargs_[1]
+ return dumper.s_obj
def find_dumper(s_obj):
# select a suitable dumper - the condition is that the dumper must
@@ -120,12 +127,29 @@
add_dumper(annmodel.s_None, dump_none)
def load_none(loader):
- if loader.readchr() != TYPE_NONE:
+ if readchr(loader) != TYPE_NONE:
raise ValueError("expected a None")
return None
#add_loader(annmodel.s_None, load_none) -- cannot install it as a regular
# loader, because it will also match any annotation that can be None
+def dump_bool(buf, x):
+ if x:
+ buf.append(TYPE_TRUE)
+ else:
+ buf.append(TYPE_FALSE)
+add_dumper(annmodel.s_Bool, dump_bool)
+
+def load_bool(loader):
+ t = readchr(loader)
+ if t == TYPE_TRUE:
+ return True
+ elif t == TYPE_FALSE:
+ return False
+ else:
+ raise ValueError("expected a bool")
+add_loader(annmodel.s_Bool, load_bool)
+
def dump_int(buf, x):
buf.append(TYPE_INT)
w_long(buf, x)
@@ -139,11 +163,27 @@
add_loader(annmodel.SomeInteger(nonneg=True), load_int_nonneg)
def load_int(loader):
- if loader.readchr() != TYPE_INT:
+ if readchr(loader) != TYPE_INT:
raise ValueError("expected an int")
- return loader.r_long()
+ return readlong(loader)
add_loader(annmodel.SomeInteger(), load_int)
+def dump_longlong(buf, x):
+ buf.append(TYPE_INT64)
+ w_long(buf, intmask(x))
+ w_long(buf, intmask(x>>32))
+add_dumper(annotation(r_longlong), dump_longlong)
+
+r_32bits_mask = r_longlong(0xFFFFFFFF)
+
+def load_longlong(loader):
+ if readchr(loader) != TYPE_INT64:
+ raise ValueError("expected a longlong")
+ x = r_longlong(readlong(loader)) & r_32bits_mask
+ x |= (r_longlong(readlong(loader)) << 32)
+ return x
+add_loader(annotation(r_longlong), load_longlong)
+
def dump_float(buf, x):
buf.append(TYPE_FLOAT)
s = formatd("%.17g", x)
@@ -151,13 +191,13 @@
buf += s
add_dumper(annmodel.SomeFloat(), dump_float)
-##def load_float(loader):
-## if loader.readchr() != TYPE_FLOAT:
-## raise ValueError("expected a float")
-## length = ord(loader.readchr())
-## s = loader.read(length)
-## return ...mess...
-##add_loader(annmodel.SomeFloat(), load_float)
+def load_float(loader):
+ if loader.readchr() != TYPE_FLOAT:
+ raise ValueError("expected a float")
+ length = ord(loader.readchr())
+ s = loader.read(length)
+ return xxx # ...mess...
+add_loader(annmodel.SomeFloat(), load_float)
def dump_string_or_none(buf, x):
if x is None:
@@ -169,23 +209,23 @@
add_dumper(annmodel.SomeString(can_be_None=True), dump_string_or_none)
def load_single_char(loader):
- if loader.readchr() != TYPE_STRING or loader.r_long() != 1:
+ if readchr(loader) != TYPE_STRING or readlong(loader) != 1:
raise ValueError("expected a character")
- return loader.readchr()
+ return readchr(loader)
add_loader(annmodel.SomeChar(), load_single_char)
def load_string(loader):
- if loader.readchr() != TYPE_STRING:
+ if readchr(loader) != TYPE_STRING:
raise ValueError("expected a string")
- length = loader.r_long()
- return loader.read(length)
+ length = readlong(loader)
+ return readstr(loader, length)
add_loader(annmodel.SomeString(can_be_None=False), load_string)
def load_string_or_none(loader):
- t = loader.readchr()
+ t = readchr(loader)
if t == TYPE_STRING:
- length = loader.r_long()
- return loader.read(length)
+ length = readlong(loader)
+ return readstr(loader, length)
elif t == TYPE_NONE:
return None
else:
@@ -202,29 +242,41 @@
self.buf = buf
self.pos = 0
- def read(self, count):
- pos = self.pos
- end = pos + count
- if end > len(self.buf):
- raise ValueError("not enough data")
- self.pos = end
- return self.buf[pos:end]
-
- def readchr(self):
- pos = self.pos
- if pos >= len(self.buf):
- raise ValueError("not enough data")
- self.pos = pos + 1
- return self.buf[pos]
-
- def r_long(self):
- a = ord(self.readchr())
- b = ord(self.readchr())
- c = ord(self.readchr())
- d = ord(self.readchr())
- if d >= 0x80:
- d -= 0x100
- return a | (b<<8) | (c<<16) | (d<<24)
+ def check_finished(self):
+ if self.pos != len(self.buf):
+ raise ValueError("not all data consumed")
+
+ def need_more_data(self):
+ raise ValueError("not enough data") # can be overridden
+
+# the rest are not method on the Loader class, because it causes troubles
+# in pypy.translator.rsandbox if new methods are discovered after some
+# sandboxed-enabled graphs are produced
+def readstr(loader, count):
+ if count < 0:
+ raise ValueError("negative count")
+ pos = loader.pos
+ end = pos + count
+ while end > len(loader.buf):
+ loader.need_more_data()
+ loader.pos = end
+ return loader.buf[pos:end]
+
+def readchr(loader):
+ pos = loader.pos
+ while pos >= len(loader.buf):
+ loader.need_more_data()
+ loader.pos = pos + 1
+ return loader.buf[pos]
+
+def readlong(loader):
+ a = ord(readchr(loader))
+ b = ord(readchr(loader))
+ c = ord(readchr(loader))
+ d = ord(readchr(loader))
+ if d >= 0x80:
+ d -= 0x100
+ return a | (b<<8) | (c<<16) | (d<<24)
# ____________________________________________________________
#
@@ -279,9 +331,9 @@
def install_unmarshaller((tag, s_list)):
def load_list_or_none(loader):
- t = loader.readchr()
+ t = readchr(loader)
if t == TYPE_LIST:
- length = loader.r_long()
+ length = readlong(loader)
result = []
for i in range(length):
result.append(itemloader(loader))
@@ -313,9 +365,9 @@
def install_unmarshaller((tag, s_tuple)):
def load_tuple(loader):
- if loader.readchr() != TYPE_TUPLE:
+ if readchr(loader) != TYPE_TUPLE:
raise ValueError("expected a tuple")
- if loader.r_long() != expected_length:
+ if readlong(loader) != expected_length:
raise ValueError("wrong tuple length")
result = ()
for i, itemloader in unroll_item_loaders:
@@ -326,3 +378,25 @@
expected_length = len(itemloaders)
unroll_item_loaders = unrolling_iterable(enumerate(itemloaders))
add_loader(s_tuple, load_tuple)
+
+
+class __extend__(pairtype(MTag, controllerentry.SomeControlledInstance)):
+ # marshal a ControlledInstance by marshalling the underlying object
+
+ def install_marshaller((tag, s_obj)):
+ def dump_controlled_instance(buf, x):
+ real_obj = controllerentry.controlled_instance_unbox(controller, x)
+ realdumper(buf, real_obj)
+
+ controller = s_obj.controller
+ realdumper = get_marshaller(s_obj.s_real_obj)
+ add_dumper(s_obj, dump_controlled_instance)
+
+ def install_unmarshaller((tag, s_obj)):
+ def load_controlled_instance(loader):
+ real_obj = realloader(loader)
+ return controllerentry.controlled_instance_box(controller,
+ real_obj)
+ controller = s_obj.controller
+ realloader = get_loader(s_obj.s_real_obj)
+ add_loader(s_obj, load_controlled_instance)
Modified: pypy/branch/pypy-more-rtti-inprogress/rlib/test/test_rmarshal.py
==============================================================================
--- pypy/branch/pypy-more-rtti-inprogress/rlib/test/test_rmarshal.py (original)
+++ pypy/branch/pypy-more-rtti-inprogress/rlib/test/test_rmarshal.py Fri Aug 17 14:58:59 2007
@@ -24,6 +24,18 @@
assert marshal.loads(''.join(buf)) == "hello, world"
buf = []
+ get_marshaller(bool)(buf, False)
+ assert marshal.loads(''.join(buf)) is False
+
+ buf = []
+ get_marshaller(bool)(buf, True)
+ assert marshal.loads(''.join(buf)) is True
+
+ buf = []
+ get_marshaller(r_longlong)(buf, r_longlong(0x12380000007))
+ assert marshal.loads(''.join(buf)) == 0x12380000007
+
+ buf = []
get_marshaller([int])(buf, [2, 5, -7])
assert marshal.loads(''.join(buf)) == [2, 5, -7]
@@ -53,6 +65,15 @@
buf = 'i\x05\x00\x00\x00'
py.test.raises(ValueError, get_unmarshaller(str), buf)
+ buf = 'F'
+ assert get_unmarshaller(bool)(buf) is False
+
+ buf = 'T'
+ assert get_unmarshaller(bool)(buf) is True
+
+ buf = 'I\x07\x00\x00\x80\x23\x01\x00\x00'
+ assert get_unmarshaller(r_longlong)(buf) == 0x12380000007
+
buf = ('[\x03\x00\x00\x00i\x02\x00\x00\x00i\x05\x00\x00\x00'
'i\xf9\xff\xff\xff')
assert get_unmarshaller([int])(buf) == [2, 5, -7]
@@ -65,3 +86,70 @@
for typ in types_that_can_be_none:
buf = 'N'
assert get_unmarshaller(typ)(buf) is None
+
+
+def test_llinterp_marshal():
+ from pypy.rpython.test.test_llinterp import interpret
+ marshaller = get_marshaller([(int, str)])
+ def f():
+ buf = []
+ marshaller(buf, [(5, "hello"), (7, "world")])
+ return ''.join(buf)
+ res = interpret(f, [])
+ res = ''.join(res.chars)
+ assert res == ('[\x02\x00\x00\x00(\x02\x00\x00\x00i\x05\x00\x00\x00'
+ 's\x05\x00\x00\x00hello(\x02\x00\x00\x00i\x07\x00\x00\x00'
+ 's\x05\x00\x00\x00world')
+
+def test_llinterp_unmarshal():
+ from pypy.rpython.test.test_llinterp import interpret
+ unmarshaller = get_unmarshaller([(int, str)])
+ buf = ('[\x02\x00\x00\x00(\x02\x00\x00\x00i\x05\x00\x00\x00'
+ 's\x05\x00\x00\x00hello(\x02\x00\x00\x00i\x07\x00\x00\x00'
+ 's\x05\x00\x00\x00world')
+ def f():
+ result = ''
+ for num, string in unmarshaller(buf):
+ result += '%d=%s;' % (num, string)
+ return result
+ res = interpret(f, [])
+ res = ''.join(res.chars)
+ assert res == '5=hello;7=world;'
+
+def test_tuple_stat_result():
+ from pypy.rpython.module.ll_os_stat import STAT_FIELDS, s_tuple_StatResult
+ marshaller = get_marshaller(s_tuple_StatResult)
+ unmarshaller = get_unmarshaller(s_tuple_StatResult)
+
+ sttuple = (0,) * len(STAT_FIELDS)
+ buf = []
+ marshaller(buf, sttuple)
+ res = unmarshaller(''.join(buf))
+ assert res == sttuple
+
+def test_stat_result():
+ import os
+ from pypy.translator.c.test.test_genc import compile
+ from pypy.rpython.module.ll_os_stat import STAT_FIELDS, s_StatResult
+ marshal_stat_result = get_marshaller(s_StatResult)
+ unmarshal_stat_result = get_unmarshaller(s_StatResult)
+ def f(path):
+ st = os.stat(path)
+ buf = []
+ marshal_stat_result(buf, st)
+ buf = ''.join(buf)
+ st2 = unmarshal_stat_result(buf)
+ assert st2.st_mode == st.st_mode
+ assert st2[9] == st[9]
+ return buf
+ fn = compile(f, [str])
+ res = fn('.')
+ st = os.stat('.')
+ sttuple = marshal.loads(res)
+ assert sttuple[0] == st[0]
+ assert sttuple[1] == st[1]
+ assert sttuple[2] == st[2]
+ assert sttuple[3] == st[3]
+ assert sttuple[4] == st[4]
+ assert sttuple[5] == st[5]
+ assert len(sttuple) == len(STAT_FIELDS)
More information about the Pypy-commit
mailing list