[pypy-svn] r15175 - in pypy/dist/pypy: lib lib/test2 module/__builtin__/test objspace/std

tismer at codespeak.net tismer at codespeak.net
Wed Jul 27 14:02:05 CEST 2005


Author: tismer
Date: Wed Jul 27 14:02:01 2005
New Revision: 15175

Added:
   pypy/dist/pypy/lib/_marshal.py
      - copied, changed from r15173, pypy/dist/pypy/lib/marshal.py
   pypy/dist/pypy/lib/_osfilewrapper.py
      - copied unchanged from r15174, pypy/dist/pypy/lib/osfilewrapper.py
Removed:
   pypy/dist/pypy/lib/marshal.py
   pypy/dist/pypy/lib/osfilewrapper.py
Modified:
   pypy/dist/pypy/lib/test2/test_marshal_extra.py
   pypy/dist/pypy/module/__builtin__/test/test_import.py
   pypy/dist/pypy/objspace/std/objspace.py
Log:
made marshal an early builtin module.

- changed marshal to _marshal
- changed osfilewrapper to _osfilewrapper
- adjusted tests
- refactored std/objspace to use some helper functions,
  because there are now several applications of this

I will doo a little more folding of stuff.


Copied: pypy/dist/pypy/lib/_marshal.py (from r15173, pypy/dist/pypy/lib/marshal.py)
==============================================================================
--- pypy/dist/pypy/lib/marshal.py	(original)
+++ pypy/dist/pypy/lib/_marshal.py	Wed Jul 27 14:02:01 2005
@@ -4,7 +4,7 @@
 """
 
 import types
-from osfilewrapper import OsFileWrapper
+from _osfilewrapper import OsFileWrapper
 
 try:
     import new

Deleted: /pypy/dist/pypy/lib/marshal.py
==============================================================================
--- /pypy/dist/pypy/lib/marshal.py	Wed Jul 27 14:02:01 2005
+++ (empty file)
@@ -1,475 +0,0 @@
-"""Internal Python object serialization
-
-This module contains functions that can read and write Python values in a binary format. The format is specific to Python, but independent of machine architecture issues (e.g., you can write a Python value to a file on a PC, transport the file to a Sun, and read it back there). Details of the format may change between Python versions.
-"""
-
-import types
-from osfilewrapper import OsFileWrapper
-
-try:
-    import new
-except ImportError:
-    new = None
-
-TYPE_NULL     = '0'
-TYPE_NONE     = 'N'
-TYPE_FALSE    = 'F'
-TYPE_TRUE     = 'T'
-TYPE_STOPITER = 'S'
-TYPE_ELLIPSIS = '.'
-TYPE_INT      = 'i'
-TYPE_INT64    = 'I'
-TYPE_FLOAT    = 'f'
-TYPE_COMPLEX  = 'x'
-TYPE_LONG     = 'l'
-TYPE_STRING   = 's'
-TYPE_INTERNED = 't'
-TYPE_STRINGREF= 'R'
-TYPE_TUPLE    = '('
-TYPE_LIST     = '['
-TYPE_DICT     = '{'
-TYPE_CODE     = 'c'
-TYPE_UNICODE  = 'u'
-TYPE_UNKNOWN  = '?'
-TYPE_SET      = '<'
-TYPE_FROZENSET= '>'
-
-class Marshaller:
-
-    dispatch = {}
-
-    def __init__(self, fd):
-        if fd >= 0:
-            self.fd_wrapper = OsFileWrapper(fd)
-        else:
-            self.fd_wrapper = None
-        self.buffer = []
-
-    def _write(self, data):
-        if self.fd_wrapper is not None:
-            self.fd_wrapper.write(data)
-        else:
-            self.buffer.append(data)
-
-    def getvalue(self):
-        return ''.join(self.buffer)
-
-    def dump(self, x):
-        try:
-            self.dispatch[type(x)](self, x)
-        except KeyError:
-            for tp in type(x).mro():
-                func = self.dispatch.get(tp)
-                if func:
-                    break
-            else:
-                raise ValueError, "unmarshallable object"
-            func(self, x)
-
-    def w_long64(self, x):
-        self.w_long(x)
-        self.w_long(x>>32)
-
-    def w_long(self, x):
-        self._write(chr((x)     & 0xff))
-        self._write(chr((x>> 8) & 0xff))
-        self._write(chr((x>>16) & 0xff))
-        self._write(chr((x>>24) & 0xff))
-
-    def w_short(self, x):
-        self._write(chr((x)     & 0xff))
-        self._write(chr((x>> 8) & 0xff))
-
-    def dump_none(self, x):
-        self._write(TYPE_NONE)
-    dispatch[types.NoneType] = dump_none
-
-    def dump_bool(self, x):
-        if x:
-            self._write(TYPE_TRUE)
-        else:
-            self._write(TYPE_FALSE)
-    dispatch[bool] = dump_bool
-
-    def dump_stopiter(self, x):
-        if x is not StopIteration:
-            raise ValueError, "unmarshallable object"
-        self._write(TYPE_STOPITER)
-    dispatch[type(StopIteration)] = dump_stopiter
-
-    def dump_ellipsis(self, x):
-        self._write(TYPE_ELLIPSIS)
-    
-    try:
-        dispatch[types.EllipsisType] = dump_ellipsis
-    except NameError:
-        pass
-
-    def dump_int(self, x):
-        y = x>>31
-        if y and y != -1:
-            self._write(TYPE_INT64)
-            self.w_long64(x)
-        else:
-            self._write(TYPE_INT)
-            self.w_long(x)
-    dispatch[types.IntType] = dump_int
-
-    def dump_long(self, x):
-        self._write(TYPE_LONG)
-        sign = 1
-        if x < 0:
-            sign = -1
-            x = -x
-        digits = []
-        while x:
-            digits.append(x & 0x7FFF)
-            x = x>>15
-        self.w_long(len(digits) * sign)
-        for d in digits:
-            self.w_short(d)
-    dispatch[types.LongType] = dump_long
-
-    def dump_float(self, x):
-        write = self._write
-        write(TYPE_FLOAT)
-        s = `x`
-        write(chr(len(s)))
-        write(s)
-    dispatch[types.FloatType] = dump_float
-
-    def dump_complex(self, x):
-        write = self._write
-        write(TYPE_COMPLEX)
-        s = `x.real`
-        write(chr(len(s)))
-        write(s)
-        s = `x.imag`
-        write(chr(len(s)))
-        write(s)
-    try:
-        dispatch[types.ComplexType] = dump_complex
-    except NameError:
-        pass
-
-    def dump_string(self, x):
-        # XXX we can't check for interned strings, yet,
-        # so we (for now) never create TYPE_INTERNED or TYPE_STRINGREF
-        self._write(TYPE_STRING)
-        self.w_long(len(x))
-        self._write(x)
-    dispatch[types.StringType] = dump_string
-
-    def dump_unicode(self, x):
-        self._write(TYPE_UNICODE)
-        s = x.encode('utf8')
-        self.w_long(len(s))
-        self._write(s)
-    dispatch[types.UnicodeType] = dump_unicode
-
-    def dump_tuple(self, x):
-        self._write(TYPE_TUPLE)
-        self.w_long(len(x))
-        for item in x:
-            self.dump(item)
-    dispatch[types.TupleType] = dump_tuple
-
-    def dump_list(self, x):
-        self._write(TYPE_LIST)
-        self.w_long(len(x))
-        for item in x:
-            self.dump(item)
-    dispatch[types.ListType] = dump_list
-
-    def dump_dict(self, x):
-        self._write(TYPE_DICT)
-        for key, value in x.items():
-            self.dump(key)
-            self.dump(value)
-        self._write(TYPE_NULL)
-    dispatch[types.DictionaryType] = dump_dict
-
-    def dump_code(self, x):
-        self._write(TYPE_CODE)
-        self.w_long(x.co_argcount)
-        self.w_long(x.co_nlocals)
-        self.w_long(x.co_stacksize)
-        self.w_long(x.co_flags)
-        self.dump(x.co_code)
-        self.dump(x.co_consts)
-        self.dump(x.co_names)
-        self.dump(x.co_varnames)
-        self.dump(x.co_freevars)
-        self.dump(x.co_cellvars)
-        self.dump(x.co_filename)
-        self.dump(x.co_name)
-        self.w_long(x.co_firstlineno)
-        self.dump(x.co_lnotab)
-    try:
-        dispatch[types.CodeType] = dump_code
-    except NameError:
-        pass
-
-    def dump_set(self, x):
-        self._write(TYPE_SET)
-        self.w_long(len(x))
-        for each in x:
-            self.dump(each)
-    try:
-        dispatch[set] = dump_set
-    except NameError:
-        pass
-
-    def dump_frozenset(self, x):
-        self._write(TYPE_FROZENSET)
-        self.w_long(len(x))
-        for each in x:
-            self.dump(each)
-    try:
-        dispatch[frozenset] = dump_frozenset
-    except NameError:
-        pass
-
-class NULL:
-    pass
-
-class Unmarshaller:
-
-    dispatch = {}
-
-    def __init__(self, fd):
-        if fd >= 0:
-            self.fd_wrapper = OsFileWrapper(fd)
-        else:
-            self.fd_wrapper = None
-        self.bufstr = ''
-        self.bufpos = 0
-        self._stringtable = []
-
-    def _read(self):
-        if self.fd_wrapper is not None:
-            return self.fd_wrapper.read(1)
-        else:
-            ret = self.bufstr[self.bufpos]
-            self.bufpos += 1
-            return ret
-
-    def _readn(self, n):
-        if self.fd_wrapper is not None:
-            return self.fd_wrapper.read(n)
-        else:
-            ret = self.bufstr[self.bufpos : self.bufpos+n]
-            self.bufpos += n
-            if self.bufpos > len(self.bufstr):
-                raise EOFError, "read past buffer"
-            return ret
-
-    def setvalue(self, data):
-        self.bufstr = data
-
-    def load(self):
-        c = self._read()
-        if not c:
-            raise EOFError
-        try:
-            return self.dispatch[c](self)
-        except KeyError:
-            raise ValueError, "bad marshal code: %c (%d)" % (c, ord(c))
-
-    def r_short(self):
-        lo = ord(self._read())
-        hi = ord(self._read())
-        x = lo | (hi<<8)
-        if x & 0x8000:
-            x = x - 0x10000
-        return x
-
-    def r_long(self):
-        a = ord(self._read())
-        b = ord(self._read())
-        c = ord(self._read())
-        d = ord(self._read())
-        x = a | (b<<8) | (c<<16) | (d<<24)
-        if d & 0x80 and x > 0:
-            x = -((1L<<32) - x)
-        return int(x)
-
-    def r_long64(self):
-        a = ord(self._read())
-        b = ord(self._read())
-        c = ord(self._read())
-        d = ord(self._read())
-        e = long(ord(self._read()))
-        f = long(ord(self._read()))
-        g = long(ord(self._read()))
-        h = long(ord(self._read()))
-        x = a | (b<<8) | (c<<16) | (d<<24)
-        x = x | (e<<32) | (f<<40) | (g<<48) | (h<<56)
-        if h & 0x80 and x > 0:
-            x = -((1L<<64) - x)
-        return x
-
-    def load_null(self):
-        return NULL
-    dispatch[TYPE_NULL] = load_null
-
-    def load_none(self):
-        return None
-    dispatch[TYPE_NONE] = load_none
-
-    def load_true(self):
-        return True
-    dispatch[TYPE_TRUE] = load_true
-
-    def load_false(self):
-        return False
-    dispatch[TYPE_FALSE] = load_false
-
-    def load_stopiter(self):
-        return StopIteration
-    dispatch[TYPE_STOPITER] = load_stopiter
-
-    def load_ellipsis(self):
-        return Ellipsis
-    dispatch[TYPE_ELLIPSIS] = load_ellipsis
-
-    def load_int(self):
-        return self.r_long()
-    dispatch[TYPE_INT] = load_int
-
-    def load_int64(self):
-        return self.r_long64()
-    dispatch[TYPE_INT64] = load_int64
-
-    def load_long(self):
-        size = self.r_long()
-        sign = 1
-        if size < 0:
-            sign = -1
-            size = -size
-        x = 0L
-        for i in range(size):
-            d = self.r_short()
-            x = x | (d<<(i*15L))
-        return x * sign
-    dispatch[TYPE_LONG] = load_long
-
-    def load_float(self):
-        n = ord(self._read())
-        s = self._readn(n)
-        return float(s)
-    dispatch[TYPE_FLOAT] = load_float
-
-    def load_complex(self):
-        n = ord(self._read())
-        s = self._readn(n)
-        real = float(s)
-        n = ord(self._read())
-        s = self._readn(n)
-        imag = float(s)
-        return complex(real, imag)
-    dispatch[TYPE_COMPLEX] = load_complex
-
-    def load_string(self):
-        n = self.r_long()
-        return self._readn(n)
-    dispatch[TYPE_STRING] = load_string
-
-    def load_interned(self):
-        n = self.r_long()
-        ret = intern(self._readn(n))
-        self._stringtable.append(ret)
-        return ret
-    dispatch[TYPE_INTERNED] = load_interned
-
-    def load_stringref(self):
-        n = self.r_long()
-        return self._stringtable[n]
-    dispatch[TYPE_STRINGREF] = load_stringref
-
-    def load_unicode(self):
-        n = self.r_long()
-        s = self._readn(n)
-        ret = s.decode('utf8')
-        return ret
-    dispatch[TYPE_UNICODE] = load_unicode
-
-    def load_tuple(self):
-        return tuple(self.load_list())
-    dispatch[TYPE_TUPLE] = load_tuple
-
-    def load_list(self):
-        n = self.r_long()
-        list = []
-        for i in range(n):
-            list.append(self.load())
-        return list
-    dispatch[TYPE_LIST] = load_list
-
-    def load_dict(self):
-        d = {}
-        while 1:
-            key = self.load()
-            if key is NULL:
-                break
-            value = self.load()
-            d[key] = value
-        return d
-    dispatch[TYPE_DICT] = load_dict
-
-    def load_code(self):
-        argcount = self.r_long()
-        nlocals = self.r_long()
-        stacksize = self.r_long()
-        flags = self.r_long()
-        code = self.load()
-        consts = self.load()
-        names = self.load()
-        varnames = self.load()
-        freevars = self.load()
-        cellvars = self.load()
-        filename = self.load()
-        name = self.load()
-        firstlineno = self.r_long()
-        lnotab = self.load()
-        if not new:
-            raise RuntimeError, "can't unmarshal code objects; no 'new' module"
-        return new.code(argcount, nlocals, stacksize, flags, code, consts,
-                        names, varnames, filename, name, firstlineno, lnotab,
-                        freevars, cellvars)
-    dispatch[TYPE_CODE] = load_code
-
-    def load_set(self):
-        n = self.r_long()
-        args = [self.load() for i in range(n)]
-        return set(args)
-    dispatch[TYPE_SET] = load_set
-
-    def load_frozenset(self):
-        n = self.r_long()
-        args = [self.load() for i in range(n)]
-        return frozenset(args)
-    dispatch[TYPE_FROZENSET] = load_frozenset
-
-try:
-    set
-except NameError:
-    def set(x):
-        raise ValueError("cannot unmarshal set objects on Python < 2.4")
-    frozenset = set
-
-def dump(x, f):
-    Marshaller(f.fileno()).dump(x)
-
-def load(f):
-    return Unmarshaller(f.fileno()).load()
-
-def dumps(x):
-    m = Marshaller(-1)
-    m.dump(x)
-    return m.getvalue()
-
-def loads(s):
-    um = Unmarshaller(-1)
-    um.setvalue(s)
-    return um.load()

Deleted: /pypy/dist/pypy/lib/osfilewrapper.py
==============================================================================
--- /pypy/dist/pypy/lib/osfilewrapper.py	Wed Jul 27 14:02:01 2005
+++ (empty file)
@@ -1,36 +0,0 @@
-import os
-
-class OsFileWrapper(object):
-    """ Very simple os file wrapper.
-    Note user is responsible for closing.
-    XXX Could add a simple buffer mechanism. """
-    
-    def __init__(self, fd):
-        self.fd = fd
-        
-    def read(self, expected):
-        readcount = 0
-        bufs = []
-        while readcount < expected:
-            # os.read will raise an error itself
-            buf = os.read(self.fd, expected - readcount)
-            if not buf:
-                break
-            readcount += len(buf)
-            bufs.append(buf)
-        return "".join(bufs)
-
-    def write(self, buf):
-        writecount = 0
-        towrite = len(buf)
-        while writecount < towrite:
-            # os.write will raise an error itself
-            writecount += os.write(self.fd, buf[writecount:])
-
-    def close(self):
-        os.close(self.fd)
-
-    
-def create_wrapper(filename, flag, mode=0777):
-    fd = os.open(filename, flag, mode)
-    return OsFileWrapper(fd)

Modified: pypy/dist/pypy/lib/test2/test_marshal_extra.py
==============================================================================
--- pypy/dist/pypy/lib/test2/test_marshal_extra.py	(original)
+++ pypy/dist/pypy/lib/test2/test_marshal_extra.py	Wed Jul 27 14:02:01 2005
@@ -2,7 +2,7 @@
 import py
 import sys
 import marshal as cpy_marshal
-from pypy.lib import marshal
+from pypy.lib import _marshal as marshal
 
 from pypy.tool.udir import udir 
 

Modified: pypy/dist/pypy/module/__builtin__/test/test_import.py
==============================================================================
--- pypy/dist/pypy/module/__builtin__/test/test_import.py	(original)
+++ pypy/dist/pypy/module/__builtin__/test/test_import.py	Wed Jul 27 14:02:01 2005
@@ -5,7 +5,7 @@
 from pypy.tool.udir import udir 
 import sys, os
 import tempfile, marshal
-from pypy.tool.osfilewrapper import OsFileWrapper
+from pypy.lib._osfilewrapper import OsFileWrapper
 
 from pypy.module.__builtin__.importing import BIN_READMASK
 

Modified: pypy/dist/pypy/objspace/std/objspace.py
==============================================================================
--- pypy/dist/pypy/objspace/std/objspace.py	(original)
+++ pypy/dist/pypy/objspace/std/objspace.py	Wed Jul 27 14:02:01 2005
@@ -80,6 +80,10 @@
         self.w_instance = W_TypeObject(self, 'instance', [self.w_object], {})
         self.setup_old_style_classes()
 
+        #early bootstrap for marshal
+        mod = self.setup_marshal()
+        self.sys.setmodule(self.wrap(mod))
+
         # fix up a problem where multimethods apparently don't 
         # like to define this at interp-level 
         self.appexec([self.w_dict], """
@@ -111,21 +115,34 @@
         # sanity check that this approach is working and is not too late
         assert not self.is_true(self.contains(self.builtin.w_dict,self.wrap('_classobj'))),"app-level code has seen dummy old style classes"
         assert not self.is_true(self.contains(self.builtin.w_dict,self.wrap('_instance'))),"app-level code has seen dummy old style classes"
+        mod, w_dic = self.create_builtin_module('_classobj.py', 'classobj')
+        w_purify = self.getitem(w_dic, self.wrap('purify'))
+        w_classobj = self.getitem(w_dic, self.wrap('classobj'))
+        w_instance = self.getitem(w_dic, self.wrap('instance'))
+        self.call_function(w_purify)
+        self.w_classobj = w_classobj
+        self.w_instance = w_instance
+
+    def setup_marshal(self):
+        mod, w_dic = self.create_builtin_module('_marshal.py', 'marshal')
+        return mod
+
+    def create_builtin_module(self, pyname, publicname):
+        """NOT_RPYTHON
+        helper function which returns the new unwrapped module and its waapped dict.
+        """
         # generate on-the-fly
         class Fake: pass
         fake = Fake()
         import pypy.lib as lib
-        fname = os.path.join(os.path.split(lib.__file__)[0], '_classobj.py')
+        fname = os.path.join(os.path.split(lib.__file__)[0], pyname)
         fake.filename = fname
         fake.source = file(fname).read()
-        fake.modname = 'classobj'
+        fake.modname = publicname
         w_dic = PyPyCacheDir.build_applevelinterp_dict(fake, self)
-        w_purify = self.getitem(w_dic, self.wrap('purify'))
-        w_classobj = self.getitem(w_dic, self.wrap('classobj'))
-        w_instance = self.getitem(w_dic, self.wrap('instance'))
-        self.call_function(w_purify)
-        self.w_classobj = w_classobj
-        self.w_instance = w_instance
+        from pypy.interpreter.module import Module
+        mod = Module(self, self.wrap(publicname), w_dic)
+        return mod, w_dic
 
     def setuselibfile(self): 
         """ NOT_RPYTHON use our application level file implementation
@@ -172,18 +189,10 @@
                 bases = [space.w_object]
             res = W_TypeObject(space, name, bases, dic)
             return res
-        # generate on-the-fly
-        class Fake: pass
-        fake = Fake()
-        import pypy.lib as lib
-        fname = os.path.join(os.path.split(lib.__file__)[0], '_exceptions.py')
-        fake.filename = fname
-        fake.source = file(fname).read()
-        fake.modname = 'exceptions'
         try:
             # note that we hide the real call method by an instance variable!
             self.call = call
-            w_dic = PyPyCacheDir.build_applevelinterp_dict(fake, self)
+            mod, w_dic = self.create_builtin_module('_exceptions.py', 'exceptions')
 
             self.w_IndexError = self.getitem(w_dic, self.wrap("IndexError"))
             self.w_StopIteration = self.getitem(w_dic, self.wrap("StopIteration"))
@@ -198,11 +207,8 @@
                 excname = name
                 w_exc = self.getitem(w_dic, w_name)
                 setattr(self, "w_"+excname, w_exc)
-                        
-        # XXX refine things, clean up, create a builtin module...
-        # but for now, we do a regular one.
-        from pypy.interpreter.module import Module
-        return Module(self, self.wrap("exceptions"), w_dic)
+
+        return mod
 
     def createexecutioncontext(self):
         # add space specific fields to execution context



More information about the Pypy-commit mailing list