[pypy-svn] r47181 - in pypy/dist/pypy/module/bz2: . test

fijal at codespeak.net fijal at codespeak.net
Fri Oct 5 11:15:33 CEST 2007


Author: fijal
Date: Fri Oct  5 11:15:32 2007
New Revision: 47181

Added:
   pypy/dist/pypy/module/bz2/test/__init__.py   (contents, props changed)
   pypy/dist/pypy/module/bz2/test/support.py   (contents, props changed)
Removed:
   pypy/dist/pypy/module/bz2/bz2module.c
   pypy/dist/pypy/module/bz2/bzlib.py
Modified:
   pypy/dist/pypy/module/bz2/interp_bz2.py
   pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py
   pypy/dist/pypy/module/bz2/test/test_bz2_file.py
Log:
* move bz2 module from rctypes to rffi
* check allocation at the end of every function, unfortunately
  this makes py.test choke when C-c


Modified: pypy/dist/pypy/module/bz2/interp_bz2.py
==============================================================================
--- pypy/dist/pypy/module/bz2/interp_bz2.py	(original)
+++ pypy/dist/pypy/module/bz2/interp_bz2.py	Fri Oct  5 11:15:32 2007
@@ -1,34 +1,39 @@
-from pypy.rpython.rctypes.tool import ctypes_platform
-import pypy.rpython.rctypes.implementation # this defines rctypes magic
+from pypy.rpython.tool import rffi_platform as platform
+from pypy.rpython.lltypesystem import rffi
+from pypy.rpython.lltypesystem import lltype
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.typedef import interp_attrproperty
 from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, interp2app
 from pypy.rlib.streamio import Stream
-from ctypes import *
-import ctypes.util
 import sys
 
-from bzlib import bz_stream, BZFILE, FILE
-
-#libbz2 = cdll.LoadLibrary(ctypes.util.find_library("bz2"))
-
-c_void = None
-
 class CConfig:
-    _header_ = """
-    #include <stdio.h>
-    #include <sys/types.h>
-    #include <bzlib.h>
-    """
-
-    _includes_ = ["bzlib.h"]
-    libbz2 = ctypes_platform.Library('bz2')
-    off_t = ctypes_platform.SimpleType("off_t", c_longlong)
-    size_t = ctypes_platform.SimpleType("size_t", c_ulong)
-    BUFSIZ = ctypes_platform.ConstantInteger("BUFSIZ")
-    SEEK_SET = ctypes_platform.ConstantInteger("SEEK_SET")
+    _includes_ = ['stdio.h', 'sys/types.h', 'bzlib.h']
+    _libraries_ = ['bz2']
+    calling_conv = 'c'
+
+    off_t = platform.SimpleType("off_t", rffi.LONGLONG)
+    size_t = platform.SimpleType("size_t", rffi.ULONG)
+    BUFSIZ = platform.ConstantInteger("BUFSIZ")
+    _alloc_type = lltype.FuncType([rffi.VOIDP, rffi.INT, rffi.INT], rffi.VOIDP)
+    _free_type = lltype.FuncType([rffi.VOIDP, rffi.VOIDP], lltype.Void)
+    SEEK_SET = platform.ConstantInteger("SEEK_SET")
+    bz_stream = platform.Struct('bz_stream',
+                                [('next_in', rffi.CCHARP),
+                                 ('avail_in', rffi.UINT),
+                                 ('total_in_lo32', rffi.UINT),
+                                 ('total_in_hi32', rffi.UINT),
+                                 ('next_out', rffi.CCHARP),
+                                 ('avail_out', rffi.UINT),
+                                 ('total_out_lo32', rffi.UINT),
+                                 ('total_out_hi32', rffi.UINT),
+                                 ('state', rffi.VOIDP),
+                                 ('bzalloc', lltype.Ptr(_alloc_type)),
+                                 ('bzfree', lltype.Ptr(_free_type)),
+                                 ('opaque', rffi.VOIDP),
+                                 ])
 
 constants = {}
 constant_names = ['BZ_RUN', 'BZ_FLUSH', 'BZ_FINISH', 'BZ_OK',
@@ -37,11 +42,11 @@
     'BZ_DATA_ERROR_MAGIC', 'BZ_IO_ERROR', 'BZ_UNEXPECTED_EOF',
     'BZ_OUTBUFF_FULL', 'BZ_CONFIG_ERROR']
 for name in constant_names:
-    setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name))
+    setattr(CConfig, name, platform.DefinedConstantInteger(name))
     
 class cConfig:
     pass
-cConfig.__dict__.update(ctypes_platform.configure(CConfig))
+cConfig.__dict__.update(platform.configure(CConfig))
 
 for name in constant_names:
     value = getattr(cConfig, name)
@@ -49,8 +54,8 @@
         constants[name] = value
 locals().update(constants)
 
-libbz2 = cConfig.libbz2
 off_t = cConfig.off_t
+bz_stream = lltype.Ptr(cConfig.bz_stream)
 BUFSIZ = cConfig.BUFSIZ
 SEEK_SET = cConfig.SEEK_SET
 BZ_OK = cConfig.BZ_OK
@@ -69,56 +74,53 @@
 else:
     SMALLCHUNK = BUFSIZ
     
-if sizeof(c_int) > 4:
+if rffi.sizeof(rffi.INT) > 4:
     BIGCHUNK = 512 * 32
 else:
     BIGCHUNK = 512 * 1024
-    
+
 MAXINT = sys.maxint
 
 if BZ_CONFIG_ERROR:
-    if sizeof(c_long) >= 8 or sizeof(c_longlong) >= 8:
+    if rffi.sizeof(rffi.LONG) >= 8 or rffi.sizeof(rffi.LONGLONG) >= 8:
         def _bzs_total_out(bzs):
-            return (bzs.total_out_hi32 << 32) + bzs.total_out_lo32
+            return (bzs.c_total_out_hi32 << 32) + bzs.c_total_out_lo32
     else:
         def _bzs_total_out(bzs):
-            return bzs.total_out_lo32
+            return bzs.c_total_out_lo32
 else:
     def _bzs_total_out(bzs):
         return bzs.total_out
 
-# the least but one parameter should be c_void_p but it's not used
+def external(name, args, result):
+    return rffi.llexternal(name, args, result, includes=CConfig._includes_,
+                           libraries=['bz2'])
+
+FILE = rffi.COpaquePtr('FILE')
+BZFILE = rffi.COpaquePtr('BZFILE')
+
+# the least but one parameter should be rffi.VOIDP but it's not used
 # so I trick the compiler to not complain about constanst pointer passed
 # to void* arg
-libbz2.BZ2_bzReadOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int,
-    c_int, POINTER(c_int), c_int]
-libbz2.BZ2_bzReadOpen.restype = POINTER(BZFILE)
-libbz2.BZ2_bzWriteOpen.argtypes = [POINTER(c_int), POINTER(FILE), c_int,
-    c_int, c_int]
-libbz2.BZ2_bzWriteOpen.restype = POINTER(BZFILE)
-libbz2.BZ2_bzReadClose.argtypes = [POINTER(c_int), POINTER(BZFILE)]
-libbz2.BZ2_bzReadClose.restype = c_void
-libbz2.BZ2_bzWriteClose.argtypes = [POINTER(c_int), POINTER(BZFILE),
-    c_int, POINTER(c_uint), POINTER(c_uint)]
-libbz2.BZ2_bzWriteClose.restype = c_void
-libbz2.BZ2_bzRead.argtypes = [POINTER(c_int), POINTER(BZFILE), POINTER(c_char), c_int]
-libbz2.BZ2_bzRead.restype = c_int
-libbz2.BZ2_bzWrite.argtypes = [POINTER(c_int), POINTER(BZFILE), c_char_p, c_int]
-libbz2.BZ2_bzWrite.restype = c_void
-
-libbz2.BZ2_bzCompressInit.argtypes = [POINTER(bz_stream), c_int, c_int, c_int]
-libbz2.BZ2_bzCompressInit.restype = c_int
-libbz2.BZ2_bzCompressEnd.argtypes = [POINTER(bz_stream)]
-libbz2.BZ2_bzCompressEnd.restype = c_int
-libbz2.BZ2_bzCompress.argtypes = [POINTER(bz_stream), c_int]
-libbz2.BZ2_bzCompress.restype = c_int
-
-libbz2.BZ2_bzDecompressInit.argtypes = [POINTER(bz_stream), c_int, c_int]
-libbz2.BZ2_bzDecompressInit.restype = c_int
-libbz2.BZ2_bzDecompressEnd.argtypes = [POINTER(bz_stream)]
-libbz2.BZ2_bzDecompressEnd.restype = c_int
-libbz2.BZ2_bzDecompress.argtypes = [POINTER(bz_stream)]
-libbz2.BZ2_bzDecompress.restype = c_int
+BZ2_bzReadOpen = external('BZ2_bzReadOpen', [rffi.INTP, FILE, rffi.INT,
+    rffi.INT, rffi.INTP, rffi.INT], BZFILE)
+BZ2_bzWriteOpen = external('BZ2_bzWriteOpen', [rffi.INTP, FILE, rffi.INT,
+    rffi.INT, rffi.INT], BZFILE)
+BZ2_bzReadClose = external('BZ2_bzReadClose', [rffi.INTP, BZFILE], lltype.Void)
+BZ2_bzWriteClose = external('BZ2_bzWriteClose', [rffi.INTP, BZFILE,
+    rffi.INT, rffi.UINTP, rffi.UINTP], lltype.Void)
+BZ2_bzRead = external('BZ2_bzRead', [rffi.INTP, BZFILE, rffi.CCHARP, rffi.INT],
+                      rffi.INT)
+BZ2_bzWrite = external('BZ2_bzWrite', [rffi.INTP, BZFILE, rffi.CCHARP,
+                                       rffi.INT], lltype.Void)
+BZ2_bzCompressInit = external('BZ2_bzCompressInit', [bz_stream, rffi.INT,
+                              rffi.INT, rffi.INT], rffi.INT)
+BZ2_bzCompressEnd = external('BZ2_bzCompressEnd', [bz_stream], rffi.INT)
+BZ2_bzCompress = external('BZ2_bzCompress', [bz_stream, rffi.INT], rffi.INT)
+BZ2_bzDecompressInit = external('BZ2_bzDecompressInit', [bz_stream, rffi.INT,
+                                                         rffi.INT], rffi.INT)
+BZ2_bzDecompressEnd = external('BZ2_bzDecompressEnd', [bz_stream], rffi.INT)
+BZ2_bzDecompress = external('BZ2_bzDecompress', [bz_stream], rffi.INT)
 
 def _catch_bz2_error(space, bzerror):
     if BZ_CONFIG_ERROR and bzerror == BZ_CONFIG_ERROR:
@@ -258,7 +260,6 @@
         raise OperationError(self.space.w_IOError,
                              self.space.wrap("file is not ready for writing"))
 
-
 class WriteBZ2Filter(Stream):
     """Standard I/O stream filter that compresses the stream with bz2."""
 
@@ -294,7 +295,6 @@
     def try_to_find_file_descriptor(self):
         return self.stream.try_to_find_file_descriptor()
 
-
 def descr_compressor__new__(space, w_subtype, compresslevel=9):
     x = space.allocate_instance(W_BZ2Compressor, w_subtype)
     x = space.interp_w(W_BZ2Compressor, x)
@@ -311,7 +311,7 @@
     must be a number between 1 and 9."""
     def __init__(self, space, compresslevel):
         self.space = space
-        self.bzs = bz_stream()
+        self.bzs = lltype.malloc(bz_stream.TO, flavor='raw', zero=True)
         self.running = False
         self._init_bz2comp(compresslevel)
     __init__.unwrap_spec = ['self', ObjSpace, int]
@@ -320,15 +320,16 @@
         if compresslevel < 1 or compresslevel > 9:
             raise OperationError(self.space.w_ValueError,
                 self.space.wrap("compresslevel must be between 1 and 9"))
-                
-        bzerror = libbz2.BZ2_bzCompressInit(byref(self.bzs), compresslevel, 0, 0)
+
+        bzerror = BZ2_bzCompressInit(self.bzs, compresslevel, 0, 0)
         if bzerror != BZ_OK:
             _catch_bz2_error(self.space, bzerror)
         
         self.running = True
         
     def __del__(self):
-        libbz2.BZ2_bzCompressEnd(byref(self.bzs))
+        BZ2_bzCompressEnd(self.bzs)
+        lltype.free(self.bzs, flavor='raw')
     
     def compress(self, data):
         """compress(data) -> string
@@ -348,44 +349,54 @@
                 self.space.wrap("this object was already flushed"))
         
         out_bufsize = SMALLCHUNK
-        out_buf = create_string_buffer(out_bufsize)
+        out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw',
+                                zero=True)
         
         in_bufsize = datasize
-        in_buf = create_string_buffer(in_bufsize)
-        in_buf.value = data
-        
-        self.bzs.next_in = cast(in_buf, POINTER(c_char))
-        self.bzs.avail_in = in_bufsize
-        self.bzs.next_out = cast(out_buf, POINTER(c_char))
-        self.bzs.avail_out = out_bufsize
-        
-        temp = []
-        while True:
-            bzerror = libbz2.BZ2_bzCompress(byref(self.bzs), BZ_RUN)
-            if bzerror != BZ_RUN_OK:
-                _catch_bz2_error(self.space, bzerror)
+        in_buf = lltype.malloc(rffi.CCHARP.TO, in_bufsize, flavor='raw')
+        for i in range(datasize):
+            in_buf[i] = data[i]
+
+        try:
+        
+            self.bzs.c_next_in = in_buf
+            self.bzs.c_avail_in = rffi.cast(rffi.UINT, in_bufsize)
+            self.bzs.c_next_out = out_buf
+            self.bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize)
+        
+            temp = []
+            while True:
+                bzerror = BZ2_bzCompress(self.bzs, BZ_RUN)
+                if bzerror != BZ_RUN_OK:
+                    _catch_bz2_error(self.space, bzerror)
 
-            if self.bzs.avail_in == 0:
-                break
-            elif self.bzs.avail_out == 0:
+                if self.bzs.c_avail_in == 0:
+                    break
+                elif self.bzs.c_avail_out == 0:
+                    total_out = _bzs_total_out(self.bzs)
+                    data = "".join([out_buf[i] for i in range(total_out)])
+                    temp.append(data)
+                
+                    out_bufsize = _new_buffer_size(out_bufsize)
+                    lltype.free(out_buf, flavor='raw')
+                    out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize,
+                                            flavor='raw', zero=True)
+                    self.bzs.c_next_out = out_buf
+                    self.bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize)
+
+            if temp:
                 total_out = _bzs_total_out(self.bzs)
-                data = "".join([out_buf[i] for i in range(total_out)])
+                data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))])
                 temp.append(data)
-                
-                out_bufsize = _new_buffer_size(out_bufsize)
-                out_buf = create_string_buffer(out_bufsize)
-                self.bzs.next_out = cast(out_buf, POINTER(c_char))
-                self.bzs.avail_out = out_bufsize
+                return self.space.wrap("".join(temp))
 
-        if temp:
             total_out = _bzs_total_out(self.bzs)
-            data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))])
-            temp.append(data)
-            return self.space.wrap("".join(temp))
+            res = "".join([out_buf[i] for i in range(total_out)])
+            return self.space.wrap(res)
+        finally:
+            lltype.free(in_buf, flavor='raw')
+            lltype.free(out_buf, flavor='raw')
 
-        total_out = _bzs_total_out(self.bzs)
-        res = "".join([out_buf[i] for i in range(total_out)])
-        return self.space.wrap(res)
     compress.unwrap_spec = ['self', str]
     
     def flush(self):
@@ -395,41 +406,47 @@
         self.running = False
         
         out_bufsize = SMALLCHUNK
-        out_buf = create_string_buffer(out_bufsize)
+        out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw',
+                                zero=True)
+
+        try:
     
-        self.bzs.next_out = cast(out_buf, POINTER(c_char))
-        self.bzs.avail_out = out_bufsize
-        
-        total_out = _bzs_total_out(self.bzs)
+            self.bzs.c_next_out = out_buf
+            self.bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize)
         
-        temp = []
-        while True:
-            bzerror = libbz2.BZ2_bzCompress(byref(self.bzs), BZ_FINISH)
-            if bzerror == BZ_STREAM_END:
-                break
-            elif bzerror != BZ_FINISH_OK:
-                _catch_bz2_error(self.space, bzerror)
-                
-            if self.bzs.avail_out == 0:
-                data = "".join([out_buf[i] for i in range(_bzs_total_out(self.bzs))])
-                temp.append(data)
+            total_out = _bzs_total_out(self.bzs)
+            
+            temp = []
+            while True:
+                bzerror = BZ2_bzCompress(self.bzs, BZ_FINISH)
+                if bzerror == BZ_STREAM_END:
+                    break
+                elif bzerror != BZ_FINISH_OK:
+                    _catch_bz2_error(self.space, bzerror)
                 
-                out_bufsize = _new_buffer_size(out_bufsize)
-                out_buf = create_string_buffer(out_bufsize)
-                self.bzs.next_out = cast(out_buf, POINTER(c_char))
-                self.bzs.avail_out = out_bufsize
+                if self.bzs.c_avail_out == 0:
+                    data = "".join([out_buf[i] for i in range(_bzs_total_out(self.bzs))])
+                    temp.append(data)
+                    
+                    out_bufsize = _new_buffer_size(out_bufsize)
+                    out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize,
+                                            flavor='raw', zero=True)
+                    self.bzs.c_next_out = out_buf
+                    self.bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize)
         
-        if temp:
-            return self.space.wrap("".join(temp))
+            if temp:
+                return self.space.wrap("".join(temp))
             
-        if self.bzs.avail_out:
-            size = _bzs_total_out(self.bzs) - total_out
-            res = "".join([out_buf[i] for i in range(size)])
-            return self.space.wrap(res)
+            if self.bzs.c_avail_out:
+                size = _bzs_total_out(self.bzs) - total_out
+                res = "".join([out_buf[i] for i in range(size)])
+                return self.space.wrap(res)
     
-        total_out = _bzs_total_out(self.bzs)
-        res = "".join([out_buf[i] for i in range(total_out)])
-        return self.space.wrap(res)
+            total_out = _bzs_total_out(self.bzs)
+            res = "".join([out_buf[i] for i in range(total_out)])
+            return self.space.wrap(res)
+        finally:
+            lltype.free(out_buf, flavor='raw')
     flush.unwrap_spec = ['self']
 
 W_BZ2Compressor.typedef = TypeDef("BZ2Compressor",
@@ -456,25 +473,26 @@
     
     def __init__(self, space):
         self.space = space
-        
-        self.bzs = bz_stream()
+
+        self.bzs = lltype.malloc(bz_stream.TO, flavor='raw', zero=True)
         self.running = False
         self.unused_data = ""
         
         self._init_bz2decomp()
     
     def _init_bz2decomp(self):
-        bzerror = libbz2.BZ2_bzDecompressInit(byref(self.bzs), 0, 0)
+        bzerror = BZ2_bzDecompressInit(self.bzs, 0, 0)
         if bzerror != BZ_OK:
             _catch_bz2_error(self.space, bzerror)
         
         self.running = True
     
     def __del__(self):
-        libbz2.BZ2_bzDecompressEnd(byref(self.bzs))
+        BZ2_bzDecompressEnd(self.bzs)
+        lltype.free(self.bzs, flavor='raw')
     
     def decompress(self, data):
-        """"decompress(data) -> string
+        """decompress(data) -> string
 
         Provide more data to the decompressor object. It will return chunks
         of decompressed data whenever possible. If you try to decompress data
@@ -487,50 +505,59 @@
                 self.space.wrap("end of stream was already found"))
         
         in_bufsize = len(data)
-        in_buf = create_string_buffer(in_bufsize)
-        in_buf.value = data
+        in_buf = lltype.malloc(rffi.CCHARP.TO, in_bufsize, flavor='raw')
+        for i in range(in_bufsize):
+            in_buf[i] = data[i]
 
         out_bufsize = SMALLCHUNK
-        out_buf = create_string_buffer(out_bufsize)
-        
-        self.bzs.next_in = cast(in_buf, POINTER(c_char))
-        self.bzs.avail_in = in_bufsize
-        self.bzs.next_out = cast(out_buf, POINTER(c_char))
-        self.bzs.avail_out = out_bufsize
+        out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw',
+                                zero=True)
+
+        try:
         
-        temp = []
-        while True:
-            bzerror = libbz2.BZ2_bzDecompress(byref(self.bzs))
-            if bzerror == BZ_STREAM_END:
-                if self.bzs.avail_in != 0:
-                    unused = [self.bzs.next_in[i] for i in range(self.bzs.avail_in)]
-                    self.unused_data = "".join(unused)
-                self.running = False
-                break
-            if bzerror != BZ_OK:
-                _catch_bz2_error(self.space, bzerror)
+            self.bzs.c_next_in = in_buf
+            self.bzs.c_avail_in = rffi.cast(rffi.UINT, in_bufsize)
+            self.bzs.c_next_out = out_buf
+            self.bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize)
+        
+            temp = []
+            while True:
+                bzerror = BZ2_bzDecompress(self.bzs)
+                if bzerror == BZ_STREAM_END:
+                    if self.bzs.c_avail_in != 0:
+                        unused = [self.bzs.c_next_in[i] for i in range(self.bzs.c_avail_in)]
+                        self.unused_data = "".join(unused)
+                    self.running = False
+                    break
+                if bzerror != BZ_OK:
+                    _catch_bz2_error(self.space, bzerror)
             
-            if self.bzs.avail_in == 0:
-                break
-            elif self.bzs.avail_out == 0:
+                if self.bzs.c_avail_in == 0:
+                    break
+                elif self.bzs.c_avail_out == 0:
+                    total_out = _bzs_total_out(self.bzs)
+                    data = "".join([out_buf[i] for i in range(total_out)])
+                    temp.append(data)
+
+                    lltype.free(out_buf, flavor='raw')
+                    out_bufsize = _new_buffer_size(out_bufsize)
+                    out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw')
+                    self.bzs.c_next_out = out_buf
+                    self.bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize)
+                
+            if temp:
                 total_out = _bzs_total_out(self.bzs)
-                data = "".join([out_buf[i] for i in range(total_out)])
+                data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))])
                 temp.append(data)
-                
-                out_bufsize = _new_buffer_size(out_bufsize)
-                out_buf = create_string_buffer(out_bufsize)
-                self.bzs.next_out = cast(out_buf, POINTER(c_char))
-                self.bzs.avail_out = out_bufsize
-                
-        if temp:
+                return self.space.wrap("".join(temp))
+
             total_out = _bzs_total_out(self.bzs)
-            data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))])
-            temp.append(data)
-            return self.space.wrap("".join(temp))
+            res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00'])
+            return self.space.wrap(res)
+        finally:
+            lltype.free(in_buf, flavor='raw')
+            lltype.free(out_buf, flavor='raw')
 
-        total_out = _bzs_total_out(self.bzs)
-        res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00'])
-        return self.space.wrap(res)
     decompress.unwrap_spec = ['self', str]
 
 
@@ -553,57 +580,65 @@
         raise OperationError(space.w_ValueError,
             space.wrap("compresslevel must be between 1 and 9"))
             
-    bzs = bz_stream()
-    
+    bzs = lltype.malloc(bz_stream.TO, flavor='raw', zero=True)
     in_bufsize = len(data)
     # conforming to bz2 manual, this is large enough to fit compressed
-        # data in one shot. We will check it later anyway.
+    # data in one shot. We will check it later anyway.
     out_bufsize = in_bufsize + (in_bufsize / 100 + 1) + 600
     
-    out_buf = create_string_buffer(out_bufsize)
-    in_buf = create_string_buffer(in_bufsize)
-    in_buf.value = data
-    
-    bzs.next_in = cast(in_buf, POINTER(c_char))
-    bzs.avail_in = in_bufsize
-    bzs.next_out = cast(out_buf, POINTER(c_char))
-    bzs.avail_out = out_bufsize
-
-    bzerror = libbz2.BZ2_bzCompressInit(byref(bzs), compresslevel, 0, 0)
-    if bzerror != BZ_OK:
-        _catch_bz2_error(space, bzerror)
-    
-    total_out = _bzs_total_out(bzs)
-    temp = []
-    while True:
-        bzerror = libbz2.BZ2_bzCompress(byref(bzs), BZ_FINISH)
-        if bzerror == BZ_STREAM_END:
-            break
-        elif bzerror != BZ_FINISH_OK:
-            libbz2.BZ2_bzCompressEnd(byref(bzs))
+    out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw',
+                            zero=True)
+    in_buf = lltype.malloc(rffi.CCHARP.TO, in_bufsize, flavor='raw')
+    for i in range(in_bufsize):
+        in_buf[i] = data[i]
+
+    try:
+        bzs.c_next_in = in_buf
+        bzs.c_avail_in = rffi.cast(rffi.UINT, in_bufsize)
+        bzs.c_next_out = out_buf
+        bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize)
+
+        bzerror = BZ2_bzCompressInit(bzs, compresslevel, 0, 0)
+        if bzerror != BZ_OK:
             _catch_bz2_error(space, bzerror)
-            
-        if bzs.avail_out == 0:
-            data = "".join([out_buf[i] for i in range(_bzs_total_out(bzs))])
-            temp.append(data)
-            
-            out_bufsize = _new_buffer_size(out_bufsize)
-            out_buf = create_string_buffer(out_bufsize)
-            bzs.next_out = cast(out_buf, POINTER(c_char))
-            bzs.avail_out = out_bufsize
-    
-    if temp:
-        res = "".join(temp)
-        
-    if bzs.avail_out:
-        size = _bzs_total_out(bzs) - total_out
-        res = "".join([out_buf[i] for i in range(size)])
-    else:
+    
         total_out = _bzs_total_out(bzs)
-        res = "".join([out_buf[i] for i in range(total_out)])
+        temp = []
+        while True:
+            bzerror = BZ2_bzCompress(bzs, BZ_FINISH)
+            if bzerror == BZ_STREAM_END:
+                break
+            elif bzerror != BZ_FINISH_OK:
+                BZ2_bzCompressEnd(bzs)
+                _catch_bz2_error(space, bzerror)
+            
+            if bzs.c_avail_out == 0:
+                data = "".join([out_buf[i] for i in range(_bzs_total_out(bzs))])
+                temp.append(data)
+
+                lltype.free(out_buf, flavor='raw')
+                out_bufsize = _new_buffer_size(out_bufsize)
+                out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize,
+                                        flavor='raw', zero=True)
+                bzs.c_next_out = out_buf
+                bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize)
     
-    libbz2.BZ2_bzCompressEnd(byref(bzs))
-    return space.wrap(res)
+        if temp:
+            res = "".join(temp)
+        
+        if bzs.c_avail_out:
+            size = _bzs_total_out(bzs) - total_out
+            res = "".join([out_buf[i] for i in range(size)])
+        else:
+            total_out = _bzs_total_out(bzs)
+            res = "".join([out_buf[i] for i in range(total_out)])
+    
+        BZ2_bzCompressEnd(bzs)
+        return space.wrap(res)
+    finally:
+        lltype.free(bzs, flavor='raw')
+        lltype.free(in_buf, flavor='raw')
+        lltype.free(out_buf, flavor='raw')
 compress.unwrap_spec = [ObjSpace, str, int]
 
 def decompress(space, data):
@@ -615,55 +650,64 @@
     in_bufsize = len(data)
     if in_bufsize == 0:
         return space.wrap("")
-    
-    bzs = bz_stream()
-    
-    in_buf = create_string_buffer(in_bufsize)
-    in_buf.value = data
+
+    bzs = lltype.malloc(bz_stream.TO, flavor='raw', zero=True)
+    in_buf = lltype.malloc(rffi.CCHARP.TO, in_bufsize, flavor='raw')
+    for i in range(in_bufsize):
+        in_buf[i] = data[i]
 
     out_bufsize = SMALLCHUNK
-    out_buf = create_string_buffer(out_bufsize)
+    out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize, flavor='raw',
+                            zero=True)
+    try:
+    
     
-    bzs.next_in = cast(in_buf, POINTER(c_char))
-    bzs.avail_in = in_bufsize
-    bzs.next_out = cast(out_buf, POINTER(c_char))
-    bzs.avail_out = out_bufsize
-    
-    bzerror = libbz2.BZ2_bzDecompressInit(byref(bzs), 0, 0)
-    if bzerror != BZ_OK:
-        _catch_bz2_error(space, bzerror)
-        
-    temp = []
-    while True:
-        bzerror = libbz2.BZ2_bzDecompress(byref(bzs))
-        if bzerror == BZ_STREAM_END:
-            break
+        bzs.c_next_in = in_buf
+        bzs.c_avail_in = rffi.cast(rffi.UINT, in_bufsize)
+        bzs.c_next_out = out_buf
+        bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize)
+    
+        bzerror = BZ2_bzDecompressInit(bzs, 0, 0)
         if bzerror != BZ_OK:
-            libbz2.BZ2_bzDecompressEnd(byref(bzs))
             _catch_bz2_error(space, bzerror)
         
-        if bzs.avail_in == 0:
-            libbz2.BZ2_bzDecompressEnd(byref(bzs))
-            raise OperationError(space.w_ValueError,
-                space.wrap("couldn't find end of stream"))
-        elif bzs.avail_out == 0:
-            total_out = _bzs_total_out(bzs)
-            data = "".join([out_buf[i] for i in range(total_out)])
+        temp = []
+        while True:
+            bzerror = BZ2_bzDecompress(bzs)
+            if bzerror == BZ_STREAM_END:
+                break
+            if bzerror != BZ_OK:
+                BZ2_bzDecompressEnd(bzs)
+                _catch_bz2_error(space, bzerror)
+        
+            if bzs.c_avail_in == 0:
+                BZ2_bzDecompressEnd(bzs)
+                raise OperationError(space.w_ValueError,
+                                     space.wrap("couldn't find end of stream"))
+            elif bzs.c_avail_out == 0:
+                total_out = _bzs_total_out(bzs)
+                data = "".join([out_buf[i] for i in range(total_out)])
+                temp.append(data)
+
+                lltype.free(out_buf, flavor='raw')
+                out_bufsize = _new_buffer_size(out_bufsize)
+                out_buf = lltype.malloc(rffi.CCHARP.TO, out_bufsize,
+                                        flavor='raw', zero=True)
+                bzs.c_next_out = out_buf
+                bzs.c_avail_out = rffi.cast(rffi.UINT, out_bufsize)
+    
+        total_out = _bzs_total_out(bzs)
+        if temp:
+            data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))])
             temp.append(data)
-            
-            out_bufsize = _new_buffer_size(out_bufsize)
-            out_buf = create_string_buffer(out_bufsize)
-            bzs.next_out = cast(out_buf, POINTER(c_char))
-            bzs.avail_out = out_bufsize
-    
-    total_out = _bzs_total_out(bzs)
-    if temp:
-        data = "".join([out_buf[i] for i in range(total_out - len(temp[0]))])
-        temp.append(data)
-        res = "".join(temp)
-    else:
-        res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00'])
+            res = "".join(temp)
+        else:
+            res = "".join([out_buf[i] for i in range(total_out) if out_buf[i] != '\x00'])
     
-    libbz2.BZ2_bzDecompressEnd(byref(bzs))
-    return space.wrap(res)
+        BZ2_bzDecompressEnd(bzs)
+        return space.wrap(res)
+    finally:
+        lltype.free(bzs, flavor='raw')
+        lltype.free(out_buf, flavor='raw')
+        lltype.free(in_buf, flavor='raw')
 decompress.unwrap_spec = [ObjSpace, str]

Added: pypy/dist/pypy/module/bz2/test/__init__.py
==============================================================================

Added: pypy/dist/pypy/module/bz2/test/support.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/module/bz2/test/support.py	Fri Oct  5 11:15:32 2007
@@ -0,0 +1,8 @@
+class CheckAllocation:
+    def teardown_method(self, fun):
+        from pypy.rpython.lltypesystem import ll2ctypes
+        import gc
+        gc.collect()
+        gc.collect()
+        gc.collect() # to make sure we disallocate buffers
+        assert not ll2ctypes.ALLOCATED

Modified: pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py
==============================================================================
--- pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py	(original)
+++ pypy/dist/pypy/module/bz2/test/test_bz2_compdecomp.py	Fri Oct  5 11:15:32 2007
@@ -1,4 +1,5 @@
 from pypy.conftest import gettestobjspace
+from pypy.module.bz2.test.support import CheckAllocation
 import os
 
 if os.name == "nt":
@@ -28,8 +29,7 @@
     if os.path.exists("foo"):
         os.unlink("foo")
 
-
-class AppTestBZ2Compressor:
+class AppTestBZ2Compressor(CheckAllocation):
     def setup_class(cls):
         space = gettestobjspace(usemodules=('bz2',))
         cls.space = space
@@ -46,7 +46,7 @@
         BZ2Compressor(9)
         
     def test_compress(self):
-        from bz2 import BZ2Compressor            
+        from bz2 import BZ2Compressor
         
         bz2c = BZ2Compressor()
         raises(TypeError, bz2c.compress)
@@ -79,7 +79,7 @@
         data = "%s%s" % (data, bz2c.flush())
         assert self.decompress(data) == self.TEXT
 
-class AppTestBZ2Decompressor:
+class AppTestBZ2Decompressor(CheckAllocation):
     def setup_class(cls):
         space = gettestobjspace(usemodules=('bz2',))
         cls.space = space
@@ -139,7 +139,7 @@
         bz2d.decompress(self.DATA)
         raises(EOFError, bz2d.decompress, "foo")
 
-class AppTestBZ2ModuleFunctions:
+class AppTestBZ2ModuleFunctions(CheckAllocation):
     def setup_class(cls):
         space = gettestobjspace(usemodules=('bz2',))
         cls.space = space

Modified: pypy/dist/pypy/module/bz2/test/test_bz2_file.py
==============================================================================
--- pypy/dist/pypy/module/bz2/test/test_bz2_file.py	(original)
+++ pypy/dist/pypy/module/bz2/test/test_bz2_file.py	Fri Oct  5 11:15:32 2007
@@ -1,5 +1,6 @@
 import py
 from pypy.conftest import gettestobjspace
+from pypy.module.bz2.test.support import CheckAllocation
 import os
 
 if os.name == "nt":
@@ -34,7 +35,9 @@
     mod.create_temp_file = create_temp_file
     mod.decompress = decompress
 
-class AppTestBZ2File:
+class AppTestBZ2File: #(CheckAllocation):
+    # XXX for unknown reasons, we cannot do allocation checks, as sth is
+    # keeping those objects alive (BZ2File objects)
     def setup_class(cls):
         space = gettestobjspace(usemodules=('bz2',))
         cls.space = space
@@ -92,6 +95,7 @@
         
         bz2f = BZ2File(self.temppath, mode='w')
         pos = bz2f.tell()
+        bz2f.close()
         assert pos == 0
     
     def test_seek(self):



More information about the Pypy-commit mailing list