[pypy-commit] pypy jit-targets: hg merge default

hakanardo noreply at buildbot.pypy.org
Fri Nov 4 21:14:44 CET 2011


Author: Hakan Ardo <hakan at debian.org>
Branch: jit-targets
Changeset: r48762:67b1506999f5
Date: 2011-11-04 07:43 +0100
http://bitbucket.org/pypy/pypy/changeset/67b1506999f5/

Log:	hg merge default

diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -362,7 +362,7 @@
     def descr_reversed(self):
         lastitem = self.start + (self.len-1) * self.step
         return self.space.wrap(W_XRangeIterator(self.space, lastitem,
-                                                self.start - 1, -self.step))
+                                                self.start, -self.step, True))
 
     def descr_reduce(self):
         space = self.space
@@ -389,21 +389,26 @@
 )
 
 class W_XRangeIterator(Wrappable):
-    def __init__(self, space, start, stop, step):
+    def __init__(self, space, start, stop, step, inclusive=False):
         self.space = space
         self.current = start
         self.stop = stop
         self.step = step
+        self.inclusive = inclusive
 
     def descr_iter(self):
         return self.space.wrap(self)
 
     def descr_next(self):
-        if (self.step > 0 and self.current < self.stop) or (self.step < 0 and self.current > self.stop):
-            item = self.current
-            self.current = item + self.step
-            return self.space.wrap(item)
-        raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        if self.inclusive:
+            if not ((self.step > 0 and self.current <= self.stop) or (self.step < 0 and self.current >= self.stop)):
+                raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        else:
+            if not ((self.step > 0 and self.current < self.stop) or (self.step < 0 and self.current > self.stop)):
+                raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        item = self.current
+        self.current = item + self.step
+        return self.space.wrap(item)
 
     #def descr_len(self):
     #    return self.space.wrap(self.remaining)
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -157,7 +157,8 @@
        raises(OverflowError, xrange, a)
        raises(OverflowError, xrange, 0, a)
        raises(OverflowError, xrange, 0, 1, a)
-
+       assert list(reversed(xrange(-sys.maxint-1, -sys.maxint-1, -2))) == []
+       
    def test_xrange_reduce(self):
       x = xrange(2, 9, 3)
       callable, args = x.__reduce__()
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -4,32 +4,44 @@
 from pypy.interpreter.error import OperationError
 from pypy.tool.sourcetools import func_renamer
 from pypy.interpreter.baseobjspace import Wrappable
-from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.rpython.lltypesystem import lltype, llmemory, rffi
+from pypy.rlib import rgc, ropenssl
 from pypy.rlib.objectmodel import keepalive_until_here
-from pypy.rlib import ropenssl
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.thread.os_lock import Lock
 
 algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
 
+# HASH_MALLOC_SIZE is the size of EVP_MD, EVP_MD_CTX plus their points
+# Used for adding memory pressure. Last number is an (under?)estimate of
+# EVP_PKEY_CTX's size.
+# XXX: Make a better estimate here
+HASH_MALLOC_SIZE = ropenssl.EVP_MD_SIZE + ropenssl.EVP_MD_CTX_SIZE \
+        + rffi.sizeof(ropenssl.EVP_MD) * 2 + 208
+
 class W_Hash(Wrappable):
     ctx = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)
+    _block_size = -1
 
     def __init__(self, space, name):
         self.name = name
+        self.digest_size = self.compute_digest_size()
 
         # Allocate a lock for each HASH object.
         # An optimization would be to not release the GIL on small requests,
         # and use a custom lock only when needed.
         self.lock = Lock(space)
 
+        ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
+        rgc.add_memory_pressure(HASH_MALLOC_SIZE + self.digest_size)
+        self.ctx = ctx
+
+    def initdigest(self, space, name):
         digest = ropenssl.EVP_get_digestbyname(name)
         if not digest:
             raise OperationError(space.w_ValueError,
                                  space.wrap("unknown hash function"))
-        ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
-        ropenssl.EVP_DigestInit(ctx, digest)
-        self.ctx = ctx
+        ropenssl.EVP_DigestInit(self.ctx, digest)
 
     def __del__(self):
         # self.lock.free()
@@ -65,33 +77,29 @@
         "Return the digest value as a string of hexadecimal digits."
         digest = self._digest(space)
         hexdigits = '0123456789abcdef'
-        result = StringBuilder(self._digest_size() * 2)
+        result = StringBuilder(self.digest_size * 2)
         for c in digest:
             result.append(hexdigits[(ord(c) >> 4) & 0xf])
             result.append(hexdigits[ ord(c)       & 0xf])
         return space.wrap(result.build())
 
     def get_digest_size(self, space):
-        return space.wrap(self._digest_size())
+        return space.wrap(self.digest_size)
 
     def get_block_size(self, space):
-        return space.wrap(self._block_size())
+        return space.wrap(self.compute_block_size())
 
     def _digest(self, space):
-        copy = self.copy(space)
-        ctx = copy.ctx
-        digest_size = self._digest_size()
-        digest = lltype.malloc(rffi.CCHARP.TO, digest_size, flavor='raw')
+        with lltype.scoped_alloc(ropenssl.EVP_MD_CTX.TO) as ctx:
+            with self.lock:
+                ropenssl.EVP_MD_CTX_copy(ctx, self.ctx)
+            digest_size = self.digest_size
+            with lltype.scoped_alloc(rffi.CCHARP.TO, digest_size) as digest:
+                ropenssl.EVP_DigestFinal(ctx, digest, None)
+                ropenssl.EVP_MD_CTX_cleanup(ctx)
+                return rffi.charpsize2str(digest, digest_size)
 
-        try:
-            ropenssl.EVP_DigestFinal(ctx, digest, None)
-            return rffi.charpsize2str(digest, digest_size)
-        finally:
-            keepalive_until_here(copy)
-            lltype.free(digest, flavor='raw')
-
-
-    def _digest_size(self):
+    def compute_digest_size(self):
         # XXX This isn't the nicest way, but the EVP_MD_size OpenSSL
         # XXX function is defined as a C macro on OS X and would be
         # XXX significantly harder to implement in another way.
@@ -105,12 +113,14 @@
             'sha512': 64, 'SHA512': 64,
             }.get(self.name, 0)
 
-    def _block_size(self):
+    def compute_block_size(self):
+        if self._block_size != -1:
+            return self._block_size
         # XXX This isn't the nicest way, but the EVP_MD_CTX_block_size
         # XXX OpenSSL function is defined as a C macro on some systems
         # XXX and would be significantly harder to implement in
         # XXX another way.
-        return {
+        self._block_size = {
             'md5':     64, 'MD5':     64,
             'sha1':    64, 'SHA1':    64,
             'sha224':  64, 'SHA224':  64,
@@ -118,6 +128,7 @@
             'sha384': 128, 'SHA384': 128,
             'sha512': 128, 'SHA512': 128,
             }.get(self.name, 0)
+        return self._block_size
 
 W_Hash.typedef = TypeDef(
     'HASH',
@@ -135,6 +146,7 @@
 @unwrap_spec(name=str, string='bufferstr')
 def new(space, name, string=''):
     w_hash = W_Hash(space, name)
+    w_hash.initdigest(space, name)
     w_hash.update(space, string)
     return space.wrap(w_hash)
 
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import wrap_oserror, OperationError
 from pypy.rpython.lltypesystem import rffi, lltype
+from pypy.rlib import rgc
 from pypy.rlib.rarithmetic import r_uint
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rpython.tool import rffi_platform as platform
@@ -23,6 +24,8 @@
     _CreateSemaphore = rwin32.winexternal(
         'CreateSemaphoreA', [rffi.VOIDP, rffi.LONG, rffi.LONG, rwin32.LPCSTR],
         rwin32.HANDLE)
+    _CloseHandle = rwin32.winexternal('CloseHandle', [rwin32.HANDLE],
+        rwin32.BOOL, threadsafe=False)
     _ReleaseSemaphore = rwin32.winexternal(
         'ReleaseSemaphore', [rwin32.HANDLE, rffi.LONG, rffi.LONGP],
         rwin32.BOOL)
@@ -51,6 +54,7 @@
         SEM_FAILED = platform.ConstantInteger('SEM_FAILED')
         SEM_VALUE_MAX = platform.ConstantInteger('SEM_VALUE_MAX')
         SEM_TIMED_WAIT = platform.Has('sem_timedwait')
+        SEM_T_SIZE = platform.SizeOf('sem_t')
 
     config = platform.configure(CConfig)
     TIMEVAL        = config['TIMEVAL']
@@ -61,18 +65,21 @@
     SEM_FAILED     = config['SEM_FAILED'] # rffi.cast(SEM_T, config['SEM_FAILED'])
     SEM_VALUE_MAX  = config['SEM_VALUE_MAX']
     SEM_TIMED_WAIT = config['SEM_TIMED_WAIT']
+    SEM_T_SIZE = config['SEM_T_SIZE']
     if sys.platform == 'darwin':
         HAVE_BROKEN_SEM_GETVALUE = True
     else:
         HAVE_BROKEN_SEM_GETVALUE = False
 
-    def external(name, args, result):
+    def external(name, args, result, **kwargs):
         return rffi.llexternal(name, args, result,
-                               compilation_info=eci)
+                               compilation_info=eci, **kwargs)
 
     _sem_open = external('sem_open',
                          [rffi.CCHARP, rffi.INT, rffi.INT, rffi.UINT],
                          SEM_T)
+    # tread sem_close as not threadsafe for now to be able to use the __del__
+    _sem_close = external('sem_close', [SEM_T], rffi.INT, threadsafe=False)
     _sem_unlink = external('sem_unlink', [rffi.CCHARP], rffi.INT)
     _sem_wait = external('sem_wait', [SEM_T], rffi.INT)
     _sem_trywait = external('sem_trywait', [SEM_T], rffi.INT)
@@ -90,6 +97,11 @@
             raise OSError(rposix.get_errno(), "sem_open failed")
         return res
 
+    def sem_close(handle):
+        res = _sem_close(handle)
+        if res < 0:
+            raise OSError(rposix.get_errno(), "sem_close failed")
+
     def sem_unlink(name):
         res = _sem_unlink(name)
         if res < 0:
@@ -205,6 +217,11 @@
             raise WindowsError(err, "CreateSemaphore")
         return handle
 
+    def delete_semaphore(handle):
+        if not _CloseHandle(handle):
+            err = rwin32.GetLastError()
+            raise WindowsError(err, "CloseHandle")
+
     def semlock_acquire(self, space, block, w_timeout):
         if not block:
             full_msecs = 0
@@ -291,8 +308,13 @@
             sem_unlink(name)
         except OSError:
             pass
+        else:
+            rgc.add_memory_pressure(SEM_T_SIZE)
         return sem
 
+    def delete_semaphore(handle):
+        sem_close(handle)
+
     def semlock_acquire(self, space, block, w_timeout):
         if not block:
             deadline = lltype.nullptr(TIMESPECP.TO)
@@ -483,6 +505,9 @@
     def exit(self, space, __args__):
         self.release(space)
 
+    def __del__(self):
+        delete_semaphore(self.handle)
+
 @unwrap_spec(kind=int, value=int, maxvalue=int)
 def descr_new(space, w_subtype, kind, value, maxvalue):
     if kind != RECURSIVE_MUTEX and kind != SEMAPHORE:
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -4,9 +4,9 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError
 from pypy.objspace.descroperation import object_setattr
+from pypy.rlib import rgc
+from pypy.rlib.unroll import unrolling_iterable
 from pypy.rpython.lltypesystem import rffi, lltype
-from pypy.rlib.unroll import unrolling_iterable
-
 from pypy.rpython.tool import rffi_platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.translator.platform import platform
@@ -118,6 +118,19 @@
         locals()[name] = rffi_platform.ConstantInteger(name)
     for name in xml_model_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
+    XML_Parser_SIZE = rffi_platform.SizeOf("XML_Parser")
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
@@ -793,7 +806,10 @@
             rffi.cast(rffi.CHAR, namespace_separator))
     else:
         xmlparser = XML_ParserCreate(encoding)
-
+    # Currently this is just the size of the pointer and some estimated bytes.
+    # The struct isn't actually defined in expat.h - it is in xmlparse.c
+    # XXX: find a good estimate of the XML_ParserStruct
+    rgc.add_memory_pressure(XML_Parser_SIZE + 300)
     if not xmlparser:
         raise OperationError(space.w_RuntimeError,
                              space.wrap('XML_ParserCreate failed'))
diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -2,10 +2,11 @@
 from pypy.rpython.lltypesystem import rffi, lltype, llmemory
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 import py
-from pypy.rlib import jit
+from pypy.rlib import jit, rgc
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem.lloperation import llop
+from pypy.rpython.tool import rffi_platform
 from pypy.tool import autopath
 
 class error(Exception):
@@ -49,7 +50,7 @@
 
 TLOCKP = rffi.COpaquePtr('struct RPyOpaque_ThreadLock',
                           compilation_info=eci)
-
+TLOCKP_SIZE = rffi_platform.sizeof('struct RPyOpaque_ThreadLock', eci)
 c_thread_lock_init = llexternal('RPyThreadLockInit', [TLOCKP], rffi.INT,
                                 threadsafe=False)   # may add in a global list
 c_thread_lock_dealloc_NOAUTO = llexternal('RPyOpaqueDealloc_ThreadLock',
@@ -164,6 +165,9 @@
     if rffi.cast(lltype.Signed, res) <= 0:
         lltype.free(ll_lock, flavor='raw', track_allocation=False)
         raise error("out of resources")
+    # Add some memory pressure for the size of the lock because it is an
+    # Opaque object
+    rgc.add_memory_pressure(TLOCKP_SIZE)
     return ll_lock
 
 def free_ll_lock(ll_lock):
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -259,6 +259,24 @@
     except Exception:
         return False      # don't keep objects whose _freeze_() method explodes
 
+def add_memory_pressure(estimate):
+    """Add memory pressure for OpaquePtrs."""
+    pass
+
+class AddMemoryPressureEntry(ExtRegistryEntry):
+    _about_ = add_memory_pressure
+
+    def compute_result_annotation(self, s_nbytes):
+        from pypy.annotation import model as annmodel
+        return annmodel.s_None
+
+    def specialize_call(self, hop):
+        [v_size] = hop.inputargs(lltype.Signed)
+        hop.exception_cannot_occur()
+        return hop.genop('gc_add_memory_pressure', [v_size],
+                         resulttype=lltype.Void)
+
+
 def get_rpy_memory_usage(gcref):
     "NOT_RPYTHON"
     # approximate implementation using CPython's type info
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -25,6 +25,7 @@
     'openssl/err.h',
     'openssl/rand.h',
     'openssl/evp.h',
+    'openssl/ossl_typ.h',
     'openssl/x509v3.h']
 
 eci = ExternalCompilationInfo(
@@ -108,7 +109,9 @@
     GENERAL_NAME_st = rffi_platform.Struct(
         'struct GENERAL_NAME_st',
         [('type', rffi.INT),
-         ]) 
+         ])
+    EVP_MD_SIZE = rffi_platform.SizeOf('EVP_MD')
+    EVP_MD_CTX_SIZE = rffi_platform.SizeOf('EVP_MD_CTX')
 
 
 for k, v in rffi_platform.configure(CConfig).items():
@@ -154,7 +157,7 @@
 ssl_external('CRYPTO_set_id_callback',
              [lltype.Ptr(lltype.FuncType([], rffi.LONG))],
              lltype.Void)
-             
+
 if HAVE_OPENSSL_RAND:
     ssl_external('RAND_add', [rffi.CCHARP, rffi.INT, rffi.DOUBLE], lltype.Void)
     ssl_external('RAND_status', [], rffi.INT)
@@ -255,7 +258,7 @@
              [BIO, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP], X509)
 
 EVP_MD_CTX = rffi.COpaquePtr('EVP_MD_CTX', compilation_info=eci)
-EVP_MD     = rffi.COpaquePtr('EVP_MD')
+EVP_MD     = rffi.COpaquePtr('EVP_MD', compilation_info=eci)
 
 OpenSSL_add_all_digests = external(
     'OpenSSL_add_all_digests', [], lltype.Void)
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -172,7 +172,7 @@
 
 def checkadr(addr):
     assert lltype.typeOf(addr) is llmemory.Address
-    
+
 def is_inst(inst):
     return isinstance(lltype.typeOf(inst), (ootype.Instance, ootype.BuiltinType, ootype.StaticMethod))
 
@@ -657,7 +657,7 @@
                 raise TypeError("graph with %r args called with wrong func ptr type: %r" %
                                 (tuple([v.concretetype for v in args_v]), ARGS)) 
         frame = self.newsubframe(graph, args)
-        return frame.eval()        
+        return frame.eval()
 
     def op_direct_call(self, f, *args):
         FTYPE = self.llinterpreter.typer.type_system.derefType(lltype.typeOf(f))
@@ -698,13 +698,13 @@
             return ptr
         except MemoryError:
             self.make_llexception()
-            
+
     def op_malloc_nonmovable(self, TYPE, flags):
         flavor = flags['flavor']
         assert flavor == 'gc'
         zero = flags.get('zero', False)
         return self.heap.malloc_nonmovable(TYPE, zero=zero)
-        
+
     def op_malloc_nonmovable_varsize(self, TYPE, flags, size):
         flavor = flags['flavor']
         assert flavor == 'gc'
@@ -716,6 +716,9 @@
         track_allocation = flags.get('track_allocation', True)
         self.heap.free(obj, flavor='raw', track_allocation=track_allocation)
 
+    def op_gc_add_memory_pressure(self, size):
+        self.heap.add_memory_pressure(size)
+
     def op_shrink_array(self, obj, smallersize):
         return self.heap.shrink_array(obj, smallersize)
 
@@ -1318,7 +1321,7 @@
         func_graph = fn.graph
     else:
         # obj is an instance, we want to call 'method_name' on it
-        assert fn is None        
+        assert fn is None
         self_arg = [obj]
         func_graph = obj._TYPE._methods[method_name._str].graph
 
diff --git a/pypy/rpython/lltypesystem/llheap.py b/pypy/rpython/lltypesystem/llheap.py
--- a/pypy/rpython/lltypesystem/llheap.py
+++ b/pypy/rpython/lltypesystem/llheap.py
@@ -5,8 +5,7 @@
 
 setfield = setattr
 from operator import setitem as setarrayitem
-from pypy.rlib.rgc import collect
-from pypy.rlib.rgc import can_move
+from pypy.rlib.rgc import can_move, collect, add_memory_pressure
 
 def setinterior(toplevelcontainer, inneraddr, INNERTYPE, newvalue,
                 offsets=None):
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -473,6 +473,7 @@
     'gc_is_rpy_instance'  : LLOp(),
     'gc_dump_rpy_heap'    : LLOp(),
     'gc_typeids_z'        : LLOp(),
+    'gc_add_memory_pressure': LLOp(),
 
     # ------- JIT & GC interaction, only for some GCs ----------
 
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -48,7 +48,7 @@
         self.TYPE = TYPE
     def __repr__(self):
         return '<Uninitialized %r>'%(self.TYPE,)
-        
+
 
 def saferecursive(func, defl, TLS=TLS):
     def safe(*args):
@@ -537,9 +537,9 @@
         return "Func ( %s ) -> %s" % (args, self.RESULT)
     __str__ = saferecursive(__str__, '...')
 
-    def _short_name(self):        
+    def _short_name(self):
         args = ', '.join([ARG._short_name() for ARG in self.ARGS])
-        return "Func(%s)->%s" % (args, self.RESULT._short_name())        
+        return "Func(%s)->%s" % (args, self.RESULT._short_name())
     _short_name = saferecursive(_short_name, '...')
 
     def _container_example(self):
@@ -553,7 +553,7 @@
 
 class OpaqueType(ContainerType):
     _gckind = 'raw'
-    
+
     def __init__(self, tag, hints={}):
         """ if hints['render_structure'] is set, the type is internal and not considered
             to come from somewhere else (it should be rendered as a structure) """
@@ -723,10 +723,10 @@
 
     def __str__(self):
         return '* %s' % (self.TO, )
-    
+
     def _short_name(self):
         return 'Ptr %s' % (self.TO._short_name(), )
-    
+
     def _is_atomic(self):
         return self.TO._gckind == 'raw'
 
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -377,17 +377,24 @@
             self.malloc_varsize_nonmovable_ptr = None
 
         if getattr(GCClass, 'raw_malloc_memory_pressure', False):
-            def raw_malloc_memory_pressure(length, itemsize):
+            def raw_malloc_memory_pressure_varsize(length, itemsize):
                 totalmem = length * itemsize
                 if totalmem > 0:
                     gcdata.gc.raw_malloc_memory_pressure(totalmem)
                 #else: probably an overflow -- the following rawmalloc
                 #      will fail then
+            def raw_malloc_memory_pressure(sizehint):
+                gcdata.gc.raw_malloc_memory_pressure(sizehint)
+            self.raw_malloc_memory_pressure_varsize_ptr = getfn(
+                raw_malloc_memory_pressure_varsize,
+                [annmodel.SomeInteger(), annmodel.SomeInteger()],
+                annmodel.s_None, minimal_transform = False)
             self.raw_malloc_memory_pressure_ptr = getfn(
                 raw_malloc_memory_pressure,
-                [annmodel.SomeInteger(), annmodel.SomeInteger()],
+                [annmodel.SomeInteger()],
                 annmodel.s_None, minimal_transform = False)
 
+
         self.identityhash_ptr = getfn(GCClass.identityhash.im_func,
                                       [s_gc, s_gcref],
                                       annmodel.SomeInteger(),
diff --git a/pypy/rpython/memory/gctransform/transform.py b/pypy/rpython/memory/gctransform/transform.py
--- a/pypy/rpython/memory/gctransform/transform.py
+++ b/pypy/rpython/memory/gctransform/transform.py
@@ -63,7 +63,7 @@
                     gct.push_alive(v_result, self.llops)
             elif opname not in ('direct_call', 'indirect_call'):
                 gct.push_alive(v_result, self.llops)
-        
+
 
 
     def rename(self, newopname):
@@ -118,7 +118,7 @@
             self.minimalgctransformer = self.MinimalGCTransformer(self)
         else:
             self.minimalgctransformer = None
-            
+
     def get_lltype_of_exception_value(self):
         if self.translator is not None:
             exceptiondata = self.translator.rtyper.getexceptiondata()
@@ -399,7 +399,7 @@
 
     def gct_gc_heap_stats(self, hop):
         from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP
-        
+
         return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP),
                                         lltype.nullptr(ARRAY_TYPEID_MAP)))
 
@@ -427,7 +427,7 @@
         assert flavor == 'raw'
         assert not flags.get('zero')
         return self.parenttransformer.gct_malloc_varsize(hop)
-    
+
     def gct_free(self, hop):
         flags = hop.spaceop.args[1].value
         flavor = flags['flavor']
@@ -502,7 +502,7 @@
         stack_mh = mallocHelpers()
         stack_mh.allocate = lambda size: llop.stack_malloc(llmemory.Address, size)
         ll_stack_malloc_fixedsize = stack_mh._ll_malloc_fixedsize
-        
+
         if self.translator:
             self.raw_malloc_fixedsize_ptr = self.inittime_helper(
                 ll_raw_malloc_fixedsize, [lltype.Signed], llmemory.Address)
@@ -541,7 +541,7 @@
                           resulttype=llmemory.Address)
         if flags.get('zero'):
             hop.genop("raw_memclear", [v_raw, c_size])
-        return v_raw        
+        return v_raw
 
     def gct_malloc_varsize(self, hop, add_flags=None):
         flags = hop.spaceop.args[1].value
@@ -559,6 +559,14 @@
     def gct_malloc_nonmovable_varsize(self, *args, **kwds):
         return self.gct_malloc_varsize(*args, **kwds)
 
+    def gct_gc_add_memory_pressure(self, hop):
+        if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
+            op = hop.spaceop
+            size = op.args[0]
+            return hop.genop("direct_call",
+                          [self.raw_malloc_memory_pressure_ptr,
+                           size])
+
     def varsize_malloc_helper(self, hop, flags, meth, extraargs):
         def intconst(c): return rmodel.inputconst(lltype.Signed, c)
         op = hop.spaceop
@@ -590,9 +598,9 @@
     def gct_fv_raw_malloc_varsize(self, hop, flags, TYPE, v_length, c_const_size, c_item_size,
                                                                     c_offset_to_length):
         if flags.get('add_memory_pressure', False):
-            if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
+            if hasattr(self, 'raw_malloc_memory_pressure_varsize_ptr'):
                 hop.genop("direct_call",
-                          [self.raw_malloc_memory_pressure_ptr,
+                          [self.raw_malloc_memory_pressure_varsize_ptr,
                            v_length, c_item_size])
         if c_offset_to_length is None:
             if flags.get('zero'):
@@ -625,7 +633,7 @@
                 hop.genop("track_alloc_stop", [v])
             hop.genop('raw_free', [v])
         else:
-            assert False, "%s has no support for free with flavor %r" % (self, flavor)           
+            assert False, "%s has no support for free with flavor %r" % (self, flavor)
 
     def gct_gc_can_move(self, hop):
         return hop.cast_result(rmodel.inputconst(lltype.Bool, False))
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -66,6 +66,10 @@
             gctypelayout.zero_gc_pointers(result)
         return result
 
+    def add_memory_pressure(self, size):
+        if hasattr(self.gc, 'raw_malloc_memory_pressure'):
+            self.gc.raw_malloc_memory_pressure(size)
+
     def shrink_array(self, p, smallersize):
         if hasattr(self.gc, 'shrink_array'):
             addr = llmemory.cast_ptr_to_adr(p)
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -592,7 +592,7 @@
             return rgc.can_move(lltype.malloc(TP, 1))
         assert self.interpret(func, []) == self.GC_CAN_MOVE
 
-    
+
     def test_malloc_nonmovable(self):
         TP = lltype.GcArray(lltype.Char)
         def func():
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -27,7 +27,7 @@
     t.config.set(**extraconfigopts)
     ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy())
     ann.build_types(func, inputtypes)
-                                   
+
     if specialize:
         t.buildrtyper().specialize()
     if backendopt:
@@ -44,7 +44,7 @@
     GC_CAN_MOVE = False
     GC_CAN_MALLOC_NONMOVABLE = True
     taggedpointers = False
-    
+
     def setup_class(cls):
         funcs0 = []
         funcs2 = []
@@ -155,7 +155,7 @@
             return run, gct
         else:
             return run
-        
+
 class GenericGCTests(GCTest):
     GC_CAN_SHRINK_ARRAY = False
 
@@ -190,7 +190,7 @@
                     j += 1
             return 0
         return malloc_a_lot
-    
+
     def test_instances(self):
         run, statistics = self.runner("instances", statistics=True)
         run([])
@@ -276,7 +276,7 @@
         for i in range(1, 5):
             res = run([i, i - 1])
             assert res == i - 1 # crashes if constants are not considered roots
-            
+
     def define_string_concatenation(cls):
         def concat(j, dummy):
             lst = []
@@ -656,7 +656,7 @@
             #    return 2
 
         return func
-    
+
     def test_malloc_nonmovable(self):
         run = self.runner("malloc_nonmovable")
         assert int(self.GC_CAN_MALLOC_NONMOVABLE) == run([])
@@ -676,7 +676,7 @@
                 return 2
 
         return func
-    
+
     def test_malloc_nonmovable_fixsize(self):
         run = self.runner("malloc_nonmovable_fixsize")
         assert run([]) == int(self.GC_CAN_MALLOC_NONMOVABLE)
@@ -757,7 +757,7 @@
             lltype.free(idarray, flavor='raw')
             return 0
         return f
-    
+
     def test_many_ids(self):
         if not self.GC_CAN_TEST_ID:
             py.test.skip("fails for bad reasons in lltype.py :-(")
@@ -813,7 +813,7 @@
             else:
                 assert 0, "oups, not found"
         return f, None, fix_graph_of_g
-            
+
     def test_do_malloc_operations(self):
         run = self.runner("do_malloc_operations")
         run([])
@@ -850,7 +850,7 @@
             else:
                 assert 0, "oups, not found"
         return f, None, fix_graph_of_g
-        
+
     def test_do_malloc_operations_in_call(self):
         run = self.runner("do_malloc_operations_in_call")
         run([])
@@ -861,7 +861,7 @@
         l2 = []
         l3 = []
         l4 = []
-        
+
         def f():
             for i in range(10):
                 s = lltype.malloc(S)
@@ -1026,7 +1026,7 @@
             llop.gc__collect(lltype.Void)
             return static.p.x + i
         def cleanup():
-            static.p = lltype.nullptr(T1)        
+            static.p = lltype.nullptr(T1)
         return f, cleanup, None
 
     def test_nongc_static_root_minor_collect(self):
@@ -1081,7 +1081,7 @@
             return 0
 
         return f
-        
+
     def test_many_weakrefs(self):
         run = self.runner("many_weakrefs")
         run([])
@@ -1131,7 +1131,7 @@
     def define_adr_of_nursery(cls):
         class A(object):
             pass
-        
+
         def f():
             # we need at least 1 obj to allocate a nursery
             a = A()
@@ -1147,9 +1147,9 @@
             assert nt1 > nf1
             assert nt1 == nt0
             return 0
-        
+
         return f
-    
+
     def test_adr_of_nursery(self):
         run = self.runner("adr_of_nursery")
         res = run([])        
@@ -1175,7 +1175,7 @@
             def _teardown(self):
                 self.__ready = False # collecting here is expected
                 GenerationGC._teardown(self)
-                
+
             GC_PARAMS = {'space_size': 512*WORD,
                          'nursery_size': 128*WORD,
                          'translated_to_c': False}
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -37,7 +37,7 @@
             else:
                 print res
             return 0
-        
+
         t = Translation(main, standalone=True, gc=cls.gcpolicy,
                         policy=annpolicy.StrictAnnotatorPolicy(),
                         taggedpointers=cls.taggedpointers,
@@ -128,10 +128,10 @@
         if not args:
             args = (-1, )
         res = self.allfuncs(name, *args)
-        num = self.name_to_func[name]        
+        num = self.name_to_func[name]
         if self.funcsstr[num]:
             return res
-        return int(res)        
+        return int(res)
 
     def define_empty_collect(cls):
         def f():
@@ -228,7 +228,7 @@
         T = lltype.GcStruct("T", ('y', lltype.Signed),
                                  ('s', lltype.Ptr(S)))
         ARRAY_Ts = lltype.GcArray(lltype.Ptr(T))
-        
+
         def f():
             r = 0
             for i in range(30):
@@ -250,7 +250,7 @@
     def test_framework_varsized(self):
         res = self.run('framework_varsized')
         assert res == self.run_orig('framework_varsized')
-            
+
     def define_framework_using_lists(cls):
         class A(object):
             pass
@@ -271,7 +271,7 @@
         N = 1000
         res = self.run('framework_using_lists')
         assert res == N*(N - 1)/2
-    
+
     def define_framework_static_roots(cls):
         class A(object):
             def __init__(self, y):
@@ -318,8 +318,8 @@
     def test_framework_void_array(self):
         res = self.run('framework_void_array')
         assert res == 44
-        
-        
+
+
     def define_framework_malloc_failure(cls):
         def f():
             a = [1] * (sys.maxint//2)
@@ -342,7 +342,7 @@
     def test_framework_array_of_void(self):
         res = self.run('framework_array_of_void')
         assert res == 43 + 1000000
-        
+
     def define_framework_opaque(cls):
         A = lltype.GcStruct('A', ('value', lltype.Signed))
         O = lltype.GcOpaqueType('test.framework')
@@ -437,7 +437,7 @@
             b = B()
             return 0
         return func
-    
+
     def test_del_raises(self):
         self.run('del_raises') # does not raise
 
@@ -712,7 +712,7 @@
 
     def test_callback_with_collect(self):
         assert self.run('callback_with_collect')
-    
+
     def define_can_move(cls):
         class A:
             pass
@@ -1255,7 +1255,7 @@
         l1 = []
         l2 = []
         l3 = []
-        
+
         def f():
             for i in range(10):
                 s = lltype.malloc(S)
@@ -1298,7 +1298,7 @@
     def test_string_builder(self):
         res = self.run('string_builder')
         assert res == "aabcbdddd"
-    
+
     def definestr_string_builder_over_allocation(cls):
         import gc
         def fn(_):
@@ -1458,6 +1458,37 @@
         res = self.run("nongc_attached_to_gc")
         assert res == -99997
 
+    def define_nongc_opaque_attached_to_gc(cls):
+        from pypy.module._hashlib.interp_hashlib import HASH_MALLOC_SIZE
+        from pypy.rlib import rgc, ropenssl
+        from pypy.rpython.lltypesystem import rffi
+
+        class A:
+            def __init__(self):
+                self.ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO,
+                    flavor='raw')
+                digest = ropenssl.EVP_get_digestbyname('sha1')
+                ropenssl.EVP_DigestInit(self.ctx, digest)
+                rgc.add_memory_pressure(HASH_MALLOC_SIZE + 64)
+
+            def __del__(self):
+                ropenssl.EVP_MD_CTX_cleanup(self.ctx)
+                lltype.free(self.ctx, flavor='raw')
+        A()
+        def f():
+            am1 = am2 = am3 = None
+            for i in range(100000):
+                am3 = am2
+                am2 = am1
+                am1 = A()
+            # what can we use for the res?
+            return 0
+        return f
+
+    def test_nongc_opaque_attached_to_gc(self):
+        res = self.run("nongc_opaque_attached_to_gc")
+        assert res == 0
+
 # ____________________________________________________________________
 
 class TaggedPointersTest(object):


More information about the pypy-commit mailing list