[pypy-svn] r48349 - in pypy/dist/pypy: config doc/config rpython rpython/lltypesystem rpython/memory rpython/memory/gc rpython/memory/gctransform rpython/memory/test rpython/test translator/backendopt
cfbolz at codespeak.net
cfbolz at codespeak.net
Wed Nov 7 01:21:20 CET 2007
Author: cfbolz
Date: Wed Nov 7 01:21:18 2007
New Revision: 48349
Added:
pypy/dist/pypy/doc/config/translation.backendopt.coalloc.txt (contents, props changed)
Modified:
pypy/dist/pypy/config/translationoption.py
pypy/dist/pypy/rpython/llinterp.py
pypy/dist/pypy/rpython/lltypesystem/llheap.py
pypy/dist/pypy/rpython/memory/gc/base.py
pypy/dist/pypy/rpython/memory/gc/generation.py
pypy/dist/pypy/rpython/memory/gctransform/framework.py
pypy/dist/pypy/rpython/memory/gctransform/transform.py
pypy/dist/pypy/rpython/memory/gcwrapper.py
pypy/dist/pypy/rpython/memory/test/test_gc.py
pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py
pypy/dist/pypy/rpython/test/test_llinterp.py
pypy/dist/pypy/translator/backendopt/all.py
Log:
add coallocation support to the GCs. needs changes in various places.
Modified: pypy/dist/pypy/config/translationoption.py
==============================================================================
--- pypy/dist/pypy/config/translationoption.py (original)
+++ pypy/dist/pypy/config/translationoption.py Wed Nov 7 01:21:18 2007
@@ -160,6 +160,9 @@
BoolOption("heap2stack", "Escape analysis and stack allocation",
default=False,
requires=[("translation.stackless", False)]),
+ BoolOption("coalloc", "Try to replace mallocs by coallocation",
+ default=False,
+ suggests=[("translation.gc", "generation")]),
# control profile based inlining
StrOption("profile_based_inline",
"Use call count profiling to drive inlining"
Added: pypy/dist/pypy/doc/config/translation.backendopt.coalloc.txt
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/doc/config/translation.backendopt.coalloc.txt Wed Nov 7 01:21:18 2007
@@ -0,0 +1,4 @@
+turn allocations into coallocations when appropriate. Requires the use of a
+generational GC. See the paper `Finding your Cronies`_.
+
+.. _`Finding your Cronies`:: http://www.cs.utexas.edu/~sammy/cronies-oopsla-2004.pdf
Modified: pypy/dist/pypy/rpython/llinterp.py
==============================================================================
--- pypy/dist/pypy/rpython/llinterp.py (original)
+++ pypy/dist/pypy/rpython/llinterp.py Wed Nov 7 01:21:18 2007
@@ -676,8 +676,11 @@
return ptr
def op_coalloc(self, obj, coallocator, flags):
- # it's always safe to ignore the coallocator
- return self.op_malloc(obj, flags)
+ flavor = flags['flavor']
+ assert flavor == "gc"
+ zero = flags.get('zero', False)
+ ptr = self.heap.coalloc(obj, coallocator, zero=zero)
+ return ptr
# only after gc transform
def op_cpy_malloc(self, obj, cpytype): # xxx
@@ -699,8 +702,12 @@
self.make_llexception()
def op_coalloc_varsize(self, obj, coallocator, flags, size):
- # it's always safe to ignore the coallocator
- return self.op_malloc_varsize(obj, flags, size)
+ flavor = flags['flavor']
+ zero = flags.get('zero', False)
+ assert flavor == "gc"
+ zero = flags.get('zero', False)
+ ptr = self.heap.coalloc(obj, coallocator, size, zero=zero)
+ return ptr
def op_free(self, obj, flavor):
assert isinstance(flavor, str)
Modified: pypy/dist/pypy/rpython/lltypesystem/llheap.py
==============================================================================
--- pypy/dist/pypy/rpython/lltypesystem/llheap.py (original)
+++ pypy/dist/pypy/rpython/lltypesystem/llheap.py Wed Nov 7 01:21:18 2007
@@ -19,3 +19,7 @@
def weakref_create_getlazy(objgetter):
return weakref_create(objgetter())
+
+def coalloc(T, coallocator, n=None, zero=True):
+ # ignore the coallocator
+ return malloc(T, n, zero=zero)
Modified: pypy/dist/pypy/rpython/memory/gc/base.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gc/base.py (original)
+++ pypy/dist/pypy/rpython/memory/gc/base.py Wed Nov 7 01:21:18 2007
@@ -36,9 +36,10 @@
def size_gc_header(self, typeid=0):
return self.gcheaderbuilder.size_gc_header
- def malloc(self, typeid, length=0, zero=False):
+ def malloc(self, typeid, length=0, zero=False, coallocator=None):
"""For testing. The interface used by the gctransformer is
the four malloc_[fixed,var]size[_clear]() functions.
+ And (if they exist) to the coalloc_[fixed,var]size functios
"""
size = self.fixed_size(typeid)
needs_finalizer = bool(self.getfinalizer(typeid))
@@ -59,15 +60,26 @@
malloc_varsize = self.malloc_varsize_clear
else:
malloc_varsize = self.malloc_varsize
- ref = malloc_varsize(typeid, length, size, itemsize,
- offset_to_length, True, needs_finalizer)
+ if coallocator is not None and hasattr(self, "coalloc_varsize"):
+ assert not needs_finalizer
+ coallocator = llmemory.cast_ptr_to_adr(coallocator)
+ ref = self.coalloc_varsize(coallocator, typeid, length, size,
+ itemsize, offset_to_length)
+ else:
+ ref = malloc_varsize(typeid, length, size, itemsize,
+ offset_to_length, True, needs_finalizer)
else:
if zero:
malloc_fixedsize = self.malloc_fixedsize_clear
else:
malloc_fixedsize = self.malloc_fixedsize
- ref = malloc_fixedsize(typeid, size, True, needs_finalizer,
- contains_weakptr)
+ if coallocator is not None and hasattr(self, "coalloc_fixedsize"):
+ assert not needs_finalizer
+ coallocator = llmemory.cast_ptr_to_adr(coallocator)
+ ref = self.coalloc_fixedsize(coallocator, typeid, size)
+ else:
+ ref = malloc_fixedsize(typeid, size, True, needs_finalizer,
+ contains_weakptr)
# lots of cast and reverse-cast around...
return llmemory.cast_ptr_to_adr(ref)
Modified: pypy/dist/pypy/rpython/memory/gc/generation.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gc/generation.py (original)
+++ pypy/dist/pypy/rpython/memory/gc/generation.py Wed Nov 7 01:21:18 2007
@@ -77,6 +77,17 @@
self.young_objects_with_weakrefs.append(result + size_gc_header)
return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF)
+ def coalloc_fixedsize(self, coallocator, typeid, size):
+ # note: a coallocated object can never return a weakref, since the
+ # coallocation analysis is done at a time where weakrefs are
+ # represented as opaque objects which aren't allocated using malloc but
+ # with weakref_create
+ if self.is_in_nursery(coallocator):
+ return self.malloc_fixedsize(typeid, size, True, False, False)
+ else:
+ return SemiSpaceGC.malloc_fixedsize(self, typeid, size, True,
+ False, False)
+
def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length,
can_collect, has_finalizer=False):
# only use the nursery if there are not too many items
@@ -101,6 +112,16 @@
self.nursery_free = result + llarena.round_up_for_allocation(totalsize)
return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF)
+ def coalloc_varsize(self, coallocator, typeid, length, size, itemsize,
+ offset_to_length):
+ if self.is_in_nursery(coallocator):
+ return self.malloc_varsize(typeid, length, size, itemsize,
+ offset_to_length, True, False)
+ else:
+ return SemiSpaceGC.malloc_varsize(self, typeid, length, size,
+ itemsize, offset_to_length,
+ True, False)
+
# override the init_gc_object methods to change the default value of 'flags',
# used by objects that are directly created outside the nursery by the SemiSpaceGC.
# These objects must have the GCFLAG_NO_YOUNG_PTRS flag set immediately.
Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original)
+++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Wed Nov 7 01:21:18 2007
@@ -27,6 +27,8 @@
if op.opname in ('malloc', 'malloc_varsize'):
flags = op.args[1].value
return flags['flavor'] == 'gc' and not flags.get('nocollect', False)
+ if op.opname in ('coalloc', 'coalloc_varsize'):
+ return True
def find_initializing_stores(collect_analyzer, graph):
from pypy.objspace.flow.model import mkentrymap
@@ -323,6 +325,20 @@
inline=True)
else:
self.write_barrier_ptr = None
+ if hasattr(GCClass, "coalloc_fixedsize"):
+ self.coalloc_clear_ptr = getfn(
+ GCClass.coalloc_fixedsize.im_func,
+ [s_gc, annmodel.SomeAddress(),
+ annmodel.SomeInteger(nonneg=True),
+ annmodel.SomeInteger(nonneg=True)],
+ s_gcref, inline=True)
+ self.coalloc_varsize_clear_ptr = getfn(
+ GCClass.coalloc_varsize.im_func,
+ [s_gc, annmodel.SomeAddress()] +
+ [annmodel.SomeInteger(nonneg=True) for i in range(5)],
+ s_gcref, inline=True)
+ else:
+ self.coalloc_clear_ptr = self.coalloc_varsize_clear_ptr = None
self.statistics_ptr = getfn(GCClass.statistics.im_func,
[s_gc, annmodel.SomeInteger()],
annmodel.SomeInteger())
@@ -563,6 +579,43 @@
gct_fv_gc_malloc_varsize = gct_fv_gc_malloc
+ def gct_fv_gc_coalloc(self, hop, coallocator, flags, TYPE, *args):
+ if self.coalloc_clear_ptr is None:
+ return self.gct_fv_gc_malloc(
+ hop, flags, TYPE, *args)
+ op = hop.spaceop
+ flavor = flags['flavor']
+ assert not flags.get("nocollect", False)
+
+ PTRTYPE = op.result.concretetype
+ assert PTRTYPE.TO == TYPE
+ type_id = self.get_type_id(TYPE)
+
+ c_type_id = rmodel.inputconst(lltype.Signed, type_id)
+ info = self.layoutbuilder.type_info_list[type_id]
+ c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"])
+ has_finalizer = bool(self.finalizer_funcptr_for_type(TYPE))
+ assert not has_finalizer
+
+ v_coallocator = gen_cast(hop.llops, llmemory.Address, coallocator)
+
+ if not op.opname.endswith('_varsize'):
+ malloc_ptr = self.coalloc_clear_ptr
+ args = [self.c_const_gc, v_coallocator, c_type_id, c_size]
+ else:
+ v_length = op.args[-1]
+ c_ofstolength = rmodel.inputconst(lltype.Signed, info['ofstolength'])
+ c_varitemsize = rmodel.inputconst(lltype.Signed, info['varitemsize'])
+ malloc_ptr = self.coalloc_varsize_clear_ptr
+ args = [self.c_const_gc, v_coallocator, c_type_id, v_length, c_size,
+ c_varitemsize, c_ofstolength]
+ livevars = self.push_roots(hop)
+ v_result = hop.genop("direct_call", [malloc_ptr] + args,
+ resulttype=llmemory.GCREF)
+ self.pop_roots(hop, livevars)
+ return v_result
+ gct_fv_gc_coalloc_varsize = gct_fv_gc_coalloc
+
def gct_gc__collect(self, hop):
op = hop.spaceop
livevars = self.push_roots(hop)
Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original)
+++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Wed Nov 7 01:21:18 2007
@@ -491,6 +491,21 @@
c_size = rmodel.inputconst(lltype.Signed, llmemory.sizeof(TYPE))
v_raw = meth(hop, flags, TYPE, c_size)
hop.cast_result(v_raw)
+
+ def gct_coalloc(self, hop):
+ TYPE = hop.spaceop.result.concretetype.TO
+ assert not TYPE._is_varsize()
+ flags = hop.spaceop.args[2].value
+ flavor = flags['flavor']
+ c_size = rmodel.inputconst(lltype.Signed, llmemory.sizeof(TYPE))
+ meth = getattr(self, 'gct_fv_%s_comalloc' % flavor, None)
+ if meth is None:
+ meth = getattr(self, 'gct_fv_%s_malloc' % flavor, None)
+ assert meth, "%s has no support for comalloc with flavor %r" % (self, flavor)
+ v_raw = meth(hop, flags, TYPE, c_size)
+ else:
+ v_raw = meth(hop, hop.spaceop.args[1], flags, TYPE, c_size)
+ hop.cast_result(v_raw)
def gct_fv_raw_malloc(self, hop, flags, TYPE, c_size):
v_raw = hop.genop("direct_call", [self.raw_malloc_fixedsize_ptr, c_size],
@@ -514,16 +529,32 @@
return hop.genop('cpy_malloc', args, resulttype=op.result.concretetype)
def gct_malloc_varsize(self, hop):
- def intconst(c): return rmodel.inputconst(lltype.Signed, c)
- op = hop.spaceop
- TYPE = op.result.concretetype.TO
- assert TYPE._is_varsize()
flags = hop.spaceop.args[1].value
flavor = flags['flavor']
meth = getattr(self, 'gct_fv_%s_malloc_varsize' % flavor, None)
assert meth, "%s has no support for malloc_varsize with flavor %r" % (self, flavor)
+ return self.varsize_malloc_helper(hop, flags, meth, [])
+
+ def gct_coalloc_varsize(self, hop):
+ flags = hop.spaceop.args[2].value
+ flavor = flags['flavor']
+ meth = getattr(self, 'gct_fv_%s_coalloc_varsize' % flavor, None)
+ if meth is None:
+ meth = getattr(self, 'gct_fv_%s_malloc_varsize' % flavor, None)
+ assert meth, "%s has no support for malloc_varsize with flavor %r" % (self, flavor)
+ return self.varsize_malloc_helper(hop, flags, meth, [])
+ else:
+ return self.varsize_malloc_helper(hop, flags, meth,
+ [hop.spaceop.args[1]])
+
+
+ def varsize_malloc_helper(self, hop, flags, meth, extraargs):
+ def intconst(c): return rmodel.inputconst(lltype.Signed, c)
+ op = hop.spaceop
+ TYPE = op.result.concretetype.TO
+ assert TYPE._is_varsize()
if isinstance(TYPE, lltype.Struct):
ARRAY = TYPE._flds[TYPE._arrayfld]
else:
@@ -545,11 +576,13 @@
offset_to_length = llmemory.ArrayLengthOffset(ARRAY)
c_offset_to_length = intconst(offset_to_length)
- v_raw = meth(hop, flags, TYPE, op.args[-1], c_const_size, c_item_size,
- c_offset_to_length)
+ args = [hop] + extraargs + [flags, TYPE,
+ op.args[-1], c_const_size, c_item_size, c_offset_to_length]
+ v_raw = meth(*args)
hop.cast_result(v_raw)
+
def gct_fv_raw_malloc_varsize(self, hop, flags, TYPE, v_length, c_const_size, c_item_size,
c_offset_to_length):
if c_offset_to_length is None:
Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/gcwrapper.py (original)
+++ pypy/dist/pypy/rpython/memory/gcwrapper.py Wed Nov 7 01:21:18 2007
@@ -62,6 +62,17 @@
assert flavor != 'gc'
return lltype.free(TYPE, flavor=flavor)
+ def coalloc(self, TYPE, coallocator, size=None, zero=False):
+ if hasattr(self.gc, "coalloc_fixedsize"):
+ typeid = self.get_type_id(TYPE)
+ addr = self.gc.malloc(typeid, size, zero=zero,
+ coallocator=coallocator)
+ result = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE))
+ if self.gc.needs_zero_gc_pointers:
+ gctypelayout.zero_gc_pointers(result)
+ return result
+ return self.malloc(TYPE, size, 'gc', zero)
+
def setfield(self, obj, fieldname, fieldvalue):
STRUCT = lltype.typeOf(obj).TO
addr = llmemory.cast_ptr_to_adr(obj)
@@ -104,6 +115,7 @@
ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr)
return self.gc.id(ptr)
+
# ____________________________________________________________
class RootLinkedList(object):
Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/test/test_gc.py (original)
+++ pypy/dist/pypy/rpython/memory/test/test_gc.py Wed Nov 7 01:21:18 2007
@@ -368,3 +368,17 @@
class TestGenerationalGC(GCTest):
from pypy.rpython.memory.gc.generation import GenerationGC as GCClass
+
+ def test_coalloc(self):
+ def malloc_a_lot():
+ i = 0
+ while i < 10:
+ i += 1
+ a = [1] * 10
+ j = 0
+ while j < 30:
+ j += 1
+ a.append(j)
+ return 0
+ res = self.interpret(malloc_a_lot, [], backendopt=True, coalloc=True)
+ assert res == 0
Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py
==============================================================================
--- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original)
+++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Wed Nov 7 01:21:18 2007
@@ -13,24 +13,29 @@
INT_SIZE = struct.calcsize("i") # only for estimates
-def rtype(func, inputtypes, specialize=True, gcname='ref', stacklessgc=False):
+def rtype(func, inputtypes, specialize=True, gcname='ref', stacklessgc=False,
+ backendopt=False, **extraconfigopts):
from pypy.translator.translator import TranslationContext
t = TranslationContext()
# XXX XXX XXX mess
t.config.translation.gc = gcname
t.config.translation.stacklessgc = stacklessgc
+ t.config.set(**extraconfigopts)
t.buildannotator().build_types(func, inputtypes)
if specialize:
t.buildrtyper().specialize()
+ if backendopt:
+ from pypy.translator.backendopt.all import backend_optimizations
+ backend_optimizations(t)
if conftest.option.view:
- t.view()
+ t.viewcg()
return t
class GCTest(object):
gcpolicy = None
stacklessgc = False
- def runner(self, f, nbargs=0, statistics=False):
+ def runner(self, f, nbargs=0, statistics=False, **extraconfigopts):
if nbargs == 2:
def entrypoint(args):
x = args[0]
@@ -49,7 +54,8 @@
ARGS = lltype.FixedSizeArray(lltype.Signed, nbargs)
s_args = annmodel.SomePtr(lltype.Ptr(ARGS))
t = rtype(entrypoint, [s_args], gcname=self.gcname,
- stacklessgc=self.stacklessgc)
+ stacklessgc=self.stacklessgc,
+ **extraconfigopts)
cbuild = CStandaloneBuilder(t, entrypoint, config=t.config,
gcpolicy=self.gcpolicy)
db = cbuild.generate_graphs_for_llinterp()
@@ -688,6 +694,21 @@
res = run([3, 0])
assert res == 1
+ def test_coalloc(self):
+ def malloc_a_lot():
+ i = 0
+ while i < 10:
+ i += 1
+ a = [1] * 10
+ j = 0
+ while j < 30:
+ j += 1
+ a.append(j)
+ return 0
+ run, statistics = self.runner(malloc_a_lot, statistics=True,
+ backendopt=True, coalloc=True)
+ run([])
+
class TestStacklessMarkSweepGC(TestMarkSweepGC):
@@ -838,3 +859,4 @@
i += 1
run = self.runner(f, nbargs=0)
run([])
+
Modified: pypy/dist/pypy/rpython/test/test_llinterp.py
==============================================================================
--- pypy/dist/pypy/rpython/test/test_llinterp.py (original)
+++ pypy/dist/pypy/rpython/test/test_llinterp.py Wed Nov 7 01:21:18 2007
@@ -37,8 +37,10 @@
return res
def gengraph(func, argtypes=[], viewbefore='auto', policy=None,
- type_system="lltype", backendopt=False, config=None):
+ type_system="lltype", backendopt=False, config=None,
+ **extraconfigopts):
t = TranslationContext(config=config)
+ t.config.set(**extraconfigopts)
a = t.buildannotator(policy=policy)
timelog("annotating", a.build_types, func, argtypes)
if viewbefore == 'auto':
@@ -69,9 +71,12 @@
def get_interpreter(func, values, view='auto', viewbefore='auto', policy=None,
someobjects=False, type_system="lltype", backendopt=False,
- config=None, malloc_check=True):
- key = (func,) + tuple([typeOf(x) for x in values])+ (someobjects,
- backendopt)
+ config=None, malloc_check=True, **extraconfigopts):
+ extra_key = [(key, value) for key, value in extraconfigopts.iteritems()]
+ extra_key.sort()
+ extra_key = tuple(extra_key)
+ key = ((func,) + tuple([typeOf(x) for x in values]) +
+ (someobjects, backendopt, extra_key))
try:
(t, interp, graph) = _tcache[key]
except KeyError:
@@ -91,7 +96,8 @@
t, typer, graph = gengraph(func, [annotation(x) for x in values],
viewbefore, policy, type_system=type_system,
- backendopt=backendopt, config=config)
+ backendopt=backendopt, config=config,
+ **extraconfigopts)
interp = LLInterpreter(typer, malloc_check=malloc_check)
_tcache[key] = (t, interp, graph)
# keep the cache small
Modified: pypy/dist/pypy/translator/backendopt/all.py
==============================================================================
--- pypy/dist/pypy/translator/backendopt/all.py (original)
+++ pypy/dist/pypy/translator/backendopt/all.py Wed Nov 7 01:21:18 2007
@@ -125,6 +125,10 @@
print "after if-to-switch:"
print_statistics(translator.graphs[0], translator)
+ if config.coalloc and not secondary:
+ from pypy.translator.backendopt import coalloc
+ coalloc.malloc_to_coalloc(translator)
+
for graph in graphs:
checkgraph(graph)
More information about the Pypy-commit
mailing list