[pypy-svn] r21144 - in pypy/dist/pypy: annotation tool translator
arigo at codespeak.net
arigo at codespeak.net
Tue Dec 13 20:01:03 CET 2005
Author: arigo
Date: Tue Dec 13 20:00:58 2005
New Revision: 21144
Modified:
pypy/dist/pypy/annotation/bookkeeper.py
pypy/dist/pypy/annotation/policy.py
pypy/dist/pypy/annotation/specialize.py
pypy/dist/pypy/tool/cache.py
pypy/dist/pypy/translator/annrpython.py
Log:
(pedronis, arigo)
Yet Another Refacoring of The Memo Mess (tm).
There are again two phases, discovery of the possible argument values (at
which point the specializer returns the most precise annotation as a result,
but not a complete graph to compute it); and then, just before fixpoint is
reached, the memo tables built so far are "forced" and become graphs that can
be called. This allows the general N-arguments logic to be written in a
kind-of-sane-but-a-bit-longish way. It would be easy to add support for Bool
arguments now.
Modified: pypy/dist/pypy/annotation/bookkeeper.py
==============================================================================
--- pypy/dist/pypy/annotation/bookkeeper.py (original)
+++ pypy/dist/pypy/annotation/bookkeeper.py Tue Dec 13 20:00:58 2005
@@ -168,6 +168,8 @@
self.pbc_maximal_call_families = UnionFind(description.CallFamily)
self.emulated_pbc_calls = {}
+ self.all_specializations = {} # {FuncDesc: specialization-info}
+ self.pending_specializations = [] # list of callbacks
self.needs_hash_support = {}
self.needs_generic_instantiate = {}
Modified: pypy/dist/pypy/annotation/policy.py
==============================================================================
--- pypy/dist/pypy/annotation/policy.py (original)
+++ pypy/dist/pypy/annotation/policy.py Tue Dec 13 20:00:58 2005
@@ -1,7 +1,7 @@
# base annotation policy for overrides and specialization
from pypy.annotation.specialize import default_specialize as default
from pypy.annotation.specialize import argtype, argvalue, arglistitemtype
-from pypy.annotation.specialize import memo, methodmemo
+from pypy.annotation.specialize import memo
# for some reason, model must be imported first,
# or we create a cycle.
from pypy.annotation import model as annmodel
@@ -20,8 +20,11 @@
def no_specialization(pol, funcdesc, args_s):
return funcdesc.cachedgraph(None)
- def compute_at_fixpoint(pol, annotator):
- annotator.bookkeeper.compute_at_fixpoint()
+ def no_more_blocks_to_annotate(pol, annotator):
+ # hint to all pending specializers that we are done
+ for callback in annotator.bookkeeper.pending_specializations:
+ callback()
+ del annotator.bookkeeper.pending_specializations[:]
class AnnotatorPolicy(BasicAnnotatorPolicy):
@@ -52,7 +55,6 @@
default_specialize = staticmethod(default)
specialize__memo = staticmethod(memo)
- specialize__methodmemo = staticmethod(methodmemo)
specialize__arg0 = staticmethod(argvalue(0))
specialize__argtype0 = staticmethod(argtype(0))
specialize__arglistitemtype0 = staticmethod(arglistitemtype(0))
Modified: pypy/dist/pypy/annotation/specialize.py
==============================================================================
--- pypy/dist/pypy/annotation/specialize.py (original)
+++ pypy/dist/pypy/annotation/specialize.py Tue Dec 13 20:00:58 2005
@@ -1,6 +1,9 @@
# specialization support
import types
+import py
from pypy.tool.uid import uid
+from pypy.tool.sourcetools import func_with_new_name
+from pypy.tool.algo.unionfind import UnionFind
from pypy.objspace.flow.model import Block, Link, Variable, SpaceOperation
from pypy.objspace.flow.model import Constant, checkgraph
@@ -54,126 +57,298 @@
# ____________________________________________________________________________
# specializations
+class MemoTable:
+ def __init__(self, funcdesc, args, value):
+ self.funcdesc = funcdesc
+ self.table = {args: value}
+ self.graph = None
+
+ def update(self, other):
+ self.table.update(other.table)
+ self.graph = None # just in case
+
+ fieldnamecounter = 0
+
+ def getuniquefieldname(self, descs):
+ name = self.funcdesc.name
+ fieldname = 'memofield_%s_%d' % (name, MemoTable.fieldnamecounter)
+ MemoTable.fieldnamecounter += 1
+ # look for name clashes
+ for desc in descs:
+ try:
+ desc.read_attribute(fieldname)
+ except AttributeError:
+ pass # no clash
+ else:
+ # clash! try again...
+ return self.getuniquefieldname(descs)
+ else:
+ return fieldname
+
+ def finish(self):
+ from pypy.annotation.model import unionof
+ # list of which argument positions can take more than one value
+ example_args, example_value = self.table.iteritems().next()
+ nbargs = len(example_args)
+ # list of sets of possible argument values -- one set per argument index
+ sets = [{} for i in range(nbargs)]
+ for args in self.table:
+ for i in range(nbargs):
+ sets[i][args[i]] = True
+
+ bookkeeper = self.funcdesc.bookkeeper
+ annotator = bookkeeper.annotator
+ name = self.funcdesc.name
+ argnames = ['a%d' % i for i in range(nbargs)]
+
+ def make_helper(firstarg, expr, miniglobals):
+ source = """
+ def f(%s):
+ return %s
+ """ % (', '.join(argnames[firstarg:]), expr)
+ exec py.code.Source(source).compile() in miniglobals
+ f = miniglobals['f']
+ return func_with_new_name(f, 'memo_%s_%d' % (name, firstarg))
+
+ def make_constant_subhelper(firstarg, result):
+ # make a function that just returns the constant answer 'result'
+ f = make_helper(firstarg, 'result', {'result': result})
+ f.constant_result = result
+ return f
+
+ def make_subhelper(args_so_far=()):
+ firstarg = len(args_so_far)
+ if firstarg == nbargs:
+ # no argument left, return the known result
+ # (or a dummy value if none corresponds exactly)
+ result = self.table.get(args_so_far, example_value)
+ return make_constant_subhelper(firstarg, result)
+ else:
+ nextargvalues = list(sets[len(args_so_far)])
+ nextfns = [make_subhelper(args_so_far + (arg,))
+ for arg in nextargvalues]
+ # do all graphs return a constant?
+ try:
+ constants = [fn.constant_result for fn in nextfns]
+ except AttributeError:
+ constants = None # one of the 'fn' has no constant_result
+
+ # is there actually only one possible value for the current arg?
+ if len(nextargvalues) == 1:
+ if constants: # is the result a constant?
+ result = constants[0]
+ return make_constant_subhelper(firstarg, result)
+ else:
+ # ignore the first argument and just call the subhelper
+ expr = 'subhelper(%s)' % (
+ ', '.join(argnames[firstarg+1:]),)
+ return make_helper(firstarg, expr,
+ {'subhelper': nextfns[0]})
+ else:
+ descs = [bookkeeper.getdesc(pbc) for pbc in nextargvalues]
+ fieldname = self.getuniquefieldname(descs)
+ expr = 'getattr(%s, %r)' % (argnames[firstarg],
+ fieldname)
+ if constants:
+ # instead of calling these subhelpers indirectly,
+ # we store what they would return directly in the
+ # pbc memo fields
+ store = constants
+ else:
+ store = nextfns
+ # call the result of the getattr()
+ expr += '(%s)' % (', '.join(argnames[firstarg+1:]),)
+
+ # store the memo field values
+ for desc, value_to_store in zip(descs, store):
+ desc.create_new_attribute(fieldname, value_to_store)
+
+ return make_helper(firstarg, expr, {})
+
+ entrypoint = make_subhelper(args_so_far = ())
+ self.graph = annotator.translator.buildflowgraph(entrypoint)
+
+ # schedule this new graph for being annotated
+ args_s = []
+ for set in sets:
+ values_s = [bookkeeper.immutablevalue(x) for x in set]
+ args_s.append(unionof(*values_s))
+ annotator.addpendinggraph(self.graph, args_s)
+
+
def memo(funcdesc, arglist_s):
- """NOT_RPYTHON"""
- from pypy.annotation.model import SomePBC, SomeImpossibleValue
+ from pypy.annotation.model import SomePBC, SomeImpossibleValue, unionof
# call the function now, and collect possible results
+ argvalues = []
for s in arglist_s:
if not isinstance(s, SomePBC):
if isinstance(s, SomeImpossibleValue):
return s # we will probably get more possible args later
raise Exception("memo call: argument must be a class or a frozen "
"PBC, got %r" % (s,))
- if len(arglist_s) != 1:
- raise Exception("memo call: only 1 argument functions supported"
- " at the moment (%r)" % (funcdesc,))
- s, = arglist_s
- from pypy.annotation.model import SomeImpossibleValue
- func = funcdesc.pyobj
- if func is None:
- raise Exception("memo call: no Python function object to call (%r)" %
- (funcdesc,))
- return memo1(funcdesc, func, s)
-
-# XXX OBSCURE to support methodmemo()... needs to find something more
-# reasonable :-(
-KEY_NUMBERS = {}
-
-def memo1(funcdesc, func, s, key='memo1'):
- from pypy.annotation.model import SomeImpossibleValue
- # compute the concrete results and store them directly on the descs,
- # using a strange attribute name
- num = KEY_NUMBERS.setdefault(key, len(KEY_NUMBERS))
- attrname = '$memo%d_%d_%s' % (uid(funcdesc), num, funcdesc.name)
- for desc in s.descriptions:
- s_result = desc.s_read_attribute(attrname)
- if isinstance(s_result, SomeImpossibleValue):
- # first time we see this 'desc'
+ assert not s.can_be_None, "memo call: arguments must never be None"
+ values = []
+ for desc in s.descriptions:
if desc.pyobj is None:
raise Exception("memo call with a class or PBC that has no "
"corresponding Python object (%r)" % (desc,))
- result = func(desc.pyobj)
- desc.create_new_attribute(attrname, result)
- # get or build the graph of the function that reads this strange attr
- def memoized(x, y=None):
- return getattr(x, attrname)
- def builder(translator, func):
- return translator.buildflowgraph(memoized) # instead of 'func'
- return funcdesc.cachedgraph(key, alt_name='memo_%s' % funcdesc.name,
- builder=builder)
-
-def methodmemo(funcdesc, arglist_s):
- """NOT_RPYTHON"""
- from pypy.annotation.model import SomePBC, SomeImpossibleValue
- # call the function now, and collect possible results
- for s in arglist_s:
- if not isinstance(s, SomePBC):
- if isinstance(s, SomeImpossibleValue):
- return s # we will probably get more possible args later
- raise Exception("method-memo call: argument must be a class or"
- " a frozen PBC, got %r" % (s,))
- if len(arglist_s) != 2:
- raise Exception("method-memo call: expected 2 arguments function"
- " at the moment (%r)" % (funcdesc,))
- from pypy.annotation.model import SomeImpossibleValue
- from pypy.annotation.description import FrozenDesc
- func = funcdesc.pyobj
- if func is None:
- raise Exception("method-memo call: no Python function object to call"
- " (%r)" % (funcdesc,))
- # compute the concrete results and store them directly on the descs,
- # using a strange attribute name. The goal is to store in the pbcs of
- # 's1' under the common 'attrname' a reader function; each reader function
- # will read a field 'attrname2' from the pbcs of 's2', where 'attrname2'
- # differs for each pbc of 's1'. This is all specialized also
- # considering the type of s1 to support return value
- # polymorphism.
- s1, s2 = arglist_s
- s1_type = s1.knowntype
- if s2.is_constant():
- return memo1(funcdesc, lambda val1: func(val1, s2.const),
- s1, ('memo1of2', s1_type, Constant(s2.const)))
- memosig = "%d_%d_%s" % (uid(funcdesc), uid(s1_type), funcdesc.name)
-
- attrname = '$memoreader%s' % memosig
- for desc1 in s1.descriptions:
- attrname2 = '$memofield%d_%s' % (uid(desc1), memosig)
- s_reader = desc1.s_read_attribute(attrname)
- if isinstance(s_reader, SomeImpossibleValue):
- # first time we see this 'desc1': sanity-check 'desc1' and
- # create its reader function
- assert isinstance(desc1, FrozenDesc), (
- "XXX not implemented: memo call with a class as first arg")
- if desc1.pyobj is None:
- raise Exception("method-memo call with a class or PBC"
- " that has no "
- "corresponding Python object (%r)" % (desc1,))
- def reader(y, attrname2=attrname2):
- return getattr(y, attrname2)
- desc1.create_new_attribute(attrname, reader)
- for desc2 in s2.descriptions:
- s_result = desc2.s_read_attribute(attrname2)
- if isinstance(s_result, SomeImpossibleValue):
- # first time we see this 'desc1+desc2' combination
- if desc2.pyobj is None:
- raise Exception("method-memo call with a class or PBC"
- " that has no "
- "corresponding Python object (%r)" % (desc2,))
- # concrete call, to get the concrete result
- result = func(desc1.pyobj, desc2.pyobj)
- #print 'func(%s, %s) -> %s' % (desc1.pyobj, desc2.pyobj, result)
- #print 'goes into %s.%s'% (desc2,attrname2)
- #print 'with reader %s.%s'% (desc1,attrname)
- desc2.create_new_attribute(attrname2, result)
- # get or build the graph of the function that reads this indirect
- # settings of attributes
- def memoized(x, y):
- reader_fn = getattr(x, attrname)
- return reader_fn(y)
- def builder(translator, func):
- return translator.buildflowgraph(memoized) # instead of 'func'
- return funcdesc.cachedgraph(s1_type, alt_name='memo_%s' % funcdesc.name,
- builder=builder)
+ values.append(desc.pyobj)
+ argvalues.append(values)
+ # the list of all possible tuples of arguments to give to the memo function
+ possiblevalues = cartesian_product(argvalues)
+
+ # a MemoTable factory -- one MemoTable per family of arguments that can
+ # be called together, merged via a UnionFind.
+ bookkeeper = funcdesc.bookkeeper
+ try:
+ memotables = bookkeeper.all_specializations[funcdesc]
+ except KeyError:
+ func = funcdesc.pyobj
+ if func is None:
+ raise Exception("memo call: no Python function object to call "
+ "(%r)" % (funcdesc,))
+
+ def compute_one_result(args):
+ value = func(*args)
+ return MemoTable(funcdesc, args, value)
+
+ def finish():
+ for memotable in memotables.infos():
+ memotable.finish()
+
+ memotables = UnionFind(compute_one_result)
+ bookkeeper.all_specializations[funcdesc] = memotables
+ bookkeeper.pending_specializations.append(finish)
+
+ # merge the MemoTables for the individual argument combinations
+ firstvalues = possiblevalues.next()
+ _, _, memotable = memotables.find(firstvalues)
+ for values in possiblevalues:
+ _, _, memotable = memotables.union(firstvalues, values)
+
+ if memotable.graph is not None:
+ return memotable.graph # if already computed
+ else:
+ # otherwise, for now, return the union of each possible result
+ return unionof(*[bookkeeper.immutablevalue(v)
+ for v in memotable.table.values()])
+
+def cartesian_product(lstlst):
+ if not lstlst:
+ yield ()
+ return
+ for tuple_tail in cartesian_product(lstlst[1:]):
+ for value in lstlst[0]:
+ yield (value,) + tuple_tail
+
+## """NOT_RPYTHON"""
+## if len(arglist_s) != 1:
+## raise Exception("memo call: only 1 argument functions supported"
+## " at the moment (%r)" % (funcdesc,))
+## s, = arglist_s
+## from pypy.annotation.model import SomeImpossibleValue
+## return memo1(funcdesc, func, s)
+
+### XXX OBSCURE to support methodmemo()... needs to find something more
+### reasonable :-(
+##KEY_NUMBERS = {}
+
+##def memo1(funcdesc, func, s, key='memo1'):
+## from pypy.annotation.model import SomeImpossibleValue
+## # compute the concrete results and store them directly on the descs,
+## # using a strange attribute name
+## num = KEY_NUMBERS.setdefault(key, len(KEY_NUMBERS))
+## attrname = '$memo%d_%d_%s' % (uid(funcdesc), num, funcdesc.name)
+## for desc in s.descriptions:
+## s_result = desc.s_read_attribute(attrname)
+## if isinstance(s_result, SomeImpossibleValue):
+## # first time we see this 'desc'
+## if desc.pyobj is None:
+## raise Exception("memo call with a class or PBC that has no "
+## "corresponding Python object (%r)" % (desc,))
+## result = func(desc.pyobj)
+## desc.create_new_attribute(attrname, result)
+## # get or build the graph of the function that reads this strange attr
+## def memoized(x, y=None):
+## return getattr(x, attrname)
+## def builder(translator, func):
+## return translator.buildflowgraph(memoized) # instead of 'func'
+## return funcdesc.cachedgraph(key, alt_name='memo_%s' % funcdesc.name,
+## builder=builder)
+
+##def methodmemo(funcdesc, arglist_s):
+## """NOT_RPYTHON"""
+## from pypy.annotation.model import SomePBC, SomeImpossibleValue
+## # call the function now, and collect possible results
+## for s in arglist_s:
+## if not isinstance(s, SomePBC):
+## if isinstance(s, SomeImpossibleValue):
+## return s # we will probably get more possible args later
+## raise Exception("method-memo call: argument must be a class or"
+## " a frozen PBC, got %r" % (s,))
+## if len(arglist_s) != 2:
+## raise Exception("method-memo call: expected 2 arguments function"
+## " at the moment (%r)" % (funcdesc,))
+## from pypy.annotation.model import SomeImpossibleValue
+## from pypy.annotation.description import FrozenDesc
+## func = funcdesc.pyobj
+## if func is None:
+## raise Exception("method-memo call: no Python function object to call"
+## " (%r)" % (funcdesc,))
+## # compute the concrete results and store them directly on the descs,
+## # using a strange attribute name. The goal is to store in the pbcs of
+## # 's1' under the common 'attrname' a reader function; each reader function
+## # will read a field 'attrname2' from the pbcs of 's2', where 'attrname2'
+## # differs for each pbc of 's1'. This is all specialized also
+## # considering the type of s1 to support return value
+## # polymorphism.
+## s1, s2 = arglist_s
+## s1_type = s1.knowntype
+## if s2.is_constant():
+## return memo1(funcdesc, lambda val1: func(val1, s2.const),
+## s1, ('memo1of2', s1_type, Constant(s2.const)))
+## memosig = "%d_%d_%s" % (uid(funcdesc), uid(s1_type), funcdesc.name)
+
+## attrname = '$memoreader%s' % memosig
+## for desc1 in s1.descriptions:
+## attrname2 = '$memofield%d_%s' % (uid(desc1), memosig)
+## s_reader = desc1.s_read_attribute(attrname)
+## if isinstance(s_reader, SomeImpossibleValue):
+## # first time we see this 'desc1': sanity-check 'desc1' and
+## # create its reader function
+## assert isinstance(desc1, FrozenDesc), (
+## "XXX not implemented: memo call with a class as first arg")
+## if desc1.pyobj is None:
+## raise Exception("method-memo call with a class or PBC"
+## " that has no "
+## "corresponding Python object (%r)" % (desc1,))
+## def reader(y, attrname2=attrname2):
+## return getattr(y, attrname2)
+## desc1.create_new_attribute(attrname, reader)
+## for desc2 in s2.descriptions:
+## s_result = desc2.s_read_attribute(attrname2)
+## if isinstance(s_result, SomeImpossibleValue):
+## # first time we see this 'desc1+desc2' combination
+## if desc2.pyobj is None:
+## raise Exception("method-memo call with a class or PBC"
+## " that has no "
+## "corresponding Python object (%r)" % (desc2,))
+## # concrete call, to get the concrete result
+## result = func(desc1.pyobj, desc2.pyobj)
+## #print 'func(%s, %s) -> %s' % (desc1.pyobj, desc2.pyobj, result)
+## #print 'goes into %s.%s'% (desc2,attrname2)
+## #print 'with reader %s.%s'% (desc1,attrname)
+## desc2.create_new_attribute(attrname2, result)
+## # get or build the graph of the function that reads this indirect
+## # settings of attributes
+## def memoized(x, y):
+## reader_fn = getattr(x, attrname)
+## return reader_fn(y)
+## def builder(translator, func):
+## return translator.buildflowgraph(memoized) # instead of 'func'
+## return funcdesc.cachedgraph(s1_type, alt_name='memo_%s' % funcdesc.name,
+## builder=builder)
+
def argvalue(i):
def specialize_argvalue(funcdesc, args_s):
Modified: pypy/dist/pypy/tool/cache.py
==============================================================================
--- pypy/dist/pypy/tool/cache.py (original)
+++ pypy/dist/pypy/tool/cache.py Tue Dec 13 20:00:58 2005
@@ -37,7 +37,7 @@
result = self._build(key)
self.content[key] = result
return result
- getorbuild._annspecialcase_ = "specialize:methodmemo"
+ getorbuild._annspecialcase_ = "specialize:memo"
def _freeze_(self):
# needs to be SomePBC, but otherwise we can't really freeze the
Modified: pypy/dist/pypy/translator/annrpython.py
==============================================================================
--- pypy/dist/pypy/translator/annrpython.py (original)
+++ pypy/dist/pypy/translator/annrpython.py Tue Dec 13 20:00:58 2005
@@ -94,7 +94,6 @@
def build_graph_types(self, flowgraph, inputcells):
checkgraph(flowgraph)
- self._register_returnvar(flowgraph)
nbarg = len(flowgraph.getargs())
if len(inputcells) != nbarg:
@@ -102,7 +101,7 @@
flowgraph, nbarg, len(inputcells)))
# register the entry point
- self.addpendingblock(flowgraph, flowgraph.startblock, inputcells)
+ self.addpendinggraph(flowgraph, inputcells)
# recursively proceed until no more pending block is left
self.complete()
return self.binding(flowgraph.getreturnvar(), extquery=True)
@@ -128,6 +127,10 @@
#___ medium-level interface ____________________________
+ def addpendinggraph(self, flowgraph, inputcells):
+ self._register_returnvar(flowgraph)
+ self.addpendingblock(flowgraph, flowgraph.startblock, inputcells)
+
def addpendingblock(self, graph, block, cells, called_from_graph=None):
"""Register an entry point into block with the given input cells."""
assert not self.frozen
@@ -142,9 +145,13 @@
def complete(self):
"""Process pending blocks until none is left."""
- while self.pendingblocks:
- block, graph = self.pendingblocks.popitem()
- self.processblock(graph, block)
+ while True:
+ while self.pendingblocks:
+ block, graph = self.pendingblocks.popitem()
+ self.processblock(graph, block)
+ self.policy.no_more_blocks_to_annotate(self)
+ if not self.pendingblocks:
+ break # finished
if False in self.annotated.values():
if annmodel.DEBUG:
for block in self.annotated:
@@ -175,7 +182,7 @@
if v not in self.bindings:
self.setbinding(v, annmodel.SomeImpossibleValue())
# policy-dependent computation
- self.policy.compute_at_fixpoint(self)
+ self.bookkeeper.compute_at_fixpoint()
def binding(self, arg, extquery=False):
"Gives the SomeValue corresponding to the given Variable or Constant."
More information about the Pypy-commit
mailing list