[pypy-svn] r18833 - in pypy/dist/pypy/translator: c locality
tismer at codespeak.net
tismer at codespeak.net
Sat Oct 22 01:32:52 CEST 2005
Author: tismer
Date: Sat Oct 22 01:32:49 2005
New Revision: 18833
Added:
pypy/dist/pypy/translator/locality/support.py (contents, props changed)
Modified:
pypy/dist/pypy/translator/c/genc.py
pypy/dist/pypy/translator/locality/calltree.py
pypy/dist/pypy/translator/locality/projection.py
pypy/dist/pypy/translator/locality/simulation.py
Log:
current state of locality development.
There were more problems than expected.
Things solved here:
* replaced debug prints with logging
* kept the interface between simulation andflowgraphs to the minimum.
Simulation/projection do not need to know anything about PyPy at all.
* support for disjoint graphs being optimized independently
* filtering of external functions, which appeared as infinite recursions
* working in the projection space, only. Optimizing in multi dimensions
is still supported by the code, but it turned out that this is overkill
for the problem to be solved.
* optimization of the lonelyness parameter works fine. But there is a problem
with the starting conditions. I had cases where I started in a local minimum,
and lonelyness was going up a bit, before showing monotonic decrease.
Introduced a random initialization, which made this vanish.
This is anyway no guarantee, but I think this is good enough.
Open issues
-----------
* tested with small targets only, yet
* recursion is still weak. Need a way to isolate them, or they would
dominate the simulation (although that might be no problem at all, have to play more)
* convergence is achieved on the few tested graphs, only.
The contraction algorithm seems to be very valid in general.
I'm still researching about correct parameters on how much
of the correction vectors to allow to be used. Several estimators
failed so far and are not checked in.
* functions held in variables are not captured, yet. This will be tried
after a positive result.
* the simulation approach seems to be quite good. I planned for a real
benchmark, but meanwhile I like the simulation idea almost better.
It might make sense to provide per-function calling probabilities
by some heuristics; maybe we could also add properties to functions
or even modules.
* an initial test on stand-alone PyPy is still not done.
Will hopefully be done, tomorrow. I'm a bit of fearful
if this approach is useful at all. That's why I put so much
preliminary work into it. If it fails, it should not be
a mistake of my implementation.
Note: I'm again over time budget with this, and tomorrow will be the last
day spent on it. I hereby put a 5 % minimum speed gain on the simulated
thing. If this is not met, the whole locality folder gets into the attic.
I think I have spent four full days on this, now.
Modified: pypy/dist/pypy/translator/c/genc.py
==============================================================================
--- pypy/dist/pypy/translator/c/genc.py (original)
+++ pypy/dist/pypy/translator/c/genc.py Sat Oct 22 01:32:49 2005
@@ -10,6 +10,7 @@
from pypy.rpython.rmodel import getfunctionptr
from pypy.rpython import lltype
from pypy.tool.udir import udir
+from pypy.translator.locality.calltree import CallTree
class CBuilder(object):
c_source_filename = None
@@ -175,6 +176,9 @@
self.funcnodes = funcnodes
self.othernodes = othernodes
self.path = path
+ return # the below is under development
+ graph = CallTree(self.funcnodes, self.database)
+ graph.simulate()
def uniquecname(self, name):
assert name.endswith('.c')
Modified: pypy/dist/pypy/translator/locality/calltree.py
==============================================================================
--- pypy/dist/pypy/translator/locality/calltree.py (original)
+++ pypy/dist/pypy/translator/locality/calltree.py Sat Oct 22 01:32:49 2005
@@ -28,10 +28,24 @@
"""
from pypy.objspace.flow.model import Variable, Constant
+from pypy.translator.locality.support import log
+from pypy.translator.locality.simulation import SimNode, SimGraph
+from pypy.translator.locality.projection import SpaceNode, SpaceGraph
+
+
+class FlowSimNode(SimNode):
+ def _get_name(self, func):
+ return func.name
+
+ def _find_callee_names(self):
+ calls = self.sim.clientdata[self.func]
+ return [func.name for func in calls]
+
class CallTree:
- def __init__(self, funcnodes):
+ def __init__(self, funcnodes, database):
self.nodes = funcnodes
+ self.database = database
self.graphs2nodes = self._build_graph2nodes()
self.calls = {}
for node in self.nodes:
@@ -46,6 +60,10 @@
def find_callees(self, node):
graph = node.obj.graph
res = []
+ if node.obj._callable in self.database.externalfuncs:
+ s = "skipped external function %s" % node.obj._callable.__name__
+ log.calltree.findCallees(s)
+ return res
for block in graph.iterblocks():
for op in block.operations:
if op.opname == 'direct_call':
@@ -57,10 +75,20 @@
try:
callednode = self.graphs2nodes[graph]
except KeyError:
- print "No node found for graph %s" % graph.name
+ s = "No node found for graph %s" % graph.name
+ log.calltree.findCallees(s)
continue
else:
res.append(callednode)
else:
- print "Node %s calls Variable %s" % (node, fnarg)
+ s = "Node %s calls Variable %s" % (node.name, fnarg)
+ log.calltree.findCallees(s)
return res
+
+ def simulate(self):
+ log.calltree('building CallTree...')
+ sim = SimGraph(self.nodes, FlowSimNode, self.calls)
+ log.calltree('simulating...')
+ sim.sim_all(0.9, 50)
+ import pdb
+ pdb.set_trace()
Modified: pypy/dist/pypy/translator/locality/projection.py
==============================================================================
--- pypy/dist/pypy/translator/locality/projection.py (original)
+++ pypy/dist/pypy/translator/locality/projection.py Sat Oct 22 01:32:49 2005
@@ -25,49 +25,279 @@
The weighted distance between two nodes
"""
-from pypy.translator.locality.simulation import DemoNode, DemoSim
from math import sqrt
+import random
+
+
+def zipextend(v1, v2):
+ adjust = len(v2) - len(v1)
+ if adjust:
+ if adjust > 0:
+ v1 += [0.0] * adjust
+ else:
+ v2 = v2[:] + [0.0] * -adjust
+ return zip(v1, v2)
+
+
+class Vector:
+ # a really dumb little helper class
+
+ def __init__(self, seq=None):
+ self.coords = list(seq or [])
+
+ def __inplace_add__(self, other):
+ self.coords = [p + q for p, q in zipextend(self.coords, other.coords)]
+
+ def __inplace_sub__(self, other):
+ self.coords = [p - q for p, q in zipextend(self.coords, other.coords)]
+
+ def __inplace_mul__(self, scalar):
+ if isinstance(scalar, Vector):
+ # dot product. zip is correct here, zero cancels.
+ other = scalar
+ self.coords = [p * q for p, q in zip(self.coords, other.coords)]
+ else:
+ # scalar product
+ self.coords = [p * scalar for p in self.coords]
+
+ def __inplace_div__(self, scalar):
+ self.coords = [p / scalar for p in self.coords]
+
+ def __add__(self, other):
+ vec = Vector(self.coords)
+ vec.__inplace_add__(other)
+ return vec
+
+ def __sub__(self, other):
+ vec = Vector(self.coords)
+ vec.__inplace_sub__(other)
+ return vec
+
+ def __mul__(self, scalar):
+ vec = Vector(self.coords)
+ vec.__inplace_mul__(scalar)
+ return vec
+
+ def __div__(self, scalar):
+ vec = Vector(self.coords)
+ vec.__inplace_div__(scalar)
+ return vec
+
+ def __neg__(self):
+ return Vector([-k for k in self.coords])
+
+ def norm2(self):
+ return sqrt(sum([k * k for k in self.coords]))
+
+ def getdim(self):
+ return len(self.coords)
+
+ # access to coordinates
+ def __getitem__(self, idx):
+ return self.coords[idx]
+
+ def __setitem__(self, idx, value):
+ self.coords[idx] = value
+
+ def __iter__(self):
+ return iter(self.coords)
+
+ def __repr__(self):
+ return 'Vector(%r)' % self.coords
class SpaceNode:
def __init__(self, node):
self.func = node.func
self.name = node.name
- def setup(self, relations, weights):
+ def setup(self, relations, weights, initpos):
self.relations = relations
self.weights = weights
- self.position = weights[:] # just anything to start with
+ self.position = initpos
def distance(self, other):
# using the nice property of zip to give the minimum length
- dist = 0.0
- for x1, x2 in zip(self.position, other.position):
- d = x2 - x1
- dist += d * d
- return sqrt(dist)
+ return (other.position - self.position).norm2()
+
+ def scale(self, factor):
+ self.position *= factor
+
+ def shift(self, delta):
+ self.position += delta
+
+ def shiftx(self, deltax):
+ self.position[0] += deltax
def lonelyness(self):
- # get the sum of weighted distances
- lonely = 0.0
+ # get the square norm of weighted distances
+ lonely = []
for weight, relative in zip(self.weights, self.relations):
- lonely += weight * self.distance(relative)
- return lonely
+ lonely.append(weight * self.distance(relative))
+ return Vector(lonely).norm2()
+
+ def forcevector(self):
+ # weighted implementation of the "rubber2" algorithm,
+ # from "PolyTop", (C) Christian Tismer / Gerhard G. Thomas 1992
+ vec = self.position * 0.0
+ for w, rel in zip(self.weights, self.relations):
+ tmp = rel.position - self.position
+ lng = tmp.norm2()
+ tmp *= w * lng
+ vec += tmp
+ # this is a little faster than
+ # vec += (rel.position - self.position) * w * self.distance(rel)
+ return vec
- def corrvector(self):
- pass # XXX continue here
class SpaceGraph:
+ random = random.Random(42).random
+
def __init__(self, simgraph):
+ self.nodes = []
+ self.addgraph(simgraph)
+ self.lastdim = 0 # calculated by normalize
+ self.subgraphs = []
+
+ def addgraph(self, simgraph):
mapping = {}
for simnode in simgraph.nodes:
mapping[simnode] = SpaceNode(simnode)
- self.nodes = [mapping[simnode] for simnode in simgraph.nodes]
+ i = len(self.nodes)
+ self.nodes += [mapping[simnode] for simnode in simgraph.nodes]
for simnode in simgraph.nodes:
relations, weights = simnode.get_relations()
relations = [mapping[rel] for rel in relations]
node = mapping[simnode]
- node.setup(relations, weights)
+ # extreme simplification:
+ # use just one dimension
+ # scamble as much as possible to avoid
+ # starting in a local minimum
+ #node.setup(relations, weights, Vector([i]))
+ node.setup(relations, weights, Vector([self.random()]))
+ i += 1
+ self.subgraphs = []
+
+ def xminmax(self, nodes=None):
+ nodes = nodes or self.nodes
+ xaxis = [node.position[0] for node in nodes]
+ xmin = min(xaxis)
+ xmax = max(xaxis)
+ return float(xmin), float(xmax)
+
+ def compute_subgraphs(self):
+ nodes = {}
+ for node in self.nodes:
+ nodes[node] = node
+ self.subgraphs = []
+ while nodes:
+ for node in nodes:
+ break
+ todo = [node]
+ del nodes[node]
+ for node in todo:
+ for rel in node.relations:
+ if rel in nodes:
+ del nodes[rel]
+ todo.append(rel)
+ self.subgraphs.append(todo)
+
+ def normalize(self):
+ # identify disjoint subgraphs.
+ # for every subgraph:
+ # move the graph center to zero
+ # scale the graph to make the x-axis as long as the number of nodes.
+ # shift all graphs to be in disjoint intervals on the x-axis.
+ if not self.subgraphs:
+ self.compute_subgraphs()
+ def distort(nodes):
+ # stretch collapsed x-axis
+ for i, node in enumerate(nodes):
+ node.position[0] = i
+ return nodes
+ def norm_subgraph(nodes, start):
+ # normalize a subgraph, return the dimensionality as side effect
+ xmin, xmax = self.xminmax(nodes)
+ xwidth = xmax - xmin
+ if not xwidth: # degenerated
+ return norm_subgraph(distort(nodes))
+ factor = (len(nodes) - 1) / xwidth
+ mean = Vector()
+ for node in nodes:
+ mean += node.position
+ mean /= len(nodes)
+ shift = -mean
+ dim = shift.getdim()
+ for node in nodes:
+ node.shift(shift)
+ node.scale(factor)
+ shiftx = start - (xmin + shift[0]) * factor
+ for node in nodes:
+ node.shiftx(shiftx)
+ return dim
+
+ start = 0.0
+ dim = 0
+ for nodes in self.subgraphs:
+ dim = max(dim, norm_subgraph(nodes, start))
+ start += len(nodes)
+ self.lastdim = dim
+
+ def do_correction(self, korr=0.13):
+ forcevecs = [node.forcevector() for node in self.nodes]
+ corrx = [vec[0] for vec in forcevecs]
+ maxcorr = abs(max(corrx))
+ xmin, xmax = self.xminmax()
+ xwidth = xmax - xmin
+ scale = xwidth / maxcorr
+ scale = scale * korr
+ for node, forcevec in zip(self.nodes, forcevecs):
+ corrvec = forcevec * scale
+ node.shift(corrvec)
+
+ def squeeze_dim(self):
+ scale = []
+ ndim = self.lastdim
+ for i in range(ndim):
+ scale.append( 1.01 ** -i )
+ scale = Vector(scale)
+ for node in self.nodes:
+ node.scale(scale)
+
+ def lonelyness2(self):
+ # square norm of lonelynesses
+ lonely = []
+ for node in self.nodes:
+ lonely.append(node.lonelyness())
+ return Vector(lonely).norm2()
+
+ def lonelyness(self):
+ # square norm of lonelynesses
+ lonely = 0.0
+ for node in self.nodes:
+ lonely += node.lonelyness()
+ return lonely / len(self.nodes)
+
+ def order(self):
+ sorter = [(node.position[0], node) for node in self.nodes]
+ sorter.sort()
+ return [node for x, node in sorter]
+
+ def display(self):
+ for node in self.order():
+ print node.name, node.lonelyness(), node.position
if __name__ == '__main__':
- from pypy.translator.locality.simulation import test
+ from pypy.translator.locality.simulation import SimGraph
+ def test():
+ def a(): b()
+ def b(): c()
+ def c(): d()
+ def d(): e()
+ def e(): f()
+ def f(): a()
+ sim = DemoSim([a, b, c, d, e, f])
+ sim.sim_all(0.9, 50)
+ return sim
g = SpaceGraph(test())
+ g.addgraph(test())
+ g.addgraph(test())
Modified: pypy/dist/pypy/translator/locality/simulation.py
==============================================================================
--- pypy/dist/pypy/translator/locality/simulation.py (original)
+++ pypy/dist/pypy/translator/locality/simulation.py Sat Oct 22 01:32:49 2005
@@ -7,20 +7,20 @@
in the call-graph of a program, to gather information
about frequencies of transitions between functions.
-The following DemoNode/DemoSim classes show an example of the
+The following SimNode/SimGraph classes show an example of the
simulation performed. They can be subclassed to connect them
to client structures like flowgraphs.
-- DemoSim.run was used to get an obviously correct reference implementation.
+- SimGraph.run was used to get an obviously correct reference implementation.
-- DemoSim.sim_all simulates the calls of the run method. The results are
+- SimGraph.sim_all simulates the calls of the run method. The results are
exactly the same, although the computation time ir orders of magnitudes
- smaller, and the DemoSim.simulate method is able to handle recursions
+ smaller, and the SimGraph.simulate method is able to handle recursions
and function call probabilities which are fractions.
"""
-class DemoNode:
+class SimNode:
def __init__(self, sim, func):
self.sim = sim
self.func = func
@@ -84,11 +84,13 @@
freqs, nodes = zip(*ret)
return nodes, [-freq for freq in freqs]
-class DemoSim:
- def __init__(self, funcnodes, nodefactory=DemoNode):
+
+class SimGraph:
+ def __init__(self, funcnodes, nodefactory=SimNode, clientdata=None):
self.nodes = []
self.transitions = {}
self.pending = {}
+ self.clientdata = clientdata
name2node = {}
for func in funcnodes:
@@ -127,15 +129,16 @@
self.transitions[key] = 0
for node in self.nodes:
node.clear()
+ self.pending.clear()
def display(self):
d = {'w': max(self._names_width, 6) }
- print '%%%(w)ds %%%(w)ds repetition' % d % ('caller', 'callee')
+ print '%%%(w)ds %%%(w)gs repetition' % d % ('caller', 'callee')
for caller, callee, reps in self.get_state():
- print '%%%(w)ds %%%(w)ds %%6d' % d % (caller, callee, reps)
- print '%%%(w)ds calls' % d % 'node'
+ print '%%%(w)ds %%%(w)gs %%6g' % d % (caller, callee, reps)
+ print '%%%(w)gs calls' % d % 'node'
for node in self.nodes:
- print '%%%(w)ds %%6d' % d % (node.name, node.calls)
+ print '%%%(w)gs %%6g' % d % (node.name, node.calls)
def get_state(self):
lst = []
@@ -166,11 +169,15 @@
pending[callee] = pending.get(callee, 0) + ntrans * call_prob
self.pending = pending
- def sim_all(self, call_prob=1, root=None):
- # for testing, only. Would run infinitely with recursions.
+ def sim_all(self, call_prob=1, maxrun=None, root=None):
+ # simulate and stop after maxrun loops
self.simulate(call_prob, root)
+ i = 0
while self.pending:
self.simulate(call_prob)
+ i += 1
+ if maxrun and i >= maxrun:
+ break
def _compute_callers(self):
nodes = {}
@@ -192,12 +199,12 @@
def c(): pass
def d(): c(); e()
def e(): c()
- sim = DemoSim([a, b, c, d, e])
+ sim = SimGraph([a, b, c, d, e])
if debug:
globals().update(locals())
sim.clear()
- for prob in 1, 2, 3:
+ for prob in 1, 3, 2:
sim.clear()
sim.run_all(prob)
state1 = sim.get_state()
Added: pypy/dist/pypy/translator/locality/support.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/translator/locality/support.py Sat Oct 22 01:32:49 2005
@@ -0,0 +1,6 @@
+# logging
+
+import py
+from pypy.tool.ansi_print import ansi_log
+log = py.log.Producer("locality")
+py.log.setconsumer("locality", ansi_log)
More information about the Pypy-commit
mailing list