[pypy-svn] r24783 - in pypy/dist/pypy: jit/codegen jit/codegen/llvm jit/codegen/llvm/test translator/llvm/pyllvm

ericvrp at codespeak.net ericvrp at codespeak.net
Wed Mar 22 12:19:24 CET 2006


Author: ericvrp
Date: Wed Mar 22 12:19:20 2006
New Revision: 24783

Added:
   pypy/dist/pypy/jit/codegen/
   pypy/dist/pypy/jit/codegen/__init__.py
   pypy/dist/pypy/jit/codegen/llvm/
   pypy/dist/pypy/jit/codegen/llvm/__init__.py
   pypy/dist/pypy/jit/codegen/llvm/jitcode.py
   pypy/dist/pypy/jit/codegen/llvm/rgenop.py
   pypy/dist/pypy/jit/codegen/llvm/test/
   pypy/dist/pypy/jit/codegen/llvm/test/test_rgenop.py
Modified:
   pypy/dist/pypy/translator/llvm/pyllvm/build.py
Log:
Generate JIT code from the graphs generated by rgenop.py


Added: pypy/dist/pypy/jit/codegen/__init__.py
==============================================================================

Added: pypy/dist/pypy/jit/codegen/llvm/__init__.py
==============================================================================

Added: pypy/dist/pypy/jit/codegen/llvm/jitcode.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/jit/codegen/llvm/jitcode.py	Wed Mar 22 12:19:20 2006
@@ -0,0 +1,76 @@
+"""
+All code for using LLVM's JIT in one place
+"""
+
+import py
+from pypy.translator.translator import TranslationContext
+from pypy.translator.llvm.database import Database
+from pypy.translator.llvm.codewriter import CodeWriter
+from pypy.translator.llvm.opwriter import OpWriter
+from pypy.translator.llvm.funcnode import FuncNode
+from pypy.translator.llvm.gc import GcPolicy
+from pypy.translator.llvm.exception import ExceptionPolicy
+from pypy.translator.llvm.pyllvm.build import pyllvm
+from cStringIO import StringIO
+
+from pypy.tool.ansi_print import ansi_log
+log = py.log.Producer("llvm/jitcode")
+py.log.setconsumer("llvm/jitcode", ansi_log)
+
+
+# Custom database, codewriter and opwriter
+class JITDatabase(Database): pass
+class JITCodeWriter(CodeWriter): pass
+class JITOpWriter(OpWriter): pass
+
+class GraphContainer(object):   # XXX what should this really be?
+    def __init__(self, graph):
+        self.graph = graph
+
+# LLVM execution engine (llvm module) that is going to contain the code
+ee = pyllvm.get_ee()
+
+
+# Create a LLVM 'execution engine' which wrap the LLVM JIT
+class JITcode(object):
+
+    def __init__(self, typer):
+        self.typer = typer
+        self.db = JITDatabase(genllvm=None, translator=None) #XXX fake
+        self.db.gcpolicy = GcPolicy.new(self.db,'raw')
+        self.db.exceptionpolicy = ExceptionPolicy.new(self.db, 'explicit')
+        self.code = StringIO()
+        self.codewriter = JITCodeWriter(self.code, self.db)
+        self.opwriter = JITOpWriter(self.db, self.codewriter)
+        self.graph_ref = {} #name by which LLVM knowns a graph
+
+    def backendoptimizations(self, graph, translator=None):
+        from pypy.translator.backendopt.removenoops import remove_same_as
+        from pypy.translator import simplify
+        from pypy.translator.unsimplify import remove_double_links
+        remove_same_as(graph)
+        simplify.eliminate_empty_blocks(graph)
+        simplify.transform_dead_op_vars(graph, translator)
+        remove_double_links(translator, graph)
+        #translator.checkgraph(graph)
+
+    def codegen(self, graph):
+        self.backendoptimizations(graph)
+        node = FuncNode(self.db, GraphContainer(graph))
+        node.writeimpl(self.codewriter)
+        log('code =' + self.code.getvalue())
+        ee.parse(self.code.getvalue())
+        return node.ref[1:] #strip of % prefix
+
+    def eval_graph(self, graph, args=()):
+        """
+        From a graph this generates llvm code that gets parsed.
+        It would be faster and use less memory to generate the code
+        without first generating the entire graph.
+        It would also be faster (probably) to extend pyllvm so blocks/instructions/etc.
+        could be added directly instead of using intermediate llvm sourcecode.
+        """
+        log('eval_graph: %s(%s)' % (graph.name, args))
+        if graph not in self.graph_ref:
+            self.graph_ref[graph] = self.codegen(graph)
+        return ee.call(self.graph_ref[graph], *args)

Added: pypy/dist/pypy/jit/codegen/llvm/rgenop.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/jit/codegen/llvm/rgenop.py	Wed Mar 22 12:19:20 2006
@@ -0,0 +1,285 @@
+"""
+Functions that generate flow graphs and operations.
+The functions below produce L2 graphs, but they define an interface
+that can be used to produce any other kind of graph.
+"""
+
+from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.objspace.flow import model as flowmodel
+from pypy.translator.simplify import eliminate_empty_blocks, join_blocks
+from pypy.rpython.module.support import init_opaque_object
+from pypy.rpython.module.support import to_opaque_object, from_opaque_object
+from pypy.rpython.module.support import from_rstr
+from pypy.jit.codegen.llvm.jitcode import JITcode
+
+
+# for debugging, sanity checks in non-RPython code
+reveal = from_opaque_object
+
+def initblock(opaqueptr):
+    init_opaque_object(opaqueptr, flowmodel.Block([]))
+
+def newblock():
+    blockcontainer = lltype.malloc(BLOCKCONTAINERTYPE)
+    initblock(blockcontainer.obj)
+    return blockcontainer
+
+def geninputarg(blockcontainer, gv_CONCRETE_TYPE):
+    block = from_opaque_object(blockcontainer.obj)
+    CONCRETE_TYPE = from_opaque_object(gv_CONCRETE_TYPE).value
+    v = flowmodel.Variable()
+    v.concretetype = CONCRETE_TYPE
+    block.inputargs.append(v)
+    return to_opaque_object(v)
+
+def _inputvars(vars):
+    if not isinstance(vars, list):
+        n = vars.ll_length()
+        vars = vars.ll_items()        
+    else:
+        n = len(vars)
+    res = []
+    for i in range(n):
+        v = from_opaque_object(vars[i])
+        assert isinstance(v, (flowmodel.Constant, flowmodel.Variable))
+        res.append(v)
+    return res
+
+# is opname a runtime value?
+def genop(blockcontainer, opname, vars, gv_RESULT_TYPE):
+    if not isinstance(opname, str):
+        opname = from_rstr(opname)
+    block = from_opaque_object(blockcontainer.obj)
+    RESULT_TYPE = from_opaque_object(gv_RESULT_TYPE).value
+    opvars = _inputvars(vars)    
+    v = flowmodel.Variable()
+    v.concretetype = RESULT_TYPE
+    op = flowmodel.SpaceOperation(opname, opvars, v)
+    block.operations.append(op)
+    return to_opaque_object(v)
+
+def gencallableconst(blockcontainer, name, targetcontainer, gv_FUNCTYPE):
+    # is name useful, is it runtime variable?
+    target = from_opaque_object(targetcontainer.obj)
+    FUNCTYPE = from_opaque_object(gv_FUNCTYPE).value
+    fptr = lltype.functionptr(FUNCTYPE, name,
+                              graph=_buildgraph(target))
+    return genconst(fptr)
+
+def genconst(llvalue):
+    v = flowmodel.Constant(llvalue)
+    v.concretetype = lltype.typeOf(llvalue)
+    if v.concretetype == lltype.Void: # XXX genconst should not really be used for Void constants
+        assert not isinstance(llvalue, str) and not isinstance(llvalue, lltype.LowLevelType)
+    return to_opaque_object(v)
+
+def revealconst(T, gv_value):
+    c = from_opaque_object(gv_value)
+    assert isinstance(c, flowmodel.Constant)
+    if isinstance(T, lltype.Ptr):
+        return lltype.cast_pointer(T, c.value)
+    elif T == llmemory.Address:
+        return llmemory.cast_ptr_to_adr(c.value)
+    else:
+        return lltype.cast_primitive(T, c.value)
+                                    
+# XXX
+# temporary interface; it's unclera if genop itself should change to ease dinstinguishing
+# Void special args from the rest. Or there should be variation for the ops involving them
+
+def placeholder(dummy):
+    c = flowmodel.Constant(dummy)
+    c.concretetype = lltype.Void
+    return to_opaque_object(c)    
+
+def constFieldName(name):
+    assert isinstance(name, str)
+    c = flowmodel.Constant(name)
+    c.concretetype = lltype.Void
+    return to_opaque_object(c)
+
+def constTYPE(TYPE):
+    assert isinstance(TYPE, lltype.LowLevelType)
+    c = flowmodel.Constant(TYPE)
+    c.concretetype = lltype.Void
+    return to_opaque_object(c)
+
+def closeblock1(blockcontainer):
+    block = from_opaque_object(blockcontainer.obj)
+    link = flowmodel.Link([], None)
+    block.closeblock(link)
+    return to_opaque_object(link)
+
+def closeblock2into(blockcontainer, exitswitch, linkpair):
+    block = from_opaque_object(blockcontainer.obj)
+    exitswitch = from_opaque_object(exitswitch)
+    assert isinstance(exitswitch, flowmodel.Variable)
+    block.exitswitch = exitswitch
+    false_link = flowmodel.Link([], None)
+    false_link.exitcase = False
+    false_link.llexitcase = False
+    true_link = flowmodel.Link([], None)
+    true_link.exitcase = True
+    true_link.llexitcase = True
+    block.closeblock(false_link, true_link)
+    linkpair.item0 = to_opaque_object(false_link)
+    linkpair.item1 = to_opaque_object(true_link)
+
+def closeblock2(blockcontainer, exitswitch):
+    linkpair = lltype.malloc(LINKPAIR)
+    closeblock2into(blockcontainer, exitswitch, linkpair) 
+    return linkpair
+
+def _closelink(link, vars, targetblock):
+    if isinstance(link, flowmodel.Link):
+        for v in vars:
+            assert isinstance(v, (flowmodel.Variable, flowmodel.Constant))
+        assert ([v.concretetype for v in vars] ==
+                [v.concretetype for v in targetblock.inputargs])
+        link.args[:] = vars
+        link.target = targetblock
+    elif isinstance(link, flowmodel.FunctionGraph):
+        graph = link
+        graph.startblock = targetblock
+        targetblock.isstartblock = True
+    else:
+        raise TypeError
+
+def closelink(link, vars, targetblockcontainer):
+    link = from_opaque_object(link)
+    targetblock = from_opaque_object(targetblockcontainer.obj)
+    vars = _inputvars(vars)
+    return _closelink(link, vars, targetblock) 
+
+def closereturnlink(link, returnvar):
+    returnvar = from_opaque_object(returnvar)
+    link = from_opaque_object(link)
+    v = flowmodel.Variable()
+    v.concretetype = returnvar.concretetype
+    pseudoreturnblock = flowmodel.Block([v])
+    pseudoreturnblock.operations = ()
+    _closelink(link, [returnvar], pseudoreturnblock)
+
+def _patchgraph(graph):
+    returntype = None
+    for link in graph.iterlinks():
+        if link.target.operations == ():
+            assert len(link.args) == 1    # for now
+            if returntype is None:
+                returntype = link.target.inputargs[0].concretetype
+            else:
+                assert returntype == link.target.inputargs[0].concretetype
+            link.target = graph.returnblock
+    if returntype is None:
+        returntype = lltype.Void
+    graph.returnblock.inputargs[0].concretetype = returntype
+
+class PseudoRTyper(object):
+    def __init__(self):
+        from pypy.rpython.typesystem import LowLevelTypeSystem
+        self.type_system = LowLevelTypeSystem.instance
+
+def _buildgraph(block):
+    graph = flowmodel.FunctionGraph('generated', block)
+    _patchgraph(graph)
+    flowmodel.checkgraph(graph)
+    eliminate_empty_blocks(graph)
+    join_blocks(graph)
+    graph.rgenop = True
+    return graph
+
+def buildgraph(blockcontainer):
+    block = from_opaque_object(blockcontainer.obj)
+    return _buildgraph(block)
+
+def testgengraph(gengraph, args, viewbefore=False):
+    from pypy.rpython.llinterp import LLInterpreter
+    if viewbefore:
+        gengraph.show()
+    #llinterp = LLInterpreter(PseudoRTyper())
+    #return llinterp.eval_graph(gengraph, args)
+    jitcode = JITcode(PseudoRTyper())
+    return jitcode.eval_graph(gengraph, args)
+    
+def runblock(blockcontainer, args, viewbefore=False):
+    graph = buildgraph(blockcontainer)
+    return testgengraph(graph, args, viewbefore)
+
+# ____________________________________________________________
+# RTyping of the above functions
+
+from pypy.rpython.extfunctable import declaretype, declareptrtype, declare
+
+blocktypeinfo = declaretype(flowmodel.Block, "Block")
+consttypeinfo = declareptrtype(flowmodel.Constant, "ConstOrVar")
+vartypeinfo   = declareptrtype(flowmodel.Variable, "ConstOrVar")
+vartypeinfo.set_lltype(consttypeinfo.get_lltype())   # force same lltype
+linktypeinfo  = declareptrtype(flowmodel.Link, "Link")
+
+CONSTORVAR = lltype.Ptr(consttypeinfo.get_lltype())
+BLOCKCONTAINERTYPE = blocktypeinfo.get_lltype()
+BLOCK = lltype.Ptr(BLOCKCONTAINERTYPE)
+LINK = lltype.Ptr(linktypeinfo.get_lltype())
+
+fieldnames = ['item%d' % i for i in range(2)]
+lltypes = [LINK]*2
+fields = tuple(zip(fieldnames, lltypes))    
+LINKPAIR = lltype.GcStruct('tuple2', *fields)
+
+# helpers
+def setannotation(func, TYPE):
+    func.compute_result_annotation = lambda *args_s: TYPE 
+
+def setspecialize(func):
+    # for now
+    def specialize_as_direct_call(hop):
+        FUNCTYPE = lltype.FuncType([r.lowleveltype for r in hop.args_r], hop.r_result.lowleveltype)
+        args_v = hop.inputargs(*hop.args_r)
+        funcptr = lltype.functionptr(FUNCTYPE, func.__name__, _callable=func)
+        cfunc = hop.inputconst(lltype.Ptr(FUNCTYPE), funcptr)
+        return hop.genop('direct_call', [cfunc] + args_v, hop.r_result)
+    func.specialize = specialize_as_direct_call
+
+# annotations
+from pypy.annotation import model as annmodel
+
+s_ConstOrVar = annmodel.SomePtr(CONSTORVAR)#annmodel.SomeExternalObject(flowmodel.Variable)
+s_Link = annmodel.SomePtr(LINK)#annmodel.SomeExternalObject(flowmodel.Link)
+s_LinkPair = annmodel.SomePtr(lltype.Ptr(LINKPAIR))
+
+setannotation(initblock, None)
+setannotation(geninputarg, s_ConstOrVar)
+setannotation(genop, s_ConstOrVar)
+setannotation(genconst, s_ConstOrVar)
+revealconst.compute_result_annotation = lambda s_T, s_gv: annmodel.lltype_to_annotation(s_T.const)
+setannotation(closeblock1, s_Link)
+setannotation(closeblock2, s_LinkPair)
+setannotation(closelink, None)
+setannotation(closereturnlink, None)
+
+# specialize
+setspecialize(initblock)
+setspecialize(geninputarg)
+setspecialize(genop)
+setspecialize(genconst)
+setspecialize(revealconst)
+setspecialize(closeblock1)
+setspecialize(closeblock2)
+setspecialize(closelink)
+setspecialize(closereturnlink)
+
+# XXX(for now) void constant constructors
+setannotation(constFieldName, s_ConstOrVar)
+setannotation(constTYPE, s_ConstOrVar)
+setannotation(placeholder, s_ConstOrVar)
+
+def set_specialize_void_constant_constructor(func):
+    # for now
+    def specialize_as_constant(hop):
+        llvalue = func(hop.args_s[0].const)
+        return hop.inputconst(lltype.typeOf(llvalue), llvalue)
+    func.specialize = specialize_as_constant
+
+set_specialize_void_constant_constructor(placeholder)
+set_specialize_void_constant_constructor(constFieldName)
+set_specialize_void_constant_constructor(constTYPE)

Added: pypy/dist/pypy/jit/codegen/llvm/test/test_rgenop.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/jit/codegen/llvm/test/test_rgenop.py	Wed Mar 22 12:19:20 2006
@@ -0,0 +1,166 @@
+from pypy.translator.llvm.buildllvm import llvm_is_on_path
+if not llvm_is_on_path():
+    import py
+    py.test.skip("llvm not found")
+
+from pypy.jit.codegen.llvm.rgenop import *
+from pypy.rpython.lltypesystem.lltype import *
+from pypy.rpython.test.test_llinterp import interpret
+from pypy.rpython.module.support import from_opaque_object
+from pypy.objspace.flow import model as flowmodel
+
+
+def build_square():
+    """def square(v0): return v0*v0"""
+    block = newblock()
+    v0 = geninputarg(block, constTYPE(Signed))
+    v1 = genop(block, 'int_mul', [v0, v0], constTYPE(Signed))
+    link = closeblock1(block)
+    closereturnlink(link, v1)
+    return block
+
+def test_square():
+    block = build_square()
+    res = runblock(block, [17])
+    assert res == 289
+
+def test_rtype_newblock():
+    def emptyblock():
+        return newblock()
+    blockcontainer = interpret(emptyblock, [])
+    block = from_opaque_object(blockcontainer.obj)
+    assert isinstance(block, flowmodel.Block)
+
+def test_rtype_geninputarg():
+    def onearg():
+        block = newblock()
+        v0 = geninputarg(block, constTYPE(Signed))
+        return v0
+    opaquev = interpret(onearg, [])
+    v = from_opaque_object(opaquev)
+    assert isinstance(v, flowmodel.Variable)
+    
+def test_rtype_build_square():
+    blockcontainer = interpret(build_square, [])
+    res = runblock(blockcontainer, [17])
+    assert res == 289
+
+def build_if():
+    """
+    def f(v0):
+        if v0 < 0:
+            return 0
+        else:
+            return v0
+    """
+    block = newblock()
+    v0 = geninputarg(block, constTYPE(Signed))
+    const0 = genconst(0)
+    v1 = genop(block, 'int_lt', [v0, const0], constTYPE(Bool))
+    exitspair = closeblock2(block, v1)
+    false_link, true_link = exitspair.item0, exitspair.item1
+    closereturnlink(true_link, const0)
+    closereturnlink(false_link, v0)
+    return block
+
+def test_if():
+    block = build_if()
+    res = runblock(block, [-1])
+    assert res == 0
+    res = runblock(block, [42])
+    assert res == 42
+
+def test_rtype_build_if():
+    blockcontainer = interpret(build_if, [])
+    res = runblock(blockcontainer, [-1])
+    assert res == 0
+    res = runblock(blockcontainer, [42])
+    assert res == 42
+
+def build_loop():
+    """
+    def f(v0):
+        i = 1
+        result = 1
+        while i <= v0:
+            result *= i
+            i += 1
+        return result
+    """
+    block = newblock()
+    v0 = geninputarg(block, constTYPE(Signed))
+    const1 = genconst(1)
+    link = closeblock1(block)
+    loopblock = newblock()
+    result0 = geninputarg(loopblock, constTYPE(Signed))
+    i0 = geninputarg(loopblock, constTYPE(Signed))
+    v1 = geninputarg(loopblock, constTYPE(Signed))
+    closelink(link, [const1, const1, v0], loopblock)
+    const1 = genconst(1)
+    result1 = genop(loopblock, 'int_mul', [result0, i0], constTYPE(Signed))
+    i1 = genop(loopblock, 'int_add', [i0, const1], constTYPE(Signed))
+    v2 = genop(loopblock, 'int_le', [i1, v1], constTYPE(Bool))
+    exitspair = closeblock2(loopblock, v2)
+    false_link, true_link = exitspair.item0, exitspair.item1
+    closereturnlink(false_link, result1)
+    closelink(true_link, [result1, i1, v1], loopblock)
+    return block    
+
+def test_loop():
+    block = build_loop()
+    res = runblock(block, [0])
+    assert res == 1
+    res = runblock(block, [1])
+    assert res == 1
+    res = runblock(block, [7])
+    assert res == 5040
+
+def test_rtype_build_loop():
+    blockcontainer = interpret(build_loop, [])
+    res = runblock(blockcontainer, [0])
+    assert res == 1
+    res = runblock(blockcontainer, [1])
+    assert res == 1
+    res = runblock(blockcontainer, [7])
+    assert res == 5040
+
+def test_rtype_void_constant_construction():
+    def fieldname_foo():
+        return constFieldName("foo")
+    res = interpret(fieldname_foo, [])
+    c = from_opaque_object(res)
+    assert isinstance(c, flowmodel.Constant)
+    assert c.concretetype == lltype.Void
+    assert c.value == "foo"
+
+    def type_Signed():
+        return constTYPE(lltype.Signed)
+    res = interpret(type_Signed, [])
+    c = from_opaque_object(res)
+    assert isinstance(c, flowmodel.Constant)
+    assert c.concretetype == lltype.Void
+    assert c.value == lltype.Signed
+
+    def dummy():
+        return placeholder(None)
+    res = interpret(dummy, [])
+    c = from_opaque_object(res)
+    assert isinstance(c, flowmodel.Constant)
+    assert c.concretetype == lltype.Void
+    assert c.value == None
+
+def test_rtype_revealcosnt():
+    def hide_and_reveal(v):
+        gv = genconst(v)
+        return revealconst(lltype.Signed, gv)
+    res = interpret(hide_and_reveal, [42])
+    assert res == 42
+
+    S = lltype.GcStruct('s', ('x', lltype.Signed))
+    S_PTR = lltype.Ptr(S)
+    def hide_and_reveal_p(p):
+        gv = genconst(p)
+        return revealconst(S_PTR, gv)
+    s = malloc(S)
+    res = interpret(hide_and_reveal_p, [s])
+    assert res == s

Modified: pypy/dist/pypy/translator/llvm/pyllvm/build.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/pyllvm/build.py	(original)
+++ pypy/dist/pypy/translator/llvm/pyllvm/build.py	Wed Mar 22 12:19:20 2006
@@ -1,11 +1,5 @@
 try:
     from pypy.translator.llvm.pyllvm import pyllvm
 except:
-    try:
-        import sys
-        sys.argv = "setup.py build_ext -i".split()
-        from pypy.translator.llvm.pyllvm import setup
-    except:
-        import py
-        py.test.skip("pyllvm failed to build")
-    from pypy.translator.llvm.pyllvm import pyllvm
+    import py
+    py.test.skip("pyllvm not found: run 'python setup.py build_ext -i' in translator/llvm/pyllvm")



More information about the Pypy-commit mailing list