[pypy-svn] r19944 - in pypy/dist/pypy/translator/llvm: . demo test
rxe at codespeak.net
rxe at codespeak.net
Wed Nov 16 20:11:28 CET 2005
Author: rxe
Date: Wed Nov 16 20:11:23 2005
New Revision: 19944
Modified:
pypy/dist/pypy/translator/llvm/codewriter.py
pypy/dist/pypy/translator/llvm/demo/richards.py
pypy/dist/pypy/translator/llvm/demo/run.py
pypy/dist/pypy/translator/llvm/externs2ll.py
pypy/dist/pypy/translator/llvm/funcnode.py
pypy/dist/pypy/translator/llvm/genllvm.py
pypy/dist/pypy/translator/llvm/test/runtest.py
Log:
Start refactoring entry points to genllvm - and general refactors and tidies.
This to make life easier in adding new features.
Modified: pypy/dist/pypy/translator/llvm/codewriter.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/codewriter.py (original)
+++ pypy/dist/pypy/translator/llvm/codewriter.py Wed Nov 16 20:11:23 2005
@@ -22,6 +22,11 @@
else:
self.append(line)
+ def header_comment(self, s):
+ self.newline()
+ self.comment(s)
+ self.newline()
+
def newline(self):
self.append("")
Modified: pypy/dist/pypy/translator/llvm/demo/richards.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/demo/richards.py (original)
+++ pypy/dist/pypy/translator/llvm/demo/richards.py Wed Nov 16 20:11:23 2005
@@ -360,7 +360,7 @@
class Richards:
- iterations = 25
+ iterations = 750
def run(self):
for i in xrange(self.iterations):
Modified: pypy/dist/pypy/translator/llvm/demo/run.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/demo/run.py (original)
+++ pypy/dist/pypy/translator/llvm/demo/run.py Wed Nov 16 20:11:23 2005
@@ -18,7 +18,7 @@
f()
def l(name):
- exe_path = compile_module(entry_point, [], standalone=True, exe_name=name)
+ exe_path = compile_module(entry_point, [], exe_name=name)
print 'Running standalone (llvm-based) executable:'
print exe_path
os.system(exe_path)
Modified: pypy/dist/pypy/translator/llvm/externs2ll.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/externs2ll.py (original)
+++ pypy/dist/pypy/translator/llvm/externs2ll.py Wed Nov 16 20:11:23 2005
@@ -9,8 +9,21 @@
from pypy.tool.udir import udir
+support_functions = [
+ "%raisePyExc_IOError",
+ "%raisePyExc_ValueError",
+ "%raisePyExc_OverflowError",
+ "%raisePyExc_ZeroDivisionError",
+ "%raisePyExc_RuntimeError",
+ "%prepare_ZeroDivisionError",
+ "%prepare_OverflowError",
+ "%prepare_ValueError",
+ "%RPyString_FromString",
+ "%RPyString_AsString",
+ "%RPyString_Size"]
def get_ll(ccode, function_names):
+ function_names += support_functions
filename = str(udir.join("ccode.c"))
f = open(filename, "w")
f.write(ccode)
@@ -126,7 +139,7 @@
path = os.path.join(path, p)
return path
-def generate_llfile(db, extern_decls, support_functions, debug=False):
+def generate_llfile(db, extern_decls):
ccode = []
function_names = []
@@ -179,4 +192,4 @@
ccode.append(s)
ccode = "".join(ccode)
- return get_ll(ccode, function_names + support_functions)
+ return get_ll(ccode, function_names)
Modified: pypy/dist/pypy/translator/llvm/funcnode.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/funcnode.py (original)
+++ pypy/dist/pypy/translator/llvm/funcnode.py Wed Nov 16 20:11:23 2005
@@ -40,7 +40,7 @@
self.graph = value.graph
self.db.genllvm.exceptionpolicy.transform(self.db.translator, self.graph)
- remove_exception_mallocs(self.db.translator, self.graph, self.ref)
+ #remove_exception_mallocs(self.db.translator, self.graph, self.ref)
#merge_mallocs(self.db.translator, self.graph, self.ref)
remove_double_links(self.db.translator, self.graph)
Modified: pypy/dist/pypy/translator/llvm/genllvm.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/genllvm.py (original)
+++ pypy/dist/pypy/translator/llvm/genllvm.py Wed Nov 16 20:11:23 2005
@@ -17,229 +17,265 @@
from pypy.translator.translator import Translator
from pypy.translator.llvm.log import log
-function_count = {}
-llexterns_header = llexterns_functions = None
-
+# keep for propersity sake
+"""run_pypy-llvm.sh [aug 29th 2005]
+before slotifying: 350Mb
+after slotifying: 300Mb, 35 minutes until the .ll file is fully written.
+STATS (1, "<class 'pypy.translator.llvm.arraynode.VoidArrayTypeNode'>")
+STATS (1, "<class 'pypy.translator.llvm.opaquenode.OpaqueTypeNode'>")
+STATS (9, "<class 'pypy.translator.llvm.structnode.StructVarsizeTypeNode'>")
+STATS (46, "<class 'pypy.translator.llvm.extfuncnode.ExternalFuncNode'>")
+STATS (52, "<class 'pypy.translator.llvm.arraynode.ArrayTypeNode'>")
+STATS (189, "<class 'pypy.translator.llvm.arraynode.VoidArrayNode'>")
+STATS (819, "<class 'pypy.translator.llvm.opaquenode.OpaqueNode'>")
+STATS (1250, "<class 'pypy.translator.llvm.funcnode.FuncTypeNode'>")
+STATS (1753, "<class 'pypy.translator.llvm.structnode.StructTypeNode'>")
+STATS (5896, "<class 'pypy.translator.llvm.funcnode.FuncNode'>")
+STATS (24013, "<class 'pypy.translator.llvm.arraynode.ArrayNode'>")
+STATS (25411, "<class 'pypy.translator.llvm.structnode.StructVarsizeNode'>")
+STATS (26210, "<class 'pypy.translator.llvm.arraynode.StrArrayNode'>")
+STATS (268884, "<class 'pypy.translator.llvm.structnode.StructNode'>")
+
+init took 00m00s
+setup_all took 08m14s
+setup_all externs took 00m00s
+generate_ll took 00m02s
+write externs type declarations took 00m00s
+write data type declarations took 00m02s
+write global constants took 09m49s
+write function prototypes took 00m00s
+write declarations took 00m03s
+write implementations took 01m54s
+write support functions took 00m00s
+write external functions took 00m00s
+"""
class GenLLVM(object):
- def __init__(self, translator, gcpolicy=None, exceptionpolicy=None, debug=False):
+ # see open_file() below
+ function_count = {}
+ llexterns_header = llexterns_functions = None
+
+ def __init__(self, translator, gcpolicy=None, exceptionpolicy=None,
+ debug=False, logging=True):
# reset counters
LLVMNode.nodename_count = {}
+ extfuncnode.ExternalFuncNode.used_external_functions = {}
+
+ # create and set internals
self.db = Database(self, translator)
- self.translator = translator
self.gcpolicy = gcpolicy
+ self.translator = translator
self.exceptionpolicy = exceptionpolicy
- extfuncnode.ExternalFuncNode.used_external_functions = {}
- self.debug = debug # for debug we create comments of every operation that may be executed
- if debug:
- translator.checkgraphs()
- def _checkpoint(self, msg=None):
- if self.debug:
- if msg:
- t = (time.time() - self.starttime)
- log('\t%s took %02dm%02ds' % (msg, t/60, t%60))
- else:
- log('GenLLVM:')
- self.starttime = time.time()
+ # the debug flag is for creating comments of every operation
+ # that may be executed
+ self.debug = debug
+
+ # the logging flag is for logging information statistics in the build
+ # process
+ self.logging = logging
- def _print_node_stats(self):
- """run_pypy-llvm.sh [aug 29th 2005]
- before slotifying: 350Mb
- after slotifying: 300Mb, 35 minutes until the .ll file is fully written.
- STATS (1, "<class 'pypy.translator.llvm.arraynode.VoidArrayTypeNode'>")
- STATS (1, "<class 'pypy.translator.llvm.opaquenode.OpaqueTypeNode'>")
- STATS (9, "<class 'pypy.translator.llvm.structnode.StructVarsizeTypeNode'>")
- STATS (46, "<class 'pypy.translator.llvm.extfuncnode.ExternalFuncNode'>")
- STATS (52, "<class 'pypy.translator.llvm.arraynode.ArrayTypeNode'>")
- STATS (189, "<class 'pypy.translator.llvm.arraynode.VoidArrayNode'>")
- STATS (819, "<class 'pypy.translator.llvm.opaquenode.OpaqueNode'>")
- STATS (1250, "<class 'pypy.translator.llvm.funcnode.FuncTypeNode'>")
- STATS (1753, "<class 'pypy.translator.llvm.structnode.StructTypeNode'>")
- STATS (5896, "<class 'pypy.translator.llvm.funcnode.FuncNode'>")
- STATS (24013, "<class 'pypy.translator.llvm.arraynode.ArrayNode'>")
- STATS (25411, "<class 'pypy.translator.llvm.structnode.StructVarsizeNode'>")
- STATS (26210, "<class 'pypy.translator.llvm.arraynode.StrArrayNode'>")
- STATS (268884, "<class 'pypy.translator.llvm.structnode.StructNode'>")
- """
- return #disable node stats output
- if not self.debug:
- return
- nodecount = {}
- for node in self.db.getnodes():
- typ = type(node)
- try:
- nodecount[typ] += 1
- except:
- nodecount[typ] = 1
- stats = [(count, str(typ)) for typ, count in nodecount.iteritems()]
- stats.sort()
- for s in stats:
- log('STATS %s' % str(s))
def gen_llvm_source(self, func=None):
- """
- init took 00m00s
- setup_all took 08m14s
- setup_all externs took 00m00s
- generate_ll took 00m02s
- write externs type declarations took 00m00s
- write data type declarations took 00m02s
- write global constants took 09m49s
- write function prototypes took 00m00s
- write declarations took 00m03s
- write implementations took 01m54s
- write support functions took 00m00s
- write external functions took 00m00s
- """
+
self._checkpoint()
- if func is None:
- func = self.translator.entrypoint
- self.entrypoint = func
-
- ptr = getfunctionptr(self.translator, func)
- c = inputconst(lltype.typeOf(ptr), ptr)
- entry_point = c.value._obj
- self.db.prepare_arg_value(c)
- self._checkpoint('init')
+ # get entry point
+ entry_point, func_name = self.get_entry_point(func)
+ self._checkpoint('get_entry_point')
+
+ # open file & create codewriter
+ codewriter, filename = self.create_codewrite(func_name)
+ self._checkpoint('open file and create codewriter')
# set up all nodes
self.db.setup_all()
self.entrynode = self.db.set_entrynode(entry_point)
- entryfunc_name = self.entrynode.getdecl().split('%', 1)[1].split('(')[0]
- self._checkpoint('setup_all')
+ self._checkpoint('setup_all first pass')
- # post set up externs
+ # post set up nodes
extern_decls = post_setup_externs(self.db)
self.translator.rtyper.specialize_more_blocks()
self.db.setup_all()
- using_external_functions = extfuncnode.ExternalFuncNode.used_external_functions.keys() != []
- self._print_node_stats()
- self._checkpoint('setup_all externs')
+ self._checkpoint('setup_all second pass')
- support_functions = "%raisePyExc_IOError %raisePyExc_ValueError "\
- "%raisePyExc_OverflowError %raisePyExc_ZeroDivisionError "\
- "%raisePyExc_RuntimeError "\
- "%prepare_ZeroDivisionError %prepare_OverflowError %prepare_ValueError "\
- "%RPyString_FromString %RPyString_AsString %RPyString_Size".split()
-
- global llexterns_header, llexterns_functions
- if llexterns_header is None and using_external_functions:
- llexterns_header, llexterns_functions = generate_llfile(self.db, extern_decls, support_functions, self.debug)
- self._checkpoint('generate_ll')
-
- # prevent running the same function twice in a test
- if func.func_name in function_count:
- postfix = '_%d' % function_count[func.func_name]
- function_count[func.func_name] += 1
- else:
- postfix = ''
- function_count[func.func_name] = 1
- filename = udir.join(func.func_name + postfix).new(ext='.ll')
- f = open(str(filename),'w')
- codewriter = CodeWriter(f, self)
- comment = codewriter.comment
- nl = codewriter.newline
+ self._print_node_stats()
+ # create ll file from c code
+ using_external_functions = self.setup_externs(extern_decls)
+ self._checkpoint('setup_externs')
+
+ # write external function headers
if using_external_functions:
- nl(); comment("External Function Declarations") ; nl()
- codewriter.append(llexterns_header)
+ codewriter.header_comment('External Function Headers')
+ codewriter.append(self.llexterns_header)
- nl(); comment("Type Declarations"); nl()
- for c_name, obj in extern_decls:
- if isinstance(obj, lltype.LowLevelType):
- if isinstance(obj, lltype.Ptr):
- obj = obj.TO
- l = "%%%s = type %s" % (c_name, self.db.repr_type(obj))
- codewriter.append(l)
+ codewriter.header_comment("Type Declarations")
+
+ # write extern type declarations
+ self.write_extern_decls(codewriter, extern_decls)
self._checkpoint('write externs type declarations')
+ # write node type declarations
for typ_decl in self.db.getnodes():
typ_decl.writedatatypedecl(codewriter)
self._checkpoint('write data type declarations')
- nl(); comment("Global Data") ; nl()
+ codewriter.header_comment("Global Data")
+
+ # write pbcs
for typ_decl in self.db.getnodes():
typ_decl.writeglobalconstants(codewriter)
self._checkpoint('write global constants')
- nl(); comment("Function Prototypes") ; nl()
+ codewriter.header_comment("Function Prototypes")
+
+ # write external protos
codewriter.append(extdeclarations)
+
+ # write garbage collection protos
codewriter.append(self.gcpolicy.declarations())
- self._checkpoint('write function prototypes')
+ # write node protos
for typ_decl in self.db.getnodes():
typ_decl.writedecl(codewriter)
- self._checkpoint('write declarations')
- nl(); comment("Function Implementation")
+ self._checkpoint('write function prototypes')
+
codewriter.startimpl()
-
- for typ_decl in self.db.getnodes():
- typ_decl.writeimpl(codewriter)
- self._checkpoint('write implementations')
- codewriter.append(self.exceptionpolicy.llvmcode(self.entrynode))
+ codewriter.header_comment("Function Implementation")
- # XXX we need to create our own main() that calls the actual entry_point function
- if entryfunc_name == 'pypy_entry_point': #XXX just to get on with translate_pypy
- extfuncnode.ExternalFuncNode.used_external_functions['%main'] = True
+ # write external function implementations
+ if using_external_functions:
+ codewriter.header_comment('External Function Implementation')
+ codewriter.append(self.llexterns_functions)
- elif entryfunc_name == 'pypy_main_noargs': #XXX just to get on with bpnn & richards
- extfuncnode.ExternalFuncNode.used_external_functions['%main_noargs'] = True
+ self._checkpoint('write external functions')
- for f in support_functions:
- extfuncnode.ExternalFuncNode.used_external_functions[f] = True
+ # write exception implementaions
+ codewriter.append(self.exceptionpolicy.llvmcode(self.entrynode))
- depdone = {}
- for funcname,value in extfuncnode.ExternalFuncNode.used_external_functions.iteritems():
- deps = dependencies(funcname,[])
- deps.reverse()
- for dep in deps:
- if dep not in depdone:
- if dep in extfunctions: #else external function that is shared with genc
- codewriter.append(extfunctions[dep][1])
- depdone[dep] = True
- self._checkpoint('write support functions')
-
- if using_external_functions:
- nl(); comment("External Function Implementation") ; nl()
- codewriter.append(llexterns_functions)
- self._checkpoint('write external functions')
+ # write all node implementations
+ for key, (deps, impl) in extfunctions.items():
+ print key
+ if key in ["%main_noargs", "%main"]:
+ continue
+ codewriter.append(impl)
- comment("End of file") ; nl()
+ for typ_decl in self.db.getnodes():
+ typ_decl.writeimpl(codewriter)
+ self._checkpoint('write node implementations')
+
+ self.write_entry_point(codewriter)
+ self._checkpoint('write support implentations')
+
+ codewriter.comment("End of file")
return filename
+
+ def get_entry_point(self, func):
+ if func is None:
+ func = self.translator.entrypoint
+ self.entrypoint = func
- def create_module(self,
- filename,
- really_compile=True,
- standalone=False,
- optimize=True,
- exe_name=None):
-
- if standalone:
- return build_llvm_module.make_module_from_llvm(self, filename,
- optimize=optimize,
- exe_name=exe_name)
+ ptr = getfunctionptr(self.translator, func)
+ c = inputconst(lltype.typeOf(ptr), ptr)
+ self.db.prepare_arg_value(c)
+ return c.value._obj, func.func_name
+
+ def setup_externs(self, extern_decls):
+ extern_funcs = extfuncnode.ExternalFuncNode.used_external_functions
+ using_external_functions = extern_funcs.keys() != []
+
+ # we cache the llexterns to make tests run faster
+ if using_external_functions and self.llexterns_header is None:
+ assert self.llexterns_functions is None
+ self.llexterns_header, self.llexterns_functions = \
+ generate_llfile(self.db, extern_decls)
+ return using_external_functions
+
+ def create_codewrite(self, func_name):
+ # prevent running the same function twice in a test
+ if func_name in self.function_count:
+ postfix = '_%d' % self.function_count[func_name]
+ self.function_count[func_name] += 1
else:
postfix = ''
- basename = filename.purebasename + '_wrapper' + postfix + '.pyx'
- pyxfile = filename.new(basename = basename)
- write_pyx_wrapper(self, pyxfile)
- return build_llvm_module.make_module_from_llvm(self, filename,
- pyxfile=pyxfile,
- optimize=optimize)
+ self.function_count[func_name] = 1
+ filename = udir.join(func_name + postfix).new(ext='.ll')
+ f = open(str(filename), 'w')
+ return CodeWriter(f, self), filename
+ def write_extern_decls(self, codewriter, extern_decls):
+ for c_name, obj in extern_decls:
+ if isinstance(obj, lltype.LowLevelType):
+ if isinstance(obj, lltype.Ptr):
+ obj = obj.TO
-def genllvm(translator, gcpolicy=None, exceptionpolicy=None, log_source=False, **kwds):
- gen = GenLLVM(translator, GcPolicy.new(gcpolicy), ExceptionPolicy.new(exceptionpolicy))
+ l = "%%%s = type %s" % (c_name, self.db.repr_type(obj))
+ codewriter.append(l)
+
+ def write_entry_point(self, codewriter):
+ # XXX we need to create our own main() that calls the actual entry_point function
+ entryfunc_name = self.entrynode.getdecl().split('%pypy_', 1)[1].split('(')[0]
+ llcode = extfunctions["%" + entryfunc_name][1]
+ codewriter.append(llcode)
+
+ def _checkpoint(self, msg=None):
+ if not self.logging:
+ return
+ if msg:
+ t = (time.time() - self.starttime)
+ log('\t%s took %02dm%02ds' % (msg, t/60, t%60))
+ else:
+ log('GenLLVM:')
+ self.starttime = time.time()
+
+ def _print_node_stats(self):
+ # disable node stats output
+ if not self.logging:
+ return
+
+ nodecount = {}
+ for node in self.db.getnodes():
+ typ = type(node)
+ try:
+ nodecount[typ] += 1
+ except:
+ nodecount[typ] = 1
+ stats = [(count, str(typ)) for typ, count in nodecount.iteritems()]
+ stats.sort()
+ for s in stats:
+ log('STATS %s' % str(s))
+
+def genllvm(translator, gcpolicy=None, exceptionpolicy=None,
+ log_source=False, optimize=True, exe_name=None, logging=False):
+
+ gen = GenLLVM(translator,
+ GcPolicy.new(gcpolicy),
+ ExceptionPolicy.new(exceptionpolicy),
+ logging=logging)
filename = gen.gen_llvm_source()
+
if log_source:
log(open(filename).read())
- return gen.create_module(filename, **kwds)
+
+ if exe_name is not None:
+ # standalone
+ return build_llvm_module.make_module_from_llvm(gen, filename,
+ optimize=optimize,
+ exe_name=exe_name)
+ else:
+ # use pyrex to create module for CPython
+ postfix = ''
+ basename = filename.purebasename + '_wrapper' + postfix + '.pyx'
+ pyxfile = filename.new(basename = basename)
+ write_pyx_wrapper(gen, pyxfile)
+ return build_llvm_module.make_module_from_llvm(gen, filename,
+ pyxfile=pyxfile,
+ optimize=optimize)
def compile_module(function, annotation, view=False, **kwds):
t = Translator(function)
@@ -247,10 +283,13 @@
a.simplify()
t.specialize()
t.backend_optimizations(ssa_form=False)
- if view: #note: this is without policy transforms
+
+ # note: this is without policy transforms
+ if view:
t.view()
return genllvm(t, **kwds)
def compile_function(function, annotation, **kwds):
+ """ Helper - which get the compiled module from CPython. """
mod = compile_module(function, annotation, **kwds)
return getattr(mod, 'pypy_' + function.func_name + "_wrapper")
Modified: pypy/dist/pypy/translator/llvm/test/runtest.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/test/runtest.py (original)
+++ pypy/dist/pypy/translator/llvm/test/runtest.py Wed Nov 16 20:11:23 2005
@@ -26,7 +26,8 @@
if v < MINIMUM_LLVM_VERSION:
py.test.skip("llvm version not up-to-date (found %.1f, should be >= %.1f)" % (v, MINIMUM_LLVM_VERSION))
- mod = compile_module(function, annotation, optimize=optimize_tests, **kwds)
+ mod = compile_module(function, annotation, optimize=optimize_tests,
+ logging=False, **kwds)
return getattr(mod, 'pypy_' + function.func_name + "_wrapper")
def compile_module_function(function, annotation, **kwds):
More information about the Pypy-commit
mailing list