[pypy-svn] r20194 - in pypy/dist/pypy/translator: goal llvm llvm/demo llvm/module llvm/test
rxe at codespeak.net
rxe at codespeak.net
Wed Nov 23 19:41:19 CET 2005
Author: rxe
Date: Wed Nov 23 19:41:12 2005
New Revision: 20194
Modified:
pypy/dist/pypy/translator/goal/driver.py
pypy/dist/pypy/translator/llvm/demo/run.py
pypy/dist/pypy/translator/llvm/exception.py
pypy/dist/pypy/translator/llvm/externs2ll.py
pypy/dist/pypy/translator/llvm/gc.py
pypy/dist/pypy/translator/llvm/genllvm.py
pypy/dist/pypy/translator/llvm/module/genexterns.c
pypy/dist/pypy/translator/llvm/module/support.py
pypy/dist/pypy/translator/llvm/test/runtest.py
pypy/dist/pypy/translator/llvm/test/test_gc.py
Log:
More entry point refinements.
* small wrapper for boehm that now uses the macros
* gentle renaming of entry points (like compile_module() to build standalone)
* tiny refactor to driver.py common code (please review)
Modified: pypy/dist/pypy/translator/goal/driver.py
==============================================================================
--- pypy/dist/pypy/translator/goal/driver.py (original)
+++ pypy/dist/pypy/translator/goal/driver.py Wed Nov 23 19:41:12 2005
@@ -215,18 +215,21 @@
#
task_source_c = taskdef(task_source_c, ['database_c'], "Generating c source")
+ def create_exe(self):
+ import shutil
+ exename = mkexename(self.c_entryp)
+ newexename = mkexename('./pypy-llvm')
+ shutil.copy(exename, newexename)
+ self.c_entryp = newexename
+ self.log.info("created: %s" % (self.c_entryp,))
+
def task_compile_c(self): # xxx messy
cbuilder = self.cbuilder
cbuilder.compile()
-
+
if self.standalone:
- c_entryp = cbuilder.executable_name
- import shutil
- exename = mkexename(c_entryp)
- newexename = mkexename('./'+'pypy-c')
- shutil.copy(exename, newexename)
- self.c_entryp = newexename
- self.log.info("created: %s" % (self.c_entryp,))
+ self.c_entryp = cbuilder.executable_name
+ self.create_exe()
else:
cbuilder.import_module()
self.c_entryp = cbuilder.get_entry_point()
@@ -274,10 +277,9 @@
from pypy.translator.llvm import genllvm
# XXX Need more options for policies/llvm-backendoptions here?
- gc_opts = self.options.gc
- self.llvmgen = genllvm.GenLLVM(translator,
- genllvm.GcPolicy.new(gc_opts),
- genllvm.ExceptionPolicy.new(None))
+ self.llvmgen = genllvm.GenLLVM(translator, self.options.gc,
+ None, self.standalone)
+
llvm_filename = self.llvmgen.gen_llvm_source()
self.log.info("written: %s" % (llvm_filename,))
#
@@ -286,14 +288,12 @@
"Generating llvm source")
def task_compile_llvm(self):
- self.c_entryp = self.llvmgen.compile_llvm_source(exe_name='pypy-llvm')
+ gen = self.llvmgen
if self.standalone:
- import shutil
- exename = mkexename(self.c_entryp)
- newexename = mkexename('./pypy-llvm')
- shutil.copy(exename, newexename)
- self.c_entryp = newexename
- self.log.info("created: %s" % (self.c_entryp,))
+ self.c_entryp = gen.compile_llvm_source(exe_name='pypy-llvm')
+ self.create_exe()
+ else:
+ self.c_entryp = gen.compile_llvm_source(return_fn=True)
#
task_compile_llvm = taskdef(task_compile_llvm,
['source_llvm'],
Modified: pypy/dist/pypy/translator/llvm/demo/run.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/demo/run.py (original)
+++ pypy/dist/pypy/translator/llvm/demo/run.py Wed Nov 23 19:41:12 2005
@@ -5,7 +5,7 @@
from pypy.annotation.model import SomeList, SomeString
from pypy.annotation.listdef import ListDef
-from pypy.translator.llvm.genllvm import compile_module
+from pypy.translator.llvm.genllvm import genllvm_compile
from pypy.translator.translator import Translator
def p():
@@ -15,7 +15,8 @@
def l(name):
s_list_of_strings = SomeList(ListDef(None, SomeString()))
s_list_of_strings.listdef.resize()
- exe_path = compile_module(entry_point, [s_list_of_strings], exe_name=name)
+ exe_path = genllvm_compile(entry_point, [s_list_of_strings],
+ exe_name=name, standalone=True)
print 'Running standalone (llvm-based) executable:'
print exe_path
os.system(exe_path)
Modified: pypy/dist/pypy/translator/llvm/exception.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/exception.py (original)
+++ pypy/dist/pypy/translator/llvm/exception.py Wed Nov 23 19:41:12 2005
@@ -19,7 +19,7 @@
ret sbyte* %%tmp.4
else:
- %%tmp.8 = call ccc sbyte* %%GC_malloc(uint %%nbytes)
+ %%tmp.8 = call ccc sbyte* %%pypy_malloc(uint %%nbytes)
ret sbyte* %%tmp.8
}
''' % (RINGBUFFER_ENTRY_MAXSIZE, RINGBUGGER_OVERSIZE, RINGBUGGER_SIZE-1)
Modified: pypy/dist/pypy/translator/llvm/externs2ll.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/externs2ll.py (original)
+++ pypy/dist/pypy/translator/llvm/externs2ll.py Wed Nov 23 19:41:12 2005
@@ -105,6 +105,7 @@
from pypy.translator.c.extfunc import predeclare_all
# hacks to make predeclare_all work
+ # XXX Rationalise this
db.standalone = True
db.externalfuncs = {}
decls = list(predeclare_all(db, rtyper))
@@ -129,7 +130,7 @@
path = os.path.join(path, p)
return path
-def generate_llfile(db, extern_decls, entrynode):
+def generate_llfile(db, extern_decls, entrynode, standalone):
ccode = []
function_names = []
@@ -144,7 +145,7 @@
for k, v in db.obj2node.items():
try:
if isinstance(lltype.typeOf(k), lltype.FuncType):
- if v == entrynode and k._name == "entry_point":
+ if v == entrynode and standalone:
predeclarefn("__ENTRY_POINT__", v.get_ref())
ccode.append('#define ENTRY_POINT_DEFINED 1\n\n')
break
Modified: pypy/dist/pypy/translator/llvm/gc.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/gc.py (original)
+++ pypy/dist/pypy/translator/llvm/gc.py Wed Nov 23 19:41:12 2005
@@ -53,11 +53,7 @@
return ['gc', 'pthread'] # XXX on windows?
def declarations(self):
- return '''
-declare ccc sbyte* %GC_malloc(uint)
-declare ccc sbyte* %GC_malloc_atomic(uint)
-%GC_all_interior_pointers = external global int
-'''
+ return ''
def malloc(self, targetvar, type_, size, is_atomic, word, uword):
s = str(size)
@@ -67,7 +63,7 @@
t = '''
%%malloc.Size%(cnt)s = getelementptr %(type_)s* null, %(uword)s %(s)s
%%malloc.SizeU%(cnt)s = cast %(type_)s* %%malloc.Size%(cnt)s to %(uword)s
-%%malloc.Ptr%(cnt)s = call ccc sbyte* %%GC_malloc%(atomic)s(%(uword)s %%malloc.SizeU%(cnt)s)
+%%malloc.Ptr%(cnt)s = call fastcc sbyte* %%pypy_malloc%(atomic)s(%(uword)s %%malloc.SizeU%(cnt)s)
%(targetvar)s = cast sbyte* %%malloc.Ptr%(cnt)s to %(type_)s*
''' % locals()
if is_atomic:
Modified: pypy/dist/pypy/translator/llvm/genllvm.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/genllvm.py (original)
+++ pypy/dist/pypy/translator/llvm/genllvm.py Wed Nov 23 19:41:12 2005
@@ -17,46 +17,13 @@
from pypy.translator.translator import Translator
from pypy.translator.llvm.log import log
-# XXX for propersity sake
-"""run_pypy-llvm.sh [aug 29th 2005]
-before slotifying: 350Mb
-after slotifying: 300Mb, 35 minutes until the .ll file is fully written.
-STATS (1, "<class 'pypy.translator.llvm.arraynode.VoidArrayTypeNode'>")
-STATS (1, "<class 'pypy.translator.llvm.opaquenode.OpaqueTypeNode'>")
-STATS (9, "<class 'pypy.translator.llvm.structnode.StructVarsizeTypeNode'>")
-STATS (46, "<class 'pypy.translator.llvm.extfuncnode.ExternalFuncNode'>")
-STATS (52, "<class 'pypy.translator.llvm.arraynode.ArrayTypeNode'>")
-STATS (189, "<class 'pypy.translator.llvm.arraynode.VoidArrayNode'>")
-STATS (819, "<class 'pypy.translator.llvm.opaquenode.OpaqueNode'>")
-STATS (1250, "<class 'pypy.translator.llvm.funcnode.FuncTypeNode'>")
-STATS (1753, "<class 'pypy.translator.llvm.structnode.StructTypeNode'>")
-STATS (5896, "<class 'pypy.translator.llvm.funcnode.FuncNode'>")
-STATS (24013, "<class 'pypy.translator.llvm.arraynode.ArrayNode'>")
-STATS (25411, "<class 'pypy.translator.llvm.structnode.StructVarsizeNode'>")
-STATS (26210, "<class 'pypy.translator.llvm.arraynode.StrArrayNode'>")
-STATS (268884, "<class 'pypy.translator.llvm.structnode.StructNode'>")
-
-init took 00m00s
-setup_all took 08m14s
-setup_all externs took 00m00s
-generate_ll took 00m02s
-write externs type declarations took 00m00s
-write data type declarations took 00m02s
-write global constants took 09m49s
-write function prototypes took 00m00s
-write declarations took 00m03s
-write implementations took 01m54s
-write support functions took 00m00s
-write external functions took 00m00s
-"""
-
class GenLLVM(object):
# see open_file() below
function_count = {}
llexterns_header = llexterns_functions = None
- def __init__(self, translator, gcpolicy=None, exceptionpolicy=None,
+ def __init__(self, translator, gcpolicy, exceptionpolicy, standalone,
debug=False, logging=True):
# reset counters
@@ -64,9 +31,11 @@
# create and set internals
self.db = Database(self, translator)
- self.gcpolicy = gcpolicy
+
+ self.gcpolicy = GcPolicy.new(gcpolicy)
+ self.standalone = standalone
self.translator = translator
- self.exceptionpolicy = exceptionpolicy
+ self.exceptionpolicy = ExceptionPolicy.new(exceptionpolicy)
# the debug flag is for creating comments of every operation
# that may be executed
@@ -76,35 +45,58 @@
# process
self.logging = logging
- def gen_llvm_source(self, func=None):
+ self.source_generated = False
+ def gen_llvm_source(self, func=None):
self._checkpoint()
+ codewriter = self.setup(func)
+
+ # write top part of llvm file
+ self.write_headers(codewriter)
+
+ codewriter.startimpl()
+
+ # write bottom part of llvm file
+ self.write_implementations(codewriter)
+
+ self.source_generated = True
+ self._checkpoint('done')
+ return self.filename
+
+ def setup(self, func):
+ """ setup all nodes
+ create c file for externs
+ create ll file for c file
+ create codewriter """
+
# set up externs nodes
- extern_decls = setup_externs(self.db)
+ self.extern_decls = setup_externs(self.db)
self.translator.rtyper.specialize_more_blocks()
self.db.setup_all()
self._checkpoint('setup_all externs')
# get entry point
- entry_point, func_name = self.get_entry_point(func)
+ entry_point = self.get_entry_point(func)
self._checkpoint('get_entry_point')
# set up all nodes
self.db.setup_all()
self.entrynode = self.db.set_entrynode(entry_point)
- self._checkpoint('setup_all first pass')
+ self._checkpoint('setup_all all nodes')
self._print_node_stats()
- # open file & create codewriter
- codewriter, filename = self.create_codewrite(func_name)
- self._checkpoint('open file and create codewriter')
-
# create ll file from c code
- self.setup_externs(extern_decls)
+ self.generate_ll_externs()
self._checkpoint('setup_externs')
+
+ # open file & create codewriter
+ codewriter, self.filename = self.create_codewriter()
+ self._checkpoint('open file and create codewriter')
+ return codewriter
+ def write_headers(self, codewriter):
# write external function headers
codewriter.header_comment('External Function Headers')
codewriter.append(self.llexterns_header)
@@ -112,7 +104,7 @@
codewriter.header_comment("Type Declarations")
# write extern type declarations
- self.write_extern_decls(codewriter, extern_decls)
+ self.write_extern_decls(codewriter)
self._checkpoint('write externs type declarations')
# write node type declarations
@@ -141,15 +133,14 @@
self._checkpoint('write function prototypes')
- codewriter.startimpl()
-
+ def write_implementations(self, codewriter):
codewriter.header_comment("Function Implementation")
# write external function implementations
codewriter.header_comment('External Function Implementation')
codewriter.append(self.llexterns_functions)
codewriter.append(extfunctions)
- self.write_extern_impls(codewriter, extern_decls)
+ self.write_extern_impls(codewriter)
self.write_setup_impl(codewriter)
self._checkpoint('write support implentations')
@@ -164,27 +155,6 @@
# write entry point if there is one
codewriter.comment("End of file")
-
- self._checkpoint('done')
- self.filename = filename
- return filename
-
- def compile_llvm_source(self, optimize=True, exe_name=None):
- assert hasattr(self, "filename")
- if exe_name is not None:
- # standalone
- return build_llvm_module.make_module_from_llvm(self, self.filename,
- optimize=optimize,
- exe_name=exe_name)
- else:
- # use pyrex to create module for CPython
- postfix = ''
- basename = self.filename.purebasename + '_wrapper' + postfix + '.pyx'
- pyxfile = self.filename.new(basename = basename)
- write_pyx_wrapper(self, pyxfile)
- return build_llvm_module.make_module_from_llvm(self, self.filename,
- pyxfile=pyxfile,
- optimize=optimize)
def get_entry_point(self, func):
if func is None:
@@ -194,40 +164,42 @@
ptr = getfunctionptr(self.translator, func)
c = inputconst(lltype.typeOf(ptr), ptr)
self.db.prepare_arg_value(c)
- return c.value._obj, func.func_name
+ self.entry_func_name = func.func_name
+ return c.value._obj
- def setup_externs(self, extern_decls):
- # we cache the llexterns to make tests run faster
+ def generate_ll_externs(self):
+ # we only cache the llexterns to make tests run faster
if self.llexterns_header is None:
assert self.llexterns_functions is None
self.llexterns_header, self.llexterns_functions = \
generate_llfile(self.db,
- extern_decls,
- self.entrynode)
+ self.extern_decls,
+ self.entrynode,
+ self.standalone)
- def create_codewrite(self, func_name):
+ def create_codewriter(self):
# prevent running the same function twice in a test
- if func_name in self.function_count:
- postfix = '_%d' % self.function_count[func_name]
- self.function_count[func_name] += 1
+ if self.entry_func_name in self.function_count:
+ postfix = '_%d' % self.function_count[self.entry_func_name]
+ self.function_count[self.entry_func_name] += 1
else:
postfix = ''
- self.function_count[func_name] = 1
- filename = udir.join(func_name + postfix).new(ext='.ll')
+ self.function_count[self.entry_func_name] = 1
+ filename = udir.join(self.entry_func_name + postfix).new(ext='.ll')
f = open(str(filename), 'w')
return CodeWriter(f, self), filename
- def write_extern_decls(self, codewriter, extern_decls):
- for c_name, obj in extern_decls:
+ def write_extern_decls(self, codewriter):
+ for c_name, obj in self.extern_decls:
if isinstance(obj, lltype.LowLevelType):
if isinstance(obj, lltype.Ptr):
obj = obj.TO
l = "%%%s = type %s" % (c_name, self.db.repr_type(obj))
codewriter.append(l)
-
- def write_extern_impls(self, codewriter, extern_decls):
- for c_name, obj in extern_decls:
+
+ def write_extern_impls(self, codewriter):
+ for c_name, obj in self.extern_decls:
if c_name.startswith("RPyExc_"):
c_name = c_name[1:]
exc_repr = self.db.repr_constant(obj)[1]
@@ -242,6 +214,34 @@
codewriter.ret("sbyte*", "null")
codewriter.closefunc()
+ def compile_llvm_source(self, optimize=True,
+ exe_name=None, return_fn=False):
+ assert self.source_generated
+
+ assert hasattr(self, "filename")
+ if exe_name is not None:
+ assert self.standalone
+ assert not return_fn
+ return build_llvm_module.make_module_from_llvm(self, self.filename,
+ optimize=optimize,
+ exe_name=exe_name)
+ else:
+ assert not self.standalone
+
+ # use pyrex to create module for CPython
+ postfix = ''
+ basename = self.filename.purebasename + '_wrapper' + postfix + '.pyx'
+ pyxfile = self.filename.new(basename = basename)
+ write_pyx_wrapper(self, pyxfile)
+ res = build_llvm_module.make_module_from_llvm(self, self.filename,
+ pyxfile=pyxfile,
+ optimize=optimize)
+ wrap_fun = getattr(res, 'pypy_' + self.entry_func_name + "_wrapper")
+ if return_fn:
+ return wrap_fun
+
+ return res, wrap_fun
+
def _checkpoint(self, msg=None):
if not self.logging:
return
@@ -269,13 +269,11 @@
for s in stats:
log('STATS %s' % str(s))
-def genllvm(translator, gcpolicy=None, exceptionpolicy=None,
+def genllvm(translator, gcpolicy=None, exceptionpolicy=None, standalone=False,
log_source=False, logging=False, **kwds):
- gen = GenLLVM(translator,
- GcPolicy.new(gcpolicy),
- ExceptionPolicy.new(exceptionpolicy),
- logging=logging)
+ gen = GenLLVM(translator, gcpolicy, exceptionpolicy,
+ standalone, logging=logging)
filename = gen.gen_llvm_source()
if log_source:
@@ -283,7 +281,7 @@
return gen.compile_llvm_source(**kwds)
-def compile_module(function, annotation, view=False, **kwds):
+def genllvm_compile(function, annotation, view=False, **kwds):
t = Translator(function)
a = t.annotate(annotation)
a.simplify()
@@ -297,5 +295,4 @@
def compile_function(function, annotation, **kwds):
""" Helper - which get the compiled module from CPython. """
- mod = compile_module(function, annotation, **kwds)
- return getattr(mod, 'pypy_' + function.func_name + "_wrapper")
+ return compile_module(function, annotation, return_fn=True, **kwds)
Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c
==============================================================================
--- pypy/dist/pypy/translator/llvm/module/genexterns.c (original)
+++ pypy/dist/pypy/translator/llvm/module/genexterns.c Wed Nov 23 19:41:12 2005
@@ -45,17 +45,29 @@
return NULL;
}
-extern GC_all_interior_pointers;
+#include <gc/gc.h>
+#define USING_BOEHM_GC
char *LLVM_RPython_StartupCode();
+char *pypy_malloc(unsigned int size) {
+ // use the macros luke
+ return GC_MALLOC(size);
+}
+
+char *pypy_malloc_atomic(unsigned int size) {
+ return GC_MALLOC_ATOMIC(size);
+}
+
+#ifdef ENTRY_POINT_DEFINED
+
+extern GC_all_interior_pointers;
char *RPython_StartupCode() {
GC_all_interior_pointers = 0;
+ GC_INIT();
return LLVM_RPython_StartupCode();
}
-#ifdef ENTRY_POINT_DEFINED
-
int __ENTRY_POINT__(RPyListOfString *);
int main(int argc, char *argv[])
@@ -93,8 +105,16 @@
}
#else
+extern GC_all_interior_pointers;
+
+char *RPython_StartupCode() {
+ GC_all_interior_pointers = 0;
+ GC_INIT();
+ return LLVM_RPython_StartupCode();
+}
int Pyrex_RPython_StartupCode() {
+
char *error = RPython_StartupCode();
if (error != NULL) {
return 0;
Modified: pypy/dist/pypy/translator/llvm/module/support.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/module/support.py (original)
+++ pypy/dist/pypy/translator/llvm/module/support.py Wed Nov 23 19:41:12 2005
@@ -57,7 +57,6 @@
ret int %result
}
-
internal fastcc double %pypyop_float_abs(double %x) {
block0:
%cond1 = setge double %x, 0.0
Modified: pypy/dist/pypy/translator/llvm/test/runtest.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/test/runtest.py (original)
+++ pypy/dist/pypy/translator/llvm/test/runtest.py Wed Nov 23 19:41:12 2005
@@ -1,5 +1,5 @@
import py
-from pypy.translator.llvm.genllvm import compile_module
+from pypy.translator.llvm.genllvm import genllvm_compile
optimize_tests = False
MINIMUM_LLVM_VERSION = 1.6
@@ -18,26 +18,23 @@
v = int(v) / 10.0
return v
-def compile_function(function, annotation, **kwds):
+def llvm_test():
if not llvm_is_on_path():
py.test.skip("llvm not found")
-
+ return False
v = llvm_version()
if v < MINIMUM_LLVM_VERSION:
- py.test.skip("llvm version not up-to-date (found %.1f, should be >= %.1f)" % (v, MINIMUM_LLVM_VERSION))
+ py.test.skip("llvm version not up-to-date (found "
+ "%.1f, should be >= %.1f)" % (v, MINIMUM_LLVM_VERSION))
+ return False
+ return True
- mod = compile_module(function, annotation, optimize=optimize_tests,
- logging=False, **kwds)
- return getattr(mod, 'pypy_' + function.func_name + "_wrapper")
+def compile_test(function, annotation, **kwds):
+ if llvm_test():
+ return genllvm_compile(function, annotation, optimize=optimize_tests,
+ logging=False, **kwds)
-def compile_module_function(function, annotation, **kwds):
- if not llvm_is_on_path():
- py.test.skip("llvm not found")
+def compile_function(function, annotation, **kwds):
+ if llvm_test():
+ return compile_test(function, annotation, return_fn=True, **kwds)
- v, minimum = llvm_version(), 1.6
- if v < minimum:
- py.test.skip("llvm version not up-to-date (found %.1f, should be >= %.1f)" % (v, minimum))
-
- mod = compile_module(function, annotation, **kwds)
- f = getattr(mod, 'pypy_' + function.func_name + "_wrapper")
- return mod, f
Modified: pypy/dist/pypy/translator/llvm/test/test_gc.py
==============================================================================
--- pypy/dist/pypy/translator/llvm/test/test_gc.py (original)
+++ pypy/dist/pypy/translator/llvm/test/test_gc.py Wed Nov 23 19:41:12 2005
@@ -1,7 +1,7 @@
import sys
import py
-from pypy.translator.llvm.test.runtest import compile_module_function
+from pypy.translator.llvm.test.runtest import compile_test
def test_GC_malloc():
#XXX how to get to gcpolicy?
@@ -16,7 +16,7 @@
x += l[2]
i += 1
return x
- mod,f = compile_module_function(tuple_getitem, [int])
+ mod, f = compile_test(tuple_getitem, [int])
n = 5000
result = tuple_getitem(n)
assert f(n) == result
More information about the Pypy-commit
mailing list