[pypy-commit] pypy vecopt2: making the backend ready to be translated. stumbled over several problems not allowed in rpython (slice[1:-1], returning multiple values)
plan_rich
noreply at buildbot.pypy.org
Tue May 5 09:45:15 CEST 2015
Author: Richard Plangger <rich at pasra.at>
Branch: vecopt2
Changeset: r77068:0627a3228d2e
Date: 2015-03-11 15:38 +0100
http://bitbucket.org/pypy/pypy/changeset/0627a3228d2e/
Log: making the backend ready to be translated. stumbled over several
problems not allowed in rpython (slice[1:-1], returning multiple
values)
diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,4 @@
release/
!pypy/tool/release/
rpython/_cache/
+__pycache__/
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -16,7 +16,8 @@
call2_driver = jit.JitDriver(
name='numpy_call2',
greens=['shapelen', 'func', 'calc_dtype', 'res_dtype'],
- reds='auto')
+ reds='auto',
+ vectorize=True)
def call2(space, shape, func, calc_dtype, res_dtype, w_lhs, w_rhs, out):
# handle array_priority
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -288,6 +288,10 @@
# The import actually creates the udir directory
from rpython.tool.udir import udir
options.builddir = udir.ensure("build", dir=True)
+ else:
+ # if a user provides a path it must be converted to a local file system path
+ # otherwise ensure in create_package will fail
+ options.builddir = py.path.local(options.builddir)
assert '/' not in options.pypy_c
return create_package(basedir, options)
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -170,6 +170,9 @@
return getkind(self.A.OF) == 'int' \
and rffi.sizeof(self.A.OF) < symbolic.WORD
+ def get_item_size_in_bytes(self):
+ return rffi.sizeof(self.A.OF)
+
def get_item_integer_min(self):
if getkind(self.A.OF) != 'int':
assert False
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -208,6 +208,9 @@
def is_item_signed(self):
return self.flag == FLAG_SIGNED
+ def get_item_size_in_bytes(self):
+ return self.itemsize
+
def is_array_of_structs(self):
return self.flag == FLAG_STRUCT
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -60,8 +60,8 @@
loop.operations)
optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
if jitdriver_sd.vectorize:
- return optimize_vector(metainterp_sd, jitdriver_sd, loop,
- optimizations, start_state, export_state)
+ optimize_vector(metainterp_sd, jitdriver_sd, loop,
+ optimizations)
elif unroll:
return optimize_unroll(metainterp_sd, jitdriver_sd, loop,
optimizations, inline_short_preamble,
diff --git a/rpython/jit/metainterp/optimizeopt/readme.md b/rpython/jit/metainterp/optimizeopt/readme.md
--- a/rpython/jit/metainterp/optimizeopt/readme.md
+++ b/rpython/jit/metainterp/optimizeopt/readme.md
@@ -65,3 +65,22 @@
* Allocation Removal by Partial Evaluation in a Tracing JIT
Link: - http://www.stups.uni-duesseldorf.de/mediawiki/images/b/b0/Pub-BoCuFiLePeRi2011.pdf
+
+
+Setting user parameters in the jit
+===
+
+It is described often in the documenation that PyPy is a Python program that runs and after a certain point starts to analyse the program (annotate, rtype) and finally generate a c code for a virtual machine to run python programs. Thus at runtime it is very often the case that the program does not provide an implementation of a function, but later insert implementation to call sites. An example for that would be the parameter passing to the jit backend.
+
+`pypy/app_main.py` parses the arguments and provides invokes `set_param` on the internal jit module (`pypy/module/interp_pypy.py`). Some steps iron out some problems on the user side and finally pass it to `rpython/rlib/jit.py`.
+Following the `set_param` calls will lead to an empty method call in `_set_param`. At first glance this is very confusing. There are two things happening while compiling that make create the actual implemenation.
+First an `ExtParam` class is consturcted deriving from `ExtRegistryEntry`. Second, the method is filled with an actual implementation in `rpython/jit/metainterpreter/warmspot.py`. The method `rewrite_set_param_and_get_stats` find methods that call 'jit_marker' (first parameter is 'set_param'). Those functions are rewritten and invoke a `set_param_XXX`. In the program I have not seen a direct invocation of jit_marker yet, but the ExtRegistryEntry (in combination with its meta class) supplies methods to control the annotator. In the case it generates an operation that calls 'jit_marker'. After that the rewrite method is able to find that invocation and exchange the dummy call site with a real python function that sets the parameter.
+
+Test vs Runtime environment
+===
+
+Optimizer module
+---
+
+* The test environment instanciates mostly fake objects for generated objects, or objects that are selected at translation time of pypy. Examples: cpu, jitdriver_sd, descriptors, ...
+For descriptors this was not that obvious to me: `rpython/jit/backend/llgraph/*.py` contains nearly all descriptors, but only for testing purpose. Find the real implementations in `rpython/jit/backend/llsupport/descr.py`.
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -38,9 +38,9 @@
loop.operations[-1].setdescr(token)
return loop
- def assert_vectorize(self, loop, unfolded_loop, call_pure_results=None):
- optloop = self._do_optimize_loop(loop, call_pure_results, export_state=True)
- self.assert_equal(optloop, unfolded_loop)
+ def assert_vectorize(self, loop, expected_loop, call_pure_results=None):
+ self._do_optimize_loop(loop, call_pure_results, export_state=True)
+ self.assert_equal(loop, expected_loop)
def assert_unroll_loop_equals(self, loop, expected_loop, \
unroll_factor = -1, call_pure_results=None):
@@ -50,8 +50,8 @@
if unroll_factor == -1:
opt._gather_trace_information(loop)
unroll_factor = opt.get_estimated_unroll_factor()
- opt_loop = opt.unroll_loop_iterations(loop, unroll_factor)
- self.assert_equal(opt_loop, expected_loop)
+ opt.unroll_loop_iterations(loop, unroll_factor)
+ self.assert_equal(loop, expected_loop)
def assert_def_use(self, graph, from_instr_index, to_instr_index):
assert graph.instr_dependency(from_instr_index,
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -1,28 +1,19 @@
import sys
from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.jit.backend.llgraph.runner import ArrayDescr
-from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const
-from rpython.jit.metainterp.inliner import Inliner
-from rpython.jit.metainterp.optimize import InvalidLoop
from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp
+from rpython.jit.metainterp.resoperation import rop
from rpython.jit.metainterp.resume import Snapshot
-from rpython.jit.metainterp import compile
from rpython.rlib.debug import debug_print, debug_start, debug_stop
-def optimize_vector(metainterp_sd, jitdriver_sd, loop, optimizations, start_state=None,
- export_state=True):
+def optimize_vector(metainterp_sd, jitdriver_sd, loop, optimizations):
opt = OptVectorize(metainterp_sd, jitdriver_sd, loop, optimizations)
- opt_loop = opt.propagate_all_forward(start_state, export_state)
- if opt.vectorized:
- return opt_loop
- # vectorization is not possible, propagate only normal optimizations
- opt = Optimizer(metainterp_sd, jitdriver_sd, loop, optimizations)
- opt.propagate_all_forward()
- return loop
-
+ opt_loop = opt.propagate_all_forward()
+ if not opt.vectorized:
+ # vectorization is not possible, propagate only normal optimizations
+ def_opt = Optimizer(metainterp_sd, jitdriver_sd, loop, optimizations)
+ def_opt.propagate_all_forward()
class VectorizeOptimizer(Optimizer):
def setup(self):
@@ -31,8 +22,6 @@
class OptVectorize(Optimization):
""" Try to unroll the loop and find instructions to group """
- inline_short_preamble = True
-
def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations):
self.optimizer = VectorizeOptimizer(metainterp_sd, jitdriver_sd,
loop, optimizations)
@@ -49,7 +38,7 @@
def unroll_loop_iterations(self, loop, unroll_factor):
label_op = loop.operations[0]
jump_op = loop.operations[-1]
- operations = loop.operations[1:-1]
+ operations = [loop.operations[i] for i in range(1,len(loop.operations)-1)]
loop.operations = []
iterations = [[op.clone() for op in operations]]
@@ -107,10 +96,9 @@
loop.operations.append(op)
loop.operations.append(jump_op)
- return loop
-
def _gather_trace_information(self, loop):
- for op in loop.operations:
+ for i,op in enumerate(loop.operations):
+ self.loop_vectorizer_checker._op_index = i
self.loop_vectorizer_checker.inspect_operation(op)
def get_estimated_unroll_factor(self, force_reg_bytes = -1):
@@ -119,13 +107,12 @@
byte_count = self.loop_vectorizer_checker.smallest_type_bytes
simd_vec_reg_bytes = 16 # TODO get from cpu
if force_reg_bytes > 0:
- simd_vec_reg_bytes = force_simd_vec_reg_bytes
+ simd_vec_reg_bytes = force_reg_bytes
unroll_factor = simd_vec_reg_bytes // byte_count
return unroll_factor
- def propagate_all_forward(self, starting_state, export_state=True):
+ def propagate_all_forward(self):
- self.optimizer.exporting_state = export_state
loop = self.optimizer.loop
self.optimizer.clear_newoperations()
@@ -143,28 +130,42 @@
self.unroll_loop_iterations(loop, unroll_factor)
-
self.vectorized = True
- return loop
-
class LoopVectorizeChecker(object):
def __init__(self):
self.smallest_type_bytes = 0
+ self._op_index = 0
+ self.mem_ref_indices = []
+
+ def add_memory_ref(self, i):
+ self.mem_ref_indices.append(i)
def count_RAW_LOAD(self, op):
+ self.add_memory_ref(self._op_index)
descr = op.getdescr()
- assert isinstance(descr, ArrayDescr) # TODO prove this right
- if not isinstance(descr.A.OF, lltype.Ptr):
- byte_count = rffi.sizeof(descr.A.OF)
+ if not descr.is_array_of_pointers():
+ byte_count = descr.get_item_size_in_bytes()
if self.smallest_type_bytes == 0 \
or byte_count < self.smallest_type_bytes:
self.smallest_type_bytes = byte_count
def default_count(self, operation):
pass
-
dispatch_opt = make_dispatcher_method(LoopVectorizeChecker, 'count_',
default=LoopVectorizeChecker.default_count)
LoopVectorizeChecker.inspect_operation = dispatch_opt
+
+"""
+Implementation of the algorithm introduced by Larsen. Refer to
+'Exploiting Superword Level Parallelism with Multimedia Instruction Sets'
+for more details.
+"""
+
+class Pack(object):
+ pass
+
+class Pair(object):
+ pass
+
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -392,6 +392,8 @@
graph.func._dont_inline_ = True
graph.func._jit_unroll_safe_ = True
jd.jitdriver = block.operations[pos].args[1].value
+ jd.vectorize = jd.jitdriver.vectorize
+ del jd.jitdriver.vectorize
jd.portal_runner_ptr = "<not set so far>"
jd.result_type = history.getkind(jd.portal_graph.getreturnvar()
.concretetype)[0]
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -297,8 +297,7 @@
self.warmrunnerdesc.memory_manager.max_unroll_recursion = value
def set_param_vectorize(self, value):
- if self.warmrunnerdesc:
- self.warmrunnerdesc.vectorize = bool(value)
+ self.vectorize = bool(value)
def disable_noninlinable_function(self, greenkey):
cell = self.JitCell.ensure_jit_cell_at_key(greenkey)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -502,7 +502,7 @@
get_jitcell_at=None, set_jitcell_at=None,
get_printable_location=None, confirm_enter_jit=None,
can_never_inline=None, should_unroll_one_iteration=None,
- name='jitdriver', check_untranslated=True):
+ name='jitdriver', check_untranslated=True, vectorize=False):
if greens is not None:
self.greens = greens
self.name = name
@@ -538,6 +538,7 @@
self.can_never_inline = can_never_inline
self.should_unroll_one_iteration = should_unroll_one_iteration
self.check_untranslated = check_untranslated
+ self.vectorize = vectorize
def _freeze_(self):
return True
More information about the pypy-commit
mailing list