[pypy-svn] r75863 - in pypy/branch/fast-forward: . dotviewer include lib-python/modified-2.5.2 lib-python/modified-2.5.2/distutils lib-python/modified-2.5.2/distutils/command lib_pypy lib_pypy/pypy_test py/_process pypy pypy/_interfaces pypy/annotation pypy/doc pypy/doc/jit pypy/interpreter pypy/interpreter/pyparser pypy/interpreter/pyparser/test pypy/interpreter/test pypy/jit/backend pypy/jit/backend/llgraph pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/jit/tl pypy/jit/tl/spli pypy/jit/tl/tinyframe pypy/module pypy/module/__builtin__ pypy/module/__builtin__/test pypy/module/_codecs pypy/module/_codecs/test pypy/module/_stackless pypy/module/_stackless/test pypy/module/cpyext pypy/module/cpyext/test pypy/module/marshal pypy/module/operator/test pypy/module/sys pypy/module/sys/test pypy/module/termios pypy/module/test_lib_pypy/ctypes_tests pypy/module/unicodedata pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/rsre/test pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/lltypesystem/test pypy/rpython/microbench pypy/rpython/module pypy/rpython/test pypy/tool/release pypy/tool/release/test pypy/tool/test pypy/translator pypy/translator/backendopt/test pypy/translator/c/gcc pypy/translator/cli pypy/translator/cli/test pypy/translator/goal/test2 pypy/translator/jvm pypy/translator/jvm/test pypy/translator/platform

benjamin at codespeak.net benjamin at codespeak.net
Tue Jul 6 00:00:40 CEST 2010


Author: benjamin
Date: Tue Jul  6 00:00:33 2010
New Revision: 75863

Added:
   pypy/branch/fast-forward/include/   (props changed)
      - copied from r75862, pypy/trunk/include/
   pypy/branch/fast-forward/lib_pypy/pypy_test/
   pypy/branch/fast-forward/lib_pypy/pypy_test/__init__.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/inprogress_test_binascii_extra.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/no_test_pickle_extra.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_binascii.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_collections.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_coroutine.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_ctypes_support.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_datetime.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_dbm_extra.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_defaultdict.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_deque_extra.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_exception_extra.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_functools.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_grp_extra.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_hashlib.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_locale.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_marshal_extra.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_md5_extra.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_pickle_extra.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_pyexpat.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_resource.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_runpy.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_sha_extra.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_stackless.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_stackless_pickling.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_struct_extra.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_structseq.py   (contents, props changed)
   pypy/branch/fast-forward/lib_pypy/pypy_test/test_syslog.py   (contents, props changed)
   pypy/branch/fast-forward/pypy/jit/metainterp/jitdriver.py
      - copied unchanged from r75862, pypy/trunk/pypy/jit/metainterp/jitdriver.py
   pypy/branch/fast-forward/pypy/jit/metainterp/test/test_jitdriver.py
      - copied unchanged from r75862, pypy/trunk/pypy/jit/metainterp/test/test_jitdriver.py
Removed:
   pypy/branch/fast-forward/pypy/_interfaces/
   pypy/branch/fast-forward/pypy/module/_codecs/app_codecs.py
   pypy/branch/fast-forward/pypy/module/unicodedata/LICENSE
Modified:
   pypy/branch/fast-forward/   (props changed)
   pypy/branch/fast-forward/LICENSE
   pypy/branch/fast-forward/dotviewer/sshgraphserver.py
   pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/cmd.py
   pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/build_ext.py
   pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/install.py
   pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py
   pypy/branch/fast-forward/lib-python/modified-2.5.2/site.py
   pypy/branch/fast-forward/lib_pypy/stackless.py
   pypy/branch/fast-forward/py/_process/cmdexec.py
   pypy/branch/fast-forward/pypy/annotation/model.py
   pypy/branch/fast-forward/pypy/annotation/unaryop.py
   pypy/branch/fast-forward/pypy/conftest.py
   pypy/branch/fast-forward/pypy/doc/index.txt
   pypy/branch/fast-forward/pypy/doc/jit/pyjitpl5.txt
   pypy/branch/fast-forward/pypy/doc/release-1.3.0.txt
   pypy/branch/fast-forward/pypy/doc/stackless.txt
   pypy/branch/fast-forward/pypy/interpreter/pyopcode.py
   pypy/branch/fast-forward/pypy/interpreter/pyparser/parsestring.py
   pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_parsestring.py
   pypy/branch/fast-forward/pypy/interpreter/test/test_module.py
   pypy/branch/fast-forward/pypy/interpreter/unicodehelper.py
   pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py
   pypy/branch/fast-forward/pypy/jit/backend/llgraph/runner.py
   pypy/branch/fast-forward/pypy/jit/backend/model.py
   pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py
   pypy/branch/fast-forward/pypy/jit/backend/test/support.py
   pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py
   pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py
   pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_zrpy_gc.py
   pypy/branch/fast-forward/pypy/jit/codewriter/assembler.py
   pypy/branch/fast-forward/pypy/jit/codewriter/call.py
   pypy/branch/fast-forward/pypy/jit/codewriter/codewriter.py
   pypy/branch/fast-forward/pypy/jit/codewriter/jitcode.py
   pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py
   pypy/branch/fast-forward/pypy/jit/codewriter/support.py
   pypy/branch/fast-forward/pypy/jit/codewriter/test/test_assembler.py
   pypy/branch/fast-forward/pypy/jit/codewriter/test/test_call.py
   pypy/branch/fast-forward/pypy/jit/codewriter/test/test_codewriter.py
   pypy/branch/fast-forward/pypy/jit/codewriter/test/test_flatten.py
   pypy/branch/fast-forward/pypy/jit/metainterp/blackhole.py
   pypy/branch/fast-forward/pypy/jit/metainterp/compile.py
   pypy/branch/fast-forward/pypy/jit/metainterp/executor.py
   pypy/branch/fast-forward/pypy/jit/metainterp/history.py
   pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py
   pypy/branch/fast-forward/pypy/jit/metainterp/resoperation.py
   pypy/branch/fast-forward/pypy/jit/metainterp/resume.py
   pypy/branch/fast-forward/pypy/jit/metainterp/test/test_basic.py
   pypy/branch/fast-forward/pypy/jit/metainterp/test/test_blackhole.py
   pypy/branch/fast-forward/pypy/jit/metainterp/test/test_compile.py
   pypy/branch/fast-forward/pypy/jit/metainterp/test/test_list.py
   pypy/branch/fast-forward/pypy/jit/metainterp/test/test_pyjitpl.py
   pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmspot.py
   pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmstate.py
   pypy/branch/fast-forward/pypy/jit/metainterp/test/test_ztranslation.py
   pypy/branch/fast-forward/pypy/jit/metainterp/virtualizable.py
   pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py
   pypy/branch/fast-forward/pypy/jit/metainterp/warmstate.py
   pypy/branch/fast-forward/pypy/jit/tl/spli/targetspli.py
   pypy/branch/fast-forward/pypy/jit/tl/targettlc.py
   pypy/branch/fast-forward/pypy/jit/tl/targettlr.py
   pypy/branch/fast-forward/pypy/jit/tl/tinyframe/targettinyframe.py
   pypy/branch/fast-forward/pypy/module/__builtin__/functional.py
   pypy/branch/fast-forward/pypy/module/__builtin__/test/test_functional.py
   pypy/branch/fast-forward/pypy/module/_codecs/__init__.py
   pypy/branch/fast-forward/pypy/module/_codecs/interp_codecs.py
   pypy/branch/fast-forward/pypy/module/_codecs/test/test_codecs.py
   pypy/branch/fast-forward/pypy/module/_stackless/interp_coroutine.py
   pypy/branch/fast-forward/pypy/module/_stackless/rcoroutine.py
   pypy/branch/fast-forward/pypy/module/_stackless/test/test_coroutine.py
   pypy/branch/fast-forward/pypy/module/conftest.py
   pypy/branch/fast-forward/pypy/module/cpyext/api.py
   pypy/branch/fast-forward/pypy/module/cpyext/test/test_api.py
   pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py
   pypy/branch/fast-forward/pypy/module/operator/test/test_operator.py
   pypy/branch/fast-forward/pypy/module/sys/state.py
   pypy/branch/fast-forward/pypy/module/sys/test/test_initialpath.py
   pypy/branch/fast-forward/pypy/module/sys/version.py
   pypy/branch/fast-forward/pypy/module/termios/interp_termios.py
   pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/   (props changed)
   pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py
   pypy/branch/fast-forward/pypy/objspace/std/stringobject.py
   pypy/branch/fast-forward/pypy/objspace/std/test/test_shadowtracking.py
   pypy/branch/fast-forward/pypy/objspace/std/unicodeobject.py
   pypy/branch/fast-forward/pypy/rlib/debug.py
   pypy/branch/fast-forward/pypy/rlib/libffi.py
   pypy/branch/fast-forward/pypy/rlib/rarithmetic.py
   pypy/branch/fast-forward/pypy/rlib/rcoroutine.py
   pypy/branch/fast-forward/pypy/rlib/rsre/test/targetrsre.py
   pypy/branch/fast-forward/pypy/rlib/rstring.py
   pypy/branch/fast-forward/pypy/rlib/runicode.py
   pypy/branch/fast-forward/pypy/rlib/test/test_debug.py
   pypy/branch/fast-forward/pypy/rlib/test/test_runicode.py
   pypy/branch/fast-forward/pypy/rpython/lltypesystem/rffi.py
   pypy/branch/fast-forward/pypy/rpython/lltypesystem/test/test_rffi.py
   pypy/branch/fast-forward/pypy/rpython/microbench/microbench.py
   pypy/branch/fast-forward/pypy/rpython/module/ll_termios.py
   pypy/branch/fast-forward/pypy/rpython/rstr.py
   pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py
   pypy/branch/fast-forward/pypy/tool/release/force-builds.py
   pypy/branch/fast-forward/pypy/tool/release/make_release.py
   pypy/branch/fast-forward/pypy/tool/release/package.py
   pypy/branch/fast-forward/pypy/tool/release/test/test_package.py
   pypy/branch/fast-forward/pypy/tool/test/test_killsubprocess.py
   pypy/branch/fast-forward/pypy/translator/backendopt/test/test_mallocprediction.py
   pypy/branch/fast-forward/pypy/translator/c/gcc/trackgcroot.py
   pypy/branch/fast-forward/pypy/translator/cli/gencli.py
   pypy/branch/fast-forward/pypy/translator/cli/query.py
   pypy/branch/fast-forward/pypy/translator/cli/rte.py
   pypy/branch/fast-forward/pypy/translator/cli/support.py
   pypy/branch/fast-forward/pypy/translator/cli/test/runtest.py
   pypy/branch/fast-forward/pypy/translator/driver.py
   pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py
   pypy/branch/fast-forward/pypy/translator/jvm/genjvm.py
   pypy/branch/fast-forward/pypy/translator/jvm/test/runtest.py
   pypy/branch/fast-forward/pypy/translator/platform/windows.py
Log:
merge from trunk

Modified: pypy/branch/fast-forward/LICENSE
==============================================================================
--- pypy/branch/fast-forward/LICENSE	(original)
+++ pypy/branch/fast-forward/LICENSE	Tue Jul  6 00:00:33 2010
@@ -152,3 +152,26 @@
 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
 TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+License for 'pypy/module/unicodedata/'
+======================================
+
+The following files are from the website of The Unicode Consortium
+at http://www.unicode.org/. For the terms of use of these files, see
+http://www.unicode.org/terms_of_use.html
+
+    CompositionExclusions-3.2.0.txt
+    CompositionExclusions-4.1.0.txt
+    CompositionExclusions-5.0.0.txt
+    EastAsianWidth-3.2.0.txt
+    EastAsianWidth-4.1.0.txt
+    EastAsianWidth-5.0.0.txt
+    UnicodeData-3.2.0.txt
+    UnicodeData-4.1.0.txt
+    UnicodeData-5.0.0.txt
+    
+The following files are derived from files from the above website. The same
+terms of use apply.
+    UnihanNumeric-3.2.0.txt
+    UnihanNumeric-4.1.0.txt
+    UnihanNumeric-5.0.0.txt

Modified: pypy/branch/fast-forward/dotviewer/sshgraphserver.py
==============================================================================
--- pypy/branch/fast-forward/dotviewer/sshgraphserver.py	(original)
+++ pypy/branch/fast-forward/dotviewer/sshgraphserver.py	Tue Jul  6 00:00:33 2010
@@ -21,7 +21,7 @@
     remoteport = random.randrange(10000, 20000)
     #  ^^^ and just hope there is no conflict
 
-    args = ['ssh', '-C', '-R%d:127.0.0.1:%d' % (remoteport, localport)]
+    args = ['ssh', '-S', 'none', '-C', '-R%d:127.0.0.1:%d' % (remoteport, localport)]
     args = args + sshargs + ['python -u -c "exec input()"']
     print ' '.join(args[:-1])
     p = subprocess.Popen(args, bufsize=0,

Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/cmd.py
==============================================================================
--- pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/cmd.py	(original)
+++ pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/cmd.py	Tue Jul  6 00:00:33 2010
@@ -17,9 +17,9 @@
 def _easy_install_get_site_dirs():
     # return a list of 'site' dirs for easy_install
     from pkg_resources import normalize_path
+    from distutils.sysconfig import get_python_lib
     sitedirs = filter(None,os.environ.get('PYTHONPATH','').split(os.pathsep))
-    pypylib = 'pypy%d.%d' % sys.pypy_version_info[:2]
-    sitedirs.append(os.path.join(sys.prefix, 'lib', pypylib, 'site-packages'))
+    sitedirs.append(get_python_lib(standard_lib=False))
     sitedirs = map(normalize_path, sitedirs)
     return sitedirs
 

Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/build_ext.py
==============================================================================
--- pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/build_ext.py	(original)
+++ pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/build_ext.py	Tue Jul  6 00:00:33 2010
@@ -167,7 +167,7 @@
         # for Release and Debug builds.
         # also Python's library directory must be appended to library_dirs
         if os.name == 'nt':
-            self.library_dirs.append(os.path.join(sys.prefix, 'pypy', '_interfaces'))
+            self.library_dirs.append(os.path.join(sys.prefix, 'include'))
             if self.debug:
                 self.build_temp = os.path.join(self.build_temp, "Debug")
             else:

Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/install.py
==============================================================================
--- pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/install.py	(original)
+++ pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/command/install.py	Tue Jul  6 00:00:33 2010
@@ -67,8 +67,8 @@
         'data'   : '$base',
         },
     'pypy': {
-        'purelib': '$base/lib/pypy$pypy_version_short/site-packages',
-        'platlib': '$base/lib/pypy$pypy_version_short/site-packages',
+        'purelib': '$base/site-packages',
+        'platlib': '$base/site-packages',
         'headers': '$base/include',
         'scripts': '$base/bin',
         'data'   : '$base',

Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py
==============================================================================
--- pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py	(original)
+++ pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py	Tue Jul  6 00:00:33 2010
@@ -13,12 +13,7 @@
 
 def get_python_inc(plat_specific=0, prefix=None):
     from os.path import join as j
-    cand = j(sys.pypy_prefix, 'include')
-    if os.path.exists(cand):
-        return cand
-    if plat_specific:
-        return j(sys.prefix, "pypy", "_interfaces")
-    return j(sys.prefix, 'pypy', 'module', 'cpyext', 'include')
+    return j(sys.prefix, 'include')
 
 def get_python_version():
     """Return a string containing the major and minor Python version,
@@ -47,8 +42,7 @@
             "calls to get_python_lib(standard_lib=1) cannot succeed")
     if prefix is None:
         prefix = PREFIX
-    pypylib = 'pypy%d.%d' % sys.pypy_version_info[:2]
-    return os.path.join(prefix, 'lib', pypylib, 'site-packages')
+    return os.path.join(prefix, 'site-packages')
 
 
 _config_vars = None

Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/site.py
==============================================================================
--- pypy/branch/fast-forward/lib-python/modified-2.5.2/site.py	(original)
+++ pypy/branch/fast-forward/lib-python/modified-2.5.2/site.py	Tue Jul  6 00:00:33 2010
@@ -176,8 +176,8 @@
 def addsitepackages(known_paths):
     """Add site-packages to sys.path, in a PyPy-specific way."""
     if hasattr(sys, 'pypy_version_info'):
-        pypylib = 'pypy%d.%d' % sys.pypy_version_info[:2]
-        sitedir = os.path.join(sys.prefix, 'lib', pypylib, 'site-packages')
+        from distutils.sysconfig import get_python_lib
+        sitedir = get_python_lib(standard_lib=False)
         if os.path.isdir(sitedir):
             addsitedir(sitedir, known_paths)
     return None

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/__init__.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/__init__.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1 @@
+#

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/inprogress_test_binascii_extra.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/inprogress_test_binascii_extra.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,19 @@
+from __future__ import absolute_import
+from .. import binascii
+
+def test_uu():
+    assert binascii.b2a_uu('1234567') == "',3(S-#4V-P  \n"
+    assert binascii.b2a_uu('123456789012345678901234567890123456789012345') == 'M,3(S-#4V-S at Y,#$R,S0U-C<X.3 Q,C,T-38W.#DP,3(S-#4V-S at Y,#$R,S0U\n'
+    try:
+        assert binascii.b2a_uu('1234567890123456789012345678901234567890123456')
+    except binascii.Error:
+        pass
+    else:
+        assert False, "Expected binascii.Error on oversize input."
+    assert binascii.b2a_uu('1234567') == "',3(S-#4V-P  \n"
+    assert binascii.b2a_uu('123456789012345678901234567890123456789012345')  == 'M,3(S-#4V-S at Y,#$R,S0U-C<X.3 Q,C,T-38W.#DP,3(S-#4V-S at Y,#$R,S0U\n'
+
+
+def test_base64():
+    assert binascii.b2a_base64('xxxx') == 'eHh4eA==\n'
+

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/no_test_pickle_extra.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/no_test_pickle_extra.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,40 @@
+"""
+Disabled for now.  This should run at app-level, too.
+"""
+import pickle
+
+class Picklable(object):
+    def __init__(self, a=555):
+        self.a = a
+    def __eq__(self, other):
+        return self.a == other.a
+    def __str__(self):
+        return '%s(%r)' % (self.__class__.__name__, self.a)
+    __repr__ = __str__
+
+class PicklableSpecial2(Picklable):
+    def __reduce__(self):
+        return self.__class__, (self.a,)
+
+class PicklableSpecial3(Picklable):
+    def __reduce__(self):
+        return self.__class__, (), self.a
+    def __setstate__(self, a):
+        self.a = a
+
+class PicklableSpecial4(Picklable):
+    def __reduce_ex__(self, proto):
+        return self.__class__, (), self.a
+    def __setstate__(self, a):
+        self.a = a
+
+def _pickle_some(x):
+    for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+        s = pickle.dumps(x, proto)
+        y = pickle.loads(s)
+        assert x == y
+
+_pickle_some(Picklable(5))
+_pickle_some(PicklableSpecial2(66))
+_pickle_some(PicklableSpecial3(7))
+_pickle_some(PicklableSpecial4(17))

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_binascii.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_binascii.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,168 @@
+from __future__ import absolute_import
+import py
+from .. import binascii
+
+# Create binary test data
+data = "The quick brown fox jumps over the lazy dog.\r\n"
+# Be slow so we don't depend on other modules
+data += "".join(map(chr, xrange(256)))
+data += "\r\nHello world.\n"
+
+def test_exceptions():
+    # Check module exceptions
+    assert issubclass(binascii.Error, Exception)
+    assert issubclass(binascii.Incomplete, Exception)
+
+def test_functions():
+    # Check presence of all functions
+    funcs = []
+    for suffix in "base64", "hqx", "uu", "hex":
+        prefixes = ["a2b_", "b2a_"]
+        if suffix == "hqx":
+            prefixes.extend(["crc_", "rlecode_", "rledecode_"])
+        for prefix in prefixes:
+            name = prefix + suffix
+            assert callable(getattr(binascii, name))
+            py.test.raises(TypeError, getattr(binascii, name))
+    for name in ("hexlify", "unhexlify"):
+        assert callable(getattr(binascii, name))
+        py.test.raises(TypeError, getattr(binascii, name))
+
+def test_base64valid():
+    # Test base64 with valid data
+    MAX_BASE64 = 57
+    lines = []
+    for i in range(0, len(data), MAX_BASE64):
+        b = data[i:i+MAX_BASE64]
+        a = binascii.b2a_base64(b)
+        lines.append(a)
+    res = ""
+    for line in lines:
+        b = binascii.a2b_base64(line)
+        res = res + b
+    assert res == data
+
+def test_base64invalid():
+    # Test base64 with random invalid characters sprinkled throughout
+    # (This requires a new version of binascii.)
+    MAX_BASE64 = 57
+    lines = []
+    for i in range(0, len(data), MAX_BASE64):
+        b = data[i:i+MAX_BASE64]
+        a = binascii.b2a_base64(b)
+        lines.append(a)
+
+    fillers = ""
+    valid = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/"
+    for i in xrange(256):
+        c = chr(i)
+        if c not in valid:
+            fillers += c
+    def addnoise(line):
+        noise = fillers
+        ratio = len(line) // len(noise)
+        res = ""
+        while line and noise:
+            if len(line) // len(noise) > ratio:
+                c, line = line[0], line[1:]
+            else:
+                c, noise = noise[0], noise[1:]
+            res += c
+        return res + noise + line
+    res = ""
+    for line in map(addnoise, lines):
+        b = binascii.a2b_base64(line)
+        res += b
+    assert res == data
+
+    # Test base64 with just invalid characters, which should return
+    # empty strings. TBD: shouldn't it raise an exception instead ?
+    assert binascii.a2b_base64(fillers) == ''
+
+def test_uu():
+    MAX_UU = 45
+    lines = []
+    for i in range(0, len(data), MAX_UU):
+        b = data[i:i+MAX_UU]
+        a = binascii.b2a_uu(b)
+        lines.append(a)
+    res = ""
+    for line in lines:
+        b = binascii.a2b_uu(line)
+        res += b
+    assert res == data
+
+    assert binascii.a2b_uu("\x7f") == "\x00"*31
+    assert binascii.a2b_uu("\x80") == "\x00"*32
+    assert binascii.a2b_uu("\xff") == "\x00"*31
+    py.test.raises(binascii.Error, binascii.a2b_uu, "\xff\x00")
+    py.test.raises(binascii.Error, binascii.a2b_uu, "!!!!")
+
+    py.test.raises(binascii.Error, binascii.b2a_uu, 46*"!")
+
+def test_crc32():
+    crc = binascii.crc32("Test the CRC-32 of")
+    crc = binascii.crc32(" this string.", crc)
+    assert crc == 1571220330
+    
+    crc = binascii.crc32('frotz\n', 0)
+    assert crc == -372923920
+
+    py.test.raises(TypeError, binascii.crc32)
+
+def test_hex():
+    # test hexlification
+    s = '{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000'
+    t = binascii.b2a_hex(s)
+    u = binascii.a2b_hex(t)
+    assert s == u
+    py.test.raises(TypeError, binascii.a2b_hex, t[:-1])
+    py.test.raises(TypeError, binascii.a2b_hex, t[:-1] + 'q')
+
+    # Verify the treatment of Unicode strings
+    assert binascii.hexlify(unicode('a', 'ascii')) == '61'
+
+def test_qp():
+    # A test for SF bug 534347 (segfaults without the proper fix)
+    try:
+        binascii.a2b_qp("", **{1:1})
+    except TypeError:
+        pass
+    else:
+        fail("binascii.a2b_qp(**{1:1}) didn't raise TypeError")
+    assert binascii.a2b_qp("= ") == "= "
+    assert binascii.a2b_qp("==") == "="
+    assert binascii.a2b_qp("=AX") == "=AX"
+    py.test.raises(TypeError, binascii.b2a_qp, foo="bar")
+    assert binascii.a2b_qp("=00\r\n=00") == "\x00\r\n\x00"
+    assert binascii.b2a_qp("\xff\r\n\xff\n\xff") == "=FF\r\n=FF\r\n=FF"
+    target = "0"*75+"=\r\n=FF\r\n=FF\r\n=FF"
+    assert binascii.b2a_qp("0"*75+"\xff\r\n\xff\r\n\xff") == target
+
+def test_empty_string():
+    # A test for SF bug #1022953.  Make sure SystemError is not raised.
+    for n in ['b2a_qp', 'a2b_hex', 'b2a_base64', 'a2b_uu', 'a2b_qp',
+              'b2a_hex', 'unhexlify', 'hexlify', 'crc32', 'b2a_hqx',
+              'a2b_hqx', 'a2b_base64', 'rlecode_hqx', 'b2a_uu',
+              'rledecode_hqx']:
+        f = getattr(binascii, n)
+        f('')
+    binascii.crc_hqx('', 0)
+
+def test_qp_bug_case():
+    assert binascii.b2a_qp('y'*77, False, False) == 'y'*75 + '=\nyy'
+    assert binascii.b2a_qp(' '*77, False, False) == ' '*75 + '=\n =20'
+    assert binascii.b2a_qp('y'*76, False, False) == 'y'*76
+    assert binascii.b2a_qp(' '*76, False, False) == ' '*75 + '=\n=20'
+
+def test_wrong_padding():
+    s = 'CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3IQ'
+    py.test.raises(binascii.Error, binascii.a2b_base64, s)
+
+def test_crap_after_padding():
+    s = 'xxx=axxxx'
+    assert binascii.a2b_base64(s) == '\xc7\x1c'
+
+def test_wrong_args():
+    # this should grow as a way longer list
+    py.test.raises(TypeError, binascii.a2b_base64, 42)

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_collections.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_collections.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,24 @@
+from __future__ import absolute_import
+from .. import collections
+import py
+
+def test_deque_remove_empty():
+    d = collections.deque([])
+    py.test.raises(ValueError, d.remove, 1)
+
+def test_deque_remove_mutating():
+    class MutatingCmp(object):
+        def __eq__(self, other):
+            d.clear()
+            return True
+
+    d = collections.deque([MutatingCmp()])
+    py.test.raises(IndexError, d.remove, 1)
+    
+class SubclassWithKwargs(collections.deque):
+    def __init__(self, newarg=1):
+        collections.deque.__init__(self)
+
+def test_subclass_with_kwargs():
+    # SF bug #1486663 -- this used to erroneously raise a TypeError
+    SubclassWithKwargs(newarg=1)

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_coroutine.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_coroutine.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,214 @@
+from __future__ import absolute_import
+from py.test import skip, raises
+
+try:
+    from ..stackless import coroutine
+except ImportError, e:
+    skip('cannot import stackless: %s' % (e,))
+
+
+class Test_Coroutine:
+
+    def test_is_zombie(self):
+        co = coroutine()
+        def f():
+            print 'in coro'
+        assert not co.is_zombie
+        co.bind(f)
+        assert not co.is_zombie
+        co.switch()
+        assert not co.is_zombie
+
+    def test_is_zombie_del_without_frame(self):
+        try:
+            import _stackless # are we on pypy with a stackless build?
+        except ImportError:
+            skip("only works on pypy-c-stackless")
+        import gc
+        res = []
+        class MyCoroutine(coroutine):
+            def __del__(self):
+                res.append(self.is_zombie)
+        def f():
+            print 'in coro'
+        co = MyCoroutine()
+        co.bind(f)
+        co.switch()
+        del co
+        for i in range(10):
+            gc.collect()
+            if res:
+                break
+        co = coroutine()
+        co.bind(f)
+        co.switch()
+        assert res[0], "is_zombie was False in __del__"
+
+    def test_is_zombie_del_with_frame(self):
+        try:
+            import _stackless # are we on pypy with a stackless build?
+        except ImportError:
+            skip("only works on pypy-c-stackless")
+        import gc
+        res = []
+        class MyCoroutine(coroutine):
+            def __del__(self):
+                res.append(self.is_zombie)
+        main = coroutine.getcurrent()
+        def f():
+            print 'in coro'
+            main.switch()
+        co = MyCoroutine()
+        co.bind(f)
+        co.switch()
+        del co
+        for i in range(10):
+            gc.collect()
+            if res:
+                break
+        co = coroutine()
+        co.bind(f)
+        co.switch()
+        assert res[0], "is_zombie was False in __del__"
+
+    def test_raise_propagate(self):
+        co = coroutine()
+        def f():
+            return 1/0
+        co.bind(f)
+        try:
+            co.switch()
+        except ZeroDivisionError:
+            pass
+        else:
+            raise AssertionError("exception not propagated")
+
+    def test_strange_test(self):
+        def f():
+            return 42
+        def create():
+            b = coroutine()
+            b.bind(f)
+            b.switch()
+            return b
+        a = coroutine()
+        a.bind(create)
+        b = a.switch()
+        def nothing():
+            pass
+        a.bind(nothing)
+        def kill():
+            a.kill()
+        b.bind(kill)
+        b.switch()
+
+    def test_kill(self):
+        co = coroutine()
+        def f():
+            pass
+        assert not co.is_alive
+        co.bind(f)
+        assert co.is_alive
+        co.kill()
+        assert not co.is_alive
+
+    def test_catch_coroutineexit(self):
+        coroutineexit = []
+        co_a = coroutine()
+        co_test = coroutine.getcurrent()
+
+        def a():
+            try:
+                co_test.switch()
+            except CoroutineExit:
+                coroutineexit.append(True)
+                raise 
+        
+        co_a.bind(a)
+        co_a.switch()
+        assert co_a.is_alive
+        
+        co_a.kill()
+        assert coroutineexit == [True]
+        assert not co_a.is_alive
+        
+    def test_throw(self):
+        exceptions = []
+        co = coroutine()
+        def f(main):
+            try:
+                main.switch()
+            except RuntimeError:
+                exceptions.append(True)
+        
+        co.bind(f, coroutine.getcurrent())
+        co.switch()
+        co.throw(RuntimeError)
+        assert exceptions == [True]
+        
+    def test_propagation(self):
+        exceptions = []
+        co = coroutine()
+        co2 = coroutine()
+        def f(main):
+            main.switch()
+        
+        co.bind(f, coroutine.getcurrent())
+        co.switch()
+        
+        try:
+            co.throw(RuntimeError)
+        except RuntimeError:
+            exceptions.append(1)
+            
+        def f2():
+            raise RuntimeError
+        
+        co2.bind(f2)
+            
+        try:
+            co2.switch()
+        except RuntimeError:
+            exceptions.append(2)
+        
+        assert exceptions == [1,2]
+
+    def test_bogus_bind(self):
+        co = coroutine()
+        def f():
+            pass
+        co.bind(f)
+        raises(ValueError, co.bind, f)
+
+    def test_simple_task(self):
+        maintask = coroutine.getcurrent()
+        def f():pass
+        co = coroutine()
+        co.bind(f)
+        co.switch()
+        assert not co.is_alive
+        assert maintask is coroutine.getcurrent()
+
+    def test_backto_main(self):
+        maintask = coroutine.getcurrent()
+        def f(task):
+            task.switch()
+        co = coroutine()
+        co.bind(f,maintask)
+        co.switch()
+
+    def test_wrapped_main(self):
+        class mwrap(object):
+            def __init__(self, coro):
+                self._coro = coro
+
+            def __getattr__(self, attr):
+                return getattr(self._coro, attr)
+
+        maintask = mwrap(coroutine.getcurrent())
+        def f(task):
+            task.switch()
+        co = coroutine()
+        co.bind(f,maintask)
+        co.switch()
+

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_ctypes_support.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_ctypes_support.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,22 @@
+from __future__ import absolute_import
+
+import py
+from ctypes import *
+try:
+    from ctypes_support import standard_c_lib, get_errno, set_errno
+except ImportError:    # on top of cpython
+    from ..ctypes_support import standard_c_lib, get_errno, set_errno
+
+
+def test_stdlib_and_errno():
+    py.test.skip("this is expected on top of pypy, we need to fix ctypes in a way that is now in 2.6 in order to make this reliable")
+    write = standard_c_lib.write
+    write.argtypes = [c_int, c_char_p, c_size_t]
+    write.restype = c_size_t
+    # clear errno first
+    set_errno(0)
+    assert get_errno() == 0
+    write(-345, "abc", 3)
+    assert get_errno() != 0
+    set_errno(0)
+    assert get_errno() == 0

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_datetime.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_datetime.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,18 @@
+from __future__ import absolute_import
+
+from .. import datetime
+
+def test_repr():
+    print datetime
+    expected = "datetime.datetime(1, 2, 3, 0, 0)"
+    assert repr(datetime.datetime(1,2,3)) == expected
+
+def test_strptime():
+    import time
+
+    string = '2004-12-01 13:02:47'
+    format = '%Y-%m-%d %H:%M:%S'
+    expected = datetime.datetime(*(time.strptime(string, format)[0:6]))
+    got = datetime.datetime.strptime(string, format)
+    assert expected == got
+    

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_dbm_extra.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_dbm_extra.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,50 @@
+from __future__ import absolute_import
+import py
+from pypy.tool.udir import udir
+try:
+    from .. import dbm
+except ImportError, e:
+    py.test.skip(e)
+
+def test_get():
+    path = str(udir.join('test_dbm_extra.test_get'))
+    d = dbm.open(path, 'c')
+    x = d.get("42")
+    assert x is None
+    d.close()
+
+def test_delitem():
+    path = str(udir.join('test_dbm_extra.test_delitem'))
+    d = dbm.open(path, 'c')
+    py.test.raises(KeyError, "del d['xyz']")
+
+def test_nonstring():
+    path = str(udir.join('test_dbm_extra.test_nonstring'))
+    d = dbm.open(path, 'c')
+    py.test.raises(TypeError, "d[123] = 'xyz'")
+    py.test.raises(TypeError, "d['xyz'] = 123")
+    py.test.raises(TypeError, "d['xyz'] = None")
+    py.test.raises(TypeError, "del d[123]")
+    py.test.raises(TypeError, "d[123]")
+    py.test.raises(TypeError, "123 in d")
+    py.test.raises(TypeError, "d.has_key(123)")
+    py.test.raises(TypeError, "d.setdefault(123, 'xyz')")
+    py.test.raises(TypeError, "d.setdefault('xyz', 123)")
+    py.test.raises(TypeError, "d.get(123)")
+    assert dict(d) == {}
+    d.setdefault('xyz', '123')
+    assert dict(d) == {'xyz': '123'}
+    d.close()
+
+def test_multiple_sets():
+    path = str(udir.join('test_dbm_extra.test_multiple_sets'))
+    d = dbm.open(path, 'c')
+    d['xyz'] = '12'
+    d['xyz'] = '3'
+    d['xyz'] = '546'
+    assert dict(d) == {'xyz': '546'}
+    assert d['xyz'] == '546'
+
+def test_extra():
+    py.test.raises(TypeError, dbm.datum, 123)
+    py.test.raises(TypeError, dbm.datum, False)

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_defaultdict.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_defaultdict.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,126 @@
+# defaultdict Tests
+# from CPython2.5
+from __future__ import absolute_import
+import py
+import sys
+if sys.version_info < (2, 5):
+    # the app-level defaultdict relies on the interp-level dict
+    # calling __missing__()
+    py.test.skip("these tests only run on top of CPython 2.5")
+
+import copy
+
+from ..collections import defaultdict
+
+def foobar():
+    return list
+
+class Test_defaultdict:
+    
+    def test_basic(self):
+        d1 = defaultdict()
+        assert d1.default_factory is None
+        d1.default_factory = list
+        d1[12].append(42)
+        assert d1 == {12: [42]}
+        d1[12].append(24)
+        assert d1 == {12: [42, 24]}
+        d1[13]
+        d1[14]
+        assert d1 == {12: [42, 24], 13: [], 14: []}
+        assert d1[12] is not d1[13] is not d1[14]
+        d2 = defaultdict(list, foo=1, bar=2)
+        assert d2.default_factory == list
+        assert d2 == {"foo": 1, "bar": 2}
+        assert d2["foo"] == 1
+        assert d2["bar"] == 2
+        assert d2[42] == []
+        assert "foo" in d2
+        assert "foo" in d2.keys()
+        assert "bar" in d2
+        assert "bar" in d2.keys()
+        assert 42 in d2
+        assert 42 in d2.keys()
+        assert 12 not in d2
+        assert 12 not in d2.keys()
+        d2.default_factory = None
+        assert d2.default_factory == None
+        py.test.raises(KeyError, d2.__getitem__, 15)
+        py.test.raises(TypeError, defaultdict, 1)
+
+    def test_missing(self):
+        d1 = defaultdict()
+        py.test.raises(KeyError, d1.__missing__, 42)
+        d1.default_factory = list
+        assert d1.__missing__(42) == []
+
+    def test_repr(self):
+        d1 = defaultdict()
+        assert d1.default_factory == None
+        assert repr(d1) == "defaultdict(None, {})"
+        d1[11] = 41
+        assert repr(d1) == "defaultdict(None, {11: 41})"
+        d2 = defaultdict(int)
+        assert d2.default_factory == int
+        d2[12] = 42
+        assert repr(d2) == "defaultdict(<type 'int'>, {12: 42})"
+        def foo(): return 43
+        d3 = defaultdict(foo)
+        assert d3.default_factory is foo
+        d3[13]
+        assert repr(d3) == "defaultdict(%s, {13: 43})" % repr(foo)
+        d4 = defaultdict(int)
+        d4[14] = defaultdict()
+        assert repr(d4) == "defaultdict(%s, {14: defaultdict(None, {})})" % repr(int)
+
+    def test_recursive_repr(self):
+        # Issue2045: stack overflow when default_factory is a bound method
+        class sub(defaultdict):
+            def __init__(self):
+                self.default_factory = self._factory
+            def _factory(self):
+                return []
+        d = sub()
+        assert repr(d).startswith(
+            "defaultdict(<bound method sub._factory of defaultdict(...")
+
+    def test_copy(self):
+        d1 = defaultdict()
+        d2 = d1.copy()
+        assert type(d2) == defaultdict
+        assert d2.default_factory == None
+        assert d2 == {}
+        d1.default_factory = list
+        d3 = d1.copy()
+        assert type(d3) == defaultdict
+        assert d3.default_factory == list
+        assert d3 == {}
+        d1[42]
+        d4 = d1.copy()
+        assert type(d4) == defaultdict
+        assert d4.default_factory == list
+        assert d4 == {42: []}
+        d4[12]
+        assert d4 == {42: [], 12: []}
+
+    def test_shallow_copy(self):
+        d1 = defaultdict(foobar, {1: 1})
+        d2 = copy.copy(d1)
+        assert d2.default_factory == foobar
+        assert d2 == d1
+        d1.default_factory = list
+        d2 = copy.copy(d1)
+        assert d2.default_factory == list
+        assert d2 == d1
+
+    def test_deep_copy(self):
+        d1 = defaultdict(foobar, {1: [1]})
+        d2 = copy.deepcopy(d1)
+        assert d2.default_factory == foobar
+        assert d2 == d1
+        assert d1[1] is not d2[1]
+        d1.default_factory = list
+        d2 = copy.deepcopy(d1)
+        assert d2.default_factory == list
+        assert d2 == d1
+

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_deque_extra.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_deque_extra.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,56 @@
+# Deque Tests
+from __future__ import absolute_import
+import py
+
+
+n = 10
+class Test_deque:
+    def setup_method(self,method):
+        
+        from ..collections import deque
+        self.deque = deque
+        self.d = deque(range(n))
+        
+    def test_deque(self):
+        
+        assert len(self.d) == n
+        for i in range(n):
+            assert i == self.d[i]
+        for i in range(n-1, -1, -1):
+            assert self.d.pop() == i
+        assert len(self.d) == 0
+            
+    def test_deque_iter(self):
+        it = iter(self.d)
+        py.test.raises(TypeError, len, it)
+        assert it.next() == 0
+        self.d.pop()
+        py.test.raises(RuntimeError, it.next)
+
+    def test_deque_reversed(self):
+        it = reversed(self.d)
+        py.test.raises(TypeError, len, it)
+        assert it.next() == n-1
+        assert it.next() == n-2
+        self.d.pop()
+        py.test.raises(RuntimeError, it.next)
+
+    def test_deque_remove(self):
+        d = self.d
+        py.test.raises(ValueError, d.remove, "foobar")
+
+    def test_mutate_during_remove(self):
+        # Handle evil mutator
+        class MutateCmp:
+            def __init__(self, deque, result):
+                self.deque = deque
+                self.result = result
+            def __eq__(self, other):
+                self.deque.clear()
+                return self.result
+
+        for match in (True, False):
+            d = self.deque(['ab'])
+            d.extend([MutateCmp(d, match), 'c'])
+            py.test.raises(IndexError, d.remove, 'c')
+            assert len(d) == 0

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_exception_extra.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_exception_extra.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,21 @@
+def app_test_environmenterror_repr():
+    import exceptions as ex 
+    e = ex.EnvironmentError("hello")
+    assert str(e) == "hello"
+    e = ex.EnvironmentError(1, "hello")
+    assert str(e) == "[Errno 1] hello"
+    e = ex.EnvironmentError(1, "hello", "world")
+    assert str(e) == "[Errno 1] hello: 'world'"
+
+def app_test_import():
+    import exceptions
+    assert exceptions.SyntaxError is SyntaxError 
+
+def app_test_baseexception():
+    assert issubclass(Exception, BaseException)
+
+def app_test_systemexit():
+    assert issubclass(SystemExit, BaseException)
+
+def app_test_keyboardinterrupt():
+    assert issubclass(KeyboardInterrupt, BaseException)

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_functools.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_functools.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,271 @@
+from __future__ import absolute_import
+from .. import functools
+import unittest
+from test import test_support
+from weakref import proxy
+import py
+
+ at staticmethod
+def PythonPartial(func, *args, **keywords):
+    'Pure Python approximation of partial()'
+    def newfunc(*fargs, **fkeywords):
+        newkeywords = keywords.copy()
+        newkeywords.update(fkeywords)
+        return func(*(args + fargs), **newkeywords)
+    newfunc.func = func
+    newfunc.args = args
+    newfunc.keywords = keywords
+    return newfunc
+
+def capture(*args, **kw):
+    """capture all positional and keyword arguments"""
+    return args, kw
+
+class TestPartial:
+
+    thetype = functools.partial
+
+    def test_basic_examples(self):
+        p = self.thetype(capture, 1, 2, a=10, b=20)
+        assert p(3, 4, b=30, c=40) == (
+                         ((1, 2, 3, 4), dict(a=10, b=30, c=40)))
+        p = self.thetype(map, lambda x: x*10)
+        assert p([1,2,3,4]) == [10, 20, 30, 40]
+
+    def test_attributes_1(self):
+        p = self.thetype(capture, 1, 2, a=10, b=20)
+        # attributes should be readable
+        assert p.func == capture
+        assert p.args == (1, 2)
+        assert p.keywords == dict(a=10, b=20)
+        # attributes should not be writable
+        if not isinstance(self.thetype, type):
+            return
+        py.test.raises(TypeError, setattr, p, 'func', map)
+        py.test.raises(TypeError, setattr, p, 'args', (1, 2))
+        py.test.raises(TypeError, setattr, p, 'keywords', dict(a=1, b=2))
+
+    def test_argument_checking(self):
+        py.test.raises(TypeError, self.thetype)     # need at least a func arg
+        try:
+            self.thetype(2)()
+        except TypeError:
+            pass
+        else:
+            raise AssertionError, 'First arg not checked for callability'
+
+    def test_protection_of_callers_dict_argument(self):
+        # a caller's dictionary should not be altered by partial
+        def func(a=10, b=20):
+            return a
+        d = {'a':3}
+        p = self.thetype(func, a=5)
+        assert p(**d) == 3
+        assert d == {'a':3}
+        p(b=7)
+        assert d == {'a':3}
+
+    def test_arg_combinations(self):
+        # exercise special code paths for zero args in either partial
+        # object or the caller
+        p = self.thetype(capture)
+        assert p() == ((), {})
+        assert p(1,2) == ((1,2), {})
+        p = self.thetype(capture, 1, 2)
+        assert p() == ((1,2), {})
+        assert p(3,4) == ((1,2,3,4), {})
+
+    def test_kw_combinations(self):
+        # exercise special code paths for no keyword args in
+        # either the partial object or the caller
+        p = self.thetype(capture)
+        assert p() == ((), {})
+        assert p(a=1) == ((), {'a':1})
+        p = self.thetype(capture, a=1)
+        assert p() == ((), {'a':1})
+        assert p(b=2) == ((), {'a':1, 'b':2})
+        # keyword args in the call override those in the partial object
+        assert p(a=3, b=2) == ((), {'a':3, 'b':2})
+
+    def test_positional(self):
+        # make sure positional arguments are captured correctly
+        for args in [(), (0,), (0,1), (0,1,2), (0,1,2,3)]:
+            p = self.thetype(capture, *args)
+            expected = args + ('x',)
+            got, empty = p('x')
+            assert expected == got and empty == {}
+
+    def test_keyword(self):
+        # make sure keyword arguments are captured correctly
+        for a in ['a', 0, None, 3.5]:
+            p = self.thetype(capture, a=a)
+            expected = {'a':a,'x':None}
+            empty, got = p(x=None)
+            assert expected == got and empty == ()
+
+    def test_no_side_effects(self):
+        # make sure there are no side effects that affect subsequent calls
+        p = self.thetype(capture, 0, a=1)
+        args1, kw1 = p(1, b=2)
+        assert args1 == (0,1) and kw1 == {'a':1,'b':2}
+        args2, kw2 = p()
+        assert args2 == (0,) and kw2 == {'a':1}
+
+    def test_error_propagation(self):
+        def f(x, y):
+            x / y
+        py.test.raises(ZeroDivisionError, self.thetype(f, 1, 0))
+        py.test.raises(ZeroDivisionError, self.thetype(f, 1), 0)
+        py.test.raises(ZeroDivisionError, self.thetype(f), 1, 0)
+        py.test.raises(ZeroDivisionError, self.thetype(f, y=0), 1)
+
+    def test_attributes_2(self):
+        p = self.thetype(hex)
+        try:
+            del p.__dict__
+        except (TypeError, AttributeError):
+            pass
+        else:
+            raise AssertionError, 'partial object allowed __dict__ to be deleted'
+
+    def test_weakref(self):
+        f = self.thetype(int, base=16)
+        p = proxy(f)
+        assert f.func == p.func
+        f = None
+        import gc
+        gc.collect()
+        py.test.raises(ReferenceError, getattr, p, 'func')
+
+    def test_with_bound_and_unbound_methods(self):
+        data = map(str, range(10))
+        join = self.thetype(str.join, '')
+        assert join(data) == '0123456789'
+        join = self.thetype(''.join)
+        assert join(data) == '0123456789'
+
+class PartialSubclass(functools.partial):
+    pass
+
+class TestPartialSubclass(TestPartial):
+
+    thetype = PartialSubclass
+
+
+class TestPythonPartial(TestPartial):
+
+    thetype = PythonPartial
+
+class TestUpdateWrapper:
+
+    def check_wrapper(self, wrapper, wrapped,
+                      assigned=functools.WRAPPER_ASSIGNMENTS,
+                      updated=functools.WRAPPER_UPDATES):
+        # Check attributes were assigned
+        for name in assigned:
+            assert getattr(wrapper, name) == getattr(wrapped, name)
+        # Check attributes were updated
+        for name in updated:
+            wrapper_attr = getattr(wrapper, name)
+            wrapped_attr = getattr(wrapped, name)
+            for key in wrapped_attr:
+                assert wrapped_attr[key] == wrapper_attr[key]
+
+    def test_default_update(self):
+        def f():
+            """This is a test"""
+            pass
+        f.attr = 'This is also a test'
+        def wrapper():
+            pass
+        functools.update_wrapper(wrapper, f)
+        self.check_wrapper(wrapper, f)
+        assert wrapper.__name__ == 'f'
+        assert wrapper.__doc__ == 'This is a test'
+        assert wrapper.attr == 'This is also a test'
+
+    def test_no_update(self):
+        def f():
+            """This is a test"""
+            pass
+        f.attr = 'This is also a test'
+        def wrapper():
+            pass
+        functools.update_wrapper(wrapper, f, (), ())
+        self.check_wrapper(wrapper, f, (), ())
+        assert wrapper.__name__ == 'wrapper'
+        assert wrapper.__doc__ == None
+        assert not hasattr(wrapper, 'attr')
+
+    def test_selective_update(self):
+        def f():
+            pass
+        f.attr = 'This is a different test'
+        f.dict_attr = dict(a=1, b=2, c=3)
+        def wrapper():
+            pass
+        wrapper.dict_attr = {}
+        assign = ('attr',)
+        update = ('dict_attr',)
+        functools.update_wrapper(wrapper, f, assign, update)
+        self.check_wrapper(wrapper, f, assign, update)
+        assert wrapper.__name__ == 'wrapper'
+        assert wrapper.__doc__ == None
+        assert wrapper.attr == 'This is a different test'
+        assert wrapper.dict_attr == f.dict_attr
+
+    def test_builtin_update(self):
+        # Test for bug #1576241
+        def wrapper():
+            pass
+        functools.update_wrapper(wrapper, max)
+        assert wrapper.__name__ == 'max'
+        assert wrapper.__doc__ == max.__doc__
+
+class TestWraps(TestUpdateWrapper):
+
+    def test_default_update(self):
+        def f():
+            """This is a test"""
+            pass
+        f.attr = 'This is also a test'
+        @functools.wraps(f)
+        def wrapper():
+            pass
+        self.check_wrapper(wrapper, f)
+        assert wrapper.__name__ == 'f'
+        assert wrapper.__doc__ == 'This is a test'
+        assert wrapper.attr == 'This is also a test'
+
+    def test_no_update(self):
+        def f():
+            """This is a test"""
+            pass
+        f.attr = 'This is also a test'
+        @functools.wraps(f, (), ())
+        def wrapper():
+            pass
+        self.check_wrapper(wrapper, f, (), ())
+        assert wrapper.__name__ == 'wrapper'
+        assert wrapper.__doc__ == None
+        assert not hasattr(wrapper, 'attr')
+
+    def test_selective_update(self):
+        def f():
+            pass
+        f.attr = 'This is a different test'
+        f.dict_attr = dict(a=1, b=2, c=3)
+        def add_dict_attr(f):
+            f.dict_attr = {}
+            return f
+        assign = ('attr',)
+        update = ('dict_attr',)
+        @functools.wraps(f, assign, update)
+        @add_dict_attr
+        def wrapper():
+            pass
+        self.check_wrapper(wrapper, f, assign, update)
+        assert wrapper.__name__ == 'wrapper'
+        assert wrapper.__doc__ == None
+        assert wrapper.attr == 'This is a different test'
+        assert wrapper.dict_attr == f.dict_attr

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_grp_extra.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_grp_extra.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,10 @@
+from __future__ import absolute_import
+import py
+try:
+    from .. import grp
+except ImportError:
+    py.test.skip("No grp module on this platform")
+
+def test_extra():
+    py.test.raises(TypeError, grp.getgrnam, False)
+    py.test.raises(TypeError, grp.getgrnam, None)

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_hashlib.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_hashlib.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,45 @@
+from __future__ import absolute_import
+from ..ctypes_config_cache import rebuild
+rebuild.rebuild_one('hashlib.ctc.py')
+
+from .. import hashlib, _hashlib
+
+def test_unicode():
+    assert isinstance(hashlib.new('sha1', u'xxx'), _hashlib.hash)
+
+def test_attributes():
+    for name, expected_size in {'md5': 16,
+                                'sha1': 20,
+                                'sha224': 28,
+                                'sha256': 32,
+                                'sha384': 48,
+                                'sha512': 64,
+                                }.items():
+        h = hashlib.new(name)
+        assert h.digest_size == expected_size
+        assert h.digestsize == expected_size
+        #
+        h.update('abc')
+        h2 = h.copy()
+        h.update('def')
+        digest = h.digest()
+        hexdigest = h.hexdigest()
+        h2.update('d')
+        h2.update('ef')
+        assert digest == h.digest()
+        assert hexdigest == h.hexdigest()
+
+        # also test the pure Python implementation
+        h = hashlib.__get_builtin_constructor(name)('')
+        assert h.digest_size == expected_size
+        assert h.digestsize == expected_size
+        #
+        h.update('abc')
+        h2 = h.copy()
+        h.update('def')
+        digest = h.digest()
+        hexdigest = h.hexdigest()
+        h2.update('d')
+        h2.update('ef')
+        assert digest == h.digest()
+        assert hexdigest == h.hexdigest()

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_locale.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_locale.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,79 @@
+from __future__ import absolute_import
+import py
+import sys
+
+from ..ctypes_config_cache import rebuild
+rebuild.rebuild_one('locale.ctc.py')
+
+from .. import _locale
+
+
+def setup_module(mod):
+    if sys.platform == 'darwin':
+        py.test.skip("Locale support on MacOSX is minimal and cannot be tested")
+
+class TestLocale:
+    def setup_class(cls):
+        cls.oldlocale = _locale.setlocale(_locale.LC_NUMERIC)
+        if sys.platform.startswith("win"):
+            cls.tloc = "en"
+        elif sys.platform.startswith("freebsd"):
+            cls.tloc = "en_US.US-ASCII"
+        else:
+            cls.tloc = "en_US.UTF8"
+        try:
+            _locale.setlocale(_locale.LC_NUMERIC, cls.tloc)
+        except _locale.Error:
+            py.test.skip("test locale %s not supported" % cls.tloc)
+            
+    def teardown_class(cls):
+        _locale.setlocale(_locale.LC_NUMERIC, cls.oldlocale)
+
+    def test_format(self):
+        py.test.skip("XXX fix or kill me")
+
+        def testformat(formatstr, value, grouping = 0, output=None):
+            if output:
+                print "%s %% %s =? %s ..." %\
+                      (repr(formatstr), repr(value), repr(output)),
+            else:
+                print "%s %% %s works? ..." % (repr(formatstr), repr(value)),
+            result = locale.format(formatstr, value, grouping = grouping)
+            assert result == output
+
+        testformat("%f", 1024, grouping=1, output='1,024.000000')
+        testformat("%f", 102, grouping=1, output='102.000000')
+        testformat("%f", -42, grouping=1, output='-42.000000')
+        testformat("%+f", -42, grouping=1, output='-42.000000')
+        testformat("%20.f", -42, grouping=1, output='                 -42')
+        testformat("%+10.f", -4200, grouping=1, output='    -4,200')
+        testformat("%-10.f", 4200, grouping=1, output='4,200     ')
+
+    def test_getpreferredencoding(self):
+        py.test.skip("XXX fix or kill me")
+        # Invoke getpreferredencoding to make sure it does not cause exceptions
+        _locale.getpreferredencoding()
+
+    # Test BSD Rune locale's bug for isctype functions.
+    def test_bsd_bug(self):
+        def teststrop(s, method, output):
+            print "%s.%s() =? %s ..." % (repr(s), method, repr(output)),
+            result = getattr(s, method)()
+            assert result == output
+
+        oldlocale = _locale.setlocale(_locale.LC_CTYPE)
+        _locale.setlocale(_locale.LC_CTYPE, self.tloc)
+        try:
+            teststrop('\x20', 'isspace', True)
+            teststrop('\xa0', 'isspace', False)
+            teststrop('\xa1', 'isspace', False)
+            teststrop('\xc0', 'isalpha', False)
+            teststrop('\xc0', 'isalnum', False)
+            teststrop('\xc0', 'isupper', False)
+            teststrop('\xc0', 'islower', False)
+            teststrop('\xec\xa0\xbc', 'split', ['\xec\xa0\xbc'])
+            teststrop('\xed\x95\xa0', 'strip', '\xed\x95\xa0')
+            teststrop('\xcc\x85', 'lower', '\xcc\x85')
+            teststrop('\xed\x95\xa0', 'upper', '\xed\x95\xa0')
+        finally:
+            _locale.setlocale(_locale.LC_CTYPE, oldlocale)

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_marshal_extra.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_marshal_extra.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,145 @@
+from __future__ import absolute_import
+import py
+import sys
+import marshal as cpy_marshal
+from .. import _marshal as marshal
+
+from pypy.tool.udir import udir 
+
+hello = "he"
+hello += "llo"
+def func(x):
+    return lambda y: x+y
+scopefunc = func(42)
+
+TESTCASES = [
+    None,
+    False,
+    True,
+    StopIteration,
+    Ellipsis,
+    42,
+    sys.maxint,
+    -1.25,
+    2+5j,
+    42L,
+    -1234567890123456789012345678901234567890L,
+    hello,   # not interned
+    "hello",
+    (),
+    (1, 2),
+    [],
+    [3, 4],
+    {},
+    {5: 6, 7: 8},
+    func.func_code,
+    scopefunc.func_code,
+    u'hello',
+    ]
+
+try:
+    TESTCASES += [
+        set(),
+        set([1, 2]),
+        frozenset(),
+        frozenset([3, 4]),
+        ]
+except NameError:
+    pass    # Python < 2.4
+
+if getattr(cpy_marshal, 'version', 0) > 1:
+    cpy_dump_version = (1,)
+else:
+    cpy_dump_version = ()
+
+
+def test_cases():
+    for case in TESTCASES:
+        yield dumps_and_reload, case
+        yield loads_from_cpython, case
+        yield dumps_to_cpython, case
+        if case is not StopIteration:
+            yield dumps_subclass, case
+        yield load_from_cpython, case
+        yield dump_to_cpython, case
+
+def dumps_and_reload(case):
+    print 'dump_and_reload', `case`
+    s = marshal.dumps(case)
+    obj = marshal.loads(s)
+    assert obj == case
+
+def loads_from_cpython(case):
+    print 'load_from_cpython', `case`
+    try:
+        s = cpy_marshal.dumps(case, *cpy_dump_version)
+    except ValueError:
+        py.test.skip("this version of CPython doesn't support this object") 
+    obj = marshal.loads(s)
+    assert obj == case
+
+def dumps_to_cpython(case):
+    print 'dump_to_cpython', `case`
+    try:
+        cpy_marshal.dumps(case, *cpy_dump_version)
+    except ValueError:
+        py.test.skip("this version of CPython doesn't support this object") 
+    s = marshal.dumps(case)
+    obj = cpy_marshal.loads(s)
+    assert obj == case
+
+def dumps_subclass(case):
+    try:
+        class Subclass(type(case)):
+            pass
+        case = Subclass(case)
+    except TypeError:
+        py.test.skip("this version of CPython doesn't support this object") 
+    s = marshal.dumps(case)
+    obj = marshal.loads(s)
+    assert obj == case
+
+def load_from_cpython(case):
+    p = str(udir.join('test.dat'))
+
+    f1 = open(p, "w")
+    try:
+        try:
+            s = cpy_marshal.dump(case, f1, *cpy_dump_version)
+        finally:
+            f1.close()
+    except ValueError:
+        py.test.skip("this version of CPython doesn't support this object") 
+
+    f2 = open(p, "r")
+    try:
+        obj = marshal.load(f2)
+    finally:
+        f2.close()
+    assert obj == case
+
+def dump_to_cpython(case):
+
+    try:
+        cpy_marshal.dumps(case, *cpy_dump_version)
+    except ValueError:
+        py.test.skip("this version of CPython doesn't support this object") 
+
+    p = str(udir.join('test.dat'))
+    f1 = open(p, "w")
+    try:
+        try:
+            s = marshal.dump(case, f1)
+        finally:
+            f1.close()
+    except ValueError:
+        py.test.skip("this version of CPython doesn't support this object") 
+
+    f2 = open(p, "r")
+    try:
+        obj = cpy_marshal.load(f2)
+    finally:
+        f2.close()
+    assert obj == case
+
+

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_md5_extra.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_md5_extra.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,228 @@
+"""A test script to compare MD5 implementations.
+
+A note about performance: the pure Python MD5 takes roughly
+160 sec. per MB of data on a 233 MHz Intel Pentium CPU.
+"""
+
+from __future__ import absolute_import
+import md5                              # CPython's implementation in C.
+from .. import md5 as pymd5  
+
+
+# Helpers...
+
+def formatHex(str):
+    "Print a string's HEX code in groups of two digits."
+
+    d = map(None, str)
+    d = map(ord, d)
+    d = map(lambda x:"%02x" % x, d)
+    return ' '.join(d)
+
+
+def format(str):
+    "Print a string as-is in groups of two characters."
+
+    s = ''
+    for i in range(0, len(str)-1, 2):
+        s = s + "%03s" % str[i:i+2] 
+    return s[1:] 
+
+
+def printDiff(message, d1, d2, expectedResult=None):
+    "Print different outputs for same message."
+    
+    print "Message: '%s'" % message
+    print "Message length: %d" % len(message)
+    if expectedResult:
+        print "%-48s (expected)" % format(expectedResult)
+    print "%-48s (Std. lib. MD5)" % formatHex(d1)
+    print "%-48s (Pure Python MD5)" % formatHex(d2)
+    print
+
+
+# The real comparison function.
+
+def compareImp(message):
+    """Compare two MD5 implementations, C vs. pure Python module.
+
+    For equal digests this returns None, otherwise it returns
+    a tuple of both digests.
+    """
+
+    # Use Python's standard library MD5 compiled C module.    
+    m1 = md5.md5()
+    m1.update(message)
+    d1 = m1.digest()
+    d1h = m1.hexdigest()
+    
+    # Use MD5 module in pure Python.
+    m2 = pymd5.md5()
+    m2.update(message)
+    d2 = m2.digest()
+    d2h = m2.hexdigest()
+
+    # Return None if equal or the different digests if not equal.
+    if d1 == d2 and d1h == d2h:
+        return
+    else:
+        return d1, d2
+
+
+class TestMD5Compare:
+    "Compare pure Python MD5 against Python's std. lib. version."
+    
+    def test1(self):
+        "Test cases with known digest result."
+        
+        cases = (
+          ("",
+           "d41d8cd98f00b204e9800998ecf8427e"),
+          ("a",
+           "0cc175b9c0f1b6a831c399e269772661"),
+          ("abc",
+           "900150983cd24fb0d6963f7d28e17f72"),
+          ("message digest",
+           "f96b697d7cb7938d525a2f31aaf161d0"),
+          ("abcdefghijklmnopqrstuvwxyz",
+           "c3fcd3d76192e4007dfb496cca67e13b"),
+          ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
+           "d174ab98d277d9f5a5611c2c9f419d9f"),
+          ("1234567890"*8,
+           "57edf4a22be3c955ac49da2e2107b67a"),
+        )
+
+        for i in xrange(len(cases)):
+            res = compareImp(cases[i][0])
+            if res is not None:
+                d1, d2 = res
+                message, expectedResult = cases[i][0], None
+                if len(cases[i]) == 2:
+                    expectedResult = cases[i][1]
+                printDiff(message, d1, d2, expectedResult)
+            assert res is None
+
+
+    def test2(self):
+        "Test cases without known digest result."
+        
+        cases = (
+          "123",
+          "1234",
+          "12345",
+          "123456",
+          "1234567",
+          "12345678",
+          "123456789 123456789 123456789 ",
+          "123456789 123456789 ",
+          "123456789 123456789 1",
+          "123456789 123456789 12",
+          "123456789 123456789 123",
+          "123456789 123456789 1234",
+          "123456789 123456789 123456789 1",
+          "123456789 123456789 123456789 12",
+          "123456789 123456789 123456789 123",
+          "123456789 123456789 123456789 1234",
+          "123456789 123456789 123456789 12345",
+          "123456789 123456789 123456789 123456",
+          "123456789 123456789 123456789 1234567",
+          "123456789 123456789 123456789 12345678",
+         )
+
+        for i in xrange(len(cases)):
+            res = compareImp(cases[i][0])
+            if res is not None:
+                d1, d2 = res
+                message = cases[i][0]
+                printDiff(message, d1, d2)
+            assert res is None
+
+
+    def test3(self):
+        "Test cases with long messages (can take a while)."
+        
+        cases = (
+          (2**10*'a',),
+          (2**10*'abcd',),
+##          (2**20*'a',),  ## 1 MB, takes about 160 sec. on a 233 Mhz Pentium.
+         )
+
+        for i in xrange(len(cases)):
+            res = compareImp(cases[i][0])
+            if res is not None:
+                d1, d2 = res
+                message = cases[i][0]
+                printDiff(message, d1, d2)
+            assert res is None
+
+
+    def test4(self):
+        "Test cases with increasingly growing message lengths."
+
+        i = 0
+        while i  < 2**5:
+            message = i * 'a'
+            res = compareImp(message)
+            if res is not None:
+                d1, d2 = res
+                printDiff(message, d1, d2)
+            assert res is None
+            i = i + 1
+
+
+    def test5(self):
+        "Test updating cloned objects."
+
+        cases = (
+          "123",
+          "1234",
+          "12345",
+          "123456",
+          "1234567",
+          "12345678",
+          "123456789 123456789 123456789 ",
+          "123456789 123456789 ",
+          "123456789 123456789 1",
+          "123456789 123456789 12",
+          "123456789 123456789 123",
+          "123456789 123456789 1234",
+          "123456789 123456789 123456789 1",
+          "123456789 123456789 123456789 12",
+          "123456789 123456789 123456789 123",
+          "123456789 123456789 123456789 1234",
+          "123456789 123456789 123456789 12345",
+          "123456789 123456789 123456789 123456",
+          "123456789 123456789 123456789 1234567",
+          "123456789 123456789 123456789 12345678",
+         )
+
+        # Load both with same prefix.    
+        prefix1 = 2**10 * 'a'
+
+        m1 = md5.md5()
+        m1.update(prefix1)
+        m1c = m1.copy()
+
+        m2 = pymd5.md5()
+        m2.update(prefix1)
+        m2c = m2.copy()
+
+        # Update and compare...
+        for i in xrange(len(cases)):
+            message = cases[i][0]
+
+            m1c.update(message)
+            d1 = m1c.hexdigest()
+
+            m2c.update(message)
+            d2 = m2c.hexdigest()
+
+            assert d1 == d2
+
+
+def test_attributes():
+    assert pymd5.digest_size == 16
+    assert pymd5.digestsize == 16
+    assert pymd5.blocksize == 1
+    assert pymd5.md5().digest_size == 16
+    assert pymd5.md5().digestsize == 16

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_pickle_extra.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_pickle_extra.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,15 @@
+import pickle
+
+def test_pickle_moduledict():
+    if "_pickle_moduledict" not in pickle.Pickler.__dict__:
+        import py
+        py.test.skip("test the _pickle_moduledict() addition to pickle.py")
+    #
+    s1 = pickle.dumps(pickle.__dict__)
+    import gc; gc.collect()
+    s2 = pickle.dumps(pickle.__dict__)
+    #
+    d1 = pickle.loads(s1)
+    assert d1 is pickle.__dict__
+    d2 = pickle.loads(s2)
+    assert d2 is pickle.__dict__

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_pyexpat.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_pyexpat.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,665 @@
+# XXX TypeErrors on calling handlers, or on bad return values from a
+# handler, are obscure and unhelpful.
+
+from __future__ import absolute_import
+import StringIO, sys
+import unittest, py
+
+from ..ctypes_config_cache import rebuild
+rebuild.rebuild_one('pyexpat.ctc.py')
+
+from .. import pyexpat
+#from xml.parsers import expat
+expat = pyexpat
+
+from test.test_support import sortdict, run_unittest
+
+
+class TestSetAttribute:
+    def setup_method(self, meth):
+        self.parser = expat.ParserCreate(namespace_separator='!')
+        self.set_get_pairs = [
+            [0, 0],
+            [1, 1],
+            [2, 1],
+            [0, 0],
+            ]
+
+    def test_returns_unicode(self):
+        for x, y in self.set_get_pairs:
+            self.parser.returns_unicode = x
+            assert self.parser.returns_unicode == y
+
+    def test_ordered_attributes(self):
+        for x, y in self.set_get_pairs:
+            self.parser.ordered_attributes = x
+            assert self.parser.ordered_attributes == y
+
+    def test_specified_attributes(self):
+        for x, y in self.set_get_pairs:
+            self.parser.specified_attributes = x
+            assert self.parser.specified_attributes == y
+
+
+data = '''\
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<?xml-stylesheet href="stylesheet.css"?>
+<!-- comment data -->
+<!DOCTYPE quotations SYSTEM "quotations.dtd" [
+<!ELEMENT root ANY>
+<!ATTLIST root attr1 CDATA #REQUIRED attr2 CDATA #IMPLIED>
+<!NOTATION notation SYSTEM "notation.jpeg">
+<!ENTITY acirc "&#226;">
+<!ENTITY external_entity SYSTEM "entity.file">
+<!ENTITY unparsed_entity SYSTEM "entity.file" NDATA notation>
+%unparsed_entity;
+]>
+
+<root attr1="value1" attr2="value2&#8000;">
+<myns:subelement xmlns:myns="http://www.python.org/namespace">
+     Contents of subelements
+</myns:subelement>
+<sub2><![CDATA[contents of CDATA section]]></sub2>
+&external_entity;
+&skipped_entity;
+</root>
+'''
+
+
+# Produce UTF-8 output
+class TestParse:
+    class Outputter:
+        def __init__(self):
+            self.out = []
+
+        def StartElementHandler(self, name, attrs):
+            self.out.append('Start element: ' + repr(name) + ' ' +
+                            sortdict(attrs))
+
+        def EndElementHandler(self, name):
+            self.out.append('End element: ' + repr(name))
+
+        def CharacterDataHandler(self, data):
+            data = data.strip()
+            if data:
+                self.out.append('Character data: ' + repr(data))
+
+        def ProcessingInstructionHandler(self, target, data):
+            self.out.append('PI: ' + repr(target) + ' ' + repr(data))
+
+        def StartNamespaceDeclHandler(self, prefix, uri):
+            self.out.append('NS decl: ' + repr(prefix) + ' ' + repr(uri))
+
+        def EndNamespaceDeclHandler(self, prefix):
+            self.out.append('End of NS decl: ' + repr(prefix))
+
+        def StartCdataSectionHandler(self):
+            self.out.append('Start of CDATA section')
+
+        def EndCdataSectionHandler(self):
+            self.out.append('End of CDATA section')
+
+        def CommentHandler(self, text):
+            self.out.append('Comment: ' + repr(text))
+
+        def NotationDeclHandler(self, *args):
+            name, base, sysid, pubid = args
+            self.out.append('Notation declared: %s' %(args,))
+
+        def UnparsedEntityDeclHandler(self, *args):
+            entityName, base, systemId, publicId, notationName = args
+            self.out.append('Unparsed entity decl: %s' %(args,))
+
+        def NotStandaloneHandler(self):
+            self.out.append('Not standalone')
+            return 1
+
+        def ExternalEntityRefHandler(self, *args):
+            context, base, sysId, pubId = args
+            self.out.append('External entity ref: %s' %(args[1:],))
+            return 1
+
+        def StartDoctypeDeclHandler(self, *args):
+            self.out.append(('Start doctype', args))
+            return 1
+
+        def EndDoctypeDeclHandler(self):
+            self.out.append("End doctype")
+            return 1
+
+        def EntityDeclHandler(self, *args):
+            self.out.append(('Entity declaration', args))
+            return 1
+
+        def XmlDeclHandler(self, *args):
+            self.out.append(('XML declaration', args))
+            return 1
+
+        def ElementDeclHandler(self, *args):
+            self.out.append(('Element declaration', args))
+            return 1
+
+        def AttlistDeclHandler(self, *args):
+            self.out.append(('Attribute list declaration', args))
+            return 1
+
+        def SkippedEntityHandler(self, *args):
+            self.out.append(("Skipped entity", args))
+            return 1
+
+        def DefaultHandler(self, userData):
+            pass
+
+        def DefaultHandlerExpand(self, userData):
+            pass
+
+    handler_names = [
+        'StartElementHandler', 'EndElementHandler', 'CharacterDataHandler',
+        'ProcessingInstructionHandler', 'UnparsedEntityDeclHandler',
+        'NotationDeclHandler', 'StartNamespaceDeclHandler',
+        'EndNamespaceDeclHandler', 'CommentHandler',
+        'StartCdataSectionHandler', 'EndCdataSectionHandler', 'DefaultHandler',
+        'DefaultHandlerExpand', 'NotStandaloneHandler',
+        'ExternalEntityRefHandler', 'StartDoctypeDeclHandler',
+        'EndDoctypeDeclHandler', 'EntityDeclHandler', 'XmlDeclHandler',
+        'ElementDeclHandler', 'AttlistDeclHandler', 'SkippedEntityHandler',
+        ]
+
+    def test_utf8(self):
+
+        out = self.Outputter()
+        parser = expat.ParserCreate(namespace_separator='!')
+        for name in self.handler_names:
+            setattr(parser, name, getattr(out, name))
+        parser.returns_unicode = 0
+        parser.Parse(data, 1)
+
+        # Verify output
+        operations = out.out
+        expected_operations = [
+            ('XML declaration', (u'1.0', u'iso-8859-1', 0)),
+            'PI: \'xml-stylesheet\' \'href="stylesheet.css"\'',
+            "Comment: ' comment data '",
+            "Not standalone",
+            ("Start doctype", ('quotations', 'quotations.dtd', None, 1)),
+            ('Element declaration', (u'root', (2, 0, None, ()))),
+            ('Attribute list declaration', ('root', 'attr1', 'CDATA', None,
+                1)),
+            ('Attribute list declaration', ('root', 'attr2', 'CDATA', None,
+                0)),
+            "Notation declared: ('notation', None, 'notation.jpeg', None)",
+            ('Entity declaration', ('acirc', 0, '\xc3\xa2', None, None, None, None)),
+            ('Entity declaration', ('external_entity', 0, None, None,
+                'entity.file', None, None)),
+            "Unparsed entity decl: ('unparsed_entity', None, 'entity.file', None, 'notation')",
+            "Not standalone",
+            "End doctype",
+            "Start element: 'root' {'attr1': 'value1', 'attr2': 'value2\\xe1\\xbd\\x80'}",
+            "NS decl: 'myns' 'http://www.python.org/namespace'",
+            "Start element: 'http://www.python.org/namespace!subelement' {}",
+            "Character data: 'Contents of subelements'",
+            "End element: 'http://www.python.org/namespace!subelement'",
+            "End of NS decl: 'myns'",
+            "Start element: 'sub2' {}",
+            'Start of CDATA section',
+            "Character data: 'contents of CDATA section'",
+            'End of CDATA section',
+            "End element: 'sub2'",
+            "External entity ref: (None, 'entity.file', None)",
+            ('Skipped entity', ('skipped_entity', 0)),
+            "End element: 'root'",
+        ]
+        for operation, expected_operation in zip(operations, expected_operations):
+            assert operation == expected_operation
+
+    def test_unicode(self):
+        # Try the parse again, this time producing Unicode output
+        out = self.Outputter()
+        parser = expat.ParserCreate(namespace_separator='!')
+        parser.returns_unicode = 1
+        for name in self.handler_names:
+            setattr(parser, name, getattr(out, name))
+
+        parser.Parse(data, 1)
+
+        operations = out.out
+        expected_operations = [
+            ('XML declaration', (u'1.0', u'iso-8859-1', 0)),
+            'PI: u\'xml-stylesheet\' u\'href="stylesheet.css"\'',
+            "Comment: u' comment data '",
+            "Not standalone",
+            ("Start doctype", ('quotations', 'quotations.dtd', None, 1)),
+            ('Element declaration', (u'root', (2, 0, None, ()))),
+            ('Attribute list declaration', ('root', 'attr1', 'CDATA', None,
+                1)),
+            ('Attribute list declaration', ('root', 'attr2', 'CDATA', None,
+                0)),
+            "Notation declared: (u'notation', None, u'notation.jpeg', None)",
+            ('Entity declaration', (u'acirc', 0, u'\xe2', None, None, None,
+                None)),
+            ('Entity declaration', (u'external_entity', 0, None, None,
+                 u'entity.file', None, None)),
+            "Unparsed entity decl: (u'unparsed_entity', None, u'entity.file', None, u'notation')",
+            "Not standalone",
+            "End doctype",
+            "Start element: u'root' {u'attr1': u'value1', u'attr2': u'value2\\u1f40'}",
+            "NS decl: u'myns' u'http://www.python.org/namespace'",
+            "Start element: u'http://www.python.org/namespace!subelement' {}",
+            "Character data: u'Contents of subelements'",
+            "End element: u'http://www.python.org/namespace!subelement'",
+            "End of NS decl: u'myns'",
+            "Start element: u'sub2' {}",
+            'Start of CDATA section',
+            "Character data: u'contents of CDATA section'",
+            'End of CDATA section',
+            "End element: u'sub2'",
+            "External entity ref: (None, u'entity.file', None)",
+            ('Skipped entity', ('skipped_entity', 0)),
+            "End element: u'root'",
+        ]
+        for operation, expected_operation in zip(operations, expected_operations):
+            assert operation == expected_operation
+
+    def test_parse_file(self):
+        # Try parsing a file
+        out = self.Outputter()
+        parser = expat.ParserCreate(namespace_separator='!')
+        parser.returns_unicode = 1
+        for name in self.handler_names:
+            setattr(parser, name, getattr(out, name))
+        file = StringIO.StringIO(data)
+
+        parser.ParseFile(file)
+
+        operations = out.out
+        expected_operations = [
+            ('XML declaration', (u'1.0', u'iso-8859-1', 0)),
+            'PI: u\'xml-stylesheet\' u\'href="stylesheet.css"\'',
+            "Comment: u' comment data '",
+            "Not standalone",
+            ("Start doctype", ('quotations', 'quotations.dtd', None, 1)),
+            ('Element declaration', (u'root', (2, 0, None, ()))),
+            ('Attribute list declaration', ('root', 'attr1', 'CDATA', None,
+                1)),
+            ('Attribute list declaration', ('root', 'attr2', 'CDATA', None,
+                0)),
+            "Notation declared: (u'notation', None, u'notation.jpeg', None)",
+            ('Entity declaration', ('acirc', 0, u'\xe2', None, None, None, None)),
+            ('Entity declaration', (u'external_entity', 0, None, None, u'entity.file', None, None)),
+            "Unparsed entity decl: (u'unparsed_entity', None, u'entity.file', None, u'notation')",
+            "Not standalone",
+            "End doctype",
+            "Start element: u'root' {u'attr1': u'value1', u'attr2': u'value2\\u1f40'}",
+            "NS decl: u'myns' u'http://www.python.org/namespace'",
+            "Start element: u'http://www.python.org/namespace!subelement' {}",
+            "Character data: u'Contents of subelements'",
+            "End element: u'http://www.python.org/namespace!subelement'",
+            "End of NS decl: u'myns'",
+            "Start element: u'sub2' {}",
+            'Start of CDATA section',
+            "Character data: u'contents of CDATA section'",
+            'End of CDATA section',
+            "End element: u'sub2'",
+            "External entity ref: (None, u'entity.file', None)",
+            ('Skipped entity', ('skipped_entity', 0)),
+            "End element: u'root'",
+        ]
+        for operation, expected_operation in zip(operations, expected_operations):
+            assert operation == expected_operation
+
+
+class TestNamespaceSeparator:
+    def test_legal(self):
+        # Tests that make sure we get errors when the namespace_separator value
+        # is illegal, and that we don't for good values:
+        expat.ParserCreate()
+        expat.ParserCreate(namespace_separator=None)
+        expat.ParserCreate(namespace_separator=' ')
+
+    def test_illegal(self):
+        try:
+            expat.ParserCreate(namespace_separator=42)
+            raise AssertionError
+        except TypeError, e:
+            assert str(e) == (
+                'ParserCreate() argument 2 must be string or None, not int')
+
+        try:
+            expat.ParserCreate(namespace_separator='too long')
+            raise AssertionError
+        except ValueError, e:
+            assert str(e) == (
+                'namespace_separator must be at most one character, omitted, or None')
+
+    def test_zero_length(self):
+        # ParserCreate() needs to accept a namespace_separator of zero length
+        # to satisfy the requirements of RDF applications that are required
+        # to simply glue together the namespace URI and the localname.  Though
+        # considered a wart of the RDF specifications, it needs to be supported.
+        #
+        # See XML-SIG mailing list thread starting with
+        # http://mail.python.org/pipermail/xml-sig/2001-April/005202.html
+        #
+        expat.ParserCreate(namespace_separator='') # too short
+
+
+class TestInterning:
+    def test(self):
+        py.test.skip("Not working")
+        # Test the interning machinery.
+        p = expat.ParserCreate()
+        L = []
+        def collector(name, *args):
+            L.append(name)
+        p.StartElementHandler = collector
+        p.EndElementHandler = collector
+        p.Parse("<e> <e/> <e></e> </e>", 1)
+        tag = L[0]
+        assert len(L) == 6
+        for entry in L:
+            # L should have the same string repeated over and over.
+            assert tag is entry
+
+
+class TestBufferText:
+    def setup_method(self, meth):
+        self.stuff = []
+        self.parser = expat.ParserCreate()
+        self.parser.buffer_text = 1
+        self.parser.CharacterDataHandler = self.CharacterDataHandler
+
+    def check(self, expected, label):
+        assert self.stuff == expected, (
+                "%s\nstuff    = %r\nexpected = %r"
+                % (label, self.stuff, map(unicode, expected)))
+
+    def CharacterDataHandler(self, text):
+        self.stuff.append(text)
+
+    def StartElementHandler(self, name, attrs):
+        self.stuff.append("<%s>" % name)
+        bt = attrs.get("buffer-text")
+        if bt == "yes":
+            self.parser.buffer_text = 1
+        elif bt == "no":
+            self.parser.buffer_text = 0
+
+    def EndElementHandler(self, name):
+        self.stuff.append("</%s>" % name)
+
+    def CommentHandler(self, data):
+        self.stuff.append("<!--%s-->" % data)
+
+    def setHandlers(self, handlers=[]):
+        for name in handlers:
+            setattr(self.parser, name, getattr(self, name))
+
+    def test_default_to_disabled(self):
+        parser = expat.ParserCreate()
+        assert not parser.buffer_text
+
+    def test_buffering_enabled(self):
+        # Make sure buffering is turned on
+        assert self.parser.buffer_text
+        self.parser.Parse("<a>1<b/>2<c/>3</a>", 1)
+        assert self.stuff == ['123'], (
+                          "buffered text not properly collapsed")
+
+    def test1(self):
+        # XXX This test exposes more detail of Expat's text chunking than we
+        # XXX like, but it tests what we need to concisely.
+        self.setHandlers(["StartElementHandler"])
+        self.parser.Parse("<a>1<b buffer-text='no'/>2\n3<c buffer-text='yes'/>4\n5</a>", 1)
+        assert self.stuff == (
+                          ["<a>", "1", "<b>", "2", "\n", "3", "<c>", "4\n5"]), (
+                          "buffering control not reacting as expected")
+
+    def test2(self):
+        self.parser.Parse("<a>1<b/>&lt;2&gt;<c/>&#32;\n&#x20;3</a>", 1)
+        assert self.stuff == ["1<2> \n 3"], (
+                          "buffered text not properly collapsed")
+
+    def test3(self):
+        self.setHandlers(["StartElementHandler"])
+        self.parser.Parse("<a>1<b/>2<c/>3</a>", 1)
+        assert self.stuff == ["<a>", "1", "<b>", "2", "<c>", "3"], (
+                          "buffered text not properly split")
+
+    def test4(self):
+        self.setHandlers(["StartElementHandler", "EndElementHandler"])
+        self.parser.CharacterDataHandler = None
+        self.parser.Parse("<a>1<b/>2<c/>3</a>", 1)
+        assert self.stuff == (
+                          ["<a>", "<b>", "</b>", "<c>", "</c>", "</a>"])
+
+    def test5(self):
+        self.setHandlers(["StartElementHandler", "EndElementHandler"])
+        self.parser.Parse("<a>1<b></b>2<c/>3</a>", 1)
+        assert self.stuff == (
+            ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3", "</a>"])
+
+    def test6(self):
+        self.setHandlers(["CommentHandler", "EndElementHandler",
+                    "StartElementHandler"])
+        self.parser.Parse("<a>1<b/>2<c></c>345</a> ", 1)
+        assert self.stuff == (
+            ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "345", "</a>"]), (
+            "buffered text not properly split")
+
+    def test7(self):
+        self.setHandlers(["CommentHandler", "EndElementHandler",
+                    "StartElementHandler"])
+        self.parser.Parse("<a>1<b/>2<c></c>3<!--abc-->4<!--def-->5</a> ", 1)
+        assert self.stuff == (
+                          ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3",
+                           "<!--abc-->", "4", "<!--def-->", "5", "</a>"]), (
+                          "buffered text not properly split")
+
+
+# Test handling of exception from callback:
+class TestHandlerException:
+    def StartElementHandler(self, name, attrs):
+        raise RuntimeError(name)
+
+    def test(self):
+        parser = expat.ParserCreate()
+        parser.StartElementHandler = self.StartElementHandler
+        try:
+            parser.Parse("<a><b><c/></b></a>", 1)
+            raise AssertionError
+        except RuntimeError, e:
+            assert e.args[0] == 'a', (
+                              "Expected RuntimeError for element 'a', but" + \
+                              " found %r" % e.args[0])
+
+
+# Test Current* members:
+class TestPosition:
+    def StartElementHandler(self, name, attrs):
+        self.check_pos('s')
+
+    def EndElementHandler(self, name):
+        self.check_pos('e')
+
+    def check_pos(self, event):
+        pos = (event,
+               self.parser.CurrentByteIndex,
+               self.parser.CurrentLineNumber,
+               self.parser.CurrentColumnNumber)
+        assert self.upto < len(self.expected_list)
+        expected = self.expected_list[self.upto]
+        assert pos == expected, (
+                'Expected position %s, got position %s' %(pos, expected))
+        self.upto += 1
+
+    def test(self):
+        self.parser = expat.ParserCreate()
+        self.parser.StartElementHandler = self.StartElementHandler
+        self.parser.EndElementHandler = self.EndElementHandler
+        self.upto = 0
+        self.expected_list = [('s', 0, 1, 0), ('s', 5, 2, 1), ('s', 11, 3, 2),
+                              ('e', 15, 3, 6), ('e', 17, 4, 1), ('e', 22, 5, 0)]
+
+        xml = '<a>\n <b>\n  <c/>\n </b>\n</a>'
+        self.parser.Parse(xml, 1)
+
+
+class Testsf1296433:
+    def test_parse_only_xml_data(self):
+        # http://python.org/sf/1296433
+        #
+        xml = "<?xml version='1.0' encoding='iso8859'?><s>%s</s>" % ('a' * 1025)
+        # this one doesn't crash
+        #xml = "<?xml version='1.0'?><s>%s</s>" % ('a' * 10000)
+
+        class SpecificException(Exception):
+            pass
+
+        def handler(text):
+            raise SpecificException
+
+        parser = expat.ParserCreate()
+        parser.CharacterDataHandler = handler
+
+        py.test.raises(Exception, parser.Parse, xml)
+
+class TestChardataBuffer:
+    """
+    test setting of chardata buffer size
+    """
+
+    def test_1025_bytes(self):
+        assert self.small_buffer_test(1025) == 2
+
+    def test_1000_bytes(self):
+        assert self.small_buffer_test(1000) == 1
+
+    def test_wrong_size(self):
+        parser = expat.ParserCreate()
+        parser.buffer_text = 1
+        def f(size):
+            parser.buffer_size = size
+
+        py.test.raises(TypeError, f, sys.maxint+1)
+        py.test.raises(ValueError, f, -1)
+        py.test.raises(ValueError, f, 0)
+
+    def test_unchanged_size(self):
+        xml1 = ("<?xml version='1.0' encoding='iso8859'?><s>%s" % ('a' * 512))
+        xml2 = 'a'*512 + '</s>'
+        parser = expat.ParserCreate()
+        parser.CharacterDataHandler = self.counting_handler
+        parser.buffer_size = 512
+        parser.buffer_text = 1
+
+        # Feed 512 bytes of character data: the handler should be called
+        # once.
+        self.n = 0
+        parser.Parse(xml1)
+        assert self.n == 1
+
+        # Reassign to buffer_size, but assign the same size.
+        parser.buffer_size = parser.buffer_size
+        assert self.n == 1
+
+        # Try parsing rest of the document
+        parser.Parse(xml2)
+        assert self.n == 2
+
+
+    def test_disabling_buffer(self):
+        xml1 = "<?xml version='1.0' encoding='iso8859'?><a>%s" % ('a' * 512)
+        xml2 = ('b' * 1024)
+        xml3 = "%s</a>" % ('c' * 1024)
+        parser = expat.ParserCreate()
+        parser.CharacterDataHandler = self.counting_handler
+        parser.buffer_text = 1
+        parser.buffer_size = 1024
+        assert parser.buffer_size == 1024
+
+        # Parse one chunk of XML
+        self.n = 0
+        parser.Parse(xml1, 0)
+        assert parser.buffer_size == 1024
+        assert self.n == 1
+
+        # Turn off buffering and parse the next chunk.
+        parser.buffer_text = 0
+        assert not parser.buffer_text
+        assert parser.buffer_size == 1024
+        for i in range(10):
+            parser.Parse(xml2, 0)
+        assert self.n == 11
+
+        parser.buffer_text = 1
+        assert parser.buffer_text
+        assert parser.buffer_size == 1024
+        parser.Parse(xml3, 1)
+        assert self.n == 12
+
+
+
+    def make_document(self, bytes):
+        return ("<?xml version='1.0'?><tag>" + bytes * 'a' + '</tag>')
+
+    def counting_handler(self, text):
+        self.n += 1
+
+    def small_buffer_test(self, buffer_len):
+        xml = "<?xml version='1.0' encoding='iso8859'?><s>%s</s>" % ('a' * buffer_len)
+        parser = expat.ParserCreate()
+        parser.CharacterDataHandler = self.counting_handler
+        parser.buffer_size = 1024
+        parser.buffer_text = 1
+
+        self.n = 0
+        parser.Parse(xml)
+        return self.n
+
+    def test_change_size_1(self):
+        xml1 = "<?xml version='1.0' encoding='iso8859'?><a><s>%s" % ('a' * 1024)
+        xml2 = "aaa</s><s>%s</s></a>" % ('a' * 1025)
+        parser = expat.ParserCreate()
+        parser.CharacterDataHandler = self.counting_handler
+        parser.buffer_text = 1
+        parser.buffer_size = 1024
+        assert parser.buffer_size == 1024
+
+        self.n = 0
+        parser.Parse(xml1, 0)
+        parser.buffer_size *= 2
+        assert parser.buffer_size == 2048
+        parser.Parse(xml2, 1)
+        assert self.n == 2
+
+    def test_change_size_2(self):
+        xml1 = "<?xml version='1.0' encoding='iso8859'?><a>a<s>%s" % ('a' * 1023)
+        xml2 = "aaa</s><s>%s</s></a>" % ('a' * 1025)
+        parser = expat.ParserCreate()
+        parser.CharacterDataHandler = self.counting_handler
+        parser.buffer_text = 1
+        parser.buffer_size = 2048
+        assert parser.buffer_size == 2048
+
+        self.n=0
+        parser.Parse(xml1, 0)
+        parser.buffer_size /= 2
+        assert parser.buffer_size == 1024
+        parser.Parse(xml2, 1)
+        assert self.n == 4
+
+    def test_segfault(self):
+        py.test.raises(TypeError, expat.ParserCreate, 1234123123)
+
+def test_invalid_data():
+    parser = expat.ParserCreate()
+    parser.Parse('invalid.xml', 0)
+    try:
+        parser.Parse("", 1)
+    except expat.ExpatError, e:
+        assert e.code == 2 # XXX is this reliable?
+        assert e.lineno == 1
+        assert e.message.startswith('syntax error')
+    else:
+        py.test.fail("Did not raise")
+

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_resource.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_resource.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,31 @@
+from __future__ import absolute_import
+from ..ctypes_config_cache import rebuild
+rebuild.rebuild_one('resource.ctc.py')
+
+from .. import resource
+
+def test_resource():
+    x = resource.getrusage(resource.RUSAGE_SELF)
+    assert len(x) == 16
+    assert x[0] == x[-16] == x.ru_utime
+    assert x[1] == x[-15] == x.ru_stime
+    assert x[2] == x[-14] == x.ru_maxrss
+    assert x[3] == x[-13] == x.ru_ixrss
+    assert x[4] == x[-12] == x.ru_idrss
+    assert x[5] == x[-11] == x.ru_isrss
+    assert x[6] == x[-10] == x.ru_minflt
+    assert x[7] == x[-9] == x.ru_majflt
+    assert x[8] == x[-8] == x.ru_nswap
+    assert x[9] == x[-7] == x.ru_inblock
+    assert x[10] == x[-6] == x.ru_oublock
+    assert x[11] == x[-5] == x.ru_msgsnd
+    assert x[12] == x[-4] == x.ru_msgrcv
+    assert x[13] == x[-3] == x.ru_nsignals
+    assert x[14] == x[-2] == x.ru_nvcsw
+    assert x[15] == x[-1] == x.ru_nivcsw
+    for i in range(16):
+        if i < 2:
+            expected_type = float
+        else:
+            expected_type = (int, long)
+        assert isinstance(x[i], expected_type)

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_runpy.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_runpy.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,162 @@
+# Test the runpy module
+from __future__ import absolute_import
+import py
+import unittest
+import os
+import os.path
+import sys
+import tempfile
+from .. import runpy
+from ..runpy import _run_module_code, run_module
+
+sys.modules['my_runpy'] = runpy # so that code in exec can freely import it
+                                # without messing with __name__ and/or sys.path
+
+verbose = 0
+
+# Set up the test code and expected results
+class TestRunModuleCode:
+
+    expected_result = ["Top level assignment", "Lower level reference"]
+    test_source = py.code.Source("""
+        # Check basic code execution
+        result = ['Top level assignment']
+        def f():
+            result.append('Lower level reference')
+        f()
+        # Check the sys module
+        import sys
+        run_argv0 = sys.argv[0]
+        if __name__ in sys.modules:
+            run_name = sys.modules[__name__].__name__
+        # Check nested operation
+        import my_runpy # this is imported as ..runpy as manually saved into sys.modules, see above
+        nested = my_runpy._run_module_code('x=1\', mod_name='<run>',
+                                                  alter_sys=True)
+    """).compile()
+
+
+    def test_run_module_code(self):
+        initial = object()
+        name = "<Nonsense>"
+        file = "Some other nonsense"
+        loader = "Now you're just being silly"
+        d1 = dict(initial=initial)
+        saved_argv0 = sys.argv[0]
+        d2 = _run_module_code(self.test_source,
+                              d1,
+                              name,
+                              file,
+                              loader,
+                              True)
+        assert "result" not in d1
+        assert d2["initial"] is initial
+        assert d2["result"] == self.expected_result
+        assert d2["nested"]["x"] == 1
+        assert d2["__name__"] is name
+        assert d2["run_name"] is name
+        assert d2["__file__"] is file
+        assert d2["run_argv0"] is file
+        assert d2["__loader__"] is loader
+        assert sys.argv[0] is saved_argv0
+        assert name not in sys.modules
+
+    def test_run_module_code_defaults(self):
+        saved_argv0 = sys.argv[0]
+        d = _run_module_code(self.test_source)
+        assert d["result"] == self.expected_result
+        assert d["__name__"] is None
+        assert d["__file__"] is None
+        assert d["__loader__"] is None
+        assert d["run_argv0"] is saved_argv0
+        assert "run_name" not in d
+        assert sys.argv[0] is saved_argv0
+
+class TestRunModule:
+
+    def expect_import_error(self, mod_name):
+        try:
+            run_module(mod_name)
+        except ImportError:
+            pass
+        else:
+            assert false, "Expected import error for " + mod_name
+
+    def test_invalid_names(self):
+        self.expect_import_error("sys")
+        self.expect_import_error("sys.imp.eric")
+        self.expect_import_error("os.path.half")
+        self.expect_import_error("a.bee")
+        self.expect_import_error(".howard")
+        self.expect_import_error("..eaten")
+
+    def test_library_module(self):
+        run_module("runpy")
+
+    def _make_pkg(self, source, depth):
+        pkg_name = "__runpy_pkg__"
+        init_fname = "__init__"+os.extsep+"py"
+        test_fname = "runpy_test"+os.extsep+"py"
+        pkg_dir = sub_dir = tempfile.mkdtemp()
+        if verbose: print "  Package tree in:", sub_dir
+        sys.path.insert(0, pkg_dir)
+        if verbose: print "  Updated sys.path:", sys.path[0]
+        for i in range(depth):
+            sub_dir = os.path.join(sub_dir, pkg_name)
+            os.mkdir(sub_dir)
+            if verbose: print "  Next level in:", sub_dir
+            pkg_fname = os.path.join(sub_dir, init_fname)
+            pkg_file = open(pkg_fname, "w")
+            pkg_file.write("__path__ = [%r]\n" % sub_dir)
+            pkg_file.close()
+            if verbose: print "  Created:", pkg_fname
+        mod_fname = os.path.join(sub_dir, test_fname)
+        mod_file = open(mod_fname, "w")
+        mod_file.write(source)
+        mod_file.close()
+        if verbose: print "  Created:", mod_fname
+        mod_name = (pkg_name+".")*depth + "runpy_test"
+        return pkg_dir, mod_fname, mod_name
+
+    def _del_pkg(self, top, depth, mod_name):
+        for root, dirs, files in os.walk(top, topdown=False):
+            for name in files:
+                os.remove(os.path.join(root, name))
+            for name in dirs:
+                os.rmdir(os.path.join(root, name))
+        os.rmdir(top)
+        if verbose: print "  Removed package tree"
+        for i in range(depth+1): # Don't forget the module itself
+            parts = mod_name.rsplit(".", i)
+            entry = parts[0]
+            del sys.modules[entry]
+        if verbose: print "  Removed sys.modules entries"
+        del sys.path[0]
+        if verbose: print "  Removed sys.path entry"
+
+    def _check_module(self, depth):
+        pkg_dir, mod_fname, mod_name = (
+               self._make_pkg("x=1\n", depth))
+        try:
+            if verbose: print "Running from source:", mod_name
+            d1 = run_module(mod_name) # Read from source
+            __import__(mod_name)
+            os.remove(mod_fname)
+
+            #--- the block below is to check that "imp.find_module"
+            #--- manages to import the .pyc file alone.  We don't
+            #--- support it in PyPy in the default configuration.
+            return
+
+            if verbose: print "Running from compiled:", mod_name
+            d2 = run_module(mod_name) # Read from bytecode
+        finally:
+            self._del_pkg(pkg_dir, depth, mod_name)
+        assert d1["x"] == d2["x"] == 1
+        if verbose: print "Module executed successfully"
+
+    def test_run_module(self):
+        for depth in range(4):
+            if verbose: print "Testing package depth:", depth
+            self._check_module(depth)
+

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_sha_extra.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_sha_extra.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,32 @@
+# Testing sha module (NIST's Secure Hash Algorithm)
+
+# use the three examples from Federal Information Processing Standards
+# Publication 180-1, Secure Hash Standard,  1995 April 17
+# http://www.itl.nist.gov/div897/pubs/fip180-1.htm
+from __future__ import absolute_import
+from .. import sha 
+
+class TestSHA: 
+    def check(self, data, digest):
+        computed = sha.new(data).hexdigest()
+        assert computed == digest
+
+    def test_case_1(self):
+        self.check("abc",
+                   "a9993e364706816aba3e25717850c26c9cd0d89d")
+
+    def test_case_2(self):
+        self.check("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
+                   "84983e441c3bd26ebaae4aa1f95129e5e54670f1")
+
+    def disabled_too_slow_test_case_3(self):
+        self.check("a" * 1000000,
+                   "34aa973cd4c4daa4f61eeb2bdbad27316534016f")
+
+
+def test_attributes():
+    assert sha.digest_size == 20
+    assert sha.digestsize == 20
+    assert sha.blocksize == 1
+    assert sha.sha().digest_size == 20
+    assert sha.sha().digestsize == 20

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_stackless.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_stackless.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,603 @@
+"""
+These tests are supposed to run on the following platforms:
+1. CStackless
+2. CPython (with the stackless_new module in the path
+3. pypy-c
+"""
+from __future__ import absolute_import
+from py.test import skip
+try:
+    import stackless
+except ImportError:
+    try:
+        from .. import stackless
+    except ImportError, e:
+        skip('cannot import stackless: %s' % (e,))
+
+SHOW_STRANGE = False
+
+def dprint(txt):
+    if SHOW_STRANGE:
+        print txt
+
+class Test_Stackless:
+
+    def test_simple(self):
+        rlist = []
+
+        def f():
+            rlist.append('f')
+
+        def g():
+            rlist.append('g')
+            stackless.schedule()
+
+        def main():
+            rlist.append('m')
+            cg = stackless.tasklet(g)()
+            cf = stackless.tasklet(f)()
+            stackless.run()
+            rlist.append('m')
+
+        main()
+
+        assert stackless.getcurrent() is stackless.getmain()
+        assert rlist == 'm g f m'.split()
+
+    def test_with_channel(self):
+        pref = {}
+        pref[-1] = ['s0', 'r0', 's1', 'r1', 's2', 'r2', 
+                    's3', 'r3', 's4', 'r4', 's5', 'r5', 
+                    's6', 'r6', 's7', 'r7', 's8', 'r8', 
+                    's9', 'r9']
+        pref[0] =  ['s0', 'r0', 's1', 's2', 'r1', 'r2', 
+                    's3', 's4', 'r3', 'r4', 's5', 's6', 
+                    'r5', 'r6', 's7', 's8', 'r7', 'r8', 
+                    's9', 'r9']
+        pref[1] =  ['s0', 's1', 'r0', 's2', 'r1', 's3', 
+                    'r2', 's4', 'r3', 's5', 'r4', 's6', 
+                    'r5', 's7', 'r6', 's8', 'r7', 's9', 
+                    'r8', 'r9']
+        rlist = []
+
+        def f(outchan):
+            for i in range(10):
+                rlist.append('s%s' % i)
+                outchan.send(i)
+            outchan.send(-1)
+
+        def g(inchan):
+            while 1:
+                val = inchan.receive()
+                if val == -1:
+                    break
+                rlist.append('r%s' % val)
+
+        for preference in [-1, 0, 1]:
+            rlist = []
+            ch = stackless.channel()
+            ch.preference = preference
+            t1 = stackless.tasklet(f)(ch)
+            t2 = stackless.tasklet(g)(ch)
+
+            stackless.run()
+
+            assert len(rlist) == 20
+            assert rlist == pref[preference]
+
+    def test_send_counter(self):
+        import random
+
+        numbers = range(20)
+        random.shuffle(numbers)
+
+        def counter(n, ch):
+            for i in xrange(n):
+                stackless.schedule()
+            ch.send(n)
+
+        ch = stackless.channel()
+        for each in numbers:
+            stackless.tasklet(counter)(each, ch)
+
+        stackless.run()
+
+        rlist = []
+        while ch.balance:
+            rlist.append(ch.receive())
+
+        numbers.sort()
+        assert rlist == numbers
+
+    def test_receive_counter(self):
+        import random
+
+        numbers = range(20)
+        random.shuffle(numbers)
+
+        rlist = []
+        def counter(n, ch):
+            for i in xrange(n):
+                stackless.schedule()
+            ch.receive()
+            rlist.append(n)
+
+        ch = stackless.channel()
+        for each in numbers:
+            stackless.tasklet(counter)(each, ch)
+
+        stackless.run()
+
+        while ch.balance:
+            ch.send(None)
+
+        numbers.sort()
+        assert rlist == numbers
+
+    def test_scheduling_cleanup(self):
+        rlist = []
+        def f():
+            rlist.append('fb')
+            stackless.schedule()
+            rlist.append('fa')
+
+        def g():
+            rlist.append('gb')
+            stackless.schedule()
+            rlist.append('ga')
+
+        def h():
+            rlist.append('hb')
+            stackless.schedule()
+            rlist.append('ha')
+
+        tf = stackless.tasklet(f)()
+        tg = stackless.tasklet(g)()
+        th = stackless.tasklet(h)()
+
+        rlist.append('mb')
+        stackless.run()
+        rlist.append('ma')
+
+        assert rlist == 'mb fb gb hb fa ga ha ma'.split()
+
+    def test_except(self):
+        rlist = []
+        def f():
+            rlist.append('f')
+            return 1/0
+
+        def g():
+            rlist.append('bg')
+            stackless.schedule()
+            rlist.append('ag')
+
+        def h():
+            rlist.append('bh')
+            stackless.schedule()
+            rlist.append('ah')
+
+        tg = stackless.tasklet(g)()
+        tf = stackless.tasklet(f)()
+        th = stackless.tasklet(h)()
+
+        try:
+            stackless.run()
+        # cheating, can't test for ZeroDivisionError
+        except Exception, e:
+            rlist.append('E')
+        stackless.schedule()
+        stackless.schedule()
+
+        assert rlist == "bg f E bh ag ah".split()
+
+    def test_except_full(self):
+        rlist = []
+        def f():
+            rlist.append('f')
+            return 1/0
+
+        def g():
+            rlist.append('bg')
+            stackless.schedule()
+            rlist.append('ag')
+
+        def h():
+            rlist.append('bh')
+            stackless.schedule()
+            rlist.append('ah')
+
+        tg = stackless.tasklet(g)()
+        tf = stackless.tasklet(f)()
+        th = stackless.tasklet(h)()
+
+        try:
+            stackless.run()
+        except ZeroDivisionError:
+            rlist.append('E')
+        stackless.schedule()
+        stackless.schedule()
+
+        assert rlist == "bg f E bh ag ah".split()
+
+    def test_kill(self):
+        def f():pass
+        t =  stackless.tasklet(f)()
+        t.kill()
+        assert not t.alive
+
+    def test_catch_taskletexit(self):
+        # Tests if TaskletExit can be caught in the tasklet being killed.
+        global taskletexit
+        taskletexit = False
+        
+        def f():
+            try:
+                stackless.schedule()
+            except TaskletExit:
+                global TaskletExit
+                taskletexit = True
+                raise
+            
+            t =  stackless.tasklet(f)()
+            t.run()
+            assert t.alive
+            t.kill()
+            assert not t.alive
+            assert taskletexit
+            
+    def test_autocatch_taskletexit(self):
+        # Tests if TaskletExit is caught correctly in stackless.tasklet.setup(). 
+        def f():
+            stackless.schedule()
+        
+        t = stackless.tasklet(f)()
+        t.run()
+        t.kill()
+
+
+    # tests inspired from simple stackless.com examples
+
+    def test_construction(self):
+        output = []
+        def print_(*args):
+            output.append(args)
+
+        def aCallable(value):
+            print_("aCallable:", value)
+
+        task = stackless.tasklet(aCallable)
+        task.setup('Inline using setup')
+
+        stackless.run()
+        assert output == [("aCallable:", 'Inline using setup')]
+
+
+        del output[:]
+        task = stackless.tasklet(aCallable)
+        task('Inline using ()')
+
+        stackless.run()
+        assert output == [("aCallable:", 'Inline using ()')]
+        
+        del output[:]
+        task = stackless.tasklet()
+        task.bind(aCallable)
+        task('Bind using ()')
+
+        stackless.run()
+        assert output == [("aCallable:", 'Bind using ()')]
+
+    def test_simple_channel(self):
+        output = []
+        def print_(*args):
+            output.append(args)
+            
+        def Sending(channel):
+            print_("sending")
+            channel.send("foo")
+
+        def Receiving(channel):
+            print_("receiving")
+            print_(channel.receive())
+
+        ch=stackless.channel()
+
+        task=stackless.tasklet(Sending)(ch)
+
+        # Note: the argument, schedule is taking is the value,
+        # schedule returns, not the task that runs next
+
+        #stackless.schedule(task)
+        stackless.schedule()
+        task2=stackless.tasklet(Receiving)(ch)
+        #stackless.schedule(task2)
+        stackless.schedule()
+
+        stackless.run()
+
+        assert output == [('sending',), ('receiving',), ('foo',)]
+
+    def test_balance_zero(self):
+        ch=stackless.channel()
+        assert ch.balance == 0
+        
+    def test_balance_send(self):
+        def Sending(channel):
+            channel.send("foo")
+
+        ch=stackless.channel()
+
+        task=stackless.tasklet(Sending)(ch)
+        stackless.run()
+
+        assert ch.balance == 1
+
+    def test_balance_recv(self):
+        def Receiving(channel):
+            channel.receive()
+
+        ch=stackless.channel()
+
+        task=stackless.tasklet(Receiving)(ch)
+        stackless.run()
+
+        assert ch.balance == -1
+
+    def test_run(self):
+        output = []
+        def print_(*args):
+            output.append(args)
+
+        def f(i):
+            print_(i)
+
+        stackless.tasklet(f)(1)
+        stackless.tasklet(f)(2)
+        stackless.run()
+
+        assert output == [(1,), (2,)]
+
+    def test_schedule(self):
+        output = []
+        def print_(*args):
+            output.append(args)
+
+        def f(i):
+            print_(i)
+
+        stackless.tasklet(f)(1)
+        stackless.tasklet(f)(2)
+        stackless.schedule()
+
+        assert output == [(1,), (2,)]
+
+
+    def test_cooperative(self):
+        output = []
+        def print_(*args):
+            output.append(args)
+
+        def Loop(i):
+            for x in range(3):
+                stackless.schedule()
+                print_("schedule", i)
+
+        stackless.tasklet(Loop)(1)
+        stackless.tasklet(Loop)(2)
+        stackless.run()
+
+        assert output == [('schedule', 1), ('schedule', 2),
+                          ('schedule', 1), ('schedule', 2),
+                          ('schedule', 1), ('schedule', 2),]
+
+    def test_channel_callback(self):
+        res = []
+        cb = []
+        def callback_function(chan, task, sending, willblock):
+            cb.append((chan, task, sending, willblock))
+        stackless.set_channel_callback(callback_function)
+        def f(chan):
+            chan.send('hello')
+            val = chan.receive()
+            res.append(val)
+
+        chan = stackless.channel()
+        task = stackless.tasklet(f)(chan)
+        val = chan.receive()
+        res.append(val)
+        chan.send('world')
+        assert res == ['hello','world']
+        maintask = stackless.getmain()
+        assert cb == [
+            (chan, maintask, 0, 1), 
+            (chan, task, 1, 0), 
+            (chan, maintask, 1, 1), 
+            (chan, task, 0, 0)
+        ]
+
+    def test_schedule_callback(self):
+        res = []
+        cb = []
+        def schedule_cb(prev, next):
+            cb.append((prev, next))
+
+        stackless.set_schedule_callback(schedule_cb)
+        def f(i):
+            res.append('A_%s' % i)
+            stackless.schedule()
+            res.append('B_%s' % i)
+
+        t1 = stackless.tasklet(f)(1)
+        t2 = stackless.tasklet(f)(2)
+        maintask = stackless.getmain()
+        stackless.run()
+        assert res == ['A_1', 'A_2', 'B_1', 'B_2']
+        assert len(cb) == 5
+        assert cb[0] == (maintask, t1)
+        assert cb[1] == (t1, t2)
+        assert cb[2] == (t2, t1)
+        assert cb[3] == (t1, t2)
+        assert cb[4] == (t2, maintask)
+
+    def test_bomb(self):
+        try:
+            1/0
+        except:
+            import sys
+            b = stackless.bomb(*sys.exc_info())
+        assert b.type is ZeroDivisionError
+        assert str(b.value).startswith('integer division')
+        assert b.traceback is not None
+
+    def test_send_exception(self):
+        def exp_sender(chan):
+            chan.send_exception(Exception, 'test')
+
+        def exp_recv(chan):
+            try:
+                val = chan.receive()
+            except Exception, exp:
+                assert exp.__class__ is Exception
+                assert str(exp) == 'test'
+
+        chan = stackless.channel()
+        t1 = stackless.tasklet(exp_recv)(chan)
+        t2 = stackless.tasklet(exp_sender)(chan)
+        stackless.run()
+
+    def test_send_sequence(self):
+        res = []
+        lst = [1,2,3,4,5,6,None]
+        iterable = iter(lst)
+        chan = stackless.channel()
+        def f(chan):
+            r = chan.receive()
+            while r:
+                res.append(r)
+                r = chan.receive()
+
+        t = stackless.tasklet(f)(chan)
+        chan.send_sequence(iterable)
+        assert res == [1,2,3,4,5,6]
+
+    def test_getruncount(self):
+        assert stackless.getruncount() == 1
+        def with_schedule():
+            assert stackless.getruncount() == 2
+
+        t1 = stackless.tasklet(with_schedule)()
+        assert stackless.getruncount() == 2
+        stackless.schedule()
+        def with_run():
+            assert stackless.getruncount() == 1
+
+        t2 = stackless.tasklet(with_run)()
+        stackless.run()
+
+    def test_schedule_return(self):
+        def f():pass
+        t1= stackless.tasklet(f)()
+        r = stackless.schedule()
+        assert r is stackless.getmain()
+        t2 = stackless.tasklet(f)()
+        r = stackless.schedule('test')
+        assert r == 'test'
+
+    def test_simple_pipe(self):
+        def pipe(X_in, X_out):
+            foo = X_in.receive()
+            X_out.send(foo)
+
+        X, Y = stackless.channel(), stackless.channel()
+        t = stackless.tasklet(pipe)(X, Y)
+        stackless.run()
+        X.send(42)
+        assert Y.receive() == 42
+
+    def test_nested_pipe(self):
+        dprint('tnp ==== 1')
+        def pipe(X, Y):
+            dprint('tnp_P ==== 1')
+            foo = X.receive()
+            dprint('tnp_P ==== 2')
+            Y.send(foo)
+            dprint('tnp_P ==== 3')
+
+        def nest(X, Y):
+            X2, Y2 = stackless.channel(), stackless.channel()
+            t = stackless.tasklet(pipe)(X2, Y2)
+            dprint('tnp_N ==== 1')
+            X_Val = X.receive()
+            dprint('tnp_N ==== 2')
+            X2.send(X_Val)
+            dprint('tnp_N ==== 3')
+            Y2_Val = Y2.receive() 
+            dprint('tnp_N ==== 4')
+            Y.send(Y2_Val)
+            dprint('tnp_N ==== 5')
+
+        X, Y = stackless.channel(), stackless.channel()
+        t1 = stackless.tasklet(nest)(X, Y)
+        X.send(13)
+        dprint('tnp ==== 2')
+        res = Y.receive() 
+        dprint('tnp ==== 3')
+        assert res == 13
+        if SHOW_STRANGE:
+            raise Exception('force prints')
+
+    def test_wait_two(self):
+        """
+        A tasklets/channels adaptation of the test_wait_two from the
+        logic object space
+        """
+        def sleep(X, Y):
+            dprint('twt_S ==== 1')
+            value = X.receive()
+            dprint('twt_S ==== 2')
+            Y.send((X, value))
+            dprint('twt_S ==== 3')
+
+        def wait_two(X, Y, Ret_chan):
+            Barrier = stackless.channel()
+            stackless.tasklet(sleep)(X, Barrier)
+            stackless.tasklet(sleep)(Y, Barrier)
+            dprint('twt_W ==== 1')
+            ret = Barrier.receive()
+            dprint('twt_W ==== 2')
+            if ret[0] == X:
+                Ret_chan.send((1, ret[1]))
+            else:
+                Ret_chan.send((2, ret[1]))
+            dprint('twt_W ==== 3')
+
+        X = stackless.channel()
+        Y = stackless.channel()
+        Ret_chan = stackless.channel()
+
+        stackless.tasklet(wait_two)(X, Y, Ret_chan)
+
+        dprint('twt ==== 1')
+        Y.send(42)
+
+        dprint('twt ==== 2')
+        X.send(42)
+        dprint('twt ==== 3')
+        value = Ret_chan.receive() 
+        dprint('twt ==== 4')
+        assert value == (2, 42)
+        
+
+    def test_schedule_return_value(self):
+
+        def task(val):
+            value = stackless.schedule(val)
+            assert value == val
+
+        stackless.tasklet(task)(10)
+        stackless.tasklet(task)(5)
+
+        stackless.run()
+
+

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_stackless_pickling.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_stackless_pickling.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,66 @@
+"""
+this test should probably not run from CPython or py.py.
+I'm not entirely sure, how to do that.
+"""
+from __future__ import absolute_import
+from py.test import skip
+try:
+    import stackless
+except ImportError:
+    try:
+        from .. import stackless as stackless
+    except ImportError, e:
+        skip('cannot import stackless: %s' % (e,))
+
+
+
+class Test_StacklessPickling:
+
+    def test_basic_tasklet_pickling(self):
+        try:
+            import stackless
+        except ImportError:
+            skip("can't load stackless and don't know why!!!")
+        from stackless import run, schedule, tasklet
+        import pickle
+
+        output = []
+
+        import new
+
+        mod = new.module('mod')
+        mod.output = output
+
+        exec """from stackless import schedule
+        
+def aCallable(name):
+    output.append(('b', name))
+    schedule()
+    output.append(('a', name))
+""" in mod.__dict__
+        import sys
+        sys.modules['mod'] = mod
+        aCallable = mod.aCallable
+
+
+        tasks = []
+        for name in "ABCDE":
+            tasks.append(tasklet(aCallable)(name))
+
+        schedule()
+
+        assert output == [('b', x) for x in "ABCDE"]
+        del output[:]
+        pickledTasks = pickle.dumps(tasks)
+
+        schedule()
+        assert output == [('a', x) for x in "ABCDE"]
+        del output[:]
+        
+        unpickledTasks = pickle.loads(pickledTasks)
+        for task in unpickledTasks:
+            task.insert()
+
+        schedule()
+        assert output == [('a', x) for x in "ABCDE"]
+

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_struct_extra.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_struct_extra.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,25 @@
+from __future__ import absolute_import
+from .. import struct 
+
+def test_simple():
+    morezeros = '\x00' * (struct.calcsize('l')-4)
+    assert struct.pack('<l', 16) == '\x10\x00\x00\x00' + morezeros
+    assert struct.pack('4s', 'WAVE') == 'WAVE'
+    assert struct.pack('<4sl', 'WAVE', 16) == 'WAVE\x10\x00\x00\x00' + morezeros
+    s = 'ABCD01234567\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00'
+    assert struct.unpack('<4s4H2lH', s) == ('ABCD', 0x3130, 0x3332, 0x3534,
+                                            0x3736, 1, 2, 3)
+
+def test_infinity():
+    INFINITY = 1e200 * 1e200
+    assert str(struct.unpack("!d", struct.pack("!d", INFINITY))[0]) \
+           == str(INFINITY)
+    assert str(struct.unpack("!d", struct.pack("!d", -INFINITY))[0]) \
+           == str(-INFINITY)
+
+def test_nan():
+    INFINITY = 1e200 * 1e200
+    NAN = INFINITY / INFINITY
+    assert str(struct.unpack("!d", '\xff\xf8\x00\x00\x00\x00\x00\x00')[0]) \
+           == str(NAN)
+    assert str(struct.unpack("!d", struct.pack("!d", NAN))[0]) == str(NAN)

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_structseq.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_structseq.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,80 @@
+from __future__ import absolute_import
+import py
+from .._structseq import *
+
+
+class mydata:
+    __metaclass__ = structseqtype
+
+    st_mode  = structseqfield(0, "protection bits")
+    st_ino   = structseqfield(1)
+    st_dev   = structseqfield(2)
+    st_nlink = structseqfield(3)
+    st_uid   = structseqfield(4)
+    st_gid   = structseqfield(5)
+    st_size  = structseqfield(6)
+    _st_atime_as_int = structseqfield(7)
+    _st_mtime_as_int = structseqfield(8)
+    _st_ctime_as_int = structseqfield(9)
+    # skip to higher numbers for fields not part of the sequence.
+    # the numbers are only used to ordering
+    st_rdev  = structseqfield(50, "device type (if inode device)")
+    st_atime = structseqfield(57, default=lambda self: self._st_atime_as_int)
+    st_mtime = structseqfield(58, default=lambda self: self._st_mtime_as_int)
+    st_ctime = structseqfield(59, default=lambda self: self._st_ctime_as_int)
+
+
+def test_class():
+    assert mydata.st_mode.__doc__ == "protection bits"
+    assert mydata.n_fields == 14
+    assert mydata.n_sequence_fields == 10
+    assert mydata.n_unnamed_fields == 0
+
+def test_mydata():
+    x = mydata(range(100, 111))
+    assert x.n_sequence_fields == type(x).n_sequence_fields == 10
+    assert x.n_fields == type(x).n_fields == 14
+    assert x.st_mode  == 100
+    assert x.st_size  == 106
+    assert x.st_ctime == 109    # copied by the default=lambda...
+    assert x.st_rdev  == 110
+    assert len(x)     == 10
+    assert list(x)    == range(100, 110)
+    assert x + (5,)   == tuple(range(100, 110)) + (5,)
+    assert x[4:12:2]  == (104, 106, 108)
+    assert 104 in x
+    assert 110 not in x
+
+def test_default_None():
+    x = mydata(range(100, 110))
+    assert x.st_rdev is None
+
+def test_constructor():
+    x = mydata(range(100, 111), {'st_mtime': 12.25})
+    assert x[8] == 108
+    assert x.st_mtime == 12.25
+
+def test_compare_like_tuple():
+    x = mydata(range(100, 111))
+    y = mydata(range(100, 110) + [555])
+    assert x == tuple(range(100, 110))
+    assert x == y    # blame CPython
+    assert hash(x) == hash(y) == hash(tuple(range(100, 110)))
+
+def test_pickle():
+    import pickle
+    x = mydata(range(100, 111))
+    s = pickle.dumps(x)
+    y = pickle.loads(s)
+    assert x == y
+    assert x.st_rdev == y.st_rdev == 110
+
+def test_readonly():
+    x = mydata(range(100, 113))
+    py.test.raises((TypeError, AttributeError), "x.st_mode = 1")
+    py.test.raises((TypeError, AttributeError), "x.st_mtime = 1")
+    py.test.raises((TypeError, AttributeError), "x.st_rdev = 1")
+
+def test_no_extra_assignments():
+    x = mydata(range(100, 113))
+    py.test.raises((TypeError, AttributeError), "x.some_random_attribute = 1")

Added: pypy/branch/fast-forward/lib_pypy/pypy_test/test_syslog.py
==============================================================================
--- (empty file)
+++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_syslog.py	Tue Jul  6 00:00:33 2010
@@ -0,0 +1,11 @@
+from __future__ import absolute_import
+# XXX very minimal test
+
+from ..ctypes_config_cache import rebuild
+rebuild.rebuild_one('syslog.ctc.py')
+
+from .. import syslog
+
+
+def test_syslog():
+    assert hasattr(syslog, 'LOG_ALERT')

Modified: pypy/branch/fast-forward/lib_pypy/stackless.py
==============================================================================
--- pypy/branch/fast-forward/lib_pypy/stackless.py	(original)
+++ pypy/branch/fast-forward/lib_pypy/stackless.py	Tue Jul  6 00:00:33 2010
@@ -114,7 +114,7 @@
 
 import operator
 __all__ = 'run getcurrent getmain schedule tasklet channel coroutine \
-                TaskletExit greenlet'.split()
+                greenlet'.split()
 
 _global_task_id = 0
 _squeue = None
@@ -153,12 +153,12 @@
     _last_task = next
     assert not next.blocked
     if next is not current:
-        next.switch()
+        try:
+            next.switch()
+        except CoroutineExit:
+            raise TaskletExit
     return current
 
-
-class TaskletExit(Exception):pass
-
 def set_schedule_callback(callback):
     global _schedule_callback
     _schedule_callback = callback
@@ -435,6 +435,7 @@
         the tasklet will silently die.
         """
         if not self.is_zombie:
+            # Killing the tasklet by throwing TaskletExit exception.
             coroutine.kill(self)
             _scheduler_remove(self)
             self.alive = False

Modified: pypy/branch/fast-forward/py/_process/cmdexec.py
==============================================================================
--- pypy/branch/fast-forward/py/_process/cmdexec.py	(original)
+++ pypy/branch/fast-forward/py/_process/cmdexec.py	Tue Jul  6 00:00:33 2010
@@ -8,20 +8,25 @@
 from subprocess import Popen, PIPE
 
 def cmdexec(cmd):
-    """ return output of executing 'cmd' in a separate process.
+    """ return unicode output of executing 'cmd' in a separate process.
 
     raise cmdexec.ExecutionFailed exeception if the command failed.
     the exception will provide an 'err' attribute containing
     the error-output from the command.
+    if the subprocess module does not provide a proper encoding/unicode strings
+    sys.getdefaultencoding() will be used, if that does not exist, 'UTF-8'.
     """
     process = subprocess.Popen(cmd, shell=True, 
             universal_newlines=True,
             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     out, err = process.communicate()
-    out = py.builtin._totext(out, sys.stdout.encoding or
-                             sys.getdefaultencoding())
-    err = py.builtin._totext(err, sys.stderr.encoding or
-                             sys.getdefaultencoding())
+    if sys.version_info[0] < 3: # on py3 we get unicode strings, on py2 not 
+        try:
+            default_encoding = sys.getdefaultencoding() # jython may not have it
+        except AttributeError:
+            default_encoding = sys.stdout.encoding or 'UTF-8'
+        out = unicode(out, process.stdout.encoding or default_encoding)
+        err = unicode(err, process.stderr.encoding or default_encoding)
     status = process.poll()
     if status:
         raise ExecutionFailed(status, status, cmd, out, err)

Modified: pypy/branch/fast-forward/pypy/annotation/model.py
==============================================================================
--- pypy/branch/fast-forward/pypy/annotation/model.py	(original)
+++ pypy/branch/fast-forward/pypy/annotation/model.py	Tue Jul  6 00:00:33 2010
@@ -229,11 +229,15 @@
 
 class SomeChar(SomeString):
     "Stands for an object known to be a string of length 1."
+    can_be_None = False
+    def __init__(self):    # no 'can_be_None' argument here
+        pass
 
 class SomeUnicodeCodePoint(SomeUnicodeString):
     "Stands for an object known to be a unicode codepoint."
-    def can_be_none(self):
-        return False
+    can_be_None = False
+    def __init__(self):    # no 'can_be_None' argument here
+        pass
 
 SomeString.basestringclass = SomeString
 SomeString.basecharclass = SomeChar

Modified: pypy/branch/fast-forward/pypy/annotation/unaryop.py
==============================================================================
--- pypy/branch/fast-forward/pypy/annotation/unaryop.py	(original)
+++ pypy/branch/fast-forward/pypy/annotation/unaryop.py	Tue Jul  6 00:00:33 2010
@@ -5,7 +5,7 @@
 from types import MethodType
 from pypy.annotation.model import \
      SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, \
-     SomeDict, SomeTuple, SomeImpossibleValue, \
+     SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, \
      SomeInstance, SomeBuiltin, SomeFloat, SomeIterator, SomePBC, \
      SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \
      s_ImpossibleValue, s_Bool, s_None, \
@@ -494,9 +494,6 @@
     def getanyitem(str):
         return str.basecharclass()
 
-    def ord(str):
-        return SomeInteger(nonneg=True)
-
     def method_split(str, patt): # XXX
         getbookkeeper().count("str_split", str, patt)
         return getbookkeeper().newlist(str.basestringclass())
@@ -541,11 +538,16 @@
         return SomeUnicodeString()
     method_decode.can_only_throw = [UnicodeDecodeError]
 
-class __extend__(SomeChar):
+class __extend__(SomeChar, SomeUnicodeCodePoint):
 
     def len(chr):
         return immutablevalue(1)
 
+    def ord(str):
+        return SomeInteger(nonneg=True)
+
+class __extend__(SomeChar):
+
     def method_isspace(chr):
         return s_Bool
 

Modified: pypy/branch/fast-forward/pypy/conftest.py
==============================================================================
--- pypy/branch/fast-forward/pypy/conftest.py	(original)
+++ pypy/branch/fast-forward/pypy/conftest.py	Tue Jul  6 00:00:33 2010
@@ -40,6 +40,13 @@
            default="host", callback=_set_platform,
            help="set up tests to use specified platform as compile/run target")
 
+def pytest_sessionstart():
+    # have python subprocesses avoid startup customizations by default
+    try:
+        del os.environ['PYTHONSTARTUP']
+    except KeyError:
+        pass
+
 def pytest_funcarg__space(request):
     spaceconfig = getattr(request.cls, 'spaceconfig', {})
     return gettestobjspace(**spaceconfig)

Modified: pypy/branch/fast-forward/pypy/doc/index.txt
==============================================================================
--- pypy/branch/fast-forward/pypy/doc/index.txt	(original)
+++ pypy/branch/fast-forward/pypy/doc/index.txt	Tue Jul  6 00:00:33 2010
@@ -8,7 +8,7 @@
 Getting into PyPy ... 
 =============================================
 
-* `Release 1.2`_: the latest official release
+* `Release 1.3`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy 
 
@@ -56,4 +56,4 @@
 .. _`Documentation`: docindex.html 
 .. _`Getting Started`: getting-started.html
 .. _papers: extradoc.html
-.. _`Release 1.2`: release-1.2.0.html
+.. _`Release 1.3`: http://pypy.org/download.html

Modified: pypy/branch/fast-forward/pypy/doc/jit/pyjitpl5.txt
==============================================================================
--- pypy/branch/fast-forward/pypy/doc/jit/pyjitpl5.txt	(original)
+++ pypy/branch/fast-forward/pypy/doc/jit/pyjitpl5.txt	Tue Jul  6 00:00:33 2010
@@ -8,15 +8,15 @@
 Implementation of the JIT
 =========================
 
-The JIT's `theory`_ is great in principle, but actual code is a different
+The JIT's `theory`_ is great in principle, but the actual code is a different
 story. This section tries to give a high level overview of how PyPy's JIT is
-implemented.  It's helpful to have a basic understanding of how the PyPy
-`translation tool chain`_ works before digging into the sources.
+implemented.  It's helpful to have an understanding of how the PyPy `translation
+tool chain`_ works before digging into the sources.
 
-Almost all JIT specific code is found in the two pypy/jit subdirectories,
-metainterp, and backend.  The metainterp directory holds platform independent
-code including the translation generator, the tracer, and the optimizer.  Code
-in the backend directory is responsible for generating machine code.
+Almost all JIT specific code is found in pypy/jit subdirectories.  Translation
+time code is in the codewriter directory.  The metainterp directory holds
+platform independent code including the the tracer and the optimizer.  Code in
+the backend directory is responsible for generating machine code.
 
 .. _`theory`: overview.html
 .. _`translation tool chain`: ../translation.html
@@ -34,7 +34,7 @@
 interpreter defines its hints in pypy/module/pypyjit/interp_jit.py in a few
 overriden methods of the defaut interpreter loop.
 
-The interpreter wishing to use the PyPy's JIT must define a list of *green*
+An interpreter wishing to use the PyPy's JIT must define a list of *green*
 variables and a list of *red* variables.  The *green* variables are loop
 constants.  They are used to identify the current loop.  Red variables are for
 everything else used in the execution loop.  For example, the Python interpreter
@@ -58,11 +58,11 @@
 is replaced with a call to a function, maybe_compile_and_run in warmstate.py,
 that checks if current loop is "hot" and should be compiled.
 
-Next, starting with the portal graph, metainterp/codewriter.py converts the
-graphs of the interpreter into JIT bytecode.  Since this bytecode is stored in
-the final binary, it's designed to be concise rather than fast.  The bytecode
-codewriter doesn't "see" (what it sees is defined by the JIT's policy) every
-part of the interpreter.  In these cases, it simply inserts an opaque call.
+Next, starting with the portal graph, codewriter/\*.py converts the graphs of the
+interpreter into JIT bytecode.  Since this bytecode is stored in the final
+binary, it's designed to be concise rather than fast.  The bytecode codewriter
+doesn't "see" (what it sees is defined by the JIT's policy) every part of the
+interpreter.  In these cases, it simply inserts an opaque call.
 
 Finally, translation finishes, including the bytecode of the interpreter in the
 final binary, and interpreter is ready to use the runtime component of the JIT.

Modified: pypy/branch/fast-forward/pypy/doc/release-1.3.0.txt
==============================================================================
--- pypy/branch/fast-forward/pypy/doc/release-1.3.0.txt	(original)
+++ pypy/branch/fast-forward/pypy/doc/release-1.3.0.txt	Tue Jul  6 00:00:33 2010
@@ -13,8 +13,8 @@
 stability, this feature is in alpha stage and it is not yet suited for
 production environments.
 
-Highlight of this release
-=========================
+Highlights of this release
+==========================
 
 * We introduced support for CPython extension modules written in C. As of now,
   this support is in alpha, and it's very unlikely unaltered C extensions will
@@ -39,3 +39,6 @@
 
 * Various small improvements have been added to the JIT code, as well as a great
   speedup of compiling time.
+
+Cheers,
+Maciej Fijalkowski, Armin Rigo, Alex Gaynor, Amaury Forgeot d'Arc and the PyPy team

Modified: pypy/branch/fast-forward/pypy/doc/stackless.txt
==============================================================================
--- pypy/branch/fast-forward/pypy/doc/stackless.txt	(original)
+++ pypy/branch/fast-forward/pypy/doc/stackless.txt	Tue Jul  6 00:00:33 2010
@@ -124,10 +124,17 @@
 
 * ``coro.kill()``
 
-    Kill ``coro`` by sending an exception to it.  (At the moment, the
-    exception is not visible to app-level, which means that you cannot
-    catch it, and that ``try: finally:`` clauses are not honored.  This
-    will be fixed in the future.)
+    Kill ``coro`` by sending a CoroutineExit exception and switching
+    execution immediately to it. This exception can be caught in the 
+    coroutine itself and can be raised from any call to ``coro.switch()``. 
+    This exception isn't propagated to the parent coroutine.
+
+* ``coro.throw(type, value)``
+
+    Insert an exception in ``coro`` an resume switches execution
+    immediately to it. In the coroutine itself, this exception
+    will come from any call to ``coro.switch()`` and can be caught. If the
+    exception isn't caught, it will be propagated to the parent coroutine.
 
 Example
 ~~~~~~~

Modified: pypy/branch/fast-forward/pypy/interpreter/pyopcode.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyopcode.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyopcode.py	Tue Jul  6 00:00:33 2010
@@ -14,6 +14,7 @@
 from pypy.rlib import jit, rstackovf, rstack
 from pypy.rlib.rarithmetic import r_uint, intmask
 from pypy.rlib.unroll import unrolling_iterable
+from pypy.rlib.debug import check_nonneg
 from pypy.tool.stdlib_opcode import (bytecode_spec, host_bytecode_spec,
                                      unrolling_all_opcode_descs, opmap,
                                      host_opmap)
@@ -186,7 +187,7 @@
                 lo = ord(co_code[next_instr])
                 hi = ord(co_code[next_instr+1])
                 next_instr += 2
-                oparg = (hi << 8) | lo
+                oparg = (hi * 256) | lo
             else:
                 oparg = 0
 
@@ -197,7 +198,7 @@
                 lo = ord(co_code[next_instr+1])
                 hi = ord(co_code[next_instr+2])
                 next_instr += 3
-                oparg = (oparg << 16) | (hi << 8) | lo
+                oparg = (oparg * 65536) | (hi * 256) | lo
 
             if opcode == self.opcodedesc.RETURN_VALUE.index:
                 w_returnvalue = self.popvalue()
@@ -323,7 +324,9 @@
     ##
 
     def NOP(self, oparg, next_instr):
-        pass
+        # annotation-time check: if it fails, it means that the decoding
+        # of oparg failed to produce an integer which is annotated as non-neg
+        check_nonneg(oparg)
 
     def LOAD_FAST(self, varindex, next_instr):
         # access a local variable directly

Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/parsestring.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyparser/parsestring.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyparser/parsestring.py	Tue Jul  6 00:00:33 2010
@@ -76,12 +76,12 @@
             bufp = 0
             bufq = len(buf)
         assert 0 <= bufp <= bufq
-        w_substr = space.wrap(buf[bufp : bufq])
+        substr = buf[bufp:bufq]
         if rawmode:
-            w_v = unicodehelper.PyUnicode_DecodeRawUnicodeEscape(space, w_substr)
+            v = unicodehelper.PyUnicode_DecodeRawUnicodeEscape(space, substr)
         else:
-            w_v = unicodehelper.PyUnicode_DecodeUnicodeEscape(space, w_substr)
-        return w_v
+            v = unicodehelper.PyUnicode_DecodeUnicodeEscape(space, substr)
+        return space.wrap(v)
 
     need_encoding = (encoding is not None and
                      encoding != "utf-8" and encoding != "iso-8859-1")
@@ -90,7 +90,7 @@
     substr = s[ps : q]
     if rawmode or '\\' not in s[ps:]:
         if need_encoding:
-            w_u = unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(substr))
+            w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, substr))
             #w_v = space.wrap(space.unwrap(w_u).encode(encoding)) this works
             w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
             return w_v
@@ -100,7 +100,7 @@
     enc = None
     if need_encoding:
          enc = encoding
-    v = PyString_DecodeEscape(space, substr, unicode, enc)
+    v = PyString_DecodeEscape(space, substr, enc)
     return space.wrap(v)
 
 def hexbyte(val):
@@ -109,10 +109,9 @@
         result = "0" + result
     return result
 
-def PyString_DecodeEscape(space, s, unicode, recode_encoding):
+def PyString_DecodeEscape(space, s, recode_encoding):
     """
-    Unescape a backslash-escaped string. If unicode is non-zero,
-    the string is a u-literal. If recode_encoding is non-zero,
+    Unescape a backslash-escaped string. If recode_encoding is non-zero,
     the string is UTF-8 encoded and should be re-encoded in the
     specified encoding.
     """
@@ -165,19 +164,22 @@
             span = ps
             span += (span < end) and (s[span] in '01234567')
             span += (span < end) and (s[span] in '01234567')
-            lis.append(chr(int(s[prevps : span], 8)))
+            octal = s[prevps : span]
+            # emulate a strange wrap-around behavior of CPython:
+            # \400 is the same as \000 because 0400 == 256
+            num = int(octal, 8) & 0xFF
+            lis.append(chr(num))
             ps = span
         elif ch == 'x':
             if ps+2 <= end and isxdigit(s[ps]) and isxdigit(s[ps + 1]):
-                lis.append(chr(int(s[ps : ps + 2], 16)))
+                hexa = s[ps : ps + 2]
+                num = int(hexa, 16)
+                lis.append(chr(num))
                 ps += 2
             else:
                 raise_app_valueerror(space, 'invalid \\x escape')
             # ignored replace and ignore for now
 
-        elif unicode and (ch == 'u' or ch == 'U' or ch == 'N'):
-            raise_app_valueerror(space, 'Unicode escapes not legal '
-                                        'when Unicode disabled')
         else:
             # this was not an escape, so the backslash
             # has to be added, and we start over in
@@ -204,7 +206,7 @@
     # while (s < end && *s != '\\') s++; */ /* inefficient for u".."
     while ps < end and ord(s[ps]) & 0x80:
         ps += 1
-    w_u = unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(s[pt : ps]))
+    w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, s[pt:ps]))
     w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
     v = space.str_w(w_v)
     return v, ps

Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_parsestring.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_parsestring.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_parsestring.py	Tue Jul  6 00:00:33 2010
@@ -2,67 +2,51 @@
 import py
 
 class TestParsetring:
+    def parse_and_compare(self, literal, value):
+        space = self.space
+        w_ret = parsestring.parsestr(space, None, literal)
+        if isinstance(value, str):
+            assert space.type(w_ret) == space.w_str
+            assert space.str_w(w_ret) == value
+        elif isinstance(value, unicode):
+            assert space.type(w_ret) == space.w_unicode
+            assert space.unicode_w(w_ret) == value
+        else:
+            assert False
+
     def test_simple(self):
         space = self.space
-        s = 'hello world'
-        w_ret = parsestring.parsestr(space, None, repr(s))
-        assert space.str_w(w_ret) == s
-        s = 'hello\n world'
-        w_ret = parsestring.parsestr(space, None, repr(s))
-        assert space.str_w(w_ret) == s
-        s = "'''hello\\x42 world'''"
-        w_ret = parsestring.parsestr(space, None, s)
-        assert space.str_w(w_ret) == 'hello\x42 world'
-        s = r'"\0"'
-        w_ret = parsestring.parsestr(space, None, s)
-        assert space.str_w(w_ret) == chr(0)
-        s = r'"\07"'
-        w_ret = parsestring.parsestr(space, None, s)
-        assert space.str_w(w_ret) == chr(7)
-        s = r'"\123"'
-        w_ret = parsestring.parsestr(space, None, s)
-        assert space.str_w(w_ret) == chr(0123)
-        s = r'"\x"'
-        space.raises_w(space.w_ValueError, parsestring.parsestr, space, None, s)
-        s = r'"\x7"'
-        space.raises_w(space.w_ValueError, parsestring.parsestr, space, None, s)
-        s = r'"\x7g"'
-        space.raises_w(space.w_ValueError, parsestring.parsestr, space, None, s)
-        s = r'"\xfF"'
-        w_ret = parsestring.parsestr(space, None, s)
-        assert space.str_w(w_ret) == chr(0xFF)
+        for s in ['hello world', 'hello\n world']:
+            self.parse_and_compare(repr(s), s)
+
+        self.parse_and_compare("'''hello\\x42 world'''", 'hello\x42 world')
+
+        # octal
+        self.parse_and_compare(r'"\0"', chr(0))
+        self.parse_and_compare(r'"\07"', chr(7))
+        self.parse_and_compare(r'"\123"', chr(0123))
+        self.parse_and_compare(r'"\400"', chr(0))
+        self.parse_and_compare(r'"\9"', '\\' + '9')
+        self.parse_and_compare(r'"\08"', chr(0) + '8')
+
+        # hexadecimal
+        self.parse_and_compare(r'"\xfF"', chr(0xFF))
+        self.parse_and_compare(r'"\""', '"')
+        self.parse_and_compare(r"'\''", "'")
+        for s in (r'"\x"', r'"\x7"', r'"\x7g"'):
+            space.raises_w(space.w_ValueError,
+                           parsestring.parsestr, space, None, s)
 
-        s = r'"\""'
-        w_ret = parsestring.parsestr(space, None, s)
-        assert space.str_w(w_ret) == '"'
-        
-        s = r"'\''"
-        w_ret = parsestring.parsestr(space, None, s)
-        assert space.str_w(w_ret) == "'"
-        
-        
     def test_unicode(self):
         space = self.space
-        s = u'hello world'
-        w_ret = parsestring.parsestr(space, None, repr(s))
-        ret = space.unwrap(w_ret)
-        assert isinstance(ret, unicode)
-        assert ret == s
-        s = u'hello\n world'
-        w_ret = parsestring.parsestr(self.space, None, repr(s))
-        ret = space.unwrap(w_ret)
-        assert isinstance(ret, unicode)
-        assert ret == s
-        s = "u'''hello\\x42 world'''"
-        w_ret = parsestring.parsestr(self.space, None, s)
-        ret = space.unwrap(w_ret)
-        assert isinstance(ret, unicode)
-        assert ret == u'hello\x42 world'
-        s = "u'''hello\\u0842 world'''"
-        w_ret = parsestring.parsestr(self.space, None, s)
-        ret = space.unwrap(w_ret)
-        assert isinstance(ret, unicode)
-        assert ret == u'hello\u0842 world'
+        for s in [u'hello world', u'hello\n world']:
+            self.parse_and_compare(repr(s), s)
+
+        self.parse_and_compare("u'''hello\\x42 world'''",
+                               u'hello\x42 world')
+        self.parse_and_compare("u'''hello\\u0842 world'''",
+                               u'hello\u0842 world')
+
         s = "u'\x81'"
         s = s.decode("koi8-u").encode("utf8")
         w_ret = parsestring.parsestr(self.space, 'koi8-u', s)
@@ -88,7 +72,6 @@
         assert space.unwrap(w_ret) == "hello"
 
     def test_simple_enc_roundtrip(self):
-        #py.test.skip("crashes in app_codecs, but when cheating using .encode at interp-level passes?!")
         space = self.space
         s = "'\x81'"
         s = s.decode("koi8-u").encode("utf8")

Modified: pypy/branch/fast-forward/pypy/interpreter/test/test_module.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/test/test_module.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/test/test_module.py	Tue Jul  6 00:00:33 2010
@@ -58,7 +58,7 @@
         r = repr(_pypy_interact)
         assert (r.startswith("<module '_pypy_interact' from ") and
                 ('lib_pypy/_pypy_interact.py' in r or
-                 r'lib_pypy\\_interact.py' in r.lower()) and
+                 r'lib_pypy\\_pypy_interact.py' in r.lower()) and
                 r.endswith('>'))
         nofile = type(_pypy_interact)('nofile', 'foo')
         assert repr(nofile) == "<module 'nofile' from ?>"

Modified: pypy/branch/fast-forward/pypy/interpreter/unicodehelper.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/unicodehelper.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/unicodehelper.py	Tue Jul  6 00:00:33 2010
@@ -1,30 +1,10 @@
-from pypy.interpreter import gateway
+from pypy.module._codecs import interp_codecs
 
-app = gateway.applevel(r'''
-    def PyUnicode_DecodeUnicodeEscape(data):
-        import _codecs
-        return _codecs.unicode_escape_decode(data)[0]
+def PyUnicode_AsEncodedString(space, w_data, w_encoding):
+    return interp_codecs.encode(space, w_data, w_encoding)
 
-    def PyUnicode_DecodeRawUnicodeEscape(data):
-        import _codecs
-        return _codecs.raw_unicode_escape_decode(data)[0]
-
-    def PyUnicode_DecodeUTF8(data):
-        import _codecs
-        return _codecs.utf_8_decode(data)[0]
-
-    def PyUnicode_AsEncodedString(data, encoding):
-        import _codecs
-        return _codecs.encode(data, encoding)
-
-    def PyUnicode_EncodeUTF8(data):
-        import _codecs
-        return _codecs.utf_8_encode(data)[0]
-
-''')
-
-PyUnicode_DecodeUnicodeEscape = app.interphook('PyUnicode_DecodeUnicodeEscape')
-PyUnicode_DecodeRawUnicodeEscape = app.interphook('PyUnicode_DecodeRawUnicodeEscape')
-PyUnicode_DecodeUTF8 = app.interphook('PyUnicode_DecodeUTF8')
-PyUnicode_AsEncodedString = app.interphook('PyUnicode_AsEncodedString')
-PyUnicode_EncodeUTF8 = app.interphook('PyUnicode_EncodeUTF8') 
+# These functions take and return unwrapped rpython strings and unicodes
+PyUnicode_DecodeUnicodeEscape = interp_codecs.make_raw_decoder('unicode_escape')
+PyUnicode_DecodeRawUnicodeEscape = interp_codecs.make_raw_decoder('raw_unicode_escape')
+PyUnicode_DecodeUTF8 = interp_codecs.make_raw_decoder('utf_8')
+PyUnicode_EncodeUTF8 = interp_codecs.make_raw_encoder('utf_8')

Modified: pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py	Tue Jul  6 00:00:33 2010
@@ -832,13 +832,16 @@
                     raise Exception("Nonsense type %s" % TYPE)
 
             failindex = self.cpu._execute_token(loop_token)
+            jd = loop_token.outermost_jitdriver_sd
+            assert jd is not None, ("call_assembler(): the loop_token needs "
+                                    "to have 'outermost_jitdriver_sd'")
+            if jd.index_of_virtualizable != -1:
+                vable = args[jd.index_of_virtualizable]
+            else:
+                vable = lltype.nullptr(llmemory.GCREF.TO)
+            assembler_helper_ptr = jd.assembler_helper_adr.ptr  # fish
             try:
-                if self.cpu.index_of_virtualizable != -1:
-                    return self.cpu.assembler_helper_ptr(failindex,
-                        args[self.cpu.index_of_virtualizable])
-                else:
-                    return self.cpu.assembler_helper_ptr(failindex,
-                        lltype.nullptr(llmemory.GCREF.TO))
+                return assembler_helper_ptr(failindex, vable)
             except LLException, lle:
                 assert _last_exception is None, "exception left behind"
                 _last_exception = lle
@@ -1196,10 +1199,18 @@
     array = array._obj.container
     return cast_to_int(array.getitem(index))
 
+def do_getarrayitem_raw_int(array, index):
+    array = array.adr.ptr._obj
+    return cast_to_int(array.getitem(index))
+
 def do_getarrayitem_gc_float(array, index):
     array = array._obj.container
     return cast_to_float(array.getitem(index))
 
+def do_getarrayitem_raw_float(array, index):
+    array = array.adr.ptr._obj
+    return cast_to_float(array.getitem(index))
+
 def do_getarrayitem_gc_ptr(array, index):
     array = array._obj.container
     return cast_to_ptr(array.getitem(index))
@@ -1248,12 +1259,24 @@
     newvalue = cast_from_int(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
+def do_setarrayitem_raw_int(array, index, newvalue):
+    array = array.adr.ptr
+    ITEMTYPE = lltype.typeOf(array).TO.OF
+    newvalue = cast_from_int(ITEMTYPE, newvalue)
+    array._obj.setitem(index, newvalue)
+
 def do_setarrayitem_gc_float(array, index, newvalue):
     array = array._obj.container
     ITEMTYPE = lltype.typeOf(array).OF
     newvalue = cast_from_float(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
+def do_setarrayitem_raw_float(array, index, newvalue):
+    array = array.adr.ptr
+    ITEMTYPE = lltype.typeOf(array).TO.OF
+    newvalue = cast_from_int(ITEMTYPE, newvalue)
+    array._obj.setitem(index, newvalue)
+
 def do_setarrayitem_gc_ptr(array, index, newvalue):
     array = array._obj.container
     ITEMTYPE = lltype.typeOf(array).OF

Modified: pypy/branch/fast-forward/pypy/jit/backend/llgraph/runner.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/backend/llgraph/runner.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/backend/llgraph/runner.py	Tue Jul  6 00:00:33 2010
@@ -294,7 +294,6 @@
         return llimpl.grab_exc_value()
 
     def arraydescrof(self, A):
-        assert isinstance(A, lltype.GcArray)
         assert A.OF != lltype.Void
         size = symbolic.get_size(A)
         token = history.getkind(A.OF)
@@ -317,12 +316,18 @@
     def bh_getarrayitem_gc_i(self, arraydescr, array, index):
         assert isinstance(arraydescr, Descr)
         return llimpl.do_getarrayitem_gc_int(array, index)
+    def bh_getarrayitem_raw_i(self, arraydescr, array, index):
+        assert isinstance(arraydescr, Descr)
+        return llimpl.do_getarrayitem_raw_int(array, index)
     def bh_getarrayitem_gc_r(self, arraydescr, array, index):
         assert isinstance(arraydescr, Descr)
         return llimpl.do_getarrayitem_gc_ptr(array, index)
     def bh_getarrayitem_gc_f(self, arraydescr, array, index):
         assert isinstance(arraydescr, Descr)
         return llimpl.do_getarrayitem_gc_float(array, index)
+    def bh_getarrayitem_raw_f(self, arraydescr, array, index):
+        assert isinstance(arraydescr, Descr)
+        return llimpl.do_getarrayitem_raw_float(array, index)
 
     def bh_getfield_gc_i(self, struct, fielddescr):
         assert isinstance(fielddescr, Descr)
@@ -372,6 +377,10 @@
         assert isinstance(arraydescr, Descr)
         llimpl.do_setarrayitem_gc_int(array, index, newvalue)
 
+    def bh_setarrayitem_raw_i(self, arraydescr, array, index, newvalue):
+        assert isinstance(arraydescr, Descr)
+        llimpl.do_setarrayitem_raw_int(array, index, newvalue)
+
     def bh_setarrayitem_gc_r(self, arraydescr, array, index, newvalue):
         assert isinstance(arraydescr, Descr)
         llimpl.do_setarrayitem_gc_ptr(array, index, newvalue)
@@ -380,6 +389,10 @@
         assert isinstance(arraydescr, Descr)
         llimpl.do_setarrayitem_gc_float(array, index, newvalue)
 
+    def bh_setarrayitem_raw_f(self, arraydescr, array, index, newvalue):
+        assert isinstance(arraydescr, Descr)
+        llimpl.do_setarrayitem_raw_float(array, index, newvalue)
+
     def bh_setfield_gc_i(self, struct, fielddescr, newvalue):
         assert isinstance(fielddescr, Descr)
         llimpl.do_setfield_gc_int(struct, fielddescr.ofs, newvalue)

Modified: pypy/branch/fast-forward/pypy/jit/backend/model.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/backend/model.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/backend/model.py	Tue Jul  6 00:00:33 2010
@@ -3,10 +3,6 @@
 
 class AbstractCPU(object):
     supports_floats = False
-    _got_exception = None
-    # assembler_helper_ptr - a pointer to helper to call after a direct
-    #                        assembler call
-    portal_calldescr = None
     done_with_this_frame_void_v = -1
     done_with_this_frame_int_v = -1
     done_with_this_frame_ref_v = -1

Modified: pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py	Tue Jul  6 00:00:33 2010
@@ -1698,16 +1698,22 @@
             assert self.cpu.get_latest_value_int(0) == 10
             called.append(failindex)
             return 4 + 9
-        self.cpu.index_of_virtualizable = -1
-        self.cpu.assembler_helper_ptr = llhelper(lltype.Ptr(lltype.FuncType
-            ([lltype.Signed, llmemory.GCREF], lltype.Signed)), assembler_helper)
-        
+
+        FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF],
+                                             lltype.Signed))
+        class FakeJitDriverSD:
+            index_of_virtualizable = -1
+            _assembler_helper_ptr = llhelper(FUNCPTR, assembler_helper)
+            assembler_helper_adr = llmemory.cast_ptr_to_adr(
+                _assembler_helper_ptr)
+
         ops = '''
         [i0, i1]
         i2 = int_add(i0, i1)
         finish(i2)'''
         loop = parse(ops)
         looptoken = LoopToken()
+        looptoken.outermost_jitdriver_sd = FakeJitDriverSD()
         self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         ARGS = [lltype.Signed, lltype.Signed]
         RES = lltype.Signed
@@ -1739,9 +1745,15 @@
             assert self.cpu.get_latest_value_float(0) == 1.2 + 3.2
             called.append(failindex)
             return 13.5
-        self.cpu.index_of_virtualizable = -1
-        self.cpu.assembler_helper_ptr = llhelper(lltype.Ptr(lltype.FuncType
-            ([lltype.Signed, llmemory.GCREF], lltype.Float)), assembler_helper)
+
+        FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF],
+                                             lltype.Float))
+        class FakeJitDriverSD:
+            index_of_virtualizable = -1
+            _assembler_helper_ptr = llhelper(FUNCPTR, assembler_helper)
+            assembler_helper_adr = llmemory.cast_ptr_to_adr(
+                _assembler_helper_ptr)
+
         ARGS = [lltype.Float, lltype.Float]
         RES = lltype.Float
         self.cpu.portal_calldescr = self.cpu.calldescrof(
@@ -1753,6 +1765,7 @@
         finish(f2)'''
         loop = parse(ops)
         looptoken = LoopToken()
+        looptoken.outermost_jitdriver_sd = FakeJitDriverSD()
         self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
         self.cpu.set_future_value_float(0, 1.2)
         self.cpu.set_future_value_float(1, 2.3)

Modified: pypy/branch/fast-forward/pypy/jit/backend/test/support.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/backend/test/support.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/backend/test/support.py	Tue Jul  6 00:00:33 2010
@@ -62,11 +62,12 @@
         warmrunnerdesc = WarmRunnerDesc(t, translate_support_code=True,
                                         CPUClass=self.CPUClass,
                                         **kwds)
-        warmrunnerdesc.state.set_param_threshold(3)          # for tests
-        warmrunnerdesc.state.set_param_trace_eagerness(2)    # for tests
-        warmrunnerdesc.state.set_param_trace_limit(trace_limit)
-        warmrunnerdesc.state.set_param_inlining(inline)
-        warmrunnerdesc.state.set_param_optimizer(OPTIMIZER_FULL)
+        for jd in warmrunnerdesc.jitdrivers_sd:
+            jd.warmstate.set_param_threshold(3)          # for tests
+            jd.warmstate.set_param_trace_eagerness(2)    # for tests
+            jd.warmstate.set_param_trace_limit(trace_limit)
+            jd.warmstate.set_param_inlining(inline)
+            jd.warmstate.set_param_optimizer(OPTIMIZER_FULL)
         mixlevelann = warmrunnerdesc.annhelper
         entry_point_graph = mixlevelann.getgraph(entry_point, [s_list_of_strings],
                                                  annmodel.SomeInteger())

Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py	Tue Jul  6 00:00:33 2010
@@ -109,7 +109,6 @@
         self.malloc_array_func_addr = 0
         self.malloc_str_func_addr = 0
         self.malloc_unicode_func_addr = 0
-        self.assembler_helper_adr = 0
         self.fail_boxes_int = values_array(lltype.Signed, failargs_limit)
         self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit)
         self.fail_boxes_float = values_array(lltype.Float, failargs_limit)
@@ -144,14 +143,6 @@
                 ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode()
                 self.malloc_unicode_func_addr = rffi.cast(lltype.Signed,
                                                           ll_new_unicode)
-            if we_are_translated():
-                self.assembler_helper_adr = self.cpu.cast_ptr_to_int(
-                    self.cpu.assembler_helper_ptr)
-            else:
-                if getattr(self.cpu, 'assembler_helper_ptr', None):
-                    self.assembler_helper_adr = self.cpu.cast_ptr_to_int(
-                        self.cpu.assembler_helper_ptr)
-        
             # done
             # we generate the loop body in 'mc'
             # 'mc2' is for guard recovery code
@@ -1389,7 +1380,10 @@
         je_location = mc.get_relative_pos()
         #
         # Path A: use assembler_helper_adr
-        self._emit_call(rel32(self.assembler_helper_adr), [eax, arglocs[1]], 0,
+        jd = descr.outermost_jitdriver_sd
+        assert jd is not None
+        asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr)
+        self._emit_call(rel32(asm_helper_adr), [eax, arglocs[1]], 0,
                         tmp=ecx, force_mc=True, mc=mc)
         if isinstance(result_loc, MODRM64):
             mc.FSTP(result_loc)
@@ -1403,9 +1397,9 @@
         mc.overwrite(je_location - 1, [chr(offset)])
         #
         # Reset the vable token --- XXX really too much special logic here:-(
-        if self.cpu.index_of_virtualizable >= 0:
+        if jd.index_of_virtualizable >= 0:
             from pypy.jit.backend.llsupport.descr import BaseFieldDescr
-            fielddescr = self.cpu.vable_token_descr
+            fielddescr = jd.vable_token_descr
             assert isinstance(fielddescr, BaseFieldDescr)
             ofs = fielddescr.offset
             mc.MOV(eax, arglocs[1])

Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py	Tue Jul  6 00:00:33 2010
@@ -636,10 +636,14 @@
         self._consider_call(op, guard_op)
 
     def consider_call_assembler(self, op, guard_op):
-        descr = op.descr
         portal_calldescr = self.assembler.cpu.portal_calldescr
         size = portal_calldescr.get_result_size(self.translate_support_code)
-        vable_index = self.assembler.cpu.index_of_virtualizable
+        #
+        descr = op.descr
+        assert isinstance(descr, LoopToken)
+        jd = descr.outermost_jitdriver_sd
+        assert jd is not None
+        vable_index = jd.index_of_virtualizable
         if vable_index >= 0:
             self.rm._sync_var(op.args[vable_index])
             vable = self.fm.loc(op.args[vable_index], 1)
@@ -779,12 +783,14 @@
             arglocs.append(self.loc(op.args[0]))
             return self._call(op, arglocs)
         # boehm GC (XXX kill the following code at some point)
-        scale_of_field, basesize, _ = self._unpack_arraydescr(op.descr)
-        return self._malloc_varsize(basesize, 0, scale_of_field, op.args[0],
-                                    op.result)
+        scale_of_field, basesize, ofs_length, _ = (
+            self._unpack_arraydescr(op.descr))
+        return self._malloc_varsize(basesize, ofs_length, scale_of_field,
+                                    op.args[0], op.result)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         ofs = arraydescr.get_base_size(self.translate_support_code)
         size = arraydescr.get_item_size(self.translate_support_code)
         ptr = arraydescr.is_array_of_pointers()
@@ -792,7 +798,7 @@
         while (1 << scale) < size:
             scale += 1
         assert (1 << scale) == size
-        return scale, ofs, ptr
+        return scale, ofs, ofs_length, ptr
 
     def _unpack_fielddescr(self, fielddescr):
         assert isinstance(fielddescr, BaseFieldDescr)
@@ -827,7 +833,7 @@
     consider_unicodesetitem = consider_strsetitem
 
     def consider_setarrayitem_gc(self, op):
-        scale, ofs, ptr = self._unpack_arraydescr(op.descr)
+        scale, ofs, _, ptr = self._unpack_arraydescr(op.descr)
         base_loc  = self.rm.make_sure_var_in_reg(op.args[0], op.args)
         if scale == 0:
             need_lower_byte = True
@@ -854,7 +860,7 @@
     consider_getfield_gc_pure = consider_getfield_gc
 
     def consider_getarrayitem_gc(self, op):
-        scale, ofs, _ = self._unpack_arraydescr(op.descr)
+        scale, ofs, _, _ = self._unpack_arraydescr(op.descr)
         base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args)
         ofs_loc = self.rm.make_sure_var_in_reg(op.args[1], op.args)
         self.rm.possibly_free_vars(op.args)

Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_zrpy_gc.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_zrpy_gc.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_zrpy_gc.py	Tue Jul  6 00:00:33 2010
@@ -105,6 +105,12 @@
 
 def test_compile_boehm():
     myjitdriver = JitDriver(greens = [], reds = ['n', 'x'])
+    @dont_look_inside
+    def see(lst, n):
+        assert len(lst) == 3
+        assert lst[0] == n+10
+        assert lst[1] == n+20
+        assert lst[2] == n+30
     def main(n, x):
         while n > 0:
             myjitdriver.can_enter_jit(n=n, x=x)
@@ -112,6 +118,7 @@
             y = X()
             y.foo = x.foo
             n -= y.foo
+            see([n+10, n+20, n+30], n)
     res = compile_and_run(get_entry(get_g(main)), "boehm", jit=True)
     assert int(res) >= 16
 

Modified: pypy/branch/fast-forward/pypy/jit/codewriter/assembler.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/codewriter/assembler.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/codewriter/assembler.py	Tue Jul  6 00:00:33 2010
@@ -60,37 +60,38 @@
         self.code.append(chr(reg.index))
 
     def emit_const(self, const, kind, allow_short=False):
-        if const not in self.constants_dict:
-            value = const.value
-            TYPE = lltype.typeOf(value)
-            if kind == 'int':
-                if isinstance(TYPE, lltype.Ptr):
-                    assert TYPE.TO._gckind == 'raw'
-                    self.see_raw_object(value)
-                    value = llmemory.cast_ptr_to_adr(value)
-                    TYPE = llmemory.Address
-                if TYPE == llmemory.Address:
-                    value = heaptracker.adr2int(value)
-                elif not isinstance(value, ComputedIntSymbolic):
-                    value = lltype.cast_primitive(lltype.Signed, value)
-                    if allow_short and -128 <= value <= 127:
-                        # emit the constant as a small integer
-                        self.code.append(chr(value & 0xFF))
-                        return True
-                constants = self.constants_i
-            elif kind == 'ref':
-                value = lltype.cast_opaque_ptr(llmemory.GCREF, value)
-                constants = self.constants_r
-            elif kind == 'float':
-                assert TYPE == lltype.Float
-                constants = self.constants_f
-            else:
-                raise NotImplementedError(const)
+        value = const.value
+        TYPE = lltype.typeOf(value)
+        if kind == 'int':
+            if isinstance(TYPE, lltype.Ptr):
+                assert TYPE.TO._gckind == 'raw'
+                self.see_raw_object(value)
+                value = llmemory.cast_ptr_to_adr(value)
+                TYPE = llmemory.Address
+            if TYPE == llmemory.Address:
+                value = heaptracker.adr2int(value)
+            elif not isinstance(value, ComputedIntSymbolic):
+                value = lltype.cast_primitive(lltype.Signed, value)
+                if allow_short and -128 <= value <= 127:
+                    # emit the constant as a small integer
+                    self.code.append(chr(value & 0xFF))
+                    return True
+            constants = self.constants_i
+        elif kind == 'ref':
+            value = lltype.cast_opaque_ptr(llmemory.GCREF, value)
+            constants = self.constants_r
+        elif kind == 'float':
+            assert TYPE == lltype.Float
+            constants = self.constants_f
+        else:
+            raise NotImplementedError(const)
+        key = (kind, Constant(value))
+        if key not in self.constants_dict:
             constants.append(value)
-            self.constants_dict[const] = 256 - len(constants)
+            self.constants_dict[key] = 256 - len(constants)
         # emit the constant normally, as one byte that is an index in the
         # list of constants
-        self.code.append(chr(self.constants_dict[const]))
+        self.code.append(chr(self.constants_dict[key]))
         return False
 
     def write_insn(self, insn):

Modified: pypy/branch/fast-forward/pypy/jit/codewriter/call.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/codewriter/call.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/codewriter/call.py	Tue Jul  6 00:00:33 2010
@@ -16,21 +16,22 @@
 
 class CallControl(object):
     virtualref_info = None     # optionally set from outside
-    virtualizable_info = None  # optionally set from outside
-    portal_runner_ptr = None   # optionally set from outside
 
-    def __init__(self, cpu=None, portal_graph=None):
+    def __init__(self, cpu=None, jitdrivers_sd=[]):
+        assert isinstance(jitdrivers_sd, list)   # debugging
         self.cpu = cpu
-        self.portal_graph = portal_graph
+        self.jitdrivers_sd = jitdrivers_sd
         self.jitcodes = {}             # map {graph: jitcode}
         self.unfinished_graphs = []    # list of graphs with pending jitcodes
-        self.jitdriver = None
         if hasattr(cpu, 'rtyper'):     # for tests
             self.rtyper = cpu.rtyper
             translator = self.rtyper.annotator.translator
             self.raise_analyzer = RaiseAnalyzer(translator)
             self.readwrite_analyzer = ReadWriteAnalyzer(translator)
             self.virtualizable_analyzer = VirtualizableAnalyzer(translator)
+        #
+        for index, jd in enumerate(jitdrivers_sd):
+            jd.index = index
 
     def find_all_graphs(self, policy):
         try:
@@ -41,8 +42,8 @@
         def is_candidate(graph):
             return policy.look_inside_graph(graph)
 
-        assert self.portal_graph is not None
-        todo = [self.portal_graph]
+        assert len(self.jitdrivers_sd) > 0
+        todo = [jd.portal_graph for jd in self.jitdrivers_sd]
         if hasattr(self, 'rtyper'):
             for oopspec_name, ll_args, ll_res in support.inline_calls_to:
                 c_func, _ = support.builtin_func_for_spec(self.rtyper,
@@ -122,7 +123,7 @@
     def guess_call_kind(self, op, is_candidate=None):
         if op.opname == 'direct_call':
             funcptr = op.args[0].value
-            if funcptr is self.portal_runner_ptr:
+            if self.jitdriver_sd_from_portal_runner_ptr(funcptr) is not None:
                 return 'recursive'
             funcobj = get_funcobj(funcptr)
             if getattr(funcobj, 'graph', None) is None:
@@ -143,6 +144,11 @@
         # used only after find_all_graphs()
         return graph in self.candidate_graphs
 
+    def grab_initial_jitcodes(self):
+        for jd in self.jitdrivers_sd:
+            jd.mainjitcode = self.get_jitcode(jd.portal_graph)
+            jd.mainjitcode.is_portal = True
+
     def enum_pending_graphs(self):
         while self.unfinished_graphs:
             graph = self.unfinished_graphs.pop()
@@ -241,12 +247,32 @@
         return (effectinfo is None or
                 effectinfo.extraeffect >= EffectInfo.EF_CAN_RAISE)
 
-    def found_jitdriver(self, jitdriver):
-        if self.jitdriver is None:
-            self.jitdriver = jitdriver
-        else:
-            assert self.jitdriver is jitdriver
+    def jitdriver_sd_from_portal_graph(self, graph):
+        for jd in self.jitdrivers_sd:
+            if jd.portal_graph is graph:
+                return jd
+        return None
 
-    def getjitdriver(self):
-        assert self.jitdriver is not None, "order dependency issue?"
-        return self.jitdriver
+    def jitdriver_sd_from_portal_runner_ptr(self, funcptr):
+        for jd in self.jitdrivers_sd:
+            if funcptr is jd.portal_runner_ptr:
+                return jd
+        return None
+
+    def jitdriver_sd_from_jitdriver(self, jitdriver):
+        for jd in self.jitdrivers_sd:
+            if jd.jitdriver is jitdriver:
+                return jd
+        return None
+
+    def get_vinfo(self, VTYPEPTR):
+        seen = set()
+        for jd in self.jitdrivers_sd:
+            if jd.virtualizable_info is not None:
+                if jd.virtualizable_info.is_vtypeptr(VTYPEPTR):
+                    seen.add(jd.virtualizable_info)
+        if seen:
+            assert len(seen) == 1
+            return seen.pop()
+        else:
+            return None

Modified: pypy/branch/fast-forward/pypy/jit/codewriter/codewriter.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/codewriter/codewriter.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/codewriter/codewriter.py	Tue Jul  6 00:00:33 2010
@@ -14,29 +14,30 @@
 class CodeWriter(object):
     callcontrol = None    # for tests
 
-    def __init__(self, cpu=None, maingraph=None):
+    def __init__(self, cpu=None, jitdrivers_sd=[]):
         self.cpu = cpu
         self.assembler = Assembler()
-        self.portal_graph = maingraph
-        self.callcontrol = CallControl(cpu, maingraph)
+        self.callcontrol = CallControl(cpu, jitdrivers_sd)
+        self._seen_files = set()
 
     def transform_func_to_jitcode(self, func, values, type_system='lltype'):
         """For testing."""
         rtyper = support.annotate(func, values, type_system=type_system)
         graph = rtyper.annotator.translator.graphs[0]
         jitcode = JitCode("test")
-        self.transform_graph_to_jitcode(graph, jitcode, True, True)
+        self.transform_graph_to_jitcode(graph, jitcode, True)
         return jitcode
 
-    def transform_graph_to_jitcode(self, graph, jitcode, portal, verbose):
+    def transform_graph_to_jitcode(self, graph, jitcode, verbose):
         """Transform a graph into a JitCode containing the same bytecode
         in a different format.
         """
+        portal_jd = self.callcontrol.jitdriver_sd_from_portal_graph(graph)
         graph = copygraph(graph, shallowvars=True)
         #
         # step 1: mangle the graph so that it contains the final instructions
         # that we want in the JitCode, but still as a control flow graph
-        transform_graph(graph, self.cpu, self.callcontrol, portal)
+        transform_graph(graph, self.cpu, self.callcontrol, portal_jd)
         #
         # step 2: perform register allocation on it
         regallocs = {}
@@ -59,16 +60,14 @@
         self.assembler.assemble(ssarepr, jitcode)
         #
         # print the resulting assembler
-        self.print_ssa_repr(ssarepr, portal, verbose)
+        self.print_ssa_repr(ssarepr, portal_jd, verbose)
 
     def make_jitcodes(self, verbose=False):
         log.info("making JitCodes...")
-        maingraph = self.portal_graph
-        self.mainjitcode = self.callcontrol.get_jitcode(maingraph)
+        self.callcontrol.grab_initial_jitcodes()
         count = 0
         for graph, jitcode in self.callcontrol.enum_pending_graphs():
-            self.transform_graph_to_jitcode(graph, jitcode,
-                                            graph is maingraph, verbose)
+            self.transform_graph_to_jitcode(graph, jitcode, verbose)
             count += 1
             if not count % 500:
                 log.info("Produced %d jitcodes" % count)
@@ -76,33 +75,35 @@
         log.info("there are %d JitCode instances." % count)
 
     def setup_vrefinfo(self, vrefinfo):
+        # must be called at most once
+        assert self.callcontrol.virtualref_info is None
         self.callcontrol.virtualref_info = vrefinfo
 
-    def setup_virtualizable_info(self, vinfo):
-        self.callcontrol.virtualizable_info = vinfo
-
-    def setup_portal_runner_ptr(self, portal_runner_ptr):
-        self.callcontrol.portal_runner_ptr = portal_runner_ptr
+    def setup_jitdriver(self, jitdriver_sd):
+        # Must be called once per jitdriver.  Usually jitdriver_sd is an
+        # instance of pypy.jit.metainterp.jitdriver.JitDriverStaticData.
+        self.callcontrol.jitdrivers_sd.append(jitdriver_sd)
 
     def find_all_graphs(self, policy):
         return self.callcontrol.find_all_graphs(policy)
 
-    def print_ssa_repr(self, ssarepr, portal, verbose):
+    def print_ssa_repr(self, ssarepr, portal_jitdriver, verbose):
         if verbose:
             print '%s:' % (ssarepr.name,)
             print format_assembler(ssarepr)
         else:
             dir = udir.ensure("jitcodes", dir=1)
-            if portal:
-                name = "00_portal_runner"
+            if portal_jitdriver:
+                name = "%02d_portal_runner" % (portal_jitdriver.index,)
             elif ssarepr.name and ssarepr.name != '?':
                 name = ssarepr.name
             else:
                 name = 'unnamed' % id(ssarepr)
             i = 1
             extra = ''
-            while dir.join(name+extra).check(exists=1):
+            while name+extra in self._seen_files:
                 i += 1
                 extra = '.%d' % i
+            self._seen_files.add(name+extra)
             dir.join(name+extra).write(format_assembler(ssarepr))
             log.dot()

Modified: pypy/branch/fast-forward/pypy/jit/codewriter/jitcode.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/codewriter/jitcode.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/codewriter/jitcode.py	Tue Jul  6 00:00:33 2010
@@ -13,6 +13,7 @@
         self.name = name
         self.fnaddr = fnaddr
         self.calldescr = calldescr
+        self.is_portal = False
         self._called_from = called_from   # debugging
         self._ssarepr     = None          # debugging
 
@@ -24,11 +25,11 @@
         self.constants_i = constants_i or self._empty_i
         self.constants_r = constants_r or self._empty_r
         self.constants_f = constants_f or self._empty_f
-        # encode the three num_regs into a single integer
+        # encode the three num_regs into a single char each
         assert num_regs_i < 256 and num_regs_r < 256 and num_regs_f < 256
-        self.num_regs_encoded = ((num_regs_i << 16) |
-                                 (num_regs_f << 8) |
-                                 (num_regs_r << 0))
+        self.c_num_regs_i = chr(num_regs_i)
+        self.c_num_regs_r = chr(num_regs_r)
+        self.c_num_regs_f = chr(num_regs_f)
         self.liveness = make_liveness_cache(liveness)
         self._startpoints = startpoints   # debugging
         self._alllabels = alllabels       # debugging
@@ -37,13 +38,13 @@
         return heaptracker.adr2int(self.fnaddr)
 
     def num_regs_i(self):
-        return self.num_regs_encoded >> 16
-
-    def num_regs_f(self):
-        return (self.num_regs_encoded >> 8) & 0xFF
+        return ord(self.c_num_regs_i)
 
     def num_regs_r(self):
-        return self.num_regs_encoded & 0xFF
+        return ord(self.c_num_regs_r)
+
+    def num_regs_f(self):
+        return ord(self.c_num_regs_f)
 
     def has_liveness_info(self, pc):
         return pc in self.liveness

Modified: pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py	Tue Jul  6 00:00:33 2010
@@ -13,23 +13,23 @@
 from pypy.translator.simplify import get_funcobj
 
 
-def transform_graph(graph, cpu=None, callcontrol=None, portal=True):
+def transform_graph(graph, cpu=None, callcontrol=None, portal_jd=None):
     """Transform a control flow graph to make it suitable for
     being flattened in a JitCode.
     """
-    t = Transformer(cpu, callcontrol)
-    t.transform(graph, portal)
+    t = Transformer(cpu, callcontrol, portal_jd)
+    t.transform(graph)
 
 
 class Transformer(object):
 
-    def __init__(self, cpu=None, callcontrol=None):
+    def __init__(self, cpu=None, callcontrol=None, portal_jd=None):
         self.cpu = cpu
         self.callcontrol = callcontrol
+        self.portal_jd = portal_jd   # non-None only for the portal graph(s)
 
-    def transform(self, graph, portal):
+    def transform(self, graph):
         self.graph = graph
-        self.portal = portal
         for block in list(graph.iterblocks()):
             self.optimize_block(block)
 
@@ -325,10 +325,13 @@
         return op1
 
     def handle_recursive_call(self, op):
-        ops = self.promote_greens(op.args[1:])
-        targetgraph = self.callcontrol.portal_graph
-        num_green_args = len(self.callcontrol.getjitdriver().greens)
-        args = (self.make_three_lists(op.args[1:1+num_green_args]) +
+        jitdriver_sd = self.callcontrol.jitdriver_sd_from_portal_runner_ptr(
+            op.args[0].value)
+        assert jitdriver_sd is not None
+        ops = self.promote_greens(op.args[1:], jitdriver_sd.jitdriver)
+        num_green_args = len(jitdriver_sd.jitdriver.greens)
+        args = ([Constant(jitdriver_sd.index, lltype.Signed)] +
+                self.make_three_lists(op.args[1:1+num_green_args]) +
                 self.make_three_lists(op.args[1+num_green_args:]))
         kind = getkind(op.result.concretetype)[0]
         op0 = SpaceOperation('recursive_call_%s' % kind, args, op.result)
@@ -400,7 +403,12 @@
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
     def rewrite_op_malloc_varsize(self, op):
-        assert op.args[1].value == {'flavor': 'gc'}
+        if op.args[1].value['flavor'] == 'raw':
+            ARRAY = op.args[0].value
+            return self._do_builtin_call(op, 'raw_malloc',
+                                         [op.args[2]],
+                                         extra = (ARRAY,),
+                                         extrakey = ARRAY)
         if op.args[0].value == rstr.STR:
             return SpaceOperation('newstr', [op.args[2]], op.result)
         elif op.args[0].value == rstr.UNICODE:
@@ -412,9 +420,14 @@
             return SpaceOperation('new_array', [arraydescr, op.args[2]],
                                   op.result)
 
+    def rewrite_op_free(self, op):
+        assert op.args[1].value == 'raw'
+        ARRAY = op.args[0].concretetype.TO
+        return self._do_builtin_call(op, 'raw_free', [op.args[0]],
+                                     extra = (ARRAY,), extrakey = ARRAY)
+
     def rewrite_op_getarrayitem(self, op):
         ARRAY = op.args[0].concretetype.TO
-        assert ARRAY._gckind == 'gc'
         if self._array_of_voids(ARRAY):
             return []
         if op.args[0] in self.vable_array_vars:     # for virtualizables
@@ -428,13 +441,12 @@
         # normal case follows
         arraydescr = self.cpu.arraydescrof(ARRAY)
         kind = getkind(op.result.concretetype)
-        return SpaceOperation('getarrayitem_gc_%s' % kind[0],
+        return SpaceOperation('getarrayitem_%s_%s' % (ARRAY._gckind, kind[0]),
                               [op.args[0], arraydescr, op.args[1]],
                               op.result)
 
     def rewrite_op_setarrayitem(self, op):
         ARRAY = op.args[0].concretetype.TO
-        assert ARRAY._gckind == 'gc'
         if self._array_of_voids(ARRAY):
             return []
         if op.args[0] in self.vable_array_vars:     # for virtualizables
@@ -447,7 +459,7 @@
                                     op.args[1], op.args[2]], None)]
         arraydescr = self.cpu.arraydescrof(ARRAY)
         kind = getkind(op.args[2].concretetype)
-        return SpaceOperation('setarrayitem_gc_%s' % kind[0],
+        return SpaceOperation('setarrayitem_%s_%s' % (ARRAY._gckind, kind[0]),
                               [op.args[0], arraydescr, op.args[1], op.args[2]],
                               None)
 
@@ -483,14 +495,14 @@
         # check for virtualizable
         try:
             if self.is_virtualizable_getset(op):
-                descr = self.get_virtualizable_field_descr(op.args[1].value)
+                descr = self.get_virtualizable_field_descr(op)
                 kind = getkind(RESULT)[0]
                 return [SpaceOperation('-live-', [], None),
                         SpaceOperation('getfield_vable_%s' % kind,
                                        [v_inst, descr], op.result)]
-        except VirtualizableArrayField:
+        except VirtualizableArrayField, e:
             # xxx hack hack hack
-            vinfo = self.callcontrol.virtualizable_info
+            vinfo = e.args[1]
             arrayindex = vinfo.array_field_counter[op.args[1].value]
             arrayfielddescr = vinfo.array_field_descrs[arrayindex]
             arraydescr = vinfo.array_descrs[arrayindex]
@@ -527,7 +539,7 @@
             return
         # check for virtualizable
         if self.is_virtualizable_getset(op):
-            descr = self.get_virtualizable_field_descr(op.args[1].value)
+            descr = self.get_virtualizable_field_descr(op)
             kind = getkind(RESULT)[0]
             return [SpaceOperation('-live-', [], None),
                     SpaceOperation('setfield_vable_%s' % kind,
@@ -544,21 +556,23 @@
         return (op.args[1].value == 'typeptr' and
                 op.args[0].concretetype.TO._hints.get('typeptr'))
 
+    def get_vinfo(self, v_virtualizable):
+        if self.callcontrol is None:      # for tests
+            return None
+        return self.callcontrol.get_vinfo(v_virtualizable.concretetype)
+
     def is_virtualizable_getset(self, op):
         # every access of an object of exactly the type VTYPEPTR is
         # likely to be a virtualizable access, but we still have to
         # check it in pyjitpl.py.
-        try:
-            vinfo = self.callcontrol.virtualizable_info
-        except AttributeError:
-            return False
-        if vinfo is None or not vinfo.is_vtypeptr(op.args[0].concretetype):
+        vinfo = self.get_vinfo(op.args[0])
+        if vinfo is None:
             return False
         res = False
         if op.args[1].value in vinfo.static_field_to_extra_box:
             res = True
         if op.args[1].value in vinfo.array_fields:
-            res = VirtualizableArrayField(self.graph)
+            res = VirtualizableArrayField(self.graph, vinfo)
 
         if res:
             flags = self.vable_flags[op.args[0]]
@@ -568,8 +582,9 @@
             raise res
         return res
 
-    def get_virtualizable_field_descr(self, fieldname):
-        vinfo = self.callcontrol.virtualizable_info
+    def get_virtualizable_field_descr(self, op):
+        fieldname = op.args[1].value
+        vinfo = self.get_vinfo(op.args[0])
         index = vinfo.static_field_to_extra_box[fieldname]
         return vinfo.static_field_descrs[index]
 
@@ -750,9 +765,10 @@
             return Constant(value, lltype.Bool)
         return op
 
-    def promote_greens(self, args):
+    def promote_greens(self, args, jitdriver):
         ops = []
-        num_green_args = len(self.callcontrol.getjitdriver().greens)
+        num_green_args = len(jitdriver.greens)
+        assert len(args) == num_green_args + len(jitdriver.reds)
         for v in args[:num_green_args]:
             if isinstance(v, Variable) and v.concretetype is not lltype.Void:
                 kind = getkind(v.concretetype)
@@ -762,21 +778,28 @@
         return ops
 
     def rewrite_op_jit_marker(self, op):
-        self.callcontrol.found_jitdriver(op.args[1].value)
         key = op.args[0].value
-        return getattr(self, 'handle_jit_marker__%s' % key)(op)
+        jitdriver = op.args[1].value
+        return getattr(self, 'handle_jit_marker__%s' % key)(op, jitdriver)
 
-    def handle_jit_marker__jit_merge_point(self, op):
-        assert self.portal, "jit_merge_point in non-main graph!"
-        ops = self.promote_greens(op.args[2:])
-        num_green_args = len(self.callcontrol.getjitdriver().greens)
-        args = (self.make_three_lists(op.args[2:2+num_green_args]) +
+    def handle_jit_marker__jit_merge_point(self, op, jitdriver):
+        assert self.portal_jd is not None, (
+            "'jit_merge_point' in non-portal graph!")
+        assert jitdriver is self.portal_jd.jitdriver, (
+            "general mix-up of jitdrivers?")
+        ops = self.promote_greens(op.args[2:], jitdriver)
+        num_green_args = len(jitdriver.greens)
+        args = ([Constant(self.portal_jd.index, lltype.Signed)] +
+                self.make_three_lists(op.args[2:2+num_green_args]) +
                 self.make_three_lists(op.args[2+num_green_args:]))
         op1 = SpaceOperation('jit_merge_point', args, None)
         return ops + [op1]
 
-    def handle_jit_marker__can_enter_jit(self, op):
-        return SpaceOperation('can_enter_jit', [], None)
+    def handle_jit_marker__can_enter_jit(self, op, jitdriver):
+        jd = self.callcontrol.jitdriver_sd_from_jitdriver(jitdriver)
+        assert jd is not None
+        c_index = Constant(jd.index, lltype.Signed)
+        return SpaceOperation('can_enter_jit', [c_index], None)
 
     def rewrite_op_debug_assert(self, op):
         log.WARNING("found debug_assert in %r; should have be removed" %
@@ -974,9 +997,8 @@
 
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
-        vinfo = self.callcontrol.virtualizable_info
+        vinfo = self.get_vinfo(op.args[0])
         assert vinfo is not None
-        assert vinfo.is_vtypeptr(op.args[0].concretetype)
         self.vable_flags[op.args[0]] = op.args[2].value
         return []
 

Modified: pypy/branch/fast-forward/pypy/jit/codewriter/support.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/codewriter/support.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/codewriter/support.py	Tue Jul  6 00:00:33 2010
@@ -282,6 +282,9 @@
 
     # ---------- malloc with del ----------
 
+    def _ll_2_raw_malloc(TP, size):
+        return lltype.malloc(TP, size, flavor='raw')
+
     def build_ll_0_alloc_with_del(RESULT, vtable):
         def _ll_0_alloc_with_del():
             p = lltype.malloc(RESULT.TO)
@@ -289,6 +292,15 @@
             return p
         return _ll_0_alloc_with_del
 
+    def build_ll_1_raw_malloc(ARRAY):
+        def _ll_1_raw_malloc(n):
+            return lltype.malloc(ARRAY, n, flavor='raw')
+        return _ll_1_raw_malloc
+
+    def build_ll_1_raw_free(ARRAY):
+        def _ll_1_raw_free(p):
+            lltype.free(p, flavor='raw')
+        return _ll_1_raw_free
 
 class OOtypeHelpers:
 

Modified: pypy/branch/fast-forward/pypy/jit/codewriter/test/test_assembler.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/codewriter/test/test_assembler.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/codewriter/test/test_assembler.py	Tue Jul  6 00:00:33 2010
@@ -127,6 +127,25 @@
     assert assembler.insns == {'foobar/IR': 0}
     assert jitcode.constants_i == [42]
 
+def test_assemble_list_semibug():
+    # the semibug is that after forcing 42 into the dict of constants,
+    # it would be reused for all future 42's, even ones that can be
+    # encoded directly.
+    ssarepr = SSARepr("test")
+    ssarepr.insns = [
+        ('foobar', ListOfKind('int', [Constant(42, lltype.Signed)])),
+        ('foobar', ListOfKind('int', [Constant(42, lltype.Signed)])),
+        ('baz', Constant(42, lltype.Signed)),
+        ]
+    assembler = Assembler()
+    jitcode = assembler.assemble(ssarepr)
+    assert jitcode.code == ("\x00\x01\xFF"
+                            "\x00\x01\xFF"
+                            "\x01\x2A")
+    assert assembler.insns == {'foobar/I': 0,
+                               'baz/c': 1}
+    assert jitcode.constants_i == [42]
+
 def test_assemble_descr():
     class FooDescr(AbstractDescr):
         pass

Modified: pypy/branch/fast-forward/pypy/jit/codewriter/test/test_call.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/codewriter/test/test_call.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/codewriter/test/test_call.py	Tue Jul  6 00:00:33 2010
@@ -52,13 +52,19 @@
 
 # ____________________________________________________________
 
+class FakeJitDriverSD:
+    def __init__(self, portal_graph):
+        self.portal_graph = portal_graph
+        self.portal_runner_ptr = "???"
+
 def test_find_all_graphs():
     def g(x):
         return x + 2
     def f(x):
         return g(x) + 1
     rtyper = support.annotate(f, [7])
-    cc = CallControl(portal_graph=rtyper.annotator.translator.graphs[0])
+    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
+    cc = CallControl(jitdrivers_sd=[jitdriver_sd])
     res = cc.find_all_graphs(FakePolicy())
     funcs = set([graph.func for graph in res])
     assert funcs == set([f, g])
@@ -69,7 +75,8 @@
     def f(x):
         return g(x) + 1
     rtyper = support.annotate(f, [7])
-    cc = CallControl(portal_graph=rtyper.annotator.translator.graphs[0])
+    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
+    cc = CallControl(jitdrivers_sd=[jitdriver_sd])
     class CustomFakePolicy:
         def look_inside_graph(self, graph):
             assert graph.name == 'g'
@@ -83,10 +90,11 @@
 def test_guess_call_kind_and_calls_from_graphs():
     class portal_runner_obj:
         graph = object()
+    class FakeJitDriverSD:
+        portal_runner_ptr = portal_runner_obj
     g = object()
     g1 = object()
-    cc = CallControl()
-    cc.portal_runner_ptr = portal_runner_obj
+    cc = CallControl(jitdrivers_sd=[FakeJitDriverSD()])
     cc.candidate_graphs = [g, g1]
 
     op = SpaceOperation('direct_call', [Constant(portal_runner_obj)],

Modified: pypy/branch/fast-forward/pypy/jit/codewriter/test/test_codewriter.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/codewriter/test/test_codewriter.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/codewriter/test/test_codewriter.py	Tue Jul  6 00:00:33 2010
@@ -2,7 +2,7 @@
 from pypy.jit.codewriter.codewriter import CodeWriter
 from pypy.jit.codewriter import support
 from pypy.jit.metainterp.history import AbstractDescr
-from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 
 class FakeCallDescr(AbstractDescr):
     def __init__(self, FUNC, ARGS, RESULT, effectinfo=None):
@@ -24,17 +24,28 @@
     def as_vtable_size_descr(self):
         return self
 
+class FakeArrayDescr(AbstractDescr):
+    def __init__(self, ARRAY):
+        self.ARRAY = ARRAY
+
 class FakeCPU:
     def __init__(self, rtyper):
         self.rtyper = rtyper
     calldescrof = FakeCallDescr
     fielddescrof = FakeFieldDescr
     sizeof = FakeSizeDescr
+    arraydescrof = FakeArrayDescr
 
 class FakePolicy:
     def look_inside_graph(self, graph):
         return graph.name != 'dont_look'
 
+class FakeJitDriverSD:
+    def __init__(self, portal_graph):
+        self.portal_graph = portal_graph
+        self.portal_runner_ptr = "???"
+        self.virtualizable_info = None
+
 
 def test_loop():
     def f(a, b):
@@ -70,11 +81,11 @@
     def fff(a, b):
         return ggg(b) - ggg(a)
     rtyper = support.annotate(fff, [35, 42])
-    maingraph = rtyper.annotator.translator.graphs[0]
-    cw = CodeWriter(FakeCPU(rtyper), maingraph)
+    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
+    cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd])
     cw.find_all_graphs(FakePolicy())
     cw.make_jitcodes(verbose=True)
-    jitcode = cw.mainjitcode
+    jitcode = jitdriver_sd.mainjitcode
     print jitcode.dump()
     [jitcode2] = cw.assembler.descrs
     print jitcode2.dump()
@@ -100,7 +111,7 @@
     blackholeinterp.setarg_i(0, 6)
     blackholeinterp.setarg_i(1, 100)
     blackholeinterp.run()
-    assert blackholeinterp.final_result_i() == 100+6+5+4+3
+    assert blackholeinterp.get_tmpreg_i() == 100+6+5+4+3
 
 def test_instantiate():
     class A1:     id = 651
@@ -117,7 +128,7 @@
         return x().id + y().id + dont_look(n)
     rtyper = support.annotate(f, [35])
     maingraph = rtyper.annotator.translator.graphs[0]
-    cw = CodeWriter(FakeCPU(rtyper), maingraph)
+    cw = CodeWriter(FakeCPU(rtyper), [FakeJitDriverSD(maingraph)])
     cw.find_all_graphs(FakePolicy())
     cw.make_jitcodes(verbose=True)
     #
@@ -144,10 +155,32 @@
     def f(n):
         return abs(n)
     rtyper = support.annotate(f, [35])
-    maingraph = rtyper.annotator.translator.graphs[0]
-    cw = CodeWriter(FakeCPU(rtyper), maingraph)
+    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
+    cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd])
     cw.find_all_graphs(FakePolicy())
     cw.make_jitcodes(verbose=True)
     #
-    s = cw.mainjitcode.dump()
+    s = jitdriver_sd.mainjitcode.dump()
     assert "inline_call_ir_i <JitCode '_ll_1_int_abs__Signed'>" in s
+
+def test_raw_malloc_and_access():
+    TP = rffi.CArray(lltype.Signed)
+    
+    def f(n):
+        a = lltype.malloc(TP, n, flavor='raw')
+        a[0] = n
+        res = a[0]
+        lltype.free(a, flavor='raw')
+        return res
+
+    rtyper = support.annotate(f, [35])
+    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
+    cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd])
+    cw.find_all_graphs(FakePolicy())
+    cw.make_jitcodes(verbose=True)
+    #
+    s = jitdriver_sd.mainjitcode.dump()
+    assert 'residual_call_ir_i $<* fn _ll_1_raw_malloc__Signed>' in s
+    assert 'setarrayitem_raw_i' in s
+    assert 'getarrayitem_raw_i' in s
+    assert 'residual_call_ir_v $<* fn _ll_1_raw_free__arrayPtr>' in s

Modified: pypy/branch/fast-forward/pypy/jit/codewriter/test/test_flatten.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/codewriter/test/test_flatten.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/codewriter/test/test_flatten.py	Tue Jul  6 00:00:33 2010
@@ -68,11 +68,8 @@
         return FakeDescr()
     def calldescr_canraise(self, calldescr):
         return calldescr is not self._descr_cannot_raise
-    def found_jitdriver(self, jitdriver):
-        assert isinstance(jitdriver, JitDriver)
-        self.jitdriver = jitdriver
-    def getjitdriver(self):
-        return self.jitdriver
+    def get_vinfo(self, VTYPEPTR):
+        return None
 
 class FakeCallControlWithVRefInfo:
     class virtualref_info:
@@ -118,13 +115,13 @@
         return self.rtyper.annotator.translator.graphs
 
     def encoding_test(self, func, args, expected,
-                      transform=False, liveness=False, cc=None):
+                      transform=False, liveness=False, cc=None, jd=None):
         graphs = self.make_graphs(func, args)
         #graphs[0].show()
         if transform:
             from pypy.jit.codewriter.jtransform import transform_graph
             cc = cc or FakeCallControl()
-            transform_graph(graphs[0], FakeCPU(self.rtyper), cc)
+            transform_graph(graphs[0], FakeCPU(self.rtyper), cc, jd)
         ssarepr = flatten_graph(graphs[0], fake_regallocs(),
                                 _include_all_exc_links=not transform)
         if liveness:
@@ -584,13 +581,21 @@
         def f(x, y):
             myjitdriver.jit_merge_point(x=x, y=y)
             myjitdriver.can_enter_jit(x=y, y=x)
+        class FakeJitDriverSD:
+            jitdriver = myjitdriver
+            index = 27
+        jd = FakeJitDriverSD()
+        class MyFakeCallControl(FakeCallControl):
+            def jitdriver_sd_from_jitdriver(self, jitdriver):
+                assert jitdriver == myjitdriver
+                return jd
         self.encoding_test(f, [4, 5], """
             -live- %i0, %i1
             int_guard_value %i0
-            jit_merge_point I[%i0], R[], F[], I[%i1], R[], F[]
-            can_enter_jit
+            jit_merge_point $27, I[%i0], R[], F[], I[%i1], R[], F[]
+            can_enter_jit $27
             void_return
-        """, transform=True, liveness=True)
+        """, transform=True, liveness=True, cc=MyFakeCallControl(), jd=jd)
 
     def test_keepalive(self):
         S = lltype.GcStruct('S')

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/blackhole.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/blackhole.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/blackhole.py	Tue Jul  6 00:00:33 2010
@@ -315,27 +315,12 @@
     def get_tmpreg_f(self):
         return self.tmpreg_f
 
-    def final_result_i(self):
-        assert self._return_type == 'i'
-        return self.get_tmpreg_i()
-
-    def final_result_r(self):
-        assert self._return_type == 'r'
-        return self.get_tmpreg_r()
-
-    def final_result_f(self):
-        assert self._return_type == 'f'
-        return self.get_tmpreg_f()
-
-    def final_result_v(self):
-        assert self._return_type == 'v'
-
     def _final_result_anytype(self):
         "NOT_RPYTHON"
-        if self._return_type == 'i': return self.final_result_i()
-        if self._return_type == 'r': return self.final_result_r()
-        if self._return_type == 'f': return self.final_result_f()
-        if self._return_type == 'v': return self.final_result_v()
+        if self._return_type == 'i': return self.get_tmpreg_i()
+        if self._return_type == 'r': return self.get_tmpreg_r()
+        if self._return_type == 'f': return self.get_tmpreg_f()
+        if self._return_type == 'v': return None
         raise ValueError(self._return_type)
 
     def cleanup_registers(self):
@@ -774,12 +759,12 @@
     # ----------
     # the main hints and recursive calls
 
-    @arguments()
-    def bhimpl_can_enter_jit():
+    @arguments("i")
+    def bhimpl_can_enter_jit(jdindex):
         pass
 
-    @arguments("self", "I", "R", "F", "I", "R", "F")
-    def bhimpl_jit_merge_point(self, *args):
+    @arguments("self", "i", "I", "R", "F", "I", "R", "F")
+    def bhimpl_jit_merge_point(self, jdindex, *args):
         if self.nextblackholeinterp is None:    # we are the last level
             CRN = self.builder.metainterp_sd.ContinueRunningNormally
             raise CRN(*args)
@@ -793,55 +778,55 @@
             # call the interpreter main loop from here, and just return its
             # result.
             sd = self.builder.metainterp_sd
-            if sd.result_type == 'void':
-                self.bhimpl_recursive_call_v(*args)
+            result_type = sd.jitdrivers_sd[jdindex].result_type
+            if result_type == 'v':
+                self.bhimpl_recursive_call_v(jdindex, *args)
                 self.bhimpl_void_return()
-            elif sd.result_type == 'int':
-                x = self.bhimpl_recursive_call_i(*args)
+            elif result_type == 'i':
+                x = self.bhimpl_recursive_call_i(jdindex, *args)
                 self.bhimpl_int_return(x)
-            elif sd.result_type == 'ref':
-                x = self.bhimpl_recursive_call_r(*args)
+            elif result_type == 'r':
+                x = self.bhimpl_recursive_call_r(jdindex, *args)
                 self.bhimpl_ref_return(x)
-            elif sd.result_type == 'float':
-                x = self.bhimpl_recursive_call_f(*args)
+            elif result_type == 'f':
+                x = self.bhimpl_recursive_call_f(jdindex, *args)
                 self.bhimpl_float_return(x)
             assert False
 
-    def get_portal_runner(self):
-        metainterp_sd = self.builder.metainterp_sd
-        fnptr = llmemory.cast_ptr_to_adr(metainterp_sd._portal_runner_ptr)
-        fnptr = heaptracker.adr2int(fnptr)
-        calldescr = metainterp_sd.portal_code.calldescr
+    def get_portal_runner(self, jdindex):
+        jitdriver_sd = self.builder.metainterp_sd.jitdrivers_sd[jdindex]
+        fnptr = heaptracker.adr2int(jitdriver_sd.portal_runner_adr)
+        calldescr = jitdriver_sd.mainjitcode.calldescr
         return fnptr, calldescr
 
-    @arguments("self", "I", "R", "F", "I", "R", "F", returns="i")
-    def bhimpl_recursive_call_i(self, greens_i, greens_r, greens_f,
-                                      reds_i,   reds_r,   reds_f):
-        fnptr, calldescr = self.get_portal_runner()
+    @arguments("self", "i", "I", "R", "F", "I", "R", "F", returns="i")
+    def bhimpl_recursive_call_i(self, jdindex, greens_i, greens_r, greens_f,
+                                               reds_i,   reds_r,   reds_f):
+        fnptr, calldescr = self.get_portal_runner(jdindex)
         return self.cpu.bh_call_i(fnptr, calldescr,
                                   greens_i + reds_i,
                                   greens_r + reds_r,
                                   greens_f + reds_f)
-    @arguments("self", "I", "R", "F", "I", "R", "F", returns="r")
-    def bhimpl_recursive_call_r(self, greens_i, greens_r, greens_f,
-                                      reds_i,   reds_r,   reds_f):
-        fnptr, calldescr = self.get_portal_runner()
+    @arguments("self", "i", "I", "R", "F", "I", "R", "F", returns="r")
+    def bhimpl_recursive_call_r(self, jdindex, greens_i, greens_r, greens_f,
+                                               reds_i,   reds_r,   reds_f):
+        fnptr, calldescr = self.get_portal_runner(jdindex)
         return self.cpu.bh_call_r(fnptr, calldescr,
                                   greens_i + reds_i,
                                   greens_r + reds_r,
                                   greens_f + reds_f)
-    @arguments("self", "I", "R", "F", "I", "R", "F", returns="f")
-    def bhimpl_recursive_call_f(self, greens_i, greens_r, greens_f,
-                                      reds_i,   reds_r,   reds_f):
-        fnptr, calldescr = self.get_portal_runner()
+    @arguments("self", "i", "I", "R", "F", "I", "R", "F", returns="f")
+    def bhimpl_recursive_call_f(self, jdindex, greens_i, greens_r, greens_f,
+                                               reds_i,   reds_r,   reds_f):
+        fnptr, calldescr = self.get_portal_runner(jdindex)
         return self.cpu.bh_call_f(fnptr, calldescr,
                                   greens_i + reds_i,
                                   greens_r + reds_r,
                                   greens_f + reds_f)
-    @arguments("self", "I", "R", "F", "I", "R", "F")
-    def bhimpl_recursive_call_v(self, greens_i, greens_r, greens_f,
-                                      reds_i,   reds_r,   reds_f):
-        fnptr, calldescr = self.get_portal_runner()
+    @arguments("self", "i", "I", "R", "F", "I", "R", "F")
+    def bhimpl_recursive_call_v(self, jdindex, greens_i, greens_r, greens_f,
+                                               reds_i,   reds_r,   reds_f):
+        fnptr, calldescr = self.get_portal_runner(jdindex)
         return self.cpu.bh_call_v(fnptr, calldescr,
                                   greens_i + reds_i,
                                   greens_r + reds_r,
@@ -1005,6 +990,13 @@
     bhimpl_getarrayitem_gc_pure_r = bhimpl_getarrayitem_gc_r
     bhimpl_getarrayitem_gc_pure_f = bhimpl_getarrayitem_gc_f
 
+    @arguments("cpu", "i", "d", "i", returns="i")
+    def bhimpl_getarrayitem_raw_i(cpu, array, arraydescr, index):
+        return cpu.bh_getarrayitem_raw_i(arraydescr, array, index)
+    @arguments("cpu", "i", "d", "i", returns="f")
+    def bhimpl_getarrayitem_raw_f(cpu, array, arraydescr, index):
+        return cpu.bh_getarrayitem_raw_f(arraydescr, array, index)
+
     @arguments("cpu", "r", "d", "i", "i")
     def bhimpl_setarrayitem_gc_i(cpu, array, arraydescr, index, newvalue):
         cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue)
@@ -1015,6 +1007,15 @@
     def bhimpl_setarrayitem_gc_f(cpu, array, arraydescr, index, newvalue):
         cpu.bh_setarrayitem_gc_f(arraydescr, array, index, newvalue)
 
+    @arguments("cpu", "i", "d", "i", "i")
+    def bhimpl_setarrayitem_raw_i(cpu, array, arraydescr, index, newvalue):
+        cpu.bh_setarrayitem_raw_i(arraydescr, array, index, newvalue)
+    @arguments("cpu", "i", "d", "i", "f")
+    def bhimpl_setarrayitem_raw_f(cpu, array, arraydescr, index, newvalue):
+        cpu.bh_setarrayitem_raw_f(arraydescr, array, index, newvalue)
+
+    # note, there is no 'r' here, since it can't happen
+
     @arguments("cpu", "r", "d", returns="i")
     def bhimpl_arraylen_gc(cpu, array, arraydescr):
         return cpu.bh_arraylen_gc(arraydescr, array)
@@ -1178,11 +1179,11 @@
             self._done_with_this_frame()
         kind = self._return_type
         if kind == 'i':
-            caller._setup_return_value_i(self.final_result_i())
+            caller._setup_return_value_i(self.get_tmpreg_i())
         elif kind == 'r':
-            caller._setup_return_value_r(self.final_result_r())
+            caller._setup_return_value_r(self.get_tmpreg_r())
         elif kind == 'f':
-            caller._setup_return_value_f(self.final_result_f())
+            caller._setup_return_value_f(self.get_tmpreg_f())
         else:
             assert kind == 'v'
         return lltype.nullptr(rclass.OBJECTPTR.TO)
@@ -1248,15 +1249,15 @@
         # rare case: we only get there if the blackhole interps all returned
         # normally (in general we get a ContinueRunningNormally exception).
         sd = self.builder.metainterp_sd
-        if sd.result_type == 'void':
-            self.final_result_v()
+        kind = self._return_type
+        if kind == 'v':
             raise sd.DoneWithThisFrameVoid()
-        elif sd.result_type == 'int':
-            raise sd.DoneWithThisFrameInt(self.final_result_i())
-        elif sd.result_type == 'ref':
-            raise sd.DoneWithThisFrameRef(self.cpu, self.final_result_r())
-        elif sd.result_type == 'float':
-            raise sd.DoneWithThisFrameFloat(self.final_result_f())
+        elif kind == 'i':
+            raise sd.DoneWithThisFrameInt(self.get_tmpreg_i())
+        elif kind == 'r':
+            raise sd.DoneWithThisFrameRef(self.cpu, self.get_tmpreg_r())
+        elif kind == 'f':
+            raise sd.DoneWithThisFrameFloat(self.get_tmpreg_f())
         else:
             assert False
 
@@ -1290,12 +1291,14 @@
             blackholeinterp.builder.release_interp(blackholeinterp)
         blackholeinterp = blackholeinterp.nextblackholeinterp
 
-def resume_in_blackhole(metainterp_sd, resumedescr, all_virtuals=None):
+def resume_in_blackhole(metainterp_sd, jitdriver_sd, resumedescr,
+                        all_virtuals=None):
     from pypy.jit.metainterp.resume import blackhole_from_resumedata
     debug_start('jit-blackhole')
     metainterp_sd.profiler.start_blackhole()
     blackholeinterp = blackhole_from_resumedata(
         metainterp_sd.blackholeinterpbuilder,
+        jitdriver_sd,
         resumedescr,
         all_virtuals)
     current_exc = blackholeinterp._prepare_resume_from_failure(

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/compile.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/compile.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/compile.py	Tue Jul  6 00:00:33 2010
@@ -39,9 +39,10 @@
     name = metainterp.staticdata.stats.name_for_new_loop()
     return TreeLoop(name)
 
-def make_loop_token(nb_args):
+def make_loop_token(nb_args, jitdriver_sd):
     loop_token = LoopToken()
     loop_token.specnodes = [prebuiltNotSpecNode] * nb_args
+    loop_token.outermost_jitdriver_sd = jitdriver_sd
     return loop_token
 
 # ____________________________________________________________
@@ -63,11 +64,12 @@
     # make a copy, because optimize_loop can mutate the ops and descrs
     loop.operations = [op.clone() for op in ops]
     metainterp_sd = metainterp.staticdata
-    loop_token = make_loop_token(len(loop.inputargs))
+    jitdriver_sd = metainterp.jitdriver_sd
+    loop_token = make_loop_token(len(loop.inputargs), jitdriver_sd)
     loop.token = loop_token
     loop.operations[-1].descr = loop_token     # patch the target of the JUMP
     try:
-        old_loop_token = metainterp_sd.state.optimize_loop(
+        old_loop_token = jitdriver_sd.warmstate.optimize_loop(
             metainterp_sd, old_loop_tokens, loop)
     except InvalidLoop:
         return None
@@ -141,32 +143,32 @@
     pass
 
 class DoneWithThisFrameDescrVoid(_DoneWithThisFrameDescr):
-    def handle_fail(self, metainterp_sd):
-        assert metainterp_sd.result_type == 'void'
+    def handle_fail(self, metainterp_sd, jitdriver_sd):
+        assert jitdriver_sd.result_type == history.VOID
         raise metainterp_sd.DoneWithThisFrameVoid()
 
 class DoneWithThisFrameDescrInt(_DoneWithThisFrameDescr):
-    def handle_fail(self, metainterp_sd):
-        assert metainterp_sd.result_type == 'int'
+    def handle_fail(self, metainterp_sd, jitdriver_sd):
+        assert jitdriver_sd.result_type == history.INT
         result = metainterp_sd.cpu.get_latest_value_int(0)
         raise metainterp_sd.DoneWithThisFrameInt(result)
 
 class DoneWithThisFrameDescrRef(_DoneWithThisFrameDescr):
-    def handle_fail(self, metainterp_sd):
-        assert metainterp_sd.result_type == 'ref'
+    def handle_fail(self, metainterp_sd, jitdriver_sd):
+        assert jitdriver_sd.result_type == history.REF
         cpu = metainterp_sd.cpu
         result = cpu.get_latest_value_ref(0)
         cpu.clear_latest_values(1)
         raise metainterp_sd.DoneWithThisFrameRef(cpu, result)
 
 class DoneWithThisFrameDescrFloat(_DoneWithThisFrameDescr):
-    def handle_fail(self, metainterp_sd):
-        assert metainterp_sd.result_type == 'float'
+    def handle_fail(self, metainterp_sd, jitdriver_sd):
+        assert jitdriver_sd.result_type == history.FLOAT
         result = metainterp_sd.cpu.get_latest_value_float(0)
         raise metainterp_sd.DoneWithThisFrameFloat(result)
 
 class ExitFrameWithExceptionDescrRef(_DoneWithThisFrameDescr):
-    def handle_fail(self, metainterp_sd):
+    def handle_fail(self, metainterp_sd, jitdriver_sd):
         cpu = metainterp_sd.cpu
         value = cpu.get_latest_value_ref(0)
         cpu.clear_latest_values(1)
@@ -210,8 +212,6 @@
 class ResumeDescr(AbstractFailDescr):
     def __init__(self, original_greenkey):
         self.original_greenkey = original_greenkey
-    def _clone_if_mutable(self):
-        raise NotImplementedError
 
 class ResumeGuardDescr(ResumeDescr):
     _counter = 0        # if < 0, there is one counter per value;
@@ -258,22 +258,27 @@
             # a negative value
             self._counter = cnt | i
 
-    def handle_fail(self, metainterp_sd):
-        if self.must_compile(metainterp_sd):
-            return self._trace_and_compile_from_bridge(metainterp_sd)
+    def handle_fail(self, metainterp_sd, jitdriver_sd):
+        if self.must_compile(metainterp_sd, jitdriver_sd):
+            return self._trace_and_compile_from_bridge(metainterp_sd,
+                                                       jitdriver_sd)
         else:
             from pypy.jit.metainterp.blackhole import resume_in_blackhole
-            resume_in_blackhole(metainterp_sd, self)
+            resume_in_blackhole(metainterp_sd, jitdriver_sd, self)
             assert 0, "unreachable"
 
-    def _trace_and_compile_from_bridge(self, metainterp_sd):
+    def _trace_and_compile_from_bridge(self, metainterp_sd, jitdriver_sd):
+        # 'jitdriver_sd' corresponds to the outermost one, i.e. the one
+        # of the jit_merge_point where we started the loop, even if the
+        # loop itself may contain temporarily recursion into other
+        # jitdrivers.
         from pypy.jit.metainterp.pyjitpl import MetaInterp
-        metainterp = MetaInterp(metainterp_sd)
+        metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
         return metainterp.handle_guard_failure(self)
     _trace_and_compile_from_bridge._dont_inline_ = True
 
-    def must_compile(self, metainterp_sd):
-        trace_eagerness = metainterp_sd.state.trace_eagerness
+    def must_compile(self, metainterp_sd, jitdriver_sd):
+        trace_eagerness = jitdriver_sd.warmstate.trace_eagerness
         if self._counter >= 0:
             self._counter += 1
             return self._counter >= trace_eagerness
@@ -320,8 +325,7 @@
         send_bridge_to_backend(metainterp.staticdata, self, inputargs,
                                new_loop.operations)
 
-    def _clone_if_mutable(self):
-        res = self.__class__(self.metainterp_sd, self.original_greenkey)
+    def copy_all_attrbutes_into(self, res):
         # XXX a bit ugly to have to list them all here
         res.rd_snapshot = self.rd_snapshot
         res.rd_frame_info_list = self.rd_frame_info_list
@@ -329,11 +333,19 @@
         res.rd_consts = self.rd_consts
         res.rd_virtuals = self.rd_virtuals
         res.rd_pendingfields = self.rd_pendingfields
+
+    def _clone_if_mutable(self):
+        res = ResumeGuardDescr(self.metainterp_sd, self.original_greenkey)
+        self.copy_all_attrbutes_into(res)
         return res
 
 class ResumeGuardForcedDescr(ResumeGuardDescr):
 
-    def handle_fail(self, metainterp_sd):
+    def __init__(self, metainterp_sd, original_greenkey, jitdriver_sd):
+        ResumeGuardDescr.__init__(self, metainterp_sd, original_greenkey)
+        self.jitdriver_sd = jitdriver_sd
+
+    def handle_fail(self, metainterp_sd, jitdriver_sd):
         # Failures of a GUARD_NOT_FORCED are never compiled, but
         # always just blackholed.  First fish for the data saved when
         # the virtualrefs and virtualizable have been forced by
@@ -343,7 +355,8 @@
         all_virtuals = self.fetch_data(token)
         if all_virtuals is None:
             all_virtuals = []
-        resume_in_blackhole(metainterp_sd, self, all_virtuals)
+        assert jitdriver_sd is self.jitdriver_sd
+        resume_in_blackhole(metainterp_sd, jitdriver_sd, self, all_virtuals)
         assert 0, "unreachable"
 
     @staticmethod
@@ -358,7 +371,8 @@
     def handle_async_forcing(self, force_token):
         from pypy.jit.metainterp.resume import force_from_resumedata
         metainterp_sd = self.metainterp_sd
-        all_virtuals = force_from_resumedata(metainterp_sd, self)
+        vinfo = self.jitdriver_sd.virtualizable_info
+        all_virtuals = force_from_resumedata(metainterp_sd, self, vinfo)
         # The virtualizable data was stored on the real virtualizable above.
         # Handle all_virtuals: keep them for later blackholing from the
         # future failure of the GUARD_NOT_FORCED
@@ -392,6 +406,13 @@
                 assert 0, "not found: %r" % (key,)
         return data
 
+    def _clone_if_mutable(self):
+        res = ResumeGuardForcedDescr(self.metainterp_sd,
+                                     self.original_greenkey,
+                                     self.jitdriver_sd)
+        self.copy_all_attrbutes_into(res)
+        return res
+
 
 class AbstractResumeGuardCounters(object):
     # Completely custom algorithm for now: keep 5 pairs (value, counter),
@@ -464,19 +485,19 @@
         # a loop at all but ends in a jump to the target loop.  It starts
         # with completely unoptimized arguments, as in the interpreter.
         metainterp_sd = metainterp.staticdata
+        jitdriver_sd = metainterp.jitdriver_sd
         metainterp.history.inputargs = self.redkey
-        new_loop_token = make_loop_token(len(self.redkey))
+        new_loop_token = make_loop_token(len(self.redkey), jitdriver_sd)
         new_loop.greenkey = self.original_greenkey
         new_loop.inputargs = self.redkey
         new_loop.token = new_loop_token
         send_loop_to_backend(metainterp_sd, new_loop, "entry bridge")
         # send the new_loop to warmspot.py, to be called directly the next time
-        metainterp_sd.state.attach_unoptimized_bridge_from_interp(
+        jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp(
             self.original_greenkey,
             new_loop_token)
         # store the new loop in compiled_merge_points too
-        glob = metainterp_sd.globaldata
-        old_loop_tokens = glob.get_compiled_merge_points(
+        old_loop_tokens = metainterp.get_compiled_merge_points(
             self.original_greenkey)
         # it always goes at the end of the list, as it is the most
         # general loop token
@@ -500,10 +521,11 @@
     # clone ops, as optimize_bridge can mutate the ops
     new_loop.operations = [op.clone() for op in metainterp.history.operations]
     metainterp_sd = metainterp.staticdata
+    state = metainterp.jitdriver_sd.warmstate
     try:
-        target_loop_token = metainterp_sd.state.optimize_bridge(metainterp_sd,
-                                                                old_loop_tokens,
-                                                                new_loop)
+        target_loop_token = state.optimize_bridge(metainterp_sd,
+                                                  old_loop_tokens,
+                                                  new_loop)
     except InvalidLoop:
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
         # InvalidLoop

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/executor.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/executor.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/executor.py	Tue Jul  6 00:00:33 2010
@@ -90,6 +90,15 @@
     else:
         return BoxInt(cpu.bh_getarrayitem_gc_i(arraydescr, array, index))
 
+def do_getarrayitem_raw(cpu, _, arraybox, indexbox, arraydescr):
+    array = arraybox.getint()
+    index = indexbox.getint()
+    assert not arraydescr.is_array_of_pointers()
+    if arraydescr.is_array_of_floats():
+        return BoxFloat(cpu.bh_getarrayitem_raw_f(arraydescr, array, index))
+    else:
+        return BoxInt(cpu.bh_getarrayitem_raw_i(arraydescr, array, index))
+
 def do_setarrayitem_gc(cpu, _, arraybox, indexbox, itembox, arraydescr):
     array = arraybox.getref_base()
     index = indexbox.getint()
@@ -101,6 +110,15 @@
     else:
         cpu.bh_setarrayitem_gc_i(arraydescr, array, index, itembox.getint())
 
+def do_setarrayitem_raw(cpu, _, arraybox, indexbox, itembox, arraydescr):
+    array = arraybox.getint()
+    index = indexbox.getint()
+    assert not arraydescr.is_array_of_pointers()
+    if arraydescr.is_array_of_floats():
+        cpu.bh_setarrayitem_raw_f(arraydescr, array, index, itembox.getfloat())
+    else:
+        cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint())
+
 def do_getfield_gc(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
     if fielddescr.is_pointer_field():

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/history.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/history.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/history.py	Tue Jul  6 00:00:33 2010
@@ -125,9 +125,6 @@
     def repr_of_descr(self):
         return '%r' % (self,)
 
-    def _clone_if_mutable(self):
-        return self
-
     def get_arg_types(self):
         """ Implement in call descr.
         Must return a string of INT, REF and FLOAT ('i', 'r', 'f').
@@ -171,10 +168,18 @@
         """
         raise NotImplementedError
 
+    def _clone_if_mutable(self):
+        return self
+    def clone_if_mutable(self):
+        clone = self._clone_if_mutable()
+        if not we_are_translated():
+            assert clone.__class__ is self.__class__
+        return clone
+
 class AbstractFailDescr(AbstractDescr):
     index = -1
 
-    def handle_fail(self, metainterp_sd):
+    def handle_fail(self, metainterp_sd, jitdriver_sd):
         raise NotImplementedError
     def compile_and_attach(self, metainterp, new_loop):
         raise NotImplementedError
@@ -694,6 +699,7 @@
     generated assembler.
     """
     terminating = False # see TerminatingLoopToken in compile.py
+    outermost_jitdriver_sd = None
     # specnodes = ...
     # and more data specified by the backend when the loop is compiled
     number = 0

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py	Tue Jul  6 00:00:33 2010
@@ -56,6 +56,7 @@
         self.parent_resumedata_snapshot = None
         self.parent_resumedata_frame_info_list = None
 
+    @specialize.arg(3)
     def copy_constants(self, registers, constants, ConstClass):
         """Copy jitcode.constants[0] to registers[255],
                 jitcode.constants[1] to registers[254],
@@ -68,7 +69,6 @@
             assert j >= 0
             registers[j] = ConstClass(constants[i])
             i -= 1
-    copy_constants._annspecialcase_ = 'specialize:arg(3)'
 
     def cleanup_registers(self):
         # To avoid keeping references alive, this cleans up the registers_r.
@@ -80,6 +80,7 @@
     # ------------------------------
     # Decoding of the JitCode
 
+    @specialize.arg(4)
     def prepare_list_of_boxes(self, outvalue, startindex, position, argcode):
         assert argcode in 'IRF'
         code = self.bytecode
@@ -92,7 +93,6 @@
             elif argcode == 'F': reg = self.registers_f[index]
             else: raise AssertionError(argcode)
             outvalue[startindex+i] = reg
-    prepare_list_of_boxes._annspecialcase_ = 'specialize:arg(4)'
 
     def get_current_position_info(self):
         return self.jitcode.get_live_vars_info(self.pc)
@@ -378,6 +378,14 @@
     opimpl_getarrayitem_gc_f = _opimpl_getarrayitem_gc_any
 
     @arguments("box", "descr", "box")
+    def _opimpl_getarrayitem_raw_any(self, arraybox, arraydescr, indexbox):
+        return self.execute_with_descr(rop.GETARRAYITEM_RAW,
+                                       arraydescr, arraybox, indexbox)
+
+    opimpl_getarrayitem_raw_i = _opimpl_getarrayitem_raw_any
+    opimpl_getarrayitem_raw_f = _opimpl_getarrayitem_raw_any
+
+    @arguments("box", "descr", "box")
     def _opimpl_getarrayitem_gc_pure_any(self, arraybox, arraydescr, indexbox):
         return self.execute_with_descr(rop.GETARRAYITEM_GC_PURE,
                                        arraydescr, arraybox, indexbox)
@@ -396,6 +404,15 @@
     opimpl_setarrayitem_gc_r = _opimpl_setarrayitem_gc_any
     opimpl_setarrayitem_gc_f = _opimpl_setarrayitem_gc_any
 
+    @arguments("box", "descr", "box", "box")
+    def _opimpl_setarrayitem_raw_any(self, arraybox, arraydescr,
+                                    indexbox, itembox):
+        self.execute_with_descr(rop.SETARRAYITEM_RAW, arraydescr, arraybox,
+                                indexbox, itembox)
+
+    opimpl_setarrayitem_raw_i = _opimpl_setarrayitem_raw_any
+    opimpl_setarrayitem_raw_f = _opimpl_setarrayitem_raw_any
+
     @arguments("box", "descr")
     def opimpl_arraylen_gc(self, arraybox, arraydescr):
         return self.execute_with_descr(rop.ARRAYLEN_GC, arraydescr, arraybox)
@@ -516,6 +533,8 @@
     def _nonstandard_virtualizable(self, pc, box):
         # returns True if 'box' is actually not the "standard" virtualizable
         # that is stored in metainterp.virtualizable_boxes[-1]
+        if self.metainterp.jitdriver_sd.virtualizable_info is None:
+            return True      # can occur in case of multiple JITs
         standard_box = self.metainterp.virtualizable_boxes[-1]
         if standard_box is box:
             return False
@@ -528,17 +547,11 @@
         return not isstandard
 
     def _get_virtualizable_field_index(self, fielddescr):
-        vinfo = self.metainterp.staticdata.virtualizable_info
+        # Get the index of a fielddescr.  Must only be called for
+        # the "standard" virtualizable.
+        vinfo = self.metainterp.jitdriver_sd.virtualizable_info
         return vinfo.static_field_by_descrs[fielddescr]
 
-    def _get_virtualizable_array_field_descr(self, index):
-        vinfo = self.metainterp.staticdata.virtualizable_info
-        return vinfo.array_field_descrs[index]
-
-    def _get_virtualizable_array_descr(self, index):
-        vinfo = self.metainterp.staticdata.virtualizable_info
-        return vinfo.array_descrs[index]
-
     @arguments("orgpc", "box", "descr")
     def _opimpl_getfield_vable(self, pc, box, fielddescr):
         if self._nonstandard_virtualizable(pc, box):
@@ -566,8 +579,11 @@
     opimpl_setfield_vable_f = _opimpl_setfield_vable
 
     def _get_arrayitem_vable_index(self, pc, arrayfielddescr, indexbox):
+        # Get the index of an array item: the index'th of the array
+        # described by arrayfielddescr.  Must only be called for
+        # the "standard" virtualizable.
         indexbox = self.implement_guard_value(pc, indexbox)
-        vinfo = self.metainterp.staticdata.virtualizable_info
+        vinfo = self.metainterp.jitdriver_sd.virtualizable_info
         virtualizable_box = self.metainterp.virtualizable_boxes[-1]
         virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
         arrayindex = vinfo.array_field_by_descrs[arrayfielddescr]
@@ -616,7 +632,7 @@
             arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC,
                                                           fdescr, box)
             return self.execute_with_descr(rop.ARRAYLEN_GC, adescr, arraybox)
-        vinfo = self.metainterp.staticdata.virtualizable_info
+        vinfo = self.metainterp.jitdriver_sd.virtualizable_info
         virtualizable_box = self.metainterp.virtualizable_boxes[-1]
         virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
         arrayindex = vinfo.array_field_by_descrs[fdescr]
@@ -665,12 +681,12 @@
     opimpl_residual_call_irf_f = _opimpl_residual_call3
     opimpl_residual_call_irf_v = _opimpl_residual_call3
 
-    @arguments("boxes3", "boxes3")
-    def _opimpl_recursive_call(self, greenboxes, redboxes):
+    @arguments("int", "boxes3", "boxes3")
+    def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes):
+        targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex]
         allboxes = greenboxes + redboxes
-        metainterp_sd = self.metainterp.staticdata
-        portal_code = metainterp_sd.portal_code
-        warmrunnerstate = metainterp_sd.state
+        portal_code = targetjitdriver_sd.mainjitcode
+        warmrunnerstate = targetjitdriver_sd.warmstate
         token = None
         if warmrunnerstate.inlining:
             if warmrunnerstate.can_inline_callable(greenboxes):
@@ -679,12 +695,13 @@
             token = warmrunnerstate.get_assembler_token(greenboxes)
             # verify that we have all green args, needed to make sure
             # that assembler that we call is still correct
-            self.verify_green_args(greenboxes)
+            self.verify_green_args(targetjitdriver_sd, greenboxes)
         #
-        k = llmemory.cast_ptr_to_adr(metainterp_sd._portal_runner_ptr)
+        k = targetjitdriver_sd.portal_runner_adr
         funcbox = ConstInt(heaptracker.adr2int(k))
         return self.do_residual_call(funcbox, portal_code.calldescr,
-                                     allboxes, assembler_call_token=token)
+                                     allboxes, assembler_call_token=token,
+                                     assembler_call_jd=targetjitdriver_sd)
 
     opimpl_recursive_call_i = _opimpl_recursive_call
     opimpl_recursive_call_r = _opimpl_recursive_call
@@ -768,24 +785,27 @@
         self.generate_guard(rop.GUARD_CLASS, box, [clsbox], resumepc=orgpc)
         return clsbox
 
-    @arguments()
-    def opimpl_can_enter_jit(self):
+    @arguments("int")
+    def opimpl_can_enter_jit(self, jdindex):
         if self.metainterp.in_recursion:
             from pypy.jit.metainterp.warmspot import CannotInlineCanEnterJit
             raise CannotInlineCanEnterJit()
+        assert jdindex == self.metainterp.jitdriver_sd.index, (
+            "found a can_enter_jit that does not match the current jitdriver")
         self.metainterp.seen_can_enter_jit = True
 
-    def verify_green_args(self, varargs):
-        num_green_args = self.metainterp.staticdata.num_green_args
+    def verify_green_args(self, jitdriver_sd, varargs):
+        num_green_args = jitdriver_sd.num_green_args
         assert len(varargs) == num_green_args
         for i in range(num_green_args):
             assert isinstance(varargs[i], Const)
 
-    @arguments("orgpc", "boxes3", "boxes3")
-    def opimpl_jit_merge_point(self, orgpc, greenboxes, redboxes):
-        self.verify_green_args(greenboxes)
+    @arguments("orgpc", "int", "boxes3", "boxes3")
+    def opimpl_jit_merge_point(self, orgpc, jdindex, greenboxes, redboxes):
+        jitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex]
+        self.verify_green_args(jitdriver_sd, greenboxes)
         # xxx we may disable the following line in some context later
-        self.debug_merge_point(greenboxes)
+        self.debug_merge_point(jitdriver_sd, greenboxes)
         if self.metainterp.seen_can_enter_jit:
             self.metainterp.seen_can_enter_jit = False
             # Assert that it's impossible to arrive here with in_recursion
@@ -793,6 +813,7 @@
             # to True by opimpl_can_enter_jit, which should be executed
             # just before opimpl_jit_merge_point (no recursion inbetween).
             assert not self.metainterp.in_recursion
+            assert jitdriver_sd is self.metainterp.jitdriver_sd
             # Set self.pc to point to jit_merge_point instead of just after:
             # if reached_can_enter_jit() raises SwitchToBlackhole, then the
             # pc is still at the jit_merge_point, which is a point that is
@@ -802,10 +823,9 @@
             self.metainterp.reached_can_enter_jit(greenboxes, redboxes)
             self.pc = saved_pc
 
-    def debug_merge_point(self, greenkey):
+    def debug_merge_point(self, jitdriver_sd, greenkey):
         # debugging: produce a DEBUG_MERGE_POINT operation
-        sd = self.metainterp.staticdata
-        loc = sd.state.get_location_str(greenkey)
+        loc = jitdriver_sd.warmstate.get_location_str(greenkey)
         debug_print(loc)
         constloc = self.metainterp.cpu.ts.conststr(loc)
         self.metainterp.history.record(rop.DEBUG_MERGE_POINT,
@@ -948,14 +968,15 @@
         original_greenkey = metainterp.resumekey.original_greenkey
         if opnum == rop.GUARD_NOT_FORCED:
             resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd,
-                                                         original_greenkey)
+                                                   original_greenkey,
+                                                   metainterp.jitdriver_sd)
         else:
             resumedescr = compile.ResumeGuardDescr(metainterp_sd,
                                                    original_greenkey)
         guard_op = metainterp.history.record(opnum, moreargs, None,
                                              descr=resumedescr)       
         virtualizable_boxes = None
-        if metainterp.staticdata.virtualizable_info is not None:
+        if metainterp.jitdriver_sd.virtualizable_info is not None:
             virtualizable_boxes = metainterp.virtualizable_boxes
         saved_pc = self.pc
         if resumepc >= 0:
@@ -1007,7 +1028,8 @@
         return resbox
 
     def do_residual_call(self, funcbox, descr, argboxes,
-                         assembler_call_token=None):
+                         assembler_call_token=None,
+                         assembler_call_jd=None):
         # First build allboxes: it may need some reordering from the
         # list provided in argboxes, depending on the order in which
         # the arguments are expected by the function
@@ -1052,7 +1074,8 @@
                 rop.CALL_MAY_FORCE, allboxes, descr=descr)
             self.metainterp.vrefs_after_residual_call()
             if assembler_call_token is not None:
-                self.metainterp.direct_assembler_call(assembler_call_token)
+                self.metainterp.direct_assembler_call(assembler_call_token,
+                                                      assembler_call_jd)
             if resbox is not None:
                 self.make_result_of_lastop(resbox)
             self.metainterp.vable_after_residual_call()
@@ -1137,6 +1160,11 @@
         self._addr2name_keys = [key for key, value in list_of_addr2name]
         self._addr2name_values = [value for key, value in list_of_addr2name]
 
+    def setup_jitdrivers_sd(self, optimizer):
+        if optimizer is not None:
+            for jd in self.jitdrivers_sd:
+                jd.warmstate.set_param_optimizer(optimizer)
+
     def finish_setup(self, codewriter, optimizer=None):
         from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder
         self.blackholeinterpbuilder = BlackholeInterpBuilder(codewriter, self)
@@ -1147,28 +1175,21 @@
         self.setup_indirectcalltargets(asm.indirectcalltargets)
         self.setup_list_of_addr2name(asm.list_of_addr2name)
         #
-        self.portal_code = codewriter.mainjitcode
-        self._portal_runner_ptr = codewriter.callcontrol.portal_runner_ptr
+        self.jitdrivers_sd = codewriter.callcontrol.jitdrivers_sd
         self.virtualref_info = codewriter.callcontrol.virtualref_info
-        self.virtualizable_info = codewriter.callcontrol.virtualizable_info
-        RESULT = codewriter.portal_graph.getreturnvar().concretetype
-        self.result_type = history.getkind(RESULT)
+        self.setup_jitdrivers_sd(optimizer)
         #
         # store this information for fastpath of call_assembler
-        name = self.result_type
-        tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' % name)
-        num = self.cpu.get_fail_descr_number(tokens[0].finishdescr)
-        setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num)
+        # (only the paths that can actually be taken)
+        for jd in self.jitdrivers_sd:
+            name = {history.INT: 'int',
+                    history.REF: 'ref',
+                    history.FLOAT: 'float',
+                    history.VOID: 'void'}[jd.result_type]
+            tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' % name)
+            num = self.cpu.get_fail_descr_number(tokens[0].finishdescr)
+            setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num)
         #
-        warmrunnerdesc = self.warmrunnerdesc
-        if warmrunnerdesc is not None:
-            self.num_green_args = warmrunnerdesc.num_green_args
-            self.state = warmrunnerdesc.state
-            if optimizer is not None:
-                self.state.set_param_optimizer(optimizer)
-        else:
-            self.num_green_args = 0
-            self.state = None
         self.globaldata = MetaInterpGlobalData(self)
 
     def _setup_once(self):
@@ -1189,8 +1210,7 @@
                 # Build the dictionary at run-time.  This is needed
                 # because the keys are function/class addresses, so they
                 # can change from run to run.
-                k = llmemory.cast_ptr_to_adr(self._portal_runner_ptr)
-                d = {k: 'recursive call'}
+                d = {}
                 keys = self._addr2name_keys
                 values = self._addr2name_values
                 for i in range(len(keys)):
@@ -1248,47 +1268,31 @@
         self.loopnumbering = 0
         self.resume_virtuals = {}
         self.resume_virtuals_not_translated = []
-        #
-        state = staticdata.state
-        if state is not None:
-            self.jit_cell_at_key = state.jit_cell_at_key
-        else:
-            # for tests only; not RPython
-            class JitCell:
-                compiled_merge_points = None
-            _jitcell_dict = {}
-            def jit_cell_at_key(greenkey):
-                greenkey = tuple(greenkey)
-                return _jitcell_dict.setdefault(greenkey, JitCell())
-            self.jit_cell_at_key = jit_cell_at_key
-
-    def get_compiled_merge_points(self, greenkey):
-        cell = self.jit_cell_at_key(greenkey)
-        if cell.compiled_merge_points is None:
-            cell.compiled_merge_points = []
-        return cell.compiled_merge_points
 
 # ____________________________________________________________
 
 class MetaInterp(object):
     in_recursion = 0
 
-    def __init__(self, staticdata):
+    def __init__(self, staticdata, jitdriver_sd):
         self.staticdata = staticdata
         self.cpu = staticdata.cpu
+        self.jitdriver_sd = jitdriver_sd
+        # Note: self.jitdriver_sd is the JitDriverStaticData that corresponds
+        # to the current loop -- the outermost one.  Be careful, because
+        # during recursion we can also see other jitdrivers.
         self.portal_trace_positions = []
         self.free_frames_list = []
         self.last_exc_value_box = None
 
     def perform_call(self, jitcode, boxes, greenkey=None):
         # causes the metainterp to enter the given subfunction
-        # with a special case for recursive portal calls
         f = self.newframe(jitcode, greenkey)
         f.setup_call(boxes)
         raise ChangeFrame
 
     def newframe(self, jitcode, greenkey=None):
-        if jitcode is self.staticdata.portal_code:
+        if jitcode.is_portal:
             self.in_recursion += 1
         if greenkey is not None:
             self.portal_trace_positions.append(
@@ -1303,7 +1307,7 @@
 
     def popframe(self):
         frame = self.framestack.pop()
-        if frame.jitcode is self.staticdata.portal_code:
+        if frame.jitcode.is_portal:
             self.in_recursion -= 1
         if frame.greenkey is not None:
             self.portal_trace_positions.append(
@@ -1327,14 +1331,15 @@
             except SwitchToBlackhole, stb:
                 self.aborted_tracing(stb.reason)
             sd = self.staticdata
-            if sd.result_type == 'void':
+            result_type = self.jitdriver_sd.result_type
+            if result_type == history.VOID:
                 assert resultbox is None
                 raise sd.DoneWithThisFrameVoid()
-            elif sd.result_type == 'int':
+            elif result_type == history.INT:
                 raise sd.DoneWithThisFrameInt(resultbox.getint())
-            elif sd.result_type == 'ref':
+            elif result_type == history.REF:
                 raise sd.DoneWithThisFrameRef(self.cpu, resultbox.getref_base())
-            elif sd.result_type == 'float':
+            elif result_type == history.FLOAT:
                 raise sd.DoneWithThisFrameFloat(resultbox.getfloat())
             else:
                 assert False
@@ -1364,14 +1369,17 @@
         in_recursion = -1
         for frame in self.framestack:
             jitcode = frame.jitcode
-            if jitcode is self.staticdata.portal_code:
+            assert jitcode.is_portal == len([
+                jd for jd in self.staticdata.jitdrivers_sd
+                   if jd.mainjitcode is jitcode])
+            if jitcode.is_portal:
                 in_recursion += 1
         if in_recursion != self.in_recursion:
             print "in_recursion problem!!!"
             print in_recursion, self.in_recursion
             for frame in self.framestack:
                 jitcode = frame.jitcode
-                if jitcode is self.staticdata.portal_code:
+                if jitcode.is_portal:
                     print "P",
                 else:
                     print " ",
@@ -1379,7 +1387,6 @@
             raise AssertionError
 
     def create_empty_history(self):
-        warmrunnerstate = self.staticdata.state
         self.history = history.History()
         self.staticdata.stats.set_history(self.history)
 
@@ -1489,12 +1496,11 @@
         self.resumekey.reset_counter_from_failure()
 
     def blackhole_if_trace_too_long(self):
-        warmrunnerstate = self.staticdata.state
+        warmrunnerstate = self.jitdriver_sd.warmstate
         if len(self.history.operations) > warmrunnerstate.trace_limit:
             greenkey_of_huge_function = self.find_biggest_function()
             self.portal_trace_positions = None
             if greenkey_of_huge_function is not None:
-                warmrunnerstate = self.staticdata.state
                 warmrunnerstate.disable_noninlinable_function(
                     greenkey_of_huge_function)
             raise SwitchToBlackhole(ABORT_TOO_LONG)
@@ -1521,21 +1527,27 @@
                     self.staticdata.log(sys.exc_info()[0].__name__)
                 raise
 
-    def compile_and_run_once(self, *args):
+    @specialize.arg(1)
+    def compile_and_run_once(self, jitdriver_sd, *args):
+        # NB. we pass explicity 'jitdriver_sd' around here, even though it
+        # is also available as 'self.jitdriver_sd', because we need to
+        # specialize this function and a few other ones for the '*args'.
         debug_start('jit-tracing')
         self.staticdata._setup_once()
         self.staticdata.profiler.start_tracing()
+        assert jitdriver_sd is self.jitdriver_sd
         self.create_empty_history()
         try:
-            return self._compile_and_run_once(*args)
+            original_boxes = self.initialize_original_boxes(jitdriver_sd,*args)
+            return self._compile_and_run_once(original_boxes)
         finally:
             self.staticdata.profiler.end_tracing()
             debug_stop('jit-tracing')
 
-    def _compile_and_run_once(self, *args):
-        original_boxes = self.initialize_state_from_start(*args)
+    def _compile_and_run_once(self, original_boxes):
+        self.initialize_state_from_start(original_boxes)
         self.current_merge_points = [(original_boxes, 0)]
-        num_green_args = self.staticdata.num_green_args
+        num_green_args = self.jitdriver_sd.num_green_args
         original_greenkey = original_boxes[:num_green_args]
         redkey = original_boxes[num_green_args:]
         self.resumekey = compile.ResumeFromInterpDescr(original_greenkey,
@@ -1602,7 +1614,7 @@
         self.remove_consts_and_duplicates(redboxes, len(redboxes),
                                           duplicates)
         live_arg_boxes = greenboxes + redboxes
-        if self.staticdata.virtualizable_info is not None:
+        if self.jitdriver_sd.virtualizable_info is not None:
             # we use pop() to remove the last item, which is the virtualizable
             # itself
             self.remove_consts_and_duplicates(self.virtualizable_boxes,
@@ -1624,11 +1636,12 @@
         # Search in current_merge_points for original_boxes with compatible
         # green keys, representing the beginning of the same loop as the one
         # we end now. 
-       
+
+        num_green_args = self.jitdriver_sd.num_green_args
         for j in range(len(self.current_merge_points)-1, -1, -1):
             original_boxes, start = self.current_merge_points[j]
             assert len(original_boxes) == len(live_arg_boxes) or start < 0
-            for i in range(self.staticdata.num_green_args):
+            for i in range(num_green_args):
                 box1 = original_boxes[i]
                 box2 = live_arg_boxes[i]
                 assert isinstance(box1, Const)
@@ -1651,7 +1664,7 @@
 
     def designate_target_loop(self, gmp):
         loop_token = gmp.target_loop_token
-        num_green_args = self.staticdata.num_green_args
+        num_green_args = self.jitdriver_sd.num_green_args
         residual_args = self.get_residual_args(loop_token.specnodes,
                                                gmp.argboxes[num_green_args:])
         history.set_future_values(self.cpu, residual_args)
@@ -1692,12 +1705,17 @@
             from pypy.jit.metainterp.resoperation import opname
             raise NotImplementedError(opname[opnum])
 
+    def get_compiled_merge_points(self, greenkey):
+        cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey)
+        if cell.compiled_merge_points is None:
+            cell.compiled_merge_points = []
+        return cell.compiled_merge_points
+
     def compile(self, original_boxes, live_arg_boxes, start):
-        num_green_args = self.staticdata.num_green_args
+        num_green_args = self.jitdriver_sd.num_green_args
         self.history.inputargs = original_boxes[num_green_args:]
         greenkey = original_boxes[:num_green_args]
-        glob = self.staticdata.globaldata
-        old_loop_tokens = glob.get_compiled_merge_points(greenkey)
+        old_loop_tokens = self.get_compiled_merge_points(greenkey)
         self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None)
         loop_token = compile.compile_new_loop(self, old_loop_tokens,
                                               greenkey, start)
@@ -1706,10 +1724,9 @@
         self.history.operations.pop()     # remove the JUMP
 
     def compile_bridge(self, live_arg_boxes):
-        num_green_args = self.staticdata.num_green_args
+        num_green_args = self.jitdriver_sd.num_green_args
         greenkey = live_arg_boxes[:num_green_args]
-        glob = self.staticdata.globaldata
-        old_loop_tokens = glob.get_compiled_merge_points(greenkey)
+        old_loop_tokens = self.get_compiled_merge_points(greenkey)
         if len(old_loop_tokens) == 0:
             return
         self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None)
@@ -1723,17 +1740,18 @@
         self.gen_store_back_in_virtualizable()
         # temporarily put a JUMP to a pseudo-loop
         sd = self.staticdata
-        if sd.result_type == 'void':
+        result_type = self.jitdriver_sd.result_type
+        if result_type == history.VOID:
             assert exitbox is None
             exits = []
             loop_tokens = sd.loop_tokens_done_with_this_frame_void
-        elif sd.result_type == 'int':
+        elif result_type == history.INT:
             exits = [exitbox]
             loop_tokens = sd.loop_tokens_done_with_this_frame_int
-        elif sd.result_type == 'ref':
+        elif result_type == history.REF:
             exits = [exitbox]
             loop_tokens = sd.loop_tokens_done_with_this_frame_ref
-        elif sd.result_type == 'float':
+        elif result_type == history.FLOAT:
             exits = [exitbox]
             loop_tokens = sd.loop_tokens_done_with_this_frame_float
         else:
@@ -1765,26 +1783,32 @@
             specnode.extract_runtime_data(self.cpu, args[i], expanded_args)
         return expanded_args
 
-    def _initialize_from_start(self, original_boxes, num_green_args, *args):
+    @specialize.arg(1)
+    def initialize_original_boxes(self, jitdriver_sd, *args):
+        original_boxes = []
+        self._fill_original_boxes(jitdriver_sd, original_boxes,
+                                  jitdriver_sd.num_green_args, *args)
+        return original_boxes
+
+    @specialize.arg(1)
+    def _fill_original_boxes(self, jitdriver_sd, original_boxes,
+                             num_green_args, *args):
         if args:
             from pypy.jit.metainterp.warmstate import wrap
             box = wrap(self.cpu, args[0], num_green_args > 0)
             original_boxes.append(box)
-            self._initialize_from_start(original_boxes, num_green_args-1,
-                                        *args[1:])
+            self._fill_original_boxes(jitdriver_sd, original_boxes,
+                                      num_green_args-1, *args[1:])
 
-    def initialize_state_from_start(self, *args):
-        self.in_recursion = -1 # always one portal around
-        num_green_args = self.staticdata.num_green_args
-        original_boxes = []
-        self._initialize_from_start(original_boxes, num_green_args, *args)
+    def initialize_state_from_start(self, original_boxes):
         # ----- make a new frame -----
+        self.in_recursion = -1 # always one portal around
         self.framestack = []
-        f = self.newframe(self.staticdata.portal_code)
+        f = self.newframe(self.jitdriver_sd.mainjitcode)
         f.setup_call(original_boxes)
+        assert self.in_recursion == 0
         self.virtualref_boxes = []
         self.initialize_virtualizable(original_boxes)
-        return original_boxes
 
     def initialize_state_from_guard_failure(self, resumedescr):
         # guard failure: rebuild a complete MIFrame stack
@@ -1794,9 +1818,11 @@
         self.history.inputargs = [box for box in inputargs_and_holes if box]
 
     def initialize_virtualizable(self, original_boxes):
-        vinfo = self.staticdata.virtualizable_info
+        vinfo = self.jitdriver_sd.virtualizable_info
         if vinfo is not None:
-            virtualizable_box = original_boxes[vinfo.index_of_virtualizable]
+            index = (self.jitdriver_sd.num_green_args +
+                     self.jitdriver_sd.index_of_virtualizable)
+            virtualizable_box = original_boxes[index]
             virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
             # The field 'virtualizable_boxes' is not even present
             # if 'virtualizable_info' is None.  Check for that first.
@@ -1807,7 +1833,7 @@
             self.initialize_virtualizable_enter()
 
     def initialize_virtualizable_enter(self):
-        vinfo = self.staticdata.virtualizable_info
+        vinfo = self.jitdriver_sd.virtualizable_info
         virtualizable_box = self.virtualizable_boxes[-1]
         virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
         vinfo.clear_vable_token(virtualizable)
@@ -1821,7 +1847,7 @@
             # the FORCE_TOKEN is already set at runtime in each vref when
             # it is created, by optimizeopt.py.
         #
-        vinfo = self.staticdata.virtualizable_info
+        vinfo = self.jitdriver_sd.virtualizable_info
         if vinfo is not None:
             virtualizable_box = self.virtualizable_boxes[-1]
             virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
@@ -1847,7 +1873,7 @@
                 self.stop_tracking_virtualref(i)
 
     def vable_after_residual_call(self):
-        vinfo = self.staticdata.virtualizable_info
+        vinfo = self.jitdriver_sd.virtualizable_info
         if vinfo is not None:
             virtualizable_box = self.virtualizable_boxes[-1]
             virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
@@ -1898,7 +1924,7 @@
         assert self.last_exc_value_box is None
 
     def rebuild_state_after_failure(self, resumedescr):
-        vinfo = self.staticdata.virtualizable_info
+        vinfo = self.jitdriver_sd.virtualizable_info
         self.framestack = []
         boxlists = resume.rebuild_from_resumedata(self, resumedescr, vinfo)
         inputargs_and_holes, virtualizable_boxes, virtualref_boxes = boxlists
@@ -1923,24 +1949,19 @@
             virtualizable_box = self.virtualizable_boxes[-1]
             virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
             assert not virtualizable.vable_token
-            if 0:  ## self._already_allocated_resume_virtuals is not None:
-                # resuming from a ResumeGuardForcedDescr: load the new values
-                # currently stored on the virtualizable fields
-                self.load_fields_from_virtualizable()
-            else:
-                # normal case: fill the virtualizable with the local boxes
-                self.synchronize_virtualizable()
+            # fill the virtualizable with the local boxes
+            self.synchronize_virtualizable()
         return inputargs_and_holes
 
     def check_synchronized_virtualizable(self):
         if not we_are_translated():
-            vinfo = self.staticdata.virtualizable_info
+            vinfo = self.jitdriver_sd.virtualizable_info
             virtualizable_box = self.virtualizable_boxes[-1]
             virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
             vinfo.check_boxes(virtualizable, self.virtualizable_boxes)
 
     def synchronize_virtualizable(self):
-        vinfo = self.staticdata.virtualizable_info
+        vinfo = self.jitdriver_sd.virtualizable_info
         virtualizable_box = self.virtualizable_boxes[-1]
         virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
         vinfo.write_boxes(virtualizable, self.virtualizable_boxes)
@@ -1949,7 +1970,7 @@
         # Force a reload of the virtualizable fields into the local
         # boxes (called only in escaping cases).  Only call this function
         # just before SwitchToBlackhole.
-        vinfo = self.staticdata.virtualizable_info
+        vinfo = self.jitdriver_sd.virtualizable_info
         if vinfo is not None:
             virtualizable_box = self.virtualizable_boxes[-1]
             virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
@@ -1958,7 +1979,7 @@
             self.virtualizable_boxes.append(virtualizable_box)
 
     def gen_store_back_in_virtualizable(self):
-        vinfo = self.staticdata.virtualizable_info
+        vinfo = self.jitdriver_sd.virtualizable_info
         if vinfo is not None:
             # xxx only write back the fields really modified
             vbox = self.virtualizable_boxes[-1]
@@ -1979,8 +2000,7 @@
                                             abox, ConstInt(j), itembox)
             assert i + 1 == len(self.virtualizable_boxes)
 
-    def gen_load_from_other_virtualizable(self, vbox):
-        vinfo = self.staticdata.virtualizable_info
+    def gen_load_from_other_virtualizable(self, vinfo, vbox):
         boxes = []
         assert vinfo is not None
         for i in range(vinfo.num_static_extra_boxes):
@@ -2004,7 +2024,7 @@
         for i in range(len(boxes)):
             if boxes[i] is oldbox:
                 boxes[i] = newbox
-        if self.staticdata.virtualizable_info is not None:
+        if self.jitdriver_sd.virtualizable_info is not None:
             boxes = self.virtualizable_boxes
             for i in range(len(boxes)):
                 if boxes[i] is oldbox:
@@ -2052,18 +2072,19 @@
         op.args = [resbox_as_const] + op.args
         return resbox
 
-    def direct_assembler_call(self, token):
+    def direct_assembler_call(self, token, targetjitdriver_sd):
         """ Generate a direct call to assembler for portal entry point,
         patching the CALL_MAY_FORCE that occurred just now.
         """
         op = self.history.operations.pop()
         assert op.opnum == rop.CALL_MAY_FORCE
-        num_green_args = self.staticdata.num_green_args
+        num_green_args = targetjitdriver_sd.num_green_args
         args = op.args[num_green_args + 1:]
-        if self.staticdata.virtualizable_info is not None:
-            vindex = self.staticdata.virtualizable_info.index_of_virtualizable
-            vbox = args[vindex - num_green_args]
-            args = args + self.gen_load_from_other_virtualizable(vbox)
+        vinfo = targetjitdriver_sd.virtualizable_info
+        if vinfo is not None:
+            index = targetjitdriver_sd.index_of_virtualizable
+            vbox = args[index]
+            args = args + self.gen_load_from_other_virtualizable(vinfo, vbox)
             # ^^^ and not "+=", which makes 'args' a resizable list
         op.opnum = rop.CALL_ASSEMBLER
         op.args = args
@@ -2166,6 +2187,16 @@
                 position = position3 + 1 + length3
             elif argtype == "orgpc":
                 value = orgpc
+            elif argtype == "int":
+                argcode = argcodes[next_argcode]
+                next_argcode = next_argcode + 1
+                if argcode == 'i':
+                    value = self.registers_i[ord(code[position])].getint()
+                elif argcode == 'c':
+                    value = signedord(code[position])
+                else:
+                    raise AssertionError("bad argcode")
+                position += 1
             else:
                 raise AssertionError("bad argtype: %r" % (argtype,))
             args += (value,)

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/resoperation.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/resoperation.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/resoperation.py	Tue Jul  6 00:00:33 2010
@@ -34,7 +34,7 @@
     def clone(self):
         descr = self.descr
         if descr is not None:
-            descr = descr._clone_if_mutable()
+            descr = descr.clone_if_mutable()
         op = ResOperation(self.opnum, self.args, self.result, descr)
         op.fail_args = self.fail_args
         if not we_are_translated():
@@ -199,6 +199,7 @@
     '_ALWAYS_PURE_LAST',  # ----- end of always_pure operations -----
 
     'GETARRAYITEM_GC/2d',
+    'GETARRAYITEM_RAW/2d',
     'GETFIELD_GC/1d',
     'GETFIELD_RAW/1d',
     'NEW/0d',
@@ -209,7 +210,7 @@
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----
 
     'SETARRAYITEM_GC/3d',
-    'SETARRAYITEM_RAW/3d',#only added by backend.llsupport.gc.rewrite_assembler
+    'SETARRAYITEM_RAW/3d',
     'SETFIELD_GC/2d',
     'SETFIELD_RAW/2d',
     'ARRAYCOPY/7d',      # removed before it's passed to the backend

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/resume.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/resume.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/resume.py	Tue Jul  6 00:00:33 2010
@@ -709,11 +709,11 @@
 
 # ---------- when resuming for blackholing, get direct values ----------
 
-def blackhole_from_resumedata(blackholeinterpbuilder, storage,
+def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage,
                               all_virtuals=None):
     resumereader = ResumeDataDirectReader(blackholeinterpbuilder.cpu, storage,
                                           all_virtuals)
-    vinfo = blackholeinterpbuilder.metainterp_sd.virtualizable_info
+    vinfo = jitdriver_sd.virtualizable_info
     vrefinfo = blackholeinterpbuilder.metainterp_sd.virtualref_info
     resumereader.consume_vref_and_vable(vrefinfo, vinfo)
     #
@@ -745,10 +745,9 @@
     resumereader.done()
     return firstbh
 
-def force_from_resumedata(metainterp_sd, storage):
+def force_from_resumedata(metainterp_sd, storage, vinfo=None):
     resumereader = ResumeDataDirectReader(metainterp_sd.cpu, storage)
     resumereader.handling_async_forcing()
-    vinfo = metainterp_sd.virtualizable_info
     vrefinfo = metainterp_sd.virtualref_info
     resumereader.consume_vref_and_vable(vrefinfo, vinfo)
     return resumereader.virtuals

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_basic.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_basic.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_basic.py	Tue Jul  6 00:00:33 2010
@@ -16,10 +16,18 @@
     from pypy.jit.codewriter import support, codewriter
     from pypy.jit.metainterp import simple_optimize
 
+    class FakeJitCell:
+        compiled_merge_points = None
+
     class FakeWarmRunnerState:
         def attach_unoptimized_bridge_from_interp(self, greenkey, newloop):
             pass
 
+        def jit_cell_at_key(self, greenkey):
+            assert greenkey == []
+            return self._cell
+        _cell = FakeJitCell()
+
         # pick the optimizer this way
         optimize_loop = staticmethod(simple_optimize.optimize_loop)
         optimize_bridge = staticmethod(simple_optimize.optimize_bridge)
@@ -30,14 +38,24 @@
     func._jit_unroll_safe_ = True
     rtyper = support.annotate(func, values, type_system=type_system)
     graphs = rtyper.annotator.translator.graphs
+    result_kind = history.getkind(graphs[0].getreturnvar().concretetype)[0]
+
+    class FakeJitDriverSD:
+        num_green_args = 0
+        portal_graph = graphs[0]
+        virtualizable_info = None
+        result_type = result_kind
+        portal_runner_ptr = "???"
+
     stats = history.Stats()
     cpu = CPUClass(rtyper, stats, None, False)
-    cw = codewriter.CodeWriter(cpu, graphs[0])
+    cw = codewriter.CodeWriter(cpu, [FakeJitDriverSD()])
     testself.cw = cw
     cw.find_all_graphs(JitPolicy())
     #
     testself.warmrunnerstate = FakeWarmRunnerState()
     testself.warmrunnerstate.cpu = cpu
+    FakeJitDriverSD.warmstate = testself.warmrunnerstate
     if hasattr(testself, 'finish_setup_for_interp_operations'):
         testself.finish_setup_for_interp_operations()
     #
@@ -62,7 +80,8 @@
             count_f += 1
         else:
             raise TypeError(T)
-    blackholeinterp.setposition(cw.mainjitcode, 0)
+    [jitdriver_sd] = cw.callcontrol.jitdrivers_sd
+    blackholeinterp.setposition(jitdriver_sd.mainjitcode, 0)
     blackholeinterp.run()
     return blackholeinterp._final_result_anytype()
 
@@ -78,16 +97,15 @@
     cw = testself.cw
     opt = history.Options(listops=True)
     metainterp_sd = pyjitpl.MetaInterpStaticData(cw.cpu, opt)
-    metainterp_sd.finish_setup(cw, optimizer="bogus")
-    metainterp_sd.state = testself.warmrunnerstate
-    metainterp_sd.state.cpu = metainterp_sd.cpu
-    metainterp = pyjitpl.MetaInterp(metainterp_sd)
+    metainterp_sd.finish_setup(cw)
+    [jitdriver_sd] = metainterp_sd.jitdrivers_sd
+    metainterp = pyjitpl.MetaInterp(metainterp_sd, jitdriver_sd)
     metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrame
     metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef
     metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrame
     testself.metainterp = metainterp
     try:
-        metainterp.compile_and_run_once(*args)
+        metainterp.compile_and_run_once(jitdriver_sd, *args)
     except DoneWithThisFrame, e:
         #if conftest.option.view:
         #    metainterp.stats.view()
@@ -864,8 +882,9 @@
         translator.config.translation.gc = "boehm"
         warmrunnerdesc = WarmRunnerDesc(translator,
                                         CPUClass=self.CPUClass)
-        warmrunnerdesc.state.set_param_threshold(3)          # for tests
-        warmrunnerdesc.state.set_param_trace_eagerness(0)    # for tests
+        state = warmrunnerdesc.jitdrivers_sd[0].warmstate
+        state.set_param_threshold(3)          # for tests
+        state.set_param_trace_eagerness(0)    # for tests
         warmrunnerdesc.finish()
         for n, k in [(20, 0), (20, 1)]:
             interp.eval_graph(graph, [n, k])
@@ -1543,6 +1562,35 @@
         res = self.interp_operations(f, [3, 2])
         assert res == 1
 
+    def test_raw_malloc_and_access(self):
+        from pypy.rpython.lltypesystem import rffi
+        
+        TP = rffi.CArray(lltype.Signed)
+        
+        def f(n):
+            a = lltype.malloc(TP, n, flavor='raw')
+            a[0] = n
+            res = a[0]
+            lltype.free(a, flavor='raw')
+            return res
+
+        res = self.interp_operations(f, [10])
+        assert res == 10
+
+    def test_raw_malloc_and_access_float(self):
+        from pypy.rpython.lltypesystem import rffi
+        
+        TP = rffi.CArray(lltype.Float)
+        
+        def f(n, f):
+            a = lltype.malloc(TP, n, flavor='raw')
+            a[0] = f
+            res = a[0]
+            lltype.free(a, flavor='raw')
+            return res
+
+        res = self.interp_operations(f, [10, 3.5])
+        assert res == 3.5
 
 class TestOOtype(BasicTests, OOJitMixin):
 

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_blackhole.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_blackhole.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_blackhole.py	Tue Jul  6 00:00:33 2010
@@ -42,7 +42,7 @@
     blackholeinterp.setarg_i(0, 40)
     blackholeinterp.setarg_i(1, 2)
     blackholeinterp.run()
-    assert blackholeinterp.final_result_i() == 42
+    assert blackholeinterp._final_result_anytype() == 42
 
 def test_simple_const():
     jitcode = JitCode("test")
@@ -54,7 +54,7 @@
     blackholeinterp.setposition(jitcode, 0)
     blackholeinterp.setarg_i(1, 6)
     blackholeinterp.run()
-    assert blackholeinterp.final_result_i() == 42
+    assert blackholeinterp._final_result_anytype() == 42
 
 def test_simple_bigconst():
     jitcode = JitCode("test")
@@ -66,7 +66,7 @@
     blackholeinterp.setposition(jitcode, 0)
     blackholeinterp.setarg_i(1, 10000)
     blackholeinterp.run()
-    assert blackholeinterp.final_result_i() == 42
+    assert blackholeinterp._final_result_anytype() == 42
 
 def test_simple_loop():
     jitcode = JitCode("test")
@@ -85,7 +85,7 @@
     blackholeinterp.setarg_i(0x16, 6)    # %i0
     blackholeinterp.setarg_i(0x17, 100)  # %i1
     blackholeinterp.run()
-    assert blackholeinterp.final_result_i() == 100+6+5+4+3
+    assert blackholeinterp._final_result_anytype() == 100+6+5+4+3
 
 def test_simple_exception():
     jitcode = JitCode("test")
@@ -104,12 +104,12 @@
     blackholeinterp.setposition(jitcode, 0)
     blackholeinterp.setarg_i(0x9, 100)
     blackholeinterp.run()
-    assert blackholeinterp.final_result_i() == 200
+    assert blackholeinterp._final_result_anytype() == 200
     #
     blackholeinterp.setposition(jitcode, 0)
     blackholeinterp.setarg_i(0x9, -100)
     blackholeinterp.run()
-    assert blackholeinterp.final_result_i() == 42
+    assert blackholeinterp._final_result_anytype() == 42
 
 def test_convert_and_run_from_pyjitpl():
     class MyMIFrame:

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_compile.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_compile.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_compile.py	Tue Jul  6 00:00:33 2010
@@ -51,14 +51,14 @@
     logger_noopt = FakeLogger()
     logger_ops = FakeLogger()
 
-    state = FakeState()
     stats = Stats()
     profiler = jitprof.EmptyProfiler()
     def log(self, msg, event_kind=None):
         pass
 
 class FakeMetaInterp:
-    pass
+    class jitdriver_sd:
+        warmstate = FakeState()
 
 def test_compile_new_loop():
     cpu = FakeCPU()

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_list.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_list.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_list.py	Tue Jul  6 00:00:33 2010
@@ -143,6 +143,51 @@
         assert res == 7
         self.check_loops(call=0)
 
+    def test_fold_getitem_1(self):
+        jitdriver = JitDriver(greens = ['pc', 'n', 'l'], reds = ['total'])
+        def f(n):
+            l = [100, n, 300, n, 500]
+            total = 0
+            pc = n
+            while True:
+                jitdriver.can_enter_jit(l=l, pc=pc, n=n, total=total)
+                jitdriver.jit_merge_point(l=l, pc=pc, n=n, total=total)
+                total += l[pc]
+                if total > 10000:
+                    return total
+                pc -= 1
+                if pc < 0:
+                    pc = n
+
+        res = self.meta_interp(f, [4], listops=True)
+        assert res == f(4)
+        self.check_loops(call=0)
+
+    def test_fold_getitem_2(self):
+        jitdriver = JitDriver(greens = ['pc', 'n', 'l'], reds = ['total', 'x'])
+        class X:
+            pass
+        def f(n):
+            l = [100, n, 300, n, 500]
+            total = 0
+            x = X()
+            x.pc = n
+            while True:
+                pc = x.pc
+                jitdriver.can_enter_jit(l=l, pc=pc, n=n, total=total, x=x)
+                jitdriver.jit_merge_point(l=l, pc=pc, n=n, total=total, x=x)
+                x.pc = pc
+                total += l[x.pc]
+                if total > 10000:
+                    return total
+                x.pc -= 1
+                if x.pc < 0:
+                    x.pc = n
+
+        res = self.meta_interp(f, [4], listops=True)
+        assert res == f(4)
+        self.check_loops(call=0, getfield_gc=0)
+
 class TestOOtype(ListTests, OOJitMixin):
     pass
 

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_pyjitpl.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_pyjitpl.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_pyjitpl.py	Tue Jul  6 00:00:33 2010
@@ -17,9 +17,8 @@
     portal.setup(None)
     class FakeStaticData:
         cpu = None
-        portal_code = portal
 
-    metainterp = pyjitpl.MetaInterp(FakeStaticData())
+    metainterp = pyjitpl.MetaInterp(FakeStaticData(), None)
     metainterp.framestack = []
     class FakeHistory:
         operations = []
@@ -59,7 +58,7 @@
         assert isinstance(box, referencebox.clonebox().__class__)
         assert box.value == referencebox.value
         return True
-    metainterp = pyjitpl.MetaInterp(FakeStaticData())
+    metainterp = pyjitpl.MetaInterp(FakeStaticData(), None)
     metainterp.history = History()
     b1 = BoxInt(1)
     b2 = BoxInt(2)

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmspot.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmspot.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmspot.py	Tue Jul  6 00:00:33 2010
@@ -128,8 +128,9 @@
 
         from pypy.jit.metainterp import optimize
 
-        assert warmrunnerdescr.state.optimize_loop is optimize.optimize_loop
-        assert warmrunnerdescr.state.optimize_bridge is optimize.optimize_bridge
+        state = warmrunnerdescr.jitdrivers_sd[0].warmstate
+        assert state.optimize_loop is optimize.optimize_loop
+        assert state.optimize_bridge is optimize.optimize_bridge
 
     def test_static_debug_level(self, capfd):
         py.test.skip("debug_level is being deprecated")
@@ -294,7 +295,7 @@
             def __init__(self, no):
                 self.no = no
             
-            def handle_fail(self, metainterp_sd):
+            def handle_fail(self, metainterp_sd, jitdrivers_sd):
                 if self.no == 0:
                     raise metainterp_sd.warmrunnerdesc.DoneWithThisFrameInt(3)
                 if self.no == 1:
@@ -355,12 +356,13 @@
 
     def test_call_helper(self):
         from pypy.rpython.llinterp import LLException
-        
-        assert self.desc.assembler_call_helper(0, 0) == 3
-        assert self.desc.assembler_call_helper(1, 0) == 10
-        assert self.desc.assembler_call_helper(2, 0) == 10
+
+        [jd] = self.desc.jitdrivers_sd
+        assert jd._assembler_call_helper(0, 0) == 3
+        assert jd._assembler_call_helper(1, 0) == 10
+        assert jd._assembler_call_helper(2, 0) == 10
         try:
-            self.desc.assembler_call_helper(3, 0)
+            jd._assembler_call_helper(3, 0)
         except LLException, lle:
             assert lle[0] == self.exc_vtable
         else:

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmstate.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmstate.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_warmstate.py	Tue Jul  6 00:00:33 2010
@@ -4,10 +4,9 @@
 from pypy.rpython.annlowlevel import llhelper
 from pypy.jit.metainterp.warmstate import wrap, unwrap
 from pypy.jit.metainterp.warmstate import equal_whatever, hash_whatever
-from pypy.jit.metainterp.warmstate import WarmEnterState
+from pypy.jit.metainterp.warmstate import WarmEnterState, JitCell
 from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from pypy.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr
-from pypy.rlib.jit import BaseJitCell
 
 
 def test_unwrap():
@@ -58,14 +57,12 @@
 
 
 def test_make_jitcell_getter_default():
-    class FakeWarmRunnerDesc:
-        green_args_spec = [lltype.Signed, lltype.Float]
-    class FakeJitCell(BaseJitCell):
-        pass
-    state = WarmEnterState(FakeWarmRunnerDesc())
-    get_jitcell = state._make_jitcell_getter_default(FakeJitCell)
+    class FakeJitDriverSD:
+        _green_args_spec = [lltype.Signed, lltype.Float]
+    state = WarmEnterState(None, FakeJitDriverSD())
+    get_jitcell = state._make_jitcell_getter_default()
     cell1 = get_jitcell(42, 42.5)
-    assert isinstance(cell1, FakeJitCell)
+    assert isinstance(cell1, JitCell)
     cell2 = get_jitcell(42, 42.5)
     assert cell1 is cell2
     cell3 = get_jitcell(41, 42.5)
@@ -73,10 +70,10 @@
     assert cell1 is not cell3 is not cell4 is not cell1
 
 def test_make_jitcell_getter():
-    class FakeWarmRunnerDesc:
-        green_args_spec = [lltype.Float]
-        get_jitcell_at_ptr = None
-    state = WarmEnterState(FakeWarmRunnerDesc())
+    class FakeJitDriverSD:
+        _green_args_spec = [lltype.Float]
+        _get_jitcell_at_ptr = None
+    state = WarmEnterState(None, FakeJitDriverSD())
     get_jitcell = state.make_jitcell_getter()
     cell1 = get_jitcell(1.75)
     cell2 = get_jitcell(1.75)
@@ -87,8 +84,6 @@
     from pypy.rpython.typesystem import LowLevelTypeSystem
     class FakeRTyper:
         type_system = LowLevelTypeSystem.instance
-    class FakeJitCell(BaseJitCell):
-        pass
     celldict = {}
     def getter(x, y):
         return celldict.get((x, y))
@@ -102,14 +97,14 @@
                                          lltype.Float], lltype.Void))
     class FakeWarmRunnerDesc:
         rtyper = FakeRTyper()
-        cpu = None
-        get_jitcell_at_ptr = llhelper(GETTER, getter)
-        set_jitcell_at_ptr = llhelper(SETTER, setter)
+    class FakeJitDriverSD:
+        _get_jitcell_at_ptr = llhelper(GETTER, getter)
+        _set_jitcell_at_ptr = llhelper(SETTER, setter)
     #
-    state = WarmEnterState(FakeWarmRunnerDesc())
-    get_jitcell = state._make_jitcell_getter_custom(FakeJitCell)
+    state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD())
+    get_jitcell = state._make_jitcell_getter_custom()
     cell1 = get_jitcell(5, 42.5)
-    assert isinstance(cell1, FakeJitCell)
+    assert isinstance(cell1, JitCell)
     assert cell1.x == 5
     assert cell1.y == 42.5
     cell2 = get_jitcell(5, 42.5)
@@ -127,10 +122,11 @@
             future_values[j] = "float", value
     class FakeWarmRunnerDesc:
         cpu = FakeCPU()
-        red_args_types = ["int", "float"]
+    class FakeJitDriverSD:
+        _red_args_types = ["int", "float"]
         virtualizable_info = None
     #
-    state = WarmEnterState(FakeWarmRunnerDesc())
+    state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD())
     set_future_values = state.make_set_future_values()
     set_future_values(5, 42.5)
     assert future_values == {
@@ -140,19 +136,19 @@
     assert set_future_values is state.make_set_future_values()
 
 def test_make_unwrap_greenkey():
-    class FakeWarmRunnerDesc:
-        green_args_spec = [lltype.Signed, lltype.Float]
-    state = WarmEnterState(FakeWarmRunnerDesc())
+    class FakeJitDriverSD:
+        _green_args_spec = [lltype.Signed, lltype.Float]
+    state = WarmEnterState(None, FakeJitDriverSD())
     unwrap_greenkey = state.make_unwrap_greenkey()
     greenargs = unwrap_greenkey([ConstInt(42), ConstFloat(42.5)])
     assert greenargs == (42, 42.5)
     assert type(greenargs[0]) is int
 
 def test_attach_unoptimized_bridge_from_interp():
-    class FakeWarmRunnerDesc:
-        green_args_spec = [lltype.Signed, lltype.Float]
-        get_jitcell_at_ptr = None
-    state = WarmEnterState(FakeWarmRunnerDesc())
+    class FakeJitDriverSD:
+        _green_args_spec = [lltype.Signed, lltype.Float]
+        _get_jitcell_at_ptr = None
+    state = WarmEnterState(None, FakeJitDriverSD())
     get_jitcell = state.make_jitcell_getter()
     state.attach_unoptimized_bridge_from_interp([ConstInt(5),
                                                  ConstFloat(2.25)],
@@ -162,14 +158,14 @@
     assert cell1.entry_loop_token == "entry loop token"
 
 def test_make_jitdriver_callbacks_1():
-    class FakeWarmRunnerDesc:
-        can_inline_ptr = None
-        get_printable_location_ptr = None
-        confirm_enter_jit_ptr = None
-        green_args_spec = [lltype.Signed, lltype.Float]
+    class FakeJitDriverSD:
+        _green_args_spec = [lltype.Signed, lltype.Float]
+        _can_inline_ptr = None
+        _get_printable_location_ptr = None
+        _confirm_enter_jit_ptr = None
     class FakeCell:
         dont_trace_here = False
-    state = WarmEnterState(FakeWarmRunnerDesc())
+    state = WarmEnterState(None, FakeJitDriverSD())
     def jit_getter(*args):
         return FakeCell()
     state.jit_getter = jit_getter
@@ -190,11 +186,12 @@
         dont_trace_here = False
     class FakeWarmRunnerDesc:
         rtyper = None
-        green_args_spec = [lltype.Signed, lltype.Float]
-        can_inline_ptr = llhelper(CAN_INLINE, can_inline)
-        get_printable_location_ptr = None
-        confirm_enter_jit_ptr = None
-    state = WarmEnterState(FakeWarmRunnerDesc())
+    class FakeJitDriverSD:
+        _green_args_spec = [lltype.Signed, lltype.Float]
+        _can_inline_ptr = llhelper(CAN_INLINE, can_inline)
+        _get_printable_location_ptr = None
+        _confirm_enter_jit_ptr = None
+    state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD())
     def jit_getter(*args):
         return FakeCell()
     state.jit_getter = jit_getter
@@ -211,12 +208,13 @@
                                               lltype.Ptr(rstr.STR)))
     class FakeWarmRunnerDesc:
         rtyper = None
-        green_args_spec = [lltype.Signed, lltype.Float]
-        can_inline_ptr = None
-        get_printable_location_ptr = llhelper(GET_LOCATION, get_location)
-        confirm_enter_jit_ptr = None
-        get_jitcell_at_ptr = None
-    state = WarmEnterState(FakeWarmRunnerDesc())
+    class FakeJitDriverSD:
+        _green_args_spec = [lltype.Signed, lltype.Float]
+        _can_inline_ptr = None
+        _get_printable_location_ptr = llhelper(GET_LOCATION, get_location)
+        _confirm_enter_jit_ptr = None
+        _get_jitcell_at_ptr = None
+    state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD())
     state.make_jitdriver_callbacks()
     res = state.get_location_str([ConstInt(5), ConstFloat(42.5)])
     assert res == "hi there"
@@ -231,13 +229,14 @@
                                             lltype.Signed], lltype.Bool))
     class FakeWarmRunnerDesc:
         rtyper = None
-        green_args_spec = [lltype.Signed, lltype.Float]
-        can_inline_ptr = None
-        get_printable_location_ptr = None
-        confirm_enter_jit_ptr = llhelper(ENTER_JIT, confirm_enter_jit)
-        get_jitcell_at_ptr = None
+    class FakeJitDriverSD:
+        _green_args_spec = [lltype.Signed, lltype.Float]
+        _can_inline_ptr = None
+        _get_printable_location_ptr = None
+        _confirm_enter_jit_ptr = llhelper(ENTER_JIT, confirm_enter_jit)
+        _get_jitcell_at_ptr = None
 
-    state = WarmEnterState(FakeWarmRunnerDesc())
+    state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD())
     state.make_jitdriver_callbacks()
     res = state.confirm_enter_jit(5, 42.5, 3)
     assert res is True

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_ztranslation.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_ztranslation.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_ztranslation.py	Tue Jul  6 00:00:33 2010
@@ -20,6 +20,7 @@
         # - profiler
         # - full optimizer
         # - jitdriver hooks
+        # - two JITs
 
         class Frame(object):
             _virtualizable2_ = ['i']
@@ -61,14 +62,28 @@
                     frame.i -= 2
                 frame.i -= 1
             return total * 10
-        res = ll_meta_interp(f, [40], CPUClass=self.CPUClass,
+        #
+        myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x'],
+                                 can_inline = lambda *args: False)
+        def f2(g, m, x):
+            while m > 0:
+                myjitdriver2.can_enter_jit(g=g, m=m, x=x)
+                myjitdriver2.jit_merge_point(g=g, m=m, x=x)
+                m -= 1
+                x += 3
+            return x
+        #
+        def main(i, j):
+            return f(i) - f2(i+j, i, j)
+        res = ll_meta_interp(main, [40, 5], CPUClass=self.CPUClass,
                              type_system=self.type_system)
-        assert res == f(40)
-        res = rpython_ll_meta_interp(f, [40], loops=2, CPUClass=self.CPUClass,
+        assert res == main(40, 5)
+        res = rpython_ll_meta_interp(main, [40, 5], loops=2,
+                                     CPUClass=self.CPUClass,
                                      type_system=self.type_system,
                                      optimizer=OPTIMIZER_FULL,
                                      ProfilerClass=Profiler)
-        assert res == f(40)
+        assert res == main(40, 5)
 
     def test_external_exception_handling_translates(self):
         jitdriver = JitDriver(greens = [], reds = ['n', 'total'])

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/virtualizable.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/virtualizable.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/virtualizable.py	Tue Jul  6 00:00:33 2010
@@ -14,9 +14,8 @@
     TOKEN_NONE            = 0      # must be 0 -- see also x86.call_assembler
     TOKEN_TRACING_RESCALL = -1
 
-    def __init__(self, warmrunnerdesc):
+    def __init__(self, warmrunnerdesc, VTYPEPTR):
         self.warmrunnerdesc = warmrunnerdesc
-        jitdriver = warmrunnerdesc.jitdriver
         cpu = warmrunnerdesc.cpu
         if cpu.ts.name == 'ootype':
             import py
@@ -24,11 +23,6 @@
         self.cpu = cpu
         self.BoxArray = cpu.ts.BoxRef
         #
-        assert len(jitdriver.virtualizables) == 1    # for now
-        [vname] = jitdriver.virtualizables
-        index = len(jitdriver.greens) + jitdriver.reds.index(vname)
-        self.index_of_virtualizable = index
-        VTYPEPTR = warmrunnerdesc.JIT_ENTER_FUNCTYPE.ARGS[index]
         while 'virtualizable2_accessor' not in deref(VTYPEPTR)._hints:
             VTYPEPTR = cpu.ts.get_superclass(VTYPEPTR)
         self.VTYPEPTR = VTYPEPTR

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py	Tue Jul  6 00:00:33 2010
@@ -21,6 +21,7 @@
 from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
 from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler
 from pypy.jit.metainterp.jitexc import JitException
+from pypy.jit.metainterp.jitdriver import JitDriverStaticData
 from pypy.jit.codewriter import support, codewriter
 from pypy.jit.codewriter.policy import JitPolicy
 from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE
@@ -44,8 +45,9 @@
                                     no_stats = True,
                                     ProfilerClass = ProfilerClass,
                                     **kwds)
-    warmrunnerdesc.state.set_param_inlining(inline)
-    warmrunnerdesc.state.set_param_debug(debug_level)
+    for jd in warmrunnerdesc.jitdrivers_sd:
+        jd.warmstate.set_param_inlining(inline)
+        jd.warmstate.set_param_debug(debug_level)
     warmrunnerdesc.finish()
     translator.warmrunnerdesc = warmrunnerdesc    # for later debugging
 
@@ -69,11 +71,12 @@
     translator.config.translation.gc = "boehm"
     translator.config.translation.list_comprehension_operations = True
     warmrunnerdesc = WarmRunnerDesc(translator, backendopt=backendopt, **kwds)
-    warmrunnerdesc.state.set_param_threshold(3)          # for tests
-    warmrunnerdesc.state.set_param_trace_eagerness(2)    # for tests
-    warmrunnerdesc.state.set_param_trace_limit(trace_limit)
-    warmrunnerdesc.state.set_param_inlining(inline)
-    warmrunnerdesc.state.set_param_debug(debug_level)
+    for jd in warmrunnerdesc.jitdrivers_sd:
+        jd.warmstate.set_param_threshold(3)          # for tests
+        jd.warmstate.set_param_trace_eagerness(2)    # for tests
+        jd.warmstate.set_param_trace_limit(trace_limit)
+        jd.warmstate.set_param_inlining(inline)
+        jd.warmstate.set_param_debug(debug_level)
     warmrunnerdesc.finish()
     res = interp.eval_graph(graph, args)
     if not kwds.get('translate_support_code', False):
@@ -110,12 +113,11 @@
         raise Exception("no can_enter_jit found!")
     return results
 
-def find_jit_merge_point(graphs):
+def find_jit_merge_points(graphs):
     results = _find_jit_marker(graphs, 'jit_merge_point')
-    if len(results) != 1:
-        raise Exception("found %d jit_merge_points, need exactly one!" %
-                        (len(results),))
-    return results[0]
+    if not results:
+        raise Exception("no jit_merge_point found!")
+    return results
 
 def find_set_param(graphs):
     return _find_jit_marker(graphs, 'set_param')
@@ -146,8 +148,8 @@
         pyjitpl._warmrunnerdesc = self   # this is a global for debugging only!
         self.set_translator(translator)
         self.build_cpu(CPUClass, **kwds)
-        self.find_portal()
-        self.codewriter = codewriter.CodeWriter(self.cpu, self.portal_graph)
+        self.find_portals()
+        self.codewriter = codewriter.CodeWriter(self.cpu, self.jitdrivers_sd)
         if policy is None:
             policy = JitPolicy()
         policy.set_supports_floats(self.cpu.supports_floats)
@@ -158,35 +160,31 @@
             self.prejit_optimizations(policy, graphs)
 
         self.build_meta_interp(ProfilerClass)
-        self.make_args_specification()
+        self.make_args_specifications()
         #
         from pypy.jit.metainterp.virtualref import VirtualRefInfo
         vrefinfo = VirtualRefInfo(self)
         self.codewriter.setup_vrefinfo(vrefinfo)
-        if self.jitdriver.virtualizables:
-            from pypy.jit.metainterp.virtualizable import VirtualizableInfo
-            self.virtualizable_info = VirtualizableInfo(self)
-            self.codewriter.setup_virtualizable_info(self.virtualizable_info)
-        else:
-            self.virtualizable_info = None
         #
+        self.make_virtualizable_infos()
         self.make_exception_classes()
         self.make_driverhook_graphs()
-        self.make_enter_function()
-        self.rewrite_jit_merge_point(policy)
+        self.make_enter_functions()
+        self.rewrite_jit_merge_points(policy)
 
         verbose = not self.cpu.translate_support_code
         self.codewriter.make_jitcodes(verbose=verbose)
-        self.rewrite_can_enter_jit()
+        self.rewrite_can_enter_jits()
         self.rewrite_set_param()
         self.rewrite_force_virtual(vrefinfo)
         self.add_finish()
         self.metainterp_sd.finish_setup(self.codewriter, optimizer=optimizer)
 
     def finish(self):
-        vinfo = self.virtualizable_info
-        if vinfo is not None:
-            vinfo.finish()
+        vinfos = set([jd.virtualizable_info for jd in self.jitdrivers_sd])
+        for vinfo in vinfos:
+            if vinfo is not None:
+                vinfo.finish()
         if self.cpu.translate_support_code:
             self.annhelper.finish()
 
@@ -198,18 +196,27 @@
         self.rtyper = translator.rtyper
         self.gcdescr = gc.get_description(translator.config)
 
-    def find_portal(self):
+    def find_portals(self):
+        self.jitdrivers_sd = []
         graphs = self.translator.graphs
-        self.jit_merge_point_pos = find_jit_merge_point(graphs)
-        graph, block, pos = self.jit_merge_point_pos
+        for jit_merge_point_pos in find_jit_merge_points(graphs):
+            self.split_graph_and_record_jitdriver(*jit_merge_point_pos)
+        #
+        assert (len(set([jd.jitdriver for jd in self.jitdrivers_sd])) ==
+                len(self.jitdrivers_sd)), \
+                "there are multiple jit_merge_points with the same jitdriver"
+
+    def split_graph_and_record_jitdriver(self, graph, block, pos):
+        jd = JitDriverStaticData()
+        jd._jit_merge_point_pos = (graph, block, pos)
         op = block.operations[pos]
         args = op.args[2:]
         s_binding = self.translator.annotator.binding
-        self.portal_args_s = [s_binding(v) for v in args]
+        jd._portal_args_s = [s_binding(v) for v in args]
         graph = copygraph(graph)
         graph.startblock.isstartblock = False
-        graph.startblock = support.split_before_jit_merge_point(
-            *find_jit_merge_point([graph]))
+        [jmpp] = find_jit_merge_points([graph])
+        graph.startblock = support.split_before_jit_merge_point(*jmpp)
         graph.startblock.isstartblock = True
         # a crash in the following checkgraph() means that you forgot
         # to list some variable in greens=[] or reds=[] in JitDriver.
@@ -218,12 +225,16 @@
             assert isinstance(v, Variable)
         assert len(dict.fromkeys(graph.getargs())) == len(graph.getargs())
         self.translator.graphs.append(graph)
-        self.portal_graph = graph
+        jd.portal_graph = graph
         # it's a bit unbelievable to have a portal without func
         assert hasattr(graph, "func")
         graph.func._dont_inline_ = True
         graph.func._jit_unroll_safe_ = True
-        self.jitdriver = block.operations[pos].args[1].value
+        jd.jitdriver = block.operations[pos].args[1].value
+        jd.portal_runner_ptr = "<not set so far>"
+        jd.result_type = history.getkind(jd.portal_graph.getreturnvar()
+                                         .concretetype)[0]
+        self.jitdrivers_sd.append(jd)
 
     def check_access_directly_sanity(self, graphs):
         from pypy.translator.backendopt.inline import collect_called_graphs
@@ -268,6 +279,27 @@
                                                   ProfilerClass=ProfilerClass,
                                                   warmrunnerdesc=self)
 
+    def make_virtualizable_infos(self):
+        vinfos = {}
+        for jd in self.jitdrivers_sd:
+            if not jd.jitdriver.virtualizables:
+                jd.virtualizable_info = None
+                jd.index_of_virtualizable = -1
+                continue
+            #
+            jitdriver = jd.jitdriver
+            assert len(jitdriver.virtualizables) == 1    # for now
+            [vname] = jitdriver.virtualizables
+            # XXX skip the Voids here too
+            jd.index_of_virtualizable = jitdriver.reds.index(vname)
+            #
+            index = jd.num_green_args + jd.index_of_virtualizable
+            VTYPEPTR = jd._JIT_ENTER_FUNCTYPE.ARGS[index]
+            if VTYPEPTR not in vinfos:
+                from pypy.jit.metainterp.virtualizable import VirtualizableInfo
+                vinfos[VTYPEPTR] = VirtualizableInfo(self, VTYPEPTR)
+            jd.virtualizable_info = vinfos[VTYPEPTR]
+
     def make_exception_classes(self):
 
         class DoneWithThisFrameVoid(JitException):
@@ -317,6 +349,8 @@
                     self.green_int, self.green_ref, self.green_float,
                     self.red_int, self.red_ref, self.red_float)
 
+        # XXX there is no point any more to not just have the exceptions
+        # as globals
         self.DoneWithThisFrameVoid = DoneWithThisFrameVoid
         self.DoneWithThisFrameInt = DoneWithThisFrameInt
         self.DoneWithThisFrameRef = DoneWithThisFrameRef
@@ -330,11 +364,15 @@
         self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef
         self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally
 
-    def make_enter_function(self):
+    def make_enter_functions(self):
+        for jd in self.jitdrivers_sd:
+            self.make_enter_function(jd)
+
+    def make_enter_function(self, jd):
         from pypy.jit.metainterp.warmstate import WarmEnterState
-        state = WarmEnterState(self)
+        state = WarmEnterState(self, jd)
         maybe_compile_and_run = state.make_entry_point()
-        self.state = state
+        jd.warmstate = state
 
         def crash_in_jit(e):
             if not we_are_translated():
@@ -359,15 +397,16 @@
             def maybe_enter_jit(*args):
                 maybe_compile_and_run(*args)
             maybe_enter_jit._always_inline_ = True
-        self.maybe_enter_jit_fn = maybe_enter_jit
+        jd._maybe_enter_jit_fn = maybe_enter_jit
 
-        can_inline = self.state.can_inline_greenargs
+        can_inline = state.can_inline_greenargs
+        num_green_args = jd.num_green_args
         def maybe_enter_from_start(*args):
-            if can_inline is not None and not can_inline(*args[:self.num_green_args]):
+            if can_inline is not None and not can_inline(*args[:num_green_args]):
                 maybe_compile_and_run(*args)
         maybe_enter_from_start._always_inline_ = True
-        self.maybe_enter_from_start_fn = maybe_enter_from_start
-        
+        jd._maybe_enter_from_start_fn = maybe_enter_from_start
+
     def make_driverhook_graphs(self):
         from pypy.rlib.jit import BaseJitCell
         bk = self.rtyper.annotator.bookkeeper
@@ -378,22 +417,23 @@
         s_Str = annmodel.SomeString()
         #
         annhelper = MixLevelHelperAnnotator(self.translator.rtyper)
-        self.set_jitcell_at_ptr = self._make_hook_graph(
-            annhelper, self.jitdriver.set_jitcell_at, annmodel.s_None,
-            s_BaseJitCell_not_None)
-        self.get_jitcell_at_ptr = self._make_hook_graph(
-            annhelper, self.jitdriver.get_jitcell_at, s_BaseJitCell_or_None)
-        self.can_inline_ptr = self._make_hook_graph(
-            annhelper, self.jitdriver.can_inline, annmodel.s_Bool)
-        self.get_printable_location_ptr = self._make_hook_graph(
-            annhelper, self.jitdriver.get_printable_location, s_Str)
-        self.confirm_enter_jit_ptr = self._make_hook_graph(
-            annhelper, self.jitdriver.confirm_enter_jit, annmodel.s_Bool,
-            onlygreens=False)
+        for jd in self.jitdrivers_sd:
+            jd._set_jitcell_at_ptr = self._make_hook_graph(jd,
+                annhelper, jd.jitdriver.set_jitcell_at, annmodel.s_None,
+                s_BaseJitCell_not_None)
+            jd._get_jitcell_at_ptr = self._make_hook_graph(jd,
+                annhelper, jd.jitdriver.get_jitcell_at, s_BaseJitCell_or_None)
+            jd._can_inline_ptr = self._make_hook_graph(jd,
+                annhelper, jd.jitdriver.can_inline, annmodel.s_Bool)
+            jd._get_printable_location_ptr = self._make_hook_graph(jd,
+                annhelper, jd.jitdriver.get_printable_location, s_Str)
+            jd._confirm_enter_jit_ptr = self._make_hook_graph(jd,
+                annhelper, jd.jitdriver.confirm_enter_jit, annmodel.s_Bool,
+                onlygreens=False)
         annhelper.finish()
 
-    def _make_hook_graph(self, annhelper, func, s_result, s_first_arg=None,
-                         onlygreens=True):
+    def _make_hook_graph(self, jitdriver_sd, annhelper, func,
+                         s_result, s_first_arg=None, onlygreens=True):
         if func is None:
             return None
         #
@@ -401,38 +441,57 @@
         if s_first_arg is not None:
             extra_args_s.append(s_first_arg)
         #
-        args_s = self.portal_args_s
+        args_s = jitdriver_sd._portal_args_s
         if onlygreens:
-            args_s = args_s[:len(self.green_args_spec)]
+            args_s = args_s[:len(jitdriver_sd._green_args_spec)]
         graph = annhelper.getgraph(func, extra_args_s + args_s, s_result)
         funcptr = annhelper.graph2delayed(graph)
         return funcptr
 
-    def make_args_specification(self):
-        graph, block, index = self.jit_merge_point_pos
+    def make_args_specifications(self):
+        for jd in self.jitdrivers_sd:
+            self.make_args_specification(jd)
+
+    def make_args_specification(self, jd):
+        graph, block, index = jd._jit_merge_point_pos
         op = block.operations[index]
         greens_v, reds_v = support.decode_hp_hint_args(op)
         ALLARGS = [v.concretetype for v in (greens_v + reds_v)]
-        self.green_args_spec = [v.concretetype for v in greens_v]
-        self.red_args_types = [history.getkind(v.concretetype) for v in reds_v]
-        self.num_green_args = len(self.green_args_spec)
+        jd._green_args_spec = [v.concretetype for v in greens_v]
+        jd._red_args_types = [history.getkind(v.concretetype) for v in reds_v]
+        jd.num_green_args = len(jd._green_args_spec)
         RESTYPE = graph.getreturnvar().concretetype
-        (self.JIT_ENTER_FUNCTYPE,
-         self.PTR_JIT_ENTER_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, lltype.Void)
-        (self.PORTAL_FUNCTYPE,
-         self.PTR_PORTAL_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, RESTYPE)
-        (_, self.PTR_ASSEMBLER_HELPER_FUNCTYPE) = self.cpu.ts.get_FuncType(
+        (jd._JIT_ENTER_FUNCTYPE,
+         jd._PTR_JIT_ENTER_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, lltype.Void)
+        (jd._PORTAL_FUNCTYPE,
+         jd._PTR_PORTAL_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, RESTYPE)
+        (_, jd._PTR_ASSEMBLER_HELPER_FUNCTYPE) = self.cpu.ts.get_FuncType(
             [lltype.Signed, llmemory.GCREF], RESTYPE)
 
-    def rewrite_can_enter_jit(self):
-        FUNC = self.JIT_ENTER_FUNCTYPE
-        FUNCPTR = self.PTR_JIT_ENTER_FUNCTYPE
-        jit_enter_fnptr = self.helper_func(FUNCPTR, self.maybe_enter_jit_fn)
+    def rewrite_can_enter_jits(self):
+        can_enter_jits = find_can_enter_jit(self.translator.graphs)
+        sublists = {}
+        for jd in self.jitdrivers_sd:
+            sublists[jd.jitdriver] = []
+        for graph, block, index in can_enter_jits:
+            op = block.operations[index]
+            jitdriver = op.args[1].value
+            assert jitdriver in sublists, \
+                   "can_enter_jit with no matching jit_merge_point"
+            sublists[jitdriver].append((graph, block, index))
+        for jd in self.jitdrivers_sd:
+            sublist = sublists[jd.jitdriver]
+            assert len(sublist) > 0, \
+                   "found no can_enter_jit for %r" % (jd.jitdriver,)
+            self.rewrite_can_enter_jit(jd, sublist)
+
+    def rewrite_can_enter_jit(self, jd, can_enter_jits):
+        FUNC = jd._JIT_ENTER_FUNCTYPE
+        FUNCPTR = jd._PTR_JIT_ENTER_FUNCTYPE
+        jit_enter_fnptr = self.helper_func(FUNCPTR, jd._maybe_enter_jit_fn)
 
-        graphs = self.translator.graphs
-        can_enter_jits = find_can_enter_jit(graphs)
         for graph, block, index in can_enter_jits:
-            if graph is self.jit_merge_point_pos[0]:
+            if graph is jd._jit_merge_point_pos[0]:
                 continue
 
             op = block.operations[index]
@@ -455,7 +514,11 @@
         graph = self.annhelper.getgraph(func, args_s, s_result)
         return self.annhelper.graph2delayed(graph, FUNC)
 
-    def rewrite_jit_merge_point(self, policy):
+    def rewrite_jit_merge_points(self, policy):
+        for jd in self.jitdrivers_sd:
+            self.rewrite_jit_merge_point(jd, policy)
+
+    def rewrite_jit_merge_point(self, jd, policy):
         #
         # Mutate the original portal graph from this:
         #
@@ -486,9 +549,9 @@
         #           while 1:
         #               more stuff
         #
-        origportalgraph = self.jit_merge_point_pos[0]
-        portalgraph = self.portal_graph
-        PORTALFUNC = self.PORTAL_FUNCTYPE
+        origportalgraph = jd._jit_merge_point_pos[0]
+        portalgraph = jd.portal_graph
+        PORTALFUNC = jd._PORTAL_FUNCTYPE
 
         # ____________________________________________________________
         # Prepare the portal_runner() helper
@@ -496,12 +559,12 @@
         from pypy.jit.metainterp.warmstate import specialize_value
         portal_ptr = self.cpu.ts.functionptr(PORTALFUNC, 'portal',
                                          graph = portalgraph)
-        self.portal_ptr = portal_ptr
+        jd._portal_ptr = portal_ptr
         #
         portalfunc_ARGS = []
         nums = {}
         for i, ARG in enumerate(PORTALFUNC.ARGS):
-            if i < len(self.jitdriver.greens):
+            if i < len(jd.jitdriver.greens):
                 color = 'green'
             else:
                 color = 'red'
@@ -519,7 +582,7 @@
         def ll_portal_runner(*args):
             while 1:
                 try:
-                    self.maybe_enter_from_start_fn(*args)
+                    jd._maybe_enter_from_start_fn(*args)
                     return support.maybe_on_top_of_llinterp(rtyper,
                                                       portal_ptr)(*args)
                 except self.ContinueRunningNormally, e:
@@ -548,27 +611,26 @@
                         value = cast_base_ptr_to_instance(Exception, value)
                         raise Exception, value
 
-        self.ll_portal_runner = ll_portal_runner # for debugging
-        self.portal_runner_ptr = self.helper_func(self.PTR_PORTAL_FUNCTYPE,
-                                                  ll_portal_runner)
+        jd._ll_portal_runner = ll_portal_runner # for debugging
+        jd.portal_runner_ptr = self.helper_func(jd._PTR_PORTAL_FUNCTYPE,
+                                                ll_portal_runner)
+        jd.portal_runner_adr = llmemory.cast_ptr_to_adr(jd.portal_runner_ptr)
         self.cpu.portal_calldescr = self.cpu.calldescrof(
-            self.PTR_PORTAL_FUNCTYPE.TO,
-            self.PTR_PORTAL_FUNCTYPE.TO.ARGS,
-            self.PTR_PORTAL_FUNCTYPE.TO.RESULT)
-        self.codewriter.setup_portal_runner_ptr(self.portal_runner_ptr)
+            jd._PTR_PORTAL_FUNCTYPE.TO,
+            jd._PTR_PORTAL_FUNCTYPE.TO.ARGS,
+            jd._PTR_PORTAL_FUNCTYPE.TO.RESULT)
 
-        vinfo = self.virtualizable_info
+        vinfo = jd.virtualizable_info
 
         def assembler_call_helper(failindex, virtualizableref):
             fail_descr = self.cpu.get_fail_descr_from_number(failindex)
             while True:
+                if vinfo is not None:
+                    virtualizable = lltype.cast_opaque_ptr(
+                        vinfo.VTYPEPTR, virtualizableref)
+                    vinfo.reset_vable_token(virtualizable)
                 try:
-                    if vinfo is not None:
-                        virtualizable = lltype.cast_opaque_ptr(
-                            vinfo.VTYPEPTR, virtualizableref)
-                        vinfo.reset_vable_token(virtualizable)
-                    loop_token = fail_descr.handle_fail(self.metainterp_sd)
-                    fail_descr = self.cpu.execute_token(loop_token)
+                    loop_token = fail_descr.handle_fail(self.metainterp_sd, jd)
                 except self.ContinueRunningNormally, e:
                     args = ()
                     for ARGTYPE, attrname, count in portalfunc_ARGS:
@@ -595,29 +657,26 @@
                     else:
                         value = cast_base_ptr_to_instance(Exception, value)
                         raise Exception, value
+                fail_descr = self.cpu.execute_token(loop_token)
 
-        self.assembler_call_helper = assembler_call_helper # for debugging
-        self.cpu.assembler_helper_ptr = self.helper_func(
-            self.PTR_ASSEMBLER_HELPER_FUNCTYPE,
+        jd._assembler_call_helper = assembler_call_helper # for debugging
+        jd._assembler_helper_ptr = self.helper_func(
+            jd._PTR_ASSEMBLER_HELPER_FUNCTYPE,
             assembler_call_helper)
-        # XXX a bit ugly sticking
+        jd.assembler_helper_adr = llmemory.cast_ptr_to_adr(
+            jd._assembler_helper_ptr)
         if vinfo is not None:
-            self.cpu.index_of_virtualizable = (vinfo.index_of_virtualizable -
-                                               self.num_green_args)
-            self.cpu.vable_token_descr = vinfo.vable_token_descr
-        else:
-            self.cpu.index_of_virtualizable = -1
-            self.cpu.vable_token_descr = None
+            jd.vable_token_descr = vinfo.vable_token_descr
 
         # ____________________________________________________________
         # Now mutate origportalgraph to end with a call to portal_runner_ptr
         #
-        _, origblock, origindex = self.jit_merge_point_pos
+        _, origblock, origindex = jd._jit_merge_point_pos
         op = origblock.operations[origindex]
         assert op.opname == 'jit_marker'
         assert op.args[0].value == 'jit_merge_point'
         greens_v, reds_v = support.decode_hp_hint_args(op)
-        vlist = [Constant(self.portal_runner_ptr, self.PTR_PORTAL_FUNCTYPE)]
+        vlist = [Constant(jd.portal_runner_ptr, jd._PTR_PORTAL_FUNCTYPE)]
         vlist += greens_v
         vlist += reds_v
         v_result = Variable()
@@ -627,6 +686,18 @@
         origblock.operations.append(newop)
         origblock.exitswitch = None
         origblock.recloseblock(Link([v_result], origportalgraph.returnblock))
+        #
+        # Also kill any can_enter_jit left behind (example: see
+        # test_jitdriver.test_simple, which has a can_enter_jit in
+        # loop1's origportalgraph)
+        can_enter_jits = _find_jit_marker([origportalgraph], 'can_enter_jit')
+        for _, block, i in can_enter_jits:
+            op = block.operations[i]
+            assert op.opname == 'jit_marker'
+            block.operations[i] = SpaceOperation('same_as',
+                                                 [Constant(None, lltype.Void)],
+                                                 op.result)
+        #
         checkgraph(origportalgraph)
 
     def add_finish(self):
@@ -644,8 +715,8 @@
         graphs = self.translator.graphs
         _, PTR_SET_PARAM_FUNCTYPE = self.cpu.ts.get_FuncType([lltype.Signed],
                                                              lltype.Void)
-        def make_closure(fullfuncname):
-            state = self.state
+        def make_closure(jd, fullfuncname):
+            state = jd.warmstate
             def closure(i):
                 getattr(state, fullfuncname)(i)
             funcptr = self.helper_func(PTR_SET_PARAM_FUNCTYPE, closure)
@@ -653,12 +724,17 @@
         #
         for graph, block, i in find_set_param(graphs):
             op = block.operations[i]
-            assert op.args[1].value == self.jitdriver
+            for jd in self.jitdrivers_sd:
+                if jd.jitdriver is op.args[1].value:
+                    break
+            else:
+                assert 0, "jitdriver of set_param() not found"
             funcname = op.args[2].value
-            if funcname not in closures:
-                closures[funcname] = make_closure('set_param_' + funcname)
+            key = jd, funcname
+            if key not in closures:
+                closures[key] = make_closure(jd, 'set_param_' + funcname)
             op.opname = 'direct_call'
-            op.args[:3] = [closures[funcname]]
+            op.args[:3] = [closures[key]]
 
     def rewrite_force_virtual(self, vrefinfo):
         if self.cpu.ts.name != 'lltype':

Modified: pypy/branch/fast-forward/pypy/jit/metainterp/warmstate.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/metainterp/warmstate.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/metainterp/warmstate.py	Tue Jul  6 00:00:33 2010
@@ -1,7 +1,7 @@
 import sys
 from pypy.rpython.lltypesystem import lltype, llmemory, rstr
 from pypy.rpython.ootypesystem import ootype
-from pypy.rpython.annlowlevel import hlstr, cast_base_ptr_to_instance
+from pypy.rpython.annlowlevel import hlstr, llstr, cast_base_ptr_to_instance
 from pypy.rpython.annlowlevel import cast_object_to_ptr
 from pypy.rlib.objectmodel import specialize, we_are_translated, r_dict
 from pypy.rlib.rarithmetic import intmask
@@ -120,6 +120,16 @@
     else:
         assert False
 
+class JitCell(BaseJitCell):
+    # the counter can mean the following things:
+    #     counter >=  0: not yet traced, wait till threshold is reached
+    #     counter == -1: there is an entry bridge for this cell
+    #     counter == -2: tracing is currently going on for this cell
+    counter = 0
+    compiled_merge_points = None
+    dont_trace_here = False
+    entry_loop_token = None
+
 # ____________________________________________________________
 
 
@@ -127,9 +137,10 @@
     THRESHOLD_LIMIT = sys.maxint // 2
     default_jitcell_dict = None
 
-    def __init__(self, warmrunnerdesc):
+    def __init__(self, warmrunnerdesc, jitdriver_sd):
         "NOT_RPYTHON"
         self.warmrunnerdesc = warmrunnerdesc
+        self.jitdriver_sd = jitdriver_sd
         try:
             self.profiler = warmrunnerdesc.metainterp_sd.profiler
         except AttributeError:       # for tests
@@ -176,8 +187,7 @@
         cell = self.jit_cell_at_key(greenkey)
         cell.dont_trace_here = True
         debug_start("jit-disableinlining")
-        sd = self.warmrunnerdesc.metainterp_sd
-        loc = sd.state.get_location_str(greenkey)
+        loc = self.get_location_str(greenkey)
         debug_print("disabled inlining", loc)
         debug_stop("jit-disableinlining")
 
@@ -195,8 +205,10 @@
             return self.maybe_compile_and_run
 
         metainterp_sd = self.warmrunnerdesc.metainterp_sd
-        vinfo = self.warmrunnerdesc.virtualizable_info
-        num_green_args = self.warmrunnerdesc.num_green_args
+        jitdriver_sd = self.jitdriver_sd
+        vinfo = jitdriver_sd.virtualizable_info
+        index_of_virtualizable = jitdriver_sd.index_of_virtualizable
+        num_green_args = jitdriver_sd.num_green_args
         get_jitcell = self.make_jitcell_getter()
         set_future_values = self.make_set_future_values()
         self.make_jitdriver_callbacks()
@@ -206,7 +218,6 @@
             """Entry point to the JIT.  Called at the point with the
             can_enter_jit() hint.
             """
-            globaldata = metainterp_sd.globaldata
             if NonConstant(False):
                 # make sure we always see the saner optimizer from an
                 # annotation point of view, otherwise we get lots of
@@ -214,7 +225,7 @@
                 self.set_param_optimizer(OPTIMIZER_FULL)
 
             if vinfo is not None:
-                virtualizable = args[vinfo.index_of_virtualizable]
+                virtualizable = args[num_green_args + index_of_virtualizable]
                 virtualizable = vinfo.cast_to_vtype(virtualizable)
             else:
                 virtualizable = None
@@ -234,11 +245,12 @@
                     return
                 # bound reached; start tracing
                 from pypy.jit.metainterp.pyjitpl import MetaInterp
-                metainterp = MetaInterp(metainterp_sd)
+                metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
                 # set counter to -2, to mean "tracing in effect"
                 cell.counter = -2
                 try:
-                    loop_token = metainterp.compile_and_run_once(*args)
+                    loop_token = metainterp.compile_and_run_once(jitdriver_sd,
+                                                                 *args)
                 finally:
                     if cell.counter == -2:
                         cell.counter = 0
@@ -264,7 +276,8 @@
                 metainterp_sd.profiler.end_running()
                 if vinfo is not None:
                     vinfo.reset_vable_token(virtualizable)
-                loop_token = fail_descr.handle_fail(metainterp_sd)
+                loop_token = fail_descr.handle_fail(metainterp_sd,
+                                                    jitdriver_sd)
        
         maybe_compile_and_run._dont_inline_ = True
         self.maybe_compile_and_run = maybe_compile_and_run
@@ -277,8 +290,8 @@
         if hasattr(self, 'unwrap_greenkey'):
             return self.unwrap_greenkey
         #
-        warmrunnerdesc = self.warmrunnerdesc
-        green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec)
+        jitdriver_sd = self.jitdriver_sd
+        green_args_spec = unrolling_iterable(jitdriver_sd._green_args_spec)
         #
         def unwrap_greenkey(greenkey):
             greenargs = ()
@@ -302,20 +315,10 @@
         if hasattr(self, 'jit_getter'):
             return self.jit_getter
         #
-        class JitCell(BaseJitCell):
-            # the counter can mean the following things:
-            #     counter >=  0: not yet traced, wait till threshold is reached
-            #     counter == -1: there is an entry bridge for this cell
-            #     counter == -2: tracing is currently going on for this cell
-            counter = 0
-            compiled_merge_points = None
-            dont_trace_here = False
-            entry_loop_token = None
-        #
-        if self.warmrunnerdesc.get_jitcell_at_ptr is None:
-            jit_getter = self._make_jitcell_getter_default(JitCell)
+        if self.jitdriver_sd._get_jitcell_at_ptr is None:
+            jit_getter = self._make_jitcell_getter_default()
         else:
-            jit_getter = self._make_jitcell_getter_custom(JitCell)
+            jit_getter = self._make_jitcell_getter_custom()
         #
         unwrap_greenkey = self.make_unwrap_greenkey()
         #
@@ -327,10 +330,10 @@
         #
         return jit_getter
 
-    def _make_jitcell_getter_default(self, JitCell):
+    def _make_jitcell_getter_default(self):
         "NOT_RPYTHON"
-        warmrunnerdesc = self.warmrunnerdesc
-        green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec)
+        jitdriver_sd = self.jitdriver_sd
+        green_args_spec = unrolling_iterable(jitdriver_sd._green_args_spec)
         #
         def comparekey(greenargs1, greenargs2):
             i = 0
@@ -361,11 +364,11 @@
             return cell
         return get_jitcell
 
-    def _make_jitcell_getter_custom(self, JitCell):
+    def _make_jitcell_getter_custom(self):
         "NOT_RPYTHON"
         rtyper = self.warmrunnerdesc.rtyper
-        get_jitcell_at_ptr = self.warmrunnerdesc.get_jitcell_at_ptr
-        set_jitcell_at_ptr = self.warmrunnerdesc.set_jitcell_at_ptr
+        get_jitcell_at_ptr = self.jitdriver_sd._get_jitcell_at_ptr
+        set_jitcell_at_ptr = self.jitdriver_sd._set_jitcell_at_ptr
         lltohlhack = {}
         #
         def get_jitcell(*greenargs):
@@ -415,9 +418,10 @@
             return self.set_future_values
 
         warmrunnerdesc = self.warmrunnerdesc
+        jitdriver_sd   = self.jitdriver_sd
         cpu = warmrunnerdesc.cpu
-        vinfo = warmrunnerdesc.virtualizable_info
-        red_args_types = unrolling_iterable(warmrunnerdesc.red_args_types)
+        vinfo = jitdriver_sd.virtualizable_info
+        red_args_types = unrolling_iterable(jitdriver_sd._red_args_types)
         #
         def set_future_values(*redargs):
             i = 0
@@ -428,8 +432,9 @@
                 set_future_values_from_vinfo(*redargs)
         #
         if vinfo is not None:
-            i0 = len(warmrunnerdesc.red_args_types)
-            num_green_args = warmrunnerdesc.num_green_args
+            i0 = len(jitdriver_sd._red_args_types)
+            num_green_args = jitdriver_sd.num_green_args
+            index_of_virtualizable = jitdriver_sd.index_of_virtualizable
             vable_static_fields = unrolling_iterable(
                 zip(vinfo.static_extra_types, vinfo.static_fields))
             vable_array_fields = unrolling_iterable(
@@ -439,8 +444,7 @@
             #
             def set_future_values_from_vinfo(*redargs):
                 i = i0
-                virtualizable = redargs[vinfo.index_of_virtualizable -
-                                        num_green_args]
+                virtualizable = redargs[index_of_virtualizable]
                 virtualizable = vinfo.cast_to_vtype(virtualizable)
                 for typecode, fieldname in vable_static_fields:
                     x = getattr(virtualizable, fieldname)
@@ -464,8 +468,9 @@
         if hasattr(self, 'get_location_str'):
             return
         #
-        can_inline_ptr = self.warmrunnerdesc.can_inline_ptr
+        can_inline_ptr = self.jitdriver_sd._can_inline_ptr
         unwrap_greenkey = self.make_unwrap_greenkey()
+        jit_getter = self.make_jitcell_getter()
         if can_inline_ptr is None:
             def can_inline_callable(*greenargs):
                 # XXX shouldn't it be False by default?
@@ -477,7 +482,7 @@
                 fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr)
                 return fn(*greenargs)
         def can_inline(*greenargs):
-            cell = self.jit_getter(*greenargs)
+            cell = jit_getter(*greenargs)
             if cell.dont_trace_here:
                 return False
             return can_inline_callable(*greenargs)
@@ -486,21 +491,25 @@
             greenargs = unwrap_greenkey(greenkey)
             return can_inline(*greenargs)
         self.can_inline_callable = can_inline_greenkey
-        
-        get_jitcell = self.make_jitcell_getter()
+
         def get_assembler_token(greenkey):
             greenargs = unwrap_greenkey(greenkey)
-            cell = get_jitcell(*greenargs)
+            cell = jit_getter(*greenargs)
             if cell.counter >= 0:
                 return None
             return cell.entry_loop_token
         self.get_assembler_token = get_assembler_token
         
         #
-        get_location_ptr = self.warmrunnerdesc.get_printable_location_ptr
+        get_location_ptr = self.jitdriver_sd._get_printable_location_ptr
         if get_location_ptr is None:
+            missing = '(no jitdriver.get_printable_location!)'
+            missingll = llstr(missing)
             def get_location_str(greenkey):
-                return '(no jitdriver.get_printable_location!)'
+                if we_are_translated():
+                    return missingll
+                else:
+                    return missing
         else:
             rtyper = self.warmrunnerdesc.rtyper
             unwrap_greenkey = self.make_unwrap_greenkey()
@@ -514,7 +523,7 @@
                 return res
         self.get_location_str = get_location_str
         #
-        confirm_enter_jit_ptr = self.warmrunnerdesc.confirm_enter_jit_ptr
+        confirm_enter_jit_ptr = self.jitdriver_sd._confirm_enter_jit_ptr
         if confirm_enter_jit_ptr is None:
             def confirm_enter_jit(*args):
                 return True

Modified: pypy/branch/fast-forward/pypy/jit/tl/spli/targetspli.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/tl/spli/targetspli.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/tl/spli/targetspli.py	Tue Jul  6 00:00:33 2010
@@ -31,7 +31,7 @@
 
 def jitpolicy(driver):
     """Returns the JIT policy to use when translating."""
-    from pypy.jit.metainterp.policy import JitPolicy
+    from pypy.jit.codewriter.policy import JitPolicy
     return JitPolicy()
 
 if __name__ == '__main__':

Modified: pypy/branch/fast-forward/pypy/jit/tl/targettlc.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/tl/targettlc.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/tl/targettlc.py	Tue Jul  6 00:00:33 2010
@@ -2,7 +2,7 @@
 import py
 py.path.local(__file__)
 from pypy.jit.tl.tlc import interp, interp_nonjit, ConstantPool
-from pypy.jit.metainterp.policy import JitPolicy
+from pypy.jit.codewriter.policy import JitPolicy
 from pypy.jit.backend.hlinfo import highleveljitinfo
 
 

Modified: pypy/branch/fast-forward/pypy/jit/tl/targettlr.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/tl/targettlr.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/tl/targettlr.py	Tue Jul  6 00:00:33 2010
@@ -33,7 +33,7 @@
 
 # ____________________________________________________________
 
-from pypy.jit.metainterp.policy import JitPolicy
+from pypy.jit.codewriter.policy import JitPolicy
 
 def jitpolicy(driver):
     return JitPolicy()

Modified: pypy/branch/fast-forward/pypy/jit/tl/tinyframe/targettinyframe.py
==============================================================================
--- pypy/branch/fast-forward/pypy/jit/tl/tinyframe/targettinyframe.py	(original)
+++ pypy/branch/fast-forward/pypy/jit/tl/tinyframe/targettinyframe.py	Tue Jul  6 00:00:33 2010
@@ -1,6 +1,6 @@
 
 from pypy.jit.tl.tinyframe.tinyframe import main
-from pypy.jit.metainterp.policy import JitPolicy
+from pypy.jit.codewriter.policy import JitPolicy
 
 def jitpolicy(driver):
     return JitPolicy()

Modified: pypy/branch/fast-forward/pypy/module/__builtin__/functional.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/__builtin__/functional.py	(original)
+++ pypy/branch/fast-forward/pypy/module/__builtin__/functional.py	Tue Jul  6 00:00:33 2010
@@ -176,10 +176,75 @@
     if not collections_w:
         msg = "map() requires at least two arguments"
         raise OperationError(space.w_TypeError, space.wrap(msg))
-    num_collections = len(collections_w)
     none_func = space.is_w(w_func, space.w_None)
-    if none_func and num_collections == 1:
-        return space.call_function(space.w_list, collections_w[0])
+    if len(collections_w) == 1:
+        w_collection = collections_w[0]
+        if none_func:
+            result_w = space.unpackiterable(w_collection)
+        else:
+            result_w = map_single_collection(space, w_func, w_collection)
+    else:
+        result_w = map_multiple_collections(space, w_func, collections_w,
+                                            none_func)
+    return space.newlist(result_w)
+map.unwrap_spec = [ObjSpace, W_Root, "args_w"]
+
+def map_single_collection(space, w_func, w_collection):
+    """Special case for 'map(func, coll)', where 'func' is not None and there
+    is only one 'coll' argument."""
+    w_iter = space.iter(w_collection)
+    # xxx special hacks for speed
+    from pypy.interpreter import function, pycode
+    if isinstance(w_func, function.Function):
+        # xxx compatibility issue: what if func_code is modified in the
+        # middle of running map()??  That's far too obscure for me to care...
+        code = w_func.getcode()
+        fast_natural_arity = code.fast_natural_arity
+        if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL):
+            assert isinstance(code, pycode.PyCode)
+            return map_single_user_function(code, w_func, w_iter)
+    # /xxx end of special hacks
+    return map_single_other_callable(space, w_func, w_iter)
+
+def map_single_other_callable(space, w_func, w_iter):
+    result_w = []
+    while True:
+        try:
+            w_item = space.next(w_iter)
+        except OperationError, e:
+            if not e.match(space, space.w_StopIteration):
+                raise
+            break
+        result_w.append(space.call_function(w_func, w_item))
+    return result_w
+map_single_other_callable._dont_inline_ = True
+
+from pypy.rlib.jit import JitDriver
+mapjitdriver = JitDriver(greens = ['code'],
+                         reds = ['w_func', 'w_iter', 'result_w'],
+                         can_inline = lambda *args: False)
+def map_single_user_function(code, w_func, w_iter):
+    result_w = []
+    while True:
+        mapjitdriver.can_enter_jit(code=code, w_func=w_func,
+                                   w_iter=w_iter, result_w=result_w)
+        mapjitdriver.jit_merge_point(code=code, w_func=w_func,
+                                     w_iter=w_iter, result_w=result_w)
+        space = w_func.space
+        try:
+            w_item = space.next(w_iter)
+        except OperationError, e:
+            if not e.match(space, space.w_StopIteration):
+                raise
+            break
+        new_frame = space.createframe(code, w_func.w_func_globals,
+                                      w_func.closure)
+        new_frame.fastlocals_w[0] = w_item
+        w_res = new_frame.run()
+        result_w.append(w_res)
+    return result_w
+
+def map_multiple_collections(space, w_func, collections_w, none_func):
     result_w = []
     iterators_w = [space.iter(w_seq) for w_seq in collections_w]
     num_iterators = len(iterators_w)
@@ -196,16 +261,15 @@
                     iterators_w[i] = None
                 else:
                     cont = True
-        if cont:
-            w_args = space.newtuple(args_w)
-            if none_func:
-                result_w.append(w_args)
-            else:
-                w_res = space.call(w_func, w_args)
-                result_w.append(w_res)
+        if not cont:
+            break
+        w_args = space.newtuple(args_w)
+        if none_func:
+            w_res = w_args
         else:
-            return space.newlist(result_w)
-map.unwrap_spec = [ObjSpace, W_Root, "args_w"]
+            w_res = space.call(w_func, w_args)
+        result_w.append(w_res)
+    return result_w
 
 def sum(space, w_sequence, w_start=None):
     if space.is_w(w_start, space.w_None):

Modified: pypy/branch/fast-forward/pypy/module/__builtin__/test/test_functional.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/__builtin__/test/test_functional.py	(original)
+++ pypy/branch/fast-forward/pypy/module/__builtin__/test/test_functional.py	Tue Jul  6 00:00:33 2010
@@ -6,6 +6,9 @@
    def test_trivial_map_one_seq(self):
       assert map(lambda x: x+2, [1, 2, 3, 4]) == [3, 4, 5, 6]
 
+   def test_trivial_map_one_seq_2(self):
+      assert map(str, [1, 2, 3, 4]) == ['1', '2', '3', '4']
+
    def test_trivial_map_two_seq(self):
       assert map(lambda x,y: x+y,
                            [1, 2, 3, 4],[1, 2, 3, 4]) == (

Modified: pypy/branch/fast-forward/pypy/module/_codecs/__init__.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_codecs/__init__.py	(original)
+++ pypy/branch/fast-forward/pypy/module/_codecs/__init__.py	Tue Jul  6 00:00:33 2010
@@ -3,22 +3,43 @@
 from pypy.module._codecs import interp_codecs
 
 class Module(MixedModule):
-    appleveldefs = {
-         '__doc__' :  'app_codecs.__doc__',
-         '__name__' :  'app_codecs.__name__',
-         'charmap_encode' :  'app_codecs.charmap_encode',
-         'escape_decode' :  'app_codecs.escape_decode',
-         'escape_encode' :  'app_codecs.escape_encode',
-         'raw_unicode_escape_decode' :  'app_codecs.raw_unicode_escape_decode',
-         'raw_unicode_escape_encode' :  'app_codecs.raw_unicode_escape_encode',
-         'unicode_escape_decode' :  'app_codecs.unicode_escape_decode',
-         'unicode_escape_encode' :  'app_codecs.unicode_escape_encode',
-         'unicode_internal_decode' :  'app_codecs.unicode_internal_decode',
-         'unicode_internal_encode' :  'app_codecs.unicode_internal_encode',
-         'utf_7_decode' :  'app_codecs.utf_7_decode',
-         'utf_7_encode' :  'app_codecs.utf_7_encode',
-         'charmap_build' : 'app_codecs.charmap_build'
-    }
+    """
+   _codecs -- Provides access to the codec registry and the builtin
+              codecs.
+
+   This module should never be imported directly. The standard library
+   module "codecs" wraps this builtin module for use within Python.
+
+   The codec registry is accessible via:
+
+     register(search_function) -> None
+
+     lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer)
+
+   The builtin Unicode codecs use the following interface:
+
+     <encoding>_encode(Unicode_object[,errors='strict']) -> 
+         (string object, bytes consumed)
+
+     <encoding>_decode(char_buffer_obj[,errors='strict']) -> 
+        (Unicode object, bytes consumed)
+
+   <encoding>_encode() interfaces also accept non-Unicode object as
+   input. The objects are then converted to Unicode using
+   PyUnicode_FromObject() prior to applying the conversion.
+
+   These <encoding>s are available: utf_8, unicode_escape,
+   raw_unicode_escape, unicode_internal, latin_1, ascii (7-bit),
+   mbcs (on win32).
+
+
+Written by Marc-Andre Lemburg (mal at lemburg.com).
+
+Copyright (c) Corporation for National Research Initiatives.
+"""
+
+    appleveldefs = {}
+
     interpleveldefs = {
          'encode':         'interp_codecs.encode',
          'decode':         'interp_codecs.decode',
@@ -26,12 +47,15 @@
          'lookup_error':   'interp_codecs.lookup_error',
          'register':       'interp_codecs.register_codec',
          'register_error': 'interp_codecs.register_error',
+         'charmap_build' : 'interp_codecs.charmap_build',
 
          # encoders and decoders
          'ascii_decode'     : 'interp_codecs.ascii_decode',
          'ascii_encode'     : 'interp_codecs.ascii_encode',
          'latin_1_decode'   : 'interp_codecs.latin_1_decode',
          'latin_1_encode'   : 'interp_codecs.latin_1_encode',
+         'utf_7_decode'     : 'interp_codecs.utf_7_decode',
+         'utf_7_encode'     : 'interp_codecs.utf_7_encode',
          'utf_8_decode'     : 'interp_codecs.utf_8_decode',
          'utf_8_encode'     : 'interp_codecs.utf_8_encode',
          'utf_16_be_decode' : 'interp_codecs.utf_16_be_decode',
@@ -44,6 +68,15 @@
          'charbuffer_encode': 'interp_codecs.buffer_encode',
          'readbuffer_encode': 'interp_codecs.buffer_encode',
          'charmap_decode'   : 'interp_codecs.charmap_decode',
+         'charmap_encode'   : 'interp_codecs.charmap_encode',
+         'escape_encode'    : 'interp_codecs.escape_encode',
+         'escape_decode'    : 'interp_codecs.escape_decode',
+         'unicode_escape_decode'     :  'interp_codecs.unicode_escape_decode',
+         'unicode_escape_encode'     :  'interp_codecs.unicode_escape_encode',
+         'raw_unicode_escape_decode' :  'interp_codecs.raw_unicode_escape_decode',
+         'raw_unicode_escape_encode' :  'interp_codecs.raw_unicode_escape_encode',
+         'unicode_internal_decode'   :  'interp_codecs.unicode_internal_decode',
+         'unicode_internal_encode'   :  'interp_codecs.unicode_internal_encode',
     }
 
     def __init__(self, space, *args):

Modified: pypy/branch/fast-forward/pypy/module/_codecs/interp_codecs.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_codecs/interp_codecs.py	(original)
+++ pypy/branch/fast-forward/pypy/module/_codecs/interp_codecs.py	Tue Jul  6 00:00:33 2010
@@ -1,5 +1,6 @@
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped, applevel
+from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped, interp2app
+from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
 from pypy.rlib.objectmodel import we_are_translated
@@ -13,6 +14,8 @@
         self.decode_error_handler = self.make_errorhandler(space, True)
         self.encode_error_handler = self.make_errorhandler(space, False)
 
+        self.unicodedata_handler = None
+
     def make_errorhandler(self, space, decode):
         def unicode_call_errorhandler(errors,  encoding, reason, input,
                                       startpos, endpos):
@@ -53,6 +56,21 @@
                 return replace, newpos
         return unicode_call_errorhandler
 
+    def get_unicodedata_handler(self, space):
+        if self.unicodedata_handler:
+            return self.unicodedata_handler
+        try:
+            w_builtin = space.getbuiltinmodule('__builtin__')
+            w_import = space.getattr(w_builtin, space.wrap("__import__"))
+            w_unicodedata = space.call_function(w_import,
+                                                space.wrap("unicodedata"))
+            w_getcode = space.getattr(w_unicodedata, space.wrap("_get_code"))
+        except OperationError:
+            return None
+        else:
+            self.unicodedata_handler = UnicodeData_Handler(space, w_getcode)
+            return self.unicodedata_handler
+
     def _freeze_(self):
         assert not self.codec_search_path
         return False
@@ -114,78 +132,125 @@
         "unknown encoding: %s", encoding)
 lookup_codec.unwrap_spec = [ObjSpace, str]
 
-app_errors = applevel("""
-def check_exception(exc):
+# ____________________________________________________________
+# Register standard error handlers
+
+def check_exception(space, w_exc):
     try:
-        delta = exc.end - exc.start
-        if delta < 0 or not isinstance(exc.object, (unicode, str)):
-            raise TypeError("wrong exception")
-    except AttributeError:
-        raise TypeError("wrong exception")
-
-def strict_errors(exc):
-    if isinstance(exc, Exception):
-        raise exc
-    else:
-        raise TypeError("codec must pass exception instance")
-
-def ignore_errors(exc):
-    check_exception(exc)
-    if isinstance(exc, UnicodeEncodeError):
-        return u'', exc.end
-    elif isinstance(exc, (UnicodeDecodeError, UnicodeTranslateError)):
-        return u'', exc.end
-    else: 
-        raise TypeError("don't know how to handle %.400s in error callback"%exc)
-
-Py_UNICODE_REPLACEMENT_CHARACTER = u"\ufffd"
-
-def replace_errors(exc):
-    check_exception(exc)
-    if isinstance(exc, UnicodeEncodeError):
-        return u'?'*(exc.end-exc.start), exc.end
-    elif isinstance(exc, (UnicodeTranslateError, UnicodeDecodeError)):
-        return Py_UNICODE_REPLACEMENT_CHARACTER*(exc.end-exc.start), exc.end
-    else:
-        raise TypeError("don't know how to handle %.400s in error callback"%exc)
-
-def xmlcharrefreplace_errors(exc):
-    if isinstance(exc, UnicodeEncodeError):
-        res = []
-        for ch in exc.object[exc.start:exc.end]:
-            res += '&#'
-            res += str(ord(ch))
-            res += ';'
-        return u''.join(res), exc.end
-    else:
-        raise TypeError("don't know how to handle %.400s in error callback"%type(exc))
-
-def backslashreplace_errors(exc):
-    if isinstance(exc, UnicodeEncodeError):
-        p = []
-        for c in exc.object[exc.start:exc.end]:
-            p += '\\\\'
-            oc = ord(c)
-            if (oc >= 0x00010000):
-                p += 'U'
-                p += "%.8x" % ord(c)
+        w_start = space.getattr(w_exc, space.wrap('start'))
+        w_end = space.getattr(w_exc, space.wrap('end'))
+        w_obj = space.getattr(w_exc, space.wrap('object'))
+    except OperationError, e:
+        if not e.match(space, space.w_AttributeError):
+            raise
+        raise OperationError(space.w_TypeError, space.wrap(
+            "wrong exception"))
+
+    delta = space.int_w(w_end) - space.int_w(w_start)
+    if delta < 0 or not (space.isinstance_w(w_obj, space.w_str) or
+                         space.isinstance_w(w_obj, space.w_unicode)):
+        raise OperationError(space.w_TypeError, space.wrap(
+            "wrong exception"))
+
+def strict_errors(space, w_exc):
+    check_exception(space, w_exc)
+    if space.isinstance_w(w_exc, space.w_BaseException):
+        raise OperationError(space.type(w_exc), w_exc)
+    else:
+        raise OperationError(space.w_TypeError, space.wrap(
+            "codec must pass exception instance"))
+
+def ignore_errors(space, w_exc):
+    check_exception(space, w_exc)
+    w_end = space.getattr(w_exc, space.wrap('end'))
+    if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
+        return space.newtuple([space.wrap(''), w_end])
+    elif (space.isinstance_w(w_exc, space.w_UnicodeDecodeError) or
+          space.isinstance_w(w_exc, space.w_UnicodeTranslateError)):
+        return space.newtuple([space.wrap(u''), w_end])
+    else:
+        typename = space.type(w_exc).getname(space, '?')
+        raise operationerrfmt(space.w_TypeError,
+            "don't know how to handle %s in error callback", typename)
+
+def replace_errors(space, w_exc):
+    check_exception(space, w_exc)
+    w_start = space.getattr(w_exc, space.wrap('start'))
+    w_end = space.getattr(w_exc, space.wrap('end'))
+    size = space.int_w(w_end) - space.int_w(w_start)
+    if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
+        text = '?' * size
+        return space.newtuple([space.wrap(text), w_end])
+    elif (space.isinstance_w(w_exc, space.w_UnicodeDecodeError) or
+          space.isinstance_w(w_exc, space.w_UnicodeTranslateError)):
+        text = u'\ufffd' * size
+        return space.newtuple([space.wrap(text), w_end])
+    else:
+        typename = space.type(w_exc).getname(space, '?')
+        raise operationerrfmt(space.w_TypeError,
+            "don't know how to handle %s in error callback", typename)
+
+def xmlcharrefreplace_errors(space, w_exc):
+    check_exception(space, w_exc)
+    if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
+        obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
+        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
+        w_end = space.getattr(w_exc, space.wrap('end'))
+        end = space.int_w(w_end)
+        builder = UnicodeBuilder()
+        pos = start
+        while pos < end:
+            ch = obj[pos]
+            builder.append(u"&#")
+            builder.append(unicode(str(ord(ch))))
+            builder.append(u";")
+            pos += 1
+        return space.newtuple([space.wrap(builder.build()), w_end])
+    else:
+        typename = space.type(w_exc).getname(space, '?')
+        raise operationerrfmt(space.w_TypeError,
+            "don't know how to handle %s in error callback", typename)
+
+def backslashreplace_errors(space, w_exc):
+    check_exception(space, w_exc)
+    if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
+        obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
+        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
+        w_end = space.getattr(w_exc, space.wrap('end'))
+        end = space.int_w(w_end)
+        builder = UnicodeBuilder()
+        pos = start
+        while pos < end:
+            oc = ord(obj[pos])
+            num = hex(oc)
+            if (oc >= 0x10000):
+                builder.append(u"\\U")
+                zeros = 8
             elif (oc >= 0x100):
-                p += 'u'
-                p += "%.4x" % ord(c)
+                builder.append(u"\\u")
+                zeros = 4
             else:
-                p += 'x'
-                p += "%.2x" % ord(c)
-        return u''.join(p), exc.end
-    else:
-        raise TypeError("don't know how to handle %.400s in error callback"%type(exc))
-""")
+                builder.append(u"\\x")
+                zeros = 2
+            lnum = len(num)
+            nb = zeros + 2 - lnum # num starts with '0x'
+            if nb > 0:
+                builder.append_multiple_char(u'0', nb)
+            builder.append_slice(unicode(num), 2, lnum)
+            pos += 1
+        return space.newtuple([space.wrap(builder.build()), w_end])
+    else:
+        typename = space.type(w_exc).getname(space, '?')
+        raise operationerrfmt(space.w_TypeError,
+            "don't know how to handle %s in error callback", typename)
 
 def register_builtin_error_handlers(space):
+    "NOT_RPYTHON"
     state = space.fromcache(CodecState)
     for error in ("strict", "ignore", "replace", "xmlcharrefreplace",
                   "backslashreplace"):
         name = error + "_errors"
-        state.codec_error_registry[error] = app_errors.wget(space, name)
+        state.codec_error_registry[error] = space.wrap(interp2app(globals()[name]))
 
 
 def lookup_error(space, errors):
@@ -279,6 +344,38 @@
 
 from pypy.rlib import runicode
 
+def make_raw_encoder(name):
+    rname = "unicode_encode_%s" % (name.replace("_encode", ""), )
+    assert hasattr(runicode, rname)
+    def raw_encoder(space, uni):
+        state = space.fromcache(CodecState)
+        func = getattr(runicode, rname)
+        errors = "strict"
+        return func(uni, len(uni), errors, state.encode_error_handler)
+    raw_encoder.func_name = rname
+    return raw_encoder
+
+def make_raw_decoder(name):
+    rname = "str_decode_%s" % (name.replace("_decode", ""), )
+    assert hasattr(runicode, rname)
+    def raw_decoder(space, string):
+        final = True
+        errors = "strict"
+        state = space.fromcache(CodecState)
+        func = getattr(runicode, rname)
+        kwargs = {}
+        if name == 'unicode_escape':
+            unicodedata_handler = state.get_unicodedata_handler(space)
+            result, consumed = func(string, len(string), errors,
+                                    final, state.decode_error_handler,
+                                    unicodedata_handler=unicodedata_handler)
+        else:
+            result, consumed = func(string, len(string), errors,
+                                    final, state.decode_error_handler)
+        return result
+    raw_decoder.func_name = rname
+    return raw_decoder
+
 def make_encoder_wrapper(name):
     rname = "unicode_encode_%s" % (name.replace("_encode", ""), )
     assert hasattr(runicode, rname)
@@ -308,20 +405,26 @@
 for encoders in [
          "ascii_encode",
          "latin_1_encode",
+         "utf_7_encode",
          "utf_8_encode",
          "utf_16_encode",
          "utf_16_be_encode",
          "utf_16_le_encode",
+         "unicode_escape_encode",
+         "raw_unicode_escape_encode",
+         "unicode_internal_encode",
         ]:
     make_encoder_wrapper(encoders)
 
 for decoders in [
          "ascii_decode",
          "latin_1_decode",
+         "utf_7_decode",
          "utf_8_decode",
          "utf_16_decode",
          "utf_16_be_decode",
          "utf_16_le_decode",
+         "raw_unicode_escape_decode",
          ]:
     make_decoder_wrapper(decoders)
 
@@ -330,8 +433,6 @@
     make_decoder_wrapper('mbcs_decode')
 
 def utf_16_ex_decode(space, data, errors='strict', byteorder=0, w_final=False):
-    """None
-    """
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
     if byteorder == 0:
@@ -349,77 +450,213 @@
                            space.wrap(byteorder)])
 utf_16_ex_decode.unwrap_spec = [ObjSpace, str, str, int, W_Root]
 
-def _extract_from_mapping(space, mapping_w, w_mapping, ch):
-    if mapping_w is not None:
+# ____________________________________________________________
+# Charmap
+
+class Charmap_Decode:
+    def __init__(self, space, w_mapping):
+        self.space = space
+        self.w_mapping = w_mapping
+
+        # fast path for all the stuff in the encodings module
+        if space.is_true(space.isinstance(w_mapping, space.w_tuple)):
+            self.mapping_w = space.fixedview(w_mapping)
+        else:
+            self.mapping_w = None
+
+    def get(self, ch, errorchar):
+        space = self.space
+
+        # get the character from the mapping
+        if self.mapping_w is not None:
+            w_ch = self.mapping_w[ord(ch)]
+        else:
+            try:
+                w_ch = space.getitem(self.w_mapping, space.newint(ord(ch)))
+            except OperationError, e:
+                if not e.match(space, space.w_LookupError):
+                    raise
+                return errorchar
+
+        # Charmap may return a unicode string
         try:
-            return mapping_w[ord(ch)]
-        except IndexError:
-            pass
-    else:
+            x = space.unicode_w(w_ch)
+        except OperationError, e:
+            if not e.match(space, space.w_TypeError):
+                raise
+        else:
+            return x
+
+        # Charmap may return a number
+        try:
+            x = space.int_w(w_ch)
+        except OperationError:
+            if not e.match(space, space.w_TypeError):
+                raise
+        else:
+            if 0 <= x < 65536: # Even on wide unicode builds...
+                return unichr(x)
+            else:
+                raise OperationError(space.w_TypeError, space.wrap(
+                    "character mapping must be in range(65536)"))
+
+        # Charmap may return None
+        if space.is_w(w_ch, space.w_None):
+            return errorchar
+
+        raise OperationError(space.w_TypeError, space.wrap("invalid mapping"))
+
+class Charmap_Encode:
+    def __init__(self, space, w_mapping):
+        self.space = space
+        self.w_mapping = w_mapping
+
+    def get(self, ch, errorchar):
+        space = self.space
+
+        # get the character from the mapping
         try:
-            return space.getitem(w_mapping, space.newint(ord(ch)))
+            w_ch = space.getitem(self.w_mapping, space.newint(ord(ch)))
         except OperationError, e:
-            if (not e.match(space, space.w_KeyError) and
-                not e.match(space, space.w_IndexError)):
+            if not e.match(space, space.w_LookupError):
                 raise
-            pass
+            return errorchar
 
-def _append_unicode(space, builder, w_x):
-    try:
-        x = space.unicode_w(w_x)
-    except OperationError, e:
-        if not e.match(space, space.w_TypeError):
-            raise
-    else:
-        if x != u"\ufffe":
-            builder.append(x)
-            return True
-        return False
-    try:
-        x = space.int_w(w_x)
-    except OperationError:
-        if not e.match(space, space.w_TypeError):
-            raise
-    else:
-        if x < 65536:
-            builder.append(unichr(x))
+        # Charmap may return a string
+        try:
+            x = space.realstr_w(w_ch)
+        except OperationError, e:
+            if not e.match(space, space.w_TypeError):
+                raise
         else:
-            raise OperationError(space.w_TypeError, space.wrap("character mapping must be in range(65536)"))
-        return True
-    if not space.is_true(w_x):
-        return False
+            return x
+
+        # Charmap may return a number
+        try:
+            x = space.int_w(w_ch)
+        except OperationError:
+            if not e.match(space, space.w_TypeError):
+                raise
+        else:
+            if 0 <= x < 256:
+                return chr(x)
+            else:
+                raise OperationError(space.w_TypeError, space.wrap(
+                    "character mapping must be in range(256)"))
+
+        # Charmap may return None
+        if space.is_w(w_ch, space.w_None):
+            return errorchar
+
+        raise OperationError(space.w_TypeError, space.wrap("invalid mapping"))
+
+
+ at unwrap_spec(ObjSpace, str, str, W_Root)
+def charmap_decode(space, string, errors="strict", w_mapping=None):
+    if len(string) == 0:
+        return space.newtuple([space.wrap(u''), space.wrap(0)])
+
+    if space.is_w(w_mapping, space.w_None):
+        mapping = None
     else:
-        raise OperationError(space.w_TypeError, space.w_None)
+        mapping = Charmap_Decode(space, w_mapping)
+
+    final = True
+    state = space.fromcache(CodecState)
+    result, consumed = runicode.str_decode_charmap(
+        string, len(string), errors,
+        final, state.decode_error_handler, mapping)
+    return space.newtuple([space.wrap(result), space.wrap(consumed)])
+
+ at unwrap_spec(ObjSpace, unicode, str, W_Root)
+def charmap_encode(space, uni, errors="strict", w_mapping=None):
+    if space.is_w(w_mapping, space.w_None):
+        mapping = None
+    else:
+        mapping = Charmap_Encode(space, w_mapping)
+
+    state = space.fromcache(CodecState)
+    result = runicode.unicode_encode_charmap(
+        uni, len(uni), errors,
+        state.encode_error_handler, mapping)
+    return space.newtuple([space.wrap(result), space.wrap(len(uni))])
 
 
-def charmap_decode(space, s, errors="strict", w_mapping=None):
-    size = len(s)
-    # Default to Latin-1
-    if space.is_true(space.is_(w_mapping, space.w_None)):
-        return latin_1_decode(space, s, errors, space.w_False)
+ at unwrap_spec(ObjSpace, unicode)
+def charmap_build(space, chars):
+    # XXX CPython sometimes uses a three-level trie
+    w_charmap = space.newdict()
+    for num in range(len(chars)):
+        elem = chars[num]
+        space.setitem(w_charmap, space.newint(ord(elem)), space.newint(num))
+    return w_charmap
 
-    if (size == 0):
+# ____________________________________________________________
+# Unicode escape
+
+class UnicodeData_Handler:
+    def __init__(self, space, w_getcode):
+        self.space = space
+        self.w_getcode = w_getcode
+
+    def call(self, name):
+        space = self.space
+        try:
+            w_code = space.call_function(self.w_getcode, space.wrap(name))
+        except OperationError, e:
+            if not e.match(space, space.w_KeyError):
+                raise
+            return -1
+        return space.int_w(w_code)
+
+ at unwrap_spec(ObjSpace, 'bufferstr', str, W_Root)
+def unicode_escape_decode(space, string, errors="strict", w_final=False):
+    final = space.is_true(w_final)
+    state = space.fromcache(CodecState)
+    errorhandler=state.decode_error_handler
+
+    unicode_name_handler = state.get_unicodedata_handler(space)
+
+    result, consumed = runicode.str_decode_unicode_escape(
+        string, len(string), errors,
+        final, state.decode_error_handler,
+        unicode_name_handler)
+
+    return space.newtuple([space.wrap(result), space.wrap(consumed)])
+
+# ____________________________________________________________
+# Unicode-internal
+
+ at unwrap_spec(ObjSpace, W_Root, str)
+def unicode_internal_decode(space, w_string, errors="strict"):
+    # special case for this codec: unicodes are returned as is
+    if space.isinstance_w(w_string, space.w_unicode):
+        return space.newtuple([w_string, space.len(w_string)])
+
+    string = space.str_w(w_string)
+
+    if len(string) == 0:
         return space.newtuple([space.wrap(u''), space.wrap(0)])
-    
-    # fast path for all the stuff in the encodings module
-    if space.is_true(space.isinstance(w_mapping, space.w_tuple)):
-        mapping_w = space.fixedview(w_mapping)
-    else:
-        mapping_w = None
-
-    builder = UnicodeBuilder(size)
-    inpos = 0
-    while (inpos < len(s)):
-        #/* Get mapping_w (char ordinal -> integer, Unicode char or None) */
-        ch = s[inpos]
-        w_x = _extract_from_mapping(space, mapping_w, w_mapping, ch)
-        if w_x is not None and _append_unicode(space, builder, w_x):
-            inpos += 1
-            continue
-        state = space.fromcache(CodecState)
-        next, inpos = state.decode_error_handler(errors, "charmap",
-                   "character maps to <undefined>", s, inpos, inpos+1)
-        builder.append(next)
-    res = builder.build()
-    return space.newtuple([space.wrap(res), space.wrap(size)])
-charmap_decode.unwrap_spec = [ObjSpace, str, str, W_Root]
+
+    final = True
+    state = space.fromcache(CodecState)
+    result, consumed = runicode.str_decode_unicode_internal(
+        string, len(string), errors,
+        final, state.decode_error_handler)
+    return space.newtuple([space.wrap(result), space.wrap(consumed)])
+
+# ____________________________________________________________
+# support for the "string escape" codec
+# This is a bytes-to bytes transformation
+
+ at unwrap_spec(ObjSpace, W_Root, str)
+def escape_encode(space, w_string, errors='strict'):
+    w_repr = space.repr(w_string)
+    w_result = space.getslice(w_repr, space.wrap(1), space.wrap(-1))
+    return space.newtuple([w_result, space.len(w_string)])
+
+ at unwrap_spec(ObjSpace, str, str)
+def escape_decode(space, data, errors='strict'):
+    from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape
+    result = PyString_DecodeEscape(space, data, None)
+    return space.newtuple([space.wrap(result), space.wrap(len(data))])

Modified: pypy/branch/fast-forward/pypy/module/_codecs/test/test_codecs.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_codecs/test/test_codecs.py	(original)
+++ pypy/branch/fast-forward/pypy/module/_codecs/test/test_codecs.py	Tue Jul  6 00:00:33 2010
@@ -1,7 +1,5 @@
 import autopath
 from pypy.conftest import gettestobjspace
-from pypy.module._codecs.app_codecs import unicode_escape_encode,\
-     charmap_encode, unicode_escape_decode
 
 
 class AppTestCodecs:
@@ -14,26 +12,16 @@
         raises(TypeError, _codecs.register, 1)
 
     def test_bigU_codecs(self):
-        import sys
-        oldmaxunicode = sys.maxunicode
-        if sys.maxunicode <= 0xffff:
-            return # this test cannot run on UCS2 builds
         u = u'\U00010001\U00020002\U00030003\U00040004\U00050005'
         for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be',
                          'raw_unicode_escape',
                          'unicode_escape', 'unicode_internal'):
             assert unicode(u.encode(encoding),encoding) == u
-        sys.maxunicode = oldmaxunicode
 
     def test_ucs4(self):
-        import sys
-        oldmaxunicode = sys.maxunicode
-        if sys.maxunicode <= 0xffff:
-            sys.maxunicode = 0xffffffff
         x = u'\U00100000'
         y = x.encode("raw-unicode-escape").decode("raw-unicode-escape")
         assert x == y 
-        sys.maxunicode = oldmaxunicode
 
     def test_named_unicode(self):
         assert unicode('\\N{SPACE}','unicode-escape') == u" "
@@ -118,12 +106,20 @@
 
     def test_charmap_decode(self):
         from _codecs import charmap_decode
+        import sys
         assert charmap_decode('', 'strict', 'blablabla') == ('', 0)
         assert charmap_decode('xxx') == ('xxx', 3)
         assert charmap_decode('xxx', 'strict', {ord('x'): u'XX'}) == ('XXXXXX', 3)
         map = tuple([unichr(i) for i in range(256)])
         assert charmap_decode('xxx\xff', 'strict', map) == (u'xxx\xff', 4)
 
+        raises(TypeError, charmap_decode, '\xff', "replace",  {0xff: 0x10001})
+
+    def test_unicode_escape(self):
+        from _codecs import unicode_escape_encode, unicode_escape_decode
+        assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3)
+        assert unicode_escape_decode('abc') == (u'abc'.decode('unicode_escape'), 3)
+        assert unicode_escape_decode('\\x61\\x62\\x63') == (u'abc', 12)
 
 class AppTestPartialEvaluation:
 
@@ -275,7 +271,6 @@
         assert u"\u0663".encode("raw-unicode-escape") == "\u0663"
 
     def test_escape_decode(self):
-        
         test = 'a\n\\b\x00c\td\u2045'.encode('string_escape')
         assert test.decode('string_escape') =='a\n\\b\x00c\td\u2045'
         assert '\\077'.decode('string_escape') == '?'
@@ -283,6 +278,14 @@
         assert '\\253'.decode('string_escape') == chr(0253)
         assert '\\312'.decode('string_escape') == chr(0312)
 
+    def test_escape_decode_wrap_around(self):
+        assert '\\400'.decode('string_escape') == chr(0)
+
+    def test_escape_decode_ignore_invalid(self):
+        assert '\\9'.decode('string_escape') == '\\9'
+        assert '\\01'.decode('string_escape') == chr(01)
+        assert '\\0f'.decode('string_escape') == chr(0) + 'f'
+        assert '\\08'.decode('string_escape') == chr(0) + '8'
 
     def test_decode_utf8_different_case(self):
         constant = u"a"
@@ -377,6 +380,9 @@
 
     def test_charmap_decode_1(self):
         import codecs
+        assert codecs.charmap_encode(u'xxx') == ('xxx', 3)
+        assert codecs.charmap_encode(u'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 3)
+
         res = codecs.charmap_decode("\x00\x01\x02", "replace", u"ab")
         assert res == (u"ab\ufffd", 3)
         res = codecs.charmap_decode("\x00\x01\x02", "replace", u"ab\ufffe")
@@ -464,6 +470,9 @@
         assert '\xff'.decode('utf-7', 'ignore') == ''
         assert '\x00'.decode('unicode-internal', 'ignore') == ''
 
+    def test_backslahreplace(self):
+        assert u'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == 'a\\xac\u1234\u20ac\u8000'
+
     def test_badhandler(self):
         import codecs
         results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) )
@@ -527,9 +536,26 @@
     def test_charmap_encode(self):
         assert 'xxx'.encode('charmap') == 'xxx'
 
+        import codecs
+        raises(TypeError, codecs.charmap_encode, u'\xff', "replace",  {0xff: 300})
+        raises(UnicodeError, codecs.charmap_encode, u"\xff", "replace", {0xff: None})
+
+    def test_charmap_encode_replace(self):
+        charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"])
+        charmap[ord("?")] = "XYZ"
+        import codecs
+        sin = u"abcDEF"
+        sout = codecs.charmap_encode(sin, "replace", charmap)[0]
+        assert sout == "AABBCCXYZXYZXYZ"
+
     def test_charmap_decode_2(self):
         assert 'foo'.decode('charmap') == 'foo'
 
+    def test_charmap_build(self):
+        import codecs
+        assert codecs.charmap_build(u'123456') == {49: 0, 50: 1, 51: 2,
+                                                   52: 3, 53: 4, 54: 5}
+
     def test_utf7_start_end_in_exception(self):
         try:
             '+IC'.decode('utf-7')
@@ -537,6 +563,9 @@
             assert exc.start == 0
             assert exc.end == 3
 
+    def test_utf7_surrogate(self):
+        raises(UnicodeDecodeError, '+3ADYAA-'.decode, 'utf-7')
+
     def test_utf_16_encode_decode(self):
         import codecs
         x = u'123abc'
@@ -546,6 +575,8 @@
     def test_unicode_escape(self):        
         assert u'\\'.encode('unicode-escape') == '\\\\'
         assert '\\\\'.decode('unicode-escape') == u'\\'
+        assert u'\ud801'.encode('unicode-escape') == '\\ud801'
+        assert u'\u0013'.encode('unicode-escape') == '\\x13'
 
     def test_mbcs(self):
         import sys
@@ -555,14 +586,3 @@
         assert u'caf\xe9'.encode('mbcs') == 'caf\xe9'
         assert u'\u040a'.encode('mbcs') == '?' # some cyrillic letter
         assert 'cafx\e9'.decode('mbcs') == u'cafx\e9'
-
-
-class TestDirect:
-    def test_charmap_encode(self):
-        assert charmap_encode(u'xxx') == ('xxx', 3)
-        assert charmap_encode(u'xxx', 'strict', {ord('x'): 'XX'}) ==  ('XXXXXX', 6)
-
-    def test_unicode_escape(self):
-        assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3)
-        assert unicode_escape_decode('abc') == (u'abc'.decode('unicode_escape'), 3)
-        assert unicode_escape_decode('\\x61\\x62\\x63') == (u'abc', 12)

Modified: pypy/branch/fast-forward/pypy/module/_stackless/interp_coroutine.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_stackless/interp_coroutine.py	(original)
+++ pypy/branch/fast-forward/pypy/module/_stackless/interp_coroutine.py	Tue Jul  6 00:00:33 2010
@@ -24,7 +24,9 @@
 from pypy.interpreter.function import StaticMethod
 
 from pypy.module._stackless.stackless_flags import StacklessFlags
-from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState, AbstractThunk
+from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState, AbstractThunk, CoroutineExit
+
+from pypy.module.exceptions.interp_exceptions import W_SystemExit, _new_exception
 
 from pypy.rlib import rstack # for resume points
 from pypy.tool import stdlib_opcode as pythonopcode
@@ -48,6 +50,13 @@
         rstack.resume_point("appthunk", costate, returns=w_result)
         costate.w_tempval = w_result
 
+W_CoroutineExit = _new_exception('CoroutineExit', W_SystemExit,
+                        """Coroutine killed manually.""")
+
+# Should be moved to interp_stackless.py if it's ever implemented... Currently
+# used by pypy/lib/stackless.py.
+W_TaskletExit = _new_exception('TaskletExit', W_SystemExit, 
+            """Tasklet killed manually.""")
 
 class AppCoroutine(Coroutine): # XXX, StacklessFlags):
 
@@ -92,6 +101,13 @@
         w_ret, state.w_tempval = state.w_tempval, space.w_None
         return w_ret
 
+    def switch(self):
+        space = self.space
+        try:
+            Coroutine.switch(self)
+        except CoroutineExit:
+            raise OperationError(self.costate.w_CoroutineExit, space.w_None)
+
     def w_finished(self, w_excinfo):
         pass
 
@@ -102,6 +118,9 @@
             w_excvalue = operror.get_w_value(space)
             w_exctraceback = operror.application_traceback
             w_excinfo = space.newtuple([w_exctype, w_excvalue, w_exctraceback])
+            
+            if w_exctype is self.costate.w_CoroutineExit:
+                self.coroutine_exit = True
         else:
             w_N = space.w_None
             w_excinfo = space.newtuple([w_N, w_N, w_N])
@@ -118,6 +137,23 @@
 
     def w_kill(self):
         self.kill()
+            
+    def w_throw(self, w_type, w_value=None, w_traceback=None):
+        space = self.space
+
+        operror = OperationError(w_type, w_value)
+        operror.normalize_exception(space)
+        
+        if not space.is_w(w_traceback, space.w_None):
+            from pypy.interpreter import pytraceback
+            tb = space.interpclass_w(w_traceback)
+            if tb is None or not space.is_true(space.isinstance(tb, 
+                space.gettypeobject(pytraceback.PyTraceback.typedef))):
+                raise OperationError(space.w_TypeError,
+                      space.wrap("throw: arg 3 must be a traceback or None"))
+            operror.application_traceback = tb
+        
+        self._kill(operror)
 
     def _userdel(self):
         if self.get_is_zombie():
@@ -309,6 +345,7 @@
                       unwrap_spec=['self', W_Root, Arguments]),
     switch = interp2app(AppCoroutine.w_switch),
     kill = interp2app(AppCoroutine.w_kill),
+    throw = interp2app(AppCoroutine.w_throw),
     finished = interp2app(AppCoroutine.w_finished),
     is_alive = GetSetProperty(AppCoroutine.w_get_is_alive),
     is_zombie = GetSetProperty(AppCoroutine.w_get_is_zombie,
@@ -330,6 +367,26 @@
         BaseCoState.__init__(self)
         self.w_tempval = space.w_None
         self.space = space
+
+        # Exporting new exception to space
+        self.w_CoroutineExit = space.gettypefor(W_CoroutineExit)
+        space.setitem(
+                      space.exceptions_module.w_dict, 
+                      space.new_interned_str('CoroutineExit'), 
+                      self.w_CoroutineExit) 
+        space.setitem(space.builtin.w_dict, 
+                      space.new_interned_str('CoroutineExit'), 
+                      self.w_CoroutineExit)
+        
+        # Should be moved to interp_stackless.py if it's ever implemented...
+        self.w_TaskletExit = space.gettypefor(W_TaskletExit)
+        space.setitem(
+                      space.exceptions_module.w_dict, 
+                      space.new_interned_str('TaskletExit'), 
+                      self.w_TaskletExit) 
+        space.setitem(space.builtin.w_dict, 
+                      space.new_interned_str('TaskletExit'), 
+                      self.w_TaskletExit)  
         
     def post_install(self):
         self.current = self.main = AppCoroutine(self.space, state=self)

Modified: pypy/branch/fast-forward/pypy/module/_stackless/rcoroutine.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_stackless/rcoroutine.py	(original)
+++ pypy/branch/fast-forward/pypy/module/_stackless/rcoroutine.py	Tue Jul  6 00:00:33 2010
@@ -7,3 +7,4 @@
 BaseCoState = d['BaseCoState']
 AbstractThunk = d['AbstractThunk']
 syncstate = d['syncstate']
+CoroutineExit = d['CoroutineExit']

Modified: pypy/branch/fast-forward/pypy/module/_stackless/test/test_coroutine.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_stackless/test/test_coroutine.py	(original)
+++ pypy/branch/fast-forward/pypy/module/_stackless/test/test_coroutine.py	Tue Jul  6 00:00:33 2010
@@ -84,8 +84,7 @@
         assert not co.is_alive
 
     def test_kill_running(self):
-        skip("kill is not really working (there is only CoroutineExit, "
-             "which is not an app-level exception)")
+        coroutineexit = []
         import _stackless as stackless
         main = stackless.coroutine.getcurrent()
         result = []
@@ -96,6 +95,9 @@
                 result.append(1)
                 main.switch()
                 x = 3
+            except CoroutineExit:
+                coroutineexit.append(True)
+                raise
             finally:
                 result.append(x)
             result.append(4)
@@ -107,6 +109,7 @@
         co.kill()
         assert not co.is_alive
         assert result == [1, 2]
+        assert coroutineexit == [True]
 
     def test_bogus_bind(self):
         import _stackless as stackless

Modified: pypy/branch/fast-forward/pypy/module/conftest.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/conftest.py	(original)
+++ pypy/branch/fast-forward/pypy/module/conftest.py	Tue Jul  6 00:00:33 2010
@@ -2,16 +2,17 @@
 from pypy.tool.lib_pypy import LIB_PYPY
 
 class MultipleDirCollector(py.test.collect.Collector):
-    def __init__(self, name, fspaths, parent=None, config=None):
+    def __init__(self, name, mainfspath, fspaths, parent=None, config=None):
         super(MultipleDirCollector, self).__init__(name, parent, config)
+        self.main_collector = py.test.collect.Directory(mainfspath, self)
         self.collectors = [py.test.collect.Directory(fspath, self)
                            for fspath in fspaths]
 
     def collect(self):
-        return self.collectors
+        return self.main_collector.collect() + self.collectors
 
 
 def pytest_collect_directory(path, parent):
     if path.basename == 'test_lib_pypy':
         # collect all the test in BOTH test_lib_pypy and ../../lib_pypy
-        return MultipleDirCollector(path.basename, [path, LIB_PYPY], parent)
+        return MultipleDirCollector(path.basename, path, [LIB_PYPY], parent)

Modified: pypy/branch/fast-forward/pypy/module/cpyext/api.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/cpyext/api.py	(original)
+++ pypy/branch/fast-forward/pypy/module/cpyext/api.py	Tue Jul  6 00:00:33 2010
@@ -45,11 +45,9 @@
 pypydir = py.path.local(autopath.pypydir)
 include_dir = pypydir / 'module' / 'cpyext' / 'include'
 source_dir = pypydir / 'module' / 'cpyext' / 'src'
-interfaces_dir = pypydir / "_interfaces"
 include_dirs = [
     include_dir,
     udir,
-    interfaces_dir,
     ]
 
 class CConfig:
@@ -100,9 +98,20 @@
 udir.join('pypy_macros.h').write("/* Will be filled later */")
 globals().update(rffi_platform.configure(CConfig_constants))
 
-def copy_header_files():
+def copy_header_files(dstdir):
+    assert dstdir.check(dir=True)
+    headers = include_dir.listdir('*.h') + include_dir.listdir('*.inl')
     for name in ("pypy_decl.h", "pypy_macros.h"):
-        udir.join(name).copy(interfaces_dir / name)
+        headers.append(udir.join(name))
+    for header in headers:
+        target = dstdir.join(header.basename)
+        try:
+            header.copy(dstdir)
+        except py.error.EACCES:
+            target.remove()   # maybe it was a read-only file
+            header.copy(dstdir)
+        target.chmod(0444) # make the file read-only, to make sure that nobody
+                           # edits it by mistake
 
 _NOT_SPECIFIED = object()
 CANNOT_FAIL = object()
@@ -881,7 +890,8 @@
         deco(func.get_wrapper(space))
 
     setup_init_functions(eci)
-    copy_header_files()
+    trunk_include = pypydir.dirpath() / 'include'
+    copy_header_files(trunk_include)
 
 initfunctype = lltype.Ptr(lltype.FuncType([], lltype.Void))
 @unwrap_spec(ObjSpace, str, str)

Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_api.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/cpyext/test/test_api.py	(original)
+++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_api.py	Tue Jul  6 00:00:33 2010
@@ -1,3 +1,4 @@
+import py
 from pypy.conftest import gettestobjspace
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.interpreter.baseobjspace import W_Root
@@ -68,3 +69,13 @@
         api.PyPy_GetWrapped(space.w_None)
         api.PyPy_GetReference(space.w_None)
 
+
+def test_copy_header_files(tmpdir):
+    api.copy_header_files(tmpdir)
+    def check(name):
+        f = tmpdir.join(name)
+        assert f.check(file=True)
+        py.test.raises(py.error.EACCES, "f.open('w')") # check that it's not writable
+    check('Python.h')
+    check('modsupport.inl')
+    check('pypy_decl.h')

Modified: pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py	(original)
+++ pypy/branch/fast-forward/pypy/module/marshal/interp_marshal.py	Tue Jul  6 00:00:33 2010
@@ -359,8 +359,8 @@
         return self.reader.read(n)
 
     def get1(self):
-        # convince typer to use a char
-        return chr(ord(self.get(1)))
+        # the [0] is used to convince the annotator to return a char
+        return self.get(1)[0]
 
     def atom_str(self, typecode):
         self.start(typecode)

Modified: pypy/branch/fast-forward/pypy/module/operator/test/test_operator.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/operator/test/test_operator.py	(original)
+++ pypy/branch/fast-forward/pypy/module/operator/test/test_operator.py	Tue Jul  6 00:00:33 2010
@@ -164,7 +164,6 @@
 
     def test_irepeat(self):
         import operator
-        import py
 
         class X(object):
             def __index__(self):

Modified: pypy/branch/fast-forward/pypy/module/sys/state.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/sys/state.py	(original)
+++ pypy/branch/fast-forward/pypy/module/sys/state.py	Tue Jul  6 00:00:33 2010
@@ -32,30 +32,19 @@
     if not stat.S_ISDIR(st[0]):
         raise OSError(errno.ENOTDIR, path)
 
-def getinitialpath(prefix):
-    from pypy.module.sys.version import PYPY_VERSION
-    libdir = os.path.join(prefix, 'lib')
-    pypyxy_dir = os.path.join(libdir, 'pypy%d.%d' % PYPY_VERSION[:2])
-    # search for the stdlib both in $PREFIX/lib/pypy1.2 and $PREFIX
-    for lib_prefix in [pypyxy_dir, prefix]:
-        try:
-            return get_importlist(lib_prefix)
-        except OSError:
-            pass
-    raise OSError # stdlib not foud
 
-def get_importlist(lib_prefix):
+def getinitialpath(prefix):
     from pypy.module.sys.version import CPYTHON_VERSION
     dirname = '%d.%d.%d' % (CPYTHON_VERSION[0],
                             CPYTHON_VERSION[1],
                             CPYTHON_VERSION[2])
-    lib_python = os.path.join(lib_prefix, 'lib-python')
+    lib_python = os.path.join(prefix, 'lib-python')
     python_std_lib = os.path.join(lib_python, dirname)
     checkdir(python_std_lib)
     python_std_lib_modified = os.path.join(lib_python, 'modified-' + dirname)
     checkdir(python_std_lib_modified)
     
-    lib_pypy = os.path.join(lib_prefix, 'lib_pypy')
+    lib_pypy = os.path.join(prefix, 'lib_pypy')
     checkdir(lib_pypy)
 
     importlist = []

Modified: pypy/branch/fast-forward/pypy/module/sys/test/test_initialpath.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/sys/test/test_initialpath.py	(original)
+++ pypy/branch/fast-forward/pypy/module/sys/test/test_initialpath.py	Tue Jul  6 00:00:33 2010
@@ -12,21 +12,7 @@
     return a, b, c
 
 
-def test_stdlib_in_pypyxy(tmpdir):
-    pypyxy = tmpdir.join('lib', 'pypy%d.%d' % PYPY_VERSION[:2])
-    dirs = build_hierarchy(pypyxy)
-    path = getinitialpath(str(tmpdir))
-    assert path == map(str, dirs)
-
-
 def test_stdlib_in_prefix(tmpdir):
     dirs = build_hierarchy(tmpdir)
     path = getinitialpath(str(tmpdir))
     assert path == map(str, dirs)
-
-def test_stdlib_precedence(tmpdir):
-    pypyxy = tmpdir.join('lib', 'pypy%d.%d' % PYPY_VERSION[:2])
-    dirs1 = build_hierarchy(tmpdir)
-    dirs2 = build_hierarchy(pypyxy)
-    path = getinitialpath(str(tmpdir))
-    assert path == map(str, dirs2)

Modified: pypy/branch/fast-forward/pypy/module/sys/version.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/sys/version.py	(original)
+++ pypy/branch/fast-forward/pypy/module/sys/version.py	Tue Jul  6 00:00:33 2010
@@ -7,8 +7,7 @@
 CPYTHON_VERSION            = (2, 7, 0, "final", 42)
 CPYTHON_API_VERSION        = 1012
 
-# when changing the pypy version, remind to also change the name of trunk/lib/pypyX.Y
-PYPY_VERSION               = (1, 2, 0, "beta", '?')
+PYPY_VERSION               = (1, 3, 0, "beta", '?')
 # the last item is replaced by the svn revision ^^^
 
 TRIM_URL_UP_TO = 'svn/pypy/'

Modified: pypy/branch/fast-forward/pypy/module/termios/interp_termios.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/termios/interp_termios.py	(original)
+++ pypy/branch/fast-forward/pypy/module/termios/interp_termios.py	Tue Jul  6 00:00:33 2010
@@ -60,8 +60,8 @@
     # last one need to be chosen carefully
     cc_w = [space.wrap(i) for i in cc]
     if lflag & termios.ICANON:
-        cc_w[termios.VMIN] = space.wrap(ord(cc[termios.VMIN]))
-        cc_w[termios.VTIME] = space.wrap(ord(cc[termios.VTIME]))
+        cc_w[termios.VMIN] = space.wrap(ord(cc[termios.VMIN][0]))
+        cc_w[termios.VTIME] = space.wrap(ord(cc[termios.VTIME][0]))
     w_cc = space.newlist(cc_w)
     l_w.append(w_cc)
     return space.newlist(l_w)

Modified: pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py
==============================================================================
--- pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py	(original)
+++ pypy/branch/fast-forward/pypy/objspace/std/marshal_impl.py	Tue Jul  6 00:00:33 2010
@@ -447,11 +447,11 @@
 register(TYPE_CODE, unmarshal_pycode)
 
 def marshal_w__Unicode(space, w_unicode, m):
-    s = space.str_w(unicodehelper.PyUnicode_EncodeUTF8(space, w_unicode))
+    s = unicodehelper.PyUnicode_EncodeUTF8(space, space.unicode_w(w_unicode))
     m.atom_str(TYPE_UNICODE, s)
 
 def unmarshal_Unicode(space, u, tc):
-    return unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(u.get_str()))
+    return space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, u.get_str()))
 register(TYPE_UNICODE, unmarshal_Unicode)
 
 app = gateway.applevel(r'''

Modified: pypy/branch/fast-forward/pypy/objspace/std/stringobject.py
==============================================================================
--- pypy/branch/fast-forward/pypy/objspace/std/stringobject.py	(original)
+++ pypy/branch/fast-forward/pypy/objspace/std/stringobject.py	Tue Jul  6 00:00:33 2010
@@ -863,7 +863,7 @@
             space.w_TypeError,
             "ord() expected a character, but string "
             "of length %d found", len(u_str))
-    return space.wrap(ord(u_str))
+    return space.wrap(ord(u_str[0]))
 
 def getnewargs__String(space, w_str):
     return space.newtuple([wrapstr(space, w_str._value)])

Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_shadowtracking.py
==============================================================================
--- pypy/branch/fast-forward/pypy/objspace/std/test/test_shadowtracking.py	(original)
+++ pypy/branch/fast-forward/pypy/objspace/std/test/test_shadowtracking.py	Tue Jul  6 00:00:33 2010
@@ -209,6 +209,7 @@
         a = A()
         names = [name for name in A.__dict__.keys()
                       if not name.startswith('_')]
+        names.sort()
         names_repeated = names * 10
         result = []
         __pypy__.reset_method_cache_counter()

Modified: pypy/branch/fast-forward/pypy/objspace/std/unicodeobject.py
==============================================================================
--- pypy/branch/fast-forward/pypy/objspace/std/unicodeobject.py	(original)
+++ pypy/branch/fast-forward/pypy/objspace/std/unicodeobject.py	Tue Jul  6 00:00:33 2010
@@ -12,6 +12,7 @@
 from pypy.rlib.rarithmetic import intmask, ovfcheck
 from pypy.rlib.objectmodel import compute_hash
 from pypy.rlib.rstring import string_repeat
+from pypy.rlib.runicode import unicode_encode_unicode_escape
 from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb
 from pypy.tool.sourcetools import func_with_new_name
 
@@ -892,101 +893,11 @@
                     space.wrap("character mapping must return integer, None or unicode"))
     return W_UnicodeObject(u''.join(result))
 
-# Move this into the _codecs module as 'unicodeescape_string (Remember to cater for quotes)'
 def repr__Unicode(space, w_unicode):
-    hexdigits = "0123456789abcdef"
     chars = w_unicode._value
     size = len(chars)
-    
-    singlequote = doublequote = False
-    for c in chars:
-        if c == u'\'':
-            singlequote = True
-        elif c == u'"':
-            doublequote = True
-    if singlequote and not doublequote:
-        quote = '"'
-    else:
-        quote = '\''
-    result = ['u', quote]
-    j = 0
-    while j<len(chars):
-        ch = chars[j]
-        code = ord(ch)
-        if code >= 0x10000:
-            # Resize if needed
-            result.extend(['\\', "U",
-                           hexdigits[(code >> 28) & 0xf],
-                           hexdigits[(code >> 24) & 0xf],
-                           hexdigits[(code >> 20) & 0xf],
-                           hexdigits[(code >> 16) & 0xf],
-                           hexdigits[(code >> 12) & 0xf],
-                           hexdigits[(code >>  8) & 0xf],
-                           hexdigits[(code >>  4) & 0xf],
-                           hexdigits[(code >>  0) & 0xf],
-                           ])
-            j += 1
-            continue
-        if code >= 0xD800 and code < 0xDC00:
-            if j < size - 1:
-                ch2 = chars[j+1]
-                code2 = ord(ch2)
-                if code2 >= 0xDC00 and code2 <= 0xDFFF:
-                    code = (((code & 0x03FF) << 10) | (code2 & 0x03FF)) + 0x00010000
-                    result.extend(['\\', "U",
-                                   hexdigits[(code >> 28) & 0xf],
-                                   hexdigits[(code >> 24) & 0xf],
-                                   hexdigits[(code >> 20) & 0xf],
-                                   hexdigits[(code >> 16) & 0xf],
-                                   hexdigits[(code >> 12) & 0xf],
-                                   hexdigits[(code >>  8) & 0xf],
-                                   hexdigits[(code >>  4) & 0xf],
-                                   hexdigits[(code >>  0) & 0xf],
-                                  ])
-                    j += 2
-                    continue
-                
-        if code >= 0x100:
-            result.extend(['\\', "u",
-                           hexdigits[(code >> 12) & 0xf],
-                           hexdigits[(code >>  8) & 0xf],
-                           hexdigits[(code >>  4) & 0xf],
-                           hexdigits[(code >>  0) & 0xf],
-                          ])
-            j += 1
-            continue
-        if code == ord('\\') or code == ord(quote):
-            result.append('\\')
-            result.append(chr(code))
-            j += 1
-            continue
-        if code == ord('\t'):
-            result.append('\\')
-            result.append('t')
-            j += 1
-            continue
-        if code == ord('\r'):
-            result.append('\\')
-            result.append('r')
-            j += 1
-            continue
-        if code == ord('\n'):
-            result.append('\\')
-            result.append('n')
-            j += 1
-            continue
-        if code < ord(' ') or code >= 0x7f:
-            result.extend(['\\', "x",
-                           hexdigits[(code >> 4) & 0xf], 
-                           hexdigits[(code >> 0) & 0xf],
-                          ])
-            j += 1
-            continue
-        result.append(chr(code))
-        j += 1
-    result.append(quote)
-    return space.wrap(''.join(result))
-        
+    s = unicode_encode_unicode_escape(chars, size, "strict", quotes=True)
+    return space.wrap(s)
 
 def mod__Unicode_ANY(space, w_format, w_values):
     return mod_format(space, w_format, w_values, do_unicode=True)

Modified: pypy/branch/fast-forward/pypy/rlib/debug.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rlib/debug.py	(original)
+++ pypy/branch/fast-forward/pypy/rlib/debug.py	Tue Jul  6 00:00:33 2010
@@ -250,3 +250,16 @@
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
         return hop.inputarg(hop.args_r[0], arg=0)
+
+
+class IntegerCanBeNegative(Exception):
+    pass
+
+def _check_nonneg(ann, bk):
+    from pypy.annotation.model import SomeInteger
+    s_nonneg = SomeInteger(nonneg=True)
+    if not s_nonneg.contains(ann):
+        raise IntegerCanBeNegative
+
+def check_nonneg(x):
+    check_annotation(x, _check_nonneg)

Modified: pypy/branch/fast-forward/pypy/rlib/libffi.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rlib/libffi.py	(original)
+++ pypy/branch/fast-forward/pypy/rlib/libffi.py	Tue Jul  6 00:00:33 2010
@@ -402,7 +402,7 @@
                     restype = ffi_type_sint32
                 elif restype.c_size <= 8:
                     restype = ffi_type_sint64
-                
+
         res = c_ffi_prep_cif(self.ll_cif, cc,
                              rffi.cast(rffi.UINT, argnum), restype,
                              self.ll_argtypes)

Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py	(original)
+++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py	Tue Jul  6 00:00:33 2010
@@ -57,13 +57,10 @@
     from math import isinf, isnan, copysign
 except ImportError:
     def isinf(x):
-        return x != 0.0 and x / 2 == x
+        return x == INFINITY or x == -INFINITY
 
-    # To get isnan, working x-platform and both on 2.3 and 2.4, is a
-    # horror.  I think this works (for reasons I don't really want to talk
-    # about), and probably when implemented on top of pypy, too.
     def isnan(v):
-        return v != v*1.0 or (v == 1.0 and v == 2.0)
+        return v != v
 
     def copysign(x, y):
         """Return x with the sign of y"""

Modified: pypy/branch/fast-forward/pypy/rlib/rcoroutine.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rlib/rcoroutine.py	(original)
+++ pypy/branch/fast-forward/pypy/rlib/rcoroutine.py	Tue Jul  6 00:00:33 2010
@@ -32,6 +32,8 @@
 from pypy.rlib.rstack import yield_current_frame_to_caller, resume_point
 from pypy.rlib.objectmodel import we_are_translated
 
+from pypy.interpreter.error import OperationError
+
 try:
     from greenlet import greenlet
     main_greenlet = greenlet.getcurrent()
@@ -158,6 +160,7 @@
             self.costate = state
             self.parent = None
             self.thunk = None
+            self.coroutine_exit = False
 
         def __repr__(self):
             'NOT_RPYTHON'
@@ -236,11 +239,12 @@
                     self = state.current
                     self.finish(exc)
             except CoroutineExit:
-                # ignore a shutdown exception
                 pass
             except Exception, e:
-                # redirect all unhandled exceptions to the parent
-                syncstate.push_exception(e)
+                if self.coroutine_exit is False:
+                    # redirect all unhandled exceptions to the parent
+                    syncstate.push_exception(e)
+
             while self.parent is not None and self.parent.frame is None:
                 # greenlet behavior is fine
                 self.parent = self.parent.parent
@@ -257,10 +261,13 @@
             syncstate.switched(incoming_frame)
 
         def kill(self):
+            self._kill(CoroutineExit())
+
+        def _kill(self, exc):
             if self.frame is None:
                 return
             state = self.costate
-            syncstate.push_exception(CoroutineExit())
+            syncstate.push_exception(exc)
             # careful here - if setting self.parent to state.current would
             # create a loop, break it.  The assumption is that 'self'
             # will die, so that state.current's chain of parents can be

Modified: pypy/branch/fast-forward/pypy/rlib/rsre/test/targetrsre.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rlib/rsre/test/targetrsre.py	(original)
+++ pypy/branch/fast-forward/pypy/rlib/rsre/test/targetrsre.py	Tue Jul  6 00:00:33 2010
@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 from pypy.rlib.rarithmetic import intmask
 from pypy.rlib.rsre import rsre
 import os, time

Modified: pypy/branch/fast-forward/pypy/rlib/rstring.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rlib/rstring.py	(original)
+++ pypy/branch/fast-forward/pypy/rlib/rstring.py	Tue Jul  6 00:00:33 2010
@@ -56,6 +56,7 @@
         self.l.append(s)
 
     def append_slice(self, s, start, end):
+        assert 0 <= start <= end <= len(s)
         self.l.append(s[start:end])
 
     def append_multiple_char(self, c, times):

Modified: pypy/branch/fast-forward/pypy/rlib/runicode.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rlib/runicode.py	(original)
+++ pypy/branch/fast-forward/pypy/rlib/runicode.py	Tue Jul  6 00:00:33 2010
@@ -1,7 +1,9 @@
 import sys
 from pypy.rlib.bitmanipulation import splitter
 from pypy.rpython.lltypesystem import lltype, rffi
-from pypy.rlib.objectmodel import we_are_translated
+from pypy.rlib.objectmodel import we_are_translated, specialize
+from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
+from pypy.rlib.rarithmetic import r_uint
 
 if rffi.sizeof(lltype.UniChar) == 4:
     MAXUNICODE = 0x10ffff
@@ -42,8 +44,6 @@
     UNICHR = unichr
     ORD = ord
 
-# XXX review the functions below and think about using stringbuilders for them
-
 
 def raise_unicode_exception_decode(errors, encoding, msg, s,
                                    startingpos, endingpos):
@@ -55,8 +55,8 @@
     assert isinstance(u, unicode)
     raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
 
-# ____________________________________________________________ 
-# unicode decoding
+# ____________________________________________________________
+# utf-8
 
 utf8_code_length = [
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
@@ -81,9 +81,10 @@
                      errorhandler=None):
     if errorhandler is None:
         errorhandler = raise_unicode_exception_decode
-    if (size == 0):
+    if size == 0:
         return u'', 0
-    result = []
+
+    result = UnicodeBuilder(size)
     pos = 0
     while pos < size:
         ch = s[pos]
@@ -94,14 +95,14 @@
             continue
 
         n = utf8_code_length[ordch1]
-        if (pos + n > size):
+        if pos + n > size:
             if not final:
                 break
             else:
                 r, pos = errorhandler(errors, "utf-8",
                                       "unexpected end of data", s,  pos, size)
                 result.append(r)
-                if (pos + n > size):
+                if pos + n > size:
                     break
         if n == 0:
             r, pos = errorhandler(errors, "utf-8", "unexpected code byte",
@@ -116,7 +117,7 @@
             z, two = splitter[6, 2](ordch2)
             y, six = splitter[5, 3](ordch1)
             assert six == 6
-            if (two != 2):
+            if two != 2:
                 r, pos = errorhandler(errors, "utf-8", "invalid data",
                                       s,  pos, pos + 2)
                 result.append(r)
@@ -137,7 +138,7 @@
             y, two2 = splitter[6, 2](ordch2)
             x, fourteen = splitter[4, 4](ordch1)
             assert fourteen == 14
-            if (two1 != 2 or two2 != 2):
+            if two1 != 2 or two2 != 2:
                 r, pos = errorhandler(errors, "utf-8", "invalid data",
                                       s,  pos, pos + 3)
                 result.append(r)
@@ -166,7 +167,7 @@
             x, two3 = splitter[6, 2](ordch2)
             w, thirty = splitter[3, 5](ordch1)
             assert thirty == 30
-            if (two1 != 2 or two2 != 2 or two3 != 2):
+            if two1 != 2 or two2 != 2 or two3 != 2:
                 r, pos = errorhandler(errors, "utf-8", "invalid data",
                                       s,  pos, pos + 4)
                 result.append(r)
@@ -174,7 +175,7 @@
                 c = (w << 18) + (x << 12) + (y << 6) + z
                 # minimum value allowed for 4 byte encoding
                 # maximum value allowed for UTF-16
-                if ((c < 0x10000) or (c > 0x10ffff)):
+                if c < 0x10000 or c > 0x10ffff:
                     r, pos = errorhandler(errors, "utf-8", "illegal encoding",
                                           s,  pos, pos + 4)
                     result.append(r)
@@ -197,8 +198,53 @@
                                   s,  pos, pos + n)
             result.append(r)
 
-    return u"".join(result), pos
+    return result.build(), pos
 
+def _encodeUCS4(result, ch):
+    # Encode UCS4 Unicode ordinals
+    result.append((chr((0xf0 | (ch >> 18)))))
+    result.append((chr((0x80 | ((ch >> 12) & 0x3f)))))
+    result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))
+    result.append((chr((0x80 | (ch & 0x3f)))))
+
+def unicode_encode_utf_8(s, size, errors, errorhandler=None):
+    assert(size >= 0)
+    result = StringBuilder(size)
+    i = 0
+    while i < size:
+        ch = ord(s[i])
+        i += 1
+        if ch < 0x80:
+            # Encode ASCII
+            result.append(chr(ch))
+        elif ch < 0x0800:
+            # Encode Latin-1
+            result.append(chr((0xc0 | (ch >> 6))))
+            result.append(chr((0x80 | (ch & 0x3f))))
+        else:
+            # Encode UCS2 Unicode ordinals
+            if ch < 0x10000:
+                # Special case: check for high surrogate
+                if 0xD800 <= ch <= 0xDBFF and i != size:
+                    ch2 = ord(s[i])
+                    # Check for low surrogate and combine the two to
+                    # form a UCS4 value
+                    if 0xDC00 <= ch2 <= 0xDFFF:
+                        ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000
+                        i += 1
+                        _encodeUCS4(result, ch3)
+                        continue
+                # Fall through: handles isolated high surrogates
+                result.append((chr((0xe0 | (ch >> 12)))))
+                result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))
+                result.append((chr((0x80 | (ch & 0x3f)))))
+                continue
+            else:
+                _encodeUCS4(result, ch)
+    return result.build()
+
+# ____________________________________________________________
+# utf-16
 
 def str_decode_utf_16(s, size, errors, final=True,
                       errorhandler=None):
@@ -238,12 +284,11 @@
     #  mark is skipped, in all other modes, it is copied to the output
     #  stream as-is (giving a ZWNBSP character).
     pos = 0
-    result = []
     if byteorder == 'native':
-        if (size >= 2):
+        if size >= 2:
             bom = (ord(s[ihi]) << 8) | ord(s[ilo])
             if BYTEORDER == 'little':
-                if (bom == 0xFEFF):
+                if bom == 0xFEFF:
                     pos += 2
                     bo = -1
                 elif bom == 0xFFFE:
@@ -260,20 +305,22 @@
         bo = -1
     else:
         bo = 1
-    if (size == 0):
+    if size == 0:
         return u'', 0, bo
-    if (bo == -1):
+    if bo == -1:
         # force little endian
         ihi = 1
         ilo = 0
 
-    elif (bo == 1):
+    elif bo == 1:
         # force big endian
         ihi = 0
         ilo = 1
 
+    result = UnicodeBuilder(size // 2)
+
     #XXX I think the errors are not correctly handled here
-    while (pos < len(s)):
+    while pos < size:
         # remaining bytes at the end? (size should be even)
         if len(s) - pos < 2:
             if not final:
@@ -285,7 +332,7 @@
                 break
         ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo])
         pos += 2
-        if (ch < 0xD800 or ch > 0xDFFF):
+        if ch < 0xD800 or ch > 0xDFFF:
             result.append(unichr(ch))
             continue
         # UTF-16 code pair:
@@ -297,10 +344,10 @@
             result.append(r)
             if len(s) - pos < 2:
                 break
-        elif (0xD800 <= ch and ch <= 0xDBFF):
+        elif 0xD800 <= ch <= 0xDBFF:
             ch2 = (ord(s[pos+ihi]) << 8) | ord(s[pos+ilo])
             pos += 2
-            if (0xDC00 <= ch2 and ch2 <= 0xDFFF):
+            if 0xDC00 <= ch2 <= 0xDFFF:
                 if MAXUNICODE < 65536:
                     result.append(unichr(ch))
                     result.append(unichr(ch2))
@@ -318,17 +365,305 @@
                                   "illegal encoding",
                                   s, pos - 2, pos)
             result.append(r)
-    return u"".join(result), pos, bo
+    return result.build(), pos, bo
+
+def _STORECHAR(result, CH, byteorder):
+    hi = chr(((CH) >> 8) & 0xff)
+    lo = chr((CH) & 0xff)
+    if byteorder == 'little':
+        result.append(lo)
+        result.append(hi)
+    else:
+        result.append(hi)
+        result.append(lo)
+
+def unicode_encode_utf_16_helper(s, size, errors,
+                                 errorhandler=None,
+                                 byteorder='little'):
+    if size == 0:
+        return ""
+
+    result = StringBuilder(size * 2 + 2)
+    if byteorder == 'native':
+        _STORECHAR(result, 0xFEFF, BYTEORDER)
+        byteorder = BYTEORDER
+
+    i = 0
+    while i < size:
+        ch = ord(s[i])
+        i += 1
+        ch2 = 0
+        if ch >= 0x10000:
+            ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF)
+            ch  = 0xD800 | ((ch-0x10000) >> 10)
+
+        _STORECHAR(result, ch, byteorder)
+        if ch2:
+            _STORECHAR(result, ch2, byteorder)
+
+    return result.build()
+
+def unicode_encode_utf_16(s, size, errors,
+                          errorhandler=None):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native")
+
+
+def unicode_encode_utf_16_be(s, size, errors,
+                             errorhandler=None):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big")
+
+
+def unicode_encode_utf_16_le(s, size, errors,
+                             errorhandler=None):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little")
+
+
+# ____________________________________________________________
+# utf-7
+
+## indicate whether a UTF-7 character is special i.e. cannot be directly
+##       encoded:
+##         0 - not special
+##         1 - special
+##         2 - whitespace (optional)
+##         3 - RFC2152 Set O (optional)
+
+_utf7_special = [
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0,
+    3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3,
+    3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1,
+]
+
+def _utf7_SPECIAL(oc, encodeO=False, encodeWS=False):
+    return (oc > 127 or _utf7_special[oc] == 1 or
+            (encodeWS and _utf7_special[oc] == 2) or
+            (encodeO and _utf7_special[oc] == 3))
+
+def _utf7_B64CHAR(oc):
+    if oc > 127:
+        return False
+    c = chr(oc)
+    return c.isalnum() or c == '+' or c == '/'
+def _utf7_TO_BASE64(n):
+    "Returns the base-64 character of the bottom 6 bits of n"
+    return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[n & 0x3f]
+def _utf7_FROM_BASE64(c):
+    "Retuns the base-64 value of a base-64 character"
+    if c == '+':
+        return 62
+    elif c == '/':
+        return 63
+    elif c >= 'a':
+        return ord(c) - 71
+    elif c >= 'A':
+        return ord(c) - 65
+    else:
+        return ord(c) + 4
+
+def _utf7_ENCODE(result, ch, bits):
+    while bits >= 6:
+        result.append(_utf7_TO_BASE64(ch >> (bits - 6)))
+        bits -= 6
+    return bits
+
+def _utf7_DECODE(s, result, errorhandler, errors,
+                 pos, charsleft, bitsleft, surrogate):
+    while bitsleft >= 16:
+        outCh =  (charsleft >> (bitsleft-16)) & 0xffff
+        bitsleft -= 16
+
+        if surrogate:
+            ## We have already generated an error for the high
+            ## surrogate so let's not bother seeing if the low
+            ## surrogate is correct or not
+            surrogate = False
+        elif 0xDC00 <= outCh <= 0xDFFF:
+            ## This is a surrogate pair. Unfortunately we can't
+            ## represent it in a 16-bit character
+            surrogate = True
+            msg = "code pairs are not supported"
+            res, pos = errorhandler(errors, 'utf-7',
+                                    msg, s, pos-1, pos)
+            result.append(res)
+            bitsleft = 0
+            break
+        else:
+            result.append(unichr(outCh))
+    return pos, charsleft, bitsleft, surrogate
+
+
+def str_decode_utf_7(s, size, errors, final=False,
+                     errorhandler=None):
+    if errorhandler is None:
+        errorhandler = raise_unicode_exception_decode
+    if size == 0:
+        return u'', 0
+
+    inShift = False
+    bitsleft = 0
+    startinpos = 0
+    charsleft = 0
+    surrogate = False
+
+    result = UnicodeBuilder(size)
+    pos = 0
+    while pos < size:
+        ch = s[pos]
+        oc = ord(ch)
+
+        if inShift:
+            if ch == '-' or not _utf7_B64CHAR(oc):
+                inShift = 0
+                pos += 1
+
+                pos, charsleft, bitsleft, surrogate = _utf7_DECODE(
+                    s, result, errorhandler, errors,
+                    pos, charsleft, bitsleft, surrogate)
+                if bitsleft >= 6:
+                    ## The shift sequence has a partial character in it. If
+                    ## bitsleft < 6 then we could just classify it as padding
+                    ## but that is not the case here
+                    msg = "partial character in shift sequence"
+                    res, pos = errorhandler(errors, 'utf-7',
+                                            msg, s, pos-1, pos)
+                    result.append(res)
+                    ## According to RFC2152 the remaining bits should be
+                    ## zero. We choose to signal an error/insert a replacement
+                    ## character here so indicate the potential of a
+                    ## misencoded character.
+                if ch == '-':
+                    if pos < size and s[pos] == '-':
+                        result.append(u'-')
+                        inShift = True
+
+                elif _utf7_SPECIAL(oc):
+                    msg = "unexpected special character"
+                    res, pos = errorhandler(errors, 'utf-7',
+                                            msg, s, pos-1, pos)
+                    result.append(res)
+                else:
+                    result.append(unichr(ord(ch)))
+            else:
+                charsleft = (charsleft << 6) | _utf7_FROM_BASE64(ch)
+                bitsleft += 6
+                pos += 1
+
+                pos, charsleft, bitsleft, surrogate = _utf7_DECODE(
+                    s, result, errorhandler, errors,
+                    pos, charsleft, bitsleft, surrogate)
+        elif ch == '+':
+            startinpos = pos
+            pos += 1
+            if pos < size and s[pos] == '-':
+                pos += 1
+                result.append(u'+')
+            else:
+                inShift = 1
+                bitsleft = 0
+
+        elif _utf7_SPECIAL(oc):
+            pos += 1
+            msg = "unexpected special character"
+            res, pos = errorhandler(errors, 'utf-7', msg, s, pos-1, pos)
+            result.append(res)
+        else:
+            result.append(unichr(oc))
+            pos += 1
+
+    if inShift:
+        endinpos = size
+        msg = "unterminated shift sequence"
+        res, pos = errorhandler(errors, 'utf-7', msg, s, startinpos, pos)
+        result.append(res)
+
+    return result.build(), pos
+
+def unicode_encode_utf_7(s, size, errors, errorhandler=None):
+    if size == 0:
+        return ''
+    result = StringBuilder(size)
+
+    encodeSetO = encodeWhiteSpace = False
+
+    inShift = False
+    bitsleft = 0
+    charsleft = 0
+
+    pos = 0
+    while pos < size:
+        ch = s[pos]
+        oc = ord(ch)
+        if not inShift:
+            if ch == u'+':
+                result.append('+-')
+            elif _utf7_SPECIAL(oc, encodeSetO, encodeWhiteSpace):
+                charsleft = oc
+                bitsleft = 16
+                result.append('+')
+                bitsleft = _utf7_ENCODE(result, charsleft, bitsleft)
+                inShift = bitsleft > 0
+            else:
+                result.append(chr(oc))
+        else:
+            if not _utf7_SPECIAL(oc, encodeSetO, encodeWhiteSpace):
+                result.append(_utf7_TO_BASE64(charsleft << (6-bitsleft)))
+                charsleft = 0
+                bitsleft = 0
+                ## Characters not in the BASE64 set implicitly unshift the
+                ## sequence so no '-' is required, except if the character is
+                ## itself a '-'
+                if _utf7_B64CHAR(oc) or ch == u'-':
+                    result.append('-')
+                inShift = False
+                result.append(chr(oc))
+            else:
+                bitsleft += 16
+                charsleft = (charsleft << 16) | oc
+                bitsleft =  _utf7_ENCODE(result, charsleft, bitsleft)
+                ## If the next character is special then we dont' need to
+                ## terminate the shift sequence. If the next character is not
+                ## a BASE64 character or '-' then the shift sequence will be
+                ## terminated implicitly and we don't have to insert a '-'.
+                if bitsleft == 0:
+                    if pos + 1 < size:
+                        ch2 = s[pos + 1]
+                        oc2 = ord(ch2)
+
+                        if _utf7_SPECIAL(oc2, encodeSetO, encodeWhiteSpace):
+                            pass
+                        elif _utf7_B64CHAR(oc2) or ch2 == u'-':
+                            result.append('-')
+                            inShift = False
+                        else:
+                            inShift = False
+                    else:
+                        result.append('-')
+                        inShift = False
+        pos += 1
+
+    if bitsleft:
+        result.append(_utf7_TO_BASE64(charsleft << (6 - bitsleft)))
+        result.append('-')
+
+    return result.build()
+
+# ____________________________________________________________
+# ascii and latin-1
 
 def str_decode_latin_1(s, size, errors, final=False,
                        errorhandler=None):
     # latin1 is equivalent to the first 256 ordinals in Unicode.
     pos = 0
-    result = []
-    while (pos < size):
+    result = UnicodeBuilder(size)
+    while pos < size:
         result.append(unichr(ord(s[pos])))
         pos += 1
-    return u"".join(result), pos
+    return result.build(), pos
 
 
 def str_decode_ascii(s, size, errors, final=False,
@@ -336,9 +671,9 @@
     if errorhandler is None:
         errorhandler = raise_unicode_exception_decode
     # ASCII is equivalent to the first 128 ordinals in Unicode.
-    result = []
+    result = UnicodeBuilder(size)
     pos = 0
-    while pos < len(s):
+    while pos < size:
         c = s[pos]
         if ord(c) < 128:
             result.append(unichr(ord(c)))
@@ -347,55 +682,7 @@
             r, pos = errorhandler(errors, "ascii", "ordinal not in range(128)",
                                   s,  pos, pos + 1)
             result.append(r)
-    return u"".join(result), pos
-
-
-# ____________________________________________________________ 
-# unicode encoding 
-
-
-def unicode_encode_utf_8(s, size, errors, errorhandler=None):
-    assert(size >= 0)
-    result = []
-    i = 0
-    while i < size:
-        ch = ord(s[i])
-        i += 1
-        if (ch < 0x80):
-            # Encode ASCII 
-            result.append(chr(ch))
-        elif (ch < 0x0800) :
-            # Encode Latin-1 
-            result.append(chr((0xc0 | (ch >> 6))))
-            result.append(chr((0x80 | (ch & 0x3f))))
-        else:
-            # Encode UCS2 Unicode ordinals
-            if (ch < 0x10000):
-                # Special case: check for high surrogate
-                if (0xD800 <= ch and ch <= 0xDBFF and i != size) :
-                    ch2 = ord(s[i])
-                    # Check for low surrogate and combine the two to
-                    # form a UCS4 value
-                    if (0xDC00 <= ch2 and ch2 <= 0xDFFF) :
-                        ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000
-                        i += 1
-                        _encodeUCS4(result, ch3)
-                        continue
-                # Fall through: handles isolated high surrogates
-                result.append((chr((0xe0 | (ch >> 12)))))
-                result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))
-                result.append((chr((0x80 | (ch & 0x3f)))))
-                continue
-            else:
-                _encodeUCS4(result, ch)
-    return "".join(result)
-
-def _encodeUCS4(result, ch):
-    # Encode UCS4 Unicode ordinals
-    result.append((chr((0xf0 | (ch >> 18)))))
-    result.append((chr((0x80 | ((ch >> 12) & 0x3f)))))
-    result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))
-    result.append((chr((0x80 | (ch & 0x3f)))))
+    return result.build(), pos
 
 
 def unicode_encode_ucs1_helper(p, size, errors,
@@ -408,12 +695,12 @@
     else:
         reason = "ordinal not in range(128)"
         encoding = "ascii"
-    
-    if (size == 0):
+
+    if size == 0:
         return ''
-    result = []
+    result = StringBuilder(size)
     pos = 0
-    while pos < len(p):
+    while pos < size:
         ch = p[pos]
         
         if ord(ch) < limit:
@@ -427,9 +714,9 @@
                 collend += 1
             r, pos = errorhandler(errors, encoding, reason, p,
                                   collstart, collend)
-            result += r   # extend 'result' as a list of characters
+            result.append(r)
     
-    return "".join(result)
+    return result.build()
 
 def unicode_encode_latin_1(p, size, errors, errorhandler=None):
     res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 256)
@@ -439,57 +726,479 @@
     res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 128)
     return res
 
+# ____________________________________________________________
+# Charmap
 
-def _STORECHAR(result, CH, byteorder):
-    hi = chr(((CH) >> 8) & 0xff)
-    lo = chr((CH) & 0xff)
-    if byteorder == 'little':
-        result.append(lo)
-        result.append(hi)
+ERROR_CHAR = u'\ufffe'
+
+ at specialize.argtype(5)
+def str_decode_charmap(s, size, errors, final=False,
+                       errorhandler=None, mapping=None):
+    "mapping can be a rpython dictionary, or a dict-like object."
+
+    # Default to Latin-1
+    if mapping is None:
+        return str_decode_latin_1(s, size, errors, final=final,
+                                  errorhandler=errorhandler)
+    if errorhandler is None:
+        errorhandler = raise_unicode_exception_decode
+    if size == 0:
+        return u'', 0
+
+    pos = 0
+    result = UnicodeBuilder(size)
+    while pos < size:
+        ch = s[pos]
+
+        c = mapping.get(ch, ERROR_CHAR)
+        if c == ERROR_CHAR:
+            r, pos = errorhandler(errors, "charmap",
+                                  "character maps to <undefined>",
+                                  s,  pos, pos + 1)
+            result.append(r)
+            continue
+        result.append(c)
+        pos += 1
+    return result.build(), pos
+
+def unicode_encode_charmap(s, size, errors, errorhandler=None,
+                           mapping=None):
+    if mapping is None:
+        return unicode_encode_latin_1(s, size, errors,
+                                      errorhandler=errorhandler)
+
+    if errorhandler is None:
+        errorhandler = raise_unicode_exception_encode
+
+    if size == 0:
+        return ''
+    result = StringBuilder(size)
+    pos = 0
+    while pos < size:
+        ch = s[pos]
+
+        c = mapping.get(ch, '')
+        if len(c) == 0:
+            res, pos = errorhandler(errors, "charmap",
+                                    "character maps to <undefined>",
+                                    s, pos, pos + 1)
+            for ch2 in res:
+                c2 = mapping.get(unichr(ord(ch2)), '')
+                if len(c2) == 0:
+                    errorhandler(
+                        "strict", "charmap",
+                        "character maps to <undefined>",
+                        s,  pos, pos + 1)
+                result.append(c2)
+            continue
+        result.append(c)
+        pos += 1
+    return result.build()
+
+# ____________________________________________________________
+# Unicode escape
+
+hexdigits = "0123456789ABCDEFabcdef"
+
+def hexescape(builder, s, pos, digits,
+              encoding, errorhandler, message, errors):
+    import sys
+    chr = 0
+    if pos + digits > len(s):
+        message = "end of string in escape sequence"
+        res, pos = errorhandler(errors, "unicodeescape",
+                                message, s, pos-2, len(s))
+        builder.append(res)
     else:
-        result.append(hi)
-        result.append(lo)
+        try:
+            chr = r_uint(int(s[pos:pos+digits], 16))
+        except ValueError:
+            endinpos = pos
+            while s[endinpos] in hexdigits:
+                endinpos += 1
+            res, pos = errorhandler(errors, encoding,
+                                    message, s, pos-2, endinpos+1)
+            builder.append(res)
+        else:
+            # when we get here, chr is a 32-bit unicode character
+            if chr <= MAXUNICODE:
+                builder.append(UNICHR(chr))
+                pos += digits
+
+            elif chr <= 0x10ffff:
+                chr -= 0x10000L
+                builder.append(unichr(0xD800 + (chr >> 10)))
+                builder.append(unichr(0xDC00 + (chr & 0x03FF)))
+                pos += digits
+            else:
+                message = "illegal Unicode character"
+                res, pos = errorhandler(errors, encoding,
+                                        message, s, pos-2, pos+digits)
+                builder.append(res)
+    return pos
+
+def str_decode_unicode_escape(s, size, errors, final=False,
+                              errorhandler=False,
+                              unicodedata_handler=None):
+    if errorhandler is None:
+        errorhandler = raise_unicode_exception_decode
 
-def unicode_encode_utf_16_helper(s, size, errors,
-                                 errorhandler=None,
-                                 byteorder='little'):
-    result = []
-    if (byteorder == 'native'):
-        _STORECHAR(result, 0xFEFF, BYTEORDER)
-        byteorder = BYTEORDER
-        
     if size == 0:
-        return ""
+        return u'', 0
 
-    i = 0
-    while i < size:
-        ch = ord(s[i])
-        i += 1
-        ch2 = 0
-        if (ch >= 0x10000) :
-            ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF)
-            ch  = 0xD800 | ((ch-0x10000) >> 10)
+    builder = UnicodeBuilder(size)
+    pos = 0
+    while pos < size:
+        ch = s[pos]
 
-        _STORECHAR(result, ch, byteorder)
-        if ch2:
-            _STORECHAR(result, ch2, byteorder)
+        # Non-escape characters are interpreted as Unicode ordinals
+        if ch != '\\':
+            builder.append(unichr(ord(ch)))
+            pos += 1
+            continue
 
-    return "".join(result)
+        # - Escapes
+        pos += 1
+        if pos >= size:
+            message = "\\ at end of string"
+            res, pos = errorhandler(errors, "unicodeescape",
+                                    message, s, pos-1, size)
+            builder.append(res)
+            continue
 
-def unicode_encode_utf_16(s, size, errors,
-                          errorhandler=None):
-    return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native")
+        ch = s[pos]
+        pos += 1
+        # \x escapes
+        if ch == '\n': pass
+        elif ch == '\\': builder.append(u'\\')
+        elif ch == '\'': builder.append(u'\'')
+        elif ch == '\"': builder.append(u'\"')
+        elif ch == 'b' : builder.append(u'\b')
+        elif ch == 'f' : builder.append(u'\f')
+        elif ch == 't' : builder.append(u'\t')
+        elif ch == 'n' : builder.append(u'\n')
+        elif ch == 'r' : builder.append(u'\r')
+        elif ch == 'v' : builder.append(u'\v')
+        elif ch == 'a' : builder.append(u'\a')
+        elif '0' <= ch <= '7':
+            x = ord(ch) - ord('0')
+            if pos < size:
+                ch = s[pos]
+                if '0' <= ch <= '7':
+                    pos += 1
+                    x = (x<<3) + ord(ch) - ord('0')
+                    if pos < size:
+                        ch = s[pos]
+                        if '0' <= ch <= '7':
+                            pos += 1
+                            x = (x<<3) + ord(ch) - ord('0')
+            builder.append(unichr(x))
+        # hex escapes
+        # \xXX
+        elif ch == 'x':
+            digits = 2
+            message = "truncated \\xXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        # \uXXXX
+        elif ch == 'u':
+            digits = 4
+            message = "truncated \\uXXXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        #  \UXXXXXXXX
+        elif ch == 'U':
+            digits = 8
+            message = "truncated \\UXXXXXXXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        # \N{name}
+        elif ch == 'N':
+            message = "malformed \\N character escape"
+            look = pos
+            if unicodedata_handler is None:
+                message = ("\\N escapes not supported "
+                           "(can't load unicodedata module)")
+                res, pos = errorhandler(errors, "unicodeescape",
+                                        message, s, pos-1, size)
+                builder.append(res)
+                continue
+
+            if look < size and s[look] == '{':
+                # look for the closing brace
+                while look < size and s[look] != '}':
+                    look += 1
+                if look < size and s[look] == '}':
+                    # found a name.  look it up in the unicode database
+                    message = "unknown Unicode character name"
+                    name = s[pos+1:look]
+                    code = unicodedata_handler.call(name)
+                    if code < 0:
+                        res, pos = errorhandler(errors, "unicodeescape",
+                                                message, s, pos-1, look+1)
+                        builder.append(res)
+                        continue
+                    pos = look + 1
+                    if code <= MAXUNICODE:
+                        builder.append(UNICHR(code))
+                    else:
+                        code -= 0x10000L
+                        builder.append(unichr(0xD800 + (code >> 10)))
+                        builder.append(unichr(0xDC00 + (code & 0x03FF)))
+                else:
+                    res, pos = errorhandler(errors, "unicodeescape",
+                                            message, s, pos-1, look+1)
+                    builder.append(res)
+            else:
+                res, pos = errorhandler(errors, "unicodeescape",
+                                        message, s, pos-1, look+1)
+                builder.append(res)
+        else:
+            builder.append(u'\\')
+            builder.append(unichr(ord(ch)))
 
+    return builder.build(), pos
 
-def unicode_encode_utf_16_be(s, size, errors,
-                             errorhandler=None):
-    return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big")
+def unicode_encode_unicode_escape(s, size, errors, errorhandler=None, quotes=False):
+    # errorhandler is not used: this function cannot cause Unicode errors
+    result = StringBuilder(size)
+
+    if quotes:
+        if s.find(u'\'') != -1 and s.find(u'\"') == -1:
+            quote = ord('\"')
+            result.append('u"')
+        else:
+            quote = ord('\'')
+            result.append('u\'')
+    else:
+        quote = 0
 
+        if size == 0:
+            return ''
 
-def unicode_encode_utf_16_le(s, size, errors,
-                             errorhandler=None):
-    return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little")
+    pos = 0
+    while pos < size:
+        ch = s[pos]
+        oc = ord(ch)
+
+        # Escape quotes
+        if quotes and (oc == quote or ch == '\\'):
+            result.append('\\')
+            result.append(chr(oc))
+            pos += 1
+            continue
+
+        if 0xD800 <= oc < 0xDC00 and pos + 1 < size:
+            # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes
+            pos += 1
+            oc2 = ord(s[pos])
+
+            if 0xDC00 <= oc2 <= 0xDFFF:
+                ucs = (((oc & 0x03FF) << 10) | (oc2 & 0x03FF)) + 0x00010000
+                raw_unicode_escape_helper(result, ucs)
+                pos += 1
+                continue
+            # Fall through: isolated surrogates are copied as-is
+            pos -= 1
+
+        # Map special whitespace to '\t', \n', '\r'
+        if ch == '\t':
+            result.append('\\t')
+        elif ch == '\n':
+            result.append('\\n')
+        elif ch == '\r':
+            result.append('\\r')
+        elif ch == '\\':
+            result.append('\\\\')
+
+        # Map non-printable or non-ascii to '\xhh' or '\uhhhh'
+        elif oc < 32 or oc >= 0x7F:
+            raw_unicode_escape_helper(result, oc)
+
+        # Copy everything else as-is
+        else:
+            result.append(chr(oc))
+        pos += 1
+
+    if quotes:
+        result.append(chr(quote))
+    return result.build()
+
+# ____________________________________________________________
+# Raw unicode escape
+
+def str_decode_raw_unicode_escape(s, size, errors, final=False,
+                                  errorhandler=None):
+    if errorhandler is None:
+        errorhandler = raise_unicode_exception_decode
+    if size == 0:
+        return u'', 0
+
+    result = UnicodeBuilder(size)
+    pos = 0
+    while pos < size:
+        ch = s[pos]
+
+        # Non-escape characters are interpreted as Unicode ordinals
+        if ch != '\\':
+            result.append(unichr(ord(ch)))
+            pos += 1
+            continue
+
+        startinpos = pos
+        # \u-escapes are only interpreted iff the number of leading
+        # backslashes is odd
+        bs = pos
+        while pos < size:
+            pos += 1
+            if pos == size or s[pos] != '\\':
+                break
+            result.append(u'\\')
+
+        # we have a backslash at the end of the string, stop here
+        if pos >= size:
+            result.append(u'\\')
+            break
+
+        if ((pos - bs) & 1 == 0 or
+            pos >= size or
+            (s[pos] != 'u' and s[pos] != 'U')):
+            result.append(u'\\')
+            result.append(unichr(ord(s[pos])))
+            pos += 1
+            continue
+
+        if s[pos] == 'u':
+            digits = 4
+            message = "truncated \\uXXXX escape"
+        else:
+            digits = 8
+            message = "truncated \\UXXXXXXXX escape"
+        pos += 1
+        pos = hexescape(result, s, pos, digits,
+                        "rawunicodeescape", errorhandler, message, errors)
+
+    return result.build(), pos
+
+def raw_unicode_escape_helper(result, char):
+    num = hex(char)
+    if char >= 0x10000:
+        result.append("\\U")
+        zeros = 8
+    elif char >= 0x100:
+        result.append("\\u")
+        zeros = 4
+    else:
+        result.append("\\x")
+        zeros = 2
+    lnum = len(num)
+    nb = zeros + 2 - lnum # num starts with '0x'
+    if nb > 0:
+        result.append_multiple_char('0', nb)
+    result.append_slice(num, 2, lnum)
+
+def unicode_encode_raw_unicode_escape(s, size, errors, errorhandler=None):
+    # errorhandler is not used: this function cannot cause Unicode errors
+    if size == 0:
+        return ''
+    result = StringBuilder(size)
+    pos = 0
+    while pos < size:
+        oc = ord(s[pos])
+        if oc < 0x100:
+            result.append(chr(oc))
+        else:
+            raw_unicode_escape_helper(result, oc)
+        pos += 1
+
+    return result.build()
+
+# ____________________________________________________________
+# unicode-internal
+
+def str_decode_unicode_internal(s, size, errors, final=False,
+                                errorhandler=None):
+    if errorhandler is None:
+        errorhandler = raise_unicode_exception_decode
+    if size == 0:
+        return u'', 0
+
+    if MAXUNICODE < 65536:
+        unicode_bytes = 2
+    else:
+        unicode_bytes = 4
+    if BYTEORDER == "little":
+        start = 0
+        stop = unicode_bytes
+        step = 1
+    else:
+        start = unicode_bytes - 1
+        stop = -1
+        step = -1
+
+    result = UnicodeBuilder(size // unicode_bytes)
+    pos = 0
+    while pos < size:
+        if pos > size - unicode_bytes:
+            res, pos = errorhandler(errors, "unicode_internal",
+                                    "truncated input",
+                                    s, pos, size)
+            result.append(res)
+            if pos > size - unicode_bytes:
+                break
+            continue
+        t = r_uint(0)
+        h = 0
+        for j in range(start, stop, step):
+            t += r_uint(ord(s[pos + j])) << (h*8)
+            h += 1
+        if t > MAXUNICODE:
+            res, pos = errorhandler(errors, "unicode_internal",
+                                    "unichr(%d) not in range" % (t,),
+                                    s, pos, pos + unicode_bytes)
+            result.append(res)
+            continue
+        result.append(unichr(t))
+        pos += unicode_bytes
+    return result.build(), pos
+
+def unicode_encode_unicode_internal(s, size, errors, errorhandler=None):
+    if size == 0:
+        return ''
+
+    if MAXUNICODE < 65536:
+        unicode_bytes = 2
+    else:
+        unicode_bytes = 4
+
+    result = StringBuilder(size * unicode_bytes)
+    pos = 0
+    while pos < size:
+        oc = ord(s[pos])
+        if MAXUNICODE < 65536:
+            if BYTEORDER == "little":
+                result.append(chr(oc       & 0xFF))
+                result.append(chr(oc >>  8 & 0xFF))
+            else:
+                result.append(chr(oc >>  8 & 0xFF))
+                result.append(chr(oc       & 0xFF))
+        else:
+            if BYTEORDER == "little":
+                result.append(chr(oc       & 0xFF))
+                result.append(chr(oc >>  8 & 0xFF))
+                result.append(chr(oc >> 16 & 0xFF))
+                result.append(chr(oc >> 24 & 0xFF))
+            else:
+                result.append(chr(oc >> 24 & 0xFF))
+                result.append(chr(oc >> 16 & 0xFF))
+                result.append(chr(oc >>  8 & 0xFF))
+                result.append(chr(oc       & 0xFF))
+        pos += 1
 
+    return result.build()
 
 # ____________________________________________________________
 # MBCS codecs for Windows

Modified: pypy/branch/fast-forward/pypy/rlib/test/test_debug.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rlib/test/test_debug.py	(original)
+++ pypy/branch/fast-forward/pypy/rlib/test/test_debug.py	Tue Jul  6 00:00:33 2010
@@ -3,6 +3,7 @@
 from pypy.rlib.debug import check_annotation, make_sure_not_resized
 from pypy.rlib.debug import debug_print, debug_start, debug_stop
 from pypy.rlib.debug import have_debug_prints
+from pypy.rlib.debug import check_nonneg, IntegerCanBeNegative
 from pypy.rlib import debug
 from pypy.rpython.test.test_llinterp import interpret
 
@@ -30,6 +31,16 @@
 
     py.test.raises(Error, "interpret(g, [3])")
 
+def test_check_nonneg():
+    def f(x):
+        assert x >= 5
+        check_nonneg(x)
+    interpret(f, [9])
+
+    def g(x):
+        check_nonneg(x-1)
+    py.test.raises(IntegerCanBeNegative, interpret, g, [9])
+
 def test_make_sure_not_resized():
     from pypy.annotation.listdef import TooLateForChange
     def f():

Modified: pypy/branch/fast-forward/pypy/rlib/test/test_runicode.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rlib/test/test_runicode.py	(original)
+++ pypy/branch/fast-forward/pypy/rlib/test/test_runicode.py	Tue Jul  6 00:00:33 2010
@@ -5,11 +5,14 @@
 def test_unichr():
     a = runicode.UNICHR(0xffff)
     assert a == u'\uffff'
-    a = runicode.UNICHR(0x10000)
-    if sys.maxunicode < 0x10000:
-        assert len(a) == 2      # surrogates
+    if runicode.MAXUNICODE > 0xffff:
+        a = runicode.UNICHR(0x10000)
+        if sys.maxunicode < 0x10000:
+            assert len(a) == 2      # surrogates
+        else:
+            assert len(a) == 1
     else:
-        assert len(a) == 1
+        py.test.raises(ValueError, runicode.UNICHR, 0x10000)
 
 
 class UnicodeTests(object):
@@ -228,7 +231,7 @@
 
 class TestTranslation(object):
     def setup_class(cls):
-        if len(runicode.UNICHR(0x10000)) == 2:
+        if runicode.MAXUNICODE != sys.maxunicode:
             py.test.skip("these tests cannot run on the llinterp")
 
     def test_utf8(self):

Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/rffi.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rpython/lltypesystem/rffi.py	(original)
+++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/rffi.py	Tue Jul  6 00:00:33 2010
@@ -642,16 +642,8 @@
         allows for the process to be performed without an extra copy.
         Make sure to call keep_buffer_alive_until_here on the returned values.
         """
-        str_chars_offset = (offsetof(STRTYPE, 'chars') + \
-                            itemoffsetof(STRTYPE.chars, 0))
-        gc_buf = rgc.malloc_nonmovable(STRTYPE, count)
-        if gc_buf:
-            realbuf = cast_ptr_to_adr(gc_buf) + str_chars_offset
-            raw_buf = cast(TYPEP, realbuf)
-            return raw_buf, gc_buf
-        else:
-            raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
-            return raw_buf, lltype.nullptr(STRTYPE)
+        raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
+        return raw_buf, lltype.nullptr(STRTYPE)
     alloc_buffer._always_inline_ = True # to get rid of the returned tuple
 
     # (char*, str, int, int) -> None
@@ -794,6 +786,8 @@
         return r_wchar_t.BITS/8
     if tp is lltype.Float:
         return 8
+    if tp is lltype.SingleFloat:
+        return 4
     assert isinstance(tp, lltype.Number)
     if tp is lltype.Signed:
         return ULONG._type.BITS/8

Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/test/test_rffi.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rpython/lltypesystem/test/test_rffi.py	(original)
+++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/test/test_rffi.py	Tue Jul  6 00:00:33 2010
@@ -707,6 +707,7 @@
             lltype.UniChar:  ctypes.c_wchar,
             lltype.Char:     ctypes.c_ubyte,
             DOUBLE:     ctypes.c_double,
+            FLOAT:      ctypes.c_float,
             SIGNEDCHAR: ctypes.c_byte,
             UCHAR:      ctypes.c_ubyte,
             SHORT:      ctypes.c_short,

Modified: pypy/branch/fast-forward/pypy/rpython/microbench/microbench.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rpython/microbench/microbench.py	(original)
+++ pypy/branch/fast-forward/pypy/rpython/microbench/microbench.py	Tue Jul  6 00:00:33 2010
@@ -3,7 +3,7 @@
 import sys
 import autopath
 from time import clock
-from py.compat import subprocess
+import subprocess
 from pypy.translator.interactive import Translation
 
 LOOPS = 10000000

Modified: pypy/branch/fast-forward/pypy/rpython/module/ll_termios.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rpython/module/ll_termios.py	(original)
+++ pypy/branch/fast-forward/pypy/rpython/module/ll_termios.py	Tue Jul  6 00:00:33 2010
@@ -85,7 +85,7 @@
     c_struct.c_c_lflag, ispeed, ospeed, cc = attributes
     try:
         for i in range(NCCS):
-            c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i]))
+            c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i][0]))
         error = c_cfsetispeed(c_struct, ispeed)
         if error == -1:
             raise termios.error(error, 'tcsetattr failed')

Modified: pypy/branch/fast-forward/pypy/rpython/rstr.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rpython/rstr.py	(original)
+++ pypy/branch/fast-forward/pypy/rpython/rstr.py	Tue Jul  6 00:00:33 2010
@@ -80,17 +80,6 @@
             # defaults to checking the length
             return super(AbstractStringRepr, self).rtype_is_true(hop)
 
-    def rtype_ord(self, hop):
-        string_repr = hop.args_r[0].repr
-        v_str, = hop.inputargs(string_repr)
-        c_zero = inputconst(Signed, 0)
-        v_chr = hop.gendirectcall(self.ll.ll_stritem_nonneg, v_str, c_zero)
-        if string_repr is hop.rtyper.type_system.rstr.string_repr:
-            return hop.genop('cast_char_to_int', [v_chr], resulttype=Signed)
-        else:
-            assert string_repr is hop.rtyper.type_system.rstr.unicode_repr
-            return hop.genop('cast_unichar_to_int', [v_chr], resulttype=Signed)
-
     def rtype_method_startswith(self, hop):
         str1_repr, str2_repr = self._str_reprs(hop)
         v_str, v_value = hop.inputargs(str1_repr, str2_repr)

Modified: pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py
==============================================================================
--- pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py	(original)
+++ pypy/branch/fast-forward/pypy/rpython/test/test_rstr.py	Tue Jul  6 00:00:33 2010
@@ -131,7 +131,7 @@
             s = c * mul
             res = 0
             for i in range(len(s)):
-                res = res*10 + ord(const(s[i])) - ord(const('0'))
+                res = res*10 + ord(const(s[i])[0]) - ord(const('0')[0])
             c2 = c
             c2 *= mul
             res = 10 * res + (c2 == s)
@@ -577,7 +577,7 @@
             sum = 0
             for num in l:
                  if len(num):
-                     sum += ord(num) - ord(const('0'))
+                     sum += ord(num[0]) - ord(const('0')[0])
             return sum + len(l) * 100
         for i in range(5):
             res = self.interpret(fn, [i])

Modified: pypy/branch/fast-forward/pypy/tool/release/force-builds.py
==============================================================================
--- pypy/branch/fast-forward/pypy/tool/release/force-builds.py	(original)
+++ pypy/branch/fast-forward/pypy/tool/release/force-builds.py	Tue Jul  6 00:00:33 2010
@@ -25,7 +25,7 @@
     'pypy-c-stackless-app-level-linux-x86-32',
     'pypy-c-app-level-win-32',
     'pypy-c-jit-linux-x86-32',
-    'pypy-c-jit-macosx-x86-32',
+#    'pypy-c-jit-macosx-x86-32',
     'pypy-c-jit-win-x86-32',
 ]
 

Modified: pypy/branch/fast-forward/pypy/tool/release/make_release.py
==============================================================================
--- pypy/branch/fast-forward/pypy/tool/release/make_release.py	(original)
+++ pypy/branch/fast-forward/pypy/tool/release/make_release.py	Tue Jul  6 00:00:33 2010
@@ -16,6 +16,8 @@
 from pypy.tool.udir import udir
 from pypy.tool.release.package import package
 import tarfile
+import os
+import shutil
 
 BASEURL = 'http://buildbot.pypy.org/nightly/'
 
@@ -47,28 +49,39 @@
     to_download = browse_nightly(branch)
     tmpdir = udir.join('download')
     tmpdir.ensure(dir=True)
-    for (kind, platform), (rev, name) in to_download.iteritems():
-        if platform == 'win32':
-            print 'Ignoring %s, windows unsupported' % name
-        else:
-            print "Downloading %s at rev %d" % (name, rev)
-            url = BASEURL + branch + "/" + name
-            data = urllib2.urlopen(url).read()
-            tmpdir.join(name).write(data, mode="wb")
-            t = tarfile.open(str(tmpdir.join(name)))
-            data = t.extractfile('pypy-c').read()
-            pypy_c = tmpdir.join('pypy-c')
-            pypy_c.write(data, mode="wb")
-            if kind == 'jit':
-                kind = ''
+    alltars = []
+    try:
+        os.chdir(str(tmpdir))
+        for (kind, platform), (rev, name) in to_download.iteritems():
+            if platform == 'win32':
+                print 'Ignoring %s, windows unsupported' % name
             else:
-                kind = '-' + kind
-            name = 'pypy-%s%s-%s' % (release, kind, platform)
-            builddir = package(py.path.local(autopath.pypydir).join('..'),
-                               name=name,
-                               override_pypy_c=pypy_c)
-            print "Build %s/%s.tar.bz2" % (builddir, name)
-    print "\nLook into %s for packages" % builddir
+                print "Downloading %s at rev %d" % (name, rev)
+                url = BASEURL + branch + "/" + name
+                data = urllib2.urlopen(url).read()
+                tmpdir.join(name).write(data, mode="wb")
+                t = tarfile.open(str(tmpdir.join(name)))
+                dirname = t.getmembers()[0].name
+                t.extractall(path=str(tmpdir))
+                os.system('mv %s %s' % (str(tmpdir.join(dirname)),
+                                        str(tmpdir.join('pypy-%s' % release))))
+                if kind == 'jit':
+                    kind = ''
+                else:
+                    kind = '-' + kind
+                olddir = os.getcwd()
+                name = 'pypy-%s-%s%s.tar.bz2' % (release, platform, kind)
+                print "Building %s" % name
+                t = tarfile.open(name, 'w:bz2')
+                t.add('pypy-%s' % release)
+                alltars.append(name)
+                t.close()
+                shutil.rmtree(str(tmpdir.join('pypy-1.3')))
+        for name in alltars:
+            print "Uploading %s" % name
+            os.system('scp %s codespeak.net:/www/pypy.org/htdocs/download' % name)
+    finally:
+        os.chdir(olddir)
 
 if __name__ == '__main__':
     if len(sys.argv) != 2:

Modified: pypy/branch/fast-forward/pypy/tool/release/package.py
==============================================================================
--- pypy/branch/fast-forward/pypy/tool/release/package.py	(original)
+++ pypy/branch/fast-forward/pypy/tool/release/package.py	Tue Jul  6 00:00:33 2010
@@ -57,11 +57,11 @@
     for file in ['LICENSE', 'README']:
         shutil.copy(str(basedir.join(file)), str(pypydir))
     pypydir.ensure('include', dir=True)
-    # we want to put there all *.h from module/cpyext/include
+    # we want to put there all *.h and *.inl from trunk/include
     # and from pypy/_interfaces
-    for n in basedir.join('pypy', 'module', 'cpyext', 'include').listdir('*.h'):
-        shutil.copy(str(n), str(pypydir.join('include')))
-    for n in basedir.join('pypy', '_interfaces').listdir('*.h'):
+    includedir = basedir.join('include')
+    headers = includedir.listdir('*.h') + includedir.listdir('*.inl')
+    for n in headers:
         shutil.copy(str(n), str(pypydir.join('include')))
     pypydir.ensure('bin', dir=True)
     archive_pypy_c = pypydir.join('bin', rename_pypy_c)

Modified: pypy/branch/fast-forward/pypy/tool/release/test/test_package.py
==============================================================================
--- pypy/branch/fast-forward/pypy/tool/release/test/test_package.py	(original)
+++ pypy/branch/fast-forward/pypy/tool/release/test/test_package.py	Tue Jul  6 00:00:33 2010
@@ -26,6 +26,16 @@
         assert prefix.join('README').check()
         th = tarfile.open(str(builddir.join('test.tar.bz2')))
         assert th.getmember('test/lib_pypy/syslog.py')
+
+        # the headers file could be not there, because they are copied into
+        # trunk/include only during translation
+        includedir = py.path.local(pypydir).dirpath().join('include')
+        def check_include(name):
+            if includedir.join(name).check(file=True):
+                assert th.getmember('test/include/%s' % name)
+        check_include('Python.h')
+        check_include('modsupport.inl')
+        check_include('pypy_decl.h')
     finally:
         if fake_pypy_c:
             pypy_c.remove()

Modified: pypy/branch/fast-forward/pypy/tool/test/test_killsubprocess.py
==============================================================================
--- pypy/branch/fast-forward/pypy/tool/test/test_killsubprocess.py	(original)
+++ pypy/branch/fast-forward/pypy/tool/test/test_killsubprocess.py	Tue Jul  6 00:00:33 2010
@@ -1,5 +1,5 @@
 import sys, time
-from py.compat import subprocess
+import subprocess
 from pypy.tool.killsubprocess import killsubprocess
 
 def waitdead(process):

Modified: pypy/branch/fast-forward/pypy/translator/backendopt/test/test_mallocprediction.py
==============================================================================
--- pypy/branch/fast-forward/pypy/translator/backendopt/test/test_mallocprediction.py	(original)
+++ pypy/branch/fast-forward/pypy/translator/backendopt/test/test_mallocprediction.py	Tue Jul  6 00:00:33 2010
@@ -169,7 +169,7 @@
     t, graph = rtype(entrypoint, [int])
     total0 = preparation(t, t.graphs, heuristic=heuristic)
     total = clever_inlining_and_malloc_removal(t)
-    assert total0 + total == 16
+    assert total == 5     # XXX total0 appears to vary
 
 def test_richards():
     from pypy.translator.goal.richards import entry_point

Modified: pypy/branch/fast-forward/pypy/translator/c/gcc/trackgcroot.py
==============================================================================
--- pypy/branch/fast-forward/pypy/translator/c/gcc/trackgcroot.py	(original)
+++ pypy/branch/fast-forward/pypy/translator/c/gcc/trackgcroot.py	Tue Jul  6 00:00:33 2010
@@ -17,6 +17,7 @@
 
 class FunctionGcRootTracker(object):
     skip = 0
+    COMMENT = "([#;].*)?"
 
     @classmethod
     def init_regexp(cls):
@@ -25,10 +26,10 @@
         cls.r_globllabel    = re.compile(cls.LABEL+r"=[.][+]%d\s*$"%cls.OFFSET_LABELS)
 
         cls.r_insn          = re.compile(r"\t([a-z]\w*)\s")
-        cls.r_unaryinsn     = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*$")
+        cls.r_unaryinsn     = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*" + cls.COMMENT + "$")
         cls.r_binaryinsn    = re.compile(r"\t[a-z]\w*\s+(?P<source>"+cls.OPERAND+"),\s*(?P<target>"+cls.OPERAND+")\s*$")
 
-        cls.r_jump          = re.compile(r"\tj\w+\s+"+cls.LABEL+"\s*$")
+        cls.r_jump          = re.compile(r"\tj\w+\s+"+cls.LABEL+"\s*" + cls.COMMENT + "$")
         cls.r_jmp_switch    = re.compile(r"\tjmp\t[*]"+cls.LABEL+"[(]")
         cls.r_jmp_source    = re.compile(r"\d*[(](%[\w]+)[,)]")
 
@@ -616,7 +617,7 @@
             # tail-calls are equivalent to RET for us
             return InsnRet(self.CALLEE_SAVE_REGISTERS)
         return InsnStop()
-
+    
     def register_jump_to(self, label):
         if not isinstance(self.insns[-1], InsnStop):
             self.labels[label].previous_insns.append(self.insns[-1])
@@ -641,6 +642,7 @@
         self.register_jump_to(label)
         return []
 
+    visit_jmpl = visit_jmp
     visit_je = conditional_jump
     visit_jne = conditional_jump
     visit_jg = conditional_jump

Modified: pypy/branch/fast-forward/pypy/translator/cli/gencli.py
==============================================================================
--- pypy/branch/fast-forward/pypy/translator/cli/gencli.py	(original)
+++ pypy/branch/fast-forward/pypy/translator/cli/gencli.py	Tue Jul  6 00:00:33 2010
@@ -2,7 +2,7 @@
 import shutil
 
 import py
-from py.compat import subprocess
+import subprocess
 from pypy.config.config import Config
 from pypy.translator.oosupport.genoo import GenOO
 from pypy.translator.cli import conftest

Modified: pypy/branch/fast-forward/pypy/translator/cli/query.py
==============================================================================
--- pypy/branch/fast-forward/pypy/translator/cli/query.py	(original)
+++ pypy/branch/fast-forward/pypy/translator/cli/query.py	Tue Jul  6 00:00:33 2010
@@ -2,7 +2,7 @@
 import cPickle as pickle
 import os.path
 import py
-from py.compat import subprocess
+import subprocess
 from pypy.tool.udir import udir
 from pypy.rpython.ootypesystem import ootype
 from pypy.translator.cli.rte import Query

Modified: pypy/branch/fast-forward/pypy/translator/cli/rte.py
==============================================================================
--- pypy/branch/fast-forward/pypy/translator/cli/rte.py	(original)
+++ pypy/branch/fast-forward/pypy/translator/cli/rte.py	Tue Jul  6 00:00:33 2010
@@ -7,7 +7,7 @@
 import shutil
 
 import py
-from py.compat import subprocess
+import subprocess
 from pypy.translator.cli.sdk import SDK
 from pypy.tool.ansi_print import ansi_log
 log = py.log.Producer("cli")

Modified: pypy/branch/fast-forward/pypy/translator/cli/support.py
==============================================================================
--- pypy/branch/fast-forward/pypy/translator/cli/support.py	(original)
+++ pypy/branch/fast-forward/pypy/translator/cli/support.py	Tue Jul  6 00:00:33 2010
@@ -1,3 +1,4 @@
+import sys
 import py
 from pypy.rpython.ootypesystem import ootype
 from pypy.translator.cli.rte import Support

Modified: pypy/branch/fast-forward/pypy/translator/cli/test/runtest.py
==============================================================================
--- pypy/branch/fast-forward/pypy/translator/cli/test/runtest.py	(original)
+++ pypy/branch/fast-forward/pypy/translator/cli/test/runtest.py	Tue Jul  6 00:00:33 2010
@@ -2,7 +2,7 @@
 import platform
 
 import py
-from py.compat import subprocess
+import subprocess
 from pypy.tool.udir import udir
 from pypy.translator.translator import TranslationContext
 from pypy.rpython.test.tool import BaseRtypingTest, OORtypeMixin

Modified: pypy/branch/fast-forward/pypy/translator/driver.py
==============================================================================
--- pypy/branch/fast-forward/pypy/translator/driver.py	(original)
+++ pypy/branch/fast-forward/pypy/translator/driver.py	Tue Jul  6 00:00:33 2010
@@ -729,7 +729,7 @@
                               'Compiling JVM source')
 
     def copy_jvm_jar(self):
-        from py.compat import subprocess
+        import subprocess
         basename = self.exe_name % self.get_info()
         root = udir.join('pypy')
         manifest = self.create_manifest(root)
@@ -764,7 +764,7 @@
         return filename
 
     def create_classlist(self, root, additional_jars=[]):
-        from py.compat import subprocess
+        import subprocess
         # first, uncompress additional jars
         for jarfile in additional_jars:
             oldpwd = root.chdir()

Modified: pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py
==============================================================================
--- pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py	(original)
+++ pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py	Tue Jul  6 00:00:33 2010
@@ -8,19 +8,6 @@
 
 banner = sys.version.splitlines()[0]
 
-def relpath(path):
-    # force 'path' to be a relative path, for testing purposes
-    curdir = py.path.local()
-    p = py.path.local(path)
-    result = []
-    while not p.relto(curdir):
-        result.append(os.pardir)
-        if curdir == curdir.dirpath():
-            return str(path)     # no relative path found, give up
-        curdir = curdir.dirpath()
-    result.append(p.relto(curdir))
-    return os.path.join(*result)
-
 app_main = os.path.join(autopath.this_dir, os.pardir, 'app_main.py')
 app_main = os.path.abspath(app_main)
 
@@ -30,7 +17,8 @@
     p = udir.join('demo_test_app_main_%d.py' % (_counter,))
     _counter += 1
     p.write(str(py.code.Source(source)))
-    return relpath(p)
+    # return relative path for testing purposes 
+    return py.path.local().bestrelpath(p) 
 
 
 demo_script = getscript("""
@@ -495,16 +483,14 @@
         # setup code for test_get_library_path
         # ------------------------------------
         from pypy.module.sys.version import CPYTHON_VERSION, PYPY_VERSION
-        libroot = 'lib/pypy%d.%d' % PYPY_VERSION[:2]
         cpy_ver = '%d.%d.%d' % CPYTHON_VERSION[:3]
         
         goal_dir = os.path.dirname(app_main)
         # build a directory hierarchy like which contains both bin/pypy-c and
         # lib/pypy1.2/*
-        prefix = udir.join('pathtest')
+        prefix = udir.join('pathtest').ensure(dir=1)
         fake_exe = prefix.join('bin/pypy-c').ensure(file=1)
-        pypyxy = prefix.join(libroot).ensure(dir=1)
-        expected_path = [str(pypyxy.join(subdir).ensure(dir=1))
+        expected_path = [str(prefix.join(subdir).ensure(dir=1))
                          for subdir in ('lib_pypy',
                                         'lib-python/modified-%s' % cpy_ver,
                                         'lib-python/%s' % cpy_ver)]

Modified: pypy/branch/fast-forward/pypy/translator/jvm/genjvm.py
==============================================================================
--- pypy/branch/fast-forward/pypy/translator/jvm/genjvm.py	(original)
+++ pypy/branch/fast-forward/pypy/translator/jvm/genjvm.py	Tue Jul  6 00:00:33 2010
@@ -6,7 +6,7 @@
 import os
 
 import py
-from py.compat import subprocess
+import subprocess
 from pypy.tool.udir import udir
 from pypy.translator.translator import TranslationContext
 from pypy.translator.oosupport.genoo import GenOO

Modified: pypy/branch/fast-forward/pypy/translator/jvm/test/runtest.py
==============================================================================
--- pypy/branch/fast-forward/pypy/translator/jvm/test/runtest.py	(original)
+++ pypy/branch/fast-forward/pypy/translator/jvm/test/runtest.py	Tue Jul  6 00:00:33 2010
@@ -2,7 +2,7 @@
 import platform
 
 import py
-from py.compat import subprocess
+import subprocess
 from pypy.tool.udir import udir
 from pypy.rpython.test.tool import BaseRtypingTest, OORtypeMixin
 from pypy.rpython.lltypesystem.lltype import typeOf

Modified: pypy/branch/fast-forward/pypy/translator/platform/windows.py
==============================================================================
--- pypy/branch/fast-forward/pypy/translator/platform/windows.py	(original)
+++ pypy/branch/fast-forward/pypy/translator/platform/windows.py	Tue Jul  6 00:00:33 2010
@@ -168,7 +168,7 @@
             # Tell the linker to generate a manifest file
             temp_manifest = ofile.dirpath().join(
                 ofile.purebasename + '.manifest')
-            args += ["/MANIFESTFILE:%s" % (temp_manifest,)]
+            args += ["/MANIFEST", "/MANIFESTFILE:%s" % (temp_manifest,)]
 
         self._execute_c_compiler(self.link, args, exe_name)
 
@@ -277,7 +277,7 @@
                    '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS)')
         else:
             m.rule('$(TARGET)', '$(OBJECTS)',
-                   ['$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFESTFILE:$*.manifest',
+                   ['$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST /MANIFESTFILE:$*.manifest',
                     'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1',
                     ])
 
@@ -290,7 +290,7 @@
                    'int main(int argc, char* argv[]) '
                    '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); } > $@')
             m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.obj'],
-                   ['$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFESTFILE:$*.manifest',
+                   ['$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest',
                     'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1',
                     ])
 



More information about the Pypy-commit mailing list