From jython-checkins at python.org Sun Jun 8 14:12:58 2014 From: jython-checkins at python.org (jeff.allen) Date: Sun, 8 Jun 2014 14:12:58 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_stringlib=2EIntegerForm?= =?utf-8?q?atter_and_use_to_refactor_=25-formatting=2E?= Message-ID: <3gmc1Q1kjpz7LjQ@mail.python.org> http://hg.python.org/jython/rev/ccf7e598786a changeset: 7279:ccf7e598786a parent: 7259:60838278e668 user: Jeff Allen date: Sun May 25 20:37:09 2014 +0100 summary: Add stringlib.IntegerFormatter and use to refactor %-formatting. Updates Lib/test_format from CPython 2.7, but we still need a Jython version. Big weight-loss in PyString.StringFormatter, and numeric bin(), oct() and hex() are based on the new code. (Still using Santoso's speeded-up conversion methods.) files: Lib/test/test_format.py | 473 +++-- src/org/python/core/PyInteger.java | 29 +- src/org/python/core/PyLong.java | 34 +- src/org/python/core/PyString.java | 524 ++---- src/org/python/core/__builtin__.java | 20 +- src/org/python/core/stringlib/FloatFormatter.java | 14 +- src/org/python/core/stringlib/IntegerFormatter.java | 696 ++++++++++ src/org/python/core/stringlib/InternalFormat.java | 42 +- 8 files changed, 1231 insertions(+), 601 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -1,14 +1,17 @@ -from test.test_support import verbose, have_unicode, TestFailed, is_jython import sys +from test.test_support import verbose, have_unicode, TestFailed +from test.test_support import is_jython +import test.test_support as test_support +import unittest + +maxsize = test_support.MAX_Py_ssize_t # test string formatting operator (I am not sure if this is being tested # elsewhere but, surely, some of the given cases are *not* tested because # they crash python) # test on unicode strings as well -overflowok = 1 - -def testformat(formatstr, args, output=None): +def testformat(formatstr, args, output=None, limit=None, overflowok=False): if verbose: if output: print "%s %% %s =? %s ..." %\ @@ -23,231 +26,289 @@ if verbose: print 'overflow (this is fine)' else: - if output and result != output: + if output and limit is None and result != output: if verbose: print 'no' - print "%s %% %s == %s != %s" %\ - (repr(formatstr), repr(args), repr(result), repr(output)) + raise AssertionError("%r %% %r == %r != %r" % + (formatstr, args, result, output)) + # when 'limit' is specified, it determines how many characters + # must match exactly; lengths must always match. + # ex: limit=5, '12345678' matches '12345___' + # (mainly for floating point format tests for which an exact match + # can't be guaranteed due to rounding and representation errors) + elif output and limit is not None and ( + len(result)!=len(output) or result[:limit]!=output[:limit]): + if verbose: + print 'no' + print "%s %% %s == %s != %s" % \ + (repr(formatstr), repr(args), repr(result), repr(output)) else: if verbose: print 'yes' -def testboth(formatstr, *args): - testformat(formatstr, *args) + +def testboth(formatstr, *args, **kwargs): + testformat(formatstr, *args, **kwargs) if have_unicode: - testformat(unicode(formatstr), *args) + testformat(unicode(formatstr), *args, **kwargs) -testboth("%.1d", (1,), "1") -testboth("%.*d", (sys.maxint,1)) # expect overflow -testboth("%.100d", (1,), '0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001') -testboth("%#.117x", (1,), '0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001') -testboth("%#.118x", (1,), '0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001') +class FormatTest(unittest.TestCase): + def test_format(self): + testboth("%.1d", (1,), "1") + testboth("%.*d", (sys.maxint,1), overflowok=True) # expect overflow + testboth("%.100d", (1,), '00000000000000000000000000000000000000' + '000000000000000000000000000000000000000000000000000000' + '00000001', overflowok=True) + testboth("%#.117x", (1,), '0x00000000000000000000000000000000000' + '000000000000000000000000000000000000000000000000000000' + '0000000000000000000000000001', + overflowok=True) + testboth("%#.118x", (1,), '0x00000000000000000000000000000000000' + '000000000000000000000000000000000000000000000000000000' + '00000000000000000000000000001', + overflowok=True) -testboth("%f", (1.0,), "1.000000") -# these are trying to test the limits of the internal magic-number-length -# formatting buffer, if that number changes then these tests are less -# effective -testboth("%#.*g", (109, -1.e+49/3.)) -testboth("%#.*g", (110, -1.e+49/3.)) -testboth("%#.*g", (110, -1.e+100/3.)) + testboth("%f", (1.0,), "1.000000") + # these are trying to test the limits of the internal magic-number-length + # formatting buffer, if that number changes then these tests are less + # effective + testboth("%#.*g", (109, -1.e+49/3.)) + testboth("%#.*g", (110, -1.e+49/3.)) + testboth("%#.*g", (110, -1.e+100/3.)) -# test some ridiculously large precision, expect overflow -testboth('%12.*f', (123456, 1.0)) + # test some ridiculously large precision, expect overflow + # ... Jython remains consistent with the original comment. + testboth('%12.*f', (123456, 1.0), overflowok=is_jython) -# Formatting of long integers. Overflow is not ok -overflowok = 0 -testboth("%x", 10L, "a") -testboth("%x", 100000000000L, "174876e800") -testboth("%o", 10L, "12") -testboth("%o", 100000000000L, "1351035564000") -testboth("%d", 10L, "10") -testboth("%d", 100000000000L, "100000000000") + # check for internal overflow validation on length of precision + # these tests should no longer cause overflow in Python + # 2.7/3.1 and later. + testboth("%#.*g", (110, -1.e+100/3.)) + testboth("%#.*G", (110, -1.e+100/3.)) + testboth("%#.*f", (110, -1.e+100/3.)) + testboth("%#.*F", (110, -1.e+100/3.)) -big = 123456789012345678901234567890L -testboth("%d", big, "123456789012345678901234567890") -testboth("%d", -big, "-123456789012345678901234567890") -testboth("%5d", -big, "-123456789012345678901234567890") -testboth("%31d", -big, "-123456789012345678901234567890") -testboth("%32d", -big, " -123456789012345678901234567890") -testboth("%-32d", -big, "-123456789012345678901234567890 ") -testboth("%032d", -big, "-0123456789012345678901234567890") -testboth("%-032d", -big, "-123456789012345678901234567890 ") -testboth("%034d", -big, "-000123456789012345678901234567890") -testboth("%034d", big, "0000123456789012345678901234567890") -testboth("%0+34d", big, "+000123456789012345678901234567890") -testboth("%+34d", big, " +123456789012345678901234567890") -testboth("%34d", big, " 123456789012345678901234567890") -testboth("%.2d", big, "123456789012345678901234567890") -testboth("%.30d", big, "123456789012345678901234567890") -testboth("%.31d", big, "0123456789012345678901234567890") -testboth("%32.31d", big, " 0123456789012345678901234567890") + # Formatting of long integers. Overflow is not ok + testboth("%x", 10L, "a") + testboth("%x", 100000000000L, "174876e800") + testboth("%o", 10L, "12") + testboth("%o", 100000000000L, "1351035564000") + testboth("%d", 10L, "10") + testboth("%d", 100000000000L, "100000000000") -big = 0x1234567890abcdef12345L # 21 hex digits -testboth("%x", big, "1234567890abcdef12345") -testboth("%x", -big, "-1234567890abcdef12345") -testboth("%5x", -big, "-1234567890abcdef12345") -testboth("%22x", -big, "-1234567890abcdef12345") -testboth("%23x", -big, " -1234567890abcdef12345") -testboth("%-23x", -big, "-1234567890abcdef12345 ") -testboth("%023x", -big, "-01234567890abcdef12345") -testboth("%-023x", -big, "-1234567890abcdef12345 ") -testboth("%025x", -big, "-0001234567890abcdef12345") -testboth("%025x", big, "00001234567890abcdef12345") -testboth("%0+25x", big, "+0001234567890abcdef12345") -testboth("%+25x", big, " +1234567890abcdef12345") -testboth("%25x", big, " 1234567890abcdef12345") -testboth("%.2x", big, "1234567890abcdef12345") -testboth("%.21x", big, "1234567890abcdef12345") -testboth("%.22x", big, "01234567890abcdef12345") -testboth("%23.22x", big, " 01234567890abcdef12345") -testboth("%-23.22x", big, "01234567890abcdef12345 ") -testboth("%X", big, "1234567890ABCDEF12345") -testboth("%#X", big, "0X1234567890ABCDEF12345") -testboth("%#x", big, "0x1234567890abcdef12345") -testboth("%#x", -big, "-0x1234567890abcdef12345") -testboth("%#.23x", -big, "-0x001234567890abcdef12345") -testboth("%#+.23x", big, "+0x001234567890abcdef12345") -testboth("%# .23x", big, " 0x001234567890abcdef12345") -testboth("%#+.23X", big, "+0X001234567890ABCDEF12345") -testboth("%#-+.23X", big, "+0X001234567890ABCDEF12345") -testboth("%#-+26.23X", big, "+0X001234567890ABCDEF12345") -testboth("%#-+27.23X", big, "+0X001234567890ABCDEF12345 ") -testboth("%#+27.23X", big, " +0X001234567890ABCDEF12345") -# next one gets two leading zeroes from precision, and another from the -# 0 flag and the width -testboth("%#+027.23X", big, "+0X0001234567890ABCDEF12345") -# same, except no 0 flag -testboth("%#+27.23X", big, " +0X001234567890ABCDEF12345") + big = 123456789012345678901234567890L + testboth("%d", big, "123456789012345678901234567890") + testboth("%d", -big, "-123456789012345678901234567890") + testboth("%5d", -big, "-123456789012345678901234567890") + testboth("%31d", -big, "-123456789012345678901234567890") + testboth("%32d", -big, " -123456789012345678901234567890") + testboth("%-32d", -big, "-123456789012345678901234567890 ") + testboth("%032d", -big, "-0123456789012345678901234567890") + testboth("%-032d", -big, "-123456789012345678901234567890 ") + testboth("%034d", -big, "-000123456789012345678901234567890") + testboth("%034d", big, "0000123456789012345678901234567890") + testboth("%0+34d", big, "+000123456789012345678901234567890") + testboth("%+34d", big, " +123456789012345678901234567890") + testboth("%34d", big, " 123456789012345678901234567890") + testboth("%.2d", big, "123456789012345678901234567890") + testboth("%.30d", big, "123456789012345678901234567890") + testboth("%.31d", big, "0123456789012345678901234567890") + testboth("%32.31d", big, " 0123456789012345678901234567890") + testboth("%d", float(big), "123456________________________", 6) -big = 012345670123456701234567012345670L # 32 octal digits -testboth("%o", big, "12345670123456701234567012345670") -testboth("%o", -big, "-12345670123456701234567012345670") -testboth("%5o", -big, "-12345670123456701234567012345670") -testboth("%33o", -big, "-12345670123456701234567012345670") -testboth("%34o", -big, " -12345670123456701234567012345670") -testboth("%-34o", -big, "-12345670123456701234567012345670 ") -testboth("%034o", -big, "-012345670123456701234567012345670") -testboth("%-034o", -big, "-12345670123456701234567012345670 ") -testboth("%036o", -big, "-00012345670123456701234567012345670") -testboth("%036o", big, "000012345670123456701234567012345670") -testboth("%0+36o", big, "+00012345670123456701234567012345670") -testboth("%+36o", big, " +12345670123456701234567012345670") -testboth("%36o", big, " 12345670123456701234567012345670") -testboth("%.2o", big, "12345670123456701234567012345670") -testboth("%.32o", big, "12345670123456701234567012345670") -testboth("%.33o", big, "012345670123456701234567012345670") -testboth("%34.33o", big, " 012345670123456701234567012345670") -testboth("%-34.33o", big, "012345670123456701234567012345670 ") -testboth("%o", big, "12345670123456701234567012345670") -testboth("%#o", big, "012345670123456701234567012345670") -testboth("%#o", -big, "-012345670123456701234567012345670") -testboth("%#.34o", -big, "-0012345670123456701234567012345670") -testboth("%#+.34o", big, "+0012345670123456701234567012345670") -testboth("%# .34o", big, " 0012345670123456701234567012345670") -testboth("%#+.34o", big, "+0012345670123456701234567012345670") -testboth("%#-+.34o", big, "+0012345670123456701234567012345670") -testboth("%#-+37.34o", big, "+0012345670123456701234567012345670 ") -testboth("%#+37.34o", big, " +0012345670123456701234567012345670") -# next one gets one leading zero from precision -testboth("%.33o", big, "012345670123456701234567012345670") -# base marker shouldn't change that, since "0" is redundant -testboth("%#.33o", big, "012345670123456701234567012345670") -# but reduce precision, and base marker should add a zero -testboth("%#.32o", big, "012345670123456701234567012345670") -# one leading zero from precision, and another from "0" flag & width -testboth("%034.33o", big, "0012345670123456701234567012345670") -# base marker shouldn't change that -testboth("%0#34.33o", big, "0012345670123456701234567012345670") + big = 0x1234567890abcdef12345L # 21 hex digits + testboth("%x", big, "1234567890abcdef12345") + testboth("%x", -big, "-1234567890abcdef12345") + testboth("%5x", -big, "-1234567890abcdef12345") + testboth("%22x", -big, "-1234567890abcdef12345") + testboth("%23x", -big, " -1234567890abcdef12345") + testboth("%-23x", -big, "-1234567890abcdef12345 ") + testboth("%023x", -big, "-01234567890abcdef12345") + testboth("%-023x", -big, "-1234567890abcdef12345 ") + testboth("%025x", -big, "-0001234567890abcdef12345") + testboth("%025x", big, "00001234567890abcdef12345") + testboth("%0+25x", big, "+0001234567890abcdef12345") + testboth("%+25x", big, " +1234567890abcdef12345") + testboth("%25x", big, " 1234567890abcdef12345") + testboth("%.2x", big, "1234567890abcdef12345") + testboth("%.21x", big, "1234567890abcdef12345") + testboth("%.22x", big, "01234567890abcdef12345") + testboth("%23.22x", big, " 01234567890abcdef12345") + testboth("%-23.22x", big, "01234567890abcdef12345 ") + testboth("%X", big, "1234567890ABCDEF12345") + testboth("%#X", big, "0X1234567890ABCDEF12345") + testboth("%#x", big, "0x1234567890abcdef12345") + testboth("%#x", -big, "-0x1234567890abcdef12345") + testboth("%#.23x", -big, "-0x001234567890abcdef12345") + testboth("%#+.23x", big, "+0x001234567890abcdef12345") + testboth("%# .23x", big, " 0x001234567890abcdef12345") + testboth("%#+.23X", big, "+0X001234567890ABCDEF12345") + testboth("%#-+.23X", big, "+0X001234567890ABCDEF12345") + testboth("%#-+26.23X", big, "+0X001234567890ABCDEF12345") + testboth("%#-+27.23X", big, "+0X001234567890ABCDEF12345 ") + testboth("%#+27.23X", big, " +0X001234567890ABCDEF12345") + # next one gets two leading zeroes from precision, and another from the + # 0 flag and the width + testboth("%#+027.23X", big, "+0X0001234567890ABCDEF12345") + # same, except no 0 flag + testboth("%#+27.23X", big, " +0X001234567890ABCDEF12345") + testboth("%x", float(big), "123456_______________", 6) -# Some small ints, in both Python int and long flavors). -testboth("%d", 42, "42") -testboth("%d", -42, "-42") -testboth("%d", 42L, "42") -testboth("%d", -42L, "-42") -testboth("%#x", 1, "0x1") -testboth("%#x", 1L, "0x1") -testboth("%#X", 1, "0X1") -testboth("%#X", 1L, "0X1") -testboth("%#o", 1, "01") -testboth("%#o", 1L, "01") -testboth("%#o", 0, "0") -testboth("%#o", 0L, "0") -testboth("%o", 0, "0") -testboth("%o", 0L, "0") -testboth("%d", 0, "0") -testboth("%d", 0L, "0") -testboth("%#x", 0, "0x0") -testboth("%#x", 0L, "0x0") -testboth("%#X", 0, "0X0") -testboth("%#X", 0L, "0X0") + big = 012345670123456701234567012345670L # 32 octal digits + testboth("%o", big, "12345670123456701234567012345670") + testboth("%o", -big, "-12345670123456701234567012345670") + testboth("%5o", -big, "-12345670123456701234567012345670") + testboth("%33o", -big, "-12345670123456701234567012345670") + testboth("%34o", -big, " -12345670123456701234567012345670") + testboth("%-34o", -big, "-12345670123456701234567012345670 ") + testboth("%034o", -big, "-012345670123456701234567012345670") + testboth("%-034o", -big, "-12345670123456701234567012345670 ") + testboth("%036o", -big, "-00012345670123456701234567012345670") + testboth("%036o", big, "000012345670123456701234567012345670") + testboth("%0+36o", big, "+00012345670123456701234567012345670") + testboth("%+36o", big, " +12345670123456701234567012345670") + testboth("%36o", big, " 12345670123456701234567012345670") + testboth("%.2o", big, "12345670123456701234567012345670") + testboth("%.32o", big, "12345670123456701234567012345670") + testboth("%.33o", big, "012345670123456701234567012345670") + testboth("%34.33o", big, " 012345670123456701234567012345670") + testboth("%-34.33o", big, "012345670123456701234567012345670 ") + testboth("%o", big, "12345670123456701234567012345670") + testboth("%#o", big, "012345670123456701234567012345670") + testboth("%#o", -big, "-012345670123456701234567012345670") + testboth("%#.34o", -big, "-0012345670123456701234567012345670") + testboth("%#+.34o", big, "+0012345670123456701234567012345670") + testboth("%# .34o", big, " 0012345670123456701234567012345670") + testboth("%#+.34o", big, "+0012345670123456701234567012345670") + testboth("%#-+.34o", big, "+0012345670123456701234567012345670") + testboth("%#-+37.34o", big, "+0012345670123456701234567012345670 ") + testboth("%#+37.34o", big, " +0012345670123456701234567012345670") + # next one gets one leading zero from precision + testboth("%.33o", big, "012345670123456701234567012345670") + # base marker shouldn't change that, since "0" is redundant + testboth("%#.33o", big, "012345670123456701234567012345670") + # but reduce precision, and base marker should add a zero + testboth("%#.32o", big, "012345670123456701234567012345670") + # one leading zero from precision, and another from "0" flag & width + testboth("%034.33o", big, "0012345670123456701234567012345670") + # base marker shouldn't change that + testboth("%0#34.33o", big, "0012345670123456701234567012345670") + testboth("%o", float(big), "123456__________________________", 6) -testboth("%x", 0x42, "42") -testboth("%x", -0x42, "-42") -testboth("%x", 0x42L, "42") -testboth("%x", -0x42L, "-42") + # Some small ints, in both Python int and long flavors). + testboth("%d", 42, "42") + testboth("%d", -42, "-42") + testboth("%d", 42L, "42") + testboth("%d", -42L, "-42") + testboth("%d", 42.0, "42") + testboth("%#x", 1, "0x1") + testboth("%#x", 1L, "0x1") + testboth("%#X", 1, "0X1") + testboth("%#X", 1L, "0X1") + testboth("%#x", 1.0, "0x1") + testboth("%#o", 1, "01") + testboth("%#o", 1L, "01") + testboth("%#o", 0, "0") + testboth("%#o", 0L, "0") + testboth("%o", 0, "0") + testboth("%o", 0L, "0") + testboth("%d", 0, "0") + testboth("%d", 0L, "0") + testboth("%#x", 0, "0x0") + testboth("%#x", 0L, "0x0") + testboth("%#X", 0, "0X0") + testboth("%#X", 0L, "0X0") -testboth("%o", 042, "42") -testboth("%o", -042, "-42") -testboth("%o", 042L, "42") -testboth("%o", -042L, "-42") + testboth("%x", 0x42, "42") + testboth("%x", -0x42, "-42") + testboth("%x", 0x42L, "42") + testboth("%x", -0x42L, "-42") + testboth("%x", float(0x42), "42") -# Test exception for unknown format characters -if verbose: - print 'Testing exceptions' + testboth("%o", 042, "42") + testboth("%o", -042, "-42") + testboth("%o", 042L, "42") + testboth("%o", -042L, "-42") + testboth("%o", float(042), "42") -def test_exc(formatstr, args, exception, excmsg): - try: - testformat(formatstr, args) - except exception, exc: - if str(exc) == excmsg: - if verbose: - print "yes" - else: - if verbose: print 'no' - print 'Unexpected ', exception, ':', repr(str(exc)) - except: - if verbose: print 'no' - print 'Unexpected exception' - raise - else: - raise TestFailed, 'did not get expected exception: %s' % excmsg + # alternate float formatting + testformat('%g', 1.1, '1.1') + testformat('%#g', 1.1, '1.10000') -test_exc('abc %a', 1, ValueError, - "unsupported format character 'a' (0x61) at index 5") -if have_unicode: - test_exc(unicode('abc %\u3000','raw-unicode-escape'), 1, ValueError, - "unsupported format character '?' (0x3000) at index 5") + # Regression test for http://bugs.python.org/issue15516. + class IntFails(object): + def __int__(self): + raise TestFailed + def __long__(self): + return 0 -test_exc('%d', '1', TypeError, "int argument required") -test_exc('%g', '1', TypeError, "float argument required") -test_exc('no format', '1', TypeError, - "not all arguments converted during string formatting") -test_exc('no format', u'1', TypeError, - "not all arguments converted during string formatting") -test_exc(u'no format', '1', TypeError, - "not all arguments converted during string formatting") -test_exc(u'no format', u'1', TypeError, - "not all arguments converted during string formatting") + fst = IntFails() + testformat("%x", fst, '0') -# for Jython, do we really need to support this? what's the use case -# here! the problem in a nutshell is that it changes __oct__, __hex__ -# such that they don't return a string, but later on the exception -# will occur anyway. so seems like a lot of work for no value + # Test exception for unknown format characters + if verbose: + print 'Testing exceptions' -# class Foobar(long): -# def __oct__(self): -# # Returning a non-string should not blow up. -# return self + 1 + def test_exc(formatstr, args, exception, excmsg): + try: + testformat(formatstr, args) + except exception, exc: + if str(exc) == excmsg: + if verbose: + print "yes" + else: + if verbose: print 'no' + print 'Unexpected ', exception, ':', repr(str(exc)) + except: + if verbose: print 'no' + print 'Unexpected exception' + raise + else: + raise TestFailed, 'did not get expected exception: %s' % excmsg -#test_exc('%o', Foobar(), TypeError, -# "expected string or Unicode object, long found") + test_exc('abc %a', 1, ValueError, + "unsupported format character 'a' (0x61) at index 5") + if have_unicode: + test_exc(unicode('abc %\u3000','raw-unicode-escape'), 1, ValueError, + "unsupported format character '?' (0x3000) at index 5") -if sys.maxint == 2**31-1 and not is_jython: - # crashes 2.2.1 and earlier: - try: - "%*d"%(sys.maxint, -127) - except MemoryError: - pass - else: - raise TestFailed, '"%*d"%(sys.maxint, -127) should fail' + test_exc('%d', '1', TypeError, "%d format: a number is required, not str") + test_exc('%g', '1', TypeError, "float argument required, not str") + test_exc('no format', '1', TypeError, + "not all arguments converted during string formatting") + test_exc('no format', u'1', TypeError, + "not all arguments converted during string formatting") + test_exc(u'no format', '1', TypeError, + "not all arguments converted during string formatting") + test_exc(u'no format', u'1', TypeError, + "not all arguments converted during string formatting") + + # For Jython, we do not support this use case. The test aims at the, + # use of __oct__ within %o formatting of long. (Or __hex__ within %x + # formatting?) CPython does this for long (not int) and has dropped + # the idea again by v3. Jython's %o and %x are likewise direct. + class Foobar(long): + def __oct__(self): + # Returning a non-string should not blow up. + return self + 1 + + if not is_jython : + test_exc('%o', Foobar(), TypeError, + "expected string or Unicode object, long found") + + if maxsize == 2**31-1: + # crashes 2.2.1 and earlier: + try: + "%*d"%(maxsize, -127) + except MemoryError: + pass + else: + raise TestFailed, '"%*d"%(maxsize, -127) should fail' + +def test_main(): + test_support.run_unittest(FormatTest) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/org/python/core/PyInteger.java b/src/org/python/core/PyInteger.java --- a/src/org/python/core/PyInteger.java +++ b/src/org/python/core/PyInteger.java @@ -7,6 +7,8 @@ import java.text.NumberFormat; import java.util.Locale; +import org.python.core.stringlib.IntegerFormatter; +import org.python.core.stringlib.InternalFormat.Spec; import org.python.core.stringlib.InternalFormatSpec; import org.python.core.stringlib.InternalFormatSpecParser; import org.python.expose.ExposedGet; @@ -953,11 +955,8 @@ @ExposedMethod(doc = BuiltinDocs.int___oct___doc) final PyString int___oct__() { - if (getValue() < 0) { - return new PyString("-0" + Integer.toString(getValue() * -1, 8)); - } else { - return new PyString("0" + Integer.toString(getValue(), 8)); - } + // Use the prepared format specifier for octal. + return formatImpl(IntegerFormatter.OCT); } @Override @@ -967,11 +966,21 @@ @ExposedMethod(doc = BuiltinDocs.int___hex___doc) final PyString int___hex__() { - if (getValue() < 0) { - return new PyString("-0x" + Integer.toString(getValue() * -1, 16)); - } else { - return new PyString("0x" + Integer.toString(getValue(), 16)); - } + // Use the prepared format specifier for hexadecimal. + return formatImpl(IntegerFormatter.HEX); + } + + /** + * Common code used by the number-base conversion method __oct__ and __hex__. + * + * @param spec prepared format-specifier. + * @return converted value of this object + */ + private PyString formatImpl(Spec spec) { + // Traditional formatter (%-format) because #o means "-0123" not "-0o123". + IntegerFormatter f = new IntegerFormatter.Traditional(spec); + f.format(value); + return new PyString(f.getResult()); } @ExposedMethod(doc = BuiltinDocs.int___getnewargs___doc) diff --git a/src/org/python/core/PyLong.java b/src/org/python/core/PyLong.java --- a/src/org/python/core/PyLong.java +++ b/src/org/python/core/PyLong.java @@ -8,6 +8,9 @@ import java.math.BigDecimal; import java.math.BigInteger; +import org.python.core.stringlib.IntegerFormatter; +import org.python.core.stringlib.InternalFormat; +import org.python.core.stringlib.InternalFormat.Spec; import org.python.expose.ExposedGet; import org.python.expose.ExposedMethod; import org.python.expose.ExposedNew; @@ -977,14 +980,8 @@ @ExposedMethod(doc = BuiltinDocs.long___oct___doc) final PyString long___oct__() { - String s = PyInteger.toOctString(getValue()); - if (s.startsWith("-")) { - return new PyString("-0" + s.substring(1, s.length()) + "L"); - } else if (s.startsWith("0")) { - return new PyString(s + "L"); - } else { - return new PyString("0" + s + "L"); - } + // Use the prepared format specifier for octal. + return formatImpl(IntegerFormatter.OCT); } @Override @@ -994,12 +991,21 @@ @ExposedMethod(doc = BuiltinDocs.long___hex___doc) final PyString long___hex__() { - String s = PyInteger.toHexString(getValue()); - if (s.startsWith("-")) { - return new PyString("-0x" + s.substring(1, s.length()) + "L"); - } else { - return new PyString("0x" + s + "L"); - } + // Use the prepared format specifier for hexadecimal. + return formatImpl(IntegerFormatter.HEX); + } + + /** + * Common code used by the number-base conversion method __oct__ and __hex__. + * + * @param spec prepared format-specifier. + * @return converted value of this object + */ + private PyString formatImpl(Spec spec) { + // Traditional formatter (%-format) because #o means "-0123" not "-0o123". + IntegerFormatter f = new IntegerFormatter.Traditional(spec); + f.format(value).append('L'); + return new PyString(f.getResult()); } @ExposedMethod(doc = BuiltinDocs.long___str___doc) diff --git a/src/org/python/core/PyString.java b/src/org/python/core/PyString.java --- a/src/org/python/core/PyString.java +++ b/src/org/python/core/PyString.java @@ -9,6 +9,7 @@ import org.python.core.buffer.SimpleStringBuffer; import org.python.core.stringlib.FieldNameIterator; import org.python.core.stringlib.FloatFormatter; +import org.python.core.stringlib.IntegerFormatter; import org.python.core.stringlib.InternalFormat.Spec; import org.python.core.stringlib.InternalFormatSpec; import org.python.core.stringlib.InternalFormatSpecParser; @@ -4106,211 +4107,89 @@ } } - private void checkPrecision(String type) { - if (precision > 250) { - // A magic number. Larger than in CPython. - throw Py.OverflowError("formatted " + type + " is too long (precision too long?)"); - } - - } - /** - * Format the argument interpreted as a long, using the argument's __str__, - * __oct__, or __hex__ method according to type. If v is - * being treated as signed, the sign of v is transferred to {@link #negative} and the absolute - * value is converted. The altFlag argument controls the appearance of a "0x" or - * "0X" prefix in the hex case, or a "0" prefix in the octal case. The hexadecimal case, the - * case of characters and digits will match the type ('x' meaning lowercase, 'X' meaning - * uppercase). + * Return the argument as either a {@link PyInteger} or a {@link PyLong} according to its + * __int__ method, or its __long__ method. If the argument has neither + * method, or both raise an exception, we return the argument itself. The caller must check the + * return type. * * @param arg to convert - * @param type one of 'o' for octal, 'x' or 'X' for hex, anything else calls - * arg.__str__. - * @param altFlag if true there will be a prefix - * @return converted value as String + * @return PyInteger or PyLong if possible */ - private String formatLong(PyObject arg, char type, boolean altFlag) { - // Convert using the appropriate type - // XXX Results in behaviour divergent from CPython when any of the methods is overridden. - PyString argAsString; - switch (type) { - case 'o': - argAsString = arg.__oct__(); - break; - case 'x': - case 'X': - argAsString = arg.__hex__(); - break; - default: - argAsString = arg.__str__(); - break; - } - - checkPrecision("long"); - String s = argAsString.toString(); - int end = s.length(); - int ptr = 0; - - // In the hex case, the __hex__ return starts 0x - // XXX (we assume, perhaps falsely) - int numnondigits = 0; - if (type == 'x' || type == 'X') { - numnondigits = 2; - } - - // Strip a "long" indicator - if (s.endsWith("L")) { - end--; - } - - // Strip a possible sign to member negative - negative = s.charAt(0) == '-'; - if (negative) { - ptr++; - } - - // The formatted number is s[ptr:end] and starts with numnondigits non-digits. - int numdigits = end - numnondigits - ptr; - if (!altFlag) { - // We should have no "base tag" '0' or "0x" on the front. - switch (type) { - case 'o': - // Strip the '0' - if (numdigits > 1) { - ++ptr; - --numdigits; - } - break; - case 'x': - case 'X': - // Strip the "0x" - ptr += 2; - numnondigits -= 2; - break; - } - } - - // If necessary, add leading zeros to the numerical digits part. - if (precision > numdigits) { - // Recompose the formatted number in this buffer - StringBuilder buf = new StringBuilder(); - // The base indicator prefix - for (int i = 0; i < numnondigits; ++i) { - buf.append(s.charAt(ptr++)); - } - // The extra zeros - for (int i = 0; i < precision - numdigits; i++) { - buf.append('0'); - } - // The previously known digits - for (int i = 0; i < numdigits; i++) { - buf.append(s.charAt(ptr++)); - } - s = buf.toString(); - } else if (end < s.length() || ptr > 0) { - // It's only necessary to extract the formatted number from s - s = s.substring(ptr, end); - } - - // And finally, deal with the case, so it matches x or X. - switch (type) { - case 'X': - s = s.toUpperCase(); - break; - } - return s; - } - - /** - * Formats arg as an integer, with the specified radix. The integer value is obtained from the - * result of arg.__int__(). type and altFlag are passed - * to {@link #formatLong(PyObject, char, boolean)} in case the result is a PyLong. - * - * @param arg to convert - * @param radix in which to express arg - * @param unsigned true if required to interpret a 32-bit integer as unsigned ('u' legacy?). - * @param type of conversion ('d', 'o', 'x', or 'X') - * @param altFlag '#' present in format (causes "0x" prefix in hex, and '0' prefix in octal) - * @return string form of the value - */ - private String formatInteger(PyObject arg, int radix, boolean unsigned, char type, - boolean altFlag) { - PyObject argAsInt; + private PyObject asNumber(PyObject arg) { if (arg instanceof PyInteger || arg instanceof PyLong) { - argAsInt = arg; + // arg is already acceptable + return arg; + } else { - // use __int__ to get an int (or long) - if (arg instanceof PyFloat) { - // safe to call __int__: - argAsInt = arg.__int__(); + // use __int__ or __long__to get an int (or long) + if (arg.getClass() == PyFloat.class) { + // A common case where it is safe to return arg.__int__() + return arg.__int__(); + } else { - // We can't simply call arg.__int__() because PyString implements - // it without exposing it to python (i.e, str instances has no - // __int__ attribute). So, we would support strings as arguments - // for %d format, which is forbidden by CPython tests (on - // test_format.py). + /* + * In general, we can't simply call arg.__int__() because PyString implements it + * without exposing it to python (str has no __int__). This would make str + * acceptacle to integer format specifiers, which is forbidden by CPython tests + * (test_format.py). PyString implements __int__ perhaps only to help the int + * constructor. Maybe that was a bad idea? + */ try { - argAsInt = arg.__getattr__("__int__").__call__(); + // Result is the result of arg.__int__() if that works + return arg.__getattr__("__int__").__call__(); } catch (PyException e) { - // XXX: Swallow custom AttributeError throws from __int__ methods - // No better alternative for the moment - if (e.match(Py.AttributeError)) { - throw Py.TypeError("int argument required"); - } - throw e; + // Swallow the exception + } + + // Try again with arg.__long__() + try { + // Result is the result of arg.__long__() if that works + return arg.__getattr__("__long__").__call__(); + } catch (PyException e) { + // No __long__ defined (at Python level) + return arg; } } } - if (argAsInt instanceof PyInteger) { - // This call does not provide the prefix and will be lowercase. - return formatInteger(((PyInteger)argAsInt).getValue(), radix, unsigned); - } else { // must be a PyLong (as per __int__ contract) - // This call provides the base prefix and case-matches with 'x' or 'X'. - return formatLong(argAsInt, type, altFlag); - } } /** - * Convert a 32-bit integer (as from a {@link PyInteger}) to characters, signed or unsigned. The - * values is presented in a long. The string result is left-padded with zeros to - * the stated {@link #precision}. If v is being treated as signed, the sign of v is transferred - * to {@link #negative} and the absolute value is converted. Otherwise (unsigned case) - * 0x100000000L + v is converted. This method does not provide the '0' or "0x" - * prefix, just the padded digit string. + * Return the argument as either a {@link PyFloat} according to its __float__ + * method. If the argument has no such method, or it raises an exception, we return the argument + * itself. The caller must check the return type. * - * @param v value to convert - * @param radix of conversion - * @param unsigned if should be treated as unsigned - * @return string form + * @param arg to convert + * @return PyFloat if possible */ - private String formatInteger(long v, int radix, boolean unsigned) { - checkPrecision("integer"); - if (unsigned) { - // If the high bit was set, this will have been sign-extended: correct that. - if (v < 0) { - v = 0x100000000l + v; + private PyObject asFloat(PyObject arg) { + + if (arg instanceof PyFloat) { + // arg is already acceptable + return arg; + + } else { + // use __float__ to get a float. + if (arg.getClass() == PyFloat.class) { + // A common case where it is safe to return arg.__int__() + return arg.__float__(); + + } else { + /* + * In general, we can't simply call arg.__float__() because PyString implements it + * without exposing it to python (str has no __float__). This would make str + * acceptacle to float format specifiers, which is forbidden by CPython tests + * (test_format.py). PyString implements __float__ perhaps only to help the float + * constructor. Maybe that was a bad idea? + */ + try { + // Result is the result of arg.__float__() if that works + return arg.__getattr__("__float__").__call__(); + } catch (PyException e) { + // No __float__ defined (at Python level) + return arg; + } } - } else { - // If the high bit was set, the sign extension was correct, but we need sign + abs(v). - if (v < 0) { - negative = true; - v = -v; - } - } - // Use the method in java.lang.Long (lowercase, no prefix) - String s = Long.toString(v, radix); - // But zero pad to the requested precision - while (s.length() < precision) { - s = "0" + s; - } - return s; - } - - private double asDouble(PyObject obj) { - try { - return obj.asDouble(); - } catch (PyException pye) { - throw !pye.match(Py.TypeError) ? pye : Py.TypeError("float argument required"); } } @@ -4475,6 +4354,22 @@ fill = ' '; } + // Encode as an InternalFormat.Spec + char fill2 = ' '; + char align = ljustFlag ? '<' : '>'; + if (zeroFlag && !ljustFlag) { + // We only actually fill with zero if right-justifying + fill2 = '0'; + // And then the fill comes after the sign. + align = '='; + } + char sign = signFlag ? '+' : (blankFlag ? ' ' : Spec.NONE); + int w = width; + Spec spec = new Spec(fill2, align, sign, altFlag, w, false, precision, c); + + // Signal that the padding, sign, base prefix etc. have all been taken care of + boolean jobDone = false; + // Perform the type-specific formatting switch (c) { @@ -4503,102 +4398,65 @@ break; - case 'i': - case 'd': - // Signed integer decimal. Note floats accepted. - if (arg instanceof PyLong) { - string = formatLong(arg, c, altFlag); + case 'd': // All integer formats (+case for X). + case 'o': + case 'x': + case 'X': + case 'u': // Obsolete type identical to 'd'. + case 'i': // Compatibility with scanf(). + + // Format using this Spec the double form of the argument. + IntegerFormatter fi = new IntegerFormatter.Traditional(spec); + + // Note various types accepted here as long as they have an __int__ method. + PyObject argAsNumber = asNumber(arg); + + // We have to check what we got back.. + if (argAsNumber instanceof PyInteger) { + fi.format(((PyInteger)argAsNumber).getValue()); + } else if (argAsNumber instanceof PyLong) { + fi.format(((PyLong)argAsNumber).getValue()); } else { - string = formatInteger(arg, 10, false, c, altFlag); + // It couldn't be converted, raise the error here + throw Py.TypeError("%" + c + " format: a number is required, not " + + arg.getType().fastGetName()); } + + fi.pad(); + string = fi.getResult(); + + // Suppress subsequent attempts to insert a correct sign, done already. + jobDone = true; break; - case 'u': - // Obsolete type ? it is identical to 'd'. (Why not identical here?) - if (arg instanceof PyLong) { - string = formatLong(arg, c, altFlag); - } else if (arg instanceof PyInteger || arg instanceof PyFloat) { - string = formatInteger(arg, 10, false, c, altFlag); - } else { - throw Py.TypeError("int argument required"); - } - break; - - case 'o': - // Signed octal value. Note floats accepted. - if (arg instanceof PyLong) { - // This call provides the base prefix '0' if altFlag. - string = formatLong(arg, c, altFlag); - } else if (arg instanceof PyInteger || arg instanceof PyFloat) { - // This call does not provide the '0' prefix and will be lowercase ... - // ... except where arg.__int__ returns PyLong, then it's like formatLong. - string = formatInteger(arg, 8, false, c, altFlag); - if (altFlag && string.charAt(0) != '0') { - string = "0" + string; - } - } else { - throw Py.TypeError("int argument required"); - } - break; - - case 'x': - // Signed hexadecimal (lowercase). Note floats accepted. - if (arg instanceof PyLong) { - // This call provides the base prefix "0x" if altFlag and case-matches c. - string = formatLong(arg, c, altFlag); - } else if (arg instanceof PyInteger || arg instanceof PyFloat) { - // This call does not provide the "0x" prefix and will be lowercase. - // ... except where arg.__int__ returns PyLong, then it's like formatLong. - string = formatInteger(arg, 16, false, c, altFlag); - string = string.toLowerCase(); - if (altFlag) { - string = "0x" + string; - } - } else { - throw Py.TypeError("int argument required"); - } - break; - - case 'X': - // Signed hexadecimal (uppercase). Note floats accepted. - if (arg instanceof PyLong) { - // This call provides the base prefix "0x" if altFlag and case-matches c. - string = formatLong(arg, c, altFlag); - } else if (arg instanceof PyInteger || arg instanceof PyFloat) { - // This call does not provide the "0x" prefix and will be lowercase. - // ... except where arg.__int__ returns PyLong, then it's like formatLong. - string = formatInteger(arg, 16, false, c, altFlag); - string = string.toUpperCase(); - if (altFlag) { - string = "0X" + string; - } - } else { - throw Py.TypeError("int argument required"); - } - break; - - case 'e': + case 'e': // All floating point formats (+case). case 'E': case 'f': case 'F': case 'g': case 'G': - // All floating point formats (+case). - - // Convert the flags (local variables) to the form needed in the Spec object. - char align = ljustFlag ? '<' : '>'; - char sign = signFlag ? '+' : (blankFlag ? ' ' : Spec.NONE); - int w = Spec.UNSPECIFIED; - Spec spec = new Spec(fill, align, sign, altFlag, w, false, precision, c); // Format using this Spec the double form of the argument. - FloatFormatter f = new FloatFormatter(spec); - double v = asDouble(arg); - f.format(v); - string = f.getResult(); + FloatFormatter ff = new FloatFormatter(spec); + + // Note various types accepted here as long as they have a __float__ method. + PyObject argAsFloat = asFloat(arg); + + // We have to check what we got back.. + if (argAsFloat instanceof PyFloat) { + ff.format(((PyFloat)argAsFloat).getValue()); + } else { + // It couldn't be converted, raise the error here + throw Py.TypeError("float argument required, not " + + arg.getType().fastGetName()); + } + + ff.pad(); + string = ff.getResult(); // Suppress subsequent attempts to insert a correct sign, done already. - signFlag = blankFlag = negative = false; + // signFlag = blankFlag = negative = false; + jobDone = true; break; case 'c': @@ -4650,85 +4508,71 @@ * We have now dealt with the translation of the (absolute value of the) argument, in * variable string[]. In the next sections we deal with sign, padding and base prefix. */ - int length = string.length(); - int skip = 0; - - // Decide how to represent the sign according to format and actual sign of argument. - String signString = null; - if (negative) { - signString = "-"; + if (jobDone) { + // Type-specific formatting has already taken care of all this. + buffer.append(string); + } else { - if (signFlag) { - signString = "+"; - } else if (blankFlag) { - signString = " "; + // Legacy code still needed + int length = string.length(); + int skip = 0; + + // Decide how to represent the sign according to format and actual sign of argument. + String signString = null; + if (negative) { + signString = "-"; + } else { + if (signFlag) { + signString = "+"; + } else if (blankFlag) { + signString = " "; + } } - } - - // The width (from here on) will be the remaining width on the line. - if (width < length) { - width = length; - } - - // Insert the sign in the buffer and adjust the width. - if (signString != null) { - if (fill != ' ') { - // When the fill is not space, the sign comes before the fill. - buffer.append(signString); + + // The width (from here on) will be the remaining width on the line. + if (width < length) { + width = length; } - // Adjust width for sign. - if (width > length) { - width--; + + // Insert the sign in the buffer and adjust the width. + if (signString != null) { + if (fill != ' ') { + // When the fill is not space, the sign comes before the fill. + buffer.append(signString); + } + // Adjust width for sign. + if (width > length) { + width--; + } } - } - - // Insert base prefix used with alternate mode for hexadecimal. - if (altFlag && (c == 'x' || c == 'X')) { - if (fill != ' ') { - // When the fill is not space, this base prefix comes before the fill. - buffer.append('0'); - buffer.append(c); - skip += 2; + + // Fill on the left of the item. + if (width > length && !ljustFlag) { + do { + buffer.append(fill); + } while (--width > length); } - // Adjust width for base prefix. - width -= 2; - if (width < 0) { - width = 0; + + // If the fill is spaces, we will have deferred the sign and hex base prefix + if (fill == ' ') { + if (signString != null) { + buffer.append(signString); + } } - length -= 2; - } - - // Fill on the left of the item. - if (width > length && !ljustFlag) { - do { - buffer.append(fill); - } while (--width > length); - } - - // If the fill is spaces, we will have deferred the sign and hex base prefix - if (fill == ' ') { - if (signString != null) { - buffer.append(signString); + + // Now append the converted argument. + if (skip > 0) { + // The string contains a hex-prefix, but we have already inserted one. + buffer.append(string.substring(skip)); + } else { + buffer.append(string); } - if (altFlag && (c == 'x' || c == 'X')) { - buffer.append('0'); - buffer.append(c); - skip += 2; + + // If this hasn't filled the space required, add right-padding. + while (--width >= length) { + buffer.append(' '); } } - - // Now append the converted argument. - if (skip > 0) { - // The string contains a hex-prefix, but we have already inserted one. - buffer.append(string.substring(skip)); - } else { - buffer.append(string); - } - - // If this hasn't filled the space required, add right-padding. - while (--width >= length) { - buffer.append(' '); - } } /* diff --git a/src/org/python/core/__builtin__.java b/src/org/python/core/__builtin__.java --- a/src/org/python/core/__builtin__.java +++ b/src/org/python/core/__builtin__.java @@ -4,20 +4,20 @@ */ package org.python.core; -import java.io.EOFException; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.util.Iterator; import java.util.Map; import org.python.antlr.base.mod; +import org.python.core.stringlib.IntegerFormatter; +import org.python.core.stringlib.InternalFormat; +import org.python.core.stringlib.InternalFormat.Spec; import org.python.core.util.ExtraMath; import org.python.core.util.RelativeFile; -import org.python.core.util.StringUtil; import org.python.modules._functools._functools; class BuiltinFunctions extends PyBuiltinFunctionSet { @@ -768,7 +768,7 @@ /** * Built-in Python function ord() applicable to the string-like types str, * bytearray, unicode. - * + * * @param c string-like object of length 1 * @return ordinal value of character or byte value in * @throws PyException (TypeError) if not a string-like type @@ -1245,10 +1245,10 @@ PyObject[] args; if (level < 0) { // for backward compatibility provide only 4 arguments - args = new PyObject[] {Py.newString(name), globals, locals, + args = new PyObject[] {Py.newString(name), globals, locals, fromlist}; } else { - args = new PyObject[] {Py.newString(name), globals, locals, + args = new PyObject[] {Py.newString(name), globals, locals, fromlist, Py.newInteger(level)}; } PyObject module = __import__.__call__(args); @@ -1469,7 +1469,7 @@ endObject = useUnicode ? Py.newUnicode(end) : Py.newString(end); } - out.print(values, sepObject, endObject); + out.print(values, sepObject, endObject); } return Py.None; } @@ -1774,10 +1774,6 @@ public PyObject __call__(PyObject args[], String kwds[]) { ArgParser ap = new ArgParser("bin", args, kwds, new String[] {"number"}, 1); ap.noKeywords(); - PyObject number = ap.getPyObject(0); - - //XXX: this could be made more efficient by using a binary only formatter - // instead of using generic formatting. - return number.__format__(new PyString("#b")); + return IntegerFormatter.bin(ap.getPyObject(0)); } } diff --git a/src/org/python/core/stringlib/FloatFormatter.java b/src/org/python/core/stringlib/FloatFormatter.java --- a/src/org/python/core/stringlib/FloatFormatter.java +++ b/src/org/python/core/stringlib/FloatFormatter.java @@ -18,6 +18,10 @@ /** The rounding mode dominant in the formatter. */ static final RoundingMode ROUND_PY = RoundingMode.HALF_EVEN; + /** Limit the size of results. */ + // No-one needs more than log(Double.MAX_VALUE) - log2(Double.MIN_VALUE) = 1383 digits. + static final int MAX_PRECISION = 1400; + /** If it contains no decimal point, this length is zero, and 1 otherwise. */ private int lenPoint; /** The length of the fractional part, right of the decimal point. */ @@ -160,6 +164,12 @@ // Precision defaults to 6 (or 12 for none-format) int precision = spec.getPrecision(Spec.specified(spec.type) ? 6 : 12); + // Guard against excessive result precision + // XXX Possibly better raised before result is allocated/sized. + if (precision > MAX_PRECISION) { + throw precisionTooLarge("float"); + } + /* * By default, the prefix of a positive number is "", but the format specifier may override * it, and the built-in type complex needs to override the format. @@ -905,8 +915,8 @@ } /** - * Return the index in {@link #result} of the first letter. helper for {@link #uppercase()} and - * {@link #getExponent()} + * Return the index in {@link #result} of the first letter. This is a helper for + * {@link #uppercase()} and {@link #getExponent()} */ private int indexOfMarker() { return start + lenSign + lenWhole + lenPoint + lenFraction; diff --git a/src/org/python/core/stringlib/IntegerFormatter.java b/src/org/python/core/stringlib/IntegerFormatter.java new file mode 100644 --- /dev/null +++ b/src/org/python/core/stringlib/IntegerFormatter.java @@ -0,0 +1,696 @@ +// Copyright (c) Jython Developers +package org.python.core.stringlib; + +import java.math.BigInteger; + +import org.python.core.PyInteger; +import org.python.core.PyLong; +import org.python.core.PyObject; +import org.python.core.PyString; +import org.python.core.stringlib.InternalFormat.Spec; + +/** + * A class that provides the implementation of integer formatting. In a limited way, it acts like a + * StringBuilder to which text and one or more numbers may be appended, formatted according to the + * format specifier supplied at construction. These are ephemeral objects that are not, on their + * own, thread safe. + */ +public class IntegerFormatter extends InternalFormat.Formatter { + + /** + * Construct the formatter from a specification. A reference is held to this specification, but + * it will not be modified by the actions of this class. + * + * @param spec parsed conversion specification + */ + public IntegerFormatter(Spec spec) { + // Space for result is based on padded width, or precision, whole part & furniture. + this(spec, 12); + } + + /** + * Construct the formatter from a specification and an explicit initial buffer capacity. A + * reference is held to this specification, but it will not be modified by the actions of this + * class. + * + * @param spec parsed conversion specification + * @param width expected for the formatted result + */ + public IntegerFormatter(Spec spec, int width) { + super(spec, width); + } + + /* + * Re-implement the text appends so they return the right type. + */ + @Override + public IntegerFormatter append(char c) { + super.append(c); + return this; + } + + @Override + public IntegerFormatter append(CharSequence csq) { + super.append(csq); + return this; + } + + @Override + public IntegerFormatter append(CharSequence csq, int start, int end) // + throws IndexOutOfBoundsException { + super.append(csq, start, end); + return this; + } + + /** + * Format a {@link BigInteger}, which is the implementation type of Jython long, + * according to the specification represented by this IntegerFormatter. The + * conversion type, and flags for grouping or base prefix are dealt with here. At the point this + * is used, we know the {@link #spec} is one of the integer types. + * + * @param value to convert + * @return this object + */ + @SuppressWarnings("fallthrough") + public IntegerFormatter format(BigInteger value) { + try { + // Scratch all instance variables and start = result.length(). + setStart(); + + // Different process for each format type. + switch (spec.type) { + case 'd': + case Spec.NONE: + case 'u': + case 'i': + // None format or d-format: decimal + format_d(value); + break; + + case 'x': + // hexadecimal. + format_x(value, false); + break; + + case 'X': + // HEXADECIMAL! + format_x(value, true); + break; + + case 'o': + // Octal. + format_o(value); + break; + + case 'b': + // Binary. + format_b(value); + break; + + case 'n': + // Locale-sensitive version of d-format should be here. + format_d(value); + break; + + default: + // Should never get here, since this was checked in caller. + throw unknownFormat(spec.type, "long"); + } + + // If required to, group the whole-part digits. + if (spec.grouping) { + groupDigits(3, ','); + } + + return this; + + } catch (OutOfMemoryError eme) { + // Most probably due to excessive precision. + throw precisionTooLarge("long"); + } + } + + /** + * Format the value as decimal (into {@link #result}). The option for mandatory sign is dealt + * with by reference to the format specification. + * + * @param value to convert + */ + void format_d(BigInteger value) { + String number; + if (value.signum() < 0) { + // Negative value: deal with sign and base, and convert magnitude. + negativeSign(null); + number = value.negate().toString(); + } else { + // Positive value: deal with sign, base and magnitude. + positiveSign(null); + number = value.toString(); + } + appendNumber(number); + } + + /** + * Format the value as hexadecimal (into {@link #result}), with the option of using upper-case + * or lower-case letters. The options for mandatory sign and for the presence of a base-prefix + * "0x" or "0X" are dealt with by reference to the format specification. + * + * @param value to convert + * @param upper if the hexadecimal should be upper case + */ + void format_x(BigInteger value, boolean upper) { + String base = upper ? "0X" : "0x"; + String number; + if (value.signum() < 0) { + // Negative value: deal with sign and base, and convert magnitude. + negativeSign(base); + number = toHexString(value.negate()); + } else { + // Positive value: deal with sign, base and magnitude. + positiveSign(base); + number = toHexString(value); + } + // Append to result, case-shifted if necessary. + if (upper) { + number = number.toUpperCase(); + } + appendNumber(number); + } + + /** + * Format the value as octal (into {@link #result}). The options for mandatory sign and for the + * presence of a base-prefix "0o" are dealt with by reference to the format specification. + * + * @param value to convert + */ + void format_o(BigInteger value) { + String base = "0o"; + String number; + if (value.signum() < 0) { + // Negative value: deal with sign and base, and convert magnitude. + negativeSign(base); + number = toOctalString(value.negate()); + } else { + // Positive value: deal with sign, base and magnitude. + positiveSign(base); + number = toOctalString(value); + } + // Append to result. + appendNumber(number); + } + + /** + * Format the value as binary (into {@link #result}). The options for mandatory sign and for the + * presence of a base-prefix "0b" are dealt with by reference to the format specification. + * + * @param value to convert + */ + void format_b(BigInteger value) { + String base = "0b"; + String number; + if (value.signum() < 0) { + // Negative value: deal with sign and base, and convert magnitude. + negativeSign(base); + number = toBinaryString(value.negate()); + } else { + // Positive value: deal with sign, base and magnitude. + positiveSign(base); + number = toBinaryString(value); + } + // Append to result. + appendNumber(number); + } + + /** + * Format an integer according to the specification represented by this + * IntegerFormatter. The conversion type, and flags for grouping or base prefix are + * dealt with here. At the point this is used, we know the {@link #spec} is one of the integer + * types. + * + * @param value to convert + * @return this object + */ + @SuppressWarnings("fallthrough") + public IntegerFormatter format(int value) { + try { + // Scratch all instance variables and start = result.length(). + setStart(); + + // Different process for each format type. + switch (spec.type) { + case 'd': + case Spec.NONE: + case 'u': + case 'i': + // None format or d-format: decimal + format_d(value); + break; + + case 'x': + // hexadecimal. + format_x(value, false); + break; + + case 'X': + // HEXADECIMAL! + format_x(value, true); + break; + + case 'o': + // Octal. + format_o(value); + break; + + case 'b': + // Binary. + format_b(value); + break; + + case 'n': + // Locale-sensitive version of d-format should be here. + format_d(value); + break; + + default: + // Should never get here, since this was checked in caller. + throw unknownFormat(spec.type, "integer"); + } + + // If required to, group the whole-part digits. + if (spec.grouping) { + groupDigits(3, ','); + } + + return this; + } catch (OutOfMemoryError eme) { + // Most probably due to excessive precision. + throw precisionTooLarge("integer"); + } + } + + /** + * Format the value as hexadecimal (into {@link #result}), with the option of using upper-case + * or lower-case letters. The options for mandatory sign and for the presence of a base-prefix + * "0x" or "0X" are dealt with by reference to the format specification. + * + * @param value to convert + * @param upper if the hexadecimal should be upper case + */ + void format_x(int value, boolean upper) { + String base = upper ? "0X" : "0x"; + String number; + if (value < 0) { + // Negative value: deal with sign and base, and convert magnitude. + negativeSign(base); + number = Integer.toHexString(-value); + } else { + // Positive value: deal with sign, base and magnitude. + positiveSign(base); + number = Integer.toHexString(value); + } + // Append to result, case-shifted if necessary. + if (upper) { + number = number.toUpperCase(); + } + appendNumber(number); + } + + /** + * Format the value as decimal (into {@link #result}). The option for mandatory sign is dealt + * with by reference to the format specification. + * + * @param value to convert + */ + void format_d(int value) { + String number; + if (value < 0) { + // Negative value: deal with sign and base, and convert magnitude. + negativeSign(null); + number = Integer.toString(-value); + } else { + // Positive value: deal with sign, base and magnitude. + positiveSign(null); + number = Integer.toString(value); + } + appendNumber(number); + } + + /** + * Format the value as octal (into {@link #result}). The options for mandatory sign and for the + * presence of a base-prefix "0o" are dealt with by reference to the format specification. + * + * @param value to convert + */ + void format_o(int value) { + String base = "0o"; + String number; + if (value < 0) { + // Negative value: deal with sign and base, and convert magnitude. + negativeSign(base); + number = Integer.toOctalString(-value); + } else { + // Positive value: deal with sign, base and magnitude. + positiveSign(base); + number = Integer.toOctalString(value); + } + // Append to result. + appendNumber(number); + } + + /** + * Format the value as binary (into {@link #result}). The options for mandatory sign and for the + * presence of a base-prefix "0b" are dealt with by reference to the format specification. + * + * @param value to convert + */ + void format_b(int value) { + String base = "0b"; + String number; + if (value < 0) { + // Negative value: deal with sign and base, and convert magnitude. + negativeSign(base); + number = Integer.toBinaryString(-value); + } else { + // Positive value: deal with sign, base and magnitude. + positiveSign(base); + number = Integer.toBinaryString(value); + } + // Append to result. + appendNumber(number); + } + + /** + * Append to {@link #result} buffer a sign (if one is specified for positive numbers) and, in + * alternate mode, the base marker provided. The sign and base marker are together considered to + * be the "sign" of the converted number, spanned by {@link #lenSign}. This is relevant when we + * come to insert padding. + * + * @param base marker "0x" or "0X" for hex, "0o" for octal, "0b" for binary, "" or + * null for decimal. + */ + final void positiveSign(String base) { + // Does the format specify a sign for positive values? + char sign = spec.sign; + if (Spec.specified(sign)) { + append(sign); + lenSign = 1; + } + // Does the format call for a base prefix? + if (base != null && spec.alternate) { + append(base); + lenSign += base.length(); + } + } + + /** + * Append to {@link #result} buffer a minus sign and, in alternate mode, the base marker + * provided. The sign and base marker are together considered to be the "sign" of the converted + * number, spanned by {@link #lenSign}. This is relevant when we come to insert padding. + * + * @param base marker ("0x" or "0X" for hex, "0" for octal, null or "" for decimal. + */ + final void negativeSign(String base) { + // Insert a minus sign unconditionally. + append('-'); + lenSign = 1; + // Does the format call for a base prefix? + if (base != null && spec.alternate) { + append(base); + lenSign += base.length(); + } + } + + /** + * Append a string (number) to {@link #result} and set {@link #lenWhole} to its length . + * + * @param number to append + */ + void appendNumber(String number) { + lenWhole = number.length(); + append(number); + } + + // For hex-conversion by lookup + private static final String LOOKUP = "0123456789abcdef"; + + /** + * A more efficient algorithm for generating a hexadecimal representation of a byte array. + * {@link BigInteger#toString(int)} is too slow because it generalizes to any radix and, + * consequently, is implemented using expensive mathematical operations. + * + * @param value the value to generate a hexadecimal string from + * @return the hexadecimal representation of value, with "-" sign prepended if necessary + */ + private static final String toHexString(BigInteger value) { + int signum = value.signum(); + + // obvious shortcut + if (signum == 0) { + return "0"; + } + + // we want to work in absolute numeric value (negative sign is added afterward) + byte[] input = value.abs().toByteArray(); + StringBuilder sb = new StringBuilder(input.length * 2); + + int b; + for (int i = 0; i < input.length; i++) { + b = input[i] & 0xFF; + sb.append(LOOKUP.charAt(b >> 4)); + sb.append(LOOKUP.charAt(b & 0x0F)); + } + + // before returning the char array as string, remove leading zeroes, but not the last one + String result = sb.toString().replaceFirst("^0+(?!$)", ""); + return signum < 0 ? "-" + result : result; + } + + /** + * A more efficient algorithm for generating an octal representation of a byte array. + * {@link BigInteger#toString(int)} is too slow because it generalizes to any radix and, + * consequently, is implemented using expensive mathematical operations. + * + * @param value the value to generate an octal string from + * @return the octal representation of value, with "-" sign prepended if necessary + */ + private static final String toOctalString(BigInteger value) { + int signum = value.signum(); + + // obvious shortcut + if (signum == 0) { + return "0"; + } + + byte[] input = value.abs().toByteArray(); + if (input.length < 3) { + return value.toString(8); + } + + StringBuilder sb = new StringBuilder(input.length * 3); + + // working backwards, three bytes at a time + int threebytes; + int trip1, trip2, trip3; // most, middle, and least significant bytes in the triplet + for (int i = input.length - 1; i >= 0; i -= 3) { + trip3 = input[i] & 0xFF; + trip2 = ((i - 1) >= 0) ? (input[i - 1] & 0xFF) : 0x00; + trip1 = ((i - 2) >= 0) ? (input[i - 2] & 0xFF) : 0x00; + threebytes = trip3 | (trip2 << 8) | (trip1 << 16); + + // convert the three-byte value into an eight-character octal string + for (int j = 0; j < 8; j++) { + sb.append(LOOKUP.charAt((threebytes >> (j * 3)) & 0x000007)); + } + } + + String result = sb.reverse().toString().replaceFirst("^0+(?!%)", ""); + return signum < 0 ? "-" + result : result; + } + + /** + * A more efficient algorithm for generating a binary representation of a byte array. + * {@link BigInteger#toString(int)} is too slow because it generalizes to any radix and, + * consequently, is implemented using expensive mathematical operations. + * + * @param value the value to generate a binary string from + * @return the binary representation of value, with "-" sign prepended if necessary + */ + private static final String toBinaryString(BigInteger value) { + int signum = value.signum(); + + // obvious shortcut + if (signum == 0) { + return "0"; + } + + // we want to work in absolute numeric value (negative sign is added afterward) + byte[] input = value.abs().toByteArray(); + StringBuilder sb = new StringBuilder(value.bitCount()); + + int b; + for (int i = 0; i < input.length; i++) { + b = input[i] & 0xFF; + for (int bit = 7; bit >= 0; bit--) { + sb.append(((b >> bit) & 0x1) > 0 ? "1" : "0"); + } + } + + // before returning the char array as string, remove leading zeroes, but not the last one + String result = sb.toString().replaceFirst("^0+(?!$)", ""); + return signum < 0 ? "-" + result : result; + } + + /** Format specification used by bin(). */ + public static final Spec BIN = InternalFormat.fromText("#b"); + + /** Format specification used by oct(). */ + public static final Spec OCT = InternalFormat.fromText("#o"); + + /** Format specification used by hex(). */ + public static final Spec HEX = InternalFormat.fromText("#x"); + + /** + * Convert the object to binary according to the conventions of Python built-in + * bin(). The object's __index__ method is called, and is responsible for raising + * the appropriate error (which the base {@link PyObject#__index__()} does). + * + * @param number to convert + * @return PyString converted result + */ + // Follow this pattern in Python 3, where objects no longer have __hex__, __oct__ members. + public static PyString bin(PyObject number) { + return formatNumber(number, BIN); + } + + /** + * Convert the object according to the conventions of Python built-in hex(), or + * oct(). The object's __index__ method is called, and is responsible + * for raising the appropriate error (which the base {@link PyObject#__index__()} does). + * + * @param number to convert + * @return PyString converted result + */ + public static PyString formatNumber(PyObject number, Spec spec) { + number = number.__index__(); + IntegerFormatter f = new IntegerFormatter(spec); + if (number instanceof PyInteger) { + f.format(((PyInteger)number).getValue()); + } else { + f.format(((PyLong)number).getValue()); + } + return new PyString(f.getResult()); + } + + /** + * A minor variation on {@link IntegerFormatter} to handle "traditional" %-formatting. The + * difference is in the formatting octal in "alternate" mode (0 and 0123, not 0o0 and 0o123). + */ + public static class Traditional extends IntegerFormatter { + + /** + * Construct the formatter from a specification. A reference is held to this specification, + * but it will not be modified by the actions of this class. + * + * @param spec parsed conversion specification + */ + public Traditional(Spec spec) { + super(spec); + } + + /** + * Construct the formatter from a specification and an explicit initial buffer capacity. A + * reference is held to this specification, but it will not be modified by the actions of + * this class. + * + * @param spec parsed conversion specification + * @param width expected for the formatted result + */ + public Traditional(Spec spec, int width) { + super(spec, width); + } + + /** + * Format the value as octal (into {@link #result}). The options for mandatory sign and for + * the presence of a base-prefix "0" are dealt with by reference to the format + * specification. + * + * @param value to convert + */ + @Override + void format_o(BigInteger value) { + String number; + int signum = value.signum(); + if (signum < 0) { + // Negative value: deal with sign and base, and convert magnitude. + negativeSign(null); + number = toOctalString(value.negate()); + } else { + // Positive value: deal with sign, base and magnitude. + positiveSign(null); + number = toOctalString(value); + } + // Append to result. + appendOctalNumber(number); + } + + /** + * Format the value as octal (into {@link #result}). The options for mandatory sign and for + * the presence of a base-prefix "0" are dealt with by reference to the format + * specification. + * + * @param value to convert + */ + @Override + void format_o(int value) { + String number; + if (value < 0) { + // Negative value: deal with sign and convert magnitude. + negativeSign(null); + number = Integer.toOctalString(-value); + } else { + // Positive value: deal with sign, base and magnitude. + positiveSign(null); + number = Integer.toOctalString(value); + } + // Append to result. + appendOctalNumber(number); + } + + /** + * Append a string (number) to {@link #result}, but insert leading zeros first in order + * that, on return, the whole-part length #lenWhole should be no less than the precision. + * + * @param number to append + */ + @Override + void appendNumber(String number) { + int n, p = spec.getPrecision(0); + for (n = number.length(); n < p; n++) { + result.append('0'); + } + lenWhole = n; + append(number); + } + + /** + * Append a string (number) to {@link #result}, but insert leading zeros first in order + * that, on return, the whole-part length #lenWhole should be no less than the precision. + * Octal numbers must begin with a zero if spec.alternate==true, so if the + * number passed in does not start with a zero, at least one will be inserted. + * + * @param number to append + */ + void appendOctalNumber(String number) { + int n = number.length(), p = spec.getPrecision(0); + if (spec.alternate && number.charAt(0) != '0' && n >= p) { + p = n + 1; + } + for (; n < p; n++) { + result.append('0'); + } + lenWhole = n; + append(number); + } + + } +} diff --git a/src/org/python/core/stringlib/InternalFormat.java b/src/org/python/core/stringlib/InternalFormat.java --- a/src/org/python/core/stringlib/InternalFormat.java +++ b/src/org/python/core/stringlib/InternalFormat.java @@ -32,7 +32,7 @@ /** The number we are working on floats at the end of the result, and starts here. */ protected int start; - /** If it contains no sign, this length is zero, and 1 otherwise. */ + /** If it contains no sign, this length is zero, and >0 otherwise. */ protected int lenSign; /** The length of the whole part (to left of the decimal point or exponent) */ protected int lenWhole; @@ -98,7 +98,7 @@ * receive a new one. */ protected void reset() { - // Clear the variable describing the latest object in result. + // Clear the variables describing the latest object in result. lenSign = lenWhole = 0; } @@ -221,13 +221,13 @@ * greater than the current length. *

* When the padding method has decided that that it needs to add n padding characters, it - * will affect {@link #start} or {@link #lenSign} as follows. + * will affect {@link #start} or {@link #lenWhole} as follows. * * * * * - * + * * * * @@ -259,11 +259,10 @@ * * *
alignmeaningstartlenSignlenWholeresult.length()
+n
- * Note that we may have converted more than one value into the result buffer (for example - * when formatting a complex number). The pointer start is at the start of the - * last number converted. Padding with zeros, and the "pad after sign" mode, will produce a - * result you probably don't want. It is up to the client to disallow this (which - * complex does). + * Note that in the "pad after sign" mode, only the last number into the buffer receives the + * padding. This padding gets incorporated into the whole part of the number. (In other + * modes, the padding is around the whole buffer.) When this would not be appropriate, it is + * up to the client to disallow this (which complex does). * * @return this object */ @@ -345,7 +344,7 @@ * * * The padding has increased the overall length of the result to the target width. About one - * in three call to this method adds one to the width, because the whole part cannot start + * in three calls to this method adds one to the width, because the whole part cannot start * with a comma. * * 

@@ -355,9 +354,6 @@
          * '-0,000,000,001,200,000,000.0000'
          *
* - * Insert grouping characters (conventionally commas) into the whole part of the number. - * {@link #lenWhole} will increase correspondingly. - * * @param groupSize normally 3. * @param comma or some other character to use as a separator. */ @@ -386,10 +382,9 @@ * Suppose the format call was format(-12e8, "0=30,.4f"). At the beginning, we had * something like this in result: . [-|000000000001,200,000,000|.|0000||] * - * And now, result looks like this: [-|0000,000,001,200,000,000|.|0000||] in which - * the first zero is wrong as it stands, nor can it just be over-written with a - * comma. We have to insert another zero, even though this makes the result longer - * than we were given. + * And now, result looks like this: [-|,000,000,001,200,000,000|.|0000||] in which + * the first comma is wrong, but so would be a zero. We have to insert another zero, + * even though this makes the result longer than we were asked for. */ result.insert(firstZero, '0'); lenWhole += 1; @@ -457,6 +452,19 @@ return Py.ValueError(msg); } + /** + * Convenience method returning a {@link Py#OverflowError} reporting: + *

+ * "formatted "+type+" is too long (precision too large?)" + * + * @param type of formatting ("integer", "float") + * @return exception to throw + */ + public static PyException precisionTooLarge(String type) { + String msg = "formatted " + type + " is too long (precision too large?)"; + return Py.OverflowError(msg); + } + } /** -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sun Jun 8 14:12:59 2014 From: jython-checkins at python.org (jeff.allen) Date: Sun, 8 Jun 2014 14:12:59 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Trivial_changes_to_format_o?= =?utf-8?q?f_code=2C_ahead_of_changes_to_code_of_=5F=5Fformat=5F=5F=2E?= Message-ID: <3gmc1R47CNz7LjX@mail.python.org> http://hg.python.org/jython/rev/ae93b8016bb6 changeset: 7280:ae93b8016bb6 user: Jeff Allen date: Tue May 27 09:51:44 2014 +0100 summary: Trivial changes to format of code, ahead of changes to code of __format__. Mostly whitespace change and line-wrapping, done here so as not to create clutter in a subsequent material change set. files: src/org/python/core/PyLong.java | 110 +++++++++---------- 1 files changed, 54 insertions(+), 56 deletions(-) diff --git a/src/org/python/core/PyLong.java b/src/org/python/core/PyLong.java --- a/src/org/python/core/PyLong.java +++ b/src/org/python/core/PyLong.java @@ -1,7 +1,6 @@ -/* - * Copyright (c) Corporation for National Research Initiatives - * Copyright (c) Jython Developers - */ +// Copyright (c) Corporation for National Research Initiatives +// Copyright (c) Jython Developers + package org.python.core; import java.io.Serializable; @@ -9,7 +8,6 @@ import java.math.BigInteger; import org.python.core.stringlib.IntegerFormatter; -import org.python.core.stringlib.InternalFormat; import org.python.core.stringlib.InternalFormat.Spec; import org.python.expose.ExposedGet; import org.python.expose.ExposedMethod; @@ -27,8 +25,8 @@ public static final BigInteger MIN_LONG = BigInteger.valueOf(Long.MIN_VALUE); public static final BigInteger MAX_LONG = BigInteger.valueOf(Long.MAX_VALUE); - public static final BigInteger MAX_ULONG = - BigInteger.valueOf(1).shiftLeft(64).subtract(BigInteger.valueOf(1)); + public static final BigInteger MAX_ULONG = BigInteger.valueOf(1).shiftLeft(64) + .subtract(BigInteger.valueOf(1)); /** @deprecated Use MIN_INT instead. */ @Deprecated @@ -69,7 +67,7 @@ @ExposedNew public static PyObject long___new__(PyNewWrapper new_, boolean init, PyType subtype, - PyObject[] args, String[] keywords) { + PyObject[] args, String[] keywords) { if (new_.for_type != subtype) { return longSubtypeNew(new_, init, subtype, args, keywords); } @@ -77,7 +75,7 @@ ArgParser ap = new ArgParser("long", args, keywords, new String[] {"x", "base"}, 0); PyObject x = ap.getPyObject(0, null); if (x != null && x.getJavaProxy() instanceof BigInteger) { - return new PyLong((BigInteger) x.getJavaProxy()); + return new PyLong((BigInteger)x.getJavaProxy()); } int base = ap.getInt(1, -909); @@ -90,7 +88,7 @@ if (!(x instanceof PyString)) { throw Py.TypeError("long: can't convert non-string with explicit base"); } - return ((PyString) x).atol(base); + return ((PyString)x).atol(base); } /** @@ -111,8 +109,9 @@ if (!pye2.match(Py.AttributeError)) { throw pye2; } - throw Py.TypeError( - String.format("long() argument must be a string or a number, not '%.200s'", x.getType().fastGetName())); + throw Py.TypeError(String.format( + "long() argument must be a string or a number, not '%.200s'", x.getType() + .fastGetName())); } } } @@ -126,7 +125,7 @@ PyObject i = integral.invoke("__int__"); if (!(i instanceof PyInteger) && !(i instanceof PyLong)) { throw Py.TypeError(String.format("__trunc__ returned non-Integral (type %.200s)", - integral.getType().fastGetName())); + integral.getType().fastGetName())); } return i; } @@ -136,24 +135,22 @@ /** * Wimpy, slow approach to new calls for subtypes of long. * - * First creates a regular long from whatever arguments we got, then allocates a - * subtype instance and initializes it from the regular long. The regular long is then - * thrown away. + * First creates a regular long from whatever arguments we got, then allocates a subtype + * instance and initializes it from the regular long. The regular long is then thrown away. */ private static PyObject longSubtypeNew(PyNewWrapper new_, boolean init, PyType subtype, - PyObject[] args, String[] keywords) { + PyObject[] args, String[] keywords) { PyObject tmp = long___new__(new_, init, TYPE, args, keywords); if (tmp instanceof PyInteger) { - int intValue = ((PyInteger) tmp).getValue(); + int intValue = ((PyInteger)tmp).getValue(); return new PyLongDerived(subtype, BigInteger.valueOf(intValue)); } else { - return new PyLongDerived(subtype, ((PyLong) tmp).getValue()); + return new PyLongDerived(subtype, ((PyLong)tmp).getValue()); } } /** - * Convert a double to BigInteger, raising an OverflowError if - * infinite. + * Convert a double to BigInteger, raising an OverflowError if infinite. */ private static BigInteger toBigInteger(double value) { if (Double.isInfinite(value)) { @@ -252,7 +249,7 @@ } public double scaledDoubleValue(int[] exp) { - return scaledDoubleValue(getValue(),exp); + return scaledDoubleValue(getValue(), exp); } public long getLong(long min, long max) { @@ -276,14 +273,14 @@ @Override public int asInt(int index) { - return (int) getLong(Integer.MIN_VALUE, Integer.MAX_VALUE, - "long int too large to convert to int"); + return (int)getLong(Integer.MIN_VALUE, Integer.MAX_VALUE, + "long int too large to convert to int"); } @Override public int asInt() { - return (int) getLong(Integer.MIN_VALUE, Integer.MAX_VALUE, - "long int too large to convert to int"); + return (int)getLong(Integer.MIN_VALUE, Integer.MAX_VALUE, + "long int too large to convert to int"); } @Override @@ -295,13 +292,13 @@ public Object __tojava__(Class c) { try { if (c == Byte.TYPE || c == Byte.class) { - return new Byte((byte) getLong(Byte.MIN_VALUE, Byte.MAX_VALUE)); + return new Byte((byte)getLong(Byte.MIN_VALUE, Byte.MAX_VALUE)); } if (c == Short.TYPE || c == Short.class) { - return new Short((short) getLong(Short.MIN_VALUE, Short.MAX_VALUE)); + return new Short((short)getLong(Short.MIN_VALUE, Short.MAX_VALUE)); } if (c == Integer.TYPE || c == Integer.class) { - return new Integer((int) getLong(Integer.MIN_VALUE, Integer.MAX_VALUE)); + return new Integer((int)getLong(Integer.MIN_VALUE, Integer.MAX_VALUE)); } if (c == Long.TYPE || c == Long.class) { return new Long(getLong(Long.MIN_VALUE, Long.MAX_VALUE)); @@ -310,7 +307,7 @@ return __float__().__tojava__(c); } if (c == BigInteger.class || c == Number.class || c == Object.class - || c == Serializable.class) { + || c == Serializable.class) { return getValue(); } } catch (PyException e) { @@ -343,14 +340,14 @@ } /** - * Coercion logic for long. Implemented as a final method to avoid - * invocation of virtual methods from the exposed coerce. + * Coercion logic for long. Implemented as a final method to avoid invocation of virtual methods + * from the exposed coerce. */ final Object long___coerce_ex__(PyObject other) { if (other instanceof PyLong) { return other; } else if (other instanceof PyInteger) { - return Py.newLong(((PyInteger) other).getValue()); + return Py.newLong(((PyInteger)other).getValue()); } else { return Py.None; } @@ -362,9 +359,9 @@ private static final BigInteger coerce(PyObject other) { if (other instanceof PyLong) { - return ((PyLong) other).getValue(); + return ((PyLong)other).getValue(); } else if (other instanceof PyInteger) { - return BigInteger.valueOf(((PyInteger) other).getValue()); + return BigInteger.valueOf(((PyInteger)other).getValue()); } else { throw Py.TypeError("xxx"); } @@ -424,7 +421,7 @@ @ExposedMethod(type = MethodType.BINARY, doc = BuiltinDocs.long___mul___doc) final PyObject long___mul__(PyObject right) { if (right instanceof PySequence) { - return ((PySequence) right).repeat(coerceInt(this)); + return ((PySequence)right).repeat(coerceInt(this)); } if (!canCoerce(right)) { @@ -441,7 +438,7 @@ @ExposedMethod(type = MethodType.BINARY, doc = BuiltinDocs.long___rmul___doc) final PyObject long___rmul__(PyObject left) { if (left instanceof PySequence) { - return ((PySequence) left).repeat(coerceInt(this)); + return ((PySequence)left).repeat(coerceInt(this)); } if (!canCoerce(left)) { return null; @@ -482,7 +479,7 @@ if (Options.division_warning > 0) { Py.warning(Py.DeprecationWarning, "classic long division"); } - return Py.newLong(divide( getValue(), coerce(right))); + return Py.newLong(divide(getValue(), coerce(right))); } @Override @@ -511,7 +508,7 @@ if (!canCoerce(right)) { return null; } - return Py.newLong(divide( getValue(), coerce(right))); + return Py.newLong(divide(getValue(), coerce(right))); } @Override @@ -567,7 +564,7 @@ if (!canCoerce(right)) { return null; } - return true_divide( this.getValue(), coerce(right)); + return true_divide(this.getValue(), coerce(right)); } @Override @@ -598,7 +595,7 @@ return null; } BigInteger rightv = coerce(right); - return Py.newLong(modulo(getValue(),rightv, divide(getValue(),rightv))); + return Py.newLong(modulo(getValue(), rightv, divide(getValue(), rightv))); } @Override @@ -627,8 +624,8 @@ } BigInteger rightv = coerce(right); - BigInteger xdivy = divide(getValue(),rightv); - return new PyTuple(Py.newLong(xdivy), Py.newLong(modulo(getValue(),rightv, xdivy))); + BigInteger xdivy = divide(getValue(), rightv); + return new PyTuple(Py.newLong(xdivy), Py.newLong(modulo(getValue(), rightv, xdivy))); } @Override @@ -653,7 +650,7 @@ } @ExposedMethod(type = MethodType.BINARY, defaults = {"null"}, - doc = BuiltinDocs.long___pow___doc) + doc = BuiltinDocs.long___pow___doc) final PyObject long___pow__(PyObject right, PyObject modulo) { if (!canCoerce(right)) { return null; @@ -662,7 +659,7 @@ if (modulo != null && !canCoerce(right)) { return null; } - return _pow( getValue(), coerce(right), modulo, this, right); + return _pow(getValue(), coerce(right), modulo, this, right); } @Override @@ -679,7 +676,7 @@ } public static PyObject _pow(BigInteger value, BigInteger y, PyObject modulo, PyObject left, - PyObject right) { + PyObject right) { if (y.compareTo(BigInteger.ZERO) < 0) { if (value.compareTo(BigInteger.ZERO) != 0) { return left.__float__().__pow__(right, modulo); @@ -703,32 +700,32 @@ } if (z.compareTo(BigInteger.valueOf(0)) <= 0) { - // Handle negative modulo's specially - /*if (z.compareTo(BigInteger.valueOf(0)) == 0) { - throw Py.ValueError("pow(x, y, z) with z == 0"); - }*/ + // Handle negative modulo specially + // if (z.compareTo(BigInteger.valueOf(0)) == 0) { + // throw Py.ValueError("pow(x, y, z) with z == 0"); + // } y = value.modPow(y, z.negate()); if (y.compareTo(BigInteger.valueOf(0)) > 0) { return Py.newLong(z.add(y)); } else { return Py.newLong(y); } - //return __pow__(right).__mod__(modulo); + // return __pow__(right).__mod__(modulo); } else { // XXX: 1.1 no longer supported so review this. // This is buggy in SUN's jdk1.1.5 // Extra __mod__ improves things slightly return Py.newLong(value.modPow(y, z)); - //return __pow__(right).__mod__(modulo); + // return __pow__(right).__mod__(modulo); } } } private static final int coerceInt(PyObject other) { if (other instanceof PyLong) { - return ((PyLong) other).asInt(); + return ((PyLong)other).asInt(); } else if (other instanceof PyInteger) { - return ((PyInteger) other).getValue(); + return ((PyInteger)other).getValue(); } else { throw Py.TypeError("xxx"); } @@ -918,7 +915,8 @@ @ExposedMethod(doc = BuiltinDocs.long___int___doc) final PyObject long___int__() { - if (getValue().compareTo(PyInteger.MAX_INT) <= 0 && getValue().compareTo(PyInteger.MIN_INT) >= 0) { + if (getValue().compareTo(PyInteger.MAX_INT) <= 0 + && getValue().compareTo(PyInteger.MIN_INT) >= 0) { return Py.newInteger(getValue().intValue()); } return long___long__(); @@ -1082,7 +1080,7 @@ } return tooLow ? Integer.MIN_VALUE : Integer.MAX_VALUE; } - return (int) getValue().longValue(); + return (int)getValue().longValue(); } @Override -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sun Jun 8 14:13:01 2014 From: jython-checkins at python.org (jeff.allen) Date: Sun, 8 Jun 2014 14:13:01 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Refactor_int=2E=5F=5Fformat?= =?utf-8?q?=5F=5F_and_long=2E=5F=5Fformat=5F=5F_to_use_stringlib=2EInteger?= =?utf-8?q?Formatter=2E?= Message-ID: <3gmc1T1Zp0z7LjV@mail.python.org> http://hg.python.org/jython/rev/899940f3fcf8 changeset: 7281:899940f3fcf8 user: Jeff Allen date: Tue May 27 14:46:20 2014 +0100 summary: Refactor int.__format__ and long.__format__ to use stringlib.IntegerFormatter. Also reworks StringFormatterTest. files: src/org/python/core/PyInteger.java | 190 ++------ src/org/python/core/PyLong.java | 7 +- src/org/python/core/stringlib/FloatFormatter.java | 5 +- src/org/python/core/stringlib/IntegerFormatter.java | 132 +++++- src/org/python/core/stringlib/InternalFormat.java | 129 +++++- tests/java/org/python/core/StringFormatTest.java | 206 ++++++--- 6 files changed, 434 insertions(+), 235 deletions(-) diff --git a/src/org/python/core/PyInteger.java b/src/org/python/core/PyInteger.java --- a/src/org/python/core/PyInteger.java +++ b/src/org/python/core/PyInteger.java @@ -4,13 +4,10 @@ import java.io.Serializable; import java.math.BigInteger; -import java.text.NumberFormat; -import java.util.Locale; import org.python.core.stringlib.IntegerFormatter; +import org.python.core.stringlib.InternalFormat; import org.python.core.stringlib.InternalFormat.Spec; -import org.python.core.stringlib.InternalFormatSpec; -import org.python.core.stringlib.InternalFormatSpecParser; import org.python.expose.ExposedGet; import org.python.expose.ExposedMethod; import org.python.expose.ExposedNew; @@ -1022,151 +1019,68 @@ return int___format__(formatSpec); } + @SuppressWarnings("fallthrough") @ExposedMethod(doc = BuiltinDocs.int___format___doc) final PyObject int___format__(PyObject formatSpec) { - return formatImpl(getValue(), formatSpec); - } - - static PyObject formatImpl(Object value, PyObject formatSpec) { - if (!(formatSpec instanceof PyString)) { - throw Py.TypeError("__format__ requires str or unicode"); - } - - PyString formatSpecStr = (PyString)formatSpec; - String result; - try { - String specString = formatSpecStr.getString(); - InternalFormatSpec spec = new InternalFormatSpecParser(specString).parse(); - result = formatIntOrLong(value, spec); - } catch (IllegalArgumentException e) { - throw Py.ValueError(e.getMessage()); - } - return formatSpecStr.createInstance(result); + // Get a formatter for the specification + IntegerFormatter f = prepareFormatter(formatSpec); + // Convert as per specification. + f.format(value); + // Return a result that has the same type (str or unicode) as the formatSpec argument. + return f.pad().getPyResult(); } /** - * Formats an integer or long number according to a PEP-3101 format specification. + * Common code for PyInteger and PyLong to prepare an IntegerFormatter. This object has an + * overloaded format method {@link IntegerFormatter#format(int)} and + * {@link IntegerFormatter#format(BigInteger)} to support the two types. * - * @param value Integer or BigInteger object specifying the value to format. - * @param spec parsed PEP-3101 format specification. - * @return result of the formatting. + * @param formatSpec PEP-3101 format specification. + * @return a formatter ready to use. + * @throws PyException(ValueError) if the specification is faulty. */ - public static String formatIntOrLong(Object value, InternalFormatSpec spec) { - if (spec.precision != -1) { - throw new IllegalArgumentException("Precision not allowed in integer format specifier"); + static IntegerFormatter prepareFormatter(PyObject formatSpec) throws PyException { + + // Parse the specification + Spec spec = InternalFormat.fromText(formatSpec, "__format__"); + IntegerFormatter f; + + // Check for disallowed parts of the specification + if (Spec.specified(spec.precision)) { + throw IntegerFormatter.precisionNotAllowed("integer"); } - int sign; - if (value instanceof Integer) { - int intValue = (Integer)value; - sign = intValue < 0 ? -1 : intValue == 0 ? 0 : 1; - } else { - sign = ((BigInteger)value).signum(); + // Slight differences between format types + switch (spec.type) { + case 'c': + // Character data + if (Spec.specified(spec.sign)) { + throw IntegerFormatter.notAllowed("Sign", "integer", spec.type); + } else if (spec.alternate) { + throw IntegerFormatter.alternateFormNotAllowed("integer", spec.type); + } + // Fall through + + case Spec.NONE: + case 'd': + case 'x': + case 'X': + case 'o': + case 'b': + case 'n': + // spec may be incomplete. The defaults are those commonly used for numeric formats. + spec = spec.withDefaults(Spec.NUMERIC); + // Get a formatter for the spec. + f = new IntegerFormatter(spec, 1); + // Bytes mode if formatSpec argument is not unicode. + f.setBytes(!(formatSpec instanceof PyUnicode)); + break; + + default: + throw IntegerFormatter.unknownFormat(spec.type, "integer"); } - String strValue; - String strPrefix = ""; - String strSign = ""; - - if (spec.type == 'c') { - if (spec.sign != '\0') { - throw new IllegalArgumentException("Sign not allowed with integer format " - + "specifier 'c'"); - } - if (value instanceof Integer) { - int intValue = (Integer)value; - if (intValue > 0xffff) { - throw new IllegalArgumentException("%c arg not in range(0x10000)"); - } - strValue = Character.toString((char)intValue); - } else { - BigInteger bigInt = (BigInteger)value; - if (bigInt.intValue() > 0xffff || bigInt.bitCount() > 16) { - throw new IllegalArgumentException("%c arg not in range(0x10000)"); - } - strValue = Character.toString((char)bigInt.intValue()); - } - } else { - int radix = 10; - if (spec.type == 'o') { - radix = 8; - } else if (spec.type == 'x' || spec.type == 'X') { - radix = 16; - } else if (spec.type == 'b') { - radix = 2; - } - - if (spec.type == 'n') { - strValue = NumberFormat.getNumberInstance().format(value); - } else if (spec.thousands_separators) { - NumberFormat format = NumberFormat.getNumberInstance(Locale.US); - format.setGroupingUsed(true); - strValue = format.format(value); - } else if (value instanceof BigInteger) { - switch (radix) { - case 2: - strValue = toBinString((BigInteger)value); - break; - case 8: - strValue = toOctString((BigInteger)value); - break; - case 16: - strValue = toHexString((BigInteger)value); - break; - default: - // General case (v.slow in known implementations up to Java 7). - strValue = ((BigInteger)value).toString(radix); - break; - } - } else { - strValue = Integer.toString((Integer)value, radix); - } - - if (spec.alternate) { - switch (radix) { - case 2: - strPrefix = "0b"; - break; - case 8: - strPrefix = "0o"; - break; - case 16: - strPrefix = "0x"; - break; - } - - if (sign < 0) { - assert (strValue.startsWith("-")); - strSign = "-"; - strValue = strValue.substring(1); - } - } - - if (spec.type == 'X') { - strPrefix = strPrefix.toUpperCase(); - strValue = strValue.toUpperCase(); - } - - if (sign >= 0) { - switch (spec.sign) { - case '+': - case ' ': - strSign = Character.toString(spec.sign); - break; - } - } - } - - if (spec.align == '=' && (spec.sign == '-' || spec.sign == '+' || spec.sign == ' ')) { - assert (strSign.length() == 1); - return strSign + strPrefix + spec.pad(strValue, '>', 1 + strPrefix.length()); - } - - if (spec.fill_char == 0) { - return spec.pad(strSign + strPrefix + strValue, '>', 0); - } - - return strSign + strPrefix + spec.pad(strValue, '>', strSign.length() + strPrefix.length()); + return f; } /** diff --git a/src/org/python/core/PyLong.java b/src/org/python/core/PyLong.java --- a/src/org/python/core/PyLong.java +++ b/src/org/python/core/PyLong.java @@ -1062,7 +1062,12 @@ @ExposedMethod(doc = BuiltinDocs.long___format___doc) final PyObject long___format__(PyObject formatSpec) { - return PyInteger.formatImpl(getValue(), formatSpec); + // Get a formatter for the specification + IntegerFormatter f = PyInteger.prepareFormatter(formatSpec); + // Convert as per specification (note this supports BigDecimal). + f.format(value); + // Return a result that has the same type (str or unicode) as the formatSpec argument. + return f.pad().getPyResult(); } @Override diff --git a/src/org/python/core/stringlib/FloatFormatter.java b/src/org/python/core/stringlib/FloatFormatter.java --- a/src/org/python/core/stringlib/FloatFormatter.java +++ b/src/org/python/core/stringlib/FloatFormatter.java @@ -174,8 +174,9 @@ * By default, the prefix of a positive number is "", but the format specifier may override * it, and the built-in type complex needs to override the format. */ - if (positivePrefix == null && Spec.specified(spec.sign) && spec.sign != '-') { - positivePrefix = Character.toString(spec.sign); + char sign = spec.sign; + if (positivePrefix == null && Spec.specified(sign) && sign != '-') { + positivePrefix = Character.toString(sign); } // Different process for each format type, ignoring case for now. diff --git a/src/org/python/core/stringlib/IntegerFormatter.java b/src/org/python/core/stringlib/IntegerFormatter.java --- a/src/org/python/core/stringlib/IntegerFormatter.java +++ b/src/org/python/core/stringlib/IntegerFormatter.java @@ -3,10 +3,12 @@ import java.math.BigInteger; +import org.python.core.Py; import org.python.core.PyInteger; import org.python.core.PyLong; import org.python.core.PyObject; import org.python.core.PyString; +import org.python.core.PySystemState; import org.python.core.stringlib.InternalFormat.Spec; /** @@ -107,6 +109,11 @@ format_b(value); break; + case 'c': + // Binary. + format_c(value); + break; + case 'n': // Locale-sensitive version of d-format should be here. format_d(value); @@ -222,6 +229,26 @@ } /** + * Format the value as a character (into {@link #result}). + * + * @param value to convert + */ + void format_c(BigInteger value) { + // Limit is 256 if we're formatting for byte output, unicode range otherwise. + BigInteger limit = bytes ? LIMIT_BYTE : LIMIT_UNICODE; + if (value.signum() < 0 || value.compareTo(limit) >= 0) { + throw Py.OverflowError("%c arg not in range(0x" + toHexString(limit) + ")"); + } else { + result.appendCodePoint(value.intValue()); + } + } + + // Limits used in format_c(BigInteger) + private static final BigInteger LIMIT_UNICODE = BigInteger + .valueOf(PySystemState.maxunicode + 1); + private static final BigInteger LIMIT_BYTE = BigInteger.valueOf(256); + + /** * Format an integer according to the specification represented by this * IntegerFormatter. The conversion type, and flags for grouping or base prefix are * dealt with here. At the point this is used, we know the {@link #spec} is one of the integer @@ -266,6 +293,11 @@ format_b(value); break; + case 'c': + // Binary. + format_c(value); + break; + case 'n': // Locale-sensitive version of d-format should be here. format_d(value); @@ -289,6 +321,26 @@ } /** + * Format the value as decimal (into {@link #result}). The option for mandatory sign is dealt + * with by reference to the format specification. + * + * @param value to convert + */ + void format_d(int value) { + String number; + if (value < 0) { + // Negative value: deal with sign and base, and convert magnitude. + negativeSign(null); + number = Integer.toString(-value); + } else { + // Positive value: deal with sign, base and magnitude. + positiveSign(null); + number = Integer.toString(value); + } + appendNumber(number); + } + + /** * Format the value as hexadecimal (into {@link #result}), with the option of using upper-case * or lower-case letters. The options for mandatory sign and for the presence of a base-prefix * "0x" or "0X" are dealt with by reference to the format specification. @@ -316,26 +368,6 @@ } /** - * Format the value as decimal (into {@link #result}). The option for mandatory sign is dealt - * with by reference to the format specification. - * - * @param value to convert - */ - void format_d(int value) { - String number; - if (value < 0) { - // Negative value: deal with sign and base, and convert magnitude. - negativeSign(null); - number = Integer.toString(-value); - } else { - // Positive value: deal with sign, base and magnitude. - positiveSign(null); - number = Integer.toString(value); - } - appendNumber(number); - } - - /** * Format the value as octal (into {@link #result}). The options for mandatory sign and for the * presence of a base-prefix "0o" are dealt with by reference to the format specification. * @@ -380,6 +412,21 @@ } /** + * Format the value as a character (into {@link #result}). + * + * @param value to convert + */ + void format_c(int value) { + // Limit is 256 if we're formatting for byte output, unicode range otherwise. + int limit = bytes ? 256 : PySystemState.maxunicode + 1; + if (value < 0 || value >= limit) { + throw Py.OverflowError("%c arg not in range(0x" + Integer.toHexString(limit) + ")"); + } else { + result.appendCodePoint(value); + } + } + + /** * Append to {@link #result} buffer a sign (if one is specified for positive numbers) and, in * alternate mode, the base marker provided. The sign and base marker are together considered to * be the "sign" of the converted number, spanned by {@link #lenSign}. This is relevant when we @@ -391,7 +438,7 @@ final void positiveSign(String base) { // Does the format specify a sign for positive values? char sign = spec.sign; - if (Spec.specified(sign)) { + if (Spec.specified(sign) && sign != '-') { append(sign); lenSign = 1; } @@ -583,7 +630,8 @@ /** * A minor variation on {@link IntegerFormatter} to handle "traditional" %-formatting. The - * difference is in the formatting octal in "alternate" mode (0 and 0123, not 0o0 and 0o123). + * difference is in support for spec.precision, the formatting octal in "alternate" + * mode (0 and 0123, not 0o0 and 0o123), and in c-format (in the error logic). */ public static class Traditional extends IntegerFormatter { @@ -634,6 +682,26 @@ } /** + * Format the value as a character (into {@link #result}). + * + * @param value to convert + */ + @Override + void format_c(BigInteger value) { + if (value.signum() < 0) { + throw Py.OverflowError("unsigned byte integer is less than minimum"); + } else { + // Limit is 256 if we're formatting for byte output, unicode range otherwise. + BigInteger limit = bytes ? LIMIT_BYTE : LIMIT_UNICODE; + if (value.compareTo(limit) >= 0) { + throw Py.OverflowError("unsigned byte integer is greater than maximum"); + } else { + result.appendCodePoint(value.intValue()); + } + } + } + + /** * Format the value as octal (into {@link #result}). The options for mandatory sign and for * the presence of a base-prefix "0" are dealt with by reference to the format * specification. @@ -657,6 +725,26 @@ } /** + * Format the value as a character (into {@link #result}). + * + * @param value to convert + */ + @Override + void format_c(int value) { + if (value < 0) { + throw Py.OverflowError("unsigned byte integer is less than minimum"); + } else { + // Limit is 256 if we're formatting for byte output, unicode range otherwise. + int limit = bytes ? 256 : PySystemState.maxunicode + 1; + if (value >= limit) { + throw Py.OverflowError("unsigned byte integer is greater than maximum"); + } else { + result.appendCodePoint(value); + } + } + } + + /** * Append a string (number) to {@link #result}, but insert leading zeros first in order * that, on return, the whole-part length #lenWhole should be no less than the precision. * diff --git a/src/org/python/core/stringlib/InternalFormat.java b/src/org/python/core/stringlib/InternalFormat.java --- a/src/org/python/core/stringlib/InternalFormat.java +++ b/src/org/python/core/stringlib/InternalFormat.java @@ -3,6 +3,9 @@ import org.python.core.Py; import org.python.core.PyException; +import org.python.core.PyObject; +import org.python.core.PyString; +import org.python.core.PyUnicode; public class InternalFormat { @@ -14,7 +17,25 @@ */ public static Spec fromText(String text) { Parser parser = new Parser(text); - return parser.parse(); + try { + return parser.parse(); + } catch (IllegalArgumentException e) { + throw Py.ValueError(e.getMessage()); + } + } + + /** + * Create a {@link Spec} object by parsing a format specification, supplied as an object. + * + * @param text to parse + * @return parsed equivalent to text + */ + public static Spec fromText(PyObject text, String method) { + if (text instanceof PyString) { + return fromText(((PyString)text).getString()); + } else { + throw Py.TypeError(method + " requires str or unicode"); + } } /** @@ -30,6 +51,12 @@ /** The (partial) result. */ protected StringBuilder result; + /** + * Signals the client's intention to make a PyString (or other byte-like) interpretation of + * {@link #result}, rather than a PyUnicode one. + */ + protected boolean bytes; + /** The number we are working on floats at the end of the result, and starts here. */ protected int start; /** If it contains no sign, this length is zero, and >0 otherwise. */ @@ -50,6 +77,33 @@ } /** + * Signals the client's intention to make a PyString (or other byte-like) interpretation of + * {@link #result}, rather than a PyUnicode one. Only formatters that could produce + * characters >255 are affected by this (e.g. c-format). Idiom: + * + * 

+         * MyFormatter f = new MyFormatter( InternalFormatter.fromText(formatSpec) );
+         * f.setBytes(!(formatSpec instanceof PyUnicode));
+         * // ... formatting work
+         * return f.getPyResult();
+         *
+ * + * @param bytes true to signal the intention to make a byte-like interpretation + */ + public void setBytes(boolean bytes) { + this.bytes = bytes; + } + + /** + * Whether initialised for a byte-like interpretation. + * + * @return bytes attribute + */ + public boolean isBytes() { + return bytes; + } + + /** * Current (possibly final) result of the formatting, as a String. * * @return formatted result @@ -58,6 +112,22 @@ return result.toString(); } + /** + * Convenience method to return the current result of the formatting, as a + * PyObject, either {@link PyString} or {@link PyUnicode} according to + * {@link #bytes}. + * + * @return formatted result + */ + public PyString getPyResult() { + String r = getResult(); + if (bytes) { + return new PyString(r); + } else { + return new PyUnicode(r); + } + } + /* * Implement Appendable interface by delegation to the result buffer. * @@ -86,7 +156,7 @@ * Clear the instance variables describing the latest object in {@link #result}, ready to * receive a new number */ - public void setStart() { + void setStart() { // Mark the end of the buffer as the start of the current object and reset all. start = result.length(); // Clear the variable describing the latest number in result. @@ -413,7 +483,19 @@ * @return exception to throw */ public static PyException alternateFormNotAllowed(String forType) { - return notAllowed("Alternate form (#)", forType); + return alternateFormNotAllowed(forType, '\0'); + } + + /** + * Convenience method returning a {@link Py#ValueError} reporting that alternate form is not + * allowed in a format specifier for the named type and specified typoe code. + * + * @param forType the type it was found applied to + * @param code the formatting code (or '\0' not to mention one) + * @return exception to throw + */ + public static PyException alternateFormNotAllowed(String forType, char code) { + return notAllowed("Alternate form (#)", forType, code); } /** @@ -425,7 +507,18 @@ * @return exception to throw */ public static PyException alignmentNotAllowed(char align, String forType) { - return notAllowed("'" + align + "' alignment flag", forType); + return notAllowed("'" + align + "' alignment flag", forType, '\0'); + } + + /** + * Convenience method returning a {@link Py#ValueError} reporting that specifying a + * precision is not allowed in a format specifier for the named type. + * + * @param forType the type it was found applied to + * @return exception to throw + */ + public static PyException precisionNotAllowed(String forType) { + return notAllowed("Precision", forType, '\0'); } /** @@ -436,19 +529,35 @@ * @return exception to throw */ public static PyException zeroPaddingNotAllowed(String forType) { - return notAllowed("Zero padding", forType); + return notAllowed("Zero padding", forType, '\0'); } /** * Convenience method returning a {@link Py#ValueError} reporting that some format specifier - * feature is not allowed for the named type. + * feature is not allowed for the named format code and data type. Produces a message like: + *

+ * outrage+" not allowed with "+forType+" format specifier '"+code+"'" + *

+ * outrage+" not allowed in "+forType+" format specifier" * - * @param particularOutrage committed in the present case - * @param forType the type it where it is an outrage + * @param outrage committed in the present case + * @param forType the data type (e.g. "integer") it where it is an outrage + * @param code the formatting code for which it is an outrage (or '\0' not to mention one) * @return exception to throw */ - protected static PyException notAllowed(String particularOutrage, String forType) { - String msg = particularOutrage + " is not allowed in " + forType + " format specifier"; + public static PyException notAllowed(String outrage, String forType, char code) { + // Try really hard to be like CPython + String codeAsString, withOrIn; + if (code == 0) { + withOrIn = "in "; + codeAsString = ""; + } else { + withOrIn = "with "; + codeAsString = " '" + code + "'"; + } + String msg = + outrage + " not allowed " + withOrIn + forType + " format specifier" + + codeAsString; return Py.ValueError(msg); } diff --git a/tests/java/org/python/core/StringFormatTest.java b/tests/java/org/python/core/StringFormatTest.java --- a/tests/java/org/python/core/StringFormatTest.java +++ b/tests/java/org/python/core/StringFormatTest.java @@ -1,15 +1,25 @@ package org.python.core; +import java.math.BigInteger; + import junit.framework.TestCase; + import org.python.core.stringlib.FieldNameIterator; +import org.python.core.stringlib.IntegerFormatter; +import org.python.core.stringlib.InternalFormat.Formatter; import org.python.core.stringlib.InternalFormatSpec; import org.python.core.stringlib.InternalFormatSpecParser; import org.python.core.stringlib.MarkupIterator; +import org.python.util.PythonInterpreter; /** * Tests for internal bits and pieces of string.format implementation. */ public class StringFormatTest extends TestCase { + + /** Exception-raising seems to need the interpreter to be initialised **/ + PythonInterpreter interp = new PythonInterpreter(); + public void testInternalFormatSpec() { InternalFormatSpec spec = new InternalFormatSpecParser("x").parse(); assertEquals('x', spec.type); @@ -59,69 +69,151 @@ assertEquals(expected, error); } - public void testFormatIntOrLong() { - InternalFormatSpec spec = new InternalFormatSpec(); - spec.type = 'd'; - assertEquals("123", PyInteger.formatIntOrLong(123, spec)); - spec.type = 'o'; - assertEquals("173", PyInteger.formatIntOrLong(123, spec)); - spec.type = 'x'; - assertEquals("7b", PyInteger.formatIntOrLong(123, spec)); - spec.type = 'X'; - assertEquals("7B", PyInteger.formatIntOrLong(123, spec)); - spec.type = 'b'; - assertEquals("1111011", PyInteger.formatIntOrLong(123, spec)); + /** + * Test the IntegerFormatter returned by {@link PyInteger#prepareFormat}. This is based on the original + * testFormatIntOrLong which tested PyInteger.formatIntOrLong. + */ + public void testPrepareFormatter() { + int v = 123; + IntegerFormatter f; + f = PyInteger.prepareFormatter(new PyString("d")); + assertEquals("123", f.format(v).pad().getResult()); + f = PyInteger.prepareFormatter(new PyString("o")); + assertEquals("173", f.format(v).pad().getResult()); + f = PyInteger.prepareFormatter(new PyString("x")); + assertEquals("7b", f.format(v).pad().getResult()); + f = PyInteger.prepareFormatter(new PyString("X")); + assertEquals("7B", f.format(v).pad().getResult()); + f = PyInteger.prepareFormatter(new PyString("b")); + assertEquals("1111011", f.format(v).pad().getResult()); - spec.thousands_separators = true; - spec.type = 'd'; - assertEquals("1,234", PyInteger.formatIntOrLong(1234, spec)); - spec.thousands_separators = false; + int v2 = 1234567890; + f = PyInteger.prepareFormatter(new PyString(",d")); + assertEquals("1,234,567,890", f.format(v2).pad().getResult()); - spec.alternate = true; - spec.type = 'o'; - assertEquals("0o173", PyInteger.formatIntOrLong(123, spec)); - spec.type = 'X'; - assertEquals("0X7B", PyInteger.formatIntOrLong(123, spec)); - spec.alternate = false; + f = PyInteger.prepareFormatter(new PyString("#o")); + assertEquals("0o173", f.format(v).pad().getResult()); + f = PyInteger.prepareFormatter(new PyString("#X")); + assertEquals("0X7B", f.format(v).pad().getResult()); - spec.type = 'c'; - assertEquals("{", PyInteger.formatIntOrLong(123, spec)); + f = PyInteger.prepareFormatter(new PyString("c")); + assertEquals("{", f.format(v).pad().getResult()); - spec.type = 'd'; - spec.sign = '+'; - assertEquals("+123", PyInteger.formatIntOrLong(123, spec)); - spec.sign = ' '; - assertEquals(" 123", PyInteger.formatIntOrLong(123, spec)); + f = PyInteger.prepareFormatter(new PyString("+d")); + assertEquals("+123", f.format(v).pad().getResult()); + f = PyInteger.prepareFormatter(new PyString(" d")); + assertEquals(" 123", f.format(v).pad().getResult()); - spec.sign = 0; - spec.width = 5; - assertEquals(" 123", PyInteger.formatIntOrLong(123, spec)); + f = PyInteger.prepareFormatter(new PyString("5")); + assertEquals(" 123", f.format(v).pad().getResult()); - spec.align = '^'; - spec.width = 6; - assertEquals(" 123 ", PyInteger.formatIntOrLong(123, spec)); + f = PyInteger.prepareFormatter(new PyString("^6")); + assertEquals(" 123 ", f.format(v).pad().getResult()); - spec.align = '<'; - spec.width = 5; - spec.fill_char = '~'; - assertEquals("123~~", PyInteger.formatIntOrLong(123, spec)); + f = PyInteger.prepareFormatter(new PyString("~<5")); + assertEquals("123~~", f.format(v).pad().getResult()); - spec.align = '='; - spec.width = 6; - spec.fill_char = '0'; - spec.sign = '+'; - assertEquals("+00123", PyInteger.formatIntOrLong(123, spec)); + f = PyInteger.prepareFormatter(new PyString("0=+6")); + assertEquals("+00123", f.format(v).pad().getResult()); - spec.precision = 1; - assertFormatError(123, spec, "Precision not allowed in integer format specifier"); + assertValueError("0=+6.1", "Precision not allowed in integer format specifier"); + assertValueError("+c", "Sign not allowed with integer format specifier 'c'"); - spec.precision = -1; - spec.sign = '+'; - spec.type = 'c'; - assertFormatError(123, spec, "Sign not allowed with integer format specifier 'c'"); + f = PyInteger.prepareFormatter(new PyString("c")); + assertOverflowError(256, f, "%c arg not in range(0x100)"); + assertOverflowError(-1, f, "%c arg not in range(0x100)"); + assertOverflowError(0x110000, f, "%c arg not in range(0x100)"); - spec.sign = 0; - assertFormatError(0x11111, spec, "%c arg not in range(0x10000)"); + f = PyInteger.prepareFormatter(new PyUnicode("c")); + assertOverflowError(0x110000, f, "%c arg not in range(0x110000)"); + assertOverflowError(-1, f, "%c arg not in range(0x110000)"); + } + + /** + * Test the IntegerFormatter returned by {@link PyInteger#prepareFormat}. This is based on the original + * testFormatIntOrLong which tested PyInteger.formatIntOrLong. + */ + public void testPrepareFormatterLong() { + BigInteger v = BigInteger.valueOf(123); + IntegerFormatter f; + f = PyInteger.prepareFormatter(new PyString("d")); + assertEquals("123", f.format(v).pad().getResult()); + f = PyInteger.prepareFormatter(new PyString("o")); + assertEquals("173", f.format(v).pad().getResult()); + f = PyInteger.prepareFormatter(new PyString("x")); + assertEquals("7b", f.format(v).pad().getResult()); + f = PyInteger.prepareFormatter(new PyString("X")); + assertEquals("7B", f.format(v).pad().getResult()); + f = PyInteger.prepareFormatter(new PyString("b")); + assertEquals("1111011", f.format(v).pad().getResult()); + + BigInteger v2 = BigInteger.valueOf(1234567890); + f = PyInteger.prepareFormatter(new PyString(",d")); + assertEquals("1,234,567,890", f.format(v2).pad().getResult()); + + f = PyInteger.prepareFormatter(new PyString("#o")); + assertEquals("0o173", f.format(v).pad().getResult()); + f = PyInteger.prepareFormatter(new PyString("#X")); + assertEquals("0X7B", f.format(v).pad().getResult()); + + f = PyInteger.prepareFormatter(new PyString("c")); + assertEquals("{", f.format(v).pad().getResult()); + + f = PyInteger.prepareFormatter(new PyString("+d")); + assertEquals("+123", f.format(v).pad().getResult()); + f = PyInteger.prepareFormatter(new PyString(" d")); + assertEquals(" 123", f.format(v).pad().getResult()); + + f = PyInteger.prepareFormatter(new PyString("5")); + assertEquals(" 123", f.format(v).pad().getResult()); + + f = PyInteger.prepareFormatter(new PyString("^6")); + assertEquals(" 123 ", f.format(v).pad().getResult()); + + f = PyInteger.prepareFormatter(new PyString("~<5")); + assertEquals("123~~", f.format(v).pad().getResult()); + + f = PyInteger.prepareFormatter(new PyString("0=+6")); + assertEquals("+00123", f.format(v).pad().getResult()); + + f = PyInteger.prepareFormatter(new PyString("c")); + assertOverflowError(BigInteger.valueOf(256), f, "%c arg not in range(0x100)"); + assertOverflowError(BigInteger.valueOf(-1), f, "%c arg not in range(0x100)"); + assertOverflowError(BigInteger.valueOf(0x110000), f, "%c arg not in range(0x100)"); + + f = PyInteger.prepareFormatter(new PyUnicode("c")); + assertOverflowError(BigInteger.valueOf(0x110000), f, "%c arg not in range(0x110000)"); + assertOverflowError(BigInteger.valueOf(-1), f, "%c arg not in range(0x110000)"); + } + + private void assertValueError(String formatSpec, String expected) { + try { + IntegerFormatter f = PyInteger.prepareFormatter(new PyString(formatSpec)); + // f.format(123).pad().getResult(); + fail("ValueError not thrown, expected: " + expected); + } catch (PyException pye) { + assertEquals(expected, pye.value.toString()); + } + } + + private void assertOverflowError(int v, IntegerFormatter f, String expected) { + // Test with Java int for PyInteger + try { + f.format(v).pad().getResult(); + fail("OverflowError not thrown, expected: " + expected); + } catch (PyException pye) { + assertEquals(expected, pye.value.toString()); + } + } + + private void assertOverflowError(BigInteger v, IntegerFormatter f, String expected) { + // Test with BigInteger for PyLong + try { + f.format(v).pad().getResult(); + fail("OverflowError not thrown, expected: " + expected); + } catch (PyException pye) { + assertEquals(expected, pye.value.toString()); + } } public void testFormatString() { @@ -136,16 +228,6 @@ assertEquals("abc ", PyString.formatString("abc", spec)); } - private void assertFormatError(int value, InternalFormatSpec spec, String expected) { - String error = null; - try { - PyInteger.formatIntOrLong(value, spec); - } catch (IllegalArgumentException e) { - error = e.getMessage(); - } - assertEquals(expected, error); - } - public void testMarkupIterator() { MarkupIterator iterator = new MarkupIterator("abc"); assertEquals("abc", iterator.nextChunk().literalText); -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sun Jun 8 14:13:02 2014 From: jython-checkins at python.org (jeff.allen) Date: Sun, 8 Jun 2014 14:13:02 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_some_skips_from_test?= =?utf-8?q?=5Ftypes_and_test=5Fbuiltin=2E?= Message-ID: <3gmc1V446Tz7LjQ@mail.python.org> http://hg.python.org/jython/rev/29d1ba2a675d changeset: 7282:29d1ba2a675d user: Jeff Allen date: Tue May 27 16:09:47 2014 +0100 summary: Remove some skips from test_types and test_builtin. At least some of these are made possible by recent work on formatting. files: Lib/test/test_builtin.py | 13 +-- Lib/test/test_types.py | 101 ++++++++++++-------------- 2 files changed, 49 insertions(+), 65 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -361,8 +361,7 @@ self.assertEqual(eval('a', g, m), 12) self.assertRaises(NameError, eval, 'b', g, m) self.assertEqual(eval('dir()', g, m), list('xyz')) - if not is_jython: #FIXME #1861 - self.assertEqual(eval('globals()', g, m), g) + self.assertEqual(eval('globals()', g, m), g) self.assertEqual(eval('locals()', g, m), m) # Jython allows arbitrary mappings for globals @@ -386,8 +385,7 @@ self.assertEqual(eval('a', g, d), 12) self.assertRaises(NameError, eval, 'b', g, d) self.assertEqual(eval('dir()', g, d), list('xyz')) - if not is_jython: #FIXME #1861 - self.assertEqual(eval('globals()', g, d), g) + self.assertEqual(eval('globals()', g, d), g) self.assertEqual(eval('locals()', g, d), d) # Verify locals stores (used by list comps) @@ -1320,7 +1318,6 @@ self.assertRaises(TypeError, round, t) self.assertRaises(TypeError, round, t, 0) - @unittest.skipIf(is_jython, "FIXME #1861: not working in Jython") def test_round_large(self): # Issue #1869: integral floats should remain unchanged self.assertEqual(round(5e15-1), 5e15-1) @@ -1387,7 +1384,6 @@ b = 2 return vars() - @unittest.skipIf(is_jython, "FIXME #1861: not working in Jython") def test_vars(self): self.assertEqual(set(vars()), set(dir())) import sys @@ -1491,9 +1487,8 @@ self.assertEqual(format(DerivedFromSimple2(10), 'abcdef'), '10abcdef') - if not is_jython: #FIXME #1861 check again when __format__ works better. - class_test(*classes_new()) - class_test(*classes_classic()) + class_test(*classes_new()) + class_test(*classes_classic()) def empty_format_spec(value): # test that: diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1,7 +1,9 @@ # Python test set -- part 6, built-in types from test.test_support import run_unittest, have_unicode, run_with_locale, \ - check_py3k_warnings, is_jython + check_py3k_warnings +from test.test_support import is_jython + import unittest import sys import locale @@ -90,7 +92,6 @@ if float(1) == 1.0 and float(-1) == -1.0 and float(0) == 0.0: pass else: self.fail('float() does not work properly') - @unittest.skipIf(is_jython, "FIXME: not working") def test_float_to_string(self): def test(f, result): self.assertEqual(f.__format__('e'), result) @@ -407,7 +408,7 @@ test(-123456, "#012X", '-0X00001E240') # issue 5782, commas with no specifier type - #FIXME: not working. + #FIXME: not working. Spurious: ValueError: Cannot specify ',' with '?' from parser. #test(1234, '010,', '00,001,234') # make sure these are errors @@ -424,13 +425,12 @@ self.assertRaises(ValueError, 3 .__format__, ",c") # ensure that only int and float type specifiers work - #FIXME: not working. - #for format_spec in ([chr(x) for x in range(ord('a'), ord('z')+1)] + - # [chr(x) for x in range(ord('A'), ord('Z')+1)]): - # if not format_spec in 'bcdoxXeEfFgGn%': - # self.assertRaises(ValueError, 0 .__format__, format_spec) - # self.assertRaises(ValueError, 1 .__format__, format_spec) - # self.assertRaises(ValueError, (-1) .__format__, format_spec) + for format_spec in ([chr(x) for x in range(ord('a'), ord('z')+1)] + + [chr(x) for x in range(ord('A'), ord('Z')+1)]): + if not format_spec in 'bcdoxXeEfFgGn%': + self.assertRaises(ValueError, 0 .__format__, format_spec) + self.assertRaises(ValueError, 1 .__format__, format_spec) + self.assertRaises(ValueError, (-1) .__format__, format_spec) # ensure that float type specifiers work; format converts # the int to a float @@ -534,14 +534,13 @@ self.assertRaises(ValueError, 1L .__format__, "#+5x") self.assertRaises(ValueError, 1L .__format__, "+5#x") - #FIXME: this section broken in Jython. # ensure that only int and float type specifiers work - #for format_spec in ([chr(x) for x in range(ord('a'), ord('z')+1)] + - # [chr(x) for x in range(ord('A'), ord('Z')+1)]): - # if not format_spec in 'bcdoxXeEfFgGn%': - # self.assertRaises(ValueError, 0L .__format__, format_spec) - # self.assertRaises(ValueError, 1L .__format__, format_spec) - # self.assertRaises(ValueError, (-1L) .__format__, format_spec) + for format_spec in ([chr(x) for x in range(ord('a'), ord('z')+1)] + + [chr(x) for x in range(ord('A'), ord('Z')+1)]): + if not format_spec in 'bcdoxXeEfFgGn%': + self.assertRaises(ValueError, 0L .__format__, format_spec) + self.assertRaises(ValueError, 1L .__format__, format_spec) + self.assertRaises(ValueError, (-1L) .__format__, format_spec) # ensure that float type specifiers work; format converts # the long to a float @@ -562,7 +561,6 @@ test(123456L, "1=20", '11111111111111123456') test(123456L, "*=20", '**************123456') - @unittest.skipIf(is_jython, "FIXME: not working") @run_with_locale('LC_NUMERIC', 'en_US.UTF8') def test_float__format__locale(self): # test locale support for __format__ code 'n' @@ -576,13 +574,12 @@ def test_int__format__locale(self): # test locale support for __format__ code 'n' for integers - #FIXME: not working in Jython. - #x = 123456789012345678901234567890 - #for i in range(0, 30): - # self.assertEqual(locale.format('%d', x, grouping=True), format(x, 'n')) + x = 123456789012345678901234567890 + for i in range(0, 30): + self.assertEqual(locale.format('%d', x, grouping=True), format(x, 'n')) - # # move to the next integer to test - # x = x // 10 + # move to the next integer to test + x = x // 10 rfmt = ">20n" lfmt = "<20n" @@ -661,29 +658,25 @@ # a totaly empty format specifier means something else. # So, just use a sign flag test(1e200, '+g', '+1e+200') - #FIXME: not working. - #test(1e200, '+', '+1e+200') + test(1e200, '+', '+1e+200') test(1.1e200, '+g', '+1.1e+200') - #FIXME: not working. - ##test(1.1e200, '+', '+1.1e+200') + test(1.1e200, '+', '+1.1e+200') test(1.1e200, '+g', '+1.1e+200') - #FIXME: not working. - #test(1.1e200, '+', '+1.1e+200') + test(1.1e200, '+', '+1.1e+200') # 0 padding test(1234., '010f', '1234.000000') test(1234., '011f', '1234.000000') test(1234., '012f', '01234.000000') test(-1234., '011f', '-1234.000000') - #FIXME: not working. - #test(-1234., '012f', '-1234.000000') - #test(-1234., '013f', '-01234.000000') - #test(-1234.12341234, '013f', '-01234.123412') - #test(-123456.12341234, '011.2f', '-0123456.12') + test(-1234., '012f', '-1234.000000') + test(-1234., '013f', '-01234.000000') + test(-1234.12341234, '013f', '-01234.123412') + test(-123456.12341234, '011.2f', '-0123456.12') # issue 5782, commas with no specifier type - #FIXME: not working. + #FIXME: not working. Spurious: ValueError: Cannot specify ',' with '?' from parser. #test(1.2, '010,.2', '0,000,001.2') # 0 padding with commas @@ -692,13 +685,12 @@ test(1234., '013,f', '01,234.000000') test(-1234., '012,f', '-1,234.000000') test(-1234., '013,f', '-1,234.000000') - #FIXME: not working. - #test(-1234., '014,f', '-01,234.000000') - #test(-12345., '015,f', '-012,345.000000') - #test(-123456., '016,f', '-0,123,456.000000') - #test(-123456., '017,f', '-0,123,456.000000') - #test(-123456.12341234, '017,f', '-0,123,456.123412') - #test(-123456.12341234, '013,.2f', '-0,123,456.12') + test(-1234., '014,f', '-01,234.000000') + test(-12345., '015,f', '-012,345.000000') + test(-123456., '016,f', '-0,123,456.000000') + test(-123456., '017,f', '-0,123,456.000000') + test(-123456.12341234, '017,f', '-0,123,456.123412') + test(-123456.12341234, '013,.2f', '-0,123,456.12') # % formatting test(-1.0, '%', '-100.000000%') @@ -721,23 +713,20 @@ self.assertRaises(ValueError, format, -1e-100, format_spec) # Alternate formatting is not supported - #FIXME: not working. - ##self.assertRaises(ValueError, format, 0.0, '#') + self.assertRaises(ValueError, format, 0.0, '#') self.assertRaises(ValueError, format, 0.0, '#20f') # Issue 6902 - #FIXME: not working. - #test(12345.6, "0<20", '12345.60000000000000') - #test(12345.6, "1<20", '12345.61111111111111') - #test(12345.6, "*<20", '12345.6*************') - #test(12345.6, "0>20", '000000000000012345.6') - #test(12345.6, "1>20", '111111111111112345.6') - #test(12345.6, "*>20", '*************12345.6') - #test(12345.6, "0=20", '000000000000012345.6') - #test(12345.6, "1=20", '111111111111112345.6') - #test(12345.6, "*=20", '*************12345.6') + test(12345.6, "0<20", '12345.60000000000000') + test(12345.6, "1<20", '12345.61111111111111') + test(12345.6, "*<20", '12345.6*************') + test(12345.6, "0>20", '000000000000012345.6') + test(12345.6, "1>20", '111111111111112345.6') + test(12345.6, "*>20", '*************12345.6') + test(12345.6, "0=20", '000000000000012345.6') + test(12345.6, "1=20", '111111111111112345.6') + test(12345.6, "*=20", '*************12345.6') - @unittest.skipIf(is_jython, "FIXME: not working") def test_format_spec_errors(self): # int, float, and string all share the same format spec # mini-language parser. -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sun Jun 8 14:13:04 2014 From: jython-checkins at python.org (jeff.allen) Date: Sun, 8 Jun 2014 14:13:04 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Allow_int_and_long_to_invok?= =?utf-8?q?e_floating_point_formatting=2E?= Message-ID: <3gmc1X0VLYz7Ljn@mail.python.org> http://hg.python.org/jython/rev/f0c71ac97ec8 changeset: 7283:f0c71ac97ec8 user: Jeff Allen date: Tue May 27 23:51:23 2014 +0100 summary: Allow int and long to invoke floating point formatting. Also removes further skips from test_types. files: Lib/test/test_types.py | 25 +- src/org/python/core/PyFloat.java | 79 +++++++-- src/org/python/core/PyInteger.java | 77 ++++++--- src/org/python/core/PyLong.java | 37 ++++- src/org/python/core/stringlib/InternalFormat.java | 5 - tests/java/org/python/core/StringFormatTest.java | 74 ++++---- 6 files changed, 189 insertions(+), 108 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -408,8 +408,7 @@ test(-123456, "#012X", '-0X00001E240') # issue 5782, commas with no specifier type - #FIXME: not working. Spurious: ValueError: Cannot specify ',' with '?' from parser. - #test(1234, '010,', '00,001,234') + test(1234, '010,', '00,001,234') # make sure these are errors @@ -434,11 +433,10 @@ # ensure that float type specifiers work; format converts # the int to a float - #FIXME: not working. - #for format_spec in 'eEfFgG%': - # for value in [0, 1, -1, 100, -100, 1234567890, -1234567890]: - # self.assertEqual(value.__format__(format_spec), - # float(value).__format__(format_spec)) + for format_spec in 'eEfFgG%': + for value in [0, 1, -1, 100, -100, 1234567890, -1234567890]: + self.assertEqual(value.__format__(format_spec), + float(value).__format__(format_spec)) # Issue 6902 test(123456, "0<20", '12345600000000000000') @@ -544,12 +542,10 @@ # ensure that float type specifiers work; format converts # the long to a float - - #FIXME: this section broken in Jython. - #for format_spec in 'eEfFgG%': - # for value in [0L, 1L, -1L, 100L, -100L, 1234567890L, -1234567890L]: - # self.assertEqual(value.__format__(format_spec), - # float(value).__format__(format_spec)) + for format_spec in 'eEfFgG%': + for value in [0L, 1L, -1L, 100L, -100L, 1234567890L, -1234567890L]: + self.assertEqual(value.__format__(format_spec), + float(value).__format__(format_spec)) # Issue 6902 test(123456L, "0<20", '12345600000000000000') test(123456L, "1<20", '12345611111111111111') @@ -676,8 +672,7 @@ test(-123456.12341234, '011.2f', '-0123456.12') # issue 5782, commas with no specifier type - #FIXME: not working. Spurious: ValueError: Cannot specify ',' with '?' from parser. - #test(1.2, '010,.2', '0,000,001.2') + test(1.2, '010,.2', '0,000,001.2') # 0 padding with commas test(1234., '011,f', '1,234.000000') diff --git a/src/org/python/core/PyFloat.java b/src/org/python/core/PyFloat.java --- a/src/org/python/core/PyFloat.java +++ b/src/org/python/core/PyFloat.java @@ -4,9 +4,12 @@ import java.io.Serializable; import java.math.BigDecimal; +import java.math.BigInteger; import org.python.core.stringlib.FloatFormatter; +import org.python.core.stringlib.IntegerFormatter; import org.python.core.stringlib.InternalFormat; +import org.python.core.stringlib.InternalFormat.Formatter; import org.python.core.stringlib.InternalFormat.Spec; import org.python.expose.ExposedClassMethod; import org.python.expose.ExposedGet; @@ -911,32 +914,66 @@ @ExposedMethod(doc = BuiltinDocs.float___format___doc) final PyObject float___format__(PyObject formatSpec) { - if (!(formatSpec instanceof PyString)) { - throw Py.TypeError("__format__ requires str or unicode"); + + // Parse the specification + Spec spec = InternalFormat.fromText(formatSpec, "__format__"); + + // Get a formatter for the specification + FloatFormatter f = prepareFormatter(spec); + + if (f != null) { + // Bytes mode if formatSpec argument is not unicode. + f.setBytes(!(formatSpec instanceof PyUnicode)); + // Convert as per specification. + f.format(value); + // Return a result that has the same type (str or unicode) as the formatSpec argument. + return f.pad().getPyResult(); + + } else { + // The type code was not recognised in prepareFormatter + throw Formatter.unknownFormat(spec.type, "float"); } + } - PyString formatSpecStr = (PyString)formatSpec; - String result; - try { - String specString = formatSpecStr.getString(); - Spec spec = InternalFormat.fromText(specString); - if (spec.type!=Spec.NONE && "efgEFGn%".indexOf(spec.type) < 0) { - throw FloatFormatter.unknownFormat(spec.type, "float"); - } else if (spec.alternate) { - throw FloatFormatter.alternateFormNotAllowed("float"); - } else { + /** + * Common code for PyFloat, {@link PyInteger} and {@link PyLong} to prepare a {@link FloatFormatter} from a parsed specification. + * The object returned has format method {@link FloatFormatter#format(double)}. + * + * @param spec a parsed PEP-3101 format specification. + * @return a formatter ready to use, or null if the type is not a floating point format type. + * @throws PyException(ValueError) if the specification is faulty. + */ + @SuppressWarnings("fallthrough") + static FloatFormatter prepareFormatter(Spec spec) { + + // Slight differences between format types + switch (spec.type) { + + case 'n': + if (spec.grouping) { + throw IntegerFormatter.notAllowed("Grouping", "float", spec.type); + } + // Fall through + + case Spec.NONE: + case 'e': + case 'f': + case 'g': + case 'E': + case 'F': + case 'G': + case '%': + // Check for disallowed parts of the specification + if (spec.alternate) { + throw FloatFormatter.alternateFormNotAllowed("float"); + } // spec may be incomplete. The defaults are those commonly used for numeric formats. spec = spec.withDefaults(Spec.NUMERIC); - // Get a formatter for the spec. - FloatFormatter f = new FloatFormatter(spec); - // Convert as per specification. - f.format(value).pad(); - result = f.getResult(); - } - } catch (IllegalArgumentException e) { - throw Py.ValueError(e.getMessage()); // XXX Can this be reached? + return new FloatFormatter(spec, 1); + + default: + return null; } - return formatSpecStr.createInstance(result); } @ExposedMethod(doc = BuiltinDocs.float_as_integer_ratio_doc) diff --git a/src/org/python/core/PyInteger.java b/src/org/python/core/PyInteger.java --- a/src/org/python/core/PyInteger.java +++ b/src/org/python/core/PyInteger.java @@ -5,8 +5,10 @@ import java.io.Serializable; import java.math.BigInteger; +import org.python.core.stringlib.FloatFormatter; import org.python.core.stringlib.IntegerFormatter; import org.python.core.stringlib.InternalFormat; +import org.python.core.stringlib.InternalFormat.Formatter; import org.python.core.stringlib.InternalFormat.Spec; import org.python.expose.ExposedGet; import org.python.expose.ExposedMethod; @@ -1019,13 +1021,38 @@ return int___format__(formatSpec); } - @SuppressWarnings("fallthrough") @ExposedMethod(doc = BuiltinDocs.int___format___doc) final PyObject int___format__(PyObject formatSpec) { - // Get a formatter for the specification - IntegerFormatter f = prepareFormatter(formatSpec); - // Convert as per specification. - f.format(value); + + // Parse the specification + Spec spec = InternalFormat.fromText(formatSpec, "__format__"); + InternalFormat.Formatter f; + + // Try to make an integer formatter from the specification + IntegerFormatter fi = PyInteger.prepareFormatter(spec); + if (fi != null) { + // Bytes mode if formatSpec argument is not unicode. + fi.setBytes(!(formatSpec instanceof PyUnicode)); + // Convert as per specification. + fi.format(value); + f = fi; + + } else { + // Try to make a float formatter from the specification + FloatFormatter ff = PyFloat.prepareFormatter(spec); + if (ff != null) { + // Bytes mode if formatSpec argument is not unicode. + ff.setBytes(!(formatSpec instanceof PyUnicode)); + // Convert as per specification. + ff.format(value); + f = ff; + + } else { + // The type code was not recognised in either prepareFormatter + throw Formatter.unknownFormat(spec.type, "integer"); + } + } + // Return a result that has the same type (str or unicode) as the formatSpec argument. return f.pad().getPyResult(); } @@ -1035,25 +1062,17 @@ * overloaded format method {@link IntegerFormatter#format(int)} and * {@link IntegerFormatter#format(BigInteger)} to support the two types. * - * @param formatSpec PEP-3101 format specification. - * @return a formatter ready to use. + * @param spec a parsed PEP-3101 format specification. + * @return a formatter ready to use, or null if the type is not an integer format type. * @throws PyException(ValueError) if the specification is faulty. */ - static IntegerFormatter prepareFormatter(PyObject formatSpec) throws PyException { - - // Parse the specification - Spec spec = InternalFormat.fromText(formatSpec, "__format__"); - IntegerFormatter f; - - // Check for disallowed parts of the specification - if (Spec.specified(spec.precision)) { - throw IntegerFormatter.precisionNotAllowed("integer"); - } + @SuppressWarnings("fallthrough") + static IntegerFormatter prepareFormatter(Spec spec) throws PyException { // Slight differences between format types switch (spec.type) { case 'c': - // Character data + // Character data: specific prohibitions. if (Spec.specified(spec.sign)) { throw IntegerFormatter.notAllowed("Sign", "integer", spec.type); } else if (spec.alternate) { @@ -1061,26 +1080,30 @@ } // Fall through - case Spec.NONE: - case 'd': case 'x': case 'X': case 'o': case 'b': case 'n': + if (spec.grouping) { + throw IntegerFormatter.notAllowed("Grouping", "integer", spec.type); + } + // Fall through + + case Spec.NONE: + case 'd': + // Check for disallowed parts of the specification + if (Spec.specified(spec.precision)) { + throw IntegerFormatter.precisionNotAllowed("integer"); + } // spec may be incomplete. The defaults are those commonly used for numeric formats. spec = spec.withDefaults(Spec.NUMERIC); // Get a formatter for the spec. - f = new IntegerFormatter(spec, 1); - // Bytes mode if formatSpec argument is not unicode. - f.setBytes(!(formatSpec instanceof PyUnicode)); - break; + return new IntegerFormatter(spec, 1); default: - throw IntegerFormatter.unknownFormat(spec.type, "integer"); + return null; } - - return f; } /** diff --git a/src/org/python/core/PyLong.java b/src/org/python/core/PyLong.java --- a/src/org/python/core/PyLong.java +++ b/src/org/python/core/PyLong.java @@ -7,7 +7,10 @@ import java.math.BigDecimal; import java.math.BigInteger; +import org.python.core.stringlib.FloatFormatter; import org.python.core.stringlib.IntegerFormatter; +import org.python.core.stringlib.InternalFormat; +import org.python.core.stringlib.InternalFormat.Formatter; import org.python.core.stringlib.InternalFormat.Spec; import org.python.expose.ExposedGet; import org.python.expose.ExposedMethod; @@ -1062,10 +1065,36 @@ @ExposedMethod(doc = BuiltinDocs.long___format___doc) final PyObject long___format__(PyObject formatSpec) { - // Get a formatter for the specification - IntegerFormatter f = PyInteger.prepareFormatter(formatSpec); - // Convert as per specification (note this supports BigDecimal). - f.format(value); + + // Parse the specification + Spec spec = InternalFormat.fromText(formatSpec, "__format__"); + InternalFormat.Formatter f; + + // Try to make an integer formatter from the specification + IntegerFormatter fi = PyInteger.prepareFormatter(spec); + if (fi != null) { + // Bytes mode if formatSpec argument is not unicode. + fi.setBytes(!(formatSpec instanceof PyUnicode)); + // Convert as per specification. + fi.format(value); + f = fi; + + } else { + // Try to make a float formatter from the specification + FloatFormatter ff = PyFloat.prepareFormatter(spec); + if (ff != null) { + // Bytes mode if formatSpec argument is not unicode. + ff.setBytes(!(formatSpec instanceof PyUnicode)); + // Convert as per specification. + ff.format(value.doubleValue()); + f = ff; + + } else { + // The type code was not recognised in either prepareFormatter + throw Formatter.unknownFormat(spec.type, "integer"); + } + } + // Return a result that has the same type (str or unicode) as the formatSpec argument. return f.pad().getPyResult(); } diff --git a/src/org/python/core/stringlib/InternalFormat.java b/src/org/python/core/stringlib/InternalFormat.java --- a/src/org/python/core/stringlib/InternalFormat.java +++ b/src/org/python/core/stringlib/InternalFormat.java @@ -892,11 +892,6 @@ throw new IllegalArgumentException("Invalid conversion specification"); } - // Restrict grouping to known formats. (Mirrors CPython, but misplaced?) - if (grouping && "defgEG%F\0".indexOf(type) == -1) { - throw new IllegalArgumentException("Cannot specify ',' with '" + type + "'."); - } - // Create a specification return new Spec(fill, align, sign, alternate, width, grouping, precision, type); } diff --git a/tests/java/org/python/core/StringFormatTest.java b/tests/java/org/python/core/StringFormatTest.java --- a/tests/java/org/python/core/StringFormatTest.java +++ b/tests/java/org/python/core/StringFormatTest.java @@ -6,7 +6,7 @@ import org.python.core.stringlib.FieldNameIterator; import org.python.core.stringlib.IntegerFormatter; -import org.python.core.stringlib.InternalFormat.Formatter; +import org.python.core.stringlib.InternalFormat; import org.python.core.stringlib.InternalFormatSpec; import org.python.core.stringlib.InternalFormatSpecParser; import org.python.core.stringlib.MarkupIterator; @@ -76,55 +76,56 @@ public void testPrepareFormatter() { int v = 123; IntegerFormatter f; - f = PyInteger.prepareFormatter(new PyString("d")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("d")); assertEquals("123", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("o")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("o")); assertEquals("173", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("x")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("x")); assertEquals("7b", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("X")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("X")); assertEquals("7B", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("b")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("b")); assertEquals("1111011", f.format(v).pad().getResult()); int v2 = 1234567890; - f = PyInteger.prepareFormatter(new PyString(",d")); + f = PyInteger.prepareFormatter(InternalFormat.fromText(",d")); assertEquals("1,234,567,890", f.format(v2).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("#o")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("#o")); assertEquals("0o173", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("#X")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("#X")); assertEquals("0X7B", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("c")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("c")); assertEquals("{", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("+d")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("+d")); assertEquals("+123", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString(" d")); + f = PyInteger.prepareFormatter(InternalFormat.fromText(" d")); assertEquals(" 123", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("5")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("5")); assertEquals(" 123", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("^6")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("^6")); assertEquals(" 123 ", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("~<5")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("~<5")); assertEquals("123~~", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("0=+6")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("0=+6")); assertEquals("+00123", f.format(v).pad().getResult()); assertValueError("0=+6.1", "Precision not allowed in integer format specifier"); assertValueError("+c", "Sign not allowed with integer format specifier 'c'"); - f = PyInteger.prepareFormatter(new PyString("c")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("c")); + f.setBytes(true); assertOverflowError(256, f, "%c arg not in range(0x100)"); assertOverflowError(-1, f, "%c arg not in range(0x100)"); assertOverflowError(0x110000, f, "%c arg not in range(0x100)"); - f = PyInteger.prepareFormatter(new PyUnicode("c")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("c")); assertOverflowError(0x110000, f, "%c arg not in range(0x110000)"); assertOverflowError(-1, f, "%c arg not in range(0x110000)"); } @@ -136,59 +137,60 @@ public void testPrepareFormatterLong() { BigInteger v = BigInteger.valueOf(123); IntegerFormatter f; - f = PyInteger.prepareFormatter(new PyString("d")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("d")); assertEquals("123", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("o")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("o")); assertEquals("173", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("x")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("x")); assertEquals("7b", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("X")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("X")); assertEquals("7B", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("b")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("b")); assertEquals("1111011", f.format(v).pad().getResult()); BigInteger v2 = BigInteger.valueOf(1234567890); - f = PyInteger.prepareFormatter(new PyString(",d")); + f = PyInteger.prepareFormatter(InternalFormat.fromText(",d")); assertEquals("1,234,567,890", f.format(v2).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("#o")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("#o")); assertEquals("0o173", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("#X")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("#X")); assertEquals("0X7B", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("c")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("c")); assertEquals("{", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("+d")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("+d")); assertEquals("+123", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString(" d")); + f = PyInteger.prepareFormatter(InternalFormat.fromText(" d")); assertEquals(" 123", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("5")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("5")); assertEquals(" 123", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("^6")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("^6")); assertEquals(" 123 ", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("~<5")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("~<5")); assertEquals("123~~", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("0=+6")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("0=+6")); assertEquals("+00123", f.format(v).pad().getResult()); - f = PyInteger.prepareFormatter(new PyString("c")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("c")); + f.setBytes(true); assertOverflowError(BigInteger.valueOf(256), f, "%c arg not in range(0x100)"); assertOverflowError(BigInteger.valueOf(-1), f, "%c arg not in range(0x100)"); assertOverflowError(BigInteger.valueOf(0x110000), f, "%c arg not in range(0x100)"); - f = PyInteger.prepareFormatter(new PyUnicode("c")); + f = PyInteger.prepareFormatter(InternalFormat.fromText("c")); assertOverflowError(BigInteger.valueOf(0x110000), f, "%c arg not in range(0x110000)"); assertOverflowError(BigInteger.valueOf(-1), f, "%c arg not in range(0x110000)"); } private void assertValueError(String formatSpec, String expected) { try { - IntegerFormatter f = PyInteger.prepareFormatter(new PyString(formatSpec)); + IntegerFormatter f = PyInteger.prepareFormatter(InternalFormat.fromText(formatSpec)); // f.format(123).pad().getResult(); fail("ValueError not thrown, expected: " + expected); } catch (PyException pye) { -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sun Jun 8 14:13:05 2014 From: jython-checkins at python.org (jeff.allen) Date: Sun, 8 Jun 2014 14:13:05 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Improvements_to_PyString=2E?= =?utf-8?q?=5F=5Fformat=5F=5F_and_in_StringFormatter_related_to_text=2E?= Message-ID: <3gmc1Y5ZZ9z7Ljk@mail.python.org> http://hg.python.org/jython/rev/324e1138e1f3 changeset: 7284:324e1138e1f3 user: Jeff Allen date: Tue Jun 03 21:30:16 2014 +0100 summary: Improvements to PyString.__format__ and in StringFormatter related to text. Brings %s, %r, %c, %% into the new framework. Quite some rationalisation of StringFormatter made possible. Small improvement in test conformity. files: Lib/test/test_format_jy.py | 57 +- Lib/test/test_unicode.py | 7 +- src/org/python/core/PyInteger.java | 2 +- src/org/python/core/PyString.java | 482 ++++----- src/org/python/core/stringlib/IntegerFormatter.java | 1 + src/org/python/core/stringlib/InternalFormat.java | 132 +- src/org/python/core/stringlib/TextFormatter.java | 105 ++ tests/java/org/python/core/StringFormatTest.java | 17 +- 8 files changed, 480 insertions(+), 323 deletions(-) diff --git a/Lib/test/test_format_jy.py b/Lib/test/test_format_jy.py --- a/Lib/test/test_format_jy.py +++ b/Lib/test/test_format_jy.py @@ -5,8 +5,9 @@ from test import test_support import unittest -class FormatTestCase(unittest.TestCase): - # Tests that %d converts values for custom classes implementing __int__ +class FormatSubclass(unittest.TestCase): + # Custom __int__ and __float__ should be respected by %-formatting + def test_int_conversion_support(self): class Foo(object): def __init__(self, x): self.x = x @@ -21,9 +22,59 @@ def __float__(self): return self. x self.assertEqual('1.0', '%.1f' % Foo(1.0)) +class FormatUnicodeBase(unittest.TestCase): + + # Test padding non-BMP result + def test_pad_string(self): + self.padcheck(u"architect") + self.padcheck(u'a\U00010001cde') + +class FormatUnicodeClassic(FormatUnicodeBase): + # Check using %-formatting + + def padcheck(self, s): + self.assertEqual(10, len('%10.4s' % s)) + self.assertEqual(u' '*6 + s[0:4], '%10.4s' % s) + self.assertEqual(u' '*6 + s[0:4], '% 10.4s' % s) + self.assertEqual(u' '*6 + s[0:4], '%010.4s' % s) + self.assertEqual(s[0:3] + u' '*5, '%-8.3s' % s) + +class FormatUnicodeModern(FormatUnicodeBase): + # Check using __format__ + + def padcheck(self, s): + self.assertEqual(10, len(format(s, '10.4s'))) + self.assertEqual(s[0:3] + u' '*7, format(s, '10.3s')) + self.assertEqual(s[0:3] + u'~'*7, format(s, '~<10.3s')) + self.assertEqual(s[0:3] + u'~'*7, format(s, '~<10.3')) + self.assertEqual(u' '*6 + s[0:4], format(s, '>10.4s')) + self.assertEqual(u'*'*6 + s[0:4], format(s, '*>10.4s')) + self.assertEqual(u'*'*6 + s[0:4], format(s, '*>10.4')) + + +class FormatMisc(unittest.TestCase): + # Odd tests Jython used to fail + + def test_percent_padded(self) : + self.assertEqual('%hello', '%%%s' % 'hello') + self.assertEqual(u' %hello', '%6%%s' % u'hello') + self.assertEqual(u'% hello', u'%-6%%s' % 'hello') + + self.assertEqual(' %', '%6%' % ()) + self.assertEqual(' %', '%06%' % ()) + self.assertEqual(' %', '%*%' % 4) + self.assertEqual('% ', '%-6%' % ()) + self.assertEqual('% ', '%-06%' % ()) + self.assertEqual('% ', '%*%' % -4) + def test_main(): - test_support.run_unittest(FormatTestCase) + test_support.run_unittest( + FormatSubclass, + FormatUnicodeClassic, + FormatUnicodeModern, + FormatMisc, + ) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -357,13 +357,12 @@ self.assertEqual(u"%s, %s, %i, %f, %5.2f" % (u"abc", "abc", -1, -2, 3.5), u'abc, abc, -1, -2.000000, 3.50') self.assertEqual(u"%s, %s, %i, %f, %5.2f" % (u"abc", "abc", -1, -2, 3.57), u'abc, abc, -1, -2.000000, 3.57') self.assertEqual(u"%s, %s, %i, %f, %5.2f" % (u"abc", "abc", -1, -2, 1003.57), u'abc, abc, -1, -2.000000, 1003.57') - if not sys.platform.startswith('java'): - self.assertEqual(u"%r, %r" % (u"abc", "abc"), u"u'abc', 'abc'") + self.assertEqual(u"%r, %r" % (u"abc", "abc"), u"u'abc', 'abc'") self.assertEqual(u"%(x)s, %(y)s" % {'x':u"abc", 'y':"def"}, u'abc, def') self.assertEqual(u"%(x)s, %(\xfc)s" % {'x':u"abc", u'\xfc':"def"}, u'abc, def') - # self.assertEqual(u'%c' % 0x1234, u'\u1234') - # self.assertRaises(OverflowError, u"%c".__mod__, (sys.maxunicode+1,)) + self.assertEqual(u'%c' % 0x1234, u'\u1234') + self.assertRaises(OverflowError, u"%c".__mod__, (sys.maxunicode+1,)) # formatting jobs delegated from the string implementation: self.assertEqual('...%(foo)s...' % {'foo':u"abc"}, u'...abc...') diff --git a/src/org/python/core/PyInteger.java b/src/org/python/core/PyInteger.java --- a/src/org/python/core/PyInteger.java +++ b/src/org/python/core/PyInteger.java @@ -1074,7 +1074,7 @@ case 'c': // Character data: specific prohibitions. if (Spec.specified(spec.sign)) { - throw IntegerFormatter.notAllowed("Sign", "integer", spec.type); + throw IntegerFormatter.signNotAllowed("integer", spec.type); } else if (spec.alternate) { throw IntegerFormatter.alternateFormNotAllowed("integer", spec.type); } diff --git a/src/org/python/core/PyString.java b/src/org/python/core/PyString.java --- a/src/org/python/core/PyString.java +++ b/src/org/python/core/PyString.java @@ -10,10 +10,11 @@ import org.python.core.stringlib.FieldNameIterator; import org.python.core.stringlib.FloatFormatter; import org.python.core.stringlib.IntegerFormatter; +import org.python.core.stringlib.InternalFormat; +import org.python.core.stringlib.InternalFormat.Formatter; import org.python.core.stringlib.InternalFormat.Spec; -import org.python.core.stringlib.InternalFormatSpec; -import org.python.core.stringlib.InternalFormatSpecParser; import org.python.core.stringlib.MarkupIterator; +import org.python.core.stringlib.TextFormatter; import org.python.core.util.StringUtil; import org.python.expose.ExposedMethod; import org.python.expose.ExposedNew; @@ -3898,50 +3899,68 @@ @ExposedMethod(doc = BuiltinDocs.str___format___doc) final PyObject str___format__(PyObject formatSpec) { - if (!(formatSpec instanceof PyString)) { - throw Py.TypeError("__format__ requires str or unicode"); + + // Parse the specification + Spec spec = InternalFormat.fromText(formatSpec, "__format__"); + + // Get a formatter for the specification + TextFormatter f = prepareFormatter(spec); + if (f == null) { + // The type code was not recognised + throw Formatter.unknownFormat(spec.type, "string"); } - PyString formatSpecStr = (PyString)formatSpec; - String result; - try { - String specString = formatSpecStr.getString(); - InternalFormatSpec spec = new InternalFormatSpecParser(specString).parse(); - result = formatString(getString(), spec); - } catch (IllegalArgumentException e) { - throw Py.ValueError(e.getMessage()); + // Bytes mode if neither this nor formatSpec argument is Unicode. + boolean unicode = this instanceof PyUnicode || formatSpec instanceof PyUnicode; + f.setBytes(!unicode); + + // Convert as per specification. + f.format(getString()); + + // Return a result that has the same type (str or unicode) as the formatSpec argument. + return f.pad().getPyResult(); + } + + /** + * Common code for {@link PyString} and {@link PyUnicode} to prepare a {@link TextFormatter} + * from a parsed specification. The object returned has format method + * {@link TextFormatter#format(String)} that treats its argument as UTF-16 encoded unicode (not + * just chars). That method will format its argument ( str or + * unicode) according to the PEP 3101 formatting specification supplied here. This + * would be used during text.__format__(".5s") or + * "{:.5s}".format(text) where text is this Python string. + * + * @param spec a parsed PEP-3101 format specification. + * @return a formatter ready to use, or null if the type is not a string format type. + * @throws PyException(ValueError) if the specification is faulty. + */ + @SuppressWarnings("fallthrough") + static TextFormatter prepareFormatter(Spec spec) throws PyException { + // Slight differences between format types + switch (spec.type) { + + case Spec.NONE: + case 's': + // Check for disallowed parts of the specification + if (spec.grouping) { + throw Formatter.notAllowed("Grouping", "string", spec.type); + } else if (Spec.specified(spec.sign)) { + throw Formatter.signNotAllowed("string", '\0'); + } else if (spec.alternate) { + throw Formatter.alternateFormNotAllowed("string"); + } else if (spec.align == '=') { + throw Formatter.alignmentNotAllowed('=', "string"); + } + // spec may be incomplete. The defaults are those commonly used for string formats. + spec = spec.withDefaults(Spec.STRING); + // Get a formatter for the specification + return new TextFormatter(spec); + + default: + // The type code was not recognised + return null; } - return formatSpecStr.createInstance(result); - } - - /** - * Format the given text according to a parsed PEP 3101 formatting specification, as during - * text.__format__(format_spec) or "{:s}".format(text) where - * text is a Python string. - * - * @param text to format - * @param spec the parsed PEP 3101 formatting specification - * @return the result of the formatting - */ - public static String formatString(String text, InternalFormatSpec spec) { - if (spec.sign != '\0') { - throw new IllegalArgumentException("Sign not allowed in string format specifier"); - } - if (spec.alternate) { - throw new IllegalArgumentException( - "Alternate form (#) not allowed in string format specifier"); - } - if (spec.align == '=') { - throw new IllegalArgumentException( - "'=' alignment not allowed in string format specifier"); - } - if (spec.precision >= 0 && text.length() > spec.precision) { - text = text.substring(0, spec.precision); - } - return spec.pad(text, '<', 0); - } - - /* arguments' conversion helper */ + } @Override public String asString(int index) throws PyObject.ConversionException { @@ -4006,10 +4025,6 @@ String format; /** Where the output is built. */ StringBuilder buffer; - /** Remembers that the value currently converted is negative */ - boolean negative; - /** Precision from format specification. */ - int precision; /** * Index into args of argument currently being worked, or special values indicating -1: a single * item that has not yet been used, -2: a single item that has already been used, -3: a mapping. @@ -4018,7 +4033,7 @@ /** Arguments supplied to {@link #format(PyObject)} method. */ PyObject args; /** Indicate a PyUnicode result is expected. */ - boolean unicodeCoercion; + boolean needUnicode; final char pop() { try { @@ -4054,7 +4069,7 @@ public StringFormatter(String format, boolean unicodeCoercion) { index = 0; this.format = format; - this.unicodeCoercion = unicodeCoercion; + this.needUnicode = unicodeCoercion; buffer = new StringBuilder(format.length() + 100); } @@ -4155,9 +4170,9 @@ } /** - * Return the argument as either a {@link PyFloat} according to its __float__ - * method. If the argument has no such method, or it raises an exception, we return the argument - * itself. The caller must check the return type. + * Return the argument as a {@link PyFloat} according to its __float__ method. If + * the argument has no such method, or it raises an exception, we return the argument itself. + * The caller must check the return type. * * @param arg to convert * @return PyFloat if possible @@ -4171,7 +4186,7 @@ } else { // use __float__ to get a float. if (arg.getClass() == PyFloat.class) { - // A common case where it is safe to return arg.__int__() + // A common case where it is safe to return arg.__float__() return arg.__float__(); } else { @@ -4194,6 +4209,46 @@ } /** + * Return the argument as either a {@link PyString} or a {@link PyUnicode}, and set the + * {@link #needUnicode} member accordingly. If we already know we are building a Unicode string + * (needUnicode==true), then any argument that is not already a + * PyUnicode will be converted by calling its __unicode__ method. + * Conversely, if we are not yet building a Unicode string (needUnicode==false ), + * then a PyString will pass unchanged, a PyUnicode will switch us to Unicode mode + * (needUnicode=true), and any other type will be converted by calling its + * __str__ method, which will return a PyString, or possibly a + * PyUnicode, which will switch us to Unicode mode. + * + * @param arg to convert + * @return PyString or PyUnicode equivalent + */ + private PyString asText(PyObject arg) { + + if (arg instanceof PyUnicode) { + // arg is already acceptable. + needUnicode = true; + return (PyUnicode)arg; + + } else if (needUnicode) { + // The string being built is unicode, so we need that version of the arg. + return arg.__unicode__(); + + } else if (arg instanceof PyString) { + // The string being built is not unicode, so arg is already acceptable. + return (PyString)arg; + + } else { + // The string being built is not unicode, so use __str__ to get a PyString. + PyString s = arg.__str__(); + // But __str__ might return PyUnicode, and we have to notice that. + if (s instanceof PyUnicode) { + needUnicode = true; + } + return s; + } + } + + /** * Main service of this class: format one or more arguments with the format string supplied at * construction. * @@ -4204,7 +4259,7 @@ public PyString format(PyObject args) { PyObject dict = null; this.args = args; - boolean needUnicode = unicodeCoercion; + if (args instanceof PyTuple) { // We will simply work through the tuple elements argIndex = 0; @@ -4220,16 +4275,6 @@ while (index < format.length()) { - // Attributes to be parsed from the next format specifier - boolean ljustFlag = false; - boolean signFlag = false; - boolean blankFlag = false; - boolean altFlag = false; - boolean zeroFlag = false; - - int width = -1; - precision = -1; - // Read one character from the format string char c = pop(); if (c != '%') { @@ -4239,6 +4284,14 @@ // It's a %, so the beginning of a conversion specifier. Parse it. + // Attributes to be parsed from the next format specifier + boolean altFlag = false; + char sign = Spec.NONE; + char fill = ' '; + char align = '>'; + int width = Spec.UNSPECIFIED; + int precision = Spec.UNSPECIFIED; + // A conversion specifier contains the following components, in this order: // + The '%' character, which marks the start of the specifier. // + Mapping key (optional), consisting of a parenthesised sequence of characters. @@ -4278,19 +4331,22 @@ while (true) { switch (c = pop()) { case '-': - ljustFlag = true; + align = '<'; continue; case '+': - signFlag = true; + sign = '+'; continue; case ' ': - blankFlag = true; + if (!Spec.specified(sign)) { + // Blank sign only wins if '+' not specified. + sign = ' '; + } continue; case '#': altFlag = true; continue; case '0': - zeroFlag = true; + fill = '0'; continue; } break; @@ -4307,7 +4363,7 @@ width = getNumber(); if (width < 0) { width = -width; - ljustFlag = true; + align = '<'; } /* @@ -4330,103 +4386,105 @@ c = pop(); } - // c is now the conversion type. - if (c == '%') { - // It was just a percent sign after all - buffer.append(c); - continue; + /* + * As a function of the conversion type (currently in c) override some of the formatting + * flags we read from the format specification. + */ + switch (c) { + case 's': + case 'r': + case 'c': + case '%': + // These have string-like results: fill, if needed, is always blank. + fill = ' '; + break; + + default: + if (fill == '0' && align == '>') { + // Zero-fill comes after the sign in right-justification. + align = '='; + } else { + // If left-justifying, the fill is always blank. + fill = ' '; + } } /* + * Encode as an InternalFormat.Spec. The values in the constructor always have specified + * values, except for sign, width and precision. + */ + Spec spec = new Spec(fill, align, sign, altFlag, width, false, precision, c); + + /* * Process argument according to format specification decoded from the string. It is - * important we don't read the argumnent from the list until this point because of the + * important we don't read the argument from the list until this point because of the * possibility that width and precision were specified via the argument list. */ - PyObject arg = getarg(); - String string = null; - negative = false; - - // Independent of type, decide the padding character based on decoded flags. - char fill = ' '; - if (zeroFlag) { - fill = '0'; - } else { - fill = ' '; - } - - // Encode as an InternalFormat.Spec - char fill2 = ' '; - char align = ljustFlag ? '<' : '>'; - if (zeroFlag && !ljustFlag) { - // We only actually fill with zero if right-justifying - fill2 = '0'; - // And then the fill comes after the sign. - align = '='; - } - char sign = signFlag ? '+' : (blankFlag ? ' ' : Spec.NONE); - int w = width; - Spec spec = new Spec(fill2, align, sign, altFlag, w, false, precision, c); - - // Signal that the padding, sign, base prefix etc. have all been taken care of - boolean jobDone = false; - - // Perform the type-specific formatting - switch (c) { - - case 's': - // String (converts any Python object using str()). - if (arg instanceof PyUnicode) { - needUnicode = true; - } - // fall through ... - - case 'r': - // String (converts any Python object using repr()). - fill = ' '; - if (c == 's') { - if (needUnicode) { - string = arg.__unicode__().toString(); - } else { - string = arg.__str__().toString(); - } - } else { - string = arg.__repr__().toString(); - } - if (precision >= 0 && string.length() > precision) { - string = string.substring(0, precision); - } - + + // Depending on the type of conversion, we use one of these formatters: + FloatFormatter ff; + IntegerFormatter fi; + TextFormatter ft; + Formatter f; // = ff, fi or ft, whichever we actually use. + + switch (spec.type) { + + case 's': // String: converts any object using __str__(), __unicode__() ... + case 'r': // ... or repr(). + PyObject arg = getarg(); + + // Get hold of the actual object to display (may set needUnicode) + PyString argAsString = asText(spec.type == 's' ? arg : arg.__repr__()); + // Format the str/unicode form of the argument using this Spec. + f = ft = new TextFormatter(spec); + ft.setBytes(!needUnicode); + ft.format(argAsString.getString()); break; case 'd': // All integer formats (+case for X). case 'o': case 'x': case 'X': + case 'c': // Single character (accepts integer or single character string). case 'u': // Obsolete type identical to 'd'. case 'i': // Compatibility with scanf(). - // Format using this Spec the double form of the argument. - IntegerFormatter fi = new IntegerFormatter.Traditional(spec); - - // Note various types accepted here as long as they have an __int__ method. - PyObject argAsNumber = asNumber(arg); - - // We have to check what we got back.. - if (argAsNumber instanceof PyInteger) { - fi.format(((PyInteger)argAsNumber).getValue()); - } else if (argAsNumber instanceof PyLong) { - fi.format(((PyLong)argAsNumber).getValue()); + // Format the argument using this Spec. + f = fi = new IntegerFormatter.Traditional(spec); + // If not producing PyUnicode, disallow codes >255. + fi.setBytes(!needUnicode); + + arg = getarg(); + + if (arg instanceof PyString && spec.type == 'c') { + if (arg.__len__() != 1) { + throw Py.TypeError("%c requires int or char"); + } else { + if (!needUnicode && arg instanceof PyUnicode) { + // Change of mind forced by encountering unicode object. + needUnicode = true; + fi.setBytes(false); + } + fi.format(((PyString)arg).getString().codePointAt(0)); + } + } else { - // It couldn't be converted, raise the error here - throw Py.TypeError("%" + c + " format: a number is required, not " - + arg.getType().fastGetName()); + // Note various types accepted here as long as they have an __int__ method. + PyObject argAsNumber = asNumber(arg); + + // We have to check what we got back. + if (argAsNumber instanceof PyInteger) { + fi.format(((PyInteger)argAsNumber).getValue()); + } else if (argAsNumber instanceof PyLong) { + fi.format(((PyLong)argAsNumber).getValue()); + } else { + // It couldn't be converted, raise the error here + throw Py.TypeError("%" + spec.type + + " format: a number is required, not " + + arg.getType().fastGetName()); + } } - fi.pad(); - string = fi.getResult(); - - // Suppress subsequent attempts to insert a correct sign, done already. - jobDone = true; break; case 'e': // All floating point formats (+case). @@ -4437,9 +4495,11 @@ case 'G': // Format using this Spec the double form of the argument. - FloatFormatter ff = new FloatFormatter(spec); + f = ff = new FloatFormatter(spec); + ff.setBytes(!needUnicode); // Note various types accepted here as long as they have a __float__ method. + arg = getarg(); PyObject argAsFloat = asFloat(arg); // We have to check what we got back.. @@ -4451,128 +4511,24 @@ + arg.getType().fastGetName()); } - ff.pad(); - string = ff.getResult(); - - // Suppress subsequent attempts to insert a correct sign, done already. - // signFlag = blankFlag = negative = false; - jobDone = true; break; - case 'c': - // Single character (accepts integer or single character string). - fill = ' '; - if (arg instanceof PyString) { - string = ((PyString)arg).toString(); - if (string.length() != 1) { - throw Py.TypeError("%c requires int or char"); - } - if (arg instanceof PyUnicode) { - needUnicode = true; - } - break; - } - - // arg is not a str (or unicode) - int val; - try { - // Explicitly __int__ so we can look for an AttributeError (which is - // less invasive to mask than a TypeError) - val = arg.__int__().asInt(); - } catch (PyException e) { - if (e.match(Py.AttributeError)) { - throw Py.TypeError("%c requires int or char"); - } - throw e; - } - // Range check, according to ultimate type of result as presentl;y known. - if (!needUnicode) { - if (val < 0) { - throw Py.OverflowError("unsigned byte integer is less than minimum"); - } else if (val > 255) { - throw Py.OverflowError("unsigned byte integer is greater than maximum"); - } - } else if (val < 0 || val > PySystemState.maxunicode) { - throw Py.OverflowError("%c arg not in range(0x110000) (wide Python build)"); - } - string = new String(new int[] {val}, 0, 1); + case '%': // Percent symbol, but surprisingly, padded. + + // We use an integer formatter. + f = fi = new IntegerFormatter.Traditional(spec); + fi.setBytes(!needUnicode); + fi.format('%'); break; default: throw Py.ValueError("unsupported format character '" - + codecs.encode(Py.newString(c), null, "replace") + "' (0x" - + Integer.toHexString(c) + ") at index " + (index - 1)); + + codecs.encode(Py.newString(spec.type), null, "replace") + "' (0x" + + Integer.toHexString(spec.type) + ") at index " + (index - 1)); } - /* - * We have now dealt with the translation of the (absolute value of the) argument, in - * variable string[]. In the next sections we deal with sign, padding and base prefix. - */ - if (jobDone) { - // Type-specific formatting has already taken care of all this. - buffer.append(string); - - } else { - // Legacy code still needed - int length = string.length(); - int skip = 0; - - // Decide how to represent the sign according to format and actual sign of argument. - String signString = null; - if (negative) { - signString = "-"; - } else { - if (signFlag) { - signString = "+"; - } else if (blankFlag) { - signString = " "; - } - } - - // The width (from here on) will be the remaining width on the line. - if (width < length) { - width = length; - } - - // Insert the sign in the buffer and adjust the width. - if (signString != null) { - if (fill != ' ') { - // When the fill is not space, the sign comes before the fill. - buffer.append(signString); - } - // Adjust width for sign. - if (width > length) { - width--; - } - } - - // Fill on the left of the item. - if (width > length && !ljustFlag) { - do { - buffer.append(fill); - } while (--width > length); - } - - // If the fill is spaces, we will have deferred the sign and hex base prefix - if (fill == ' ') { - if (signString != null) { - buffer.append(signString); - } - } - - // Now append the converted argument. - if (skip > 0) { - // The string contains a hex-prefix, but we have already inserted one. - buffer.append(string.substring(skip)); - } else { - buffer.append(string); - } - - // If this hasn't filled the space required, add right-padding. - while (--width >= length) { - buffer.append(' '); - } - } + // Pad the result as required in the format and append to the overall result. + buffer.append(f.pad().getResult()); } /* diff --git a/src/org/python/core/stringlib/IntegerFormatter.java b/src/org/python/core/stringlib/IntegerFormatter.java --- a/src/org/python/core/stringlib/IntegerFormatter.java +++ b/src/org/python/core/stringlib/IntegerFormatter.java @@ -294,6 +294,7 @@ break; case 'c': + case '%': // Binary. format_c(value); break; diff --git a/src/org/python/core/stringlib/InternalFormat.java b/src/org/python/core/stringlib/InternalFormat.java --- a/src/org/python/core/stringlib/InternalFormat.java +++ b/src/org/python/core/stringlib/InternalFormat.java @@ -334,63 +334,75 @@ * modes, the padding is around the whole buffer.) When this would not be appropriate, it is * up to the client to disallow this (which complex does). * - * @return this object + * @return this Formatter object */ public Formatter pad() { - // We'll need this many pad characters (if>0). Note Spec.UNDEFINED<0. int n = spec.width - result.length(); if (n > 0) { + // Note: use of leftIndex anticipates client-owned result buffer. + pad(0, n); + } + return this; + } - char align = spec.getAlign('>'); // Right for numbers (wrong for strings) - char fill = spec.getFill(' '); + /** + * Pad the last result (defined as the contents of {@link #result} from argument + * leftIndex to the end) using the alignment, by n repetitions of + * the fill character defined in {@link #spec}, and distributed according to + * spec.align. The value of leftIndex is only used if the + * alignment is '>' (left) or '^' (both). The value of the critical lengths (lenWhole, + * lenSign, etc.) are not affected, because we assume that leftIndex <= + * {@link #start}. + * + * @param leftIndex the index in result at which to insert left-fill characters. + * @param n number of fill characters to insert. + */ + protected void pad(int leftIndex, int n) { + char align = spec.getAlign('>'); // Right for numbers (strings will supply '<' align) + char fill = spec.getFill(' '); - // Start by assuming padding is all leading ('>' case or '=') - int leading = n; + // Start by assuming padding is all leading ('>' case or '=') + int leading = n; - // Split the total padding according to the alignment - if (align == '^') { - // Half the padding before - leading = n / 2; - } else if (align == '<') { - // All the padding after - leading = 0; + // Split the total padding according to the alignment + if (align == '^') { + // Half the padding before + leading = n / 2; + } else if (align == '<') { + // All the padding after + leading = 0; + } + + // All padding that is not leading is trailing + int trailing = n - leading; + + // Insert the leading space + if (leading > 0) { + if (align == '=') { + // Incorporate into the (latest) whole part + leftIndex = start + lenSign; + lenWhole += leading; + } else { + // Default is to insert at the stated leftIndex <= start. + start += leading; } - - // All padding that is not leading is trailing - int trailing = n - leading; - - // Insert the leading space - if (leading > 0) { - int pos; - if (align == '=') { - // Incorporate into the (latest) whole part - pos = start + lenSign; - lenWhole += leading; - } else { - // Insert at the very beginning (not start) by default. - pos = 0; - start += leading; - } - makeSpaceAt(pos, leading); - for (int i = 0; i < leading; i++) { - result.setCharAt(pos + i, fill); - } - } - - // Append the trailing space - for (int i = 0; i < trailing; i++) { - result.append(fill); - } - - // Check for special case - if (align == '=' && fill == '0' && spec.grouping) { - // We must extend the grouping separator into the padding - zeroPadAfterSignWithGroupingFixup(3, ','); + makeSpaceAt(leftIndex, leading); + for (int i = 0; i < leading; i++) { + result.setCharAt(leftIndex + i, fill); } } - return this; + // Append the trailing space + for (int i = 0; i < trailing; i++) { + result.append(fill); + } + + // Check for special case + if (align == '=' && fill == '0' && spec.grouping) { + // We must extend the grouping separator into the padding + zeroPadAfterSignWithGroupingFixup(3, ','); + } } /** @@ -512,6 +524,18 @@ /** * Convenience method returning a {@link Py#ValueError} reporting that specifying a + * sign is not allowed in a format specifier for the named type. + * + * @param forType the type it was found applied to + * @param code the formatting code (or '\0' not to mention one) + * @return exception to throw + */ + public static PyException signNotAllowed(String forType, char code) { + return notAllowed("Sign", forType, code); + } + + /** + * Convenience method returning a {@link Py#ValueError} reporting that specifying a * precision is not allowed in a format specifier for the named type. * * @param forType the type it was found applied to @@ -534,6 +558,18 @@ /** * Convenience method returning a {@link Py#ValueError} reporting that some format specifier + * feature is not allowed for the named data type. + * + * @param outrage committed in the present case + * @param forType the data type (e.g. "integer") it where it is an outrage + * @return exception to throw + */ + public static PyException notAllowed(String outrage, String forType) { + return notAllowed(outrage, forType, '\0'); + } + + /** + * Convenience method returning a {@link Py#ValueError} reporting that some format specifier * feature is not allowed for the named format code and data type. Produces a message like: *

* outrage+" not allowed with "+forType+" format specifier '"+code+"'" @@ -753,6 +789,12 @@ false, Spec.UNSPECIFIED, Spec.NONE); /** + * Defaults applicable to string types. Equivalent to " <" + */ + public static final Spec STRING = new Spec(' ', '<', Spec.NONE, false, Spec.UNSPECIFIED, + false, Spec.UNSPECIFIED, Spec.NONE); + + /** * Constructor offering just precision and type. * * 

diff --git a/src/org/python/core/stringlib/TextFormatter.java b/src/org/python/core/stringlib/TextFormatter.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/stringlib/TextFormatter.java
@@ -0,0 +1,105 @@
+// Copyright (c) Jython Developers
+package org.python.core.stringlib;
+
+import org.python.core.stringlib.InternalFormat.Spec;
+
+/**
+ * A class that provides the implementation of str and unicode formatting.
+ * In a limited way, it acts like a StringBuilder to which text, formatted according to the format
+ * specifier supplied at construction. These are ephemeral objects that are not, on their own,
+ * thread safe.
+ */
+public class TextFormatter extends InternalFormat.Formatter {
+
+    /**
+     * Construct the formatter from a specification and guess the initial buffer capacity. A
+     * reference is held to this specification.
+     *
+     * @param spec parsed conversion specification
+     */
+    public TextFormatter(Spec spec) {
+        // No right answer here for the buffer size, especially as non-BMP Unicode possible.
+        super(spec, Math.max(spec.width, spec.getPrecision(10)) + 6);
+    }
+
+    /*
+     * Re-implement the text appends so they return the right type.
+     */
+    @Override
+    public TextFormatter append(char c) {
+        super.append(c);
+        return this;
+    }
+
+    @Override
+    public TextFormatter append(CharSequence csq) {
+        super.append(csq);
+        return this;
+    }
+
+    @Override
+    public TextFormatter append(CharSequence csq, int start, int end) //
+            throws IndexOutOfBoundsException {
+        super.append(csq, start, end);
+        return this;
+    }
+
+    /**
+     * Format the given String into the result buffer. Largely, this is a
+     * matter of copying the value of the argument, but a subtlety arises when the string contains
+     * supplementary (non-BMP) Unicode characters, which are represented as surrogate pairs. The
+     * precision specified in the format relates to a count of Unicode characters (code points), not
+     * Java chars. The method deals with this correctly, essentially by not counting
+     * the high-surrogates in the allowance. The final value of {@link #lenWhole} counts the UTF-16
+     * units added.
+     *
+     * @param value to format
+     * @return this TextFormatter object
+     */
+    public TextFormatter format(String value) {
+        this.reset();
+        int p = spec.precision, n = value.length();
+
+        if (Spec.specified(p) && p < n) {
+            /*
+             * A precision p was specified less than the length: we may have to truncate. Note we
+             * compared p with the UTF-16 length, even though it is the code point length that
+             * matters. But the code point length cannot be greater than n.
+             */
+            int count = 0;
+            while (count < p) {
+                // count is the number of UTF-16 chars.
+                char c = value.charAt(count++);
+                result.append(c);
+                // A high-surrogate will always be followed by a low, so doesn't count.
+                if (Character.isHighSurrogate(c) && p < n) {
+                    // Accomplish "not counting" by bumping the limit p, within the array bounds.
+                    p += 1;
+                }
+            }
+            // Record the UTF-16 count as the length in buffer
+            lenWhole = count;
+
+        } else {
+            // We definitely don't need to truncate. Append the whole string.
+            lenWhole = n;
+            result.append(value);
+        }
+
+        return this;
+    }
+
+    /**
+     * Pad the result according to the specification, dealing correctly with Unicode.
+     */
+    @Override
+    public TextFormatter pad() {
+        // We'll need this many pad characters (if>0). Note Spec.UNDEFINED<0.
+        int n = spec.width - result.codePointCount(0, result.length());
+        if (n > 0) {
+            pad(0, n);
+        }
+        return this;
+    }
+
+}
diff --git a/tests/java/org/python/core/StringFormatTest.java b/tests/java/org/python/core/StringFormatTest.java
--- a/tests/java/org/python/core/StringFormatTest.java
+++ b/tests/java/org/python/core/StringFormatTest.java
@@ -10,6 +10,7 @@
 import org.python.core.stringlib.InternalFormatSpec;
 import org.python.core.stringlib.InternalFormatSpecParser;
 import org.python.core.stringlib.MarkupIterator;
+import org.python.core.stringlib.TextFormatter;
 import org.python.util.PythonInterpreter;
 
 /**
@@ -219,15 +220,17 @@
     }
 
     public void testFormatString() {
-        InternalFormatSpec spec = new InternalFormatSpec();
-        assertEquals("abc", PyString.formatString("abc", spec));
+        String v = "abc";
+        TextFormatter f;
+        f = PyString.prepareFormatter(InternalFormat.fromText(""));
+        assertEquals("abc", f.format(v).pad().getResult());
 
-        spec.precision = 3;
-        assertEquals("abc", PyString.formatString("abcdef", spec));
+        String v2 = "abcdef";
+        f = PyString.prepareFormatter(InternalFormat.fromText(".3"));
+        assertEquals("abc", f.format(v2).pad().getResult());
 
-        spec.precision = -1;
-        spec.width = 6;
-        assertEquals("abc   ", PyString.formatString("abc", spec));
+        f = PyString.prepareFormatter(InternalFormat.fromText("6"));
+        assertEquals("abc   ", f.format(v).pad().getResult());
     }
 
     public void testMarkupIterator() {

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun  8 14:13:07 2014
From: jython-checkins at python.org (jeff.allen)
Date: Sun,  8 Jun 2014 14:13:07 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Allow_formatters_to_write_r?=
 =?utf-8?q?esults_directly_into_StringFormatter=2E?=
Message-ID: <3gmc1b3pYZz7Ljl@mail.python.org>

http://hg.python.org/jython/rev/1db4ca04aeaa
changeset:   7285:1db4ca04aeaa
user:        Jeff Allen 
date:        Sat Jun 07 19:41:31 2014 +0100
summary:
  Allow formatters to write results directly into StringFormatter.
Changes the formatting framework to accept a client-owned StringBuilder. Some
related clean-up, and a test.

files:
  Lib/test/test_format_jy.py                          |   12 +
  src/org/python/core/PyComplex.java                  |  112 +++++++--
  src/org/python/core/PyFloat.java                    |    6 +-
  src/org/python/core/PyInteger.java                  |    2 +-
  src/org/python/core/PyString.java                   |   17 +-
  src/org/python/core/stringlib/FloatFormatter.java   |   50 +--
  src/org/python/core/stringlib/IntegerFormatter.java |   62 ++---
  src/org/python/core/stringlib/InternalFormat.java   |   66 +++-
  src/org/python/core/stringlib/TextFormatter.java    |   34 ++-
  9 files changed, 220 insertions(+), 141 deletions(-)


diff --git a/Lib/test/test_format_jy.py b/Lib/test/test_format_jy.py
--- a/Lib/test/test_format_jy.py
+++ b/Lib/test/test_format_jy.py
@@ -55,6 +55,18 @@
 class FormatMisc(unittest.TestCase):
     # Odd tests Jython used to fail
 
+    def test_mixtures(self) :
+        # Check formatting to a common buffer in PyString
+        result = 'The cube of 0.5 -0.866j is -1 to 0.01%.'
+        self.assertEqual(result, 'The %s of %.3g -%.3fj is -%d to %.2f%%.' %
+                          ('cube', 0.5, 0.866, 1, 0.01))
+        self.assertEqual(result, 'The %s of %.3g %.3fj is %d to %.2f%%.' %
+                          ('cube', 0.5, -0.866, -1, 0.01))
+        self.assertEqual(result, 'The%5s of%4.3g%7.3fj is%3d to%5.2f%%.' %
+                          ('cube', 0.5, -0.866, -1, 0.01))
+        self.assertEqual(result, 'The %-5.4sof %-4.3g%.3fj is %-3dto %.4g%%.' %
+                          ('cubensis', 0.5, -0.866, -1, 0.01))
+
     def test_percent_padded(self) :
         self.assertEqual('%hello', '%%%s' % 'hello')
         self.assertEqual(u'     %hello', '%6%%s' % u'hello')
diff --git a/src/org/python/core/PyComplex.java b/src/org/python/core/PyComplex.java
--- a/src/org/python/core/PyComplex.java
+++ b/src/org/python/core/PyComplex.java
@@ -4,6 +4,7 @@
 
 import org.python.core.stringlib.FloatFormatter;
 import org.python.core.stringlib.InternalFormat;
+import org.python.core.stringlib.InternalFormat.Formatter;
 import org.python.core.stringlib.InternalFormat.Spec;
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedMethod;
@@ -174,7 +175,9 @@
      * @return formatted value
      */
     private String formatComplex(Spec spec) {
-        FloatFormatter f = new FloatFormatter(spec, 2, 3); // Two elements + "(j)".length
+        int size = 2 * FloatFormatter.size(spec) + 3;  // 2 floats + "(j)"
+        FloatFormatter f = new FloatFormatter(new StringBuilder(size), spec);
+        f.setBytes(true);
         // Even in r-format, complex strips *all* the trailing zeros.
         f.setMinFracDigits(0);
         if (Double.doubleToLongBits(real) == 0L) {
@@ -816,42 +819,87 @@
 
     @ExposedMethod(doc = BuiltinDocs.complex___format___doc)
     final PyObject complex___format__(PyObject formatSpec) {
-        if (!(formatSpec instanceof PyString)) {
-            throw Py.TypeError("__format__ requires str or unicode");
+
+        // Parse the specification
+        Spec spec = InternalFormat.fromText(formatSpec, "__format__");
+
+        // fromText will have thrown if formatSpecStr is not a PyString (including PyUnicode)
+        PyString formatSpecStr = (PyString)formatSpec;
+        String result;
+
+        // Validate the specification and detect the special case for none-format
+        switch (checkSpecification(spec)) {
+
+            case 0: // None-format
+                // In none-format, we take the default type and precision from __str__.
+                spec = spec.withDefaults(SPEC_STR);
+                // And then we use the __str__ mechanism to get parentheses or real 0 elision.
+                result = formatComplex(spec);
+                break;
+
+            case 1: // Floating-point formats
+                // In any other format, defaults are those commonly used for numeric formats.
+                spec = spec.withDefaults(Spec.NUMERIC);
+                int size = 2 * FloatFormatter.size(spec) + 1; // 2 floats + "j"
+                FloatFormatter f = new FloatFormatter(new StringBuilder(size), spec);
+                f.setBytes(!(formatSpecStr instanceof PyUnicode));
+                // Convert both parts as per specification
+                f.format(real).format(imag, "+").append('j');
+                result = f.pad().getResult();
+                break;
+
+            default: // The type code was not recognised
+                throw Formatter.unknownFormat(spec.type, "complex");
         }
 
-        PyString formatSpecStr = (PyString)formatSpec;
-        String result;
-        try {
-            String specString = formatSpecStr.getString();
-            Spec spec = InternalFormat.fromText(specString);
-            if (spec.type != Spec.NONE && "efgEFGn%".indexOf(spec.type) < 0) {
-                throw FloatFormatter.unknownFormat(spec.type, "complex");
-            } else if (spec.alternate) {
-                throw FloatFormatter.alternateFormNotAllowed("complex");
-            } else if (spec.fill == '0') {
-                throw FloatFormatter.zeroPaddingNotAllowed("complex");
-            } else if (spec.align == '=') {
-                throw FloatFormatter.alignmentNotAllowed('=', "complex");
-            } else {
-                if (spec.type == Spec.NONE) {
-                    // In none-format, we take the default type and precision from __str__.
-                    spec = spec.withDefaults(SPEC_STR);
-                    // And then we use the __str__ mechanism to get parentheses or real 0 elision.
-                    result = formatComplex(spec);
+        // Wrap the result in the same type as the format string
+        return formatSpecStr.createInstance(result);
+    }
+
+    /**
+     * Validate a parsed specification, for PyComplex, returning 0 if it is a valid
+     * none-format specification, 1 if it is a valid float specification, and some other value if it
+     * not a valid type. If it has any other faults (e.g. alternate form was specified) the method
+     * raises a descriptive exception.
+     *
+     * @param spec a parsed PEP-3101 format specification.
+     * @return 0, 1, or other value for none-format, a float format, or incorrect type.
+     * @throws PyException(ValueError) if the specification is faulty.
+     */
+    @SuppressWarnings("fallthrough")
+    private static int checkSpecification(Spec spec) {
+
+        // Slight differences between format types
+        switch (spec.type) {
+
+            case 'n':
+                if (spec.grouping) {
+                    throw Formatter.notAllowed("Grouping", "complex", spec.type);
+                }
+                // Fall through
+
+            case Spec.NONE:
+            case 'e':
+            case 'f':
+            case 'g':
+            case 'E':
+            case 'F':
+            case 'G':
+                // Check for disallowed parts of the specification
+                if (spec.alternate) {
+                    throw FloatFormatter.alternateFormNotAllowed("complex");
+                } else if (spec.fill == '0') {
+                    throw FloatFormatter.zeroPaddingNotAllowed("complex");
+                } else if (spec.align == '=') {
+                    throw FloatFormatter.alignmentNotAllowed('=', "complex");
                 } else {
-                    // In any other format, defaults are those commonly used for numeric formats.
-                    spec = spec.withDefaults(Spec.NUMERIC);
-                    FloatFormatter f = new FloatFormatter(spec, 2, 1);// 2 floats + "j"
-                    // Convert both parts as per specification
-                    f.format(real).format(imag, "+").append('j');
-                    result = f.pad().getResult();
+                    return (spec.type == Spec.NONE) ? 0 : 1;
                 }
-            }
-        } catch (IllegalArgumentException e) {
-            throw Py.ValueError(e.getMessage());    // XXX Can this be reached?
+
+            default:
+                // spec.type is invalid for complex
+                return 2;
         }
-        return formatSpecStr.createInstance(result);
     }
 
     @Override
diff --git a/src/org/python/core/PyFloat.java b/src/org/python/core/PyFloat.java
--- a/src/org/python/core/PyFloat.java
+++ b/src/org/python/core/PyFloat.java
@@ -4,10 +4,8 @@
 
 import java.io.Serializable;
 import java.math.BigDecimal;
-import java.math.BigInteger;
 
 import org.python.core.stringlib.FloatFormatter;
-import org.python.core.stringlib.IntegerFormatter;
 import org.python.core.stringlib.InternalFormat;
 import org.python.core.stringlib.InternalFormat.Formatter;
 import org.python.core.stringlib.InternalFormat.Spec;
@@ -951,7 +949,7 @@
 
             case 'n':
                 if (spec.grouping) {
-                    throw IntegerFormatter.notAllowed("Grouping", "float", spec.type);
+                    throw Formatter.notAllowed("Grouping", "float", spec.type);
                 }
                 // Fall through
 
@@ -969,7 +967,7 @@
                 }
                 // spec may be incomplete. The defaults are those commonly used for numeric formats.
                 spec = spec.withDefaults(Spec.NUMERIC);
-                return new FloatFormatter(spec, 1);
+                return new FloatFormatter(spec);
 
             default:
                 return null;
diff --git a/src/org/python/core/PyInteger.java b/src/org/python/core/PyInteger.java
--- a/src/org/python/core/PyInteger.java
+++ b/src/org/python/core/PyInteger.java
@@ -1099,7 +1099,7 @@
                 // spec may be incomplete. The defaults are those commonly used for numeric formats.
                 spec = spec.withDefaults(Spec.NUMERIC);
                 // Get a formatter for the spec.
-                return new IntegerFormatter(spec, 1);
+                return new IntegerFormatter(spec);
 
             default:
                 return null;
diff --git a/src/org/python/core/PyString.java b/src/org/python/core/PyString.java
--- a/src/org/python/core/PyString.java
+++ b/src/org/python/core/PyString.java
@@ -4436,7 +4436,7 @@
                     // Get hold of the actual object to display (may set needUnicode)
                     PyString argAsString = asText(spec.type == 's' ? arg : arg.__repr__());
                     // Format the str/unicode form of the argument using this Spec.
-                    f = ft = new TextFormatter(spec);
+                    f = ft = new TextFormatter(buffer, spec);
                     ft.setBytes(!needUnicode);
                     ft.format(argAsString.getString());
                     break;
@@ -4450,7 +4450,7 @@
                 case 'i': // Compatibility with scanf().
 
                     // Format the argument using this Spec.
-                    f = fi = new IntegerFormatter.Traditional(spec);
+                    f = fi = new IntegerFormatter.Traditional(buffer, spec);
                     // If not producing PyUnicode, disallow codes >255.
                     fi.setBytes(!needUnicode);
 
@@ -4495,7 +4495,7 @@
                 case 'G':
 
                     // Format using this Spec the double form of the argument.
-                    f = ff = new FloatFormatter(spec);
+                    f = ff = new FloatFormatter(buffer, spec);
                     ff.setBytes(!needUnicode);
 
                     // Note various types accepted here as long as they have a __float__ method.
@@ -4516,7 +4516,7 @@
                 case '%': // Percent symbol, but surprisingly, padded.
 
                     // We use an integer formatter.
-                    f = fi = new IntegerFormatter.Traditional(spec);
+                    f = fi = new IntegerFormatter.Traditional(buffer, spec);
                     fi.setBytes(!needUnicode);
                     fi.format('%');
                     break;
@@ -4527,8 +4527,8 @@
                             + Integer.toHexString(spec.type) + ") at index " + (index - 1));
             }
 
-            // Pad the result as required in the format and append to the overall result.
-            buffer.append(f.pad().getResult());
+            // Pad the result as specified (in-place, in the buffer).
+            f.pad();
         }
 
         /*
@@ -4543,10 +4543,7 @@
         }
 
         // Return the final buffer contents as a str or unicode as appropriate.
-        if (needUnicode) {
-            return new PyUnicode(buffer);
-        }
-        return new PyString(buffer);
+        return needUnicode ? new PyUnicode(buffer) : new PyString(buffer);
     }
 
 }
diff --git a/src/org/python/core/stringlib/FloatFormatter.java b/src/org/python/core/stringlib/FloatFormatter.java
--- a/src/org/python/core/stringlib/FloatFormatter.java
+++ b/src/org/python/core/stringlib/FloatFormatter.java
@@ -34,26 +34,14 @@
     private int minFracDigits;
 
     /**
-     * Construct the formatter from a specification. A reference is held to this specification, but
-     * it will not be modified by the actions of this class.
+     * Construct the formatter from a client-supplied buffer, to which the result will be appended,
+     * and a specification. Sets {@link #mark} to the end of the buffer.
      *
+     * @param result destination buffer
      * @param spec parsed conversion specification
      */
-    public FloatFormatter(Spec spec) {
-        // Space for result is based on padded width, or precision, whole part & furniture.
-        this(spec, 1, 0);
-    }
-
-    /**
-     * Construct the formatter from a specification and an explicit initial buffer capacity. A
-     * reference is held to this specification, but it will not be modified by the actions of this
-     * class.
-     *
-     * @param spec parsed conversion specification
-     * @param width expected for the formatted result
-     */
-    public FloatFormatter(Spec spec, int width) {
-        super(spec, width);
+    public FloatFormatter(StringBuilder result, Spec spec) {
+        super(result, spec);
         if (spec.alternate) {
             // Alternate form means do not trim the zero fractional digits.
             minFracDigits = -1;
@@ -70,20 +58,26 @@
     }
 
     /**
-     * Construct the formatter from a specification and two extra hints about the initial buffer
-     * capacity. A reference is held to this specification, but it will not be modified by the
-     * actions of this class.
+     * Construct the formatter from a specification, allocating a buffer internally for the result.
      *
      * @param spec parsed conversion specification
-     * @param count of elements likely to be formatted
-     * @param margin for elements formatted only once
      */
-    public FloatFormatter(Spec spec, int count, int margin) {
-        /*
-         * Rule of thumb used here: in e format w = (p-1) + len("+1.e+300") = p+7; in f format w = p
-         * + len("1,000,000.") = p+10. If we're wrong, the result will have to grow. No big deal.
-         */
-        this(spec, Math.max(spec.width + 1, count * (spec.precision + 10) + margin));
+    public FloatFormatter(Spec spec) {
+        this(new StringBuilder(size(spec)), spec);
+    }
+
+    /**
+     * Recommend a buffer size for a given specification, assuming one float is converted. This will
+     * be a "right" answer for e and g-format, and for f-format with values up to 9,999,999.
+     *
+     * @param spec parsed conversion specification
+     */
+    public static int size(Spec spec) {
+        // Rule of thumb used here (no right answer):
+        // in e format each float occupies: (p-1) + len("+1.e+300") = p+7;
+        // in f format each float occupies: p + len("1,000,000.%") = p+11;
+        // or an explicit (minimum) width may be given, with one overshoot possible.
+        return Math.max(spec.width + 1, spec.getPrecision(6) + 11);
     }
 
     /**
diff --git a/src/org/python/core/stringlib/IntegerFormatter.java b/src/org/python/core/stringlib/IntegerFormatter.java
--- a/src/org/python/core/stringlib/IntegerFormatter.java
+++ b/src/org/python/core/stringlib/IntegerFormatter.java
@@ -20,26 +20,24 @@
 public class IntegerFormatter extends InternalFormat.Formatter {
 
     /**
-     * Construct the formatter from a specification. A reference is held to this specification, but
-     * it will not be modified by the actions of this class.
+     * Construct the formatter from a client-supplied buffer, to which the result will be appended,
+     * and a specification. Sets {@link #mark} to the end of the buffer.
+     *
+     * @param result destination buffer
+     * @param spec parsed conversion specification
+     */
+    public IntegerFormatter(StringBuilder result, Spec spec) {
+        super(result, spec);
+    }
+
+    /**
+     * Construct the formatter from a specification, allocating a buffer internally for the result.
      *
      * @param spec parsed conversion specification
      */
     public IntegerFormatter(Spec spec) {
-        // Space for result is based on padded width, or precision, whole part & furniture.
-        this(spec, 12);
-    }
-
-    /**
-     * Construct the formatter from a specification and an explicit initial buffer capacity. A
-     * reference is held to this specification, but it will not be modified by the actions of this
-     * class.
-     *
-     * @param spec parsed conversion specification
-     * @param width expected for the formatted result
-     */
-    public IntegerFormatter(Spec spec, int width) {
-        super(spec, width);
+        // Rule of thumb: big enough for 32-bit binary with base indicator 0b
+        this(new StringBuilder(34), spec);
     }
 
     /*
@@ -76,9 +74,6 @@
     @SuppressWarnings("fallthrough")
     public IntegerFormatter format(BigInteger value) {
         try {
-            // Scratch all instance variables and start = result.length().
-            setStart();
-
             // Different process for each format type.
             switch (spec.type) {
                 case 'd':
@@ -637,25 +632,24 @@
     public static class Traditional extends IntegerFormatter {
 
         /**
-         * Construct the formatter from a specification. A reference is held to this specification,
-         * but it will not be modified by the actions of this class.
+         * Construct the formatter from a client-supplied buffer, to which the result will be
+         * appended, and a specification. Sets {@link #mark} to the end of the buffer.
+         *
+         * @param result destination buffer
+         * @param spec parsed conversion specification
+         */
+        public Traditional(StringBuilder result, Spec spec) {
+            super(result, spec);
+        }
+
+        /**
+         * Construct the formatter from a specification, allocating a buffer internally for the
+         * result.
          *
          * @param spec parsed conversion specification
          */
         public Traditional(Spec spec) {
-            super(spec);
-        }
-
-        /**
-         * Construct the formatter from a specification and an explicit initial buffer capacity. A
-         * reference is held to this specification, but it will not be modified by the actions of
-         * this class.
-         *
-         * @param spec parsed conversion specification
-         * @param width expected for the formatted result
-         */
-        public Traditional(Spec spec, int width) {
-            super(spec, width);
+            this(new StringBuilder(), spec);
         }
 
         /**
diff --git a/src/org/python/core/stringlib/InternalFormat.java b/src/org/python/core/stringlib/InternalFormat.java
--- a/src/org/python/core/stringlib/InternalFormat.java
+++ b/src/org/python/core/stringlib/InternalFormat.java
@@ -57,7 +57,9 @@
          */
         protected boolean bytes;
 
-        /** The number we are working on floats at the end of the result, and starts here. */
+        /** The start of the formatted data for padding purposes, <={@link #start} */
+        protected int mark;
+        /** The latest number we are working on floats at the end of the result, and starts here. */
         protected int start;
         /** If it contains no sign, this length is zero, and >0 otherwise. */
         protected int lenSign;
@@ -65,15 +67,29 @@
         protected int lenWhole;
 
         /**
-         * Construct the formatter from a specification and initial buffer capacity. A reference is
-         * held to this specification, but it will not be modified by the actions of this class.
+         * Construct the formatter from a client-supplied buffer and a specification. Sets
+         * {@link #mark} and {@link #start} to the end of the buffer. The new formatted object will
+         * therefore be appended there and, when the time comes, padding will be applied to (just)
+         * the new text.
+         *
+         * @param result destination buffer
+         * @param spec parsed conversion specification
+         */
+        public Formatter(StringBuilder result, Spec spec) {
+            this.spec = spec;
+            this.result = result;
+            this.start = this.mark = result.length();
+        }
+
+        /**
+         * Construct the formatter from a specification and initial buffer capacity. Sets
+         * {@link #mark} to the end of the buffer.
          *
          * @param spec parsed conversion specification
          * @param width of buffer initially
          */
         public Formatter(Spec spec, int width) {
-            this.spec = spec;
-            result = new StringBuilder(width);
+            this(new StringBuilder(width), spec);
         }
 
         /**
@@ -154,18 +170,25 @@
 
         /**
          * Clear the instance variables describing the latest object in {@link #result}, ready to
-         * receive a new number
+         * receive a new one: sets {@link #start} and calls {@link #reset()}. This is necessary when
+         * a Formatter is to be re-used. Note that this leaves {@link #mark} where it
+         * is. In the core, we need this to support complex: two floats in the same
+         * format, but padded as a unit.
          */
-        void setStart() {
-            // Mark the end of the buffer as the start of the current object and reset all.
+        public void setStart() {
+            // The new value will float at the current end of the result buffer.
             start = result.length();
-            // Clear the variable describing the latest number in result.
-            reset();
+            // If anything has been added since construction, reset all state.
+            if (start > mark) {
+                // Clear the variable describing the latest number in result.
+                reset();
+            }
         }
 
         /**
          * Clear the instance variables describing the latest object in {@link #result}, ready to
-         * receive a new one.
+         * receive a new one. This is called from {@link #setStart()}. Subclasses override this
+         * method and call {@link #setStart()} at the start of their format method.
          */
         protected void reset() {
             // Clear the variables describing the latest object in result.
@@ -285,10 +308,10 @@
         }
 
         /**
-         * Pad the result so far (defined as the entire contents of {@link #result}) using the
-         * alignment, target width and fill character defined in {@link #spec}. The action of
-         * padding will increase the overall length of the result to the target width, if that is
-         * greater than the current length.
+         * Pad the result so far (defined as the contents of {@link #result} from {@link #mark} to
+         * the end) using the alignment, target width and fill character defined in {@link #spec}.
+         * The action of padding will increase the length of this segment to the target width, if
+         * that is greater than the current length.
          * 

          * When the padding method has decided that that it needs to add n padding characters, it
          * will affect {@link #start} or {@link #lenWhole} as follows.
@@ -331,17 +354,16 @@
          * 
          * Note that in the "pad after sign" mode, only the last number into the buffer receives the
          * padding. This padding gets incorporated into the whole part of the number. (In other
-         * modes, the padding is around the whole buffer.) When this would not be appropriate, it is
-         * up to the client to disallow this (which complex does).
+         * modes, the padding is around result[mark:].) When this would not be
+         * appropriate, it is up to the client to disallow this (which complex does).
          *
          * @return this Formatter object
          */
         public Formatter pad() {
             // We'll need this many pad characters (if>0). Note Spec.UNDEFINED<0.
-            int n = spec.width - result.length();
+            int n = spec.width - (result.length() - mark);
             if (n > 0) {
-                // Note: use of leftIndex anticipates client-owned result buffer.
-                pad(0, n);
+                pad(mark, n);
             }
             return this;
         }
@@ -523,8 +545,8 @@
         }
 
         /**
-         * Convenience method returning a {@link Py#ValueError} reporting that specifying a
-         * sign is not allowed in a format specifier for the named type.
+         * Convenience method returning a {@link Py#ValueError} reporting that specifying a sign is
+         * not allowed in a format specifier for the named type.
          *
          * @param forType the type it was found applied to
          * @param code the formatting code (or '\0' not to mention one)
diff --git a/src/org/python/core/stringlib/TextFormatter.java b/src/org/python/core/stringlib/TextFormatter.java
--- a/src/org/python/core/stringlib/TextFormatter.java
+++ b/src/org/python/core/stringlib/TextFormatter.java
@@ -1,6 +1,7 @@
 // Copyright (c) Jython Developers
 package org.python.core.stringlib;
 
+import org.python.core.stringlib.InternalFormat.Formatter;
 import org.python.core.stringlib.InternalFormat.Spec;
 
 /**
@@ -12,14 +13,23 @@
 public class TextFormatter extends InternalFormat.Formatter {
 
     /**
-     * Construct the formatter from a specification and guess the initial buffer capacity. A
-     * reference is held to this specification.
+     * Construct the formatter from a client-supplied buffer, to which the result will be appended,
+     * and a specification. Sets {@link #mark} to the end of the buffer.
+     *
+     * @param result destination buffer
+     * @param spec parsed conversion specification
+     */
+    public TextFormatter(StringBuilder result, Spec spec) {
+        super(result, spec);
+    }
+
+    /**
+     * Construct the formatter from a specification, allocating a buffer internally for the result.
      *
      * @param spec parsed conversion specification
      */
     public TextFormatter(Spec spec) {
-        // No right answer here for the buffer size, especially as non-BMP Unicode possible.
-        super(spec, Math.max(spec.width, spec.getPrecision(10)) + 6);
+        this(new StringBuilder(), spec);
     }
 
     /*
@@ -57,10 +67,16 @@
      * @return this TextFormatter object
      */
     public TextFormatter format(String value) {
-        this.reset();
+
+        // Scratch all instance variables and start = result.length().
+        setStart();
+
         int p = spec.precision, n = value.length();
 
         if (Spec.specified(p) && p < n) {
+            // Estimate the space for the converted result (preempt multiple re-allocation)
+            int space = Math.max(spec.width, p);
+            result.ensureCapacity(result.length() + space + (bytes ? 0 : space / 4));
             /*
              * A precision p was specified less than the length: we may have to truncate. Note we
              * compared p with the UTF-16 length, even though it is the code point length that
@@ -89,15 +105,13 @@
         return this;
     }
 
-    /**
-     * Pad the result according to the specification, dealing correctly with Unicode.
-     */
+    // Variant to deal with supplementary characters: other formatters don't produce them.
     @Override
     public TextFormatter pad() {
         // We'll need this many pad characters (if>0). Note Spec.UNDEFINED<0.
-        int n = spec.width - result.codePointCount(0, result.length());
+        int n = spec.width - result.codePointCount(mark, result.length());
         if (n > 0) {
-            pad(0, n);
+            pad(mark, n);
         }
         return this;
     }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun  8 14:13:09 2014
From: jython-checkins at python.org (jeff.allen)
Date: Sun,  8 Jun 2014 14:13:09 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_redundant_copies_of_?=
 =?utf-8?q?code=2C_related_to_formatting=2C_now_moved_to_stringlib=2E?=
Message-ID: <3gmc1d0YcRz7LjX@mail.python.org>

http://hg.python.org/jython/rev/234d1492dde4
changeset:   7286:234d1492dde4
user:        Jeff Allen 
date:        Sun Jun 08 07:44:48 2014 +0100
summary:
  Remove redundant copies of code, related to formatting, now moved to stringlib.

files:
  src/org/python/core/PyInteger.java                          |  107 ---------
  src/org/python/core/stringlib/InternalFormatSpec.java       |   88 -------
  src/org/python/core/stringlib/InternalFormatSpecParser.java |  118 ----------
  tests/java/org/python/core/StringFormatTest.java            |   39 +-
  4 files changed, 22 insertions(+), 330 deletions(-)


diff --git a/src/org/python/core/PyInteger.java b/src/org/python/core/PyInteger.java
--- a/src/org/python/core/PyInteger.java
+++ b/src/org/python/core/PyInteger.java
@@ -1106,113 +1106,6 @@
         }
     }
 
-    /**
-     * A more efficient algorithm for generating a hexadecimal representation of a byte array.
-     * {@link BigInteger#toString(int)} is too slow because it generalizes to any radix and,
-     * consequently, is implemented using expensive mathematical operations.
-     *
-     * @param value the value to generate a hexadecimal string from
-     * @return the hexadecimal representation of value, with "-" sign prepended if necessary
-     */
-    static final String toHexString(BigInteger value) {
-        int signum = value.signum();
-
-        // obvious shortcut
-        if (signum == 0) {
-            return "0";
-        }
-
-        // we want to work in absolute numeric value (negative sign is added afterward)
-        byte[] input = value.abs().toByteArray();
-        StringBuilder sb = new StringBuilder(input.length * 2);
-
-        int b;
-        for (int i = 0; i < input.length; i++) {
-            b = input[i] & 0xFF;
-            sb.append(LOOKUP.charAt(b >> 4));
-            sb.append(LOOKUP.charAt(b & 0x0F));
-        }
-
-        // before returning the char array as string, remove leading zeroes, but not the last one
-        String result = sb.toString().replaceFirst("^0+(?!$)", "");
-        return signum < 0 ? "-" + result : result;
-    }
-
-    /**
-     * A more efficient algorithm for generating an octal representation of a byte array.
-     * {@link BigInteger#toString(int)} is too slow because it generalizes to any radix and,
-     * consequently, is implemented using expensive mathematical operations.
-     *
-     * @param value the value to generate an octal string from
-     * @return the octal representation of value, with "-" sign prepended if necessary
-     */
-    static final String toOctString(BigInteger value) {
-        int signum = value.signum();
-
-        // obvious shortcut
-        if (signum == 0) {
-            return "0";
-        }
-
-        byte[] input = value.abs().toByteArray();
-        if (input.length < 3) {
-            return value.toString(8);
-        }
-
-        StringBuilder sb = new StringBuilder(input.length * 3);
-
-        // working backwards, three bytes at a time
-        int threebytes;
-        int trip1, trip2, trip3;    // most, middle, and least significant bytes in the triplet
-        for (int i = input.length - 1; i >= 0; i -= 3) {
-            trip3 = input[i] & 0xFF;
-            trip2 = ((i - 1) >= 0) ? (input[i - 1] & 0xFF) : 0x00;
-            trip1 = ((i - 2) >= 0) ? (input[i - 2] & 0xFF) : 0x00;
-            threebytes = trip3 | (trip2 << 8) | (trip1 << 16);
-
-            // convert the three-byte value into an eight-character octal string
-            for (int j = 0; j < 8; j++) {
-                sb.append(LOOKUP.charAt((threebytes >> (j * 3)) & 0x000007));
-            }
-        }
-
-        String result = sb.reverse().toString().replaceFirst("^0+(?!%)", "");
-        return signum < 0 ? "-" + result : result;
-    }
-
-    /**
-     * A more efficient algorithm for generating a binary representation of a byte array.
-     * {@link BigInteger#toString(int)} is too slow because it generalizes to any radix and,
-     * consequently, is implemented using expensive mathematical operations.
-     *
-     * @param value the value to generate a binary string from
-     * @return the binary representation of value, with "-" sign prepended if necessary
-     */
-    static final String toBinString(BigInteger value) {
-        int signum = value.signum();
-
-        // obvious shortcut
-        if (signum == 0) {
-            return "0";
-        }
-
-        // we want to work in absolute numeric value (negative sign is added afterward)
-        byte[] input = value.abs().toByteArray();
-        StringBuilder sb = new StringBuilder(value.bitCount());
-
-        int b;
-        for (int i = 0; i < input.length; i++) {
-            b = input[i] & 0xFF;
-            for (int bit = 7; bit >= 0; bit--) {
-                sb.append(((b >> bit) & 0x1) > 0 ? "1" : "0");
-            }
-        }
-
-        // before returning the char array as string, remove leading zeroes, but not the last one
-        String result = sb.toString().replaceFirst("^0+(?!$)", "");
-        return signum < 0 ? "-" + result : result;
-    }
-
     @Override
     public boolean isIndex() {
         return true;
diff --git a/src/org/python/core/stringlib/InternalFormatSpec.java b/src/org/python/core/stringlib/InternalFormatSpec.java
deleted file mode 100644
--- a/src/org/python/core/stringlib/InternalFormatSpec.java
+++ /dev/null
@@ -1,88 +0,0 @@
-package org.python.core.stringlib;
-
-/**
- * Parsed PEP-3101 format specification of a single field. This class holds the several attributes
- * that might be decoded from a format specifier. It provides a method
- * {@link #pad(String, char, int)} for adjusting a string using those attributes related to padding
- * to a string assumed to be the result of formatting to the given precision.
- * 

- * This structure is returned by {@link InternalFormatSpecParser#parse()} and having public members
- * is freely used by {@link InternalFormatSpecParser}, and the __format__ methods of client object
- * types.
- * 

- * The fields correspond to the elements of a format specification. The grammar of a format
- * specification is:
- *
- * 
- * [[fill]align][sign][#][0][width][,][.precision][type]
- * 

- */
-public final class InternalFormatSpec {
-
-    /** The fill specified in the grammar. */
-    public char fill_char;
-    /** Alignment indicator is 0, or one of {'<', '^', '>', '=' . */
-    public char align;
-    /** The alternative format flag '#' was given. */
-    public boolean alternate;
-    /** Sign-handling flag, one of '+', '-', or ' '. */
-    public char sign;
-    /** Width to which to pad the resault in {@link #pad(String, char, int)}. */
-    public int width = -1;
-    /** Insert the grouping separator (which in Python always indicates a group-size of 3). */
-    public boolean thousands_separators;
-    /** Precision decoded from the format. */
-    public int precision = -1;
-    /** Type key from the format. */
-    public char type;
-
-    /**
-     * Pad value, using {@link #fill_char} (or ' ') before and after, to {@link #width}
-     * -leaveWidth, aligned according to {@link #align} (or according to
-     * defaultAlign).
-     *
-     * @param value to pad
-     * @param defaultAlign to use if this.align=0 (one of '<',
-     *            '^', '>', or '=').
-     * @param leaveWidth to reduce effective this.width by
-     * @return padded value
-     */
-    public String pad(String value, char defaultAlign, int leaveWidth) {
-
-        // We'll need this many pad characters (if>0)
-        int remaining = width - value.length() - leaveWidth;
-        if (remaining <= 0) {
-            return value;
-        }
-
-        // Use this.align or defaultAlign
-        int useAlign = align;
-        if (useAlign == 0) {
-            useAlign = defaultAlign;
-        }
-
-        // By default all padding is leading padding ('<' case or '=')
-        int leading = remaining;
-        if (useAlign == '^') {
-            // Half the padding before
-            leading = remaining / 2;
-        } else if (useAlign == '<') {
-            // All the padding after
-            leading = 0;
-        }
-
-        // Now build the result
-        StringBuilder result = new StringBuilder();
-        char fill = fill_char != 0 ? fill_char : ' ';
-
-        for (int i = 0; i < leading; i++) { // before
-            result.append(fill);
-        }
-        result.append(value);
-        for (int i = 0; i < remaining - leading; i++) { // after
-            result.append(fill);
-        }
-
-        return result.toString();
-    }
-}
diff --git a/src/org/python/core/stringlib/InternalFormatSpecParser.java b/src/org/python/core/stringlib/InternalFormatSpecParser.java
deleted file mode 100644
--- a/src/org/python/core/stringlib/InternalFormatSpecParser.java
+++ /dev/null
@@ -1,118 +0,0 @@
-package org.python.core.stringlib;
-
-/**
- * Parser for PEP-3101 field format specifications. This class provides a {@link #parse()} method
- * that translates the format specification into an InternalFormatSpec object.
- */
-public class InternalFormatSpecParser {
-
-    private String spec;
-    private int index;
-
-    /**
-     * Constructor simply holds the specification streang ahead of the {@link #parse()} operation.
-     *
-     * @param spec format specifier to parse (e.g. "<+12.3f")
-     */
-    public InternalFormatSpecParser(String spec) {
-        this.spec = spec;
-        this.index = 0;
-    }
-
-    private static boolean isAlign(char c) {
-        switch (c) {
-            case '<':
-            case '>':
-            case '=':
-            case '^':
-                return true;
-            default:
-                return false;
-        }
-    }
-
-    /**
-     * Parse the specification with which this object was initialised into an
-     * {@link InternalFormatSpec}, which is an object encapsulating the format for use by formatting
-     * methods. This parser deals only with the format specifiers themselves, as accepted by the
-     * __format__ method of a type, or the format() built-in, not format
-     * strings in general as accepted by str.format(). A typical idiom is:
-     *
-     * 
-     * InternalFormatSpec spec = new InternalFormatSpecParser(specString).parse();
-     * 

-     *
-     * @return the InternalFormatSpec equivalent to the constructor argument
-     */
-    /*
-     * This method is the equivalent of CPython's parse_internal_render_format_spec() in
-     * ~/Objects/stringlib/formatter.h.
-     */
-    // XXX Better encapsulated as a constructor of InternalFormatSpec?
-    public InternalFormatSpec parse() {
-        InternalFormatSpec result = new InternalFormatSpec();
-        if (spec.length() >= 1 && isAlign(spec.charAt(0))) {
-            result.align = spec.charAt(index);
-            index++;
-        } else if (spec.length() >= 2 && isAlign(spec.charAt(1))) {
-            result.fill_char = spec.charAt(0);
-            result.align = spec.charAt(1);
-            index += 2;
-        }
-        if (isAt("+- ")) {
-            result.sign = spec.charAt(index);
-            index++;
-        }
-        if (isAt("#")) {
-            result.alternate = true;
-            index++;
-        }
-        if (result.fill_char == '\0' && isAt("0")) {
-            result.fill_char = '0';
-            if (result.align == '\0') {
-                result.align = '=';
-            }
-            index++;
-        }
-        result.width = getInteger();
-        if (isAt(",")) {
-            result.thousands_separators = true;
-            index++;
-        }
-        if (isAt(".")) {
-            index++;
-            result.precision = getInteger();
-            if (result.precision == -1) {
-                throw new IllegalArgumentException("Format specifier missing precision");
-            }
-        }
-        if (index < spec.length()) {
-            result.type = spec.charAt(index);
-            if (index + 1 != spec.length()) {
-                throw new IllegalArgumentException("Invalid conversion specification");
-            }
-        }
-        if (result.thousands_separators && "defgEG%F\0".indexOf(result.type) == -1) {
-            throw new IllegalArgumentException("Cannot specify ',' with '" + result.type + "'.");
-        }
-        return result;
-    }
-
-    private int getInteger() {
-        int value = 0;
-        boolean empty = true;
-        while (index < spec.length() && spec.charAt(index) >= '0' && spec.charAt(index) <= '9') {
-            value = value * 10 + spec.charAt(index) - '0';
-            index++;
-            empty = false;
-        }
-        if (empty) {
-            return -1;
-        }
-        return value;
-    }
-
-    private boolean isAt(String chars) {
-        return index < spec.length() && chars.indexOf(spec.charAt(index)) >= 0;
-    }
-}
diff --git a/tests/java/org/python/core/StringFormatTest.java b/tests/java/org/python/core/StringFormatTest.java
--- a/tests/java/org/python/core/StringFormatTest.java
+++ b/tests/java/org/python/core/StringFormatTest.java
@@ -7,10 +7,9 @@
 import org.python.core.stringlib.FieldNameIterator;
 import org.python.core.stringlib.IntegerFormatter;
 import org.python.core.stringlib.InternalFormat;
-import org.python.core.stringlib.InternalFormatSpec;
-import org.python.core.stringlib.InternalFormatSpecParser;
 import org.python.core.stringlib.MarkupIterator;
 import org.python.core.stringlib.TextFormatter;
+import org.python.core.stringlib.InternalFormat.Spec;
 import org.python.util.PythonInterpreter;
 
 /**
@@ -22,33 +21,38 @@
     PythonInterpreter interp = new PythonInterpreter();
 
     public void testInternalFormatSpec() {
-        InternalFormatSpec spec = new InternalFormatSpecParser("x").parse();
+        InternalFormat.Spec spec;
+        spec = InternalFormat.fromText("x");
+        assertFalse(Spec.specified(spec.align));
+        assertFalse(Spec.specified(spec.fill));
+        assertFalse(Spec.specified(spec.width));
+        assertFalse(Spec.specified(spec.precision));
         assertEquals('x', spec.type);
 
-        spec = new InternalFormatSpecParser("

http://hg.python.org/jython/rev/6cee6fef06f0
changeset:   7287:6cee6fef06f0
parent:      7278:9cd9ab75eade
parent:      7286:234d1492dde4
user:        Jeff Allen 
date:        Sun Jun 08 10:03:49 2014 +0100
summary:
  Merge of formatting work to trunk

files:
  Lib/test/test_builtin.py                                    |   13 +-
  Lib/test/test_format.py                                     |  473 +++--
  Lib/test/test_format_jy.py                                  |   69 +-
  Lib/test/test_types.py                                      |  122 +-
  Lib/test/test_unicode.py                                    |    7 +-
  src/org/python/core/PyComplex.java                          |  112 +-
  src/org/python/core/PyFloat.java                            |   77 +-
  src/org/python/core/PyInteger.java                          |  349 +---
  src/org/python/core/PyLong.java                             |  178 +-
  src/org/python/core/PyString.java                           |  811 +++------
  src/org/python/core/__builtin__.java                        |   20 +-
  src/org/python/core/stringlib/FloatFormatter.java           |   69 +-
  src/org/python/core/stringlib/IntegerFormatter.java         |  779 +++++++++
  src/org/python/core/stringlib/InternalFormat.java           |  360 +++-
  src/org/python/core/stringlib/InternalFormatSpec.java       |   88 -
  src/org/python/core/stringlib/InternalFormatSpecParser.java |  118 -
  src/org/python/core/stringlib/TextFormatter.java            |  119 +
  tests/java/org/python/core/StringFormatTest.java            |  264 ++-
  18 files changed, 2424 insertions(+), 1604 deletions(-)


diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -361,8 +361,7 @@
         self.assertEqual(eval('a', g, m), 12)
         self.assertRaises(NameError, eval, 'b', g, m)
         self.assertEqual(eval('dir()', g, m), list('xyz'))
-        if not is_jython: #FIXME #1861
-            self.assertEqual(eval('globals()', g, m), g)
+        self.assertEqual(eval('globals()', g, m), g)
         self.assertEqual(eval('locals()', g, m), m)
 
         # Jython allows arbitrary mappings for globals
@@ -386,8 +385,7 @@
         self.assertEqual(eval('a', g, d), 12)
         self.assertRaises(NameError, eval, 'b', g, d)
         self.assertEqual(eval('dir()', g, d), list('xyz'))
-        if not is_jython: #FIXME #1861
-            self.assertEqual(eval('globals()', g, d), g)
+        self.assertEqual(eval('globals()', g, d), g)
         self.assertEqual(eval('locals()', g, d), d)
 
         # Verify locals stores (used by list comps)
@@ -1320,7 +1318,6 @@
         self.assertRaises(TypeError, round, t)
         self.assertRaises(TypeError, round, t, 0)
 
-    @unittest.skipIf(is_jython, "FIXME #1861: not working in Jython")
     def test_round_large(self):
         # Issue #1869: integral floats should remain unchanged
         self.assertEqual(round(5e15-1), 5e15-1)
@@ -1387,7 +1384,6 @@
         b = 2
         return vars()
 
-    @unittest.skipIf(is_jython, "FIXME #1861: not working in Jython")
     def test_vars(self):
         self.assertEqual(set(vars()), set(dir()))
         import sys
@@ -1491,9 +1487,8 @@
             self.assertEqual(format(DerivedFromSimple2(10), 'abcdef'),
                              '10abcdef')
 
-        if not is_jython: #FIXME #1861 check again when __format__ works better.
-            class_test(*classes_new())
-            class_test(*classes_classic())
+        class_test(*classes_new())
+        class_test(*classes_classic())
 
         def empty_format_spec(value):
             # test that:
diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py
--- a/Lib/test/test_format.py
+++ b/Lib/test/test_format.py
@@ -1,14 +1,17 @@
-from test.test_support import verbose, have_unicode, TestFailed, is_jython
 import sys
+from test.test_support import verbose, have_unicode, TestFailed
+from test.test_support import is_jython
+import test.test_support as test_support
+import unittest
+
+maxsize = test_support.MAX_Py_ssize_t
 
 # test string formatting operator (I am not sure if this is being tested
 # elsewhere but, surely, some of the given cases are *not* tested because
 # they crash python)
 # test on unicode strings as well
 
-overflowok = 1
-
-def testformat(formatstr, args, output=None):
+def testformat(formatstr, args, output=None, limit=None, overflowok=False):
     if verbose:
         if output:
             print "%s %% %s =? %s ..." %\
@@ -23,231 +26,289 @@
         if verbose:
             print 'overflow (this is fine)'
     else:
-        if output and result != output:
+        if output and limit is None and result != output:
             if verbose:
                 print 'no'
-            print "%s %% %s == %s != %s" %\
-                (repr(formatstr), repr(args), repr(result), repr(output))
+            raise AssertionError("%r %% %r == %r != %r" %
+                                (formatstr, args, result, output))
+        # when 'limit' is specified, it determines how many characters
+        # must match exactly; lengths must always match.
+        # ex: limit=5, '12345678' matches '12345___'
+        # (mainly for floating point format tests for which an exact match
+        # can't be guaranteed due to rounding and representation errors)
+        elif output and limit is not None and (
+                len(result)!=len(output) or result[:limit]!=output[:limit]):
+            if verbose:
+                print 'no'
+            print "%s %% %s == %s != %s" % \
+                  (repr(formatstr), repr(args), repr(result), repr(output))
         else:
             if verbose:
                 print 'yes'
 
-def testboth(formatstr, *args):
-    testformat(formatstr, *args)
+
+def testboth(formatstr, *args, **kwargs):
+    testformat(formatstr, *args, **kwargs)
     if have_unicode:
-        testformat(unicode(formatstr), *args)
+        testformat(unicode(formatstr), *args, **kwargs)
 
 
-testboth("%.1d", (1,), "1")
-testboth("%.*d", (sys.maxint,1))  # expect overflow
-testboth("%.100d", (1,), '0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001')
-testboth("%#.117x", (1,), '0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001')
-testboth("%#.118x", (1,), '0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001')
+class FormatTest(unittest.TestCase):
+    def test_format(self):
+        testboth("%.1d", (1,), "1")
+        testboth("%.*d", (sys.maxint,1), overflowok=True)  # expect overflow
+        testboth("%.100d", (1,), '00000000000000000000000000000000000000'
+                 '000000000000000000000000000000000000000000000000000000'
+                 '00000001', overflowok=True)
+        testboth("%#.117x", (1,), '0x00000000000000000000000000000000000'
+                 '000000000000000000000000000000000000000000000000000000'
+                 '0000000000000000000000000001',
+                 overflowok=True)
+        testboth("%#.118x", (1,), '0x00000000000000000000000000000000000'
+                 '000000000000000000000000000000000000000000000000000000'
+                 '00000000000000000000000000001',
+                 overflowok=True)
 
-testboth("%f", (1.0,), "1.000000")
-# these are trying to test the limits of the internal magic-number-length
-# formatting buffer, if that number changes then these tests are less
-# effective
-testboth("%#.*g", (109, -1.e+49/3.))
-testboth("%#.*g", (110, -1.e+49/3.))
-testboth("%#.*g", (110, -1.e+100/3.))
+        testboth("%f", (1.0,), "1.000000")
+        # these are trying to test the limits of the internal magic-number-length
+        # formatting buffer, if that number changes then these tests are less
+        # effective
+        testboth("%#.*g", (109, -1.e+49/3.))
+        testboth("%#.*g", (110, -1.e+49/3.))
+        testboth("%#.*g", (110, -1.e+100/3.))
 
-# test some ridiculously large precision, expect overflow
-testboth('%12.*f', (123456, 1.0))
+        # test some ridiculously large precision, expect overflow
+        # ... Jython remains consistent with the original comment.
+        testboth('%12.*f', (123456, 1.0), overflowok=is_jython)
 
-# Formatting of long integers. Overflow is not ok
-overflowok = 0
-testboth("%x", 10L, "a")
-testboth("%x", 100000000000L, "174876e800")
-testboth("%o", 10L, "12")
-testboth("%o", 100000000000L, "1351035564000")
-testboth("%d", 10L, "10")
-testboth("%d", 100000000000L, "100000000000")
+        # check for internal overflow validation on length of precision
+        # these tests should no longer cause overflow in Python
+        # 2.7/3.1 and later.
+        testboth("%#.*g", (110, -1.e+100/3.))
+        testboth("%#.*G", (110, -1.e+100/3.))
+        testboth("%#.*f", (110, -1.e+100/3.))
+        testboth("%#.*F", (110, -1.e+100/3.))
 
-big = 123456789012345678901234567890L
-testboth("%d", big, "123456789012345678901234567890")
-testboth("%d", -big, "-123456789012345678901234567890")
-testboth("%5d", -big, "-123456789012345678901234567890")
-testboth("%31d", -big, "-123456789012345678901234567890")
-testboth("%32d", -big, " -123456789012345678901234567890")
-testboth("%-32d", -big, "-123456789012345678901234567890 ")
-testboth("%032d", -big, "-0123456789012345678901234567890")
-testboth("%-032d", -big, "-123456789012345678901234567890 ")
-testboth("%034d", -big, "-000123456789012345678901234567890")
-testboth("%034d", big, "0000123456789012345678901234567890")
-testboth("%0+34d", big, "+000123456789012345678901234567890")
-testboth("%+34d", big, "   +123456789012345678901234567890")
-testboth("%34d", big, "    123456789012345678901234567890")
-testboth("%.2d", big, "123456789012345678901234567890")
-testboth("%.30d", big, "123456789012345678901234567890")
-testboth("%.31d", big, "0123456789012345678901234567890")
-testboth("%32.31d", big, " 0123456789012345678901234567890")
+        # Formatting of long integers. Overflow is not ok
+        testboth("%x", 10L, "a")
+        testboth("%x", 100000000000L, "174876e800")
+        testboth("%o", 10L, "12")
+        testboth("%o", 100000000000L, "1351035564000")
+        testboth("%d", 10L, "10")
+        testboth("%d", 100000000000L, "100000000000")
 
-big = 0x1234567890abcdef12345L  # 21 hex digits
-testboth("%x", big, "1234567890abcdef12345")
-testboth("%x", -big, "-1234567890abcdef12345")
-testboth("%5x", -big, "-1234567890abcdef12345")
-testboth("%22x", -big, "-1234567890abcdef12345")
-testboth("%23x", -big, " -1234567890abcdef12345")
-testboth("%-23x", -big, "-1234567890abcdef12345 ")
-testboth("%023x", -big, "-01234567890abcdef12345")
-testboth("%-023x", -big, "-1234567890abcdef12345 ")
-testboth("%025x", -big, "-0001234567890abcdef12345")
-testboth("%025x", big, "00001234567890abcdef12345")
-testboth("%0+25x", big, "+0001234567890abcdef12345")
-testboth("%+25x", big, "   +1234567890abcdef12345")
-testboth("%25x", big, "    1234567890abcdef12345")
-testboth("%.2x", big, "1234567890abcdef12345")
-testboth("%.21x", big, "1234567890abcdef12345")
-testboth("%.22x", big, "01234567890abcdef12345")
-testboth("%23.22x", big, " 01234567890abcdef12345")
-testboth("%-23.22x", big, "01234567890abcdef12345 ")
-testboth("%X", big, "1234567890ABCDEF12345")
-testboth("%#X", big, "0X1234567890ABCDEF12345")
-testboth("%#x", big, "0x1234567890abcdef12345")
-testboth("%#x", -big, "-0x1234567890abcdef12345")
-testboth("%#.23x", -big, "-0x001234567890abcdef12345")
-testboth("%#+.23x", big, "+0x001234567890abcdef12345")
-testboth("%# .23x", big, " 0x001234567890abcdef12345")
-testboth("%#+.23X", big, "+0X001234567890ABCDEF12345")
-testboth("%#-+.23X", big, "+0X001234567890ABCDEF12345")
-testboth("%#-+26.23X", big, "+0X001234567890ABCDEF12345")
-testboth("%#-+27.23X", big, "+0X001234567890ABCDEF12345 ")
-testboth("%#+27.23X", big, " +0X001234567890ABCDEF12345")
-# next one gets two leading zeroes from precision, and another from the
-# 0 flag and the width
-testboth("%#+027.23X", big, "+0X0001234567890ABCDEF12345")
-# same, except no 0 flag
-testboth("%#+27.23X", big, " +0X001234567890ABCDEF12345")
+        big = 123456789012345678901234567890L
+        testboth("%d", big, "123456789012345678901234567890")
+        testboth("%d", -big, "-123456789012345678901234567890")
+        testboth("%5d", -big, "-123456789012345678901234567890")
+        testboth("%31d", -big, "-123456789012345678901234567890")
+        testboth("%32d", -big, " -123456789012345678901234567890")
+        testboth("%-32d", -big, "-123456789012345678901234567890 ")
+        testboth("%032d", -big, "-0123456789012345678901234567890")
+        testboth("%-032d", -big, "-123456789012345678901234567890 ")
+        testboth("%034d", -big, "-000123456789012345678901234567890")
+        testboth("%034d", big, "0000123456789012345678901234567890")
+        testboth("%0+34d", big, "+000123456789012345678901234567890")
+        testboth("%+34d", big, "   +123456789012345678901234567890")
+        testboth("%34d", big, "    123456789012345678901234567890")
+        testboth("%.2d", big, "123456789012345678901234567890")
+        testboth("%.30d", big, "123456789012345678901234567890")
+        testboth("%.31d", big, "0123456789012345678901234567890")
+        testboth("%32.31d", big, " 0123456789012345678901234567890")
+        testboth("%d", float(big), "123456________________________", 6)
 
-big = 012345670123456701234567012345670L  # 32 octal digits
-testboth("%o", big, "12345670123456701234567012345670")
-testboth("%o", -big, "-12345670123456701234567012345670")
-testboth("%5o", -big, "-12345670123456701234567012345670")
-testboth("%33o", -big, "-12345670123456701234567012345670")
-testboth("%34o", -big, " -12345670123456701234567012345670")
-testboth("%-34o", -big, "-12345670123456701234567012345670 ")
-testboth("%034o", -big, "-012345670123456701234567012345670")
-testboth("%-034o", -big, "-12345670123456701234567012345670 ")
-testboth("%036o", -big, "-00012345670123456701234567012345670")
-testboth("%036o", big, "000012345670123456701234567012345670")
-testboth("%0+36o", big, "+00012345670123456701234567012345670")
-testboth("%+36o", big, "   +12345670123456701234567012345670")
-testboth("%36o", big, "    12345670123456701234567012345670")
-testboth("%.2o", big, "12345670123456701234567012345670")
-testboth("%.32o", big, "12345670123456701234567012345670")
-testboth("%.33o", big, "012345670123456701234567012345670")
-testboth("%34.33o", big, " 012345670123456701234567012345670")
-testboth("%-34.33o", big, "012345670123456701234567012345670 ")
-testboth("%o", big, "12345670123456701234567012345670")
-testboth("%#o", big, "012345670123456701234567012345670")
-testboth("%#o", -big, "-012345670123456701234567012345670")
-testboth("%#.34o", -big, "-0012345670123456701234567012345670")
-testboth("%#+.34o", big, "+0012345670123456701234567012345670")
-testboth("%# .34o", big, " 0012345670123456701234567012345670")
-testboth("%#+.34o", big, "+0012345670123456701234567012345670")
-testboth("%#-+.34o", big, "+0012345670123456701234567012345670")
-testboth("%#-+37.34o", big, "+0012345670123456701234567012345670  ")
-testboth("%#+37.34o", big, "  +0012345670123456701234567012345670")
-# next one gets one leading zero from precision
-testboth("%.33o", big, "012345670123456701234567012345670")
-# base marker shouldn't change that, since "0" is redundant
-testboth("%#.33o", big, "012345670123456701234567012345670")
-# but reduce precision, and base marker should add a zero
-testboth("%#.32o", big, "012345670123456701234567012345670")
-# one leading zero from precision, and another from "0" flag & width
-testboth("%034.33o", big, "0012345670123456701234567012345670")
-# base marker shouldn't change that
-testboth("%0#34.33o", big, "0012345670123456701234567012345670")
+        big = 0x1234567890abcdef12345L  # 21 hex digits
+        testboth("%x", big, "1234567890abcdef12345")
+        testboth("%x", -big, "-1234567890abcdef12345")
+        testboth("%5x", -big, "-1234567890abcdef12345")
+        testboth("%22x", -big, "-1234567890abcdef12345")
+        testboth("%23x", -big, " -1234567890abcdef12345")
+        testboth("%-23x", -big, "-1234567890abcdef12345 ")
+        testboth("%023x", -big, "-01234567890abcdef12345")
+        testboth("%-023x", -big, "-1234567890abcdef12345 ")
+        testboth("%025x", -big, "-0001234567890abcdef12345")
+        testboth("%025x", big, "00001234567890abcdef12345")
+        testboth("%0+25x", big, "+0001234567890abcdef12345")
+        testboth("%+25x", big, "   +1234567890abcdef12345")
+        testboth("%25x", big, "    1234567890abcdef12345")
+        testboth("%.2x", big, "1234567890abcdef12345")
+        testboth("%.21x", big, "1234567890abcdef12345")
+        testboth("%.22x", big, "01234567890abcdef12345")
+        testboth("%23.22x", big, " 01234567890abcdef12345")
+        testboth("%-23.22x", big, "01234567890abcdef12345 ")
+        testboth("%X", big, "1234567890ABCDEF12345")
+        testboth("%#X", big, "0X1234567890ABCDEF12345")
+        testboth("%#x", big, "0x1234567890abcdef12345")
+        testboth("%#x", -big, "-0x1234567890abcdef12345")
+        testboth("%#.23x", -big, "-0x001234567890abcdef12345")
+        testboth("%#+.23x", big, "+0x001234567890abcdef12345")
+        testboth("%# .23x", big, " 0x001234567890abcdef12345")
+        testboth("%#+.23X", big, "+0X001234567890ABCDEF12345")
+        testboth("%#-+.23X", big, "+0X001234567890ABCDEF12345")
+        testboth("%#-+26.23X", big, "+0X001234567890ABCDEF12345")
+        testboth("%#-+27.23X", big, "+0X001234567890ABCDEF12345 ")
+        testboth("%#+27.23X", big, " +0X001234567890ABCDEF12345")
+        # next one gets two leading zeroes from precision, and another from the
+        # 0 flag and the width
+        testboth("%#+027.23X", big, "+0X0001234567890ABCDEF12345")
+        # same, except no 0 flag
+        testboth("%#+27.23X", big, " +0X001234567890ABCDEF12345")
+        testboth("%x", float(big), "123456_______________", 6)
 
-# Some small ints, in both Python int and long flavors).
-testboth("%d", 42, "42")
-testboth("%d", -42, "-42")
-testboth("%d", 42L, "42")
-testboth("%d", -42L, "-42")
-testboth("%#x", 1, "0x1")
-testboth("%#x", 1L, "0x1")
-testboth("%#X", 1, "0X1")
-testboth("%#X", 1L, "0X1")
-testboth("%#o", 1, "01")
-testboth("%#o", 1L, "01")
-testboth("%#o", 0, "0")
-testboth("%#o", 0L, "0")
-testboth("%o", 0, "0")
-testboth("%o", 0L, "0")
-testboth("%d", 0, "0")
-testboth("%d", 0L, "0")
-testboth("%#x", 0, "0x0")
-testboth("%#x", 0L, "0x0")
-testboth("%#X", 0, "0X0")
-testboth("%#X", 0L, "0X0")
+        big = 012345670123456701234567012345670L  # 32 octal digits
+        testboth("%o", big, "12345670123456701234567012345670")
+        testboth("%o", -big, "-12345670123456701234567012345670")
+        testboth("%5o", -big, "-12345670123456701234567012345670")
+        testboth("%33o", -big, "-12345670123456701234567012345670")
+        testboth("%34o", -big, " -12345670123456701234567012345670")
+        testboth("%-34o", -big, "-12345670123456701234567012345670 ")
+        testboth("%034o", -big, "-012345670123456701234567012345670")
+        testboth("%-034o", -big, "-12345670123456701234567012345670 ")
+        testboth("%036o", -big, "-00012345670123456701234567012345670")
+        testboth("%036o", big, "000012345670123456701234567012345670")
+        testboth("%0+36o", big, "+00012345670123456701234567012345670")
+        testboth("%+36o", big, "   +12345670123456701234567012345670")
+        testboth("%36o", big, "    12345670123456701234567012345670")
+        testboth("%.2o", big, "12345670123456701234567012345670")
+        testboth("%.32o", big, "12345670123456701234567012345670")
+        testboth("%.33o", big, "012345670123456701234567012345670")
+        testboth("%34.33o", big, " 012345670123456701234567012345670")
+        testboth("%-34.33o", big, "012345670123456701234567012345670 ")
+        testboth("%o", big, "12345670123456701234567012345670")
+        testboth("%#o", big, "012345670123456701234567012345670")
+        testboth("%#o", -big, "-012345670123456701234567012345670")
+        testboth("%#.34o", -big, "-0012345670123456701234567012345670")
+        testboth("%#+.34o", big, "+0012345670123456701234567012345670")
+        testboth("%# .34o", big, " 0012345670123456701234567012345670")
+        testboth("%#+.34o", big, "+0012345670123456701234567012345670")
+        testboth("%#-+.34o", big, "+0012345670123456701234567012345670")
+        testboth("%#-+37.34o", big, "+0012345670123456701234567012345670  ")
+        testboth("%#+37.34o", big, "  +0012345670123456701234567012345670")
+        # next one gets one leading zero from precision
+        testboth("%.33o", big, "012345670123456701234567012345670")
+        # base marker shouldn't change that, since "0" is redundant
+        testboth("%#.33o", big, "012345670123456701234567012345670")
+        # but reduce precision, and base marker should add a zero
+        testboth("%#.32o", big, "012345670123456701234567012345670")
+        # one leading zero from precision, and another from "0" flag & width
+        testboth("%034.33o", big, "0012345670123456701234567012345670")
+        # base marker shouldn't change that
+        testboth("%0#34.33o", big, "0012345670123456701234567012345670")
+        testboth("%o", float(big), "123456__________________________", 6)
 
-testboth("%x", 0x42, "42")
-testboth("%x", -0x42, "-42")
-testboth("%x", 0x42L, "42")
-testboth("%x", -0x42L, "-42")
+        # Some small ints, in both Python int and long flavors).
+        testboth("%d", 42, "42")
+        testboth("%d", -42, "-42")
+        testboth("%d", 42L, "42")
+        testboth("%d", -42L, "-42")
+        testboth("%d", 42.0, "42")
+        testboth("%#x", 1, "0x1")
+        testboth("%#x", 1L, "0x1")
+        testboth("%#X", 1, "0X1")
+        testboth("%#X", 1L, "0X1")
+        testboth("%#x", 1.0, "0x1")
+        testboth("%#o", 1, "01")
+        testboth("%#o", 1L, "01")
+        testboth("%#o", 0, "0")
+        testboth("%#o", 0L, "0")
+        testboth("%o", 0, "0")
+        testboth("%o", 0L, "0")
+        testboth("%d", 0, "0")
+        testboth("%d", 0L, "0")
+        testboth("%#x", 0, "0x0")
+        testboth("%#x", 0L, "0x0")
+        testboth("%#X", 0, "0X0")
+        testboth("%#X", 0L, "0X0")
 
-testboth("%o", 042, "42")
-testboth("%o", -042, "-42")
-testboth("%o", 042L, "42")
-testboth("%o", -042L, "-42")
+        testboth("%x", 0x42, "42")
+        testboth("%x", -0x42, "-42")
+        testboth("%x", 0x42L, "42")
+        testboth("%x", -0x42L, "-42")
+        testboth("%x", float(0x42), "42")
 
-# Test exception for unknown format characters
-if verbose:
-    print 'Testing exceptions'
+        testboth("%o", 042, "42")
+        testboth("%o", -042, "-42")
+        testboth("%o", 042L, "42")
+        testboth("%o", -042L, "-42")
+        testboth("%o", float(042), "42")
 
-def test_exc(formatstr, args, exception, excmsg):
-    try:
-        testformat(formatstr, args)
-    except exception, exc:
-        if str(exc) == excmsg:
-            if verbose:
-                print "yes"
-        else:
-            if verbose: print 'no'
-            print 'Unexpected ', exception, ':', repr(str(exc))
-    except:
-        if verbose: print 'no'
-        print 'Unexpected exception'
-        raise
-    else:
-        raise TestFailed, 'did not get expected exception: %s' % excmsg
+        # alternate float formatting
+        testformat('%g', 1.1, '1.1')
+        testformat('%#g', 1.1, '1.10000')
 
-test_exc('abc %a', 1, ValueError,
-         "unsupported format character 'a' (0x61) at index 5")
-if have_unicode:
-    test_exc(unicode('abc %\u3000','raw-unicode-escape'), 1, ValueError,
-             "unsupported format character '?' (0x3000) at index 5")
+        # Regression test for http://bugs.python.org/issue15516.
+        class IntFails(object):
+            def __int__(self):
+                raise TestFailed
+            def __long__(self):
+                return 0
 
-test_exc('%d', '1', TypeError, "int argument required")
-test_exc('%g', '1', TypeError, "float argument required")
-test_exc('no format', '1', TypeError,
-         "not all arguments converted during string formatting")
-test_exc('no format', u'1', TypeError,
-         "not all arguments converted during string formatting")
-test_exc(u'no format', '1', TypeError,
-         "not all arguments converted during string formatting")
-test_exc(u'no format', u'1', TypeError,
-         "not all arguments converted during string formatting")
+        fst = IntFails()
+        testformat("%x", fst, '0')
 
-# for Jython, do we really need to support this? what's the use case
-# here!  the problem in a nutshell is that it changes __oct__, __hex__
-# such that they don't return a string, but later on the exception
-# will occur anyway. so seems like a lot of work for no value
+        # Test exception for unknown format characters
+        if verbose:
+            print 'Testing exceptions'
 
-# class Foobar(long):
-#     def __oct__(self):
-#         # Returning a non-string should not blow up.
-#         return self + 1
+        def test_exc(formatstr, args, exception, excmsg):
+            try:
+                testformat(formatstr, args)
+            except exception, exc:
+                if str(exc) == excmsg:
+                    if verbose:
+                        print "yes"
+                else:
+                    if verbose: print 'no'
+                    print 'Unexpected ', exception, ':', repr(str(exc))
+            except:
+                if verbose: print 'no'
+                print 'Unexpected exception'
+                raise
+            else:
+                raise TestFailed, 'did not get expected exception: %s' % excmsg
 
-#test_exc('%o', Foobar(), TypeError,
-#         "expected string or Unicode object, long found")
+        test_exc('abc %a', 1, ValueError,
+                 "unsupported format character 'a' (0x61) at index 5")
+        if have_unicode:
+            test_exc(unicode('abc %\u3000','raw-unicode-escape'), 1, ValueError,
+                     "unsupported format character '?' (0x3000) at index 5")
 
-if sys.maxint == 2**31-1 and not is_jython:
-    # crashes 2.2.1 and earlier:
-    try:
-        "%*d"%(sys.maxint, -127)
-    except MemoryError:
-        pass
-    else:
-        raise TestFailed, '"%*d"%(sys.maxint, -127) should fail'
+        test_exc('%d', '1', TypeError, "%d format: a number is required, not str")
+        test_exc('%g', '1', TypeError, "float argument required, not str")
+        test_exc('no format', '1', TypeError,
+                 "not all arguments converted during string formatting")
+        test_exc('no format', u'1', TypeError,
+                 "not all arguments converted during string formatting")
+        test_exc(u'no format', '1', TypeError,
+                 "not all arguments converted during string formatting")
+        test_exc(u'no format', u'1', TypeError,
+                 "not all arguments converted during string formatting")
+
+        # For Jython, we do not support this use case. The test aims at the,
+        # use of __oct__ within %o formatting of long. (Or __hex__ within %x
+        # formatting?) CPython does this for long (not int) and has dropped
+        # the idea again by v3. Jython's %o and %x are likewise direct.
+        class Foobar(long):
+            def __oct__(self):
+                # Returning a non-string should not blow up.
+                return self + 1
+
+        if not is_jython :
+            test_exc('%o', Foobar(), TypeError,
+                     "expected string or Unicode object, long found")
+
+        if maxsize == 2**31-1:
+            # crashes 2.2.1 and earlier:
+            try:
+                "%*d"%(maxsize, -127)
+            except MemoryError:
+                pass
+            else:
+                raise TestFailed, '"%*d"%(maxsize, -127) should fail'
+
+def test_main():
+    test_support.run_unittest(FormatTest)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/Lib/test/test_format_jy.py b/Lib/test/test_format_jy.py
--- a/Lib/test/test_format_jy.py
+++ b/Lib/test/test_format_jy.py
@@ -5,8 +5,9 @@
 from test import test_support
 import unittest
 
-class FormatTestCase(unittest.TestCase):
-    # Tests that %d converts values for custom classes implementing __int__
+class FormatSubclass(unittest.TestCase):
+    # Custom __int__ and __float__ should be respected by %-formatting
+
     def test_int_conversion_support(self):
         class Foo(object):
             def __init__(self, x): self.x = x
@@ -21,9 +22,71 @@
             def __float__(self): return self. x
         self.assertEqual('1.0', '%.1f' % Foo(1.0))
 
+class FormatUnicodeBase(unittest.TestCase):
+
+    # Test padding non-BMP result
+    def test_pad_string(self):
+        self.padcheck(u"architect")
+        self.padcheck(u'a\U00010001cde')
+
+class FormatUnicodeClassic(FormatUnicodeBase):
+    # Check using %-formatting
+
+    def padcheck(self, s):
+        self.assertEqual(10, len('%10.4s' % s))
+        self.assertEqual(u' '*6 + s[0:4], '%10.4s' % s)
+        self.assertEqual(u' '*6 + s[0:4], '% 10.4s' % s)
+        self.assertEqual(u' '*6 + s[0:4], '%010.4s' % s)
+        self.assertEqual(s[0:3] + u' '*5, '%-8.3s' % s)
+
+class FormatUnicodeModern(FormatUnicodeBase):
+    # Check using __format__
+
+    def padcheck(self, s):
+        self.assertEqual(10, len(format(s, '10.4s')))
+        self.assertEqual(s[0:3] + u' '*7, format(s, '10.3s'))
+        self.assertEqual(s[0:3] + u'~'*7, format(s, '~<10.3s'))
+        self.assertEqual(s[0:3] + u'~'*7, format(s, '~<10.3'))
+        self.assertEqual(u' '*6 + s[0:4], format(s, '>10.4s'))
+        self.assertEqual(u'*'*6 + s[0:4], format(s, '*>10.4s'))
+        self.assertEqual(u'*'*6 + s[0:4], format(s, '*>10.4'))
+
+
+class FormatMisc(unittest.TestCase):
+    # Odd tests Jython used to fail
+
+    def test_mixtures(self) :
+        # Check formatting to a common buffer in PyString
+        result = 'The cube of 0.5 -0.866j is -1 to 0.01%.'
+        self.assertEqual(result, 'The %s of %.3g -%.3fj is -%d to %.2f%%.' %
+                          ('cube', 0.5, 0.866, 1, 0.01))
+        self.assertEqual(result, 'The %s of %.3g %.3fj is %d to %.2f%%.' %
+                          ('cube', 0.5, -0.866, -1, 0.01))
+        self.assertEqual(result, 'The%5s of%4.3g%7.3fj is%3d to%5.2f%%.' %
+                          ('cube', 0.5, -0.866, -1, 0.01))
+        self.assertEqual(result, 'The %-5.4sof %-4.3g%.3fj is %-3dto %.4g%%.' %
+                          ('cubensis', 0.5, -0.866, -1, 0.01))
+
+    def test_percent_padded(self) :
+        self.assertEqual('%hello', '%%%s' % 'hello')
+        self.assertEqual(u'     %hello', '%6%%s' % u'hello')
+        self.assertEqual(u'%     hello', u'%-6%%s' % 'hello')
+
+        self.assertEqual('     %', '%6%' % ())
+        self.assertEqual('     %', '%06%' % ())
+        self.assertEqual('   %', '%*%' % 4)
+        self.assertEqual('%     ', '%-6%' % ())
+        self.assertEqual('%     ', '%-06%' % ())
+        self.assertEqual('%   ', '%*%' % -4)
+
 
 def test_main():
-    test_support.run_unittest(FormatTestCase)
+    test_support.run_unittest(
+            FormatSubclass,
+            FormatUnicodeClassic,
+            FormatUnicodeModern,
+            FormatMisc,
+    )
 
 if __name__ == '__main__':
     test_main()
diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py
--- a/Lib/test/test_types.py
+++ b/Lib/test/test_types.py
@@ -1,7 +1,9 @@
 # Python test set -- part 6, built-in types
 
 from test.test_support import run_unittest, have_unicode, run_with_locale, \
-                              check_py3k_warnings, is_jython
+                              check_py3k_warnings
+from test.test_support import is_jython
+
 import unittest
 import sys
 import locale
@@ -90,7 +92,6 @@
         if float(1) == 1.0 and float(-1) == -1.0 and float(0) == 0.0: pass
         else: self.fail('float() does not work properly')
 
-    @unittest.skipIf(is_jython, "FIXME: not working")
     def test_float_to_string(self):
         def test(f, result):
             self.assertEqual(f.__format__('e'), result)
@@ -407,8 +408,7 @@
         test(-123456, "#012X", '-0X00001E240')
 
         # issue 5782, commas with no specifier type
-        #FIXME: not working.
-        #test(1234, '010,', '00,001,234')
+        test(1234, '010,', '00,001,234')
 
         # make sure these are errors
 
@@ -424,21 +424,19 @@
         self.assertRaises(ValueError, 3 .__format__, ",c")
 
         # ensure that only int and float type specifiers work
-        #FIXME: not working.
-        #for format_spec in ([chr(x) for x in range(ord('a'), ord('z')+1)] +
-        #                    [chr(x) for x in range(ord('A'), ord('Z')+1)]):
-        #    if not format_spec in 'bcdoxXeEfFgGn%':
-        #        self.assertRaises(ValueError, 0 .__format__, format_spec)
-        #        self.assertRaises(ValueError, 1 .__format__, format_spec)
-        #        self.assertRaises(ValueError, (-1) .__format__, format_spec)
+        for format_spec in ([chr(x) for x in range(ord('a'), ord('z')+1)] +
+                            [chr(x) for x in range(ord('A'), ord('Z')+1)]):
+            if not format_spec in 'bcdoxXeEfFgGn%':
+                self.assertRaises(ValueError, 0 .__format__, format_spec)
+                self.assertRaises(ValueError, 1 .__format__, format_spec)
+                self.assertRaises(ValueError, (-1) .__format__, format_spec)
 
         # ensure that float type specifiers work; format converts
         #  the int to a float
-        #FIXME: not working.
-        #for format_spec in 'eEfFgG%':
-        #    for value in [0, 1, -1, 100, -100, 1234567890, -1234567890]:
-        #        self.assertEqual(value.__format__(format_spec),
-        #                         float(value).__format__(format_spec))
+        for format_spec in 'eEfFgG%':
+            for value in [0, 1, -1, 100, -100, 1234567890, -1234567890]:
+                self.assertEqual(value.__format__(format_spec),
+                                 float(value).__format__(format_spec))
 
         # Issue 6902
         test(123456, "0<20", '12345600000000000000')
@@ -534,23 +532,20 @@
         self.assertRaises(ValueError, 1L .__format__, "#+5x")
         self.assertRaises(ValueError, 1L .__format__, "+5#x")
 
-        #FIXME: this section broken in Jython.
         # ensure that only int and float type specifiers work
-        #for format_spec in ([chr(x) for x in range(ord('a'), ord('z')+1)] +
-        #                    [chr(x) for x in range(ord('A'), ord('Z')+1)]):
-        #    if not format_spec in 'bcdoxXeEfFgGn%':
-        #        self.assertRaises(ValueError, 0L .__format__, format_spec)
-        #        self.assertRaises(ValueError, 1L .__format__, format_spec)
-        #        self.assertRaises(ValueError, (-1L) .__format__, format_spec)
+        for format_spec in ([chr(x) for x in range(ord('a'), ord('z')+1)] +
+                            [chr(x) for x in range(ord('A'), ord('Z')+1)]):
+            if not format_spec in 'bcdoxXeEfFgGn%':
+                self.assertRaises(ValueError, 0L .__format__, format_spec)
+                self.assertRaises(ValueError, 1L .__format__, format_spec)
+                self.assertRaises(ValueError, (-1L) .__format__, format_spec)
 
         # ensure that float type specifiers work; format converts
         #  the long to a float
-
-        #FIXME: this section broken in Jython.
-        #for format_spec in 'eEfFgG%':
-        #    for value in [0L, 1L, -1L, 100L, -100L, 1234567890L, -1234567890L]:
-        #        self.assertEqual(value.__format__(format_spec),
-        #                         float(value).__format__(format_spec))
+        for format_spec in 'eEfFgG%':
+            for value in [0L, 1L, -1L, 100L, -100L, 1234567890L, -1234567890L]:
+                self.assertEqual(value.__format__(format_spec),
+                                 float(value).__format__(format_spec))
         # Issue 6902
         test(123456L, "0<20", '12345600000000000000')
         test(123456L, "1<20", '12345611111111111111')
@@ -562,7 +557,6 @@
         test(123456L, "1=20", '11111111111111123456')
         test(123456L, "*=20", '**************123456')
 
-    @unittest.skipIf(is_jython, "FIXME: not working")
     @run_with_locale('LC_NUMERIC', 'en_US.UTF8')
     def test_float__format__locale(self):
         # test locale support for __format__ code 'n'
@@ -576,13 +570,12 @@
     def test_int__format__locale(self):
         # test locale support for __format__ code 'n' for integers
 
-        #FIXME: not working in Jython.
-        #x = 123456789012345678901234567890
-        #for i in range(0, 30):
-        #    self.assertEqual(locale.format('%d', x, grouping=True), format(x, 'n'))
+        x = 123456789012345678901234567890
+        for i in range(0, 30):
+            self.assertEqual(locale.format('%d', x, grouping=True), format(x, 'n'))
 
-        #    # move to the next integer to test
-        #    x = x // 10
+            # move to the next integer to test
+            x = x // 10
 
         rfmt = ">20n"
         lfmt = "<20n"
@@ -661,30 +654,25 @@
         # a totaly empty format specifier means something else.
         # So, just use a sign flag
         test(1e200, '+g', '+1e+200')
-        #FIXME: not working.
-        #test(1e200, '+', '+1e+200')
+        test(1e200, '+', '+1e+200')
         test(1.1e200, '+g', '+1.1e+200')
-        #FIXME: not working.
-        ##test(1.1e200, '+', '+1.1e+200')
+        test(1.1e200, '+', '+1.1e+200')
 
         test(1.1e200, '+g', '+1.1e+200')
-        #FIXME: not working.
-        #test(1.1e200, '+', '+1.1e+200')
+        test(1.1e200, '+', '+1.1e+200')
 
         # 0 padding
         test(1234., '010f', '1234.000000')
         test(1234., '011f', '1234.000000')
         test(1234., '012f', '01234.000000')
         test(-1234., '011f', '-1234.000000')
-        #FIXME: not working.
-        #test(-1234., '012f', '-1234.000000')
-        #test(-1234., '013f', '-01234.000000')
-        #test(-1234.12341234, '013f', '-01234.123412')
-        #test(-123456.12341234, '011.2f', '-0123456.12')
+        test(-1234., '012f', '-1234.000000')
+        test(-1234., '013f', '-01234.000000')
+        test(-1234.12341234, '013f', '-01234.123412')
+        test(-123456.12341234, '011.2f', '-0123456.12')
 
         # issue 5782, commas with no specifier type
-        #FIXME: not working.
-        #test(1.2, '010,.2', '0,000,001.2')
+        test(1.2, '010,.2', '0,000,001.2')
 
         # 0 padding with commas
         test(1234., '011,f', '1,234.000000')
@@ -692,13 +680,12 @@
         test(1234., '013,f', '01,234.000000')
         test(-1234., '012,f', '-1,234.000000')
         test(-1234., '013,f', '-1,234.000000')
-        #FIXME: not working.
-        #test(-1234., '014,f', '-01,234.000000')
-        #test(-12345., '015,f', '-012,345.000000')
-        #test(-123456., '016,f', '-0,123,456.000000')
-        #test(-123456., '017,f', '-0,123,456.000000')
-        #test(-123456.12341234, '017,f', '-0,123,456.123412')
-        #test(-123456.12341234, '013,.2f', '-0,123,456.12')
+        test(-1234., '014,f', '-01,234.000000')
+        test(-12345., '015,f', '-012,345.000000')
+        test(-123456., '016,f', '-0,123,456.000000')
+        test(-123456., '017,f', '-0,123,456.000000')
+        test(-123456.12341234, '017,f', '-0,123,456.123412')
+        test(-123456.12341234, '013,.2f', '-0,123,456.12')
 
          # % formatting
         test(-1.0, '%', '-100.000000%')
@@ -721,23 +708,20 @@
                 self.assertRaises(ValueError, format, -1e-100, format_spec)
 
         # Alternate formatting is not supported
-        #FIXME: not working.
-        ##self.assertRaises(ValueError, format, 0.0, '#')
+        self.assertRaises(ValueError, format, 0.0, '#')
         self.assertRaises(ValueError, format, 0.0, '#20f')
 
         # Issue 6902
-        #FIXME: not working.
-        #test(12345.6, "0<20", '12345.60000000000000')
-        #test(12345.6, "1<20", '12345.61111111111111')
-        #test(12345.6, "*<20", '12345.6*************')
-        #test(12345.6, "0>20", '000000000000012345.6')
-        #test(12345.6, "1>20", '111111111111112345.6')
-        #test(12345.6, "*>20", '*************12345.6')
-        #test(12345.6, "0=20", '000000000000012345.6')
-        #test(12345.6, "1=20", '111111111111112345.6')
-        #test(12345.6, "*=20", '*************12345.6')
+        test(12345.6, "0<20", '12345.60000000000000')
+        test(12345.6, "1<20", '12345.61111111111111')
+        test(12345.6, "*<20", '12345.6*************')
+        test(12345.6, "0>20", '000000000000012345.6')
+        test(12345.6, "1>20", '111111111111112345.6')
+        test(12345.6, "*>20", '*************12345.6')
+        test(12345.6, "0=20", '000000000000012345.6')
+        test(12345.6, "1=20", '111111111111112345.6')
+        test(12345.6, "*=20", '*************12345.6')
 
-    @unittest.skipIf(is_jython, "FIXME: not working")
     def test_format_spec_errors(self):
         # int, float, and string all share the same format spec
         # mini-language parser.
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -357,13 +357,12 @@
         self.assertEqual(u"%s, %s, %i, %f, %5.2f" % (u"abc", "abc", -1, -2, 3.5), u'abc, abc, -1, -2.000000,  3.50')
         self.assertEqual(u"%s, %s, %i, %f, %5.2f" % (u"abc", "abc", -1, -2, 3.57), u'abc, abc, -1, -2.000000,  3.57')
         self.assertEqual(u"%s, %s, %i, %f, %5.2f" % (u"abc", "abc", -1, -2, 1003.57), u'abc, abc, -1, -2.000000, 1003.57')
-        if not sys.platform.startswith('java'):
-            self.assertEqual(u"%r, %r" % (u"abc", "abc"), u"u'abc', 'abc'")
+        self.assertEqual(u"%r, %r" % (u"abc", "abc"), u"u'abc', 'abc'")
         self.assertEqual(u"%(x)s, %(y)s" % {'x':u"abc", 'y':"def"}, u'abc, def')
         self.assertEqual(u"%(x)s, %(\xfc)s" % {'x':u"abc", u'\xfc':"def"}, u'abc, def')
 
-        # self.assertEqual(u'%c' % 0x1234, u'\u1234')
-        # self.assertRaises(OverflowError, u"%c".__mod__, (sys.maxunicode+1,))
+        self.assertEqual(u'%c' % 0x1234, u'\u1234')
+        self.assertRaises(OverflowError, u"%c".__mod__, (sys.maxunicode+1,))
 
         # formatting jobs delegated from the string implementation:
         self.assertEqual('...%(foo)s...' % {'foo':u"abc"}, u'...abc...')
diff --git a/src/org/python/core/PyComplex.java b/src/org/python/core/PyComplex.java
--- a/src/org/python/core/PyComplex.java
+++ b/src/org/python/core/PyComplex.java
@@ -4,6 +4,7 @@
 
 import org.python.core.stringlib.FloatFormatter;
 import org.python.core.stringlib.InternalFormat;
+import org.python.core.stringlib.InternalFormat.Formatter;
 import org.python.core.stringlib.InternalFormat.Spec;
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedMethod;
@@ -174,7 +175,9 @@
      * @return formatted value
      */
     private String formatComplex(Spec spec) {
-        FloatFormatter f = new FloatFormatter(spec, 2, 3); // Two elements + "(j)".length
+        int size = 2 * FloatFormatter.size(spec) + 3;  // 2 floats + "(j)"
+        FloatFormatter f = new FloatFormatter(new StringBuilder(size), spec);
+        f.setBytes(true);
         // Even in r-format, complex strips *all* the trailing zeros.
         f.setMinFracDigits(0);
         if (Double.doubleToLongBits(real) == 0L) {
@@ -816,42 +819,87 @@
 
     @ExposedMethod(doc = BuiltinDocs.complex___format___doc)
     final PyObject complex___format__(PyObject formatSpec) {
-        if (!(formatSpec instanceof PyString)) {
-            throw Py.TypeError("__format__ requires str or unicode");
+
+        // Parse the specification
+        Spec spec = InternalFormat.fromText(formatSpec, "__format__");
+
+        // fromText will have thrown if formatSpecStr is not a PyString (including PyUnicode)
+        PyString formatSpecStr = (PyString)formatSpec;
+        String result;
+
+        // Validate the specification and detect the special case for none-format
+        switch (checkSpecification(spec)) {
+
+            case 0: // None-format
+                // In none-format, we take the default type and precision from __str__.
+                spec = spec.withDefaults(SPEC_STR);
+                // And then we use the __str__ mechanism to get parentheses or real 0 elision.
+                result = formatComplex(spec);
+                break;
+
+            case 1: // Floating-point formats
+                // In any other format, defaults are those commonly used for numeric formats.
+                spec = spec.withDefaults(Spec.NUMERIC);
+                int size = 2 * FloatFormatter.size(spec) + 1; // 2 floats + "j"
+                FloatFormatter f = new FloatFormatter(new StringBuilder(size), spec);
+                f.setBytes(!(formatSpecStr instanceof PyUnicode));
+                // Convert both parts as per specification
+                f.format(real).format(imag, "+").append('j');
+                result = f.pad().getResult();
+                break;
+
+            default: // The type code was not recognised
+                throw Formatter.unknownFormat(spec.type, "complex");
         }
 
-        PyString formatSpecStr = (PyString)formatSpec;
-        String result;
-        try {
-            String specString = formatSpecStr.getString();
-            Spec spec = InternalFormat.fromText(specString);
-            if (spec.type != Spec.NONE && "efgEFGn%".indexOf(spec.type) < 0) {
-                throw FloatFormatter.unknownFormat(spec.type, "complex");
-            } else if (spec.alternate) {
-                throw FloatFormatter.alternateFormNotAllowed("complex");
-            } else if (spec.fill == '0') {
-                throw FloatFormatter.zeroPaddingNotAllowed("complex");
-            } else if (spec.align == '=') {
-                throw FloatFormatter.alignmentNotAllowed('=', "complex");
-            } else {
-                if (spec.type == Spec.NONE) {
-                    // In none-format, we take the default type and precision from __str__.
-                    spec = spec.withDefaults(SPEC_STR);
-                    // And then we use the __str__ mechanism to get parentheses or real 0 elision.
-                    result = formatComplex(spec);
+        // Wrap the result in the same type as the format string
+        return formatSpecStr.createInstance(result);
+    }
+
+    /**
+     * Validate a parsed specification, for PyComplex, returning 0 if it is a valid
+     * none-format specification, 1 if it is a valid float specification, and some other value if it
+     * not a valid type. If it has any other faults (e.g. alternate form was specified) the method
+     * raises a descriptive exception.
+     *
+     * @param spec a parsed PEP-3101 format specification.
+     * @return 0, 1, or other value for none-format, a float format, or incorrect type.
+     * @throws PyException(ValueError) if the specification is faulty.
+     */
+    @SuppressWarnings("fallthrough")
+    private static int checkSpecification(Spec spec) {
+
+        // Slight differences between format types
+        switch (spec.type) {
+
+            case 'n':
+                if (spec.grouping) {
+                    throw Formatter.notAllowed("Grouping", "complex", spec.type);
+                }
+                // Fall through
+
+            case Spec.NONE:
+            case 'e':
+            case 'f':
+            case 'g':
+            case 'E':
+            case 'F':
+            case 'G':
+                // Check for disallowed parts of the specification
+                if (spec.alternate) {
+                    throw FloatFormatter.alternateFormNotAllowed("complex");
+                } else if (spec.fill == '0') {
+                    throw FloatFormatter.zeroPaddingNotAllowed("complex");
+                } else if (spec.align == '=') {
+                    throw FloatFormatter.alignmentNotAllowed('=', "complex");
                 } else {
-                    // In any other format, defaults are those commonly used for numeric formats.
-                    spec = spec.withDefaults(Spec.NUMERIC);
-                    FloatFormatter f = new FloatFormatter(spec, 2, 1);// 2 floats + "j"
-                    // Convert both parts as per specification
-                    f.format(real).format(imag, "+").append('j');
-                    result = f.pad().getResult();
+                    return (spec.type == Spec.NONE) ? 0 : 1;
                 }
-            }
-        } catch (IllegalArgumentException e) {
-            throw Py.ValueError(e.getMessage());    // XXX Can this be reached?
+
+            default:
+                // spec.type is invalid for complex
+                return 2;
         }
-        return formatSpecStr.createInstance(result);
     }
 
     @Override
diff --git a/src/org/python/core/PyFloat.java b/src/org/python/core/PyFloat.java
--- a/src/org/python/core/PyFloat.java
+++ b/src/org/python/core/PyFloat.java
@@ -7,6 +7,7 @@
 
 import org.python.core.stringlib.FloatFormatter;
 import org.python.core.stringlib.InternalFormat;
+import org.python.core.stringlib.InternalFormat.Formatter;
 import org.python.core.stringlib.InternalFormat.Spec;
 import org.python.expose.ExposedClassMethod;
 import org.python.expose.ExposedGet;
@@ -911,32 +912,66 @@
 
     @ExposedMethod(doc = BuiltinDocs.float___format___doc)
     final PyObject float___format__(PyObject formatSpec) {
-        if (!(formatSpec instanceof PyString)) {
-            throw Py.TypeError("__format__ requires str or unicode");
+
+        // Parse the specification
+        Spec spec = InternalFormat.fromText(formatSpec, "__format__");
+
+        // Get a formatter for the specification
+        FloatFormatter f = prepareFormatter(spec);
+
+        if (f != null) {
+            // Bytes mode if formatSpec argument is not unicode.
+            f.setBytes(!(formatSpec instanceof PyUnicode));
+            // Convert as per specification.
+            f.format(value);
+            // Return a result that has the same type (str or unicode) as the formatSpec argument.
+            return f.pad().getPyResult();
+
+        } else {
+            // The type code was not recognised in prepareFormatter
+            throw Formatter.unknownFormat(spec.type, "float");
         }
+    }
 
-        PyString formatSpecStr = (PyString)formatSpec;
-        String result;
-        try {
-            String specString = formatSpecStr.getString();
-            Spec spec = InternalFormat.fromText(specString);
-            if (spec.type!=Spec.NONE && "efgEFGn%".indexOf(spec.type) < 0) {
-                throw FloatFormatter.unknownFormat(spec.type, "float");
-            } else if (spec.alternate) {
-                throw FloatFormatter.alternateFormNotAllowed("float");
-            } else {
+    /**
+     * Common code for PyFloat, {@link PyInteger} and {@link PyLong} to prepare a {@link FloatFormatter} from a parsed specification.
+     * The object returned has format method {@link FloatFormatter#format(double)}.
+     *
+     * @param spec a parsed PEP-3101 format specification.
+     * @return a formatter ready to use, or null if the type is not a floating point format type.
+     * @throws PyException(ValueError) if the specification is faulty.
+     */
+    @SuppressWarnings("fallthrough")
+    static FloatFormatter prepareFormatter(Spec spec) {
+
+        // Slight differences between format types
+        switch (spec.type) {
+
+            case 'n':
+                if (spec.grouping) {
+                    throw Formatter.notAllowed("Grouping", "float", spec.type);
+                }
+                // Fall through
+
+            case Spec.NONE:
+            case 'e':
+            case 'f':
+            case 'g':
+            case 'E':
+            case 'F':
+            case 'G':
+            case '%':
+                // Check for disallowed parts of the specification
+                if (spec.alternate) {
+                    throw FloatFormatter.alternateFormNotAllowed("float");
+                }
                 // spec may be incomplete. The defaults are those commonly used for numeric formats.
                 spec = spec.withDefaults(Spec.NUMERIC);
-                // Get a formatter for the spec.
-                FloatFormatter f = new FloatFormatter(spec);
-                // Convert as per specification.
-                f.format(value).pad();
-                result = f.getResult();
-            }
-        } catch (IllegalArgumentException e) {
-            throw Py.ValueError(e.getMessage());    // XXX Can this be reached?
+                return new FloatFormatter(spec);
+
+            default:
+                return null;
         }
-        return formatSpecStr.createInstance(result);
     }
 
     @ExposedMethod(doc = BuiltinDocs.float_as_integer_ratio_doc)
diff --git a/src/org/python/core/PyInteger.java b/src/org/python/core/PyInteger.java
--- a/src/org/python/core/PyInteger.java
+++ b/src/org/python/core/PyInteger.java
@@ -4,11 +4,12 @@
 
 import java.io.Serializable;
 import java.math.BigInteger;
-import java.text.NumberFormat;
-import java.util.Locale;
 
-import org.python.core.stringlib.InternalFormatSpec;
-import org.python.core.stringlib.InternalFormatSpecParser;
+import org.python.core.stringlib.FloatFormatter;
+import org.python.core.stringlib.IntegerFormatter;
+import org.python.core.stringlib.InternalFormat;
+import org.python.core.stringlib.InternalFormat.Formatter;
+import org.python.core.stringlib.InternalFormat.Spec;
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedNew;
@@ -953,11 +954,8 @@
 
     @ExposedMethod(doc = BuiltinDocs.int___oct___doc)
     final PyString int___oct__() {
-        if (getValue() < 0) {
-            return new PyString("-0" + Integer.toString(getValue() * -1, 8));
-        } else {
-            return new PyString("0" + Integer.toString(getValue(), 8));
-        }
+        // Use the prepared format specifier for octal.
+        return formatImpl(IntegerFormatter.OCT);
     }
 
     @Override
@@ -967,11 +965,21 @@
 
     @ExposedMethod(doc = BuiltinDocs.int___hex___doc)
     final PyString int___hex__() {
-        if (getValue() < 0) {
-            return new PyString("-0x" + Integer.toString(getValue() * -1, 16));
-        } else {
-            return new PyString("0x" + Integer.toString(getValue(), 16));
-        }
+        // Use the prepared format specifier for hexadecimal.
+        return formatImpl(IntegerFormatter.HEX);
+    }
+
+    /**
+     * Common code used by the number-base conversion method __oct__ and __hex__.
+     *
+     * @param spec prepared format-specifier.
+     * @return converted value of this object
+     */
+    private PyString formatImpl(Spec spec) {
+        // Traditional formatter (%-format) because #o means "-0123" not "-0o123".
+        IntegerFormatter f = new IntegerFormatter.Traditional(spec);
+        f.format(value);
+        return new PyString(f.getResult());
     }
 
     @ExposedMethod(doc = BuiltinDocs.int___getnewargs___doc)
@@ -1015,256 +1023,87 @@
 
     @ExposedMethod(doc = BuiltinDocs.int___format___doc)
     final PyObject int___format__(PyObject formatSpec) {
-        return formatImpl(getValue(), formatSpec);
-    }
 
-    static PyObject formatImpl(Object value, PyObject formatSpec) {
-        if (!(formatSpec instanceof PyString)) {
-            throw Py.TypeError("__format__ requires str or unicode");
+        // Parse the specification
+        Spec spec = InternalFormat.fromText(formatSpec, "__format__");
+        InternalFormat.Formatter f;
+
+        // Try to make an integer formatter from the specification
+        IntegerFormatter fi = PyInteger.prepareFormatter(spec);
+        if (fi != null) {
+            // Bytes mode if formatSpec argument is not unicode.
+            fi.setBytes(!(formatSpec instanceof PyUnicode));
+            // Convert as per specification.
+            fi.format(value);
+            f = fi;
+
+        } else {
+            // Try to make a float formatter from the specification
+            FloatFormatter ff = PyFloat.prepareFormatter(spec);
+            if (ff != null) {
+                // Bytes mode if formatSpec argument is not unicode.
+                ff.setBytes(!(formatSpec instanceof PyUnicode));
+                // Convert as per specification.
+                ff.format(value);
+                f = ff;
+
+            } else {
+                // The type code was not recognised in either prepareFormatter
+                throw Formatter.unknownFormat(spec.type, "integer");
+            }
         }
 
-        PyString formatSpecStr = (PyString)formatSpec;
-        String result;
-        try {
-            String specString = formatSpecStr.getString();
-            InternalFormatSpec spec = new InternalFormatSpecParser(specString).parse();
-            result = formatIntOrLong(value, spec);
-        } catch (IllegalArgumentException e) {
-            throw Py.ValueError(e.getMessage());
-        }
-        return formatSpecStr.createInstance(result);
+        // Return a result that has the same type (str or unicode) as the formatSpec argument.
+        return f.pad().getPyResult();
     }
 
     /**
-     * Formats an integer or long number according to a PEP-3101 format specification.
+     * Common code for PyInteger and PyLong to prepare an IntegerFormatter. This object has an
+     * overloaded format method {@link IntegerFormatter#format(int)} and
+     * {@link IntegerFormatter#format(BigInteger)} to support the two types.
      *
-     * @param value Integer or BigInteger object specifying the value to format.
-     * @param spec parsed PEP-3101 format specification.
-     * @return result of the formatting.
+     * @param spec a parsed PEP-3101 format specification.
+     * @return a formatter ready to use, or null if the type is not an integer format type.
+     * @throws PyException(ValueError) if the specification is faulty.
      */
-    public static String formatIntOrLong(Object value, InternalFormatSpec spec) {
-        if (spec.precision != -1) {
-            throw new IllegalArgumentException("Precision not allowed in integer format specifier");
+    @SuppressWarnings("fallthrough")
+    static IntegerFormatter prepareFormatter(Spec spec) throws PyException {
+
+        // Slight differences between format types
+        switch (spec.type) {
+            case 'c':
+                // Character data: specific prohibitions.
+                if (Spec.specified(spec.sign)) {
+                    throw IntegerFormatter.signNotAllowed("integer", spec.type);
+                } else if (spec.alternate) {
+                    throw IntegerFormatter.alternateFormNotAllowed("integer", spec.type);
+                }
+                // Fall through
+
+            case 'x':
+            case 'X':
+            case 'o':
+            case 'b':
+            case 'n':
+                if (spec.grouping) {
+                    throw IntegerFormatter.notAllowed("Grouping", "integer", spec.type);
+                }
+                // Fall through
+
+            case Spec.NONE:
+            case 'd':
+                // Check for disallowed parts of the specification
+                if (Spec.specified(spec.precision)) {
+                    throw IntegerFormatter.precisionNotAllowed("integer");
+                }
+                // spec may be incomplete. The defaults are those commonly used for numeric formats.
+                spec = spec.withDefaults(Spec.NUMERIC);
+                // Get a formatter for the spec.
+                return new IntegerFormatter(spec);
+
+            default:
+                return null;
         }
-
-        int sign;
-        if (value instanceof Integer) {
-            int intValue = (Integer)value;
-            sign = intValue < 0 ? -1 : intValue == 0 ? 0 : 1;
-        } else {
-            sign = ((BigInteger)value).signum();
-        }
-
-        String strValue;
-        String strPrefix = "";
-        String strSign = "";
-
-        if (spec.type == 'c') {
-            if (spec.sign != '\0') {
-                throw new IllegalArgumentException("Sign not allowed with integer format "
-                        + "specifier 'c'");
-            }
-            if (value instanceof Integer) {
-                int intValue = (Integer)value;
-                if (intValue > 0xffff) {
-                    throw new IllegalArgumentException("%c arg not in range(0x10000)");
-                }
-                strValue = Character.toString((char)intValue);
-            } else {
-                BigInteger bigInt = (BigInteger)value;
-                if (bigInt.intValue() > 0xffff || bigInt.bitCount() > 16) {
-                    throw new IllegalArgumentException("%c arg not in range(0x10000)");
-                }
-                strValue = Character.toString((char)bigInt.intValue());
-            }
-        } else {
-            int radix = 10;
-            if (spec.type == 'o') {
-                radix = 8;
-            } else if (spec.type == 'x' || spec.type == 'X') {
-                radix = 16;
-            } else if (spec.type == 'b') {
-                radix = 2;
-            }
-
-            if (spec.type == 'n') {
-                strValue = NumberFormat.getNumberInstance().format(value);
-            } else if (spec.thousands_separators) {
-                NumberFormat format = NumberFormat.getNumberInstance(Locale.US);
-                format.setGroupingUsed(true);
-                strValue = format.format(value);
-            } else if (value instanceof BigInteger) {
-                switch (radix) {
-                    case 2:
-                        strValue = toBinString((BigInteger)value);
-                        break;
-                    case 8:
-                        strValue = toOctString((BigInteger)value);
-                        break;
-                    case 16:
-                        strValue = toHexString((BigInteger)value);
-                        break;
-                    default:
-                        // General case (v.slow in known implementations up to Java 7).
-                        strValue = ((BigInteger)value).toString(radix);
-                        break;
-                }
-            } else {
-                strValue = Integer.toString((Integer)value, radix);
-            }
-
-            if (spec.alternate) {
-                switch (radix) {
-                    case 2:
-                        strPrefix = "0b";
-                        break;
-                    case 8:
-                        strPrefix = "0o";
-                        break;
-                    case 16:
-                        strPrefix = "0x";
-                        break;
-                }
-
-                if (sign < 0) {
-                    assert (strValue.startsWith("-"));
-                    strSign = "-";
-                    strValue = strValue.substring(1);
-                }
-            }
-
-            if (spec.type == 'X') {
-                strPrefix = strPrefix.toUpperCase();
-                strValue = strValue.toUpperCase();
-            }
-
-            if (sign >= 0) {
-                switch (spec.sign) {
-                    case '+':
-                    case ' ':
-                        strSign = Character.toString(spec.sign);
-                        break;
-                }
-            }
-        }
-
-        if (spec.align == '=' && (spec.sign == '-' || spec.sign == '+' || spec.sign == ' ')) {
-            assert (strSign.length() == 1);
-            return strSign + strPrefix + spec.pad(strValue, '>', 1 + strPrefix.length());
-        }
-
-        if (spec.fill_char == 0) {
-            return spec.pad(strSign + strPrefix + strValue, '>', 0);
-        }
-
-        return strSign + strPrefix + spec.pad(strValue, '>', strSign.length() + strPrefix.length());
-    }
-
-    /**
-     * A more efficient algorithm for generating a hexadecimal representation of a byte array.
-     * {@link BigInteger#toString(int)} is too slow because it generalizes to any radix and,
-     * consequently, is implemented using expensive mathematical operations.
-     *
-     * @param value the value to generate a hexadecimal string from
-     * @return the hexadecimal representation of value, with "-" sign prepended if necessary
-     */
-    static final String toHexString(BigInteger value) {
-        int signum = value.signum();
-
-        // obvious shortcut
-        if (signum == 0) {
-            return "0";
-        }
-
-        // we want to work in absolute numeric value (negative sign is added afterward)
-        byte[] input = value.abs().toByteArray();
-        StringBuilder sb = new StringBuilder(input.length * 2);
-
-        int b;
-        for (int i = 0; i < input.length; i++) {
-            b = input[i] & 0xFF;
-            sb.append(LOOKUP.charAt(b >> 4));
-            sb.append(LOOKUP.charAt(b & 0x0F));
-        }
-
-        // before returning the char array as string, remove leading zeroes, but not the last one
-        String result = sb.toString().replaceFirst("^0+(?!$)", "");
-        return signum < 0 ? "-" + result : result;
-    }
-
-    /**
-     * A more efficient algorithm for generating an octal representation of a byte array.
-     * {@link BigInteger#toString(int)} is too slow because it generalizes to any radix and,
-     * consequently, is implemented using expensive mathematical operations.
-     *
-     * @param value the value to generate an octal string from
-     * @return the octal representation of value, with "-" sign prepended if necessary
-     */
-    static final String toOctString(BigInteger value) {
-        int signum = value.signum();
-
-        // obvious shortcut
-        if (signum == 0) {
-            return "0";
-        }
-
-        byte[] input = value.abs().toByteArray();
-        if (input.length < 3) {
-            return value.toString(8);
-        }
-
-        StringBuilder sb = new StringBuilder(input.length * 3);
-
-        // working backwards, three bytes at a time
-        int threebytes;
-        int trip1, trip2, trip3;    // most, middle, and least significant bytes in the triplet
-        for (int i = input.length - 1; i >= 0; i -= 3) {
-            trip3 = input[i] & 0xFF;
-            trip2 = ((i - 1) >= 0) ? (input[i - 1] & 0xFF) : 0x00;
-            trip1 = ((i - 2) >= 0) ? (input[i - 2] & 0xFF) : 0x00;
-            threebytes = trip3 | (trip2 << 8) | (trip1 << 16);
-
-            // convert the three-byte value into an eight-character octal string
-            for (int j = 0; j < 8; j++) {
-                sb.append(LOOKUP.charAt((threebytes >> (j * 3)) & 0x000007));
-            }
-        }
-
-        String result = sb.reverse().toString().replaceFirst("^0+(?!%)", "");
-        return signum < 0 ? "-" + result : result;
-    }
-
-    /**
-     * A more efficient algorithm for generating a binary representation of a byte array.
-     * {@link BigInteger#toString(int)} is too slow because it generalizes to any radix and,
-     * consequently, is implemented using expensive mathematical operations.
-     *
-     * @param value the value to generate a binary string from
-     * @return the binary representation of value, with "-" sign prepended if necessary
-     */
-    static final String toBinString(BigInteger value) {
-        int signum = value.signum();
-
-        // obvious shortcut
-        if (signum == 0) {
-            return "0";
-        }
-
-        // we want to work in absolute numeric value (negative sign is added afterward)
-        byte[] input = value.abs().toByteArray();
-        StringBuilder sb = new StringBuilder(value.bitCount());
-
-        int b;
-        for (int i = 0; i < input.length; i++) {
-            b = input[i] & 0xFF;
-            for (int bit = 7; bit >= 0; bit--) {
-                sb.append(((b >> bit) & 0x1) > 0 ? "1" : "0");
-            }
-        }
-
-        // before returning the char array as string, remove leading zeroes, but not the last one
-        String result = sb.toString().replaceFirst("^0+(?!$)", "");
-        return signum < 0 ? "-" + result : result;
     }
 
     @Override
diff --git a/src/org/python/core/PyLong.java b/src/org/python/core/PyLong.java
--- a/src/org/python/core/PyLong.java
+++ b/src/org/python/core/PyLong.java
@@ -1,13 +1,17 @@
-/*
- * Copyright (c) Corporation for National Research Initiatives
- * Copyright (c) Jython Developers
- */
+// Copyright (c) Corporation for National Research Initiatives
+// Copyright (c) Jython Developers
+
 package org.python.core;
 
 import java.io.Serializable;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 
+import org.python.core.stringlib.FloatFormatter;
+import org.python.core.stringlib.IntegerFormatter;
+import org.python.core.stringlib.InternalFormat;
+import org.python.core.stringlib.InternalFormat.Formatter;
+import org.python.core.stringlib.InternalFormat.Spec;
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedNew;
@@ -24,8 +28,8 @@
 
     public static final BigInteger MIN_LONG = BigInteger.valueOf(Long.MIN_VALUE);
     public static final BigInteger MAX_LONG = BigInteger.valueOf(Long.MAX_VALUE);
-    public static final BigInteger MAX_ULONG =
-            BigInteger.valueOf(1).shiftLeft(64).subtract(BigInteger.valueOf(1));
+    public static final BigInteger MAX_ULONG = BigInteger.valueOf(1).shiftLeft(64)
+            .subtract(BigInteger.valueOf(1));
 
     /** @deprecated Use MIN_INT instead. */
     @Deprecated
@@ -66,7 +70,7 @@
 
     @ExposedNew
     public static PyObject long___new__(PyNewWrapper new_, boolean init, PyType subtype,
-                                        PyObject[] args, String[] keywords) {
+            PyObject[] args, String[] keywords) {
         if (new_.for_type != subtype) {
             return longSubtypeNew(new_, init, subtype, args, keywords);
         }
@@ -74,7 +78,7 @@
         ArgParser ap = new ArgParser("long", args, keywords, new String[] {"x", "base"}, 0);
         PyObject x = ap.getPyObject(0, null);
         if (x != null && x.getJavaProxy() instanceof BigInteger) {
-            return new PyLong((BigInteger) x.getJavaProxy());
+            return new PyLong((BigInteger)x.getJavaProxy());
         }
         int base = ap.getInt(1, -909);
 
@@ -87,7 +91,7 @@
         if (!(x instanceof PyString)) {
             throw Py.TypeError("long: can't convert non-string with explicit base");
         }
-        return ((PyString) x).atol(base);
+        return ((PyString)x).atol(base);
     }
 
     /**
@@ -108,8 +112,9 @@
                 if (!pye2.match(Py.AttributeError)) {
                     throw pye2;
                 }
-                throw Py.TypeError(
-                    String.format("long() argument must be a string or a number, not '%.200s'", x.getType().fastGetName()));
+                throw Py.TypeError(String.format(
+                        "long() argument must be a string or a number, not '%.200s'", x.getType()
+                                .fastGetName()));
             }
         }
     }
@@ -123,7 +128,7 @@
             PyObject i = integral.invoke("__int__");
             if (!(i instanceof PyInteger) && !(i instanceof PyLong)) {
                 throw Py.TypeError(String.format("__trunc__ returned non-Integral (type %.200s)",
-                                                 integral.getType().fastGetName()));
+                        integral.getType().fastGetName()));
             }
             return i;
         }
@@ -133,24 +138,22 @@
     /**
      * Wimpy, slow approach to new calls for subtypes of long.
      *
-     * First creates a regular long from whatever arguments we got, then allocates a
-     * subtype instance and initializes it from the regular long. The regular long is then
-     * thrown away.
+     * First creates a regular long from whatever arguments we got, then allocates a subtype
+     * instance and initializes it from the regular long. The regular long is then thrown away.
      */
     private static PyObject longSubtypeNew(PyNewWrapper new_, boolean init, PyType subtype,
-                                           PyObject[] args, String[] keywords) {
+            PyObject[] args, String[] keywords) {
         PyObject tmp = long___new__(new_, init, TYPE, args, keywords);
         if (tmp instanceof PyInteger) {
-            int intValue = ((PyInteger) tmp).getValue();
+            int intValue = ((PyInteger)tmp).getValue();
             return new PyLongDerived(subtype, BigInteger.valueOf(intValue));
         } else {
-            return new PyLongDerived(subtype, ((PyLong) tmp).getValue());
+            return new PyLongDerived(subtype, ((PyLong)tmp).getValue());
         }
     }
 
     /**
-     * Convert a double to BigInteger, raising an OverflowError if
-     * infinite.
+     * Convert a double to BigInteger, raising an OverflowError if infinite.
      */
     private static BigInteger toBigInteger(double value) {
         if (Double.isInfinite(value)) {
@@ -249,7 +252,7 @@
     }
 
     public double scaledDoubleValue(int[] exp) {
-        return scaledDoubleValue(getValue(),exp);
+        return scaledDoubleValue(getValue(), exp);
     }
 
     public long getLong(long min, long max) {
@@ -273,14 +276,14 @@
 
     @Override
     public int asInt(int index) {
-        return (int) getLong(Integer.MIN_VALUE, Integer.MAX_VALUE,
-                             "long int too large to convert to int");
+        return (int)getLong(Integer.MIN_VALUE, Integer.MAX_VALUE,
+                "long int too large to convert to int");
     }
 
     @Override
     public int asInt() {
-        return (int) getLong(Integer.MIN_VALUE, Integer.MAX_VALUE,
-                             "long int too large to convert to int");
+        return (int)getLong(Integer.MIN_VALUE, Integer.MAX_VALUE,
+                "long int too large to convert to int");
     }
 
     @Override
@@ -292,13 +295,13 @@
     public Object __tojava__(Class c) {
         try {
             if (c == Byte.TYPE || c == Byte.class) {
-                return new Byte((byte) getLong(Byte.MIN_VALUE, Byte.MAX_VALUE));
+                return new Byte((byte)getLong(Byte.MIN_VALUE, Byte.MAX_VALUE));
             }
             if (c == Short.TYPE || c == Short.class) {
-                return new Short((short) getLong(Short.MIN_VALUE, Short.MAX_VALUE));
+                return new Short((short)getLong(Short.MIN_VALUE, Short.MAX_VALUE));
             }
             if (c == Integer.TYPE || c == Integer.class) {
-                return new Integer((int) getLong(Integer.MIN_VALUE, Integer.MAX_VALUE));
+                return new Integer((int)getLong(Integer.MIN_VALUE, Integer.MAX_VALUE));
             }
             if (c == Long.TYPE || c == Long.class) {
                 return new Long(getLong(Long.MIN_VALUE, Long.MAX_VALUE));
@@ -307,7 +310,7 @@
                 return __float__().__tojava__(c);
             }
             if (c == BigInteger.class || c == Number.class || c == Object.class
-                || c == Serializable.class) {
+                    || c == Serializable.class) {
                 return getValue();
             }
         } catch (PyException e) {
@@ -340,14 +343,14 @@
     }
 
     /**
-     * Coercion logic for long. Implemented as a final method to avoid
-     * invocation of virtual methods from the exposed coerce.
+     * Coercion logic for long. Implemented as a final method to avoid invocation of virtual methods
+     * from the exposed coerce.
      */
     final Object long___coerce_ex__(PyObject other) {
         if (other instanceof PyLong) {
             return other;
         } else if (other instanceof PyInteger) {
-            return Py.newLong(((PyInteger) other).getValue());
+            return Py.newLong(((PyInteger)other).getValue());
         } else {
             return Py.None;
         }
@@ -359,9 +362,9 @@
 
     private static final BigInteger coerce(PyObject other) {
         if (other instanceof PyLong) {
-            return ((PyLong) other).getValue();
+            return ((PyLong)other).getValue();
         } else if (other instanceof PyInteger) {
-            return BigInteger.valueOf(((PyInteger) other).getValue());
+            return BigInteger.valueOf(((PyInteger)other).getValue());
         } else {
             throw Py.TypeError("xxx");
         }
@@ -421,7 +424,7 @@
     @ExposedMethod(type = MethodType.BINARY, doc = BuiltinDocs.long___mul___doc)
     final PyObject long___mul__(PyObject right) {
         if (right instanceof PySequence) {
-            return ((PySequence) right).repeat(coerceInt(this));
+            return ((PySequence)right).repeat(coerceInt(this));
         }
 
         if (!canCoerce(right)) {
@@ -438,7 +441,7 @@
     @ExposedMethod(type = MethodType.BINARY, doc = BuiltinDocs.long___rmul___doc)
     final PyObject long___rmul__(PyObject left) {
         if (left instanceof PySequence) {
-            return ((PySequence) left).repeat(coerceInt(this));
+            return ((PySequence)left).repeat(coerceInt(this));
         }
         if (!canCoerce(left)) {
             return null;
@@ -479,7 +482,7 @@
         if (Options.division_warning > 0) {
             Py.warning(Py.DeprecationWarning, "classic long division");
         }
-        return Py.newLong(divide( getValue(), coerce(right)));
+        return Py.newLong(divide(getValue(), coerce(right)));
     }
 
     @Override
@@ -508,7 +511,7 @@
         if (!canCoerce(right)) {
             return null;
         }
-        return Py.newLong(divide( getValue(), coerce(right)));
+        return Py.newLong(divide(getValue(), coerce(right)));
     }
 
     @Override
@@ -564,7 +567,7 @@
         if (!canCoerce(right)) {
             return null;
         }
-        return true_divide( this.getValue(), coerce(right));
+        return true_divide(this.getValue(), coerce(right));
     }
 
     @Override
@@ -595,7 +598,7 @@
             return null;
         }
         BigInteger rightv = coerce(right);
-        return Py.newLong(modulo(getValue(),rightv, divide(getValue(),rightv)));
+        return Py.newLong(modulo(getValue(), rightv, divide(getValue(), rightv)));
     }
 
     @Override
@@ -624,8 +627,8 @@
         }
         BigInteger rightv = coerce(right);
 
-        BigInteger xdivy = divide(getValue(),rightv);
-        return new PyTuple(Py.newLong(xdivy), Py.newLong(modulo(getValue(),rightv, xdivy)));
+        BigInteger xdivy = divide(getValue(), rightv);
+        return new PyTuple(Py.newLong(xdivy), Py.newLong(modulo(getValue(), rightv, xdivy)));
     }
 
     @Override
@@ -650,7 +653,7 @@
     }
 
     @ExposedMethod(type = MethodType.BINARY, defaults = {"null"},
-                   doc = BuiltinDocs.long___pow___doc)
+            doc = BuiltinDocs.long___pow___doc)
     final PyObject long___pow__(PyObject right, PyObject modulo) {
         if (!canCoerce(right)) {
             return null;
@@ -659,7 +662,7 @@
         if (modulo != null && !canCoerce(right)) {
             return null;
         }
-        return _pow( getValue(), coerce(right), modulo, this, right);
+        return _pow(getValue(), coerce(right), modulo, this, right);
     }
 
     @Override
@@ -676,7 +679,7 @@
     }
 
     public static PyObject _pow(BigInteger value, BigInteger y, PyObject modulo, PyObject left,
-                                PyObject right) {
+            PyObject right) {
         if (y.compareTo(BigInteger.ZERO) < 0) {
             if (value.compareTo(BigInteger.ZERO) != 0) {
                 return left.__float__().__pow__(right, modulo);
@@ -700,32 +703,32 @@
             }
 
             if (z.compareTo(BigInteger.valueOf(0)) <= 0) {
-                // Handle negative modulo's specially
-                /*if (z.compareTo(BigInteger.valueOf(0)) == 0) {
-                    throw Py.ValueError("pow(x, y, z) with z == 0");
-                }*/
+                // Handle negative modulo specially
+                // if (z.compareTo(BigInteger.valueOf(0)) == 0) {
+                // throw Py.ValueError("pow(x, y, z) with z == 0");
+                // }
                 y = value.modPow(y, z.negate());
                 if (y.compareTo(BigInteger.valueOf(0)) > 0) {
                     return Py.newLong(z.add(y));
                 } else {
                     return Py.newLong(y);
                 }
-                //return __pow__(right).__mod__(modulo);
+                // return __pow__(right).__mod__(modulo);
             } else {
                 // XXX: 1.1 no longer supported so review this.
                 // This is buggy in SUN's jdk1.1.5
                 // Extra __mod__ improves things slightly
                 return Py.newLong(value.modPow(y, z));
-                //return __pow__(right).__mod__(modulo);
+                // return __pow__(right).__mod__(modulo);
             }
         }
     }
 
     private static final int coerceInt(PyObject other) {
         if (other instanceof PyLong) {
-            return ((PyLong) other).asInt();
+            return ((PyLong)other).asInt();
         } else if (other instanceof PyInteger) {
-            return ((PyInteger) other).getValue();
+            return ((PyInteger)other).getValue();
         } else {
             throw Py.TypeError("xxx");
         }
@@ -915,7 +918,8 @@
 
     @ExposedMethod(doc = BuiltinDocs.long___int___doc)
     final PyObject long___int__() {
-        if (getValue().compareTo(PyInteger.MAX_INT) <= 0 && getValue().compareTo(PyInteger.MIN_INT) >= 0) {
+        if (getValue().compareTo(PyInteger.MAX_INT) <= 0
+                && getValue().compareTo(PyInteger.MIN_INT) >= 0) {
             return Py.newInteger(getValue().intValue());
         }
         return long___long__();
@@ -977,14 +981,8 @@
 
     @ExposedMethod(doc = BuiltinDocs.long___oct___doc)
     final PyString long___oct__() {
-        String s = PyInteger.toOctString(getValue());
-        if (s.startsWith("-")) {
-            return new PyString("-0" + s.substring(1, s.length()) + "L");
-        } else if (s.startsWith("0")) {
-            return new PyString(s + "L");
-        } else {
-            return new PyString("0" + s + "L");
-        }
+        // Use the prepared format specifier for octal.
+        return formatImpl(IntegerFormatter.OCT);
     }
 
     @Override
@@ -994,12 +992,21 @@
 
     @ExposedMethod(doc = BuiltinDocs.long___hex___doc)
     final PyString long___hex__() {
-        String s = PyInteger.toHexString(getValue());
-        if (s.startsWith("-")) {
-            return new PyString("-0x" + s.substring(1, s.length()) + "L");
-        } else {
-            return new PyString("0x" + s + "L");
-        }
+        // Use the prepared format specifier for hexadecimal.
+        return formatImpl(IntegerFormatter.HEX);
+    }
+
+    /**
+     * Common code used by the number-base conversion method __oct__ and __hex__.
+     *
+     * @param spec prepared format-specifier.
+     * @return converted value of this object
+     */
+    private PyString formatImpl(Spec spec) {
+        // Traditional formatter (%-format) because #o means "-0123" not "-0o123".
+        IntegerFormatter f = new IntegerFormatter.Traditional(spec);
+        f.format(value).append('L');
+        return new PyString(f.getResult());
     }
 
     @ExposedMethod(doc = BuiltinDocs.long___str___doc)
@@ -1058,7 +1065,38 @@
 
     @ExposedMethod(doc = BuiltinDocs.long___format___doc)
     final PyObject long___format__(PyObject formatSpec) {
-        return PyInteger.formatImpl(getValue(), formatSpec);
+
+        // Parse the specification
+        Spec spec = InternalFormat.fromText(formatSpec, "__format__");
+        InternalFormat.Formatter f;
+
+        // Try to make an integer formatter from the specification
+        IntegerFormatter fi = PyInteger.prepareFormatter(spec);
+        if (fi != null) {
+            // Bytes mode if formatSpec argument is not unicode.
+            fi.setBytes(!(formatSpec instanceof PyUnicode));
+            // Convert as per specification.
+            fi.format(value);
+            f = fi;
+
+        } else {
+            // Try to make a float formatter from the specification
+            FloatFormatter ff = PyFloat.prepareFormatter(spec);
+            if (ff != null) {
+                // Bytes mode if formatSpec argument is not unicode.
+                ff.setBytes(!(formatSpec instanceof PyUnicode));
+                // Convert as per specification.
+                ff.format(value.doubleValue());
+                f = ff;
+
+            } else {
+                // The type code was not recognised in either prepareFormatter
+                throw Formatter.unknownFormat(spec.type, "integer");
+            }
+        }
+
+        // Return a result that has the same type (str or unicode) as the formatSpec argument.
+        return f.pad().getPyResult();
     }
 
     @Override
@@ -1076,7 +1114,7 @@
             }
             return tooLow ? Integer.MIN_VALUE : Integer.MAX_VALUE;
         }
-        return (int) getValue().longValue();
+        return (int)getValue().longValue();
     }
 
     @Override
diff --git a/src/org/python/core/PyString.java b/src/org/python/core/PyString.java
--- a/src/org/python/core/PyString.java
+++ b/src/org/python/core/PyString.java
@@ -9,10 +9,12 @@
 import org.python.core.buffer.SimpleStringBuffer;
 import org.python.core.stringlib.FieldNameIterator;
 import org.python.core.stringlib.FloatFormatter;
+import org.python.core.stringlib.IntegerFormatter;
+import org.python.core.stringlib.InternalFormat;
+import org.python.core.stringlib.InternalFormat.Formatter;
 import org.python.core.stringlib.InternalFormat.Spec;
-import org.python.core.stringlib.InternalFormatSpec;
-import org.python.core.stringlib.InternalFormatSpecParser;
 import org.python.core.stringlib.MarkupIterator;
+import org.python.core.stringlib.TextFormatter;
 import org.python.core.util.StringUtil;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedNew;
@@ -3897,50 +3899,68 @@
 
     @ExposedMethod(doc = BuiltinDocs.str___format___doc)
     final PyObject str___format__(PyObject formatSpec) {
-        if (!(formatSpec instanceof PyString)) {
-            throw Py.TypeError("__format__ requires str or unicode");
+
+        // Parse the specification
+        Spec spec = InternalFormat.fromText(formatSpec, "__format__");
+
+        // Get a formatter for the specification
+        TextFormatter f = prepareFormatter(spec);
+        if (f == null) {
+            // The type code was not recognised
+            throw Formatter.unknownFormat(spec.type, "string");
         }
 
-        PyString formatSpecStr = (PyString)formatSpec;
-        String result;
-        try {
-            String specString = formatSpecStr.getString();
-            InternalFormatSpec spec = new InternalFormatSpecParser(specString).parse();
-            result = formatString(getString(), spec);
-        } catch (IllegalArgumentException e) {
-            throw Py.ValueError(e.getMessage());
+        // Bytes mode if neither this nor formatSpec argument is Unicode.
+        boolean unicode = this instanceof PyUnicode || formatSpec instanceof PyUnicode;
+        f.setBytes(!unicode);
+
+        // Convert as per specification.
+        f.format(getString());
+
+        // Return a result that has the same type (str or unicode) as the formatSpec argument.
+        return f.pad().getPyResult();
+    }
+
+    /**
+     * Common code for {@link PyString} and {@link PyUnicode} to prepare a {@link TextFormatter}
+     * from a parsed specification. The object returned has format method
+     * {@link TextFormatter#format(String)} that treats its argument as UTF-16 encoded unicode (not
+     * just chars). That method will format its argument ( str or
+     * unicode) according to the PEP 3101 formatting specification supplied here. This
+     * would be used during text.__format__(".5s") or
+     * "{:.5s}".format(text) where text is this Python string.
+     *
+     * @param spec a parsed PEP-3101 format specification.
+     * @return a formatter ready to use, or null if the type is not a string format type.
+     * @throws PyException(ValueError) if the specification is faulty.
+     */
+    @SuppressWarnings("fallthrough")
+    static TextFormatter prepareFormatter(Spec spec) throws PyException {
+        // Slight differences between format types
+        switch (spec.type) {
+
+            case Spec.NONE:
+            case 's':
+                // Check for disallowed parts of the specification
+                if (spec.grouping) {
+                    throw Formatter.notAllowed("Grouping", "string", spec.type);
+                } else if (Spec.specified(spec.sign)) {
+                    throw Formatter.signNotAllowed("string", '\0');
+                } else if (spec.alternate) {
+                    throw Formatter.alternateFormNotAllowed("string");
+                } else if (spec.align == '=') {
+                    throw Formatter.alignmentNotAllowed('=', "string");
+                }
+                // spec may be incomplete. The defaults are those commonly used for string formats.
+                spec = spec.withDefaults(Spec.STRING);
+                // Get a formatter for the specification
+                return new TextFormatter(spec);
+
+            default:
+                // The type code was not recognised
+                return null;
         }
-        return formatSpecStr.createInstance(result);
-    }
-
-    /**
-     * Format the given text according to a parsed PEP 3101 formatting specification, as during
-     * text.__format__(format_spec) or "{:s}".format(text) where
-     * text is a Python string.
-     *
-     * @param text to format
-     * @param spec the parsed PEP 3101 formatting specification
-     * @return the result of the formatting
-     */
-    public static String formatString(String text, InternalFormatSpec spec) {
-        if (spec.sign != '\0') {
-            throw new IllegalArgumentException("Sign not allowed in string format specifier");
-        }
-        if (spec.alternate) {
-            throw new IllegalArgumentException(
-                    "Alternate form (#) not allowed in string format specifier");
-        }
-        if (spec.align == '=') {
-            throw new IllegalArgumentException(
-                    "'=' alignment not allowed in string format specifier");
-        }
-        if (spec.precision >= 0 && text.length() > spec.precision) {
-            text = text.substring(0, spec.precision);
-        }
-        return spec.pad(text, '<', 0);
-    }
-
-    /* arguments' conversion helper */
+    }
 
     @Override
     public String asString(int index) throws PyObject.ConversionException {
@@ -4005,10 +4025,6 @@
     String format;
     /** Where the output is built. */
     StringBuilder buffer;
-    /** Remembers that the value currently converted is negative */
-    boolean negative;
-    /** Precision from format specification. */
-    int precision;
     /**
      * Index into args of argument currently being worked, or special values indicating -1: a single
      * item that has not yet been used, -2: a single item that has already been used, -3: a mapping.
@@ -4017,7 +4033,7 @@
     /** Arguments supplied to {@link #format(PyObject)} method. */
     PyObject args;
     /** Indicate a PyUnicode result is expected. */
-    boolean unicodeCoercion;
+    boolean needUnicode;
 
     final char pop() {
         try {
@@ -4053,7 +4069,7 @@
     public StringFormatter(String format, boolean unicodeCoercion) {
         index = 0;
         this.format = format;
-        this.unicodeCoercion = unicodeCoercion;
+        this.needUnicode = unicodeCoercion;
         buffer = new StringBuilder(format.length() + 100);
     }
 
@@ -4106,211 +4122,129 @@
         }
     }
 
-    private void checkPrecision(String type) {
-        if (precision > 250) {
-            // A magic number. Larger than in CPython.
-            throw Py.OverflowError("formatted " + type + " is too long (precision too long?)");
-        }
-
-    }
-
     /**
-     * Format the argument interpreted as a long, using the argument's __str__,
-     * __oct__, or __hex__ method according to type. If v is
-     * being treated as signed, the sign of v is transferred to {@link #negative} and the absolute
-     * value is converted. The altFlag argument controls the appearance of a "0x" or
-     * "0X" prefix in the hex case, or a "0" prefix in the octal case. The hexadecimal case, the
-     * case of characters and digits will match the type ('x' meaning lowercase, 'X' meaning
-     * uppercase).
+     * Return the argument as either a {@link PyInteger} or a {@link PyLong} according to its
+     * __int__ method, or its __long__ method. If the argument has neither
+     * method, or both raise an exception, we return the argument itself. The caller must check the
+     * return type.
      *
      * @param arg to convert
-     * @param type one of 'o' for octal, 'x' or 'X' for hex, anything else calls
-     *            arg.__str__.
-     * @param altFlag if true there will be a prefix
-     * @return converted value as String
+     * @return PyInteger or PyLong if possible
      */
-    private String formatLong(PyObject arg, char type, boolean altFlag) {
-        // Convert using the appropriate type
-        // XXX Results in behaviour divergent from CPython when any of the methods is overridden.
-        PyString argAsString;
-        switch (type) {
-            case 'o':
-                argAsString = arg.__oct__();
-                break;
-            case 'x':
-            case 'X':
-                argAsString = arg.__hex__();
-                break;
-            default:
-                argAsString = arg.__str__();
-                break;
-        }
-
-        checkPrecision("long");
-        String s = argAsString.toString();
-        int end = s.length();
-        int ptr = 0;
-
-        // In the hex case, the __hex__ return starts 0x
-        // XXX (we assume, perhaps falsely)
-        int numnondigits = 0;
-        if (type == 'x' || type == 'X') {
-            numnondigits = 2;
-        }
-
-        // Strip a "long" indicator
-        if (s.endsWith("L")) {
-            end--;
-        }
-
-        // Strip a possible sign to member negative
-        negative = s.charAt(0) == '-';
-        if (negative) {
-            ptr++;
-        }
-
-        // The formatted number is s[ptr:end] and starts with numnondigits non-digits.
-        int numdigits = end - numnondigits - ptr;
-        if (!altFlag) {
-            // We should have no "base tag" '0' or "0x" on the front.
-            switch (type) {
-                case 'o':
-                    // Strip the '0'
-                    if (numdigits > 1) {
-                        ++ptr;
-                        --numdigits;
-                    }
-                    break;
-                case 'x':
-                case 'X':
-                    // Strip the "0x"
-                    ptr += 2;
-                    numnondigits -= 2;
-                    break;
-            }
-        }
-
-        // If necessary, add leading zeros to the numerical digits part.
-        if (precision > numdigits) {
-            // Recompose the formatted number in this buffer
-            StringBuilder buf = new StringBuilder();
-            // The base indicator prefix
-            for (int i = 0; i < numnondigits; ++i) {
-                buf.append(s.charAt(ptr++));
-            }
-            // The extra zeros
-            for (int i = 0; i < precision - numdigits; i++) {
-                buf.append('0');
-            }
-            // The previously known digits
-            for (int i = 0; i < numdigits; i++) {
-                buf.append(s.charAt(ptr++));
-            }
-            s = buf.toString();
-        } else if (end < s.length() || ptr > 0) {
-            // It's only necessary to extract the formatted number from s
-            s = s.substring(ptr, end);
-        }
-
-        // And finally, deal with the case, so it matches x or X.
-        switch (type) {
-            case 'X':
-                s = s.toUpperCase();
-                break;
-        }
-        return s;
-    }
-
-    /**
-     * Formats arg as an integer, with the specified radix. The integer value is obtained from the
-     * result of arg.__int__(). type and altFlag are passed
-     * to {@link #formatLong(PyObject, char, boolean)} in case the result is a PyLong.
-     *
-     * @param arg to convert
-     * @param radix in which to express arg
-     * @param unsigned true if required to interpret a 32-bit integer as unsigned ('u' legacy?).
-     * @param type of conversion ('d', 'o', 'x', or 'X')
-     * @param altFlag '#' present in format (causes "0x" prefix in hex, and '0' prefix in octal)
-     * @return string form of the value
-     */
-    private String formatInteger(PyObject arg, int radix, boolean unsigned, char type,
-            boolean altFlag) {
-        PyObject argAsInt;
+    private PyObject asNumber(PyObject arg) {
         if (arg instanceof PyInteger || arg instanceof PyLong) {
-            argAsInt = arg;
+            // arg is already acceptable
+            return arg;
+
         } else {
-            // use __int__ to get an int (or long)
-            if (arg instanceof PyFloat) {
-                // safe to call __int__:
-                argAsInt = arg.__int__();
+            // use __int__ or __long__to get an int (or long)
+            if (arg.getClass() == PyFloat.class) {
+                // A common case where it is safe to return arg.__int__()
+                return arg.__int__();
+
             } else {
-                // We can't simply call arg.__int__() because PyString implements
-                // it without exposing it to python (i.e, str instances has no
-                // __int__ attribute). So, we would support strings as arguments
-                // for %d format, which is forbidden by CPython tests (on
-                // test_format.py).
+                /*
+                 * In general, we can't simply call arg.__int__() because PyString implements it
+                 * without exposing it to python (str has no __int__). This would make str
+                 * acceptacle to integer format specifiers, which is forbidden by CPython tests
+                 * (test_format.py). PyString implements __int__ perhaps only to help the int
+                 * constructor. Maybe that was a bad idea?
+                 */
                 try {
-                    argAsInt = arg.__getattr__("__int__").__call__();
+                    // Result is the result of arg.__int__() if that works
+                    return arg.__getattr__("__int__").__call__();
                 } catch (PyException e) {
-                    // XXX: Swallow custom AttributeError throws from __int__ methods
-                    // No better alternative for the moment
-                    if (e.match(Py.AttributeError)) {
-                        throw Py.TypeError("int argument required");
-                    }
-                    throw e;
+                    // Swallow the exception
+                }
+
+                // Try again with arg.__long__()
+                try {
+                    // Result is the result of arg.__long__() if that works
+                    return arg.__getattr__("__long__").__call__();
+                } catch (PyException e) {
+                    // No __long__ defined (at Python level)
+                    return arg;
                 }
             }
         }
-        if (argAsInt instanceof PyInteger) {
-            // This call does not provide the prefix and will be lowercase.
-            return formatInteger(((PyInteger)argAsInt).getValue(), radix, unsigned);
-        } else { // must be a PyLong (as per __int__ contract)
-            // This call provides the base prefix and case-matches with 'x' or 'X'.
-            return formatLong(argAsInt, type, altFlag);
-        }
     }
 
     /**
-     * Convert a 32-bit integer (as from a {@link PyInteger}) to characters, signed or unsigned. The
-     * values is presented in a long. The string result is left-padded with zeros to
-     * the stated {@link #precision}. If v is being treated as signed, the sign of v is transferred
-     * to {@link #negative} and the absolute value is converted. Otherwise (unsigned case)
-     * 0x100000000L + v is converted. This method does not provide the '0' or "0x"
-     * prefix, just the padded digit string.
+     * Return the argument as a {@link PyFloat} according to its __float__ method. If
+     * the argument has no such method, or it raises an exception, we return the argument itself.
+     * The caller must check the return type.
      *
-     * @param v value to convert
-     * @param radix of conversion
-     * @param unsigned if should be treated as unsigned
-     * @return string form
+     * @param arg to convert
+     * @return PyFloat if possible
      */
-    private String formatInteger(long v, int radix, boolean unsigned) {
-        checkPrecision("integer");
-        if (unsigned) {
-            // If the high bit was set, this will have been sign-extended: correct that.
-            if (v < 0) {
-                v = 0x100000000l + v;
-            }
+    private PyObject asFloat(PyObject arg) {
+
+        if (arg instanceof PyFloat) {
+            // arg is already acceptable
+            return arg;
+
         } else {
-            // If the high bit was set, the sign extension was correct, but we need sign + abs(v).
-            if (v < 0) {
-                negative = true;
-                v = -v;
+            // use __float__ to get a float.
+            if (arg.getClass() == PyFloat.class) {
+                // A common case where it is safe to return arg.__float__()
+                return arg.__float__();
+
+            } else {
+                /*
+                 * In general, we can't simply call arg.__float__() because PyString implements it
+                 * without exposing it to python (str has no __float__). This would make str
+                 * acceptacle to float format specifiers, which is forbidden by CPython tests
+                 * (test_format.py). PyString implements __float__ perhaps only to help the float
+                 * constructor. Maybe that was a bad idea?
+                 */
+                try {
+                    // Result is the result of arg.__float__() if that works
+                    return arg.__getattr__("__float__").__call__();
+                } catch (PyException e) {
+                    // No __float__ defined (at Python level)
+                    return arg;
+                }
             }
         }
-        // Use the method in java.lang.Long (lowercase, no prefix)
-        String s = Long.toString(v, radix);
-        // But zero pad to the requested precision
-        while (s.length() < precision) {
-            s = "0" + s;
-        }
-        return s;
-    }
-
-    private double asDouble(PyObject obj) {
-        try {
-            return obj.asDouble();
-        } catch (PyException pye) {
-            throw !pye.match(Py.TypeError) ? pye : Py.TypeError("float argument required");
+    }
+
+    /**
+     * Return the argument as either a {@link PyString} or a {@link PyUnicode}, and set the
+     * {@link #needUnicode} member accordingly. If we already know we are building a Unicode string
+     * (needUnicode==true), then any argument that is not already a
+     * PyUnicode will be converted by calling its __unicode__ method.
+     * Conversely, if we are not yet building a Unicode string (needUnicode==false ),
+     * then a PyString will pass unchanged, a PyUnicode will switch us to Unicode mode
+     * (needUnicode=true), and any other type will be converted by calling its
+     * __str__ method, which will return a PyString, or possibly a
+     * PyUnicode, which will switch us to Unicode mode.
+     *
+     * @param arg to convert
+     * @return PyString or PyUnicode equivalent
+     */
+    private PyString asText(PyObject arg) {
+
+        if (arg instanceof PyUnicode) {
+            // arg is already acceptable.
+            needUnicode = true;
+            return (PyUnicode)arg;
+
+        } else if (needUnicode) {
+            // The string being built is unicode, so we need that version of the arg.
+            return arg.__unicode__();
+
+        } else if (arg instanceof PyString) {
+            // The string being built is not unicode, so arg is already acceptable.
+            return (PyString)arg;
+
+        } else {
+            // The string being built is not unicode, so use __str__ to get a PyString.
+            PyString s = arg.__str__();
+            // But __str__ might return PyUnicode, and we have to notice that.
+            if (s instanceof PyUnicode) {
+                needUnicode = true;
+            }
+            return s;
         }
     }
 
@@ -4325,7 +4259,7 @@
     public PyString format(PyObject args) {
         PyObject dict = null;
         this.args = args;
-        boolean needUnicode = unicodeCoercion;
+
         if (args instanceof PyTuple) {
             // We will simply work through the tuple elements
             argIndex = 0;
@@ -4341,16 +4275,6 @@
 
         while (index < format.length()) {
 
-            // Attributes to be parsed from the next format specifier
-            boolean ljustFlag = false;
-            boolean signFlag = false;
-            boolean blankFlag = false;
-            boolean altFlag = false;
-            boolean zeroFlag = false;
-
-            int width = -1;
-            precision = -1;
-
             // Read one character from the format string
             char c = pop();
             if (c != '%') {
@@ -4360,6 +4284,14 @@
 
             // It's a %, so the beginning of a conversion specifier. Parse it.
 
+            // Attributes to be parsed from the next format specifier
+            boolean altFlag = false;
+            char sign = Spec.NONE;
+            char fill = ' ';
+            char align = '>';
+            int width = Spec.UNSPECIFIED;
+            int precision = Spec.UNSPECIFIED;
+
             // A conversion specifier contains the following components, in this order:
             // + The '%' character, which marks the start of the specifier.
             // + Mapping key (optional), consisting of a parenthesised sequence of characters.
@@ -4399,19 +4331,22 @@
             while (true) {
                 switch (c = pop()) {
                     case '-':
-                        ljustFlag = true;
+                        align = '<';
                         continue;
                     case '+':
-                        signFlag = true;
+                        sign = '+';
                         continue;
                     case ' ':
-                        blankFlag = true;
+                        if (!Spec.specified(sign)) {
+                            // Blank sign only wins if '+' not specified.
+                            sign = ' ';
+                        }
                         continue;
                     case '#':
                         altFlag = true;
                         continue;
                     case '0':
-                        zeroFlag = true;
+                        fill = '0';
                         continue;
                 }
                 break;
@@ -4428,7 +4363,7 @@
             width = getNumber();
             if (width < 0) {
                 width = -width;
-                ljustFlag = true;
+                align = '<';
             }
 
             /*
@@ -4451,284 +4386,149 @@
                 c = pop();
             }
 
-            // c is now the conversion type.
-            if (c == '%') {
-                // It was just a percent sign after all
-                buffer.append(c);
-                continue;
+            /*
+             * As a function of the conversion type (currently in c) override some of the formatting
+             * flags we read from the format specification.
+             */
+            switch (c) {
+                case 's':
+                case 'r':
+                case 'c':
+                case '%':
+                    // These have string-like results: fill, if needed, is always blank.
+                    fill = ' ';
+                    break;
+
+                default:
+                    if (fill == '0' && align == '>') {
+                        // Zero-fill comes after the sign in right-justification.
+                        align = '=';
+                    } else {
+                        // If left-justifying, the fill is always blank.
+                        fill = ' ';
+                    }
             }
 
             /*
+             * Encode as an InternalFormat.Spec. The values in the constructor always have specified
+             * values, except for sign, width and precision.
+             */
+            Spec spec = new Spec(fill, align, sign, altFlag, width, false, precision, c);
+
+            /*
              * Process argument according to format specification decoded from the string. It is
-             * important we don't read the argumnent from the list until this point because of the
+             * important we don't read the argument from the list until this point because of the
              * possibility that width and precision were specified via the argument list.
              */
-            PyObject arg = getarg();
-            String string = null;
-            negative = false;
-
-            // Independent of type, decide the padding character based on decoded flags.
-            char fill = ' ';
-            if (zeroFlag) {
-                fill = '0';
-            } else {
-                fill = ' ';
-            }
-
-            // Perform the type-specific formatting
-            switch (c) {
-
-                case 's':
-                    // String (converts any Python object using str()).
-                    if (arg instanceof PyUnicode) {
-                        needUnicode = true;
+
+            // Depending on the type of conversion, we use one of these formatters:
+            FloatFormatter ff;
+            IntegerFormatter fi;
+            TextFormatter ft;
+            Formatter f; // = ff, fi or ft, whichever we actually use.
+
+            switch (spec.type) {
+
+                case 's': // String: converts any object using __str__(), __unicode__() ...
+                case 'r': // ... or repr().
+                    PyObject arg = getarg();
+
+                    // Get hold of the actual object to display (may set needUnicode)
+                    PyString argAsString = asText(spec.type == 's' ? arg : arg.__repr__());
+                    // Format the str/unicode form of the argument using this Spec.
+                    f = ft = new TextFormatter(buffer, spec);
+                    ft.setBytes(!needUnicode);
+                    ft.format(argAsString.getString());
+                    break;
+
+                case 'd': // All integer formats (+case for X).
+                case 'o':
+                case 'x':
+                case 'X':
+                case 'c': // Single character (accepts integer or single character string).
+                case 'u': // Obsolete type identical to 'd'.
+                case 'i': // Compatibility with scanf().
+
+                    // Format the argument using this Spec.
+                    f = fi = new IntegerFormatter.Traditional(buffer, spec);
+                    // If not producing PyUnicode, disallow codes >255.
+                    fi.setBytes(!needUnicode);
+
+                    arg = getarg();
+
+                    if (arg instanceof PyString && spec.type == 'c') {
+                        if (arg.__len__() != 1) {
+                            throw Py.TypeError("%c requires int or char");
+                        } else {
+                            if (!needUnicode && arg instanceof PyUnicode) {
+                                // Change of mind forced by encountering unicode object.
+                                needUnicode = true;
+                                fi.setBytes(false);
+                            }
+                            fi.format(((PyString)arg).getString().codePointAt(0));
+                        }
+
+                    } else {
+                        // Note various types accepted here as long as they have an __int__ method.
+                        PyObject argAsNumber = asNumber(arg);
+
+                        // We have to check what we got back.
+                        if (argAsNumber instanceof PyInteger) {
+                            fi.format(((PyInteger)argAsNumber).getValue());
+                        } else if (argAsNumber instanceof PyLong) {
+                            fi.format(((PyLong)argAsNumber).getValue());
+                        } else {
+                            // It couldn't be converted, raise the error here
+                            throw Py.TypeError("%" + spec.type
+                                    + " format: a number is required, not "
+                                    + arg.getType().fastGetName());
+                        }
                     }
-                    // fall through ...
-
-                case 'r':
-                    // String (converts any Python object using repr()).
-                    fill = ' ';
-                    if (c == 's') {
-                        if (needUnicode) {
-                            string = arg.__unicode__().toString();
-                        } else {
-                            string = arg.__str__().toString();
-                        }
-                    } else {
-                        string = arg.__repr__().toString();
-                    }
-                    if (precision >= 0 && string.length() > precision) {
-                        string = string.substring(0, precision);
-                    }
 
                     break;
 
-                case 'i':
-                case 'd':
-                    // Signed integer decimal. Note floats accepted.
-                    if (arg instanceof PyLong) {
-                        string = formatLong(arg, c, altFlag);
-                    } else {
-                        string = formatInteger(arg, 10, false, c, altFlag);
-                    }
-                    break;
-
-                case 'u':
-                    // Obsolete type ? it is identical to 'd'. (Why not identical here?)
-                    if (arg instanceof PyLong) {
-                        string = formatLong(arg, c, altFlag);
-                    } else if (arg instanceof PyInteger || arg instanceof PyFloat) {
-                        string = formatInteger(arg, 10, false, c, altFlag);
-                    } else {
-                        throw Py.TypeError("int argument required");
-                    }
-                    break;
-
-                case 'o':
-                    // Signed octal value. Note floats accepted.
-                    if (arg instanceof PyLong) {
-                        // This call provides the base prefix '0' if altFlag.
-                        string = formatLong(arg, c, altFlag);
-                    } else if (arg instanceof PyInteger || arg instanceof PyFloat) {
-                        // This call does not provide the '0' prefix and will be lowercase ...
-                        // ... except where arg.__int__ returns PyLong, then it's like formatLong.
-                        string = formatInteger(arg, 8, false, c, altFlag);
-                        if (altFlag && string.charAt(0) != '0') {
-                            string = "0" + string;
-                        }
-                    } else {
-                        throw Py.TypeError("int argument required");
-                    }
-                    break;
-
-                case 'x':
-                    // Signed hexadecimal (lowercase). Note floats accepted.
-                    if (arg instanceof PyLong) {
-                        // This call provides the base prefix "0x" if altFlag and case-matches c.
-                        string = formatLong(arg, c, altFlag);
-                    } else if (arg instanceof PyInteger || arg instanceof PyFloat) {
-                        // This call does not provide the "0x" prefix and will be lowercase.
-                        // ... except where arg.__int__ returns PyLong, then it's like formatLong.
-                        string = formatInteger(arg, 16, false, c, altFlag);
-                        string = string.toLowerCase();
-                        if (altFlag) {
-                            string = "0x" + string;
-                        }
-                    } else {
-                        throw Py.TypeError("int argument required");
-                    }
-                    break;
-
-                case 'X':
-                    // Signed hexadecimal (uppercase). Note floats accepted.
-                    if (arg instanceof PyLong) {
-                        // This call provides the base prefix "0x" if altFlag and case-matches c.
-                        string = formatLong(arg, c, altFlag);
-                    } else if (arg instanceof PyInteger || arg instanceof PyFloat) {
-                        // This call does not provide the "0x" prefix and will be lowercase.
-                        // ... except where arg.__int__ returns PyLong, then it's like formatLong.
-                        string = formatInteger(arg, 16, false, c, altFlag);
-                        string = string.toUpperCase();
-                        if (altFlag) {
-                            string = "0X" + string;
-                        }
-                    } else {
-                        throw Py.TypeError("int argument required");
-                    }
-                    break;
-
-                case 'e':
+                case 'e': // All floating point formats (+case).
                 case 'E':
                 case 'f':
                 case 'F':
                 case 'g':
                 case 'G':
-                    // All floating point formats (+case).
-
-                    // Convert the flags (local variables) to the form needed in the Spec object.
-                    char align = ljustFlag ? '<' : '>';
-                    char sign = signFlag ? '+' : (blankFlag ? ' ' : Spec.NONE);
-                    int w = Spec.UNSPECIFIED;
-                    Spec spec = new Spec(fill, align, sign, altFlag, w, false, precision, c);
 
                     // Format using this Spec the double form of the argument.
-                    FloatFormatter f = new FloatFormatter(spec);
-                    double v = asDouble(arg);
-                    f.format(v);
-                    string = f.getResult();
-
-                    // Suppress subsequent attempts to insert a correct sign, done already.
-                    signFlag = blankFlag = negative = false;
+                    f = ff = new FloatFormatter(buffer, spec);
+                    ff.setBytes(!needUnicode);
+
+                    // Note various types accepted here as long as they have a __float__ method.
+                    arg = getarg();
+                    PyObject argAsFloat = asFloat(arg);
+
+                    // We have to check what we got back..
+                    if (argAsFloat instanceof PyFloat) {
+                        ff.format(((PyFloat)argAsFloat).getValue());
+                    } else {
+                        // It couldn't be converted, raise the error here
+                        throw Py.TypeError("float argument required, not "
+                                + arg.getType().fastGetName());
+                    }
+
                     break;
 
-                case 'c':
-                    // Single character (accepts integer or single character string).
-                    fill = ' ';
-                    if (arg instanceof PyString) {
-                        string = ((PyString)arg).toString();
-                        if (string.length() != 1) {
-                            throw Py.TypeError("%c requires int or char");
-                        }
-                        if (arg instanceof PyUnicode) {
-                            needUnicode = true;
-                        }
-                        break;
-                    }
-
-                    // arg is not a str (or unicode)
-                    int val;
-                    try {
-                        // Explicitly __int__ so we can look for an AttributeError (which is
-                        // less invasive to mask than a TypeError)
-                        val = arg.__int__().asInt();
-                    } catch (PyException e) {
-                        if (e.match(Py.AttributeError)) {
-                            throw Py.TypeError("%c requires int or char");
-                        }
-                        throw e;
-                    }
-                    // Range check, according to ultimate type of result as presentl;y known.
-                    if (!needUnicode) {
-                        if (val < 0) {
-                            throw Py.OverflowError("unsigned byte integer is less than minimum");
-                        } else if (val > 255) {
-                            throw Py.OverflowError("unsigned byte integer is greater than maximum");
-                        }
-                    } else if (val < 0 || val > PySystemState.maxunicode) {
-                        throw Py.OverflowError("%c arg not in range(0x110000) (wide Python build)");
-                    }
-                    string = new String(new int[] {val}, 0, 1);
+                case '%': // Percent symbol, but surprisingly, padded.
+
+                    // We use an integer formatter.
+                    f = fi = new IntegerFormatter.Traditional(buffer, spec);
+                    fi.setBytes(!needUnicode);
+                    fi.format('%');
                     break;
 
                 default:
                     throw Py.ValueError("unsupported format character '"
-                            + codecs.encode(Py.newString(c), null, "replace") + "' (0x"
-                            + Integer.toHexString(c) + ") at index " + (index - 1));
+                            + codecs.encode(Py.newString(spec.type), null, "replace") + "' (0x"
+                            + Integer.toHexString(spec.type) + ") at index " + (index - 1));
             }
 
-            /*
-             * We have now dealt with the translation of the (absolute value of the) argument, in
-             * variable string[]. In the next sections we deal with sign, padding and base prefix.
-             */
-            int length = string.length();
-            int skip = 0;
-
-            // Decide how to represent the sign according to format and actual sign of argument.
-            String signString = null;
-            if (negative) {
-                signString = "-";
-            } else {
-                if (signFlag) {
-                    signString = "+";
-                } else if (blankFlag) {
-                    signString = " ";
-                }
-            }
-
-            // The width (from here on) will be the remaining width on the line.
-            if (width < length) {
-                width = length;
-            }
-
-            // Insert the sign in the buffer and adjust the width.
-            if (signString != null) {
-                if (fill != ' ') {
-                    // When the fill is not space, the sign comes before the fill.
-                    buffer.append(signString);
-                }
-                // Adjust width for sign.
-                if (width > length) {
-                    width--;
-                }
-            }
-
-            // Insert base prefix used with alternate mode for hexadecimal.
-            if (altFlag && (c == 'x' || c == 'X')) {
-                if (fill != ' ') {
-                    // When the fill is not space, this base prefix comes before the fill.
-                    buffer.append('0');
-                    buffer.append(c);
-                    skip += 2;
-                }
-                // Adjust width for base prefix.
-                width -= 2;
-                if (width < 0) {
-                    width = 0;
-                }
-                length -= 2;
-            }
-
-            // Fill on the left of the item.
-            if (width > length && !ljustFlag) {
-                do {
-                    buffer.append(fill);
-                } while (--width > length);
-            }
-
-            // If the fill is spaces, we will have deferred the sign and hex base prefix
-            if (fill == ' ') {
-                if (signString != null) {
-                    buffer.append(signString);
-                }
-                if (altFlag && (c == 'x' || c == 'X')) {
-                    buffer.append('0');
-                    buffer.append(c);
-                    skip += 2;
-                }
-            }
-
-            // Now append the converted argument.
-            if (skip > 0) {
-                // The string contains a hex-prefix, but we have already inserted one.
-                buffer.append(string.substring(skip));
-            } else {
-                buffer.append(string);
-            }
-
-            // If this hasn't filled the space required, add right-padding.
-            while (--width >= length) {
-                buffer.append(' ');
-            }
+            // Pad the result as specified (in-place, in the buffer).
+            f.pad();
         }
 
         /*
@@ -4743,10 +4543,7 @@
         }
 
         // Return the final buffer contents as a str or unicode as appropriate.
-        if (needUnicode) {
-            return new PyUnicode(buffer);
-        }
-        return new PyString(buffer);
+        return needUnicode ? new PyUnicode(buffer) : new PyString(buffer);
     }
 
 }
diff --git a/src/org/python/core/__builtin__.java b/src/org/python/core/__builtin__.java
--- a/src/org/python/core/__builtin__.java
+++ b/src/org/python/core/__builtin__.java
@@ -4,20 +4,20 @@
  */
 package org.python.core;
 
-import java.io.EOFException;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.nio.ByteBuffer;
 import java.util.Iterator;
 import java.util.Map;
 
 import org.python.antlr.base.mod;
+import org.python.core.stringlib.IntegerFormatter;
+import org.python.core.stringlib.InternalFormat;
+import org.python.core.stringlib.InternalFormat.Spec;
 import org.python.core.util.ExtraMath;
 import org.python.core.util.RelativeFile;
-import org.python.core.util.StringUtil;
 import org.python.modules._functools._functools;
 
 class BuiltinFunctions extends PyBuiltinFunctionSet {
@@ -768,7 +768,7 @@
     /**
      * Built-in Python function ord() applicable to the string-like types str,
      * bytearray, unicode.
-     * 
+     *
      * @param c string-like object of length 1
      * @return ordinal value of character or byte value in
      * @throws PyException (TypeError) if not a string-like type
@@ -1245,10 +1245,10 @@
         PyObject[] args;
         if (level < 0) {
         	// for backward compatibility provide only 4 arguments
-        	args = new PyObject[] {Py.newString(name), globals, locals, 
+        	args = new PyObject[] {Py.newString(name), globals, locals,
         			fromlist};
         } else {
-        	args = new PyObject[] {Py.newString(name), globals, locals, 
+        	args = new PyObject[] {Py.newString(name), globals, locals,
         			fromlist, Py.newInteger(level)};
         }
         PyObject module = __import__.__call__(args);
@@ -1469,7 +1469,7 @@
                 endObject = useUnicode ? Py.newUnicode(end) : Py.newString(end);
             }
 
-            out.print(values, sepObject, endObject); 
+            out.print(values, sepObject, endObject);
         }
         return Py.None;
     }
@@ -1774,10 +1774,6 @@
     public PyObject __call__(PyObject args[], String kwds[]) {
         ArgParser ap = new ArgParser("bin", args, kwds, new String[] {"number"}, 1);
         ap.noKeywords();
-        PyObject number = ap.getPyObject(0);
-
-        //XXX: this could be made more efficient by using a binary only formatter
-        //     instead of using generic formatting.
-        return number.__format__(new PyString("#b"));
+        return IntegerFormatter.bin(ap.getPyObject(0));
     }
 }
diff --git a/src/org/python/core/stringlib/FloatFormatter.java b/src/org/python/core/stringlib/FloatFormatter.java
--- a/src/org/python/core/stringlib/FloatFormatter.java
+++ b/src/org/python/core/stringlib/FloatFormatter.java
@@ -18,6 +18,10 @@
     /** The rounding mode dominant in the formatter. */
     static final RoundingMode ROUND_PY = RoundingMode.HALF_EVEN;
 
+    /** Limit the size of results. */
+    // No-one needs more than log(Double.MAX_VALUE) - log2(Double.MIN_VALUE) = 1383 digits.
+    static final int MAX_PRECISION = 1400;
+
     /** If it contains no decimal point, this length is zero, and 1 otherwise. */
     private int lenPoint;
     /** The length of the fractional part, right of the decimal point. */
@@ -30,26 +34,14 @@
     private int minFracDigits;
 
     /**
-     * Construct the formatter from a specification. A reference is held to this specification, but
-     * it will not be modified by the actions of this class.
+     * Construct the formatter from a client-supplied buffer, to which the result will be appended,
+     * and a specification. Sets {@link #mark} to the end of the buffer.
      *
+     * @param result destination buffer
      * @param spec parsed conversion specification
      */
-    public FloatFormatter(Spec spec) {
-        // Space for result is based on padded width, or precision, whole part & furniture.
-        this(spec, 1, 0);
-    }
-
-    /**
-     * Construct the formatter from a specification and an explicit initial buffer capacity. A
-     * reference is held to this specification, but it will not be modified by the actions of this
-     * class.
-     *
-     * @param spec parsed conversion specification
-     * @param width expected for the formatted result
-     */
-    public FloatFormatter(Spec spec, int width) {
-        super(spec, width);
+    public FloatFormatter(StringBuilder result, Spec spec) {
+        super(result, spec);
         if (spec.alternate) {
             // Alternate form means do not trim the zero fractional digits.
             minFracDigits = -1;
@@ -66,20 +58,26 @@
     }
 
     /**
-     * Construct the formatter from a specification and two extra hints about the initial buffer
-     * capacity. A reference is held to this specification, but it will not be modified by the
-     * actions of this class.
+     * Construct the formatter from a specification, allocating a buffer internally for the result.
      *
      * @param spec parsed conversion specification
-     * @param count of elements likely to be formatted
-     * @param margin for elements formatted only once
      */
-    public FloatFormatter(Spec spec, int count, int margin) {
-        /*
-         * Rule of thumb used here: in e format w = (p-1) + len("+1.e+300") = p+7; in f format w = p
-         * + len("1,000,000.") = p+10. If we're wrong, the result will have to grow. No big deal.
-         */
-        this(spec, Math.max(spec.width + 1, count * (spec.precision + 10) + margin));
+    public FloatFormatter(Spec spec) {
+        this(new StringBuilder(size(spec)), spec);
+    }
+
+    /**
+     * Recommend a buffer size for a given specification, assuming one float is converted. This will
+     * be a "right" answer for e and g-format, and for f-format with values up to 9,999,999.
+     *
+     * @param spec parsed conversion specification
+     */
+    public static int size(Spec spec) {
+        // Rule of thumb used here (no right answer):
+        // in e format each float occupies: (p-1) + len("+1.e+300") = p+7;
+        // in f format each float occupies: p + len("1,000,000.%") = p+11;
+        // or an explicit (minimum) width may be given, with one overshoot possible.
+        return Math.max(spec.width + 1, spec.getPrecision(6) + 11);
     }
 
     /**
@@ -160,12 +158,19 @@
         // Precision defaults to 6 (or 12 for none-format)
         int precision = spec.getPrecision(Spec.specified(spec.type) ? 6 : 12);
 
+        // Guard against excessive result precision
+        // XXX Possibly better raised before result is allocated/sized.
+        if (precision > MAX_PRECISION) {
+            throw precisionTooLarge("float");
+        }
+
         /*
          * By default, the prefix of a positive number is "", but the format specifier may override
          * it, and the built-in type complex needs to override the format.
          */
-        if (positivePrefix == null && Spec.specified(spec.sign) && spec.sign != '-') {
-            positivePrefix = Character.toString(spec.sign);
+        char sign = spec.sign;
+        if (positivePrefix == null && Spec.specified(sign) && sign != '-') {
+            positivePrefix = Character.toString(sign);
         }
 
         // Different process for each format type, ignoring case for now.
@@ -905,8 +910,8 @@
     }
 
     /**
-     * Return the index in {@link #result} of the first letter. helper for {@link #uppercase()} and
-     * {@link #getExponent()}
+     * Return the index in {@link #result} of the first letter. This is a helper for
+     * {@link #uppercase()} and {@link #getExponent()}
      */
     private int indexOfMarker() {
         return start + lenSign + lenWhole + lenPoint + lenFraction;
diff --git a/src/org/python/core/stringlib/IntegerFormatter.java b/src/org/python/core/stringlib/IntegerFormatter.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/stringlib/IntegerFormatter.java
@@ -0,0 +1,779 @@
+// Copyright (c) Jython Developers
+package org.python.core.stringlib;
+
+import java.math.BigInteger;
+
+import org.python.core.Py;
+import org.python.core.PyInteger;
+import org.python.core.PyLong;
+import org.python.core.PyObject;
+import org.python.core.PyString;
+import org.python.core.PySystemState;
+import org.python.core.stringlib.InternalFormat.Spec;
+
+/**
+ * A class that provides the implementation of integer formatting. In a limited way, it acts like a
+ * StringBuilder to which text and one or more numbers may be appended, formatted according to the
+ * format specifier supplied at construction. These are ephemeral objects that are not, on their
+ * own, thread safe.
+ */
+public class IntegerFormatter extends InternalFormat.Formatter {
+
+    /**
+     * Construct the formatter from a client-supplied buffer, to which the result will be appended,
+     * and a specification. Sets {@link #mark} to the end of the buffer.
+     *
+     * @param result destination buffer
+     * @param spec parsed conversion specification
+     */
+    public IntegerFormatter(StringBuilder result, Spec spec) {
+        super(result, spec);
+    }
+
+    /**
+     * Construct the formatter from a specification, allocating a buffer internally for the result.
+     *
+     * @param spec parsed conversion specification
+     */
+    public IntegerFormatter(Spec spec) {
+        // Rule of thumb: big enough for 32-bit binary with base indicator 0b
+        this(new StringBuilder(34), spec);
+    }
+
+    /*
+     * Re-implement the text appends so they return the right type.
+     */
+    @Override
+    public IntegerFormatter append(char c) {
+        super.append(c);
+        return this;
+    }
+
+    @Override
+    public IntegerFormatter append(CharSequence csq) {
+        super.append(csq);
+        return this;
+    }
+
+    @Override
+    public IntegerFormatter append(CharSequence csq, int start, int end) //
+            throws IndexOutOfBoundsException {
+        super.append(csq, start, end);
+        return this;
+    }
+
+    /**
+     * Format a {@link BigInteger}, which is the implementation type of Jython long,
+     * according to the specification represented by this IntegerFormatter. The
+     * conversion type, and flags for grouping or base prefix are dealt with here. At the point this
+     * is used, we know the {@link #spec} is one of the integer types.
+     *
+     * @param value to convert
+     * @return this object
+     */
+    @SuppressWarnings("fallthrough")
+    public IntegerFormatter format(BigInteger value) {
+        try {
+            // Different process for each format type.
+            switch (spec.type) {
+                case 'd':
+                case Spec.NONE:
+                case 'u':
+                case 'i':
+                    // None format or d-format: decimal
+                    format_d(value);
+                    break;
+
+                case 'x':
+                    // hexadecimal.
+                    format_x(value, false);
+                    break;
+
+                case 'X':
+                    // HEXADECIMAL!
+                    format_x(value, true);
+                    break;
+
+                case 'o':
+                    // Octal.
+                    format_o(value);
+                    break;
+
+                case 'b':
+                    // Binary.
+                    format_b(value);
+                    break;
+
+                case 'c':
+                    // Binary.
+                    format_c(value);
+                    break;
+
+                case 'n':
+                    // Locale-sensitive version of d-format should be here.
+                    format_d(value);
+                    break;
+
+                default:
+                    // Should never get here, since this was checked in caller.
+                    throw unknownFormat(spec.type, "long");
+            }
+
+            // If required to, group the whole-part digits.
+            if (spec.grouping) {
+                groupDigits(3, ',');
+            }
+
+            return this;
+
+        } catch (OutOfMemoryError eme) {
+            // Most probably due to excessive precision.
+            throw precisionTooLarge("long");
+        }
+    }
+
+    /**
+     * Format the value as decimal (into {@link #result}). The option for mandatory sign is dealt
+     * with by reference to the format specification.
+     *
+     * @param value to convert
+     */
+    void format_d(BigInteger value) {
+        String number;
+        if (value.signum() < 0) {
+            // Negative value: deal with sign and base, and convert magnitude.
+            negativeSign(null);
+            number = value.negate().toString();
+        } else {
+            // Positive value: deal with sign, base and magnitude.
+            positiveSign(null);
+            number = value.toString();
+        }
+        appendNumber(number);
+    }
+
+    /**
+     * Format the value as hexadecimal (into {@link #result}), with the option of using upper-case
+     * or lower-case letters. The options for mandatory sign and for the presence of a base-prefix
+     * "0x" or "0X" are dealt with by reference to the format specification.
+     *
+     * @param value to convert
+     * @param upper if the hexadecimal should be upper case
+     */
+    void format_x(BigInteger value, boolean upper) {
+        String base = upper ? "0X" : "0x";
+        String number;
+        if (value.signum() < 0) {
+            // Negative value: deal with sign and base, and convert magnitude.
+            negativeSign(base);
+            number = toHexString(value.negate());
+        } else {
+            // Positive value: deal with sign, base and magnitude.
+            positiveSign(base);
+            number = toHexString(value);
+        }
+        // Append to result, case-shifted if necessary.
+        if (upper) {
+            number = number.toUpperCase();
+        }
+        appendNumber(number);
+    }
+
+    /**
+     * Format the value as octal (into {@link #result}). The options for mandatory sign and for the
+     * presence of a base-prefix "0o" are dealt with by reference to the format specification.
+     *
+     * @param value to convert
+     */
+    void format_o(BigInteger value) {
+        String base = "0o";
+        String number;
+        if (value.signum() < 0) {
+            // Negative value: deal with sign and base, and convert magnitude.
+            negativeSign(base);
+            number = toOctalString(value.negate());
+        } else {
+            // Positive value: deal with sign, base and magnitude.
+            positiveSign(base);
+            number = toOctalString(value);
+        }
+        // Append to result.
+        appendNumber(number);
+    }
+
+    /**
+     * Format the value as binary (into {@link #result}). The options for mandatory sign and for the
+     * presence of a base-prefix "0b" are dealt with by reference to the format specification.
+     *
+     * @param value to convert
+     */
+    void format_b(BigInteger value) {
+        String base = "0b";
+        String number;
+        if (value.signum() < 0) {
+            // Negative value: deal with sign and base, and convert magnitude.
+            negativeSign(base);
+            number = toBinaryString(value.negate());
+        } else {
+            // Positive value: deal with sign, base and magnitude.
+            positiveSign(base);
+            number = toBinaryString(value);
+        }
+        // Append to result.
+        appendNumber(number);
+    }
+
+    /**
+     * Format the value as a character (into {@link #result}).
+     *
+     * @param value to convert
+     */
+    void format_c(BigInteger value) {
+        // Limit is 256 if we're formatting for byte output, unicode range otherwise.
+        BigInteger limit = bytes ? LIMIT_BYTE : LIMIT_UNICODE;
+        if (value.signum() < 0 || value.compareTo(limit) >= 0) {
+            throw Py.OverflowError("%c arg not in range(0x" + toHexString(limit) + ")");
+        } else {
+            result.appendCodePoint(value.intValue());
+        }
+    }
+
+    // Limits used in format_c(BigInteger)
+    private static final BigInteger LIMIT_UNICODE = BigInteger
+            .valueOf(PySystemState.maxunicode + 1);
+    private static final BigInteger LIMIT_BYTE = BigInteger.valueOf(256);
+
+    /**
+     * Format an integer according to the specification represented by this
+     * IntegerFormatter. The conversion type, and flags for grouping or base prefix are
+     * dealt with here. At the point this is used, we know the {@link #spec} is one of the integer
+     * types.
+     *
+     * @param value to convert
+     * @return this object
+     */
+    @SuppressWarnings("fallthrough")
+    public IntegerFormatter format(int value) {
+        try {
+            // Scratch all instance variables and start = result.length().
+            setStart();
+
+            // Different process for each format type.
+            switch (spec.type) {
+                case 'd':
+                case Spec.NONE:
+                case 'u':
+                case 'i':
+                    // None format or d-format: decimal
+                    format_d(value);
+                    break;
+
+                case 'x':
+                    // hexadecimal.
+                    format_x(value, false);
+                    break;
+
+                case 'X':
+                    // HEXADECIMAL!
+                    format_x(value, true);
+                    break;
+
+                case 'o':
+                    // Octal.
+                    format_o(value);
+                    break;
+
+                case 'b':
+                    // Binary.
+                    format_b(value);
+                    break;
+
+                case 'c':
+                case '%':
+                    // Binary.
+                    format_c(value);
+                    break;
+
+                case 'n':
+                    // Locale-sensitive version of d-format should be here.
+                    format_d(value);
+                    break;
+
+                default:
+                    // Should never get here, since this was checked in caller.
+                    throw unknownFormat(spec.type, "integer");
+            }
+
+            // If required to, group the whole-part digits.
+            if (spec.grouping) {
+                groupDigits(3, ',');
+            }
+
+            return this;
+        } catch (OutOfMemoryError eme) {
+            // Most probably due to excessive precision.
+            throw precisionTooLarge("integer");
+        }
+    }
+
+    /**
+     * Format the value as decimal (into {@link #result}). The option for mandatory sign is dealt
+     * with by reference to the format specification.
+     *
+     * @param value to convert
+     */
+    void format_d(int value) {
+        String number;
+        if (value < 0) {
+            // Negative value: deal with sign and base, and convert magnitude.
+            negativeSign(null);
+            number = Integer.toString(-value);
+        } else {
+            // Positive value: deal with sign, base and magnitude.
+            positiveSign(null);
+            number = Integer.toString(value);
+        }
+        appendNumber(number);
+    }
+
+    /**
+     * Format the value as hexadecimal (into {@link #result}), with the option of using upper-case
+     * or lower-case letters. The options for mandatory sign and for the presence of a base-prefix
+     * "0x" or "0X" are dealt with by reference to the format specification.
+     *
+     * @param value to convert
+     * @param upper if the hexadecimal should be upper case
+     */
+    void format_x(int value, boolean upper) {
+        String base = upper ? "0X" : "0x";
+        String number;
+        if (value < 0) {
+            // Negative value: deal with sign and base, and convert magnitude.
+            negativeSign(base);
+            number = Integer.toHexString(-value);
+        } else {
+            // Positive value: deal with sign, base and magnitude.
+            positiveSign(base);
+            number = Integer.toHexString(value);
+        }
+        // Append to result, case-shifted if necessary.
+        if (upper) {
+            number = number.toUpperCase();
+        }
+        appendNumber(number);
+    }
+
+    /**
+     * Format the value as octal (into {@link #result}). The options for mandatory sign and for the
+     * presence of a base-prefix "0o" are dealt with by reference to the format specification.
+     *
+     * @param value to convert
+     */
+    void format_o(int value) {
+        String base = "0o";
+        String number;
+        if (value < 0) {
+            // Negative value: deal with sign and base, and convert magnitude.
+            negativeSign(base);
+            number = Integer.toOctalString(-value);
+        } else {
+            // Positive value: deal with sign, base and magnitude.
+            positiveSign(base);
+            number = Integer.toOctalString(value);
+        }
+        // Append to result.
+        appendNumber(number);
+    }
+
+    /**
+     * Format the value as binary (into {@link #result}). The options for mandatory sign and for the
+     * presence of a base-prefix "0b" are dealt with by reference to the format specification.
+     *
+     * @param value to convert
+     */
+    void format_b(int value) {
+        String base = "0b";
+        String number;
+        if (value < 0) {
+            // Negative value: deal with sign and base, and convert magnitude.
+            negativeSign(base);
+            number = Integer.toBinaryString(-value);
+        } else {
+            // Positive value: deal with sign, base and magnitude.
+            positiveSign(base);
+            number = Integer.toBinaryString(value);
+        }
+        // Append to result.
+        appendNumber(number);
+    }
+
+    /**
+     * Format the value as a character (into {@link #result}).
+     *
+     * @param value to convert
+     */
+    void format_c(int value) {
+        // Limit is 256 if we're formatting for byte output, unicode range otherwise.
+        int limit = bytes ? 256 : PySystemState.maxunicode + 1;
+        if (value < 0 || value >= limit) {
+            throw Py.OverflowError("%c arg not in range(0x" + Integer.toHexString(limit) + ")");
+        } else {
+            result.appendCodePoint(value);
+        }
+    }
+
+    /**
+     * Append to {@link #result} buffer a sign (if one is specified for positive numbers) and, in
+     * alternate mode, the base marker provided. The sign and base marker are together considered to
+     * be the "sign" of the converted number, spanned by {@link #lenSign}. This is relevant when we
+     * come to insert padding.
+     *
+     * @param base marker "0x" or "0X" for hex, "0o" for octal, "0b" for binary, "" or
+     *            null for decimal.
+     */
+    final void positiveSign(String base) {
+        // Does the format specify a sign for positive values?
+        char sign = spec.sign;
+        if (Spec.specified(sign) && sign != '-') {
+            append(sign);
+            lenSign = 1;
+        }
+        // Does the format call for a base prefix?
+        if (base != null && spec.alternate) {
+            append(base);
+            lenSign += base.length();
+        }
+    }
+
+    /**
+     * Append to {@link #result} buffer a minus sign and, in alternate mode, the base marker
+     * provided. The sign and base marker are together considered to be the "sign" of the converted
+     * number, spanned by {@link #lenSign}. This is relevant when we come to insert padding.
+     *
+     * @param base marker ("0x" or "0X" for hex, "0" for octal, null or "" for decimal.
+     */
+    final void negativeSign(String base) {
+        // Insert a minus sign unconditionally.
+        append('-');
+        lenSign = 1;
+        // Does the format call for a base prefix?
+        if (base != null && spec.alternate) {
+            append(base);
+            lenSign += base.length();
+        }
+    }
+
+    /**
+     * Append a string (number) to {@link #result} and set {@link #lenWhole} to its length .
+     *
+     * @param number to append
+     */
+    void appendNumber(String number) {
+        lenWhole = number.length();
+        append(number);
+    }
+
+    // For hex-conversion by lookup
+    private static final String LOOKUP = "0123456789abcdef";
+
+    /**
+     * A more efficient algorithm for generating a hexadecimal representation of a byte array.
+     * {@link BigInteger#toString(int)} is too slow because it generalizes to any radix and,
+     * consequently, is implemented using expensive mathematical operations.
+     *
+     * @param value the value to generate a hexadecimal string from
+     * @return the hexadecimal representation of value, with "-" sign prepended if necessary
+     */
+    private static final String toHexString(BigInteger value) {
+        int signum = value.signum();
+
+        // obvious shortcut
+        if (signum == 0) {
+            return "0";
+        }
+
+        // we want to work in absolute numeric value (negative sign is added afterward)
+        byte[] input = value.abs().toByteArray();
+        StringBuilder sb = new StringBuilder(input.length * 2);
+
+        int b;
+        for (int i = 0; i < input.length; i++) {
+            b = input[i] & 0xFF;
+            sb.append(LOOKUP.charAt(b >> 4));
+            sb.append(LOOKUP.charAt(b & 0x0F));
+        }
+
+        // before returning the char array as string, remove leading zeroes, but not the last one
+        String result = sb.toString().replaceFirst("^0+(?!$)", "");
+        return signum < 0 ? "-" + result : result;
+    }
+
+    /**
+     * A more efficient algorithm for generating an octal representation of a byte array.
+     * {@link BigInteger#toString(int)} is too slow because it generalizes to any radix and,
+     * consequently, is implemented using expensive mathematical operations.
+     *
+     * @param value the value to generate an octal string from
+     * @return the octal representation of value, with "-" sign prepended if necessary
+     */
+    private static final String toOctalString(BigInteger value) {
+        int signum = value.signum();
+
+        // obvious shortcut
+        if (signum == 0) {
+            return "0";
+        }
+
+        byte[] input = value.abs().toByteArray();
+        if (input.length < 3) {
+            return value.toString(8);
+        }
+
+        StringBuilder sb = new StringBuilder(input.length * 3);
+
+        // working backwards, three bytes at a time
+        int threebytes;
+        int trip1, trip2, trip3;    // most, middle, and least significant bytes in the triplet
+        for (int i = input.length - 1; i >= 0; i -= 3) {
+            trip3 = input[i] & 0xFF;
+            trip2 = ((i - 1) >= 0) ? (input[i - 1] & 0xFF) : 0x00;
+            trip1 = ((i - 2) >= 0) ? (input[i - 2] & 0xFF) : 0x00;
+            threebytes = trip3 | (trip2 << 8) | (trip1 << 16);
+
+            // convert the three-byte value into an eight-character octal string
+            for (int j = 0; j < 8; j++) {
+                sb.append(LOOKUP.charAt((threebytes >> (j * 3)) & 0x000007));
+            }
+        }
+
+        String result = sb.reverse().toString().replaceFirst("^0+(?!%)", "");
+        return signum < 0 ? "-" + result : result;
+    }
+
+    /**
+     * A more efficient algorithm for generating a binary representation of a byte array.
+     * {@link BigInteger#toString(int)} is too slow because it generalizes to any radix and,
+     * consequently, is implemented using expensive mathematical operations.
+     *
+     * @param value the value to generate a binary string from
+     * @return the binary representation of value, with "-" sign prepended if necessary
+     */
+    private static final String toBinaryString(BigInteger value) {
+        int signum = value.signum();
+
+        // obvious shortcut
+        if (signum == 0) {
+            return "0";
+        }
+
+        // we want to work in absolute numeric value (negative sign is added afterward)
+        byte[] input = value.abs().toByteArray();
+        StringBuilder sb = new StringBuilder(value.bitCount());
+
+        int b;
+        for (int i = 0; i < input.length; i++) {
+            b = input[i] & 0xFF;
+            for (int bit = 7; bit >= 0; bit--) {
+                sb.append(((b >> bit) & 0x1) > 0 ? "1" : "0");
+            }
+        }
+
+        // before returning the char array as string, remove leading zeroes, but not the last one
+        String result = sb.toString().replaceFirst("^0+(?!$)", "");
+        return signum < 0 ? "-" + result : result;
+    }
+
+    /** Format specification used by bin(). */
+    public static final Spec BIN = InternalFormat.fromText("#b");
+
+    /** Format specification used by oct(). */
+    public static final Spec OCT = InternalFormat.fromText("#o");
+
+    /** Format specification used by hex(). */
+    public static final Spec HEX = InternalFormat.fromText("#x");
+
+    /**
+     * Convert the object to binary according to the conventions of Python built-in
+     * bin(). The object's __index__ method is called, and is responsible for raising
+     * the appropriate error (which the base {@link PyObject#__index__()} does).
+     *
+     * @param number to convert
+     * @return PyString converted result
+     */
+    // Follow this pattern in Python 3, where objects no longer have __hex__, __oct__ members.
+    public static PyString bin(PyObject number) {
+        return formatNumber(number, BIN);
+    }
+
+    /**
+     * Convert the object according to the conventions of Python built-in hex(), or
+     * oct(). The object's __index__ method is called, and is responsible
+     * for raising the appropriate error (which the base {@link PyObject#__index__()} does).
+     *
+     * @param number to convert
+     * @return PyString converted result
+     */
+    public static PyString formatNumber(PyObject number, Spec spec) {
+        number = number.__index__();
+        IntegerFormatter f = new IntegerFormatter(spec);
+        if (number instanceof PyInteger) {
+            f.format(((PyInteger)number).getValue());
+        } else {
+            f.format(((PyLong)number).getValue());
+        }
+        return new PyString(f.getResult());
+    }
+
+    /**
+     * A minor variation on {@link IntegerFormatter} to handle "traditional" %-formatting. The
+     * difference is in support for spec.precision, the formatting octal in "alternate"
+     * mode (0 and 0123, not 0o0 and 0o123), and in c-format (in the error logic).
+     */
+    public static class Traditional extends IntegerFormatter {
+
+        /**
+         * Construct the formatter from a client-supplied buffer, to which the result will be
+         * appended, and a specification. Sets {@link #mark} to the end of the buffer.
+         *
+         * @param result destination buffer
+         * @param spec parsed conversion specification
+         */
+        public Traditional(StringBuilder result, Spec spec) {
+            super(result, spec);
+        }
+
+        /**
+         * Construct the formatter from a specification, allocating a buffer internally for the
+         * result.
+         *
+         * @param spec parsed conversion specification
+         */
+        public Traditional(Spec spec) {
+            this(new StringBuilder(), spec);
+        }
+
+        /**
+         * Format the value as octal (into {@link #result}). The options for mandatory sign and for
+         * the presence of a base-prefix "0" are dealt with by reference to the format
+         * specification.
+         *
+         * @param value to convert
+         */
+        @Override
+        void format_o(BigInteger value) {
+            String number;
+            int signum = value.signum();
+            if (signum < 0) {
+                // Negative value: deal with sign and base, and convert magnitude.
+                negativeSign(null);
+                number = toOctalString(value.negate());
+            } else {
+                // Positive value: deal with sign, base and magnitude.
+                positiveSign(null);
+                number = toOctalString(value);
+            }
+            // Append to result.
+            appendOctalNumber(number);
+        }
+
+        /**
+         * Format the value as a character (into {@link #result}).
+         *
+         * @param value to convert
+         */
+        @Override
+        void format_c(BigInteger value) {
+            if (value.signum() < 0) {
+                throw Py.OverflowError("unsigned byte integer is less than minimum");
+            } else {
+                // Limit is 256 if we're formatting for byte output, unicode range otherwise.
+                BigInteger limit = bytes ? LIMIT_BYTE : LIMIT_UNICODE;
+                if (value.compareTo(limit) >= 0) {
+                    throw Py.OverflowError("unsigned byte integer is greater than maximum");
+                } else {
+                    result.appendCodePoint(value.intValue());
+                }
+            }
+        }
+
+        /**
+         * Format the value as octal (into {@link #result}). The options for mandatory sign and for
+         * the presence of a base-prefix "0" are dealt with by reference to the format
+         * specification.
+         *
+         * @param value to convert
+         */
+        @Override
+        void format_o(int value) {
+            String number;
+            if (value < 0) {
+                // Negative value: deal with sign and convert magnitude.
+                negativeSign(null);
+                number = Integer.toOctalString(-value);
+            } else {
+                // Positive value: deal with sign, base and magnitude.
+                positiveSign(null);
+                number = Integer.toOctalString(value);
+            }
+            // Append to result.
+            appendOctalNumber(number);
+        }
+
+        /**
+         * Format the value as a character (into {@link #result}).
+         *
+         * @param value to convert
+         */
+        @Override
+        void format_c(int value) {
+            if (value < 0) {
+                throw Py.OverflowError("unsigned byte integer is less than minimum");
+            } else {
+                // Limit is 256 if we're formatting for byte output, unicode range otherwise.
+                int limit = bytes ? 256 : PySystemState.maxunicode + 1;
+                if (value >= limit) {
+                    throw Py.OverflowError("unsigned byte integer is greater than maximum");
+                } else {
+                    result.appendCodePoint(value);
+                }
+            }
+        }
+
+        /**
+         * Append a string (number) to {@link #result}, but insert leading zeros first in order
+         * that, on return, the whole-part length #lenWhole should be no less than the precision.
+         *
+         * @param number to append
+         */
+        @Override
+        void appendNumber(String number) {
+            int n, p = spec.getPrecision(0);
+            for (n = number.length(); n < p; n++) {
+                result.append('0');
+            }
+            lenWhole = n;
+            append(number);
+        }
+
+        /**
+         * Append a string (number) to {@link #result}, but insert leading zeros first in order
+         * that, on return, the whole-part length #lenWhole should be no less than the precision.
+         * Octal numbers must begin with a zero if spec.alternate==true, so if the
+         * number passed in does not start with a zero, at least one will be inserted.
+         *
+         * @param number to append
+         */
+        void appendOctalNumber(String number) {
+            int n = number.length(), p = spec.getPrecision(0);
+            if (spec.alternate && number.charAt(0) != '0' && n >= p) {
+                p = n + 1;
+            }
+            for (; n < p; n++) {
+                result.append('0');
+            }
+            lenWhole = n;
+            append(number);
+        }
+
+    }
+}
diff --git a/src/org/python/core/stringlib/InternalFormat.java b/src/org/python/core/stringlib/InternalFormat.java
--- a/src/org/python/core/stringlib/InternalFormat.java
+++ b/src/org/python/core/stringlib/InternalFormat.java
@@ -3,6 +3,9 @@
 
 import org.python.core.Py;
 import org.python.core.PyException;
+import org.python.core.PyObject;
+import org.python.core.PyString;
+import org.python.core.PyUnicode;
 
 public class InternalFormat {
 
@@ -14,7 +17,25 @@
      */
     public static Spec fromText(String text) {
         Parser parser = new Parser(text);
-        return parser.parse();
+        try {
+            return parser.parse();
+        } catch (IllegalArgumentException e) {
+            throw Py.ValueError(e.getMessage());
+        }
+    }
+
+    /**
+     * Create a {@link Spec} object by parsing a format specification, supplied as an object.
+     *
+     * @param text to parse
+     * @return parsed equivalent to text
+     */
+    public static Spec fromText(PyObject text, String method) {
+        if (text instanceof PyString) {
+            return fromText(((PyString)text).getString());
+        } else {
+            throw Py.TypeError(method + " requires str or unicode");
+        }
     }
 
     /**
@@ -30,23 +51,72 @@
         /** The (partial) result. */
         protected StringBuilder result;
 
-        /** The number we are working on floats at the end of the result, and starts here. */
+        /**
+         * Signals the client's intention to make a PyString (or other byte-like) interpretation of
+         * {@link #result}, rather than a PyUnicode one.
+         */
+        protected boolean bytes;
+
+        /** The start of the formatted data for padding purposes, <={@link #start} */
+        protected int mark;
+        /** The latest number we are working on floats at the end of the result, and starts here. */
         protected int start;
-        /** If it contains no sign, this length is zero, and 1 otherwise. */
+        /** If it contains no sign, this length is zero, and >0 otherwise. */
         protected int lenSign;
         /** The length of the whole part (to left of the decimal point or exponent) */
         protected int lenWhole;
 
         /**
-         * Construct the formatter from a specification and initial buffer capacity. A reference is
-         * held to this specification, but it will not be modified by the actions of this class.
+         * Construct the formatter from a client-supplied buffer and a specification. Sets
+         * {@link #mark} and {@link #start} to the end of the buffer. The new formatted object will
+         * therefore be appended there and, when the time comes, padding will be applied to (just)
+         * the new text.
+         *
+         * @param result destination buffer
+         * @param spec parsed conversion specification
+         */
+        public Formatter(StringBuilder result, Spec spec) {
+            this.spec = spec;
+            this.result = result;
+            this.start = this.mark = result.length();
+        }
+
+        /**
+         * Construct the formatter from a specification and initial buffer capacity. Sets
+         * {@link #mark} to the end of the buffer.
          *
          * @param spec parsed conversion specification
          * @param width of buffer initially
          */
         public Formatter(Spec spec, int width) {
-            this.spec = spec;
-            result = new StringBuilder(width);
+            this(new StringBuilder(width), spec);
+        }
+
+        /**
+         * Signals the client's intention to make a PyString (or other byte-like) interpretation of
+         * {@link #result}, rather than a PyUnicode one. Only formatters that could produce
+         * characters >255 are affected by this (e.g. c-format). Idiom:
+         *
+         * 
+         * MyFormatter f = new MyFormatter( InternalFormatter.fromText(formatSpec) );
+         * f.setBytes(!(formatSpec instanceof PyUnicode));
+         * // ... formatting work
+         * return f.getPyResult();
+         * 

+         *
+         * @param bytes true to signal the intention to make a byte-like interpretation
+         */
+        public void setBytes(boolean bytes) {
+            this.bytes = bytes;
+        }
+
+        /**
+         * Whether initialised for a byte-like interpretation.
+         *
+         * @return bytes attribute
+         */
+        public boolean isBytes() {
+            return bytes;
         }
 
         /**
@@ -58,6 +128,22 @@
             return result.toString();
         }
 
+        /**
+         * Convenience method to return the current result of the formatting, as a
+         * PyObject, either {@link PyString} or {@link PyUnicode} according to
+         * {@link #bytes}.
+         *
+         * @return formatted result
+         */
+        public PyString getPyResult() {
+            String r = getResult();
+            if (bytes) {
+                return new PyString(r);
+            } else {
+                return new PyUnicode(r);
+            }
+        }
+
         /*
          * Implement Appendable interface by delegation to the result buffer.
          *
@@ -84,21 +170,28 @@
 
         /**
          * Clear the instance variables describing the latest object in {@link #result}, ready to
-         * receive a new number
+         * receive a new one: sets {@link #start} and calls {@link #reset()}. This is necessary when
+         * a Formatter is to be re-used. Note that this leaves {@link #mark} where it
+         * is. In the core, we need this to support complex: two floats in the same
+         * format, but padded as a unit.
          */
         public void setStart() {
-            // Mark the end of the buffer as the start of the current object and reset all.
+            // The new value will float at the current end of the result buffer.
             start = result.length();
-            // Clear the variable describing the latest number in result.
-            reset();
+            // If anything has been added since construction, reset all state.
+            if (start > mark) {
+                // Clear the variable describing the latest number in result.
+                reset();
+            }
         }
 
         /**
          * Clear the instance variables describing the latest object in {@link #result}, ready to
-         * receive a new one.
+         * receive a new one. This is called from {@link #setStart()}. Subclasses override this
+         * method and call {@link #setStart()} at the start of their format method.
          */
         protected void reset() {
-            // Clear the variable describing the latest object in result.
+            // Clear the variables describing the latest object in result.
             lenSign = lenWhole = 0;
         }
 
@@ -215,19 +308,19 @@
         }
 
         /**
-         * Pad the result so far (defined as the entire contents of {@link #result}) using the
-         * alignment, target width and fill character defined in {@link #spec}. The action of
-         * padding will increase the overall length of the result to the target width, if that is
-         * greater than the current length.
+         * Pad the result so far (defined as the contents of {@link #result} from {@link #mark} to
+         * the end) using the alignment, target width and fill character defined in {@link #spec}.
+         * The action of padding will increase the length of this segment to the target width, if
+         * that is greater than the current length.
          * 

          * When the padding method has decided that that it needs to add n padding characters, it
-         * will affect {@link #start} or {@link #lenSign} as follows.
+         * will affect {@link #start} or {@link #lenWhole} as follows.
          * 
          * 
          * 
          * 
          * 
-         * 
+         * 
          * 
          * 
          * 
@@ -259,69 +352,79 @@
          * 
          * 
          * 
alignmeaningstartlenSignlenWholeresult.length()
+n

-         * Note that we may have converted more than one value into the result buffer (for example
-         * when formatting a complex number). The pointer start is at the start of the
-         * last number converted. Padding with zeros, and the "pad after sign" mode, will produce a
-         * result you probably don't want. It is up to the client to disallow this (which
-         * complex does).
+         * Note that in the "pad after sign" mode, only the last number into the buffer receives the
+         * padding. This padding gets incorporated into the whole part of the number. (In other
+         * modes, the padding is around result[mark:].) When this would not be
+         * appropriate, it is up to the client to disallow this (which complex does).
          *
-         * @return this object
+         * @return this Formatter object
          */
         public Formatter pad() {
+            // We'll need this many pad characters (if>0). Note Spec.UNDEFINED<0.
+            int n = spec.width - (result.length() - mark);
+            if (n > 0) {
+                pad(mark, n);
+            }
+            return this;
+        }
 
-            // We'll need this many pad characters (if>0). Note Spec.UNDEFINED<0.
-            int n = spec.width - result.length();
-            if (n > 0) {
+        /**
+         * Pad the last result (defined as the contents of {@link #result} from argument
+         * leftIndex to the end) using the alignment, by n repetitions of
+         * the fill character defined in {@link #spec}, and distributed according to
+         * spec.align. The value of leftIndex is only used if the
+         * alignment is '>' (left) or '^' (both). The value of the critical lengths (lenWhole,
+         * lenSign, etc.) are not affected, because we assume that leftIndex <= 
+         * {@link #start}.
+         *
+         * @param leftIndex the index in result at which to insert left-fill characters.
+         * @param n number of fill characters to insert.
+         */
+        protected void pad(int leftIndex, int n) {
+            char align = spec.getAlign('>'); // Right for numbers (strings will supply '<' align)
+            char fill = spec.getFill(' ');
 
-                char align = spec.getAlign('>'); // Right for numbers (wrong for strings)
-                char fill = spec.getFill(' ');
+            // Start by assuming padding is all leading ('>' case or '=')
+            int leading = n;
 
-                // Start by assuming padding is all leading ('>' case or '=')
-                int leading = n;
+            // Split the total padding according to the alignment
+            if (align == '^') {
+                // Half the padding before
+                leading = n / 2;
+            } else if (align == '<') {
+                // All the padding after
+                leading = 0;
+            }
 
-                // Split the total padding according to the alignment
-                if (align == '^') {
-                    // Half the padding before
-                    leading = n / 2;
-                } else if (align == '<') {
-                    // All the padding after
-                    leading = 0;
+            // All padding that is not leading is trailing
+            int trailing = n - leading;
+
+            // Insert the leading space
+            if (leading > 0) {
+                if (align == '=') {
+                    // Incorporate into the (latest) whole part
+                    leftIndex = start + lenSign;
+                    lenWhole += leading;
+                } else {
+                    // Default is to insert at the stated leftIndex <= start.
+                    start += leading;
                 }
-
-                // All padding that is not leading is trailing
-                int trailing = n - leading;
-
-                // Insert the leading space
-                if (leading > 0) {
-                    int pos;
-                    if (align == '=') {
-                        // Incorporate into the (latest) whole part
-                        pos = start + lenSign;
-                        lenWhole += leading;
-                    } else {
-                        // Insert at the very beginning (not start) by default.
-                        pos = 0;
-                        start += leading;
-                    }
-                    makeSpaceAt(pos, leading);
-                    for (int i = 0; i < leading; i++) {
-                        result.setCharAt(pos + i, fill);
-                    }
-                }
-
-                // Append the trailing space
-                for (int i = 0; i < trailing; i++) {
-                    result.append(fill);
-                }
-
-                // Check for special case
-                if (align == '=' && fill == '0' && spec.grouping) {
-                    // We must extend the grouping separator into the padding
-                    zeroPadAfterSignWithGroupingFixup(3, ',');
+                makeSpaceAt(leftIndex, leading);
+                for (int i = 0; i < leading; i++) {
+                    result.setCharAt(leftIndex + i, fill);
                 }
             }
 
-            return this;
+            // Append the trailing space
+            for (int i = 0; i < trailing; i++) {
+                result.append(fill);
+            }
+
+            // Check for special case
+            if (align == '=' && fill == '0' && spec.grouping) {
+                // We must extend the grouping separator into the padding
+                zeroPadAfterSignWithGroupingFixup(3, ',');
+            }
         }
 
         /**
@@ -345,7 +448,7 @@
          *
* * The padding has increased the overall length of the result to the target width. About one - * in three call to this method adds one to the width, because the whole part cannot start + * in three calls to this method adds one to the width, because the whole part cannot start * with a comma. * * 
@@ -355,9 +458,6 @@
          * '-0,000,000,001,200,000,000.0000'
          *
* - * Insert grouping characters (conventionally commas) into the whole part of the number. - * {@link #lenWhole} will increase correspondingly. - * * @param groupSize normally 3. * @param comma or some other character to use as a separator. */ @@ -386,10 +486,9 @@ * Suppose the format call was format(-12e8, "0=30,.4f"). At the beginning, we had * something like this in result: . [-|000000000001,200,000,000|.|0000||] * - * And now, result looks like this: [-|0000,000,001,200,000,000|.|0000||] in which - * the first zero is wrong as it stands, nor can it just be over-written with a - * comma. We have to insert another zero, even though this makes the result longer - * than we were given. + * And now, result looks like this: [-|,000,000,001,200,000,000|.|0000||] in which + * the first comma is wrong, but so would be a zero. We have to insert another zero, + * even though this makes the result longer than we were asked for. */ result.insert(firstZero, '0'); lenWhole += 1; @@ -418,7 +517,19 @@ * @return exception to throw */ public static PyException alternateFormNotAllowed(String forType) { - return notAllowed("Alternate form (#)", forType); + return alternateFormNotAllowed(forType, '\0'); + } + + /** + * Convenience method returning a {@link Py#ValueError} reporting that alternate form is not + * allowed in a format specifier for the named type and specified typoe code. + * + * @param forType the type it was found applied to + * @param code the formatting code (or '\0' not to mention one) + * @return exception to throw + */ + public static PyException alternateFormNotAllowed(String forType, char code) { + return notAllowed("Alternate form (#)", forType, code); } /** @@ -430,7 +541,30 @@ * @return exception to throw */ public static PyException alignmentNotAllowed(char align, String forType) { - return notAllowed("'" + align + "' alignment flag", forType); + return notAllowed("'" + align + "' alignment flag", forType, '\0'); + } + + /** + * Convenience method returning a {@link Py#ValueError} reporting that specifying a sign is + * not allowed in a format specifier for the named type. + * + * @param forType the type it was found applied to + * @param code the formatting code (or '\0' not to mention one) + * @return exception to throw + */ + public static PyException signNotAllowed(String forType, char code) { + return notAllowed("Sign", forType, code); + } + + /** + * Convenience method returning a {@link Py#ValueError} reporting that specifying a + * precision is not allowed in a format specifier for the named type. + * + * @param forType the type it was found applied to + * @return exception to throw + */ + public static PyException precisionNotAllowed(String forType) { + return notAllowed("Precision", forType, '\0'); } /** @@ -441,22 +575,63 @@ * @return exception to throw */ public static PyException zeroPaddingNotAllowed(String forType) { - return notAllowed("Zero padding", forType); + return notAllowed("Zero padding", forType, '\0'); } /** * Convenience method returning a {@link Py#ValueError} reporting that some format specifier - * feature is not allowed for the named type. + * feature is not allowed for the named data type. * - * @param particularOutrage committed in the present case - * @param forType the type it where it is an outrage + * @param outrage committed in the present case + * @param forType the data type (e.g. "integer") it where it is an outrage * @return exception to throw */ - protected static PyException notAllowed(String particularOutrage, String forType) { - String msg = particularOutrage + " is not allowed in " + forType + " format specifier"; + public static PyException notAllowed(String outrage, String forType) { + return notAllowed(outrage, forType, '\0'); + } + + /** + * Convenience method returning a {@link Py#ValueError} reporting that some format specifier + * feature is not allowed for the named format code and data type. Produces a message like: + *

+ * outrage+" not allowed with "+forType+" format specifier '"+code+"'" + *

+ * outrage+" not allowed in "+forType+" format specifier" + * + * @param outrage committed in the present case + * @param forType the data type (e.g. "integer") it where it is an outrage + * @param code the formatting code for which it is an outrage (or '\0' not to mention one) + * @return exception to throw + */ + public static PyException notAllowed(String outrage, String forType, char code) { + // Try really hard to be like CPython + String codeAsString, withOrIn; + if (code == 0) { + withOrIn = "in "; + codeAsString = ""; + } else { + withOrIn = "with "; + codeAsString = " '" + code + "'"; + } + String msg = + outrage + " not allowed " + withOrIn + forType + " format specifier" + + codeAsString; return Py.ValueError(msg); } + /** + * Convenience method returning a {@link Py#OverflowError} reporting: + *

+ * "formatted "+type+" is too long (precision too large?)" + * + * @param type of formatting ("integer", "float") + * @return exception to throw + */ + public static PyException precisionTooLarge(String type) { + String msg = "formatted " + type + " is too long (precision too large?)"; + return Py.OverflowError(msg); + } + } /** @@ -636,6 +811,12 @@ false, Spec.UNSPECIFIED, Spec.NONE); /** + * Defaults applicable to string types. Equivalent to " <" + */ + public static final Spec STRING = new Spec(' ', '<', Spec.NONE, false, Spec.UNSPECIFIED, + false, Spec.UNSPECIFIED, Spec.NONE); + + /** * Constructor offering just precision and type. * * 

@@ -775,11 +956,6 @@
                 throw new IllegalArgumentException("Invalid conversion specification");
             }
 
-            // Restrict grouping to known formats. (Mirrors CPython, but misplaced?)
-            if (grouping && "defgEG%F\0".indexOf(type) == -1) {
-                throw new IllegalArgumentException("Cannot specify ',' with '" + type + "'.");
-            }
-
             // Create a specification
             return new Spec(fill, align, sign, alternate, width, grouping, precision, type);
         }
diff --git a/src/org/python/core/stringlib/InternalFormatSpec.java b/src/org/python/core/stringlib/InternalFormatSpec.java
deleted file mode 100644
--- a/src/org/python/core/stringlib/InternalFormatSpec.java
+++ /dev/null
@@ -1,88 +0,0 @@
-package org.python.core.stringlib;
-
-/**
- * Parsed PEP-3101 format specification of a single field. This class holds the several attributes
- * that might be decoded from a format specifier. It provides a method
- * {@link #pad(String, char, int)} for adjusting a string using those attributes related to padding
- * to a string assumed to be the result of formatting to the given precision.
- * 

- * This structure is returned by {@link InternalFormatSpecParser#parse()} and having public members
- * is freely used by {@link InternalFormatSpecParser}, and the __format__ methods of client object
- * types.
- * 

- * The fields correspond to the elements of a format specification. The grammar of a format
- * specification is:
- *
- * 
- * [[fill]align][sign][#][0][width][,][.precision][type]
- * 

- */
-public final class InternalFormatSpec {
-
-    /** The fill specified in the grammar. */
-    public char fill_char;
-    /** Alignment indicator is 0, or one of {'<', '^', '>', '=' . */
-    public char align;
-    /** The alternative format flag '#' was given. */
-    public boolean alternate;
-    /** Sign-handling flag, one of '+', '-', or ' '. */
-    public char sign;
-    /** Width to which to pad the resault in {@link #pad(String, char, int)}. */
-    public int width = -1;
-    /** Insert the grouping separator (which in Python always indicates a group-size of 3). */
-    public boolean thousands_separators;
-    /** Precision decoded from the format. */
-    public int precision = -1;
-    /** Type key from the format. */
-    public char type;
-
-    /**
-     * Pad value, using {@link #fill_char} (or ' ') before and after, to {@link #width}
-     * -leaveWidth, aligned according to {@link #align} (or according to
-     * defaultAlign).
-     *
-     * @param value to pad
-     * @param defaultAlign to use if this.align=0 (one of '<',
-     *            '^', '>', or '=').
-     * @param leaveWidth to reduce effective this.width by
-     * @return padded value
-     */
-    public String pad(String value, char defaultAlign, int leaveWidth) {
-
-        // We'll need this many pad characters (if>0)
-        int remaining = width - value.length() - leaveWidth;
-        if (remaining <= 0) {
-            return value;
-        }
-
-        // Use this.align or defaultAlign
-        int useAlign = align;
-        if (useAlign == 0) {
-            useAlign = defaultAlign;
-        }
-
-        // By default all padding is leading padding ('<' case or '=')
-        int leading = remaining;
-        if (useAlign == '^') {
-            // Half the padding before
-            leading = remaining / 2;
-        } else if (useAlign == '<') {
-            // All the padding after
-            leading = 0;
-        }
-
-        // Now build the result
-        StringBuilder result = new StringBuilder();
-        char fill = fill_char != 0 ? fill_char : ' ';
-
-        for (int i = 0; i < leading; i++) { // before
-            result.append(fill);
-        }
-        result.append(value);
-        for (int i = 0; i < remaining - leading; i++) { // after
-            result.append(fill);
-        }
-
-        return result.toString();
-    }
-}
diff --git a/src/org/python/core/stringlib/InternalFormatSpecParser.java b/src/org/python/core/stringlib/InternalFormatSpecParser.java
deleted file mode 100644
--- a/src/org/python/core/stringlib/InternalFormatSpecParser.java
+++ /dev/null
@@ -1,118 +0,0 @@
-package org.python.core.stringlib;
-
-/**
- * Parser for PEP-3101 field format specifications. This class provides a {@link #parse()} method
- * that translates the format specification into an InternalFormatSpec object.
- */
-public class InternalFormatSpecParser {
-
-    private String spec;
-    private int index;
-
-    /**
-     * Constructor simply holds the specification streang ahead of the {@link #parse()} operation.
-     *
-     * @param spec format specifier to parse (e.g. "<+12.3f")
-     */
-    public InternalFormatSpecParser(String spec) {
-        this.spec = spec;
-        this.index = 0;
-    }
-
-    private static boolean isAlign(char c) {
-        switch (c) {
-            case '<':
-            case '>':
-            case '=':
-            case '^':
-                return true;
-            default:
-                return false;
-        }
-    }
-
-    /**
-     * Parse the specification with which this object was initialised into an
-     * {@link InternalFormatSpec}, which is an object encapsulating the format for use by formatting
-     * methods. This parser deals only with the format specifiers themselves, as accepted by the
-     * __format__ method of a type, or the format() built-in, not format
-     * strings in general as accepted by str.format(). A typical idiom is:
-     *
-     * 
-     * InternalFormatSpec spec = new InternalFormatSpecParser(specString).parse();
-     * 

-     *
-     * @return the InternalFormatSpec equivalent to the constructor argument
-     */
-    /*
-     * This method is the equivalent of CPython's parse_internal_render_format_spec() in
-     * ~/Objects/stringlib/formatter.h.
-     */
-    // XXX Better encapsulated as a constructor of InternalFormatSpec?
-    public InternalFormatSpec parse() {
-        InternalFormatSpec result = new InternalFormatSpec();
-        if (spec.length() >= 1 && isAlign(spec.charAt(0))) {
-            result.align = spec.charAt(index);
-            index++;
-        } else if (spec.length() >= 2 && isAlign(spec.charAt(1))) {
-            result.fill_char = spec.charAt(0);
-            result.align = spec.charAt(1);
-            index += 2;
-        }
-        if (isAt("+- ")) {
-            result.sign = spec.charAt(index);
-            index++;
-        }
-        if (isAt("#")) {
-            result.alternate = true;
-            index++;
-        }
-        if (result.fill_char == '\0' && isAt("0")) {
-            result.fill_char = '0';
-            if (result.align == '\0') {
-                result.align = '=';
-            }
-            index++;
-        }
-        result.width = getInteger();
-        if (isAt(",")) {
-            result.thousands_separators = true;
-            index++;
-        }
-        if (isAt(".")) {
-            index++;
-            result.precision = getInteger();
-            if (result.precision == -1) {
-                throw new IllegalArgumentException("Format specifier missing precision");
-            }
-        }
-        if (index < spec.length()) {
-            result.type = spec.charAt(index);
-            if (index + 1 != spec.length()) {
-                throw new IllegalArgumentException("Invalid conversion specification");
-            }
-        }
-        if (result.thousands_separators && "defgEG%F\0".indexOf(result.type) == -1) {
-            throw new IllegalArgumentException("Cannot specify ',' with '" + result.type + "'.");
-        }
-        return result;
-    }
-
-    private int getInteger() {
-        int value = 0;
-        boolean empty = true;
-        while (index < spec.length() && spec.charAt(index) >= '0' && spec.charAt(index) <= '9') {
-            value = value * 10 + spec.charAt(index) - '0';
-            index++;
-            empty = false;
-        }
-        if (empty) {
-            return -1;
-        }
-        return value;
-    }
-
-    private boolean isAt(String chars) {
-        return index < spec.length() && chars.indexOf(spec.charAt(index)) >= 0;
-    }
-}
diff --git a/src/org/python/core/stringlib/TextFormatter.java b/src/org/python/core/stringlib/TextFormatter.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/stringlib/TextFormatter.java
@@ -0,0 +1,119 @@
+// Copyright (c) Jython Developers
+package org.python.core.stringlib;
+
+import org.python.core.stringlib.InternalFormat.Formatter;
+import org.python.core.stringlib.InternalFormat.Spec;
+
+/**
+ * A class that provides the implementation of str and unicode formatting.
+ * In a limited way, it acts like a StringBuilder to which text, formatted according to the format
+ * specifier supplied at construction. These are ephemeral objects that are not, on their own,
+ * thread safe.
+ */
+public class TextFormatter extends InternalFormat.Formatter {
+
+    /**
+     * Construct the formatter from a client-supplied buffer, to which the result will be appended,
+     * and a specification. Sets {@link #mark} to the end of the buffer.
+     *
+     * @param result destination buffer
+     * @param spec parsed conversion specification
+     */
+    public TextFormatter(StringBuilder result, Spec spec) {
+        super(result, spec);
+    }
+
+    /**
+     * Construct the formatter from a specification, allocating a buffer internally for the result.
+     *
+     * @param spec parsed conversion specification
+     */
+    public TextFormatter(Spec spec) {
+        this(new StringBuilder(), spec);
+    }
+
+    /*
+     * Re-implement the text appends so they return the right type.
+     */
+    @Override
+    public TextFormatter append(char c) {
+        super.append(c);
+        return this;
+    }
+
+    @Override
+    public TextFormatter append(CharSequence csq) {
+        super.append(csq);
+        return this;
+    }
+
+    @Override
+    public TextFormatter append(CharSequence csq, int start, int end) //
+            throws IndexOutOfBoundsException {
+        super.append(csq, start, end);
+        return this;
+    }
+
+    /**
+     * Format the given String into the result buffer. Largely, this is a
+     * matter of copying the value of the argument, but a subtlety arises when the string contains
+     * supplementary (non-BMP) Unicode characters, which are represented as surrogate pairs. The
+     * precision specified in the format relates to a count of Unicode characters (code points), not
+     * Java chars. The method deals with this correctly, essentially by not counting
+     * the high-surrogates in the allowance. The final value of {@link #lenWhole} counts the UTF-16
+     * units added.
+     *
+     * @param value to format
+     * @return this TextFormatter object
+     */
+    public TextFormatter format(String value) {
+
+        // Scratch all instance variables and start = result.length().
+        setStart();
+
+        int p = spec.precision, n = value.length();
+
+        if (Spec.specified(p) && p < n) {
+            // Estimate the space for the converted result (preempt multiple re-allocation)
+            int space = Math.max(spec.width, p);
+            result.ensureCapacity(result.length() + space + (bytes ? 0 : space / 4));
+            /*
+             * A precision p was specified less than the length: we may have to truncate. Note we
+             * compared p with the UTF-16 length, even though it is the code point length that
+             * matters. But the code point length cannot be greater than n.
+             */
+            int count = 0;
+            while (count < p) {
+                // count is the number of UTF-16 chars.
+                char c = value.charAt(count++);
+                result.append(c);
+                // A high-surrogate will always be followed by a low, so doesn't count.
+                if (Character.isHighSurrogate(c) && p < n) {
+                    // Accomplish "not counting" by bumping the limit p, within the array bounds.
+                    p += 1;
+                }
+            }
+            // Record the UTF-16 count as the length in buffer
+            lenWhole = count;
+
+        } else {
+            // We definitely don't need to truncate. Append the whole string.
+            lenWhole = n;
+            result.append(value);
+        }
+
+        return this;
+    }
+
+    // Variant to deal with supplementary characters: other formatters don't produce them.
+    @Override
+    public TextFormatter pad() {
+        // We'll need this many pad characters (if>0). Note Spec.UNDEFINED<0.
+        int n = spec.width - result.codePointCount(mark, result.length());
+        if (n > 0) {
+            pad(mark, n);
+        }
+        return this;
+    }
+
+}
diff --git a/tests/java/org/python/core/StringFormatTest.java b/tests/java/org/python/core/StringFormatTest.java
--- a/tests/java/org/python/core/StringFormatTest.java
+++ b/tests/java/org/python/core/StringFormatTest.java
@@ -1,43 +1,58 @@
 package org.python.core;
 
+import java.math.BigInteger;
+
 import junit.framework.TestCase;
+
 import org.python.core.stringlib.FieldNameIterator;
-import org.python.core.stringlib.InternalFormatSpec;
-import org.python.core.stringlib.InternalFormatSpecParser;
+import org.python.core.stringlib.IntegerFormatter;
+import org.python.core.stringlib.InternalFormat;
 import org.python.core.stringlib.MarkupIterator;
+import org.python.core.stringlib.TextFormatter;
+import org.python.core.stringlib.InternalFormat.Spec;
+import org.python.util.PythonInterpreter;
 
 /**
  * Tests for internal bits and pieces of string.format implementation.
  */
 public class StringFormatTest extends TestCase {
+
+    /** Exception-raising seems to need the interpreter to be initialised **/
+    PythonInterpreter interp = new PythonInterpreter();
+
     public void testInternalFormatSpec() {
-        InternalFormatSpec spec = new InternalFormatSpecParser("x").parse();
+        InternalFormat.Spec spec;
+        spec = InternalFormat.fromText("x");
+        assertFalse(Spec.specified(spec.align));
+        assertFalse(Spec.specified(spec.fill));
+        assertFalse(Spec.specified(spec.width));
+        assertFalse(Spec.specified(spec.precision));
         assertEquals('x', spec.type);
 
-        spec = new InternalFormatSpecParser("testFormatIntOrLong which tested PyInteger.formatIntOrLong.
+     */
+    public void testPrepareFormatter() {
+        int v = 123;
+        IntegerFormatter f;
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("d"));
+        assertEquals("123", f.format(v).pad().getResult());
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("o"));
+        assertEquals("173", f.format(v).pad().getResult());
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("x"));
+        assertEquals("7b", f.format(v).pad().getResult());
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("X"));
+        assertEquals("7B", f.format(v).pad().getResult());
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("b"));
+        assertEquals("1111011", f.format(v).pad().getResult());
 
-        spec.thousands_separators = true;
-        spec.type = 'd';
-        assertEquals("1,234", PyInteger.formatIntOrLong(1234, spec));
-        spec.thousands_separators = false;
+        int v2 = 1234567890;
+        f = PyInteger.prepareFormatter(InternalFormat.fromText(",d"));
+        assertEquals("1,234,567,890", f.format(v2).pad().getResult());
 
-        spec.alternate = true;
-        spec.type = 'o';
-        assertEquals("0o173", PyInteger.formatIntOrLong(123, spec));
-        spec.type = 'X';
-        assertEquals("0X7B", PyInteger.formatIntOrLong(123, spec));
-        spec.alternate = false;
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("#o"));
+        assertEquals("0o173", f.format(v).pad().getResult());
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("#X"));
+        assertEquals("0X7B", f.format(v).pad().getResult());
 
-        spec.type = 'c';
-        assertEquals("{", PyInteger.formatIntOrLong(123, spec));
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("c"));
+        assertEquals("{", f.format(v).pad().getResult());
 
-        spec.type = 'd';
-        spec.sign = '+';
-        assertEquals("+123", PyInteger.formatIntOrLong(123, spec));
-        spec.sign = ' ';
-        assertEquals(" 123", PyInteger.formatIntOrLong(123, spec));
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("+d"));
+        assertEquals("+123", f.format(v).pad().getResult());
+        f = PyInteger.prepareFormatter(InternalFormat.fromText(" d"));
+        assertEquals(" 123", f.format(v).pad().getResult());
 
-        spec.sign = 0;
-        spec.width = 5;
-        assertEquals("  123", PyInteger.formatIntOrLong(123, spec));
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("5"));
+        assertEquals("  123", f.format(v).pad().getResult());
 
-        spec.align = '^';
-        spec.width = 6;
-        assertEquals(" 123  ", PyInteger.formatIntOrLong(123, spec));
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("^6"));
+        assertEquals(" 123  ", f.format(v).pad().getResult());
 
-        spec.align = '<';
-        spec.width = 5;
-        spec.fill_char = '~';
-        assertEquals("123~~", PyInteger.formatIntOrLong(123, spec));
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("~<5"));
+        assertEquals("123~~", f.format(v).pad().getResult());
 
-        spec.align = '=';
-        spec.width = 6;
-        spec.fill_char = '0';
-        spec.sign = '+';
-        assertEquals("+00123", PyInteger.formatIntOrLong(123, spec));
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("0=+6"));
+        assertEquals("+00123", f.format(v).pad().getResult());
 
-        spec.precision = 1;
-        assertFormatError(123, spec, "Precision not allowed in integer format specifier");
+        assertValueError("0=+6.1", "Precision not allowed in integer format specifier");
+        assertValueError("+c", "Sign not allowed with integer format specifier 'c'");
 
-        spec.precision = -1;
-        spec.sign = '+';
-        spec.type = 'c';
-        assertFormatError(123, spec, "Sign not allowed with integer format specifier 'c'");
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("c"));
+        f.setBytes(true);
+        assertOverflowError(256, f, "%c arg not in range(0x100)");
+        assertOverflowError(-1, f, "%c arg not in range(0x100)");
+        assertOverflowError(0x110000, f, "%c arg not in range(0x100)");
 
-        spec.sign = 0;
-        assertFormatError(0x11111, spec, "%c arg not in range(0x10000)");
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("c"));
+        assertOverflowError(0x110000, f, "%c arg not in range(0x110000)");
+        assertOverflowError(-1, f, "%c arg not in range(0x110000)");
+    }
+
+    /**
+     * Test the IntegerFormatter returned by {@link PyInteger#prepareFormat}. This is based on the original
+     * testFormatIntOrLong which tested PyInteger.formatIntOrLong.
+     */
+    public void testPrepareFormatterLong() {
+        BigInteger v = BigInteger.valueOf(123);
+        IntegerFormatter f;
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("d"));
+        assertEquals("123", f.format(v).pad().getResult());
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("o"));
+        assertEquals("173", f.format(v).pad().getResult());
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("x"));
+        assertEquals("7b", f.format(v).pad().getResult());
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("X"));
+        assertEquals("7B", f.format(v).pad().getResult());
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("b"));
+        assertEquals("1111011", f.format(v).pad().getResult());
+
+        BigInteger v2 = BigInteger.valueOf(1234567890);
+        f = PyInteger.prepareFormatter(InternalFormat.fromText(",d"));
+        assertEquals("1,234,567,890", f.format(v2).pad().getResult());
+
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("#o"));
+        assertEquals("0o173", f.format(v).pad().getResult());
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("#X"));
+        assertEquals("0X7B", f.format(v).pad().getResult());
+
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("c"));
+        assertEquals("{", f.format(v).pad().getResult());
+
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("+d"));
+        assertEquals("+123", f.format(v).pad().getResult());
+        f = PyInteger.prepareFormatter(InternalFormat.fromText(" d"));
+        assertEquals(" 123", f.format(v).pad().getResult());
+
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("5"));
+        assertEquals("  123", f.format(v).pad().getResult());
+
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("^6"));
+        assertEquals(" 123  ", f.format(v).pad().getResult());
+
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("~<5"));
+        assertEquals("123~~", f.format(v).pad().getResult());
+
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("0=+6"));
+        assertEquals("+00123", f.format(v).pad().getResult());
+
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("c"));
+        f.setBytes(true);
+        assertOverflowError(BigInteger.valueOf(256), f, "%c arg not in range(0x100)");
+        assertOverflowError(BigInteger.valueOf(-1), f, "%c arg not in range(0x100)");
+        assertOverflowError(BigInteger.valueOf(0x110000), f, "%c arg not in range(0x100)");
+
+        f = PyInteger.prepareFormatter(InternalFormat.fromText("c"));
+        assertOverflowError(BigInteger.valueOf(0x110000), f, "%c arg not in range(0x110000)");
+        assertOverflowError(BigInteger.valueOf(-1), f, "%c arg not in range(0x110000)");
+    }
+
+    private void assertValueError(String formatSpec, String expected) {
+        try {
+            IntegerFormatter f = PyInteger.prepareFormatter(InternalFormat.fromText(formatSpec));
+            // f.format(123).pad().getResult();
+            fail("ValueError not thrown, expected: " + expected);
+        } catch (PyException pye) {
+            assertEquals(expected, pye.value.toString());
+        }
+    }
+
+    private void assertOverflowError(int v, IntegerFormatter f, String expected) {
+        // Test with Java int for PyInteger
+        try {
+            f.format(v).pad().getResult();
+            fail("OverflowError not thrown, expected: " + expected);
+        } catch (PyException pye) {
+            assertEquals(expected, pye.value.toString());
+        }
+    }
+
+    private void assertOverflowError(BigInteger v, IntegerFormatter f, String expected) {
+        // Test with BigInteger for PyLong
+        try {
+            f.format(v).pad().getResult();
+            fail("OverflowError not thrown, expected: " + expected);
+        } catch (PyException pye) {
+            assertEquals(expected, pye.value.toString());
+        }
     }
 
     public void testFormatString() {
-        InternalFormatSpec spec = new InternalFormatSpec();
-        assertEquals("abc", PyString.formatString("abc", spec));
+        String v = "abc";
+        TextFormatter f;
+        f = PyString.prepareFormatter(InternalFormat.fromText(""));
+        assertEquals("abc", f.format(v).pad().getResult());
 
-        spec.precision = 3;
-        assertEquals("abc", PyString.formatString("abcdef", spec));
+        String v2 = "abcdef";
+        f = PyString.prepareFormatter(InternalFormat.fromText(".3"));
+        assertEquals("abc", f.format(v2).pad().getResult());
 
-        spec.precision = -1;
-        spec.width = 6;
-        assertEquals("abc   ", PyString.formatString("abc", spec));
-    }
-
-    private void assertFormatError(int value, InternalFormatSpec spec, String expected) {
-        String error = null;
-        try {
-            PyInteger.formatIntOrLong(value, spec);
-        } catch (IllegalArgumentException e) {
-            error = e.getMessage();
-        }
-        assertEquals(expected, error);
+        f = PyString.prepareFormatter(InternalFormat.fromText("6"));
+        assertEquals("abc   ", f.format(v).pad().getResult());
     }
 
     public void testMarkupIterator() {

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Wed Jun 11 00:59:42 2014
From: jython-checkins at python.org (jeff.allen)
Date: Wed, 11 Jun 2014 00:59:42 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Use_Java_codec_throughout_p?=
 =?utf-8?q?arser_=28not_sometimes_a_Python_one=29=2C_see_issue_=232123=2E?=
Message-ID: <3gp6Gk4CQ1z7LjP@mail.python.org>

http://hg.python.org/jython/rev/eef3fcffc58a
changeset:   7288:eef3fcffc58a
user:        Jeff Allen 
date:        Tue Jun 10 23:40:28 2014 +0100
summary:
  Use Java codec throughout parser (not sometimes a Python one), see issue #2123.
Allows at least some programs to run where the console encoding is not one
supported by Jython with a Python codec.

files:
  src/org/python/antlr/GrammarActions.java |  71 ++++++-----
  src/org/python/core/ParserFacade.java    |  22 ++-
  2 files changed, 53 insertions(+), 40 deletions(-)


diff --git a/src/org/python/antlr/GrammarActions.java b/src/org/python/antlr/GrammarActions.java
--- a/src/org/python/antlr/GrammarActions.java
+++ b/src/org/python/antlr/GrammarActions.java
@@ -1,28 +1,18 @@
 package org.python.antlr;
 
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
 import org.antlr.runtime.Token;
-
-import org.python.core.Py;
-import org.python.core.PyComplex;
-import org.python.core.PyFloat;
-import org.python.core.PyInteger;
-import org.python.core.PyLong;
-import org.python.core.PyString;
-import org.python.core.PyUnicode;
-import org.python.core.codecs;
-import org.python.antlr.ast.alias;
-import org.python.antlr.ast.arguments;
-import org.python.antlr.ast.boolopType;
-import org.python.antlr.ast.cmpopType;
-import org.python.antlr.ast.expr_contextType;
-import org.python.antlr.ast.operatorType;
-import org.python.antlr.ast.unaryopType;
-import org.python.antlr.ast.Context;
-import org.python.antlr.ast.keyword;
 import org.python.antlr.ast.Attribute;
 import org.python.antlr.ast.BinOp;
 import org.python.antlr.ast.BoolOp;
 import org.python.antlr.ast.Call;
+import org.python.antlr.ast.Context;
 import org.python.antlr.ast.DictComp;
 import org.python.antlr.ast.ExtSlice;
 import org.python.antlr.ast.For;
@@ -34,26 +24,38 @@
 import org.python.antlr.ast.ListComp;
 import org.python.antlr.ast.Name;
 import org.python.antlr.ast.Num;
+import org.python.antlr.ast.Repr;
+import org.python.antlr.ast.SetComp;
 import org.python.antlr.ast.Slice;
+import org.python.antlr.ast.Str;
 import org.python.antlr.ast.TryExcept;
 import org.python.antlr.ast.TryFinally;
 import org.python.antlr.ast.Tuple;
-import org.python.antlr.ast.Repr;
-import org.python.antlr.ast.SetComp;
-import org.python.antlr.ast.Str;
 import org.python.antlr.ast.UnaryOp;
 import org.python.antlr.ast.While;
 import org.python.antlr.ast.With;
 import org.python.antlr.ast.Yield;
+import org.python.antlr.ast.alias;
+import org.python.antlr.ast.arguments;
+import org.python.antlr.ast.boolopType;
+import org.python.antlr.ast.cmpopType;
+import org.python.antlr.ast.expr_contextType;
+import org.python.antlr.ast.keyword;
+import org.python.antlr.ast.operatorType;
+import org.python.antlr.ast.unaryopType;
 import org.python.antlr.base.excepthandler;
 import org.python.antlr.base.expr;
 import org.python.antlr.base.slice;
 import org.python.antlr.base.stmt;
-
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
+import org.python.core.Py;
+import org.python.core.PyComplex;
+import org.python.core.PyFloat;
+import org.python.core.PyInteger;
+import org.python.core.PyLong;
+import org.python.core.PyString;
+import org.python.core.PyUnicode;
+import org.python.core.codecs;
+import org.python.core.util.StringUtil;
 
 public class GrammarActions {
     private ErrorHandler errorHandler = null;
@@ -183,7 +185,7 @@
         }
         return result;
     }
-    
+
     List makeElse(List elseSuite, PythonTree elif) {
         if (elseSuite != null) {
             return castStmts(elseSuite);
@@ -425,8 +427,9 @@
         }
         int ndigits = s.length();
         int i=0;
-        while (i < ndigits && s.charAt(i) == '0')
+        while (i < ndigits && s.charAt(i) == '0') {
             i++;
+        }
         if ((ndigits - i) > 11) {
             return Py.newLong(new BigInteger(s, radix));
         }
@@ -449,7 +452,7 @@
         String getString() {
             return s;
         }
-        
+
         boolean isUnicode() {
             return unicode;
         }
@@ -511,8 +514,10 @@
         // XXX: No need to re-encode when the encoding is iso-8859-1, but ParserFacade
         // needs to normalize the encoding name
         if (!ustring && encoding != null) {
-            // str with a specified encoding: first re-encode back out
-            string = new PyUnicode(string.substring(start, end)).encode(encoding);
+            // The parser used a non-latin encoding: re-encode chars to bytes.
+            Charset cs = Charset.forName(encoding);
+            ByteBuffer decoded = cs.encode(string.substring(start, end));
+            string = StringUtil.fromBytes(decoded);
             if (!raw) {
                 // Handle escapes in non-raw strs
                 string = PyString.decode_UnicodeEscape(string, 0, string.length(), "strict",
@@ -744,7 +749,7 @@
         }
         return result;
     }
-    
+
     BoolOp makeBoolOp(Token t, PythonTree left, boolopType op, List right) {
         List values = new ArrayList();
         values.add(left);
@@ -780,7 +785,7 @@
         }
         return result;
     }
- 
+
     slice castSlice(Object o) {
         if (o instanceof slice) {
             return (slice)o;
diff --git a/src/org/python/core/ParserFacade.java b/src/org/python/core/ParserFacade.java
--- a/src/org/python/core/ParserFacade.java
+++ b/src/org/python/core/ParserFacade.java
@@ -9,6 +9,7 @@
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.StringReader;
+import java.nio.ByteBuffer;
 import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
@@ -54,8 +55,10 @@
                 return text;
             }
             if (reader.encoding != null) {
-                // restore the original encoding
-                text = new PyUnicode(text).encode(reader.encoding);
+                // The parser used a non-latin encoding: re-encode chars to bytes.
+                Charset cs = Charset.forName(reader.encoding);
+                ByteBuffer decoded = cs.encode(text);
+                text = StringUtil.fromBytes(decoded);
             }
             return text + "\n";
         } catch (IOException ioe) {
@@ -100,8 +103,9 @@
                         + reader.encoding + "'";
             }
             throw Py.SyntaxError(msg);
+        } else {
+            return Py.JavaError(t);
         }
-        else return Py.JavaError(t);
     }
 
     /**
@@ -121,7 +125,9 @@
             return parse(bufReader, CompileMode.eval, filename, cflags);
         } catch (Throwable t) {
             if (bufReader == null)
+             {
                 throw Py.JavaError(t); // can't do any more
+            }
             try {
                 // then, try parsing as a module
                 bufReader.reset();
@@ -169,7 +175,7 @@
             close(bufReader);
         }
     }
-    
+
     public static mod parse(InputStream stream,
                                 CompileMode kind,
                                 String filename,
@@ -268,11 +274,12 @@
         throws IOException {
         cflags.source_is_utf8 = true;
         cflags.encoding = "utf-8";
-        
+
         BufferedReader bufferedReader = new BufferedReader(reader);
         bufferedReader.mark(MARK_LIMIT);
-        if (findEncoding(bufferedReader) != null)
+        if (findEncoding(bufferedReader) != null) {
             throw new ParseException("encoding declaration in Unicode string");
+        }
         bufferedReader.reset();
 
         return new ExpectedEncodingBufferedReader(bufferedReader, null);
@@ -345,8 +352,9 @@
             CompilerFlags cflags,
             String filename)
             throws IOException {
-        if (cflags.source_is_utf8)
+        if (cflags.source_is_utf8) {
             return prepBufReader(new StringReader(string), cflags, filename);
+        }
 
         byte[] stringBytes = StringUtil.toBytes(string);
         return prepBufReader(new ByteArrayInputStream(stringBytes), cflags, filename, true, false);

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Thu Jun 12 00:10:32 2014
From: jython-checkins at python.org (jeff.allen)
Date: Thu, 12 Jun 2014 00:10:32 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_time=2Estrptime_now_uses_?=
 =?utf-8?q?=5Fstrptime=2Epy=2C_addressing_issue_=232112=2E?=
Message-ID: <3gpj7X68xKz7LjY@mail.python.org>

http://hg.python.org/jython/rev/1f517f1e5a08
changeset:   7289:1f517f1e5a08
user:        Santoso Wijaya 
date:        Wed Jun 11 20:39:11 2014 +0100
summary:
  time.strptime now uses _strptime.py, addressing issue #2112.

files:
  Lib/test/test_strptime_jy.py          |    4 +
  src/org/python/modules/time/Time.java |  138 +-------------
  2 files changed, 5 insertions(+), 137 deletions(-)


diff --git a/Lib/test/test_strptime_jy.py b/Lib/test/test_strptime_jy.py
--- a/Lib/test/test_strptime_jy.py
+++ b/Lib/test/test_strptime_jy.py
@@ -22,6 +22,10 @@
         d = strptime('0', '%f')
         self.assertEqual(1900, d.tm_year)
 
+    def test_issue2112(self):
+        d = strptime('1', '%d')
+        self.assertEqual(1900, d.tm_year)
+
 def test_main():
     test_support.run_unittest(
         ParsingTests
diff --git a/src/org/python/modules/time/Time.java b/src/org/python/modules/time/Time.java
--- a/src/org/python/modules/time/Time.java
+++ b/src/org/python/modules/time/Time.java
@@ -694,145 +694,9 @@
     }
 
     public static PyTuple strptime(String data_string, String format) {
-        if (format == null || data_string == null) {
-            // this is not a very interesting error message, but it's the same
-            // as what CPython outputs
-            throw Py.TypeError("expected string of buffer");
-        }
-        String jformat = py2java_format(format);
-        if (jformat == null) {
-            // Format not translatable to java, fallback to _strptime
-            return pystrptime(data_string, format);
-        }
-        SimpleDateFormat d = null;
-        try {
-            d = new SimpleDateFormat(jformat);
-        } catch (IllegalArgumentException e) {
-            throwValueError(e.getLocalizedMessage());
-        }
-        Calendar cal = Calendar.getInstance();
-        try {
-            cal.setTime(d.parse(data_string));
-        } catch (ParseException e) {
-            throwValueError("time data did not match format:  data=" + data_string + "  fmt=" + format);
-        }
-        int isdst = -1;
-        if (jformat.contains("zzz")) {
-            isdst = cal.getTimeZone().inDaylightTime(cal.getTime()) ? 1 : 0;
-        }
-        return toTimeTuple(cal, isdst);
+        return pystrptime(data_string, format);
     }
 
     private static final String DEFAULT_FORMAT_PY = "%a %b %d %H:%M:%S %Y";
-    private static final HashMap py2java = new HashMap() {{
-        put('a', "EEE");
-        put('A', "EEEE");
-        put('b', "MMM");
-        put('B', "MMMM");
-        put('c', "EEE MMM dd HH:mm:ss yyyy");
-        put('d', "dd");
-        put('H', "HH");
-        put('I', "hh");
-        put('j', "DDD");
-        put('m', "MM");
-        put('M', "mm");
-        put('p', "a");
-        put('S', "ss");
-        put('U', "ww");
-        put('W', "ww"); // same as U ??
-        put('x', "MM/dd/yy");
-        put('X', "HH:mm:ss");
-        put('y', "yy");
-        put('Y', "yyyy");
-        put('Z', "zzz");
-        put('%', "%");
-    }};
 
-    // strptime formats not supported by SimpleDateFormat:
-    private static final List notSupported = new ArrayList() {{
-        add('w');
-        add('f');
-    }};
-
-
-    /**
-     * Returns a {@link SimpleDateFormat} format string equivalent to the strptime
-     * format string supplied as parameter. If there is no reliable equivalent, it returns
-     * null, and the caller will use the Python implementation.
-     *
-     * @return format equivalent or null
-     */
-    private static String py2java_format(String format) {
-        StringBuilder builder = new StringBuilder();
-        boolean directive = false;
-        boolean inQuote = false;
-        boolean containsYear = false;
-        boolean containsMonth = false;
-
-        if (format.length() == 0) {
-            return null;
-        }
-
-        for (int i = 0; i < format.length(); i++) {
-            char charAt = format.charAt(i);
-
-            if (charAt == '%' && !directive) {
-                directive = true;
-                continue;
-            }
-
-            if (!directive) {
-                // ascii letters are considered SimpleDateFormat directive patterns unless
-                // escaped
-                boolean needsQuote = (charAt >= 'A' && charAt <= 'Z') ||
-                        (charAt >= 'a' && charAt <= 'z');
-                if (needsQuote && !inQuote || !needsQuote && inQuote) {
-                    builder.append("'");
-                    inQuote = needsQuote;
-                }
-                if (charAt == '\'') {
-                    // a single quote always needs to be escaped, regardless
-                    // whether already in a quote or not
-                    builder.append("'");
-                }
-                builder.append(charAt);
-                continue;
-            } else if (inQuote) {
-                builder.append("'");
-                inQuote = false;
-            }
-
-            String translated = py2java.get(charAt);
-            if (translated == null && notSupported.contains(charAt)) {
-                return null;
-            }
-
-            switch (charAt) {
-                case 'c':
-                case 'x':
-                    containsMonth = containsYear = true;
-                    break;
-                case 'y':
-                case 'Y':
-                    containsYear = true;
-                    break;
-                case 'b':
-                case 'B':
-                case 'm':
-                    containsMonth = true;
-                    break;
-            }
-
-            builder.append(translated != null ? translated : charAt);
-            directive = false;
-        }
-        if (containsMonth && !containsYear) {
-            // Java differs from Python concerning 29 Feb with no year: safe choice is fall-back.
-            return null;
-        }
-        if (inQuote) {
-            builder.append("'");
-        }
-        return builder.toString();
-    }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 14 02:34:43 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 14 Jun 2014 02:34:43 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_codecs=2Elookup_will_use_Ja?=
 =?utf-8?q?va_encodings_=28via_java=2Enio=2Echarset=2ECharset=29?=
Message-ID: <3gr0Dz4q9Jz7LjV@mail.python.org>

http://hg.python.org/jython/rev/6c718e5e9ae9
changeset:   7290:6c718e5e9ae9
user:        Jim Baker 
date:        Fri Jun 13 18:35:06 2014 -0600
summary:
  codecs.lookup will use Java encodings (via java.nio.charset.Charset)
if the Python equivalent is not available/does not load.

Fixes http://bugs.jython.org/issue1066 support for multibytecodecs,
including CJK codecs like big5, shift_jis, and johab. Note some codecs
like hz and euc_jisx0213 remain unsupported due to their lack of
availability in Java.

files:
  Lib/encodings/__init__.py               |  167 +++++
  Lib/encodings/_java.py                  |  248 +++++++
  Lib/test/test_codecs.py                 |   57 +-
  Lib/test/test_multibytecodec_support.py |  375 ++++++++++++
  4 files changed, 820 insertions(+), 27 deletions(-)


diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py
new file mode 100644
--- /dev/null
+++ b/Lib/encodings/__init__.py
@@ -0,0 +1,167 @@
+""" Standard "encodings" Package
+
+    Standard Python encoding modules are stored in this package
+    directory.
+
+    Codec modules must have names corresponding to normalized encoding
+    names as defined in the normalize_encoding() function below, e.g.
+    'utf-8' must be implemented by the module 'utf_8.py'.
+
+    Each codec module must export the following interface:
+
+    * getregentry() -> codecs.CodecInfo object
+    The getregentry() API must a CodecInfo object with encoder, decoder,
+    incrementalencoder, incrementaldecoder, streamwriter and streamreader
+    atttributes which adhere to the Python Codec Interface Standard.
+
+    In addition, a module may optionally also define the following
+    APIs which are then used by the package's codec search function:
+
+    * getaliases() -> sequence of encoding name strings to use as aliases
+
+    Alias names returned by getaliases() must be normalized encoding
+    names as defined by normalize_encoding().
+
+Written by Marc-Andre Lemburg (mal at lemburg.com).
+
+(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
+
+"""#"
+
+import codecs
+from encodings import aliases, _java
+import __builtin__
+
+_cache = {}
+_unknown = '--unknown--'
+_import_tail = ['*']
+_norm_encoding_map = ('                                              . '
+                      '0123456789       ABCDEFGHIJKLMNOPQRSTUVWXYZ     '
+                      ' abcdefghijklmnopqrstuvwxyz                     '
+                      '                                                '
+                      '                                                '
+                      '                ')
+_aliases = aliases.aliases
+
+class CodecRegistryError(LookupError, SystemError):
+    pass
+
+def normalize_encoding(encoding):
+
+    """ Normalize an encoding name.
+
+        Normalization works as follows: all non-alphanumeric
+        characters except the dot used for Python package names are
+        collapsed and replaced with a single underscore, e.g. '  -;#'
+        becomes '_'. Leading and trailing underscores are removed.
+
+        Note that encoding names should be ASCII only; if they do use
+        non-ASCII characters, these must be Latin-1 compatible.
+
+    """
+    # Make sure we have an 8-bit string, because .translate() works
+    # differently for Unicode strings.
+    if hasattr(__builtin__, "unicode") and isinstance(encoding, unicode):
+        # Note that .encode('latin-1') does *not* use the codec
+        # registry, so this call doesn't recurse. (See unicodeobject.c
+        # PyUnicode_AsEncodedString() for details)
+        encoding = encoding.encode('latin-1')
+    return '_'.join(encoding.translate(_norm_encoding_map).split())
+
+def search_function(encoding):
+
+    # Cache lookup
+    entry = _cache.get(encoding, _unknown)
+    if entry is not _unknown:
+        return entry
+
+    # Import the module:
+    #
+    # First try to find an alias for the normalized encoding
+    # name and lookup the module using the aliased name, then try to
+    # lookup the module using the standard import scheme, i.e. first
+    # try in the encodings package, then at top-level.
+    #
+    norm_encoding = normalize_encoding(encoding)
+    aliased_encoding = _aliases.get(norm_encoding) or \
+                       _aliases.get(norm_encoding.replace('.', '_'))
+    if aliased_encoding is not None:
+        modnames = [aliased_encoding,
+                    norm_encoding]
+    else:
+        modnames = [norm_encoding]
+    for modname in modnames:
+        if not modname or '.' in modname:
+            continue
+        try:
+            # Import is absolute to prevent the possibly malicious import of a
+            # module with side-effects that is not in the 'encodings' package.
+            mod = __import__('encodings.' + modname, fromlist=_import_tail,
+                             level=0)
+        except ImportError:
+            pass
+        else:
+            break
+    else:
+        mod = None
+
+    try:
+        getregentry = mod.getregentry
+    except AttributeError:
+        # Not a codec module
+        mod = None
+
+    if mod is None:
+        # First, see if we can load the encoding using java.nio.Charset;
+        # FIXME this could include encodings not known to Python, so we should test that out as well
+        entry, codecaliases = _java._java_factory(encoding)
+        if entry is not None:
+            _cache[encoding] = entry
+            for alias in codecaliases:
+                if alias not in _aliases:
+                    _aliases[alias] = modname
+            return entry
+
+        # Cache misses
+        _cache[encoding] = None
+        return None
+
+    # Now ask the module for the registry entry
+    entry = getregentry()
+    if not isinstance(entry, codecs.CodecInfo):
+        if not 4 <= len(entry) <= 7:
+            raise CodecRegistryError,\
+                 'module "%s" (%s) failed to register' % \
+                  (mod.__name__, mod.__file__)
+        if not hasattr(entry[0], '__call__') or \
+           not hasattr(entry[1], '__call__') or \
+           (entry[2] is not None and not hasattr(entry[2], '__call__')) or \
+           (entry[3] is not None and not hasattr(entry[3], '__call__')) or \
+           (len(entry) > 4 and entry[4] is not None and not hasattr(entry[4], '__call__')) or \
+           (len(entry) > 5 and entry[5] is not None and not hasattr(entry[5], '__call__')):
+            raise CodecRegistryError,\
+                'incompatible codecs in module "%s" (%s)' % \
+                (mod.__name__, mod.__file__)
+        if len(entry)<7 or entry[6] is None:
+            entry += (None,)*(6-len(entry)) + (mod.__name__.split(".", 1)[1],)
+        entry = codecs.CodecInfo(*entry)
+
+    # Cache the codec registry entry
+    _cache[encoding] = entry
+
+    # Register its aliases (without overwriting previously registered
+    # aliases)
+    try:
+        codecaliases = mod.getaliases()
+    except AttributeError:
+        pass
+    else:
+        for alias in codecaliases:
+            if alias not in _aliases:
+                _aliases[alias] = modname
+
+    # Return the registry entry
+    return entry
+
+# Register the search_function in the Python codec registry
+codecs.register(search_function)
diff --git a/Lib/encodings/_java.py b/Lib/encodings/_java.py
new file mode 100644
--- /dev/null
+++ b/Lib/encodings/_java.py
@@ -0,0 +1,248 @@
+# implements a factory to create codec instances for a given java charset
+
+import codecs
+
+from array import array
+from functools import partial
+from java.lang import StringBuilder
+from java.nio import ByteBuffer, CharBuffer
+from java.nio.charset import Charset, IllegalCharsetNameException
+from StringIO import StringIO
+
+
+python_to_java = {
+    'cp932': 'cp942',
+    'iso2022_jp': 'ISO-2022-JP',
+    'iso2022_jp_2': 'ISO-2022-JP-2',
+    'iso2022_kr': 'ISO-2022-KR',
+    'shift_jisx0213': 'x-SJIS_0213',
+}
+
+
+
+def _java_factory(encoding):
+    encoding = python_to_java.get(encoding, encoding)
+
+    supported = False
+    try:
+        supported = Charset.isSupported(encoding)
+    except IllegalCharsetNameException:
+        pass
+    if not supported:
+        return None, set()
+
+    charset = Charset.forName(encoding)  # FIXME should we return this canonical name? could be best... TBD
+    entry = codecs.CodecInfo(
+        name=encoding,
+        encode=Codec(encoding).encode,
+        decode=Codec(encoding).decode,
+        incrementalencoder=partial(IncrementalEncoder, encoding=encoding),
+        incrementaldecoder=partial(IncrementalDecoder, encoding=encoding),
+        streamreader=partial(StreamReader, encoding=encoding),
+        streamwriter=partial(StreamWriter, encoding=encoding)
+    )
+    return entry, charset.aliases()
+
+
+class Codec(object):  # (codecs.Codec):
+
+    def __init__(self, encoding):
+        self.encoding = encoding
+
+    def decode(self, input, errors='strict', final=True):
+        error_function = codecs.lookup_error(errors)
+        input_buffer = ByteBuffer.wrap(array('b', input))
+        decoder = Charset.forName(self.encoding).newDecoder()
+        output_buffer = CharBuffer.allocate(min(max(int(len(input) / 2), 256), 1024))
+        builder = StringBuilder(int(decoder.averageCharsPerByte() * len(input)))
+
+        while True:
+            result = decoder.decode(input_buffer, output_buffer, False)
+            pos = output_buffer.position()
+            output_buffer.rewind()
+            builder.append(output_buffer.subSequence(0, pos))
+            if result.isUnderflow():
+                if final:
+                    _process_incomplete_decode(self.encoding, input, error_function, input_buffer, builder)
+                break
+            _process_decode_errors(self.encoding, input, result, error_function, input_buffer, builder)
+
+        return builder.toString(), input_buffer.position()
+
+    def encode(self, input, errors='strict'):
+        error_function = codecs.lookup_error(errors)
+        # workaround non-BMP issues - need to get the exact count of chars, not codepoints
+        input_buffer = CharBuffer.allocate(StringBuilder(input).length())
+        input_buffer.put(input)
+        input_buffer.rewind()
+        encoder = Charset.forName(self.encoding).newEncoder()
+        output_buffer = ByteBuffer.allocate(min(max(len(input) * 2, 256), 1024))
+        builder = StringIO()
+
+        while True:
+            result = encoder.encode(input_buffer, output_buffer, True)
+            pos = output_buffer.position()
+            output_buffer.rewind()
+            builder.write(output_buffer.array()[0:pos].tostring())
+            if result.isUnderflow():
+                break
+            _process_encode_errors(self.encoding, input, result, error_function, input_buffer, builder)
+
+        return builder.getvalue(), len(input)
+
+
+class NonfinalCodec(Codec):
+
+    def decode(self, input, errors='strict'):
+        return Codec.decode(self, input, errors, final=False)
+
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+
+    def __init__(self, errors='strict', encoding=None):
+        assert encoding
+        self.encoding = encoding
+        self.errors = errors
+        self.encoder = Charset.forName(self.encoding).newEncoder()
+        self.output_buffer = ByteBuffer.allocate(1024)
+
+    def encode(self, input, final=False):
+        error_function = codecs.lookup_error(self.errors)
+        # workaround non-BMP issues - need to get the exact count of chars, not codepoints
+        input_buffer = CharBuffer.allocate(StringBuilder(input).length())
+        input_buffer.put(input)
+        input_buffer.rewind()
+        self.output_buffer.rewind()
+        builder = StringIO()
+
+        while True:
+            result = self.encoder.encode(input_buffer, self.output_buffer, final)
+            pos = self.output_buffer.position()
+            self.output_buffer.rewind()
+            builder.write(self.output_buffer.array()[0:pos].tostring())
+            if result.isUnderflow():
+                break
+            _process_encode_errors(self.encoding, input, result, error_function, input_buffer, builder)
+
+        return builder.getvalue()
+
+
+class IncrementalDecoder(codecs.IncrementalDecoder):
+
+    def __init__(self, errors='strict', encoding=None,):
+        assert encoding
+        self.encoding = encoding
+        self.errors = errors
+        self.decoder = Charset.forName(self.encoding).newDecoder()
+        self.output_buffer = CharBuffer.allocate(1024)
+        self.buffer = ''
+
+    def decode(self, input, final=False):
+        error_function = codecs.lookup_error(self.errors)
+        input_array = array('b', self.buffer + str(input))
+        input_buffer = ByteBuffer.wrap(input_array)
+        builder = StringBuilder(int(self.decoder.averageCharsPerByte() * len(input)))
+        self.output_buffer.rewind()
+
+        while True:
+            result = self.decoder.decode(input_buffer, self.output_buffer, final)
+            pos = self.output_buffer.position()
+            self.output_buffer.rewind()
+            builder.append(self.output_buffer.subSequence(0, pos))
+            if result.isUnderflow():
+                if not final:
+                    # Keep around any remaining input for next call to decode
+                    self.buffer = input_array[input_buffer.position():input_buffer.limit()].tostring()
+                else:
+                    _process_incomplete_decode(self.encoding, input, error_function, input_buffer, builder)
+                break
+            _process_decode_errors(self.encoding, input, result, error_function, input_buffer, builder)
+
+        return builder.toString()
+
+    def reset(self):
+        self.buffer = ""
+
+    def getstate(self):
+        return self.buffer or 0
+
+    def setstate(self, state):
+        self.buffer = state or ""
+
+
+class StreamWriter(NonfinalCodec, codecs.StreamWriter):
+
+    def __init__(self, stream, errors='strict', encoding=None, ):
+        NonfinalCodec.__init__(self, encoding)
+        codecs.StreamWriter.__init__(self, stream, errors)
+
+
+class StreamReader(NonfinalCodec, codecs.StreamReader):
+
+    def __init__(self, stream, errors='strict', encoding=None, ):
+        NonfinalCodec.__init__(self, encoding)
+        codecs.StreamReader.__init__(self, stream, errors)
+
+
+def _process_decode_errors(encoding, input, result, error_function, input_buffer, builder):
+    if result.isError():
+        e = UnicodeDecodeError(
+            encoding,
+            input, 
+            input_buffer.position(),
+            input_buffer.position() + result.length(),
+            'illegal multibyte sequence')
+        replacement, pos = error_function(e)
+        if not isinstance(replacement, unicode):
+            raise TypeError()
+        pos = int(pos)
+        if pos < 0:
+            pos = input_buffer.limit() + pos
+        if pos > input_buffer.limit():
+            raise IndexError()
+        builder.append(replacement)
+        input_buffer.position(pos)
+
+
+def _process_incomplete_decode(encoding, input, error_function, input_buffer, builder):
+    if input_buffer.position() < input_buffer.limit():
+        e = UnicodeDecodeError(
+            encoding,
+            input, 
+            input_buffer.position(),
+            input_buffer.limit(),
+            'illegal multibyte sequence')
+        replacement, pos = error_function(e)
+        if not isinstance(replacement, unicode):
+            raise TypeError()
+        pos = int(pos)
+        if pos < 0:
+            pos = input_buffer.limit() + pos
+        if pos > input_buffer.limit():
+            raise IndexError()
+        builder.append(replacement)
+        input_buffer.position(pos)
+
+
+def _get_unicode(input_buffer, result):
+    return input_buffer.subSequence(0, result.length()).toString()
+
+
+def _process_encode_errors(encoding, input, result, error_function, input_buffer, builder):
+    if result.isError():
+        e = UnicodeEncodeError(
+            encoding,
+            input, 
+            input_buffer.position(),
+            input_buffer.position() + result.length(),
+            'illegal multibyte sequence')
+        replacement, pos = error_function(e)
+        if not isinstance(replacement, unicode):
+            raise TypeError()
+        pos = int(pos)
+        if pos < 0:
+            pos = input_buffer.limit() + pos
+        if pos > input_buffer.limit():
+            raise IndexError()
+        builder.write(str(replacement))
+        input_buffer.position(pos)
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -1258,8 +1258,8 @@
 all_unicode_encodings = [
     "ascii",
     "base64_codec",
-# FIXME: Jython issue 1066:    "big5",
-# FIXME: Jython issue 1066:    "big5hkscs",
+    "big5",
+    "big5hkscs",
     "charmap",
     "cp037",
     "cp1006",
@@ -1296,27 +1296,27 @@
     "cp869",
     "cp874",
     "cp875",
-# FIXME: Jython issue 1066:    "cp932",
-# FIXME: Jython issue 1066:    "cp949",
-# FIXME: Jython issue 1066:    "cp950",
-# FIXME: Jython issue 1066:    "euc_jis_2004",
-# FIXME: Jython issue 1066:    'euc_jisx0213',
-# FIXME: Jython issue 1066:    'euc_jp',
-# FIXME: Jython issue 1066:    'euc_kr',
-# FIXME: Jython issue 1066:    'gb18030',
-# FIXME: Jython issue 1066:    'gb2312',
-# FIXME: Jython issue 1066:    'gbk',
+    "cp932",
+    "cp949",
+    "cp950",
+    # "euc_jis_2004",  # Not available on Java
+    # 'euc_jisx0213',  # Not available on Java
+    'euc_jp',
+    'euc_kr',
+    'gb18030',
+    'gb2312',
+    'gbk',
     "hex_codec",
     "hp_roman8",
-# FIXME: Jython issue 1066:    'hz',
-# FIXME: Jython issue 1066:    "idna",
-# FIXME: Jython issue 1066:    'iso2022_jp',
-# FIXME: Jython issue 1066:    'iso2022_jp_1',
-# FIXME: Jython issue 1066:    'iso2022_jp_2',
-# FIXME: Jython issue 1066:    'iso2022_jp_2004',
-# FIXME: Jython issue 1066:    'iso2022_jp_3',
-# FIXME: Jython issue 1066:    'iso2022_jp_ext',
-# FIXME: Jython issue 1066:    'iso2022_kr',
+    # 'hz', # Not available on Java
+    "idna",
+    'iso2022_jp',
+    # 'iso2022_jp_1', # Not available on Java
+    'iso2022_jp_2',
+    # 'iso2022_jp_2004', # Not available on Java
+    # 'iso2022_jp_3', # Not available on Java
+    # 'iso2022_jp_ext', # Not available on Java
+    'iso2022_kr',
     "iso8859_1",
     "iso8859_10",
     "iso8859_11",
@@ -1332,7 +1332,7 @@
     "iso8859_7",
     "iso8859_8",
     "iso8859_9",
-# FIXME: Jython issue 1066:    'johab',
+    'johab',
     "koi8_r",
     "koi8_u",
     "latin_1",
@@ -1347,9 +1347,9 @@
     "punycode",
     "raw_unicode_escape",
     "rot_13",
-# FIXME: Jython issue 1066:    'shift_jis',
-# FIXME: Jython issue 1066:    'shift_jis_2004',
-# FIXME: Jython issue 1066:    'shift_jisx0213',
+    "shift_jis",
+    #'shift_jis_2004', # Not available on Java
+    'shift_jisx0213',
     "tis_620",
     "unicode_escape",
     "unicode_internal",
@@ -1499,11 +1499,14 @@
                         self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
 
     def test_seek(self):
-        # all codecs should be able to encode these
-        s = u"%s\n%s\n" % (100*u"abc123", 100*u"def456")
+        # all codecs - except idna on Java - should be able to encode these
+        s1 = u"%s\n%s\n" % (100*u"abc123", 100*u"def456")
         for encoding in all_unicode_encodings:
+            s = s1
             if encoding in broken_unicode_with_streams:
                 continue
+            if encoding == "idna":
+                s =  u"%s\n%s\n" % (5*u"abc123", 5*u"def456") # idna encoder rejects as being too long
             reader = codecs.getreader(encoding)(StringIO.StringIO(s.encode(encoding)))
             for t in xrange(5):
                 # Test that calling seek resets the internal codec state and buffers
diff --git a/Lib/test/test_multibytecodec_support.py b/Lib/test/test_multibytecodec_support.py
new file mode 100644
--- /dev/null
+++ b/Lib/test/test_multibytecodec_support.py
@@ -0,0 +1,375 @@
+#!/usr/bin/env python
+#
+# test_multibytecodec_support.py
+#   Common Unittest Routines for CJK codecs
+#
+
+import codecs
+import os
+import re
+import sys
+import unittest
+from httplib import HTTPException
+from test import test_support
+from StringIO import StringIO
+
+class TestBase:
+    encoding        = ''   # codec name
+    codec           = None # codec tuple (with 4 elements)
+    tstring         = ''   # string to test StreamReader
+
+    codectests      = None # must set. codec test tuple
+    roundtriptest   = 1    # set if roundtrip is possible with unicode
+    has_iso10646    = 0    # set if this encoding contains whole iso10646 map
+    xmlcharnametest = None # string to test xmlcharrefreplace
+    unmappedunicode = u'\ufffe' # a unicode codepoint that is not mapped
+                                # CPython uses an isolated surrogate, which will not work on Jython
+
+    def setUp(self):
+        if self.codec is None:
+            self.codec = codecs.lookup(self.encoding)
+        self.encode = self.codec.encode
+        self.decode = self.codec.decode
+        self.reader = self.codec.streamreader
+        self.writer = self.codec.streamwriter
+        self.incrementalencoder = self.codec.incrementalencoder
+        self.incrementaldecoder = self.codec.incrementaldecoder
+
+    def test_chunkcoding(self):
+        for native, utf8 in zip(*[StringIO(f).readlines()
+                                  for f in self.tstring]):
+            u = self.decode(native)[0]
+            self.assertEqual(u, utf8.decode('utf-8'))
+            if self.roundtriptest:
+                self.assertEqual(native, self.encode(u)[0])
+
+    def test_errorhandle(self):
+        for source, scheme, expected in self.codectests:
+            if isinstance(source, bytes):
+                func = self.decode
+            else:
+                func = self.encode
+            if expected:
+                result = func(source, scheme)[0]
+                if func is self.decode:
+                    self.assertTrue(type(result) is unicode, type(result))
+                    self.assertEqual(result, expected,
+                                     '%r.decode(%r, %r)=%r != %r'
+                                     % (source, self.encoding, scheme, result,
+                                        expected))
+                else:
+                    self.assertTrue(type(result) is bytes, type(result))
+                    self.assertEqual(result, expected,
+                                     '%r.encode(%r, %r)=%r != %r'
+                                     % (source, self.encoding, scheme, result,
+                                        expected))
+            else:
+                self.assertRaises(UnicodeError, func, source, scheme)
+
+    def test_xmlcharrefreplace(self):
+        if self.has_iso10646:
+            return
+
+        s = u"\u0b13\u0b23\u0b60 nd eggs"
+        self.assertEqual(
+            self.encode(s, "xmlcharrefreplace")[0],
+            "ଓଣୠ nd eggs"
+        )
+
+    def test_customreplace_encode(self):
+        if self.has_iso10646:
+            return
+
+        from htmlentitydefs import codepoint2name
+
+        def xmlcharnamereplace(exc):
+            if not isinstance(exc, UnicodeEncodeError):
+                raise TypeError("don't know how to handle %r" % exc)
+            l = []
+            for c in exc.object[exc.start:exc.end]:
+                if ord(c) in codepoint2name:
+                    l.append(u"&%s;" % codepoint2name[ord(c)])
+                else:
+                    l.append(u"&#%d;" % ord(c))
+            return (u"".join(l), exc.end)
+
+        codecs.register_error("test.xmlcharnamereplace", xmlcharnamereplace)
+
+        if self.xmlcharnametest:
+            sin, sout = self.xmlcharnametest
+        else:
+            sin = u"\xab\u211c\xbb = \u2329\u1234\u232a"
+            sout = "«ℜ» = ⟨ሴ⟩"
+        self.assertEqual(self.encode(sin,
+                                    "test.xmlcharnamereplace")[0], sout)
+
+    def test_callback_wrong_objects(self):
+        def myreplace(exc):
+            return (ret, exc.end)
+        codecs.register_error("test.cjktest", myreplace)
+
+        for ret in ([1, 2, 3], [], None, object(), 'string', ''):
+            self.assertRaises(TypeError, self.encode, self.unmappedunicode,
+                              'test.cjktest')
+
+    def test_callback_long_index(self):
+        def myreplace(exc):
+            return (u'x', long(exc.end))
+        codecs.register_error("test.cjktest", myreplace)
+        self.assertEqual(self.encode(u'abcd' + self.unmappedunicode + u'efgh',
+                                     'test.cjktest'), ('abcdxefgh', 9))
+
+        def myreplace(exc):
+            return (u'x', sys.maxint + 1)
+        codecs.register_error("test.cjktest", myreplace)
+        self.assertRaises(IndexError, self.encode, self.unmappedunicode,
+                          'test.cjktest')
+
+    def test_callback_None_index(self):
+        def myreplace(exc):
+            return (u'x', None)
+        codecs.register_error("test.cjktest", myreplace)
+        self.assertRaises(TypeError, self.encode, self.unmappedunicode,
+                          'test.cjktest')
+
+    def test_callback_backward_index(self):
+        def myreplace(exc):
+            if myreplace.limit > 0:
+                myreplace.limit -= 1
+                return (u'REPLACED', 0)
+            else:
+                return (u'TERMINAL', exc.end)
+        myreplace.limit = 3
+        codecs.register_error("test.cjktest", myreplace)
+        self.assertEqual(self.encode(u'abcd' + self.unmappedunicode + u'efgh',
+                                     'test.cjktest'),
+                ('abcdREPLACEDabcdREPLACEDabcdREPLACEDabcdTERMINALefgh', 9))
+
+    def test_callback_forward_index(self):
+        def myreplace(exc):
+            return (u'REPLACED', exc.end + 2)
+        codecs.register_error("test.cjktest", myreplace)
+        self.assertEqual(self.encode(u'abcd' + self.unmappedunicode + u'efgh',
+                                     'test.cjktest'), ('abcdREPLACEDgh', 9))
+
+    def test_callback_index_outofbound(self):
+        def myreplace(exc):
+            return (u'TERM', 100)
+        codecs.register_error("test.cjktest", myreplace)
+        self.assertRaises(IndexError, self.encode, self.unmappedunicode,
+                          'test.cjktest')
+
+    def test_incrementalencoder(self):
+        UTF8Reader = codecs.getreader('utf-8')
+        for sizehint in [None] + range(1, 33) + \
+                        [64, 128, 256, 512, 1024]:
+            istream = UTF8Reader(StringIO(self.tstring[1]))
+            ostream = StringIO()
+            encoder = self.incrementalencoder()
+            while 1:
+                if sizehint is not None:
+                    data = istream.read(sizehint)
+                else:
+                    data = istream.read()
+
+                if not data:
+                    break
+                e = encoder.encode(data)
+                ostream.write(e)
+
+            self.assertEqual(ostream.getvalue(), self.tstring[0])
+
+    def test_incrementaldecoder(self):
+        UTF8Writer = codecs.getwriter('utf-8')
+        for sizehint in [None, -1] + range(1, 33) + \
+                        [64, 128, 256, 512, 1024]:
+            istream = StringIO(self.tstring[0])
+            ostream = UTF8Writer(StringIO())
+            decoder = self.incrementaldecoder()
+            while 1:
+                data = istream.read(sizehint)
+                if not data:
+                    break
+                else:
+                    u = decoder.decode(data)
+                    ostream.write(u)
+
+            self.assertEqual(ostream.getvalue(), self.tstring[1])
+
+    def test_incrementalencoder_error_callback(self):
+        inv = self.unmappedunicode
+
+        e = self.incrementalencoder()
+        self.assertRaises(UnicodeEncodeError, e.encode, inv, True)
+
+        e.errors = 'ignore'
+        self.assertEqual(e.encode(inv, True), '')
+
+        e.reset()
+        def tempreplace(exc):
+            return (u'called', exc.end)
+        codecs.register_error('test.incremental_error_callback', tempreplace)
+        e.errors = 'test.incremental_error_callback'
+        self.assertEqual(e.encode(inv, True), 'called')
+
+        # again
+        e.errors = 'ignore'
+        self.assertEqual(e.encode(inv, True), '')
+
+    def test_streamreader(self):
+        UTF8Writer = codecs.getwriter('utf-8')
+        for name in ["read", "readline", "readlines"]:
+            for sizehint in [None, -1] + range(1, 33) + \
+                            [64, 128, 256, 512, 1024]:
+                istream = self.reader(StringIO(self.tstring[0]))
+                ostream = UTF8Writer(StringIO())
+                func = getattr(istream, name)
+                while 1:
+                    data = func(sizehint)
+                    if not data:
+                        break
+                    if name == "readlines":
+                        ostream.writelines(data)
+                    else:
+                        ostream.write(data)
+
+                self.assertEqual(ostream.getvalue(), self.tstring[1])
+
+    def test_streamwriter(self):
+        readfuncs = ('read', 'readline', 'readlines')
+        UTF8Reader = codecs.getreader('utf-8')
+        for name in readfuncs:
+            for sizehint in [None] + range(1, 33) + \
+                            [64, 128, 256, 512, 1024]:
+                istream = UTF8Reader(StringIO(self.tstring[1]))
+                ostream = self.writer(StringIO())
+                func = getattr(istream, name)
+                while 1:
+                    if sizehint is not None:
+                        data = func(sizehint)
+                    else:
+                        data = func()
+
+                    if not data:
+                        break
+                    if name == "readlines":
+                        ostream.writelines(data)
+                    else:
+                        ostream.write(data)
+
+                self.assertEqual(ostream.getvalue(), self.tstring[0])
+
+class TestBase_Mapping(unittest.TestCase):
+    pass_enctest = []
+    pass_dectest = []
+    supmaps = []
+    codectests = []
+
+    def __init__(self, *args, **kw):
+        unittest.TestCase.__init__(self, *args, **kw)
+        try:
+            self.open_mapping_file().close() # test it to report the error early
+        except (IOError, HTTPException):
+            self.skipTest("Could not retrieve "+self.mapfileurl)
+
+    def open_mapping_file(self):
+        return test_support.open_urlresource(self.mapfileurl)
+
+    def test_mapping_file(self):
+        if self.mapfileurl.endswith('.xml'):
+            self._test_mapping_file_ucm()
+        else:
+            self._test_mapping_file_plain()
+
+    def _test_mapping_file_plain(self):
+        _unichr = lambda c: eval("u'\\U%08x'" % int(c, 16))
+        unichrs = lambda s: u''.join(_unichr(c) for c in s.split('+'))
+        urt_wa = {}
+
+        with self.open_mapping_file() as f:
+            for line in f:
+                if not line:
+                    break
+                data = line.split('#')[0].strip().split()
+                if len(data) != 2:
+                    continue
+
+                csetval = eval(data[0])
+                if csetval <= 0x7F:
+                    csetch = chr(csetval & 0xff)
+                elif csetval >= 0x1000000:
+                    csetch = chr(csetval >> 24) + chr((csetval >> 16) & 0xff) + \
+                             chr((csetval >> 8) & 0xff) + chr(csetval & 0xff)
+                elif csetval >= 0x10000:
+                    csetch = chr(csetval >> 16) + \
+                             chr((csetval >> 8) & 0xff) + chr(csetval & 0xff)
+                elif csetval >= 0x100:
+                    csetch = chr(csetval >> 8) + chr(csetval & 0xff)
+                else:
+                    continue
+
+                unich = unichrs(data[1])
+                if unich == u'\ufffd' or unich in urt_wa:
+                    continue
+                urt_wa[unich] = csetch
+
+                self._testpoint(csetch, unich)
+
+    def _test_mapping_file_ucm(self):
+        with self.open_mapping_file() as f:
+            ucmdata = f.read()
+        uc = re.findall('', ucmdata)
+        for uni, coded in uc:
+            unich = unichr(int(uni, 16))
+            codech = ''.join(chr(int(c, 16)) for c in coded.split())
+            self._testpoint(codech, unich)
+
+    def test_mapping_supplemental(self):
+        for mapping in self.supmaps:
+            self._testpoint(*mapping)
+
+    def _testpoint(self, csetch, unich):
+        if (csetch, unich) not in self.pass_enctest:
+            try:
+                self.assertEqual(unich.encode(self.encoding), csetch)
+            except UnicodeError, exc:
+                self.fail('Encoding failed while testing %s -> %s: %s' % (
+                            repr(unich), repr(csetch), exc.reason))
+        if (csetch, unich) not in self.pass_dectest:
+            try:
+                self.assertEqual(csetch.decode(self.encoding), unich)
+            except UnicodeError, exc:
+                self.fail('Decoding failed while testing %s -> %s: %s' % (
+                            repr(csetch), repr(unich), exc.reason))
+
+    def test_errorhandle(self):
+        for source, scheme, expected in self.codectests:
+            if isinstance(source, bytes):
+                func = source.decode
+            else:
+                func = source.encode
+            if expected:
+                if isinstance(source, bytes):
+                    result = func(self.encoding, scheme)
+                    self.assertTrue(type(result) is unicode, type(result))
+                    self.assertEqual(result, expected,
+                                     '%r.decode(%r, %r)=%r != %r'
+                                     % (source, self.encoding, scheme, result,
+                                        expected))
+                else:
+                    result = func(self.encoding, scheme)
+                    self.assertTrue(type(result) is bytes, type(result))
+                    self.assertEqual(result, expected,
+                                     '%r.encode(%r, %r)=%r != %r'
+                                     % (source, self.encoding, scheme, result,
+                                        expected))
+            else:
+                self.assertRaises(UnicodeError, func, self.encoding, scheme)
+
+def load_teststring(name):
+    dir = os.path.join(os.path.dirname(__file__), 'cjkencodings')
+    with open(os.path.join(dir, name + '.txt'), 'rb') as f:
+        encoded = f.read()
+    with open(os.path.join(dir, name + '-utf8.txt'), 'rb') as f:
+        utf8 = f.read()
+    return encoded, utf8

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 14 14:02:18 2014
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 14 Jun 2014 14:02:18 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Let_PyBuffer_support_try-wi?=
	=?utf-8?q?th-resources=2E?=
Message-ID: <3grHVL29Fjz7Ljc@mail.python.org>

http://hg.python.org/jython/rev/100892193c90
changeset:   7291:100892193c90
parent:      7289:1f517f1e5a08
user:        Jeff Allen 
date:        Fri Jun 13 23:36:56 2014 +0100
summary:
  Let PyBuffer support try-with-resources.
Now extends AutoCloseable. Also adds Java test.

files:
  src/org/python/core/PyBuffer.java            |   6 +-
  src/org/python/core/buffer/BaseBuffer.java   |   5 +
  tests/java/org/python/core/PyBufferTest.java |  29 +++++++++-
  3 files changed, 38 insertions(+), 2 deletions(-)


diff --git a/src/org/python/core/PyBuffer.java b/src/org/python/core/PyBuffer.java
--- a/src/org/python/core/PyBuffer.java
+++ b/src/org/python/core/PyBuffer.java
@@ -5,7 +5,7 @@
  * the counterpart of the CPython Py_buffer struct. Several concrete types implement
  * this interface in order to provide tailored support for different storage organisations.
  */
-public interface PyBuffer extends PyBUF, BufferProtocol {
+public interface PyBuffer extends PyBUF, BufferProtocol, AutoCloseable {
 
     /*
      * The different behaviours required as the actual structure of the buffer changes (from one
@@ -183,6 +183,10 @@
      */
     void release();
 
+    /** An alias for {@link #release()} to satisfy {@link AutoCloseable}. */
+    @Override
+    void close();
+
     /**
      * True only if the buffer has been released with (the required number of calls to)
      * {@link #release()} or some equivalent operation. The consumer may be sharing the reference
diff --git a/src/org/python/core/buffer/BaseBuffer.java b/src/org/python/core/buffer/BaseBuffer.java
--- a/src/org/python/core/buffer/BaseBuffer.java
+++ b/src/org/python/core/buffer/BaseBuffer.java
@@ -522,6 +522,11 @@
     }
 
     @Override
+    public void close() {
+        release();
+    }
+
+    @Override
     public boolean isReleased() {
         return exports <= 0;
     }
diff --git a/tests/java/org/python/core/PyBufferTest.java b/tests/java/org/python/core/PyBufferTest.java
--- a/tests/java/org/python/core/PyBufferTest.java
+++ b/tests/java/org/python/core/PyBufferTest.java
@@ -701,8 +701,13 @@
 
         Set uniqueBuffers = new HashSet();
 
+        // Test a balanced sequence of acquire and release using try-with-resources
         for (BufferTestPair test : buffersToRead) {
-            // Test a pattern of acquire and release with one more release than acquire
+            doTestTryWithResources(test);
+        }
+
+        // Now test a pattern of acquire and release with one more release than acquire
+        for (BufferTestPair test : buffersToRead) {
             doTestRelease(test);
             uniqueBuffers.add(test.view);
         }
@@ -718,6 +723,28 @@
                 doTestGetAfterRelease(test);
             }
         }
+
+    }
+
+    /**
+     * Exercise try-with-resources on one BufferTestPair.
+     */
+    private void doTestTryWithResources(BufferTestPair test) {
+
+        if (verbosity > 0) {
+            System.out.println("try with resources: " + test);
+        }
+        int flags = PyBUF.STRIDES | PyBUF.FORMAT;
+        BufferProtocol sub = test.subject;
+
+        // The object will be exporting test.view and N other views we don't know about
+        try (PyBuffer c = sub.getBuffer(flags)) {   // = N+1 exports
+            try (PyBuffer b = sub.getBuffer(PyBUF.FULL_RO); PyBuffer d =c.getBuffer(flags)) {
+                checkExporting(sub);// = N+3 exports
+            }
+            checkExporting(sub);                    // = N+1 exports
+        }
+        checkExporting(sub);                        // = N export
     }
 
     /**

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 14 14:02:19 2014
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 14 Jun 2014 14:02:19 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Use_PyBuffer_try-with-resou?=
 =?utf-8?q?rces_throughout_the_core=2E?=
Message-ID: <3grHVM6hWfz7Ljk@mail.python.org>

http://hg.python.org/jython/rev/18d60bea34af
changeset:   7292:18d60bea34af
user:        Jeff Allen 
date:        Sat Jun 14 10:33:07 2014 +0100
summary:
  Use PyBuffer try-with-resources throughout the core.
Set a good example. Also, fix some places where I forgot to release().
Similar work needed on binascii, but defer to a separate patch.

files:
  src/org/python/core/BaseBytes.java            |  110 +-----
  src/org/python/core/PyArray.java              |    5 +-
  src/org/python/core/PyByteArray.java          |  147 +++++----
  src/org/python/core/PyFile.java               |    6 +-
  src/org/python/core/PyString.java             |    5 +-
  src/org/python/core/PyUnicode.java            |    5 +-
  src/org/python/core/io/TextIOBase.java        |   16 +-
  src/org/python/modules/sre/PatternObject.java |    6 +-
  8 files changed, 119 insertions(+), 181 deletions(-)


diff --git a/src/org/python/core/BaseBytes.java b/src/org/python/core/BaseBytes.java
--- a/src/org/python/core/BaseBytes.java
+++ b/src/org/python/core/BaseBytes.java
@@ -345,10 +345,11 @@
      */
     protected void init(BufferProtocol value) throws PyException {
         // Get the buffer view
-        PyBuffer view = value.getBuffer(PyBUF.FULL_RO);
-        // Create storage for the bytes and have the view drop them in
-        newStorage(view.getLen());
-        view.copyTo(storage, offset);
+        try (PyBuffer view = value.getBuffer(PyBUF.FULL_RO)) {
+            // Create storage for the bytes and have the view drop them in
+            newStorage(view.getLen());
+            view.copyTo(storage, offset);
+        }
     }
 
     /**
@@ -834,21 +835,16 @@
             return 0;
 
         } else {
-
             // Try to get a byte-oriented view
-            PyBuffer bv = getView(b);
-
-            if (bv == null) {
-                // Signifies a type mis-match. See PyObject._cmp_unsafe() and related code.
-                return -2;
-
-            } else {
-                try {
+            try (PyBuffer bv = getView(b)) {
+
+                if (bv == null) {
+                    // Signifies a type mis-match. See PyObject._cmp_unsafe() and related code.
+                    return -2;
+
+                } else {
                     // Compare this with other object viewed as a buffer
                     return compare(this, bv);
-                } finally {
-                    // Must alsways let go of the buffer
-                    bv.release();
                 }
             }
         }
@@ -871,14 +867,13 @@
         } else {
 
             // Try to get a byte-oriented view
-            PyBuffer bv = getView(b);
-
-            if (bv == null) {
-                // Signifies a type mis-match. See PyObject._cmp_unsafe() and related code.
-                return -2;
-
-            } else {
-                try {
+            try (PyBuffer bv = getView(b)) {
+
+                if (bv == null) {
+                    // Signifies a type mis-match. See PyObject._cmp_unsafe() and related code.
+                    return -2;
+
+                } else {
                     if (bv.getLen() != size) {
                         // Different size: can't be equal, and we don't care which is bigger
                         return 1;
@@ -886,9 +881,6 @@
                         // Compare this with other object viewed as a buffer
                         return compare(this, bv);
                     }
-                } finally {
-                    // Must alsways let go of the buffer
-                    bv.release();
                 }
             }
         }
@@ -1017,13 +1009,10 @@
             return index(b) >= 0;
         } else {
             // Caller is treating this as a byte-string and looking for substring 'target'
-            PyBuffer targetView = getViewOrError(target);
-            try {
+            try (PyBuffer targetView = getViewOrError(target)) {
                 Finder finder = new Finder(targetView);
                 finder.setText(this);
                 return finder.nextIndex() >= 0;
-            } finally {
-                targetView.release();
             }
         }
     }
@@ -1080,9 +1069,7 @@
     private boolean match(PyObject target, int pos, int n, boolean endswith) {
 
         // Error if not something we can treat as a view of bytes
-        PyBuffer vt = getViewOrError(target);
-
-        try {
+        try (PyBuffer vt = getViewOrError(target)) {
             int j = 0, len = vt.getLen();
 
             if (!endswith) {
@@ -1106,10 +1093,6 @@
                 }
             }
             return true; // They must all have matched
-
-        } finally {
-            // Let go of the buffer we acquired
-            vt.release();
         }
     }
 
@@ -1818,8 +1801,7 @@
      * @return count of occurrences of sub within this byte array
      */
     final int basebytes_count(PyObject sub, PyObject ostart, PyObject oend) {
-        PyBuffer vsub = getViewOrError(sub);
-        try {
+        try (PyBuffer vsub = getViewOrError(sub)) {
             Finder finder = new Finder(vsub);
 
             // Convert [ostart:oend] to integers
@@ -1827,8 +1809,6 @@
 
             // Make this slice the thing we count within.
             return finder.count(storage, offset + index[0], index[3]);
-        } finally {
-            vsub.release();
         }
     }
 
@@ -1846,12 +1826,9 @@
      * @return index of start of occurrence of sub within this byte array
      */
     final int basebytes_find(PyObject sub, PyObject ostart, PyObject oend) {
-        PyBuffer vsub = getViewOrError(sub);
-        try {
+        try (PyBuffer vsub = getViewOrError(sub)) {
             Finder finder = new Finder(vsub);
             return find(finder, ostart, oend);
-        } finally {
-            vsub.release();
         }
     }
 
@@ -2026,9 +2003,7 @@
     final synchronized PyTuple basebytes_partition(PyObject sep) {
 
         // View the separator as a byte array (or error if we can't)
-        PyBuffer separator = getViewOrError(sep);
-
-        try {
+        try (PyBuffer separator = getViewOrError(sep)) {
             // Create a Finder for the separator and set it on this byte array
             int n = checkForEmptySeparator(separator);
             Finder finder = new Finder(separator);
@@ -2043,8 +2018,6 @@
                 // Not found: choose values leading to ([0:size], '', '')
                 return partition(size, size);
             }
-        } finally {
-            separator.release();
         }
     }
 
@@ -2080,12 +2053,9 @@
      * @return index of start of occurrence of sub within this byte array
      */
     final int basebytes_rfind(PyObject sub, PyObject ostart, PyObject oend) {
-        PyBuffer vsub = getViewOrError(sub);
-        try {
+        try (PyBuffer vsub = getViewOrError(sub)) {
             Finder finder = new ReverseFinder(vsub);
             return find(finder, ostart, oend);
-        } finally {
-            vsub.release();
         }
     }
 
@@ -2128,9 +2098,7 @@
     final synchronized PyByteArray basebytes_replace(PyObject oldB, PyObject newB, int maxcount) {
 
         // View the to and from as byte arrays (or error if we can't)
-        PyBuffer to = getViewOrError(newB), from = null;
-        try {
-            from = getViewOrError(oldB);
+        try (PyBuffer to = getViewOrError(newB); PyBuffer from = getViewOrError(oldB)) {
             /*
              * The logic of the first section is copied exactly from CPython in order to get the
              * same behaviour. The "headline" description of replace is simple enough but the corner
@@ -2184,16 +2152,6 @@
                 // Otherwise use the generic algorithm
                 return replace_substring(from, to, maxcount);
             }
-
-        } finally {
-            /*
-             * Release the buffers we acquired: there must be a to buffer and there might be a from
-             * buffer.
-             */
-            to.release();
-            if (from != null) {
-                from.release();
-            }
         }
     }
 
@@ -2475,8 +2433,7 @@
     final synchronized PyTuple basebytes_rpartition(PyObject sep) {
 
         // View the separator as a byte array (or error if we can't)
-        PyBuffer separator = getViewOrError(sep);
-        try {
+        try (PyBuffer separator = getViewOrError(sep)) {
             // Create a Finder for the separtor and set it on this byte array
             int n = checkForEmptySeparator(separator);
             Finder finder = new ReverseFinder(separator);
@@ -2491,8 +2448,6 @@
                 // Not found: choose values leading to ('', '', [0:size])
                 return partition(0, 0);
             }
-        } finally {
-            separator.release();
         }
     }
 
@@ -2590,9 +2545,7 @@
     final synchronized PyList basebytes_rsplit_explicit(PyObject sep, int maxsplit) {
 
         // The separator may be presented as anything viewable as bytes
-        PyBuffer separator = getViewOrError(sep);
-
-        try {
+        try (PyBuffer separator = getViewOrError(sep)) {
             int n = checkForEmptySeparator(separator);
 
             PyList result = new PyList();
@@ -2627,8 +2580,6 @@
                 result.add(0, word);
             }
             return result;
-        } finally {
-            separator.release();
         }
     }
 
@@ -2794,8 +2745,7 @@
     final synchronized PyList basebytes_split_explicit(PyObject sep, int maxsplit) {
 
         // The separator may be presented as anything viewable as bytes
-        PyBuffer separator = getViewOrError(sep);
-        try {
+        try (PyBuffer separator = getViewOrError(sep)) {
             checkForEmptySeparator(separator);
 
             PyList result = new PyList();
@@ -2821,8 +2771,6 @@
             // Append the remaining unsplit text
             result.append(getslice(p - offset, size));
             return result;
-        } finally {
-            separator.release();
         }
     }
 
diff --git a/src/org/python/core/PyArray.java b/src/org/python/core/PyArray.java
--- a/src/org/python/core/PyArray.java
+++ b/src/org/python/core/PyArray.java
@@ -1131,8 +1131,7 @@
 
             } else {
                 // Access the bytes
-                PyBuffer pybuf = ((BufferProtocol)input).getBuffer(PyBUF.STRIDED_RO);
-                try {
+                try (PyBuffer pybuf = ((BufferProtocol)input).getBuffer(PyBUF.STRIDED_RO)) {
                     // Provide argument as stream of bytes for fromstream method
                     if (pybuf.getNdim() == 1) {
                         if (pybuf.getStrides()[0] == 1) {
@@ -1149,8 +1148,6 @@
                         // Currently don't support n-dimensional sources
                         throw Py.ValueError("multi-dimensional buffer not supported");
                     }
-                } finally {
-                    pybuf.release();
                 }
             }
 
diff --git a/src/org/python/core/PyByteArray.java b/src/org/python/core/PyByteArray.java
--- a/src/org/python/core/PyByteArray.java
+++ b/src/org/python/core/PyByteArray.java
@@ -548,24 +548,25 @@
      */
     private void setslice(int start, int stop, int step, BufferProtocol value) throws PyException {
 
-        PyBuffer view = value.getBuffer(PyBUF.FULL_RO);
+        try (PyBuffer view = value.getBuffer(PyBUF.FULL_RO)) {
 
-        int len = view.getLen();
+            int len = view.getLen();
 
-        if (step == 1) {
-            // Delete this[start:stop] and open a space of the right size
-            storageReplace(start, stop - start, len);
-            view.copyTo(storage, start + offset);
+            if (step == 1) {
+                // Delete this[start:stop] and open a space of the right size
+                storageReplace(start, stop - start, len);
+                view.copyTo(storage, start + offset);
 
-        } else {
-            // This is an extended slice which means we are replacing elements
-            int n = sliceLength(start, stop, step);
-            if (n != len) {
-                throw SliceSizeError("bytes", len, n);
-            }
+            } else {
+                // This is an extended slice which means we are replacing elements
+                int n = sliceLength(start, stop, step);
+                if (n != len) {
+                    throw SliceSizeError("bytes", len, n);
+                }
 
-            for (int io = start + offset, j = 0; j < n; io += step, j++) {
-                storage[io] = view.byteAt(j);    // Assign this[i] = value[j]
+                for (int io = start + offset, j = 0; j < n; io += step, j++) {
+                    storage[io] = view.byteAt(j);    // Assign this[i] = value[j]
+                }
             }
         }
     }
@@ -2021,66 +2022,78 @@
     @ExposedMethod(defaults = "null", doc = BuiltinDocs.bytearray_translate_doc)
     final PyByteArray bytearray_translate(PyObject table, PyObject deletechars) {
 
-        // Normalise the translation table to a View
+        // Work with the translation table (if there is one) as a PyBuffer view.
+        try (PyBuffer tab = getTranslationTable(table)) {
+
+            // Accumulate the result here
+            PyByteArray result = new PyByteArray();
+
+            // There are 4 cases depending on presence/absence of table and deletechars
+
+            if (deletechars != null) {
+
+                // Work with the deletion characters as a buffer too.
+                try (PyBuffer d = getViewOrError(deletechars)) {
+                    // Use a ByteSet to express which bytes to delete
+                    ByteSet del = new ByteSet(d);
+                    int limit = offset + size;
+                    if (tab == null) {
+                        // No translation table, so we're just copying with omissions
+                        for (int i = offset; i < limit; i++) {
+                            int b = storage[i] & 0xff;
+                            if (!del.contains(b)) {
+                                result.append((byte)b);
+                            }
+                        }
+                    } else {
+                        // Loop over this byte array and write translated bytes to the result
+                        for (int i = offset; i < limit; i++) {
+                            int b = storage[i] & 0xff;
+                            if (!del.contains(b)) {
+                                result.append(tab.byteAt(b));
+                            }
+                        }
+                    }
+                }
+
+            } else {
+                // No deletion set.
+                if (tab == null) {
+                    // ... and no translation table either: just copy
+                    result.extend(this);
+                } else {
+                    int limit = offset + size;
+                    // Loop over this byte array and write translated bytes to the result
+                    for (int i = offset; i < limit; i++) {
+                        int b = storage[i] & 0xff;
+                        result.append(tab.byteAt(b));
+                    }
+                }
+            }
+
+            return result;
+        }
+    }
+
+    /**
+     * Return a {@link PyBuffer} representing a translation table, or raise an exception if it is
+     * the wrong size. The caller is responsible for calling {@link PyBuffer#release()} on any
+     * returned buffer.
+     *
+     * @param table the translation table (or null or {@link PyNone})
+     * @return the buffer view of the table or null if there is no table
+     * @throws PyException if the table is not exacltly 256 bytes long
+     */
+    private PyBuffer getTranslationTable(PyObject table) throws PyException {
         PyBuffer tab = null;
+        // Normalise the translation table to a View (if there is one).
         if (table != null && table != Py.None) {
             tab = getViewOrError(table);
             if (tab.getLen() != 256) {
                 throw Py.ValueError("translation table must be 256 bytes long");
             }
         }
-
-        // Accumulate the result here
-        PyByteArray result = new PyByteArray();
-
-        // There are 4 cases depending on presence/absence of table and deletechars
-
-        if (deletechars != null) {
-
-            // Use a ByteSet to express which bytes to delete
-            ByteSet del;
-            del = new ByteSet(getViewOrError(deletechars));
-
-            // Now, loop over this byte array and write translated bytes to the result
-            int limit = offset + size;
-
-            if (tab != null) {
-                for (int i = offset; i < limit; i++) {
-                    int b = storage[i] & 0xff;
-                    if (!del.contains(b)) {
-                        result.append(tab.byteAt(b));
-                    }
-                }
-
-            } else {
-                // No translation table
-                for (int i = offset; i < limit; i++) {
-                    int b = storage[i] & 0xff;
-                    if (!del.contains(b)) {
-                        result.append((byte)b);
-                    }
-                }
-            }
-
-        } else {
-            // No deletion set.
-
-            // Now, loop over this byte array and write translated bytes to the result
-            int limit = offset + size;
-            if (tab != null) {
-                for (int i = offset; i < limit; i++) {
-                    int b = storage[i] & 0xff;
-                    result.append(tab.byteAt(b));
-                }
-
-            } else {
-                // No translation table or deletion set: just copy
-                result.extend(this);
-            }
-
-        }
-
-        return result;
+        return tab;
     }
 
     /**
diff --git a/src/org/python/core/PyFile.java b/src/org/python/core/PyFile.java
--- a/src/org/python/core/PyFile.java
+++ b/src/org/python/core/PyFile.java
@@ -478,13 +478,9 @@
 
         } else if (obj instanceof BufferProtocol) {
             // Try to get a byte-oriented buffer
-            PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO);
-            try {
+            try (PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO)) {
                 // ... and treat those bytes as a String
                 return buf.toString();
-            } finally {
-                // We should release the buffer
-                buf.release();
             }
         }
 
diff --git a/src/org/python/core/PyString.java b/src/org/python/core/PyString.java
--- a/src/org/python/core/PyString.java
+++ b/src/org/python/core/PyString.java
@@ -684,11 +684,8 @@
             return ((PyString)obj).getString();
         } else if (obj instanceof BufferProtocol) {
             // Other object with buffer API: briefly access the buffer
-            PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.SIMPLE);
-            try {
+            try (PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO)) {
                 return buf.toString();
-            } finally {
-                buf.release();
             }
         } else {
             return null;
diff --git a/src/org/python/core/PyUnicode.java b/src/org/python/core/PyUnicode.java
--- a/src/org/python/core/PyUnicode.java
+++ b/src/org/python/core/PyUnicode.java
@@ -443,11 +443,8 @@
             return (PyUnicode)o;
         } else if (o instanceof BufferProtocol) {
             // PyString or PyByteArray, PyMemoryView, Py2kBuffer ...
-            PyBuffer buf = ((BufferProtocol)o).getBuffer(PyBUF.FULL_RO);
-            try {
+            try (PyBuffer buf = ((BufferProtocol)o).getBuffer(PyBUF.FULL_RO)) {
                 return new PyUnicode(buf.toString());
-            } finally {
-                buf.release();
             }
         } else {
             // o is some type not allowed:
diff --git a/src/org/python/core/io/TextIOBase.java b/src/org/python/core/io/TextIOBase.java
--- a/src/org/python/core/io/TextIOBase.java
+++ b/src/org/python/core/io/TextIOBase.java
@@ -115,13 +115,11 @@
             return read.length();
 
         } else if (buf instanceof BufferProtocol) {
-            PyBuffer view = ((BufferProtocol)buf).getBuffer(PyBUF.SIMPLE);
-            if (view.isReadonly()) {
-                // More helpful than falling through to CPython message
-                throw Py.TypeError("cannot read into read-only " + buf.getType().fastGetName());
-
-            } else {
-                try {
+            try (PyBuffer view = ((BufferProtocol)buf).getBuffer(PyBUF.FULL_RO)) {
+                if (view.isReadonly()) {
+                    // More helpful than falling through to CPython message
+                    throw Py.TypeError("cannot read into read-only " + buf.getType().fastGetName());
+                } else {
                     // Inefficiently, we have to go via a String
                     String read = read(view.getLen());
                     int n = read.length();
@@ -129,11 +127,7 @@
                         view.storeAt((byte)read.charAt(i), i);
                     }
                     return read.length();
-                } finally {
-                    // We should release the buffer explicitly
-                    view.release();
                 }
-
             }
         }
 
diff --git a/src/org/python/modules/sre/PatternObject.java b/src/org/python/modules/sre/PatternObject.java
--- a/src/org/python/modules/sre/PatternObject.java
+++ b/src/org/python/modules/sre/PatternObject.java
@@ -373,13 +373,9 @@
 
         } else if (obj instanceof BufferProtocol) {
             // Try to get a byte-oriented buffer
-            PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO);
-            try {
+            try (PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO)){
                 // ... and treat those bytes as a PyString
                 return new PyString(buf.toString());
-            } finally {
-                // We should release the buffer
-                buf.release();
             }
         }
 

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 14 14:02:21 2014
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 14 Jun 2014 14:02:21 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython_=28merge_default_-=3E_default?=
 =?utf-8?q?=29=3A_Merge_try-with-resources_buffer_work_to_trunk?=
Message-ID: <3grHVP4LDxz7Ljk@mail.python.org>

http://hg.python.org/jython/rev/5a20c6838896
changeset:   7293:5a20c6838896
parent:      7290:6c718e5e9ae9
parent:      7292:18d60bea34af
user:        Jeff Allen 
date:        Sat Jun 14 10:39:37 2014 +0100
summary:
  Merge try-with-resources buffer work to trunk

files:
  src/org/python/core/BaseBytes.java            |  110 +-----
  src/org/python/core/PyArray.java              |    5 +-
  src/org/python/core/PyBuffer.java             |    6 +-
  src/org/python/core/PyByteArray.java          |  147 +++++----
  src/org/python/core/PyFile.java               |    6 +-
  src/org/python/core/PyString.java             |    5 +-
  src/org/python/core/PyUnicode.java            |    5 +-
  src/org/python/core/buffer/BaseBuffer.java    |    5 +
  src/org/python/core/io/TextIOBase.java        |   16 +-
  src/org/python/modules/sre/PatternObject.java |    6 +-
  tests/java/org/python/core/PyBufferTest.java  |   29 +-
  11 files changed, 157 insertions(+), 183 deletions(-)


diff --git a/src/org/python/core/BaseBytes.java b/src/org/python/core/BaseBytes.java
--- a/src/org/python/core/BaseBytes.java
+++ b/src/org/python/core/BaseBytes.java
@@ -345,10 +345,11 @@
      */
     protected void init(BufferProtocol value) throws PyException {
         // Get the buffer view
-        PyBuffer view = value.getBuffer(PyBUF.FULL_RO);
-        // Create storage for the bytes and have the view drop them in
-        newStorage(view.getLen());
-        view.copyTo(storage, offset);
+        try (PyBuffer view = value.getBuffer(PyBUF.FULL_RO)) {
+            // Create storage for the bytes and have the view drop them in
+            newStorage(view.getLen());
+            view.copyTo(storage, offset);
+        }
     }
 
     /**
@@ -834,21 +835,16 @@
             return 0;
 
         } else {
-
             // Try to get a byte-oriented view
-            PyBuffer bv = getView(b);
-
-            if (bv == null) {
-                // Signifies a type mis-match. See PyObject._cmp_unsafe() and related code.
-                return -2;
-
-            } else {
-                try {
+            try (PyBuffer bv = getView(b)) {
+
+                if (bv == null) {
+                    // Signifies a type mis-match. See PyObject._cmp_unsafe() and related code.
+                    return -2;
+
+                } else {
                     // Compare this with other object viewed as a buffer
                     return compare(this, bv);
-                } finally {
-                    // Must alsways let go of the buffer
-                    bv.release();
                 }
             }
         }
@@ -871,14 +867,13 @@
         } else {
 
             // Try to get a byte-oriented view
-            PyBuffer bv = getView(b);
-
-            if (bv == null) {
-                // Signifies a type mis-match. See PyObject._cmp_unsafe() and related code.
-                return -2;
-
-            } else {
-                try {
+            try (PyBuffer bv = getView(b)) {
+
+                if (bv == null) {
+                    // Signifies a type mis-match. See PyObject._cmp_unsafe() and related code.
+                    return -2;
+
+                } else {
                     if (bv.getLen() != size) {
                         // Different size: can't be equal, and we don't care which is bigger
                         return 1;
@@ -886,9 +881,6 @@
                         // Compare this with other object viewed as a buffer
                         return compare(this, bv);
                     }
-                } finally {
-                    // Must alsways let go of the buffer
-                    bv.release();
                 }
             }
         }
@@ -1017,13 +1009,10 @@
             return index(b) >= 0;
         } else {
             // Caller is treating this as a byte-string and looking for substring 'target'
-            PyBuffer targetView = getViewOrError(target);
-            try {
+            try (PyBuffer targetView = getViewOrError(target)) {
                 Finder finder = new Finder(targetView);
                 finder.setText(this);
                 return finder.nextIndex() >= 0;
-            } finally {
-                targetView.release();
             }
         }
     }
@@ -1080,9 +1069,7 @@
     private boolean match(PyObject target, int pos, int n, boolean endswith) {
 
         // Error if not something we can treat as a view of bytes
-        PyBuffer vt = getViewOrError(target);
-
-        try {
+        try (PyBuffer vt = getViewOrError(target)) {
             int j = 0, len = vt.getLen();
 
             if (!endswith) {
@@ -1106,10 +1093,6 @@
                 }
             }
             return true; // They must all have matched
-
-        } finally {
-            // Let go of the buffer we acquired
-            vt.release();
         }
     }
 
@@ -1818,8 +1801,7 @@
      * @return count of occurrences of sub within this byte array
      */
     final int basebytes_count(PyObject sub, PyObject ostart, PyObject oend) {
-        PyBuffer vsub = getViewOrError(sub);
-        try {
+        try (PyBuffer vsub = getViewOrError(sub)) {
             Finder finder = new Finder(vsub);
 
             // Convert [ostart:oend] to integers
@@ -1827,8 +1809,6 @@
 
             // Make this slice the thing we count within.
             return finder.count(storage, offset + index[0], index[3]);
-        } finally {
-            vsub.release();
         }
     }
 
@@ -1846,12 +1826,9 @@
      * @return index of start of occurrence of sub within this byte array
      */
     final int basebytes_find(PyObject sub, PyObject ostart, PyObject oend) {
-        PyBuffer vsub = getViewOrError(sub);
-        try {
+        try (PyBuffer vsub = getViewOrError(sub)) {
             Finder finder = new Finder(vsub);
             return find(finder, ostart, oend);
-        } finally {
-            vsub.release();
         }
     }
 
@@ -2026,9 +2003,7 @@
     final synchronized PyTuple basebytes_partition(PyObject sep) {
 
         // View the separator as a byte array (or error if we can't)
-        PyBuffer separator = getViewOrError(sep);
-
-        try {
+        try (PyBuffer separator = getViewOrError(sep)) {
             // Create a Finder for the separator and set it on this byte array
             int n = checkForEmptySeparator(separator);
             Finder finder = new Finder(separator);
@@ -2043,8 +2018,6 @@
                 // Not found: choose values leading to ([0:size], '', '')
                 return partition(size, size);
             }
-        } finally {
-            separator.release();
         }
     }
 
@@ -2080,12 +2053,9 @@
      * @return index of start of occurrence of sub within this byte array
      */
     final int basebytes_rfind(PyObject sub, PyObject ostart, PyObject oend) {
-        PyBuffer vsub = getViewOrError(sub);
-        try {
+        try (PyBuffer vsub = getViewOrError(sub)) {
             Finder finder = new ReverseFinder(vsub);
             return find(finder, ostart, oend);
-        } finally {
-            vsub.release();
         }
     }
 
@@ -2128,9 +2098,7 @@
     final synchronized PyByteArray basebytes_replace(PyObject oldB, PyObject newB, int maxcount) {
 
         // View the to and from as byte arrays (or error if we can't)
-        PyBuffer to = getViewOrError(newB), from = null;
-        try {
-            from = getViewOrError(oldB);
+        try (PyBuffer to = getViewOrError(newB); PyBuffer from = getViewOrError(oldB)) {
             /*
              * The logic of the first section is copied exactly from CPython in order to get the
              * same behaviour. The "headline" description of replace is simple enough but the corner
@@ -2184,16 +2152,6 @@
                 // Otherwise use the generic algorithm
                 return replace_substring(from, to, maxcount);
             }
-
-        } finally {
-            /*
-             * Release the buffers we acquired: there must be a to buffer and there might be a from
-             * buffer.
-             */
-            to.release();
-            if (from != null) {
-                from.release();
-            }
         }
     }
 
@@ -2475,8 +2433,7 @@
     final synchronized PyTuple basebytes_rpartition(PyObject sep) {
 
         // View the separator as a byte array (or error if we can't)
-        PyBuffer separator = getViewOrError(sep);
-        try {
+        try (PyBuffer separator = getViewOrError(sep)) {
             // Create a Finder for the separtor and set it on this byte array
             int n = checkForEmptySeparator(separator);
             Finder finder = new ReverseFinder(separator);
@@ -2491,8 +2448,6 @@
                 // Not found: choose values leading to ('', '', [0:size])
                 return partition(0, 0);
             }
-        } finally {
-            separator.release();
         }
     }
 
@@ -2590,9 +2545,7 @@
     final synchronized PyList basebytes_rsplit_explicit(PyObject sep, int maxsplit) {
 
         // The separator may be presented as anything viewable as bytes
-        PyBuffer separator = getViewOrError(sep);
-
-        try {
+        try (PyBuffer separator = getViewOrError(sep)) {
             int n = checkForEmptySeparator(separator);
 
             PyList result = new PyList();
@@ -2627,8 +2580,6 @@
                 result.add(0, word);
             }
             return result;
-        } finally {
-            separator.release();
         }
     }
 
@@ -2794,8 +2745,7 @@
     final synchronized PyList basebytes_split_explicit(PyObject sep, int maxsplit) {
 
         // The separator may be presented as anything viewable as bytes
-        PyBuffer separator = getViewOrError(sep);
-        try {
+        try (PyBuffer separator = getViewOrError(sep)) {
             checkForEmptySeparator(separator);
 
             PyList result = new PyList();
@@ -2821,8 +2771,6 @@
             // Append the remaining unsplit text
             result.append(getslice(p - offset, size));
             return result;
-        } finally {
-            separator.release();
         }
     }
 
diff --git a/src/org/python/core/PyArray.java b/src/org/python/core/PyArray.java
--- a/src/org/python/core/PyArray.java
+++ b/src/org/python/core/PyArray.java
@@ -1131,8 +1131,7 @@
 
             } else {
                 // Access the bytes
-                PyBuffer pybuf = ((BufferProtocol)input).getBuffer(PyBUF.STRIDED_RO);
-                try {
+                try (PyBuffer pybuf = ((BufferProtocol)input).getBuffer(PyBUF.STRIDED_RO)) {
                     // Provide argument as stream of bytes for fromstream method
                     if (pybuf.getNdim() == 1) {
                         if (pybuf.getStrides()[0] == 1) {
@@ -1149,8 +1148,6 @@
                         // Currently don't support n-dimensional sources
                         throw Py.ValueError("multi-dimensional buffer not supported");
                     }
-                } finally {
-                    pybuf.release();
                 }
             }
 
diff --git a/src/org/python/core/PyBuffer.java b/src/org/python/core/PyBuffer.java
--- a/src/org/python/core/PyBuffer.java
+++ b/src/org/python/core/PyBuffer.java
@@ -5,7 +5,7 @@
  * the counterpart of the CPython Py_buffer struct. Several concrete types implement
  * this interface in order to provide tailored support for different storage organisations.
  */
-public interface PyBuffer extends PyBUF, BufferProtocol {
+public interface PyBuffer extends PyBUF, BufferProtocol, AutoCloseable {
 
     /*
      * The different behaviours required as the actual structure of the buffer changes (from one
@@ -183,6 +183,10 @@
      */
     void release();
 
+    /** An alias for {@link #release()} to satisfy {@link AutoCloseable}. */
+    @Override
+    void close();
+
     /**
      * True only if the buffer has been released with (the required number of calls to)
      * {@link #release()} or some equivalent operation. The consumer may be sharing the reference
diff --git a/src/org/python/core/PyByteArray.java b/src/org/python/core/PyByteArray.java
--- a/src/org/python/core/PyByteArray.java
+++ b/src/org/python/core/PyByteArray.java
@@ -548,24 +548,25 @@
      */
     private void setslice(int start, int stop, int step, BufferProtocol value) throws PyException {
 
-        PyBuffer view = value.getBuffer(PyBUF.FULL_RO);
+        try (PyBuffer view = value.getBuffer(PyBUF.FULL_RO)) {
 
-        int len = view.getLen();
+            int len = view.getLen();
 
-        if (step == 1) {
-            // Delete this[start:stop] and open a space of the right size
-            storageReplace(start, stop - start, len);
-            view.copyTo(storage, start + offset);
+            if (step == 1) {
+                // Delete this[start:stop] and open a space of the right size
+                storageReplace(start, stop - start, len);
+                view.copyTo(storage, start + offset);
 
-        } else {
-            // This is an extended slice which means we are replacing elements
-            int n = sliceLength(start, stop, step);
-            if (n != len) {
-                throw SliceSizeError("bytes", len, n);
-            }
+            } else {
+                // This is an extended slice which means we are replacing elements
+                int n = sliceLength(start, stop, step);
+                if (n != len) {
+                    throw SliceSizeError("bytes", len, n);
+                }
 
-            for (int io = start + offset, j = 0; j < n; io += step, j++) {
-                storage[io] = view.byteAt(j);    // Assign this[i] = value[j]
+                for (int io = start + offset, j = 0; j < n; io += step, j++) {
+                    storage[io] = view.byteAt(j);    // Assign this[i] = value[j]
+                }
             }
         }
     }
@@ -2021,66 +2022,78 @@
     @ExposedMethod(defaults = "null", doc = BuiltinDocs.bytearray_translate_doc)
     final PyByteArray bytearray_translate(PyObject table, PyObject deletechars) {
 
-        // Normalise the translation table to a View
+        // Work with the translation table (if there is one) as a PyBuffer view.
+        try (PyBuffer tab = getTranslationTable(table)) {
+
+            // Accumulate the result here
+            PyByteArray result = new PyByteArray();
+
+            // There are 4 cases depending on presence/absence of table and deletechars
+
+            if (deletechars != null) {
+
+                // Work with the deletion characters as a buffer too.
+                try (PyBuffer d = getViewOrError(deletechars)) {
+                    // Use a ByteSet to express which bytes to delete
+                    ByteSet del = new ByteSet(d);
+                    int limit = offset + size;
+                    if (tab == null) {
+                        // No translation table, so we're just copying with omissions
+                        for (int i = offset; i < limit; i++) {
+                            int b = storage[i] & 0xff;
+                            if (!del.contains(b)) {
+                                result.append((byte)b);
+                            }
+                        }
+                    } else {
+                        // Loop over this byte array and write translated bytes to the result
+                        for (int i = offset; i < limit; i++) {
+                            int b = storage[i] & 0xff;
+                            if (!del.contains(b)) {
+                                result.append(tab.byteAt(b));
+                            }
+                        }
+                    }
+                }
+
+            } else {
+                // No deletion set.
+                if (tab == null) {
+                    // ... and no translation table either: just copy
+                    result.extend(this);
+                } else {
+                    int limit = offset + size;
+                    // Loop over this byte array and write translated bytes to the result
+                    for (int i = offset; i < limit; i++) {
+                        int b = storage[i] & 0xff;
+                        result.append(tab.byteAt(b));
+                    }
+                }
+            }
+
+            return result;
+        }
+    }
+
+    /**
+     * Return a {@link PyBuffer} representing a translation table, or raise an exception if it is
+     * the wrong size. The caller is responsible for calling {@link PyBuffer#release()} on any
+     * returned buffer.
+     *
+     * @param table the translation table (or null or {@link PyNone})
+     * @return the buffer view of the table or null if there is no table
+     * @throws PyException if the table is not exacltly 256 bytes long
+     */
+    private PyBuffer getTranslationTable(PyObject table) throws PyException {
         PyBuffer tab = null;
+        // Normalise the translation table to a View (if there is one).
         if (table != null && table != Py.None) {
             tab = getViewOrError(table);
             if (tab.getLen() != 256) {
                 throw Py.ValueError("translation table must be 256 bytes long");
             }
         }
-
-        // Accumulate the result here
-        PyByteArray result = new PyByteArray();
-
-        // There are 4 cases depending on presence/absence of table and deletechars
-
-        if (deletechars != null) {
-
-            // Use a ByteSet to express which bytes to delete
-            ByteSet del;
-            del = new ByteSet(getViewOrError(deletechars));
-
-            // Now, loop over this byte array and write translated bytes to the result
-            int limit = offset + size;
-
-            if (tab != null) {
-                for (int i = offset; i < limit; i++) {
-                    int b = storage[i] & 0xff;
-                    if (!del.contains(b)) {
-                        result.append(tab.byteAt(b));
-                    }
-                }
-
-            } else {
-                // No translation table
-                for (int i = offset; i < limit; i++) {
-                    int b = storage[i] & 0xff;
-                    if (!del.contains(b)) {
-                        result.append((byte)b);
-                    }
-                }
-            }
-
-        } else {
-            // No deletion set.
-
-            // Now, loop over this byte array and write translated bytes to the result
-            int limit = offset + size;
-            if (tab != null) {
-                for (int i = offset; i < limit; i++) {
-                    int b = storage[i] & 0xff;
-                    result.append(tab.byteAt(b));
-                }
-
-            } else {
-                // No translation table or deletion set: just copy
-                result.extend(this);
-            }
-
-        }
-
-        return result;
+        return tab;
     }
 
     /**
diff --git a/src/org/python/core/PyFile.java b/src/org/python/core/PyFile.java
--- a/src/org/python/core/PyFile.java
+++ b/src/org/python/core/PyFile.java
@@ -478,13 +478,9 @@
 
         } else if (obj instanceof BufferProtocol) {
             // Try to get a byte-oriented buffer
-            PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO);
-            try {
+            try (PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO)) {
                 // ... and treat those bytes as a String
                 return buf.toString();
-            } finally {
-                // We should release the buffer
-                buf.release();
             }
         }
 
diff --git a/src/org/python/core/PyString.java b/src/org/python/core/PyString.java
--- a/src/org/python/core/PyString.java
+++ b/src/org/python/core/PyString.java
@@ -684,11 +684,8 @@
             return ((PyString)obj).getString();
         } else if (obj instanceof BufferProtocol) {
             // Other object with buffer API: briefly access the buffer
-            PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.SIMPLE);
-            try {
+            try (PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO)) {
                 return buf.toString();
-            } finally {
-                buf.release();
             }
         } else {
             return null;
diff --git a/src/org/python/core/PyUnicode.java b/src/org/python/core/PyUnicode.java
--- a/src/org/python/core/PyUnicode.java
+++ b/src/org/python/core/PyUnicode.java
@@ -443,11 +443,8 @@
             return (PyUnicode)o;
         } else if (o instanceof BufferProtocol) {
             // PyString or PyByteArray, PyMemoryView, Py2kBuffer ...
-            PyBuffer buf = ((BufferProtocol)o).getBuffer(PyBUF.FULL_RO);
-            try {
+            try (PyBuffer buf = ((BufferProtocol)o).getBuffer(PyBUF.FULL_RO)) {
                 return new PyUnicode(buf.toString());
-            } finally {
-                buf.release();
             }
         } else {
             // o is some type not allowed:
diff --git a/src/org/python/core/buffer/BaseBuffer.java b/src/org/python/core/buffer/BaseBuffer.java
--- a/src/org/python/core/buffer/BaseBuffer.java
+++ b/src/org/python/core/buffer/BaseBuffer.java
@@ -522,6 +522,11 @@
     }
 
     @Override
+    public void close() {
+        release();
+    }
+
+    @Override
     public boolean isReleased() {
         return exports <= 0;
     }
diff --git a/src/org/python/core/io/TextIOBase.java b/src/org/python/core/io/TextIOBase.java
--- a/src/org/python/core/io/TextIOBase.java
+++ b/src/org/python/core/io/TextIOBase.java
@@ -115,13 +115,11 @@
             return read.length();
 
         } else if (buf instanceof BufferProtocol) {
-            PyBuffer view = ((BufferProtocol)buf).getBuffer(PyBUF.SIMPLE);
-            if (view.isReadonly()) {
-                // More helpful than falling through to CPython message
-                throw Py.TypeError("cannot read into read-only " + buf.getType().fastGetName());
-
-            } else {
-                try {
+            try (PyBuffer view = ((BufferProtocol)buf).getBuffer(PyBUF.FULL_RO)) {
+                if (view.isReadonly()) {
+                    // More helpful than falling through to CPython message
+                    throw Py.TypeError("cannot read into read-only " + buf.getType().fastGetName());
+                } else {
                     // Inefficiently, we have to go via a String
                     String read = read(view.getLen());
                     int n = read.length();
@@ -129,11 +127,7 @@
                         view.storeAt((byte)read.charAt(i), i);
                     }
                     return read.length();
-                } finally {
-                    // We should release the buffer explicitly
-                    view.release();
                 }
-
             }
         }
 
diff --git a/src/org/python/modules/sre/PatternObject.java b/src/org/python/modules/sre/PatternObject.java
--- a/src/org/python/modules/sre/PatternObject.java
+++ b/src/org/python/modules/sre/PatternObject.java
@@ -373,13 +373,9 @@
 
         } else if (obj instanceof BufferProtocol) {
             // Try to get a byte-oriented buffer
-            PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO);
-            try {
+            try (PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO)){
                 // ... and treat those bytes as a PyString
                 return new PyString(buf.toString());
-            } finally {
-                // We should release the buffer
-                buf.release();
             }
         }
 
diff --git a/tests/java/org/python/core/PyBufferTest.java b/tests/java/org/python/core/PyBufferTest.java
--- a/tests/java/org/python/core/PyBufferTest.java
+++ b/tests/java/org/python/core/PyBufferTest.java
@@ -701,8 +701,13 @@
 
         Set uniqueBuffers = new HashSet();
 
+        // Test a balanced sequence of acquire and release using try-with-resources
         for (BufferTestPair test : buffersToRead) {
-            // Test a pattern of acquire and release with one more release than acquire
+            doTestTryWithResources(test);
+        }
+
+        // Now test a pattern of acquire and release with one more release than acquire
+        for (BufferTestPair test : buffersToRead) {
             doTestRelease(test);
             uniqueBuffers.add(test.view);
         }
@@ -718,6 +723,28 @@
                 doTestGetAfterRelease(test);
             }
         }
+
+    }
+
+    /**
+     * Exercise try-with-resources on one BufferTestPair.
+     */
+    private void doTestTryWithResources(BufferTestPair test) {
+
+        if (verbosity > 0) {
+            System.out.println("try with resources: " + test);
+        }
+        int flags = PyBUF.STRIDES | PyBUF.FORMAT;
+        BufferProtocol sub = test.subject;
+
+        // The object will be exporting test.view and N other views we don't know about
+        try (PyBuffer c = sub.getBuffer(flags)) {   // = N+1 exports
+            try (PyBuffer b = sub.getBuffer(PyBUF.FULL_RO); PyBuffer d =c.getBuffer(flags)) {
+                checkExporting(sub);// = N+3 exports
+            }
+            checkExporting(sub);                    // = N+1 exports
+        }
+        checkExporting(sub);                        // = N export
     }
 
     /**

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 14 22:44:48 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 14 Jun 2014 22:44:48 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Update_collections=2Edeque_?=
 =?utf-8?q?to_2=2E7_compliance=2C_including_supporting?=
Message-ID: <3grW5D5KgJz7LjR@mail.python.org>

http://hg.python.org/jython/rev/759e56cfcac7
changeset:   7294:759e56cfcac7
user:        Santoso Wijaya 
date:        Sat Jun 14 14:45:14 2014 -0600
summary:
  Update collections.deque to 2.7 compliance, including supporting
maxlen kwarg. Add synchronization to ensure threadsafety guarantees.

Fixes http://bugs.jython.org/issue1949

files:
  Lib/test/test_deque.py                           |  245 +++++++-
  Lib/test/test_deque_jy.py                        |   77 ++
  src/org/python/modules/_collections/PyDeque.java |  275 +++++++--
  3 files changed, 502 insertions(+), 95 deletions(-)


diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py
--- a/Lib/test/test_deque.py
+++ b/Lib/test/test_deque.py
@@ -1,12 +1,12 @@
 from collections import deque
 import unittest
 from test import test_support, seq_tests
-from weakref import proxy
+import gc
+import weakref
 import copy
 import cPickle as pickle
-from cStringIO import StringIO
 import random
-import os
+import struct
 
 BIG = 100000
 
@@ -29,8 +29,8 @@
 class TestBasic(unittest.TestCase):
 
     def test_basics(self):
-        d = deque(xrange(100))
-        d.__init__(xrange(100, 200))
+        d = deque(xrange(-5125, -5000))
+        d.__init__(xrange(200))
         for i in xrange(200, 400):
             d.append(i)
         for i in reversed(xrange(-200, 0)):
@@ -47,6 +47,106 @@
         self.assertEqual(right, range(150, 400))
         self.assertEqual(list(d), range(50, 150))
 
+    def test_maxlen(self):
+        self.assertRaises(ValueError, deque, 'abc', -1)
+        self.assertRaises(ValueError, deque, 'abc', -2)
+        it = iter(range(10))
+        d = deque(it, maxlen=3)
+        self.assertEqual(list(it), [])
+        self.assertEqual(repr(d), 'deque([7, 8, 9], maxlen=3)')
+        self.assertEqual(list(d), range(7, 10))
+        self.assertEqual(d, deque(range(10), 3))
+        d.append(10)
+        self.assertEqual(list(d), range(8, 11))
+        d.appendleft(7)
+        self.assertEqual(list(d), range(7, 10))
+        d.extend([10, 11])
+        self.assertEqual(list(d), range(9, 12))
+        d.extendleft([8, 7])
+        self.assertEqual(list(d), range(7, 10))
+        d = deque(xrange(200), maxlen=10)
+        d.append(d)
+        test_support.unlink(test_support.TESTFN)
+        fo = open(test_support.TESTFN, "wb")
+        try:
+            print >> fo, d,
+            fo.close()
+            fo = open(test_support.TESTFN, "rb")
+            self.assertEqual(fo.read(), repr(d))
+        finally:
+            fo.close()
+            test_support.unlink(test_support.TESTFN)
+
+        d = deque(range(10), maxlen=None)
+        self.assertEqual(repr(d), 'deque([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])')
+        fo = open(test_support.TESTFN, "wb")
+        try:
+            print >> fo, d,
+            fo.close()
+            fo = open(test_support.TESTFN, "rb")
+            self.assertEqual(fo.read(), repr(d))
+        finally:
+            fo.close()
+            test_support.unlink(test_support.TESTFN)
+
+    def test_maxlen_zero(self):
+        it = iter(range(100))
+        deque(it, maxlen=0)
+        self.assertEqual(list(it), [])
+
+        it = iter(range(100))
+        d = deque(maxlen=0)
+        d.extend(it)
+        self.assertEqual(list(it), [])
+
+        it = iter(range(100))
+        d = deque(maxlen=0)
+        d.extendleft(it)
+        self.assertEqual(list(it), [])
+
+    def test_maxlen_attribute(self):
+        self.assertEqual(deque().maxlen, None)
+        self.assertEqual(deque('abc').maxlen, None)
+        self.assertEqual(deque('abc', maxlen=4).maxlen, 4)
+        self.assertEqual(deque('abc', maxlen=2).maxlen, 2)
+        self.assertEqual(deque('abc', maxlen=0).maxlen, 0)
+        with self.assertRaises(AttributeError):
+            d = deque('abc')
+            d.maxlen = 10
+
+    def test_count(self):
+        for s in ('', 'abracadabra', 'simsalabim'*500+'abc'):
+            s = list(s)
+            d = deque(s)
+            for letter in 'abcdefghijklmnopqrstuvwxyz':
+                self.assertEqual(s.count(letter), d.count(letter), (s, d, letter))
+        self.assertRaises(TypeError, d.count)       # too few args
+        self.assertRaises(TypeError, d.count, 1, 2) # too many args
+        class BadCompare:
+            def __eq__(self, other):
+                raise ArithmeticError
+        d = deque([1, 2, BadCompare(), 3])
+        self.assertRaises(ArithmeticError, d.count, 2)
+        d = deque([1, 2, 3])
+        self.assertRaises(ArithmeticError, d.count, BadCompare())
+        class MutatingCompare:
+            def __eq__(self, other):
+                self.d.pop()
+                return True
+        m = MutatingCompare()
+        d = deque([1, 2, 3, m, 4, 5])
+        m.d = d
+        self.assertRaises(RuntimeError, d.count, 3)
+
+        # test issue11004
+        # block advance failed after rotation aligned elements on right side of block
+        d = deque([None]*16)
+        for i in range(len(d)):
+            d.rotate(-1)
+        d.rotate(1)
+        self.assertEqual(d.count(1), 0)
+        self.assertEqual(d.count(None), 16)
+
     def test_comparisons(self):
         d = deque('xabc'); d.popleft()
         for e in [d, deque('abc'), deque('ab'), deque(), list(d)]:
@@ -69,12 +169,23 @@
         self.assertRaises(TypeError, d.extend, 1)
         d.extend('bcd')
         self.assertEqual(list(d), list('abcd'))
+        d.extend(d)
+        self.assertEqual(list(d), list('abcdabcd'))
+
+    def test_iadd(self):
+        d = deque('a')
+        d += 'bcd'
+        self.assertEqual(list(d), list('abcd'))
+        d += d
+        self.assertEqual(list(d), list('abcdabcd'))
 
     def test_extendleft(self):
         d = deque('a')
         self.assertRaises(TypeError, d.extendleft, 1)
         d.extendleft('bcd')
         self.assertEqual(list(d), list(reversed('abcd')))
+        d.extendleft(d)
+        self.assertEqual(list(d), list('abcddcba'))
         d = deque()
         d.extendleft(range(1000))
         self.assertEqual(list(d), list(reversed(range(1000))))
@@ -121,11 +232,23 @@
             self.assertEqual(len(d), n-i)
             j = random.randrange(-len(d), len(d))
             val = d[j]
-            self.assert_(val in d)
+            self.assertIn(val, d)
             del d[j]
-            self.assert_(val not in d)
+            self.assertNotIn(val, d)
         self.assertEqual(len(d), 0)
 
+    def test_reverse(self):
+        n = 500         # O(n**2) test, don't make this too big
+        data = [random.random() for i in range(n)]
+        for i in range(n):
+            d = deque(data[:i])
+            r = d.reverse()
+            self.assertEqual(list(d), list(reversed(data[:i])))
+            self.assertIs(r, None)
+            d.reverse()
+            self.assertEqual(list(d), data[:i])
+        self.assertRaises(TypeError, d.reverse, 1)          # Arity is zero
+
     def test_rotate(self):
         s = tuple('abcde')
         n = len(s)
@@ -224,7 +347,7 @@
         self.assertRaises(RuntimeError, d.remove, 'c')
         for x, y in zip(d, e):
             # verify that original order and values are retained.
-            self.assert_(x is y)
+            self.assertTrue(x is y)
 
         # Handle evil mutator
         for match in (True, False):
@@ -238,23 +361,24 @@
         e = eval(repr(d))
         self.assertEqual(list(d), list(e))
         d.append(d)
-        self.assert_('...' in repr(d))
+        self.assertIn('...', repr(d))
 
     def test_print(self):
         d = deque(xrange(200))
         d.append(d)
+        test_support.unlink(test_support.TESTFN)
+        fo = open(test_support.TESTFN, "wb")
         try:
-            fo = open(test_support.TESTFN, "wb")
             print >> fo, d,
             fo.close()
             fo = open(test_support.TESTFN, "rb")
             self.assertEqual(fo.read(), repr(d))
         finally:
             fo.close()
-            os.remove(test_support.TESTFN)
+            test_support.unlink(test_support.TESTFN)
 
     def test_init(self):
-        self.assertRaises(TypeError, deque, 'abc', 2);
+        self.assertRaises(TypeError, deque, 'abc', 2, 3);
         self.assertRaises(TypeError, deque, 1);
 
     def test_hash(self):
@@ -333,19 +457,19 @@
 
     def test_pickle(self):
         d = deque(xrange(200))
-        for i in (0, 1, 2):
+        for i in range(pickle.HIGHEST_PROTOCOL + 1):
             s = pickle.dumps(d, i)
             e = pickle.loads(s)
             self.assertNotEqual(id(d), id(e))
             self.assertEqual(list(d), list(e))
 
-    def test_pickle_recursive(self):
-        d = deque('abc')
-        d.append(d)
-        for i in (0, 1, 2):
-            e = pickle.loads(pickle.dumps(d, i))
-            self.assertNotEqual(id(d), id(e))
-            self.assertEqual(id(e), id(e[-1]))
+##    def test_pickle_recursive(self):
+##        d = deque('abc')
+##        d.append(d)
+##        for i in range(pickle.HIGHEST_PROTOCOL + 1):
+##            e = pickle.loads(pickle.dumps(d, i))
+##            self.assertNotEqual(id(d), id(e))
+##            self.assertEqual(id(e), id(e[-1]))
 
     def test_deepcopy(self):
         mut = [10]
@@ -378,6 +502,37 @@
             d.append(1)
             gc.collect()
 
+    def test_container_iterator(self):
+        # Bug #3680: tp_traverse was not implemented for deque iterator objects
+        class C(object):
+            pass
+        for i in range(2):
+            obj = C()
+            ref = weakref.ref(obj)
+            if i == 0:
+                container = deque([obj, 1])
+            else:
+                container = reversed(deque([obj, 1]))
+            obj.x = iter(container)
+            del obj, container
+            gc.collect()
+            self.assertTrue(ref() is None, "Cycle was not collected")
+
+    check_sizeof = test_support.check_sizeof
+
+    @test_support.cpython_only
+    def test_sizeof(self):
+        BLOCKLEN = 62
+        basesize = test_support.calcobjsize('2P4PlP')
+        blocksize = struct.calcsize('2P%dP' % BLOCKLEN)
+        self.assertEqual(object.__sizeof__(deque()), basesize)
+        check = self.check_sizeof
+        check(deque(), basesize + blocksize)
+        check(deque('a'), basesize + blocksize)
+        check(deque('a' * (BLOCKLEN // 2)), basesize + blocksize)
+        check(deque('a' * (BLOCKLEN // 2 + 1)), basesize + 2 * blocksize)
+        check(deque('a' * (42 * BLOCKLEN)), basesize + 43 * blocksize)
+
 class TestVariousIteratorArgs(unittest.TestCase):
 
     def test_constructor(self):
@@ -412,8 +567,8 @@
 class TestSubclass(unittest.TestCase):
 
     def test_basics(self):
-        d = Deque(xrange(100))
-        d.__init__(xrange(100, 200))
+        d = Deque(xrange(25))
+        d.__init__(xrange(200))
         for i in xrange(200, 400):
             d.append(i)
         for i in reversed(xrange(-200, 0)):
@@ -451,28 +606,44 @@
         self.assertEqual(type(d), type(e))
         self.assertEqual(list(d), list(e))
 
-    def test_pickle(self):
-        d = Deque('abc')
-        d.append(d)
+        d = Deque('abcde', maxlen=4)
 
-        e = pickle.loads(pickle.dumps(d))
+        e = d.__copy__()
+        self.assertEqual(type(d), type(e))
+        self.assertEqual(list(d), list(e))
+
+        e = Deque(d)
+        self.assertEqual(type(d), type(e))
+        self.assertEqual(list(d), list(e))
+
+        s = pickle.dumps(d)
+        e = pickle.loads(s)
         self.assertNotEqual(id(d), id(e))
         self.assertEqual(type(d), type(e))
-        dd = d.pop()
-        ee = e.pop()
-        self.assertEqual(id(e), id(ee))
-        self.assertEqual(d, e)
+        self.assertEqual(list(d), list(e))
 
-        d.x = d
-        e = pickle.loads(pickle.dumps(d))
-        self.assertEqual(id(e), id(e.x))
-
-        d = DequeWithBadIter('abc')
-        self.assertRaises(TypeError, pickle.dumps, d)
+##    def test_pickle(self):
+##        d = Deque('abc')
+##        d.append(d)
+##
+##        e = pickle.loads(pickle.dumps(d))
+##        self.assertNotEqual(id(d), id(e))
+##        self.assertEqual(type(d), type(e))
+##        dd = d.pop()
+##        ee = e.pop()
+##        self.assertEqual(id(e), id(ee))
+##        self.assertEqual(d, e)
+##
+##        d.x = d
+##        e = pickle.loads(pickle.dumps(d))
+##        self.assertEqual(id(e), id(e.x))
+##
+##        d = DequeWithBadIter('abc')
+##        self.assertRaises(TypeError, pickle.dumps, d)
 
     def test_weakref(self):
         d = deque('gallahad')
-        p = proxy(d)
+        p = weakref.proxy(d)
         self.assertEqual(str(p), str(d))
         d = None
         if test_support.is_jython:
diff --git a/Lib/test/test_deque_jy.py b/Lib/test/test_deque_jy.py
new file mode 100644
--- /dev/null
+++ b/Lib/test/test_deque_jy.py
@@ -0,0 +1,77 @@
+import time
+import random
+import unittest
+import threading
+from test import test_support
+from collections import deque
+
+
+class ThreadSafetyTestCase(unittest.TestCase):
+
+    def run_threads(self, f, num=10):
+        threads = []
+        for i in xrange(num):
+            t = threading.Thread(target=f)
+            t.start()
+            threads.append(t)
+        timeout = 10. # be especially generous
+        for t in threads:
+            t.join(timeout)
+            timeout = 0.
+        for t in threads:
+            self.assertFalse(t.isAlive())
+
+    def test_append_remove(self):
+        # derived from Itamar Shtull-Trauring's test for issue 521701
+        d = deque(['sentinel'])
+        def tester():
+            ct = threading.currentThread()
+            for i in range(1000):
+                d.append(ct)
+                time.sleep(0.0001)
+                d.remove(ct)
+        self.run_threads(tester)
+        self.assertEqual(d, deque(['sentinel']))
+
+    def test_append_pop(self):
+        d = deque(['sentinel'])
+        def tester():
+            ct = threading.currentThread()
+            for i in range(1000):
+                d.append(ct)
+                time.sleep(0.0001)
+                d.pop()
+        self.run_threads(tester)
+        self.assertEqual(d, deque(['sentinel']))
+
+    def test_appendleft_popleft(self):
+        d = deque()
+        def tester():
+            ct = threading.currentThread()
+            for i in range(1000):
+                d.appendleft(ct)
+                time.sleep(0.0001)
+                d.popleft()
+        self.run_threads(tester)
+        self.assertEqual(d, deque())
+
+    def test_count_reverse(self):
+        d = deque([0,1,2,3,4,5,6,7,8,9,10,0])
+        def tester():
+            ct = threading.currentThread()
+            for i in range(1000):
+                self.assertEqual(d[0], 0)
+                if random.random() > 0.5:
+                    time.sleep(0.0001)
+                d.reverse()
+                self.assertEqual(d.count(0), 2)
+                self.assert_(d[1] in (1,10))
+        self.run_threads(tester)
+
+
+def test_main():
+    test_support.run_unittest(ThreadSafetyTestCase)
+
+if __name__ == "__main__":
+    test_main()
+
diff --git a/src/org/python/modules/_collections/PyDeque.java b/src/org/python/modules/_collections/PyDeque.java
--- a/src/org/python/modules/_collections/PyDeque.java
+++ b/src/org/python/modules/_collections/PyDeque.java
@@ -1,15 +1,18 @@
 package org.python.modules._collections;
 
+import org.python.core.ArgParser;
 import org.python.core.PyIterator;
+import org.python.core.PyList;
 import org.python.core.PyObject;
 import org.python.core.PyTuple;
 import org.python.core.PyType;
 import org.python.core.Py;
 import org.python.core.PyException;
-import org.python.core.PyBuiltinCallable;
 import org.python.core.ThreadState;
+import org.python.expose.ExposedGet;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedNew;
+import org.python.expose.ExposedSet;
 import org.python.expose.ExposedType;
 import org.python.expose.MethodType;
 
@@ -23,18 +26,24 @@
  * operations and incur O(n) memory movement costs for pop(0) and insert(0, v) operations which
  * change both the size and position of the underlying data representation.
  * 
- * collections.deque([iterable]) - returns a new deque object initialized left-to-right (using
- * append()) with data from iterable. If iterable is not specified, the new deque is empty.
+ * collections.deque([iterable[, maxlen]]) - returns a new deque object initialized left-to-right
+ * (using append()) with data from iterable. If iterable is not specified, the new deque is empty.
+ * If maxlen is not specified or is None, deques may grow to an arbitrary length. Otherwise, the
+ * deque is bounded to the specified maximum length. Once a bounded length deque is full, when new
+ * items are added, a corresponding number of items are discarded from the opposite end.
  */
 @ExposedType(name = "collections.deque")
 public class PyDeque extends PyObject {
 
     public static final PyType TYPE = PyType.fromClass(PyDeque.class);
 
+    private long state = 0;
     private int size = 0;
 
-    private Node header = new Node(null, null, null); 
-    
+    private int maxlen = -1;
+
+    private Node header = new Node(null, null, null);
+
     public PyDeque() {
         this(TYPE);
     }
@@ -46,41 +55,95 @@
 
     @ExposedNew
     @ExposedMethod
-    final void deque___init__(PyObject[] args, String[] kwds) {
-        if (kwds.length > 0) {
-            throw Py.TypeError("deque() does not take keyword arguments");
+    public synchronized final void deque___init__(PyObject[] args, String[] kwds) {
+        ArgParser ap = new ArgParser("deque", args, kwds, new String[] {"iterable", "maxlen",}, 0);
+
+        PyObject maxlenobj = ap.getPyObject(1, null);
+        if (maxlenobj != null) {
+            if (maxlenobj == Py.None) {
+                maxlen = -1;
+            } else {
+                maxlen = ap.getInt(1);
+                if (maxlen < 0) {
+                    throw Py.ValueError("maxlen must be non-negative");
+                }
+            }
+        } else {
+            maxlen = -1;
         }
-        int nargs = args.length;
-        if (nargs > 1) {
-            throw PyBuiltinCallable.DefaultInfo.unexpectedCall(nargs, false, "deque", 0, 1);
-        } 
-        if (nargs == 0) {
-            return;
+
+        PyObject iterable = ap.getPyObject(0, null);
+        if (iterable != null) {
+            if (size != 0) {
+                // initializing a deque with an iterator when this deque is not empty means that we discard to empty first
+                deque_clear();
+            }
+            deque_extend(iterable);
         }
-        deque_extend(args[0]);
+    }
+
+    /**
+     * If maxlen is not specified or is None, deques may grow to an arbitrary length.
+     * Otherwise, the deque is bounded to the specified maximum length.
+     */
+    @ExposedGet(name = "maxlen")
+    public PyObject getMaxlen() {
+        if (maxlen < 0) {
+            return Py.None;
+        }
+        return Py.newInteger(maxlen);
+    }
+
+    @ExposedSet(name = "maxlen")
+    public void setMaxlen(PyObject o) {
+        // this has to be here because by default, if not defined,
+        // the Jython object model raise a TypeError, where we usually expect
+        // AttributeError here; due to a CPython 2.7 idiosyncracy that has
+        // since been fixed for 3.x in http://bugs.python.org/issue1687163
+        throw Py.AttributeError("attribute 'maxlen' of 'collections.deque' objects is not writable");
     }
 
     /**
      * Add obj to the right side of the deque.
      */	
     @ExposedMethod
-    final void deque_append(PyObject obj) {
-        addBefore(obj, header);		
+    public synchronized final void deque_append(PyObject obj) {
+        if (maxlen >= 0) {
+            assert (size <= maxlen);
+            if (maxlen == 0) {
+                // do nothing; this deque will always be empty
+                return;
+            } else if (size == maxlen) {
+                deque_popleft();
+            }
+        }
+        addBefore(obj, header);
     }
 
     /**
      * Add obj to the left side of the deque.
      */
     @ExposedMethod
-    final void deque_appendleft(PyObject obj) {		
+    public synchronized final void deque_appendleft(PyObject obj) {
+        if (maxlen >= 0) {
+            assert (size <= maxlen);
+            if (maxlen == 0) {
+                // do nothing; this deque will always be empty
+                return;
+            } else if (size == maxlen) {
+                deque_pop();
+            }
+        }
         addBefore(obj, header.right);
     }
 
     private Node addBefore(PyObject obj, Node node) {
+        // should ALWAYS be called inside a synchronized block
         Node newNode = new Node(obj, node, node.left);
         newNode.left.right = newNode;
         newNode.right.left = newNode;
         size++;
+        state++;
         return newNode;
     }
 
@@ -88,7 +151,7 @@
      * Remove all elements from the deque leaving it with length 0.
      */
     @ExposedMethod
-    final void deque_clear() {
+    public synchronized final void deque_clear() {
         Node node = header.right;
         while (node != header) {
             Node right = node.right;
@@ -96,6 +159,7 @@
             node.right = null;
             node.data = null;
             node = right;
+            state++;
         }
         header.right = header.left = header;
         size = 0;
@@ -106,10 +170,15 @@
      * iterable argument.
      */
     @ExposedMethod
-    final void deque_extend(PyObject iterable) {
-        for (PyObject item : iterable.asIterable()) {
-            deque_append(item);			
-        } 
+    public synchronized final void deque_extend(PyObject iterable) {
+        // handle case where iterable == this
+        if (this == iterable) {
+            deque_extend(new PyList(iterable));
+        } else {
+            for (PyObject item : iterable.asIterable()) {
+                deque_append(item);
+            }
+        }
     }
 
     /**
@@ -118,9 +187,14 @@
      * elements in the iterable argument.
      */
     @ExposedMethod
-    final void deque_extendleft(PyObject iterable) {
-        for (PyObject item : iterable.asIterable()) {
-            deque_appendleft(item);
+    public synchronized final void deque_extendleft(PyObject iterable) {
+        // handle case where iterable == this
+        if (this == iterable) {
+            deque_extendleft(new PyList(iterable));
+        } else {
+            for (PyObject item : iterable.asIterable()) {
+                deque_appendleft(item);
+            }
         }
     }
 
@@ -129,7 +203,7 @@
      * elements are present, raises an IndexError.
      */
     @ExposedMethod
-    final PyObject deque_pop() {
+    public synchronized final PyObject deque_pop() {
         return removeNode(header.left);
     }
 
@@ -138,11 +212,12 @@
      * elements are present, raises an IndexError.
      */
     @ExposedMethod
-    final PyObject deque_popleft() {
+    public synchronized final PyObject deque_popleft() {
         return removeNode(header.right);
     }
 
     private PyObject removeNode(Node node) {
+        // should ALWAYS be called inside a synchronized block
         if (node == header) {
             throw Py.IndexError("pop from an empty deque");
         }
@@ -153,6 +228,7 @@
         node.left = null;
         node.data = null;
         size--;
+        state++;
         return obj;
     } 
 
@@ -161,15 +237,16 @@
      * ValueError.
      */
     @ExposedMethod
-    final PyObject deque_remove(PyObject value) {
+    public synchronized final PyObject deque_remove(PyObject value) {
         int n = size;
         Node tmp = header.right;
         boolean match = false;
+        long startState = state;
         for (int i = 0; i < n; i++) {
-            if (tmp.data.equals(value)) { 
+            if (tmp.data.equals(value)) {
                 match = true;
             }
-            if (n != size) { 
+            if (startState != state) {
                 throw Py.IndexError("deque mutated during remove().");
             }
             if (match) {
@@ -181,75 +258,131 @@
     }
 
     /**
+     * Count the number of deque elements equal to x.
+     */
+    @ExposedMethod
+    public synchronized final PyObject deque_count(PyObject x) {
+        int n = size;
+        int count = 0;
+        Node tmp = header.right;
+        long startState = state;
+        for (int i = 0; i < n; i++) {
+            if (tmp.data.equals(x)) {
+                count++;
+            }
+            if (startState != state) {
+                throw Py.RuntimeError("deque mutated during count().");
+            }
+            tmp = tmp.right;
+        }
+        return Py.newInteger(count);
+    }
+
+    /**
      * Rotate the deque n steps to the right. If n is negative, rotate to the 
      * left. Rotating one step to the right is equivalent to: d.appendleft(d.pop()).
      */
     @ExposedMethod(defaults = {"1"})
-    final void deque_rotate(int steps) {
+    public synchronized final void deque_rotate(int steps) {
         if (size == 0) {
             return;
         }
 
-        int halfsize = (size + 1) >> 1; 
+        int halfsize = (size + 1) >> 1;
         if (steps > halfsize || steps < -halfsize) {
             steps %= size;
-            if (steps > halfsize) { 
+            if (steps > halfsize) {
                 steps -= size;
             } else if (steps < -halfsize) {
                 steps += size;
             }
         }
 
-        //rotate right 
+        //rotate right
         for (int i = 0; i < steps; i++) {
             deque_appendleft(deque_pop());
-        } 
+        }
         //rotate left
         for (int i = 0; i > steps; i--) {
             deque_append(deque_popleft());
-        } 
+        }
     }
 
+    /**
+     * Reverse the elements of the deque in-place and then return None.
+     * @return Py.None
+     */
+    @ExposedMethod
+    public synchronized final PyObject deque_reverse() {
+        Node headerRight = header.right;
+        Node headerLeft = header.left;
+        Node node = header.right;
+        while (node != header) {
+            Node right = node.right;
+            Node left = node.left;
+            node.right = left;
+            node.left = right;
+            node = right;
+        }
+        header.right = headerLeft;
+        header.left = headerRight;
+        state++;
+        return Py.None;
+    }
+
+    @Override
     public String toString() {
         return deque_toString();
     }
 
     @ExposedMethod(names = "__repr__")
-    final String deque_toString() {
+    synchronized final String deque_toString() {
         ThreadState ts = Py.getThreadState();
         if (!ts.enterRepr(this)) { 
             return "[...]";
         }
+        long startState = state;
         StringBuilder buf = new StringBuilder("deque").append("([");
         for (Node tmp = header.right; tmp != header; tmp = tmp.right) {
             buf.append(tmp.data.__repr__().toString());
+            if (startState != state) {
+                throw Py.RuntimeError("deque mutated during iteration.");
+            }
             if (tmp.right != header) {
                 buf.append(", ");
             }
         }
-        buf.append("])");
+        buf.append("]");
+        if (maxlen >= 0) {
+            buf.append(", maxlen=");
+            buf.append(maxlen);
+        }
+        buf.append(")");
         ts.exitRepr(this);
         return buf.toString();
     }
 
+    @Override
     public int __len__() {
         return deque___len__();
     }
 
     @ExposedMethod
-    final int deque___len__() {
+    synchronized final int deque___len__() {
         return size;
     }
 
+    @Override
     public boolean __nonzero__() {
         return deque___nonzero__();
     }
 
     @ExposedMethod
-    final boolean deque___nonzero__() {
+    synchronized final boolean deque___nonzero__() {
         return size != 0;
     }
 
+    @Override
     public PyObject __finditem__(PyObject key) {
         try {
             return deque___getitem__(key);
@@ -262,31 +395,34 @@
     }
 
     @ExposedMethod
-    final PyObject deque___getitem__(PyObject index) {
-        return getNode(index).data;				
-    }	
+    synchronized final PyObject deque___getitem__(PyObject index) {
+        return getNode(index).data;
+    }
 
+    @Override
     public void __setitem__(PyObject index, PyObject value) {
         deque___setitem__(index, value);
     }
 
     @ExposedMethod
-    final void deque___setitem__(PyObject index, PyObject value) {
+    synchronized final void deque___setitem__(PyObject index, PyObject value) {
         Node node = getNode(index).right;
         removeNode(node.left);
         addBefore(value, node);
-    }	
+    }
 
+    @Override
     public void __delitem__(PyObject key) {
         deque___delitem__(key);
     }
 
     @ExposedMethod
-    final void deque___delitem__(PyObject key) {
-        removeNode(getNode(key));
+    synchronized final void deque___delitem__(PyObject key) {
+            removeNode(getNode(key));
     }
 
     private Node getNode(PyObject index) {
+        // must ALWAYS be called inside a synchronized block
         int pos = 0;
         if (!index.isIndex()) {
             throw Py.TypeError(String.format("sequence index must be integer, not '%.200s'",
@@ -314,6 +450,7 @@
         return tmp;
     }
 
+    @Override
     public PyObject __iter__() {
         return deque___iter__();
     }
@@ -323,6 +460,7 @@
         return new PyDequeIter();
     }
 
+    @Override
     public synchronized PyObject __eq__(PyObject o) {
         return deque___eq__(o);
     }
@@ -341,6 +479,7 @@
         return (i < 0) ? Py.True : Py.False;
     }
 
+    @Override
     public synchronized PyObject __ne__(PyObject o) {
         return deque___ne__(o);
     }
@@ -359,6 +498,7 @@
         return (i < 0) ? Py.False : Py.True;
     }
 
+    @Override
     public synchronized PyObject __lt__(PyObject o) {
         return deque___lt__(o);
     }
@@ -375,6 +515,7 @@
         return __finditem__(i)._lt(o.__finditem__(i));
     }
 
+    @Override
     public synchronized PyObject __le__(PyObject o) {
         return deque___le__(o);
     }
@@ -391,6 +532,7 @@
         return __finditem__(i)._le(o.__finditem__(i));
     }
 
+    @Override
     public synchronized PyObject __gt__(PyObject o) {
         return deque___gt__(o);
     }
@@ -407,6 +549,7 @@
         return __finditem__(i)._gt(o.__finditem__(i));
     }
 
+    @Override
     public synchronized PyObject __ge__(PyObject o) {
         return deque___ge__(o);
     }
@@ -423,6 +566,17 @@
         return __finditem__(i)._ge(o.__finditem__(i));
     }
 
+    @Override
+    public synchronized PyObject __iadd__(PyObject o) {
+        return deque___iadd__(o);
+    }
+
+    @ExposedMethod(type = MethodType.BINARY)
+    final synchronized PyObject deque___iadd__(PyObject o) {
+        deque_extend(o);
+        return this;
+    }
+
     // Return value >= 0 is the index where the sequences differs.
     // -1: reached the end of o1 without a difference
     // -2: reached the end of both seqeunces without a difference
@@ -445,6 +599,7 @@
         return (ol1 < ol2) ? -1 : -3;
     }
 
+    @Override
     public int hashCode() {
         return deque_hashCode();
     }
@@ -454,6 +609,7 @@
         throw Py.TypeError("deque objects are unhashable");
     }
 
+    @Override
     public PyObject __reduce__() {
         return deque___reduce__();
     }
@@ -499,21 +655,24 @@
     private class PyDequeIter extends PyIterator {
 
         private Node lastReturned = header;
-        private int itersize;
+        private long startState;
 
         public PyDequeIter() {
-            itersize = size;
+            startState = state;
         }
 
-        public PyObject __iternext__() {        	   		
-            if (itersize != size) { 
-                throw Py.RuntimeError("deque changed size during iteration");
+        @Override
+        public PyObject __iternext__() {
+            synchronized (PyDeque.this) {
+                if (startState != state) {
+                    throw Py.RuntimeError("deque changed size during iteration");
+                }
+                if (lastReturned.right != header) {
+                    lastReturned = lastReturned.right;
+                    return lastReturned.data;
+                }
+                return null;
             }
-            if (lastReturned.right != header) {
-                lastReturned = lastReturned.right;
-                return lastReturned.data;
-            }
-            return null;
         }
     }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 14 22:58:20 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 14 Jun 2014 22:58:20 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixes_test=5Fhttplib_Source?=
 =?utf-8?q?AddressTest_regression_test?=
Message-ID: <3grWNr5Ntzz7LjN@mail.python.org>

http://hg.python.org/jython/rev/2bddcf5b03b2
changeset:   7295:2bddcf5b03b2
parent:      7167:b31e71644fa8
user:        Indra Talip 
date:        Mon Dec 09 20:19:16 2013 +1100
summary:
  Fixes test_httplib SourceAddressTest regression test
Changes test_support.bind_port() to create and listen on a socket bound to
port 0 to get the OS to assign an emphemeral port, the socket is then closed
and deleted. Necessary since due to late binding of the socket implematation
socket.bind always succeeds and never actually checks if the port is in use.

files:
  Lib/test/test_support.py |  33 ++++++++++-----------------
  1 files changed, 12 insertions(+), 21 deletions(-)


diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
--- a/Lib/test/test_support.py
+++ b/Lib/test/test_support.py
@@ -374,27 +374,18 @@
     from bind()'ing to our host/port for the duration of the test.
     """
     if is_jython:
-        # Find some random ports that hopefully no one is listening on.
-        # Ideally each test would clean up after itself and not continue
-        # listening on any ports.  However, this isn't the case.  The last port
-        # (0) is a stop-gap that asks the O/S to assign a port.  Whenever the
-        # warning message below is printed, the test that is listening on the
-        # port should be fixed to close the socket at the end of the test.
-        # Another reason why we can't use a port is another process (possibly
-        # another instance of the test suite) is using the same port.
-
-        for port in [54321, 9907, 10243, 32999, 0]:
-            try:
-                sock.bind((host, port))
-                if port == 0:
-                    port = sock.getsockname()[1]
-                return port
-            except socket.error, (err, msg):
-                if err != errno.EADDRINUSE:
-                    raise
-                print >>sys.__stderr__, \
-                    '  WARNING: failed to listen on port %d, trying another' % port
-        raise TestFailed, 'unable to find port to listen on'
+        # Late binding of the jython socket implementation to a
+        # ServerSocketChannel or SocketChannel means that it's not possible to
+        # get the port until a call to connect() or listen(). Hence why a new
+        # socket is created and listen() is called on it.
+        tempsock = socket.socket(sock.family, sock.type)
+        tempsock.bind((host, 0))
+        tempsock.listen(1)
+        port = tempsock.getsockname()[1]
+        tempsock.close()
+        del tempsock
+        sock.bind((host, port))
+        return port
 
     elif sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM:
         if hasattr(socket, 'SO_REUSEADDR'):

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 14 22:58:22 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 14 Jun 2014 22:58:22 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython_=28merge_default_-=3E_default?=
	=?utf-8?b?KTogTWVyZ2Vk?=
Message-ID: <3grWNt0581z7LjN@mail.python.org>

http://hg.python.org/jython/rev/67a4b9a0361c
changeset:   7296:67a4b9a0361c
parent:      7294:759e56cfcac7
parent:      7295:2bddcf5b03b2
user:        Jim Baker 
date:        Sat Jun 14 14:58:00 2014 -0600
summary:
  Merged https://bitbucket.org/jython/jython/pull-request/18/fixes-test_httplib-sourceaddresstest

files:
  Lib/test/test_support.py |  33 ++++++++++-----------------
  1 files changed, 12 insertions(+), 21 deletions(-)


diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
--- a/Lib/test/test_support.py
+++ b/Lib/test/test_support.py
@@ -374,27 +374,18 @@
     from bind()'ing to our host/port for the duration of the test.
     """
     if is_jython:
-        # Find some random ports that hopefully no one is listening on.
-        # Ideally each test would clean up after itself and not continue
-        # listening on any ports.  However, this isn't the case.  The last port
-        # (0) is a stop-gap that asks the O/S to assign a port.  Whenever the
-        # warning message below is printed, the test that is listening on the
-        # port should be fixed to close the socket at the end of the test.
-        # Another reason why we can't use a port is another process (possibly
-        # another instance of the test suite) is using the same port.
-
-        for port in [54321, 9907, 10243, 32999, 0]:
-            try:
-                sock.bind((host, port))
-                if port == 0:
-                    port = sock.getsockname()[1]
-                return port
-            except socket.error, (err, msg):
-                if err != errno.EADDRINUSE:
-                    raise
-                print >>sys.__stderr__, \
-                    '  WARNING: failed to listen on port %d, trying another' % port
-        raise TestFailed, 'unable to find port to listen on'
+        # Late binding of the jython socket implementation to a
+        # ServerSocketChannel or SocketChannel means that it's not possible to
+        # get the port until a call to connect() or listen(). Hence why a new
+        # socket is created and listen() is called on it.
+        tempsock = socket.socket(sock.family, sock.type)
+        tempsock.bind((host, 0))
+        tempsock.listen(1)
+        port = tempsock.getsockname()[1]
+        tempsock.close()
+        del tempsock
+        sock.bind((host, port))
+        return port
 
     elif sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM:
         if hasattr(socket, 'SO_REUSEADDR'):

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 15 02:52:21 2014
From: jython-checkins at python.org (jim.baker)
Date: Sun, 15 Jun 2014 02:52:21 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_=2Eidea/*_files_in_f?=
 =?utf-8?q?avor_of_Eclipse=2E_Update_=2Ehgignore_to_ignore?=
Message-ID: <3grcZs49Pcz7Ljd@mail.python.org>

http://hg.python.org/jython/rev/773838095016
changeset:   7297:773838095016
user:        Santoso Wijaya 
date:        Sat Jun 14 18:49:53 2014 -0600
summary:
  Remove .idea/* files in favor of Eclipse. Update .hgignore to ignore
.eml file for Eclipe/IntelliJ integration.

Applied from https://bitbucket.org/jython/jython/pull-request/36/remove-intellij-idea-files-from-repo/

files:
  .hgignore                             |    1 +
  .idea/.name                           |    1 -
  .idea/codeStyleSettings.xml           |   25 --
  .idea/compiler.xml                    |   25 --
  .idea/copyright/profiles_settings.xml |    3 -
  .idea/encodings.xml                   |    5 -
  .idea/libraries/extlibs.xml           |   10 -
  .idea/libraries/extlibs2.xml          |   47 -----
  .idea/libraries/jar.xml               |   12 -
  .idea/libraries/svnant_jars.xml       |   11 -
  .idea/libraries/test.xml              |   14 -
  .idea/modules.xml                     |    9 -
  .idea/scopes/scope_settings.xml       |    5 -
  .idea/uiDesigner.xml                  |  125 --------------
  .idea/vcs.xml                         |    7 -
  15 files changed, 1 insertions(+), 299 deletions(-)


diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -9,6 +9,7 @@
 \#*
 *~
 # IntelliJ files
+*.eml
 *.ipr
 *.iml
 *.iws
diff --git a/.idea/.name b/.idea/.name
deleted file mode 100644
--- a/.idea/.name
+++ /dev/null
@@ -1,1 +0,0 @@
-jython27
\ No newline at end of file
diff --git a/.idea/codeStyleSettings.xml b/.idea/codeStyleSettings.xml
deleted file mode 100644
--- a/.idea/codeStyleSettings.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-
-
-  
-    
-    
-
-
diff --git a/.idea/compiler.xml b/.idea/compiler.xml
deleted file mode 100644
--- a/.idea/compiler.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-
-
-  
-    
-
-
diff --git a/.idea/copyright/profiles_settings.xml b/.idea/copyright/profiles_settings.xml
deleted file mode 100644
--- a/.idea/copyright/profiles_settings.xml
+++ /dev/null
@@ -1,3 +0,0 @@
-
-  
-
\ No newline at end of file
diff --git a/.idea/encodings.xml b/.idea/encodings.xml
deleted file mode 100644
--- a/.idea/encodings.xml
+++ /dev/null
@@ -1,5 +0,0 @@
-
-
-  
-
-
diff --git a/.idea/libraries/extlibs.xml b/.idea/libraries/extlibs.xml
deleted file mode 100644
--- a/.idea/libraries/extlibs.xml
+++ /dev/null
@@ -1,10 +0,0 @@
-
-  
-    
-      
-    
-    
-    
-    
-  
-
\ No newline at end of file
diff --git a/.idea/libraries/extlibs2.xml b/.idea/libraries/extlibs2.xml
deleted file mode 100644
--- a/.idea/libraries/extlibs2.xml
+++ /dev/null
@@ -1,47 +0,0 @@
-
-  
-    
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-    
-    
-    
-  
-
\ No newline at end of file
diff --git a/.idea/libraries/jar.xml b/.idea/libraries/jar.xml
deleted file mode 100644
--- a/.idea/libraries/jar.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-
-  
-    
-      
-      
-      
-      
-    
-    
-    
-  
-
\ No newline at end of file
diff --git a/.idea/libraries/svnant_jars.xml b/.idea/libraries/svnant_jars.xml
deleted file mode 100644
--- a/.idea/libraries/svnant_jars.xml
+++ /dev/null
@@ -1,11 +0,0 @@
-
-  
-    
-      
-      
-      
-    
-    
-    
-  
-
\ No newline at end of file
diff --git a/.idea/libraries/test.xml b/.idea/libraries/test.xml
deleted file mode 100644
--- a/.idea/libraries/test.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-
-  
-    
-      
-      
-      
-      
-      
-      
-    
-    
-    
-  
-
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
deleted file mode 100644
--- a/.idea/modules.xml
+++ /dev/null
@@ -1,9 +0,0 @@
-
-
-  
-    
-      
-    
-  
-
-
diff --git a/.idea/scopes/scope_settings.xml b/.idea/scopes/scope_settings.xml
deleted file mode 100644
--- a/.idea/scopes/scope_settings.xml
+++ /dev/null
@@ -1,5 +0,0 @@
-
-  
-    
-
\ No newline at end of file
diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml
deleted file mode 100644
--- a/.idea/uiDesigner.xml
+++ /dev/null
@@ -1,125 +0,0 @@
-
-
-  
-    
-      
-        
-      
-      
-        
-      
-      
-        
-      
-      
-        
-      
-      
-        
-        
-          
-        
-      
-      
-        
-        
-          
-        
-      
-      
-        
-        
-          
-        
-      
-      
-        
-        
-          
-        
-      
-      
-        
-          
-        
-      
-      
-        
-          
-        
-      
-      
-        
-          
-        
-      
-      
-        
-          
-        
-      
-      
-        
-          
-        
-      
-      
-        
-          
-        
-      
-      
-        
-      
-      
-        
-          
-        
-      
-      
-        
-          
-        
-      
-      
-        
-          
-        
-      
-      
-        
-          
-        
-      
-      
-        
-          
-        
-      
-      
-        
-      
-      
-        
-      
-      
-        
-      
-      
-        
-      
-      
-        
-          
-        
-      
-      
-        
-      
-      
-        
-      
-    
-  
-
-
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
deleted file mode 100644
--- a/.idea/vcs.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-  
-    
-  
-
-

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 15 02:58:51 2014
From: jython-checkins at python.org (jim.baker)
Date: Sun, 15 Jun 2014 02:58:51 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Applied_remaining_pattern_f?=
	=?utf-8?q?or_hg_to_ignore_=2Eidea/*?=
Message-ID: <3grckM3T98z7LjV@mail.python.org>

http://hg.python.org/jython/rev/20a007894dcf
changeset:   7298:20a007894dcf
user:        Jim Baker 
date:        Sat Jun 14 18:59:16 2014 -0600
summary:
  Applied remaining pattern for hg to ignore .idea/*

files:
  .hgignore |  3 +--
  1 files changed, 1 insertions(+), 2 deletions(-)


diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -13,8 +13,7 @@
 *.ipr
 *.iml
 *.iws
-.idea/misc.xml
-.idea/workspace.xml
+.idea/*
 
 .AppleDouble
 .DS_Store

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 15 18:31:03 2014
From: jython-checkins at python.org (jeff.allen)
Date: Sun, 15 Jun 2014 18:31:03 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_Eclipse_IDE_configur?=
 =?utf-8?q?ation_from_source_control=2E?=
Message-ID: <3gs1Pz4m06z7LjQ@mail.python.org>

http://hg.python.org/jython/rev/ea302f9e4c3a
changeset:   7299:ea302f9e4c3a
user:        Jeff Allen 
date:        Sun Jun 15 17:26:05 2014 +0100
summary:
  Remove Eclipse IDE configuration from source control.
It's better these be private as they change a lot.

files:
  .classpath                              |  47 -------------
  .externalToolBuilders/ant_expose.launch |  19 -----
  .hgignore                               |   9 +-
  .project                                |  36 ---------
  .pydevproject                           |   7 -
  5 files changed, 6 insertions(+), 112 deletions(-)


diff --git a/.classpath b/.classpath
deleted file mode 100644
--- a/.classpath
+++ /dev/null
@@ -1,47 +0,0 @@
-
-
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-	
-
diff --git a/.externalToolBuilders/ant_expose.launch b/.externalToolBuilders/ant_expose.launch
deleted file mode 100644
--- a/.externalToolBuilders/ant_expose.launch
+++ /dev/null
@@ -1,19 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -15,11 +15,14 @@
 *.iws
 .idea/*
 
+# Eclipse files
+.classpath
+.externalToolBuilders/*
+.project
+.pydevproject
+
 .AppleDouble
 .DS_Store
-.classpath
-.externalToolBuilders
-.project
 .settings
 __pycache__
 ant.properties
diff --git a/.project b/.project
deleted file mode 100644
--- a/.project
+++ /dev/null
@@ -1,36 +0,0 @@
-
-
-	jython-trunk
-	
-	
-	
-	
-		
-			org.python.pydev.PyDevBuilder
-			
-			
-		
-		
-			org.eclipse.jdt.core.javabuilder
-			
-			
-		
-		
-			org.eclipse.ui.externaltools.ExternalToolBuilder
-			
-				
-					LaunchConfigHandle
-					<project>/.externalToolBuilders/ant_expose.launch
-				
-				
-					incclean
-					true
-				
-			
-		
-	
-	
-		org.eclipse.jdt.core.javanature
-		org.python.pydev.pythonNature
-	
-
diff --git a/.pydevproject b/.pydevproject
deleted file mode 100644
--- a/.pydevproject
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-
-
-Default
-python 2.7
-

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Tue Jun 17 00:17:17 2014
From: jython-checkins at python.org (jeff.allen)
Date: Tue, 17 Jun 2014 00:17:17 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_os=2Ewrite_accepts_objects_?=
 =?utf-8?q?having_the_buffer_API=2C_fixes_=232062=2E?=
Message-ID: <3gsn316x4cz7Ljm@mail.python.org>

http://hg.python.org/jython/rev/f55873fad858
changeset:   7300:f55873fad858
user:        Jeff Allen 
date:        Mon Jun 16 00:06:02 2014 +0100
summary:
  os.write accepts objects having the buffer API, fixes #2062.

files:
  src/org/python/core/buffer/BaseBuffer.java         |   8 +-
  src/org/python/core/buffer/SimpleStringBuffer.java |   5 +-
  src/org/python/modules/posix/PosixModule.java      |  34 ++++++---
  3 files changed, 29 insertions(+), 18 deletions(-)


diff --git a/src/org/python/core/buffer/BaseBuffer.java b/src/org/python/core/buffer/BaseBuffer.java
--- a/src/org/python/core/buffer/BaseBuffer.java
+++ b/src/org/python/core/buffer/BaseBuffer.java
@@ -662,10 +662,6 @@
 
         if ((syndrome & ND) != 0) {
             return bufferRequires("shape array");
-        } else if ((syndrome & STRIDES) != 0) {
-            return bufferRequires("strides array");
-        } else if ((syndrome & INDIRECT) != 0) {
-            return bufferRequires("suboffsets array");
         } else if ((syndrome & WRITABLE) != 0) {
             return bufferIsNot("writable");
         } else if ((syndrome & C_CONTIGUOUS) != 0) {
@@ -674,6 +670,10 @@
             return bufferIsNot("Fortran-contiguous");
         } else if ((syndrome & ANY_CONTIGUOUS) != 0) {
             return bufferIsNot("contiguous");
+        } else if ((syndrome & STRIDES) != 0) {
+            return bufferRequires("strides array");
+        } else if ((syndrome & INDIRECT) != 0) {
+            return bufferRequires("suboffsets array");
         } else {
             // Catch-all error (never in practice if this method is complete)
             return bufferIsNot("capable of matching request");
diff --git a/src/org/python/core/buffer/SimpleStringBuffer.java b/src/org/python/core/buffer/SimpleStringBuffer.java
--- a/src/org/python/core/buffer/SimpleStringBuffer.java
+++ b/src/org/python/core/buffer/SimpleStringBuffer.java
@@ -108,11 +108,12 @@
     @Override
     public PyBuffer getBufferSlice(int flags, int start, int length, int stride) {
         if (stride == 1) {
-            // Unstrided slice of simple buffer is itself simple
+            // Unstrided slice of a SimpleStringBuffer is itself a SimpleStringBuffer.
             return getBufferSlice(flags, start, length);
         } else {
-            // Force creation of the actual byte buffer be a SimpleBuffer
+            // Force creation of the actual byte array from the String.
             getBuf();
+            // Now we are effectively a SimpleBuffer, return the strided view.
             return super.getBufferSlice(flags, start, length, stride);
         }
     }
diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java
--- a/src/org/python/modules/posix/PosixModule.java
+++ b/src/org/python/modules/posix/PosixModule.java
@@ -1,9 +1,6 @@
 /* Copyright (c) Jython Developers */
 package org.python.modules.posix;
 
-import jnr.constants.Constant;
-import jnr.constants.platform.Errno;
-
 import java.io.File;
 import java.io.FileDescriptor;
 import java.io.FileNotFoundException;
@@ -20,13 +17,19 @@
 import java.util.Iterator;
 import java.util.Map;
 
+import jnr.constants.Constant;
+import jnr.constants.platform.Errno;
 import jnr.posix.FileStat;
 import jnr.posix.POSIX;
 import jnr.posix.POSIXFactory;
 import jnr.posix.util.Platform;
 
+import org.python.core.BufferProtocol;
 import org.python.core.ClassDictInit;
 import org.python.core.Py;
+import org.python.core.PyBUF;
+import org.python.core.PyBuffer;
+import org.python.core.PyBuffer.Pointer;
 import org.python.core.PyBuiltinFunctionNarrow;
 import org.python.core.PyDictionary;
 import org.python.core.PyException;
@@ -39,9 +42,9 @@
 import org.python.core.PySystemState;
 import org.python.core.PyTuple;
 import org.python.core.imp;
-import org.python.core.io.IOBase;
 import org.python.core.io.FileDescriptors;
 import org.python.core.io.FileIO;
+import org.python.core.io.IOBase;
 import org.python.core.io.RawIOBase;
 import org.python.core.util.StringUtil;
 
@@ -818,14 +821,21 @@
         return new PyTuple(Py.newInteger(pid), Py.newInteger(status[0]));
     }
 
-    public static PyString __doc__write = new PyString(
-        "write(fd, string) -> byteswritten\n\n" +
-        "Write a string to a file descriptor.");
-    public static int write(PyObject fd, String string) {
-        try {
-            return FileDescriptors.get(fd).write(ByteBuffer.wrap(StringUtil.toBytes(string)));
-        } catch (PyException pye) {
-            throw badFD();
+    public static PyString __doc__write = new PyString("write(fd, string) -> byteswritten\n\n"
+            + "Write a string to a file descriptor.");
+    public static int write(PyObject fd, BufferProtocol bytes) {
+        // Get a buffer view: we can cope with N-dimensional data, but not strided data.
+        try (PyBuffer buf = bytes.getBuffer(PyBUF.ND)) {
+            // Get the array and offset of the first real byte.
+            Pointer p = buf.getBuf();
+            // Make a ByteBuffer of that array, setting the position and limit to the real data.
+            ByteBuffer bb = ByteBuffer.wrap(p.storage, p.offset, buf.getLen());
+            try {
+                // Write the data (returning the count of bytes).
+                return FileDescriptors.get(fd).write(bb);
+            } catch (PyException pye) {
+                throw badFD();
+            }
         }
     }
 

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Wed Jun 18 00:42:59 2014
From: jython-checkins at python.org (jim.baker)
Date: Wed, 18 Jun 2014 00:42:59 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Change_itertools_classes_su?=
 =?utf-8?q?ch_that_they_extend_PyIterator=2C_not?=
Message-ID: <3gtPZC5hDSz7LjQ@mail.python.org>

http://hg.python.org/jython/rev/25d9613dbb98
changeset:   7301:25d9613dbb98
user:        Jim Baker 
date:        Tue Jun 17 15:21:10 2014 -0600
summary:
  Change itertools classes such that they extend PyIterator, not
PyObject, to allow for chaining when the itertools object is returned
by an __iter__ method in a user-defined class. This also enables
itertools classes to be directly iterated over by Java code.

Fixes http://bugs.jython.org/issue1991

files:
  Lib/test/test_iter_jy.py                                          |  267 +++++++++-
  src/org/python/modules/itertools/chain.java                       |   12 +-
  src/org/python/modules/itertools/combinations.java                |   10 +-
  src/org/python/modules/itertools/combinationsWithReplacement.java |   10 +-
  src/org/python/modules/itertools/compress.java                    |   12 +-
  src/org/python/modules/itertools/count.java                       |   22 +-
  src/org/python/modules/itertools/cycle.java                       |   10 +-
  src/org/python/modules/itertools/dropwhile.java                   |   10 +-
  src/org/python/modules/itertools/groupby.java                     |   11 +-
  src/org/python/modules/itertools/ifilter.java                     |   14 +-
  src/org/python/modules/itertools/ifilterfalse.java                |   16 +-
  src/org/python/modules/itertools/imap.java                        |   14 +-
  src/org/python/modules/itertools/islice.java                      |   11 +-
  src/org/python/modules/itertools/izip.java                        |   14 +-
  src/org/python/modules/itertools/izipLongest.java                 |   11 +-
  src/org/python/modules/itertools/permutations.java                |   10 +-
  src/org/python/modules/itertools/product.java                     |   11 +-
  src/org/python/modules/itertools/repeat.java                      |   22 +-
  src/org/python/modules/itertools/starmap.java                     |   10 +-
  src/org/python/modules/itertools/takewhile.java                   |   10 +-
  20 files changed, 377 insertions(+), 130 deletions(-)


diff --git a/Lib/test/test_iter_jy.py b/Lib/test/test_iter_jy.py
--- a/Lib/test/test_iter_jy.py
+++ b/Lib/test/test_iter_jy.py
@@ -3,6 +3,7 @@
 Made for Jython.
 """
 import itertools
+import operator
 from test import test_support
 import unittest
 
@@ -28,8 +29,272 @@
         self.assertEqual(list(itertools.chain([], [], ['foo'])), ['foo'])
 
 
+class ChainedIterationTest(unittest.TestCase):
+
+    # Test http://bugs.jython.org/issue1991
+
+    def test_chain(self):
+        
+        class TestChain(object):
+            def __init__(self, data):
+                self.data = data
+            def __iter__(self):
+                return itertools.chain(self.data)
+        
+        data = [1, 2, 3]
+        obj = TestChain(data)
+        self.assertEqual(list(obj), data)
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_compress(self):
+        
+        class TestCount(object):
+            def __init__(self, data, selectors):
+                self.data = data
+                self.selectors = selectors
+            def __iter__(self):
+                return itertools.compress(self.data, self.selectors)
+        
+        obj = TestCount((1, 2, 3, 4, 5), (1, 0, 1, 0, 1))
+        self.assertEqual(list(obj), [1, 3, 5])
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_count(self):
+        
+        class TestCount(object):
+            def __init__(self, data):
+                self.data = data
+            def __iter__(self):
+                return itertools.count(self.data)
+        
+        obj = TestCount(1)
+        self.assertEqual(list(itertools.islice(obj, 5)), [1, 2, 3, 4, 5])
+        self.assertEqual(list(itertools.islice(obj, 25)), list(itertools.islice(obj.__iter__(), 25)))
+
+    def test_cycle(self):
+        
+        class TestCycle(object):
+            def __init__(self, data):
+                self.data = data
+            def __iter__(self):
+                return itertools.cycle(self.data)
+        
+        data = [1, 2, 3]
+        obj = TestCycle(data)
+        self.assertEqual(list(itertools.islice(obj, 6)), data * 2)
+        self.assertEqual(list(itertools.islice(obj, 6)), list(itertools.islice(obj.__iter__(), 6)))
+
+    def test_dropwhile(self):
+        
+        class TestCycle(object):
+            def __init__(self, pred, data):
+                self.pred = pred
+                self.data = data
+            def __iter__(self):
+                return itertools.dropwhile(self.pred, self.data)
+        
+        obj = TestCycle(lambda x: x<5, [1, 4, 6, 4, 1])
+        self.assertEqual(list(obj), [6, 4, 1])
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_groupby(self):
+
+        def unique_justseen(iterable, key=None):
+            "List unique elements, preserving order. Remember only the element just seen."
+            # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B
+            # unique_justseen('ABBCcAD', str.lower) --> A B C A D
+            return itertools.imap(
+                next, 
+                itertools.imap(operator.itemgetter(1),
+                               itertools.groupby(iterable, key)))
+
+        class TestGroupBy(object):
+            def __init__(self, data, key):
+                self.data = data
+                self.key = key
+            def __iter__(self):
+                return unique_justseen(self.data, self.key)
+        
+        obj = TestGroupBy('ABBCcAD', str.lower)
+        self.assertEqual(list(obj), list('ABCAD'))
+        self.assertEqual(list(obj), list(obj.__iter__()))        
+
+    def test_ifilter(self):
+
+        class TestIFilter(object):
+            def __init__(self, pred, data):
+                self.pred = pred
+                self.data = data
+            def __iter__(self):
+                return itertools.ifilter(self.pred, self.data)
+
+        obj = TestIFilter(lambda x: x%2, range(10))
+        self.assertEqual(list(obj), [1, 3, 5, 7, 9])
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_ifilterfalse(self):
+
+        class TestIFilterFalse(object):
+            def __init__(self, pred, data):
+                self.pred = pred
+                self.data = data
+            def __iter__(self):
+                return itertools.ifilterfalse(self.pred, self.data)
+
+        obj = TestIFilterFalse(lambda x: x%2, range(10))
+        self.assertEqual(list(obj), [0, 2, 4, 6, 8])
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_ifilterfalse(self):
+
+        class TestIslice(object):
+            def __init__(self, seq, start, stop):
+                self.seq = seq
+                self.start = start
+                self.stop = stop
+            def __iter__(self):
+                return itertools.islice(self.seq, self.start, self.stop)
+
+        obj = TestIslice('ABCDEFG', 2, None)
+        self.assertEqual(list(obj), list('CDEFG'))
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_imap(self):
+
+        class TestImap(object):
+            def __init__(self, func, p, q):
+                self.func = func
+                self.p = p
+                self.q = q
+            def __iter__(self):
+                return itertools.imap(self.func, self.p, self.q)
+
+        obj = TestImap(pow, (2,3,10), (5,2,3))
+        self.assertEqual(list(obj), [32, 9, 1000])
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_starmap(self):
+
+        class TestStarmap(object):
+            def __init__(self, func, seq):
+                self.func = func
+                self.seq = seq
+            def __iter__(self):
+                return itertools.starmap(self.func, self.seq)
+
+        obj = TestStarmap(pow, [(2,5), (3,2), (10,3)])
+        self.assertEqual(list(obj), [32, 9, 1000])
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_takewhile(self):
+
+        class TestTakewhile(object):
+            def __init__(self, pred, seq):
+                self.pred = pred
+                self.seq = seq
+            def __iter__(self):
+                return itertools.takewhile(self.pred, self.seq)
+
+        obj = TestTakewhile(lambda x: x<5, [1, 4, 6, 4, 1])
+        self.assertEqual(list(obj), [1, 4])
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_tee(self):
+
+        def pairwise(iterable):
+            "s -> (s0,s1), (s1,s2), (s2, s3), ..."
+            a, b = itertools.tee(iterable)
+            next(b, None)
+            return itertools.izip(a, b)
+
+        class TestTee(object):
+            def __init__(self, func, it):
+                self.func = func
+                self.it = it
+            def __iter__(self):
+                return self.func(self.it)
+
+        obj = TestTee(pairwise, range(6))
+        self.assertEqual(list(obj), [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5)])
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_izip(self):
+
+        class TestIzip(object):
+            def __init__(self, p, q):
+                self.p = p
+                self.q = q
+            def __iter__(self):
+                return itertools.izip(self.p, self.q)
+
+        obj = TestIzip('ABCD', 'xy')
+        self.assertEqual(list(obj), [('A', 'x'), ('B', 'y')])
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_izip_longest(self):
+
+        class TestIzipLongest(object):
+            def __init__(self, p, q):
+                self.p = p
+                self.q = q
+            def __iter__(self):
+                return itertools.izip_longest(self.p, self.q, fillvalue='-')
+
+        obj = TestIzipLongest('ABCD', 'xy')
+        self.assertEqual(list(obj), [('A', 'x'), ('B', 'y'), ('C', '-'), ('D', '-')])
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_combinations(self):
+
+        class TestCombinations(object):
+            def __init__(self, data):
+                self.data = data
+            def __iter__(self):
+                return itertools.combinations(self.data, 2)
+
+        obj = TestCombinations('ABCD')
+        self.assertEqual(list(obj), list(itertools.combinations('ABCD', 2)))
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_combinations_with_replacement(self):
+
+        class TestCombinationsWithReplacement(object):
+            def __init__(self, data):
+                self.data = data
+            def __iter__(self):
+                return itertools.combinations_with_replacement(self.data, 2)
+
+        obj = TestCombinationsWithReplacement('ABCD')
+        self.assertEqual(list(obj), list(itertools.combinations_with_replacement('ABCD', 2)))
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_permutations(self):
+
+        class TestPermutations(object):
+            def __init__(self, data):
+                self.data = data
+            def __iter__(self):
+                return itertools.permutations(self.data, 2)
+
+        obj = TestPermutations('ABCD')
+        self.assertEqual(list(obj), list(itertools.permutations('ABCD', 2)))
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+    def test_product(self):
+
+        class TestProduct(object):
+            def __init__(self, data):
+                self.data = data
+            def __iter__(self):
+                return itertools.product(self.data, repeat=2)
+
+        obj = TestProduct('ABCD')
+        self.assertEqual(list(obj), list(itertools.product('ABCD', repeat=2)))
+        self.assertEqual(list(obj), list(obj.__iter__()))
+
+
 def test_main():
-    test_support.run_unittest(IterTestCase)
+    test_support.run_unittest(IterTestCase, ChainedIterationTest)
 
 
 if __name__ == '__main__':
diff --git a/src/org/python/modules/itertools/chain.java b/src/org/python/modules/itertools/chain.java
--- a/src/org/python/modules/itertools/chain.java
+++ b/src/org/python/modules/itertools/chain.java
@@ -16,7 +16,7 @@
 import java.util.ArrayList;
 
 @ExposedType(name = "itertools.chain", base = PyObject.class)
-public class chain extends PyObject {
+public class chain extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(chain.class);
     private itertools.ItertoolsIterator iter;
@@ -84,14 +84,14 @@
         };
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 
-}
+}
\ No newline at end of file
diff --git a/src/org/python/modules/itertools/combinations.java b/src/org/python/modules/itertools/combinations.java
--- a/src/org/python/modules/itertools/combinations.java
+++ b/src/org/python/modules/itertools/combinations.java
@@ -13,7 +13,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "itertools.combinations", base = PyObject.class)
-public class combinations extends PyObject {
+public class combinations extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(combinations.class);
     private PyIterator iter;
@@ -84,13 +84,13 @@
         };
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 }
diff --git a/src/org/python/modules/itertools/combinationsWithReplacement.java b/src/org/python/modules/itertools/combinationsWithReplacement.java
--- a/src/org/python/modules/itertools/combinationsWithReplacement.java
+++ b/src/org/python/modules/itertools/combinationsWithReplacement.java
@@ -14,7 +14,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "itertools.combinations_with_replacement", base = PyObject.class)
-public class combinationsWithReplacement extends PyObject {
+public class combinationsWithReplacement extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(combinationsWithReplacement.class);
     private itertools.ItertoolsIterator iter;
@@ -86,13 +86,13 @@
         };
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 }
diff --git a/src/org/python/modules/itertools/compress.java b/src/org/python/modules/itertools/compress.java
--- a/src/org/python/modules/itertools/compress.java
+++ b/src/org/python/modules/itertools/compress.java
@@ -3,6 +3,7 @@
 
 import org.python.core.ArgParser;
 import org.python.core.Py;
+import org.python.core.PyIterator;
 import org.python.core.PyObject;
 import org.python.core.PyString;
 import org.python.core.PyType;
@@ -12,7 +13,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "itertools.compress", base = PyObject.class)
-public class compress extends PyObject {
+public class compress extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(compress.class);
     private itertools.ItertoolsIterator iter;
@@ -69,13 +70,14 @@
         };
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
+
 }
diff --git a/src/org/python/modules/itertools/count.java b/src/org/python/modules/itertools/count.java
--- a/src/org/python/modules/itertools/count.java
+++ b/src/org/python/modules/itertools/count.java
@@ -18,7 +18,7 @@
 import java.util.ArrayList;
 
 @ExposedType(name = "itertools.count", base = PyObject.class)
-public class count extends PyObject {
+public class count extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(count.class);
     private PyIterator iter;
@@ -91,16 +91,6 @@
     }
 
     @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
-    }
-
-    @ExposedMethod
-    public PyObject next() {
-        return iter.next();
-    }
-
-    @ExposedMethod
     public PyString __repr__() {
         if (stepper == 1) {
             return (PyString)(Py.newString("count(%d)").__mod__(Py.newInteger(counter)));
@@ -111,4 +101,14 @@
         }
     }
 
+    public PyObject __iternext__() {
+        return iter.__iternext__();
+    }
+
+    @ExposedMethod
+    @Override
+    public PyObject next() {
+        return doNext(__iternext__());
+    }
+
 }
diff --git a/src/org/python/modules/itertools/cycle.java b/src/org/python/modules/itertools/cycle.java
--- a/src/org/python/modules/itertools/cycle.java
+++ b/src/org/python/modules/itertools/cycle.java
@@ -14,7 +14,7 @@
 import java.util.List;
 
 @ExposedType(name = "itertools.count", base = PyObject.class)
-public class cycle extends PyObject {
+public class cycle extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(cycle.class);
     private PyIterator iter;
@@ -83,14 +83,14 @@
         };
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 
 }
diff --git a/src/org/python/modules/itertools/dropwhile.java b/src/org/python/modules/itertools/dropwhile.java
--- a/src/org/python/modules/itertools/dropwhile.java
+++ b/src/org/python/modules/itertools/dropwhile.java
@@ -12,7 +12,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "itertools.dropwhile", base = PyObject.class)
-public class dropwhile extends PyObject {
+public class dropwhile extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(dropwhile.class);
     private PyIterator iter;
@@ -54,13 +54,13 @@
         iter = new itertools.WhileIterator(predicate, iterable, true);
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 }
diff --git a/src/org/python/modules/itertools/groupby.java b/src/org/python/modules/itertools/groupby.java
--- a/src/org/python/modules/itertools/groupby.java
+++ b/src/org/python/modules/itertools/groupby.java
@@ -4,7 +4,6 @@
 import org.python.core.ArgParser;
 import org.python.core.Py;
 import org.python.core.PyIterator;
-import org.python.core.PyNone;
 import org.python.core.PyObject;
 import org.python.core.PyString;
 import org.python.core.PyTuple;
@@ -16,7 +15,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "itertools.groupby", base = PyObject.class)
-public class groupby extends PyObject {
+public class groupby extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(groupby.class);
     private PyIterator iter;
@@ -113,13 +112,13 @@
         };
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 }
diff --git a/src/org/python/modules/itertools/ifilter.java b/src/org/python/modules/itertools/ifilter.java
--- a/src/org/python/modules/itertools/ifilter.java
+++ b/src/org/python/modules/itertools/ifilter.java
@@ -2,22 +2,18 @@
 package org.python.modules.itertools;
 
 import org.python.core.ArgParser;
-import org.python.core.Py;
 import org.python.core.PyIterator;
 import org.python.core.PyObject;
 import org.python.core.PyString;
-import org.python.core.PyTuple;
 import org.python.core.PyType;
-import org.python.expose.ExposedClassMethod;
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedNew;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedType;
 
-import java.util.ArrayList;
 
 @ExposedType(name = "itertools.ifilter", base = PyObject.class)
-public class ifilter extends PyObject {
+public class ifilter extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(ifilter.class);
     private PyIterator iter;
@@ -60,13 +56,13 @@
         iter = new itertools.FilterIterator(predicate, iterable, true);
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 }
diff --git a/src/org/python/modules/itertools/ifilterfalse.java b/src/org/python/modules/itertools/ifilterfalse.java
--- a/src/org/python/modules/itertools/ifilterfalse.java
+++ b/src/org/python/modules/itertools/ifilterfalse.java
@@ -1,22 +1,17 @@
 /* Copyright (c) Jython Developers */
 package org.python.modules.itertools;
 import org.python.core.ArgParser;
-import org.python.core.Py;
 import org.python.core.PyIterator;
 import org.python.core.PyObject;
 import org.python.core.PyString;
-import org.python.core.PyTuple;
 import org.python.core.PyType;
-import org.python.expose.ExposedClassMethod;
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedNew;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedType;
 
-import java.util.ArrayList;
-
 @ExposedType(name = "itertools.ifilterfalse", base = PyObject.class)
-public class ifilterfalse extends PyObject {
+public class ifilterfalse extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(ifilterfalse.class);
     private PyIterator iter;
@@ -59,14 +54,13 @@
         iter = new itertools.FilterIterator(predicate, iterable, false);
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
-
 }
diff --git a/src/org/python/modules/itertools/imap.java b/src/org/python/modules/itertools/imap.java
--- a/src/org/python/modules/itertools/imap.java
+++ b/src/org/python/modules/itertools/imap.java
@@ -1,23 +1,19 @@
 /* Copyright (c) Jython Developers */
 package org.python.modules.itertools;
 
-import org.python.core.ArgParser;
 import org.python.core.Py;
 import org.python.core.PyIterator;
 import org.python.core.PyObject;
 import org.python.core.PyString;
 import org.python.core.PyTuple;
 import org.python.core.PyType;
-import org.python.expose.ExposedClassMethod;
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedNew;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedType;
 
-import java.util.ArrayList;
-
 @ExposedType(name = "itertools.imap", base = PyObject.class)
-public class imap extends PyObject {
+public class imap extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(imap.class);
     private PyIterator iter;
@@ -99,13 +95,13 @@
         };
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 }
diff --git a/src/org/python/modules/itertools/islice.java b/src/org/python/modules/itertools/islice.java
--- a/src/org/python/modules/itertools/islice.java
+++ b/src/org/python/modules/itertools/islice.java
@@ -3,6 +3,7 @@
 import org.python.core.ArgParser;
 import org.python.core.Py;
 import org.python.core.PyInteger;
+import org.python.core.PyIterator;
 import org.python.core.PyNone;
 import org.python.core.PyObject;
 import org.python.core.PyString;
@@ -13,7 +14,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "itertools.islice", base = PyObject.class)
-public class islice extends PyObject {
+public class islice extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(islice.class);
     private itertools.ItertoolsIterator iter;
@@ -129,13 +130,13 @@
         };
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 }
diff --git a/src/org/python/modules/itertools/izip.java b/src/org/python/modules/itertools/izip.java
--- a/src/org/python/modules/itertools/izip.java
+++ b/src/org/python/modules/itertools/izip.java
@@ -1,7 +1,6 @@
 /* Copyright (c) Jython Developers */
 package org.python.modules.itertools;
 
-import org.python.core.ArgParser;
 import org.python.core.Py;
 import org.python.core.PyIterator;
 import org.python.core.PyObject;
@@ -9,16 +8,13 @@
 import org.python.core.PyTuple;
 import org.python.core.PyType;
 import org.python.core.PyXRange;
-import org.python.expose.ExposedClassMethod;
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedNew;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedType;
 
-import java.util.ArrayList;
-
 @ExposedType(name = "itertools.izip", base = PyObject.class)
-public class izip extends PyObject {
+public class izip extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(izip.class);
     private PyIterator iter;
@@ -106,13 +102,13 @@
 
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 }
diff --git a/src/org/python/modules/itertools/izipLongest.java b/src/org/python/modules/itertools/izipLongest.java
--- a/src/org/python/modules/itertools/izipLongest.java
+++ b/src/org/python/modules/itertools/izipLongest.java
@@ -13,7 +13,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "itertools.izip_longest", base = PyObject.class)
-public class izipLongest extends PyObject {
+public class izipLongest extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(izipLongest.class);
     private PyIterator iter;
@@ -102,14 +102,13 @@
         };
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
-
 }
diff --git a/src/org/python/modules/itertools/permutations.java b/src/org/python/modules/itertools/permutations.java
--- a/src/org/python/modules/itertools/permutations.java
+++ b/src/org/python/modules/itertools/permutations.java
@@ -14,7 +14,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "itertools.permutations", base = PyObject.class)
-public class permutations extends PyObject {
+public class permutations extends PyIterator {
     public static final PyType TYPE = PyType.fromClass(permutations.class);
     private PyIterator iter;
 
@@ -105,13 +105,13 @@
         };
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 }
diff --git a/src/org/python/modules/itertools/product.java b/src/org/python/modules/itertools/product.java
--- a/src/org/python/modules/itertools/product.java
+++ b/src/org/python/modules/itertools/product.java
@@ -13,7 +13,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "itertools.product", base = PyObject.class)
-public class product extends PyObject {
+public class product extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(product.class);
     private PyIterator iter;
@@ -113,14 +113,13 @@
         };
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
-
 }
diff --git a/src/org/python/modules/itertools/repeat.java b/src/org/python/modules/itertools/repeat.java
--- a/src/org/python/modules/itertools/repeat.java
+++ b/src/org/python/modules/itertools/repeat.java
@@ -14,7 +14,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "itertools.repeat", base = PyObject.class)
-public class repeat extends PyObject {
+public class repeat extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(repeat.class);
     private PyIterator iter;
@@ -98,16 +98,6 @@
     }
 
     @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
-    }
-
-    @ExposedMethod
-    public PyObject next() {
-        return iter.next();
-    }
-
-    @ExposedMethod
     public int __len__() {
         if (counter < 0) {
             throw Py.TypeError("object of type 'itertools.repeat' has no len()");
@@ -126,4 +116,14 @@
                     __mod__(new PyTuple(object)));
         }
     }
+
+    public PyObject __iternext__() {
+        return iter.__iternext__();
+    }
+
+    @ExposedMethod
+    @Override
+    public PyObject next() {
+        return doNext(__iternext__());
+    }
 }
diff --git a/src/org/python/modules/itertools/starmap.java b/src/org/python/modules/itertools/starmap.java
--- a/src/org/python/modules/itertools/starmap.java
+++ b/src/org/python/modules/itertools/starmap.java
@@ -13,7 +13,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "itertools.starmap", base = PyObject.class)
-public class starmap extends PyObject {
+public class starmap extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(starmap.class);
     private PyIterator iter;
@@ -76,13 +76,13 @@
         };
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 }
diff --git a/src/org/python/modules/itertools/takewhile.java b/src/org/python/modules/itertools/takewhile.java
--- a/src/org/python/modules/itertools/takewhile.java
+++ b/src/org/python/modules/itertools/takewhile.java
@@ -12,7 +12,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "itertools.takewhile", base = PyObject.class)
-public class takewhile extends PyObject {
+public class takewhile extends PyIterator {
 
     public static final PyType TYPE = PyType.fromClass(takewhile.class);
     private PyIterator iter;
@@ -54,13 +54,13 @@
         iter = new itertools.WhileIterator(predicate, iterable, false);
     }
 
-    @ExposedMethod
-    public PyObject __iter__() {
-        return iter;
+    public PyObject __iternext__() {
+        return iter.__iternext__();
     }
 
     @ExposedMethod
+    @Override
     public PyObject next() {
-        return iter.next();
+        return doNext(__iternext__());
     }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Wed Jun 18 19:22:22 2014
From: jython-checkins at python.org (jim.baker)
Date: Wed, 18 Jun 2014 19:22:22 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Enable_IP_address_objects_t?=
	=?utf-8?q?o_be_used_as_tuples=2E?=
Message-ID: <3gttPp4BVnz7LjM@mail.python.org>

http://hg.python.org/jython/rev/c57ccfce5106
changeset:   7302:c57ccfce5106
user:        Santoso Wijaya 
date:        Wed Jun 18 11:22:48 2014 -0600
summary:
  Enable IP address objects to be used as tuples.

Fixes http://bugs.jython.org/issue2155
Merged from https://bitbucket.org/jython/jython/pull-request/43/fix-issue-2155-make-tuple-like-object/

files:
  Lib/_socket.py          |  58 ++++++----------------------
  Lib/test/test_socket.py |  11 +++++
  2 files changed, 24 insertions(+), 45 deletions(-)


diff --git a/Lib/_socket.py b/Lib/_socket.py
--- a/Lib/_socket.py
+++ b/Lib/_socket.py
@@ -1388,60 +1388,28 @@
 
 
 # Define data structures to support IPV4 and IPV6.
-# FIXME are these ip address classes required by CPython API? Must they be old-style classes?
 
-class _ip_address_t: pass
+class _ip_address_t(tuple):
+    pass
+
 
 class _ipv4_address_t(_ip_address_t):
 
-    def __init__(self, sockaddr, port, jaddress):
-        self.sockaddr = sockaddr
-        self.port     = port
-        self.jaddress = jaddress
+    jaddress = None
 
-    def __getitem__(self, index):
-        if   0 == index:
-            return self.sockaddr
-        elif 1 == index:
-            return self.port
-        else:
-            raise IndexError()
-
-    def __len__(self):
-        return 2
-
-    def __str__(self):
-        return "('%s', %d)" % (self.sockaddr, self.port)
-
-    __repr__ = __str__
+    def __new__(cls, sockaddr, port, jaddress):
+        ntup = tuple.__new__(cls, (sockaddr, port))
+        ntup.jaddress = jaddress
+        return ntup
 
 class _ipv6_address_t(_ip_address_t):
 
-    def __init__(self, sockaddr, port, jaddress):
-        self.sockaddr = sockaddr
-        self.port     = port
-        self.jaddress = jaddress
+    jaddress = None
 
-    def __getitem__(self, index):
-        if   0 == index:
-            return self.sockaddr
-        elif 1 == index:
-            return self.port
-        elif 2 == index:
-            return 0
-        elif 3 == index:
-            return self.jaddress.scopeId
-        else:
-            raise IndexError()
-
-    def __len__(self):
-        return 4
-
-    def __str__(self):
-        return "('%s', %d, 0, %d)" % (self.sockaddr, self.port, self.jaddress.scopeId)
-
-    __repr__ = __str__
-
+    def __new__(cls, sockaddr, port, jaddress):
+        ntup = tuple.__new__(cls, (sockaddr, port, 0, jaddress.scopeId))
+        ntup.jaddress = jaddress
+        return ntup
 
 
 def _get_jsockaddr(address_object, family, sock_type, proto, flags):
diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -1802,6 +1802,17 @@
             self.assert_(isinstance(canonname, str))
             self.assert_(isinstance(sockaddr[0], str))
 
+    def testSockAddrAsTuple(self):
+        family, socktype, proto, canonname, sockaddr = socket.getaddrinfo(HOST, PORT, socket.AF_INET, socket.SOCK_STREAM)[0]
+        self.assertEqual(len(sockaddr), 2)
+        self.assertEqual(sockaddr[-1], PORT)
+        self.assertEqual(sockaddr[:2], ('127.0.0.1', PORT))
+
+        family, socktype, proto, canonname, sockaddr = socket.getaddrinfo('::1', PORT, socket.AF_INET6, socket.SOCK_STREAM)[0]
+        self.assertEqual(len(sockaddr), 4)
+        self.assertEqual(sockaddr[-3], PORT)
+        #self.assertEqual(sockaddr[:2], ('::1', PORT))      # FIXME: Got '0:0:...:1' instead!
+
     def testAI_PASSIVE(self):
         # Disabling this test for now; it's expectations are not portable.
         # Expected results are too dependent on system config to be made portable between systems.

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Thu Jun 19 00:12:31 2014
From: jython-checkins at python.org (jim.baker)
Date: Thu, 19 Jun 2014 00:12:31 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Modify_enumerate_such_that_?=
 =?utf-8?q?it_can_it_work_with_arbitrarily_long_integers=2E?=
Message-ID: <3gv0rb01kHz7Ljv@mail.python.org>

http://hg.python.org/jython/rev/7516d3820146
changeset:   7303:7516d3820146
user:        Santoso Wijaya 
date:        Wed Jun 18 15:37:29 2014 -0600
summary:
  Modify enumerate such that it can it work with arbitrarily long integers.

Completes fix for http://bugs.jython.org/issue1982
Merged from https://bitbucket.org/jython/jython/pull-request/37/fix-enumerate-s-start-parameter-to-accept/

files:
  Lib/test/test_enumerate_jy.py               |   4 ++
  src/org/python/core/PyEnumerate.java        |  19 ++++++---
  src/org/python/core/PyEnumerateDerived.java |   2 +-
  3 files changed, 17 insertions(+), 8 deletions(-)


diff --git a/Lib/test/test_enumerate_jy.py b/Lib/test/test_enumerate_jy.py
--- a/Lib/test/test_enumerate_jy.py
+++ b/Lib/test/test_enumerate_jy.py
@@ -24,6 +24,10 @@
         self.assertEqual(iter(e), e)
         self.assertEqual(list(e), self.res)
 
+    def test_start_maxint(self):
+        e = self.enum(self.seq, sys.maxint)
+        self.assertEqual(list(e), [(2147483647, 'a'), (2147483648L, 'b'), (2147483649L, 'c')])
+
 
 def test_main(verbose=None):
     testclasses = (EnumerateJyTestCase,)
diff --git a/src/org/python/core/PyEnumerate.java b/src/org/python/core/PyEnumerate.java
--- a/src/org/python/core/PyEnumerate.java
+++ b/src/org/python/core/PyEnumerate.java
@@ -14,7 +14,7 @@
     public static final PyType TYPE = PyType.fromClass(PyEnumerate.class);
 
     /** Current index of enumeration. */
-    private long index;
+    private PyObject index;     // using PyObject so we are not limited to sys.maxint or Integer.MAX_VALUE
 
     /** Secondary iterator of enumeration. */
     private PyObject sit;
@@ -23,13 +23,13 @@
         super(subType);
     }
 
-    public PyEnumerate(PyType subType, PyObject seq, long start) {
+    public PyEnumerate(PyType subType, PyObject seq, PyObject start) {
         super(subType);
         index = start;
         sit = seq.__iter__();
     }
 
-    public PyEnumerate(PyObject seq, long start) {
+    public PyEnumerate(PyObject seq, PyObject start) {
         this(TYPE, seq, start);
     }
 
@@ -51,13 +51,15 @@
     public final static PyObject enumerate_new(PyNewWrapper new_, boolean init, PyType subtype,
                                                PyObject[] args, String[] keywords) {
         if (args.length > 2 || args.length <= 0) {
-            throw PyBuiltinCallable.DefaultInfo.unexpectedCall(args.length, false, "enumerate", 0,
-                                                               1);
+            throw PyBuiltinCallable.DefaultInfo.unexpectedCall(args.length, true, "enumerate", 1, 2);
         }
 
         ArgParser ap = new ArgParser("enumerate", args, keywords, new String[] {"sequence", "start"});
         PyObject seq = ap.getPyObject(0);
-        long start = (long) ap.getInt(1, 0);
+        PyObject start = ap.getPyObject(1, Py.newInteger(0));
+        if (!start.isIndex()) {
+            throw Py.TypeError("an integer is required");
+        }
 
         if (new_.for_type == subtype) {
             return new PyEnumerate(seq, start);
@@ -81,6 +83,9 @@
             return null;
         }
 
-        return new PyTuple(new PyInteger((int)index++), nextItem);
+        PyObject next = new PyTuple(index, nextItem);
+        index = index.__radd__(Py.newInteger(1));
+
+        return next;
     }
 }
diff --git a/src/org/python/core/PyEnumerateDerived.java b/src/org/python/core/PyEnumerateDerived.java
--- a/src/org/python/core/PyEnumerateDerived.java
+++ b/src/org/python/core/PyEnumerateDerived.java
@@ -38,7 +38,7 @@
         dict=new PyStringMap();
     }
 
-    public PyEnumerateDerived(PyType subtype,PyObject seq,long start) {
+    public PyEnumerateDerived(PyType subtype,PyObject seq,PyObject start) {
         super(subtype,seq,start);
         slots=new PyObject[subtype.getNumSlots()];
         dict=subtype.instDict();

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Thu Jun 19 23:33:29 2014
From: jython-checkins at python.org (jeff.allen)
Date: Thu, 19 Jun 2014 23:33:29 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Test_that_os=2Ewrite_suppor?=
 =?utf-8?q?ts_objects_that_offer_the_buffer_API=2E?=
Message-ID: <3gvbx51XhTz7Ljn@mail.python.org>

http://hg.python.org/jython/rev/0944779f5d1c
changeset:   7304:0944779f5d1c
user:        Jeff Allen 
date:        Thu Jun 19 07:54:45 2014 +0100
summary:
  Test that os.write supports objects that offer the buffer API.
Backs up fix for issue #2062.

files:
  Lib/test/test_os_jy.py |  32 ++++++++++++++++++++++++++++++
  1 files changed, 32 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_os_jy.py b/Lib/test/test_os_jy.py
--- a/Lib/test/test_os_jy.py
+++ b/Lib/test/test_os_jy.py
@@ -3,6 +3,7 @@
 Made for Jython.
 """
 import os
+import array
 import unittest
 from test import test_support
 
@@ -91,11 +92,42 @@
         self.assertRaises(OSError, os.lstat, test_support.TESTFN + os.path.sep)
 
 
+class OSWriteTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.fd = os.open(test_support.TESTFN, os.O_WRONLY | os.O_CREAT)
+
+    def tearDown(self):
+        if self.fd :
+            os.close(self.fd)
+            if os.path.exists(test_support.TESTFN):
+                os.remove(test_support.TESTFN)
+
+    def do_write(self, b, nx=None):
+        if nx is None : nx = len(b)
+        n = os.write(self.fd, b)
+        self.assertEqual(n, nx, "os.write length error: " + repr(b))
+
+    def test_write_buffer(self): # Issue 2062
+        s = b"Big Red Book"
+        for type2test in (str, buffer, bytearray, (lambda x : array.array('b',x))) :
+            self.do_write(type2test(s))
+
+        with memoryview(s) as m :
+            self.do_write(m)
+            # not contiguous:
+            self.assertRaises(BufferError, self.do_write, m[1::2])
+
+        # lacks buffer api:
+        self.assertRaises(TypeError, self.do_write, 1.5, 4)
+
+
 def test_main():
     test_support.run_unittest(
         OSFileTestCase, 
         OSDirTestCase,
         OSStatTestCase,
+        OSWriteTestCase,
     )
 
 if __name__ == '__main__':

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Thu Jun 19 23:33:30 2014
From: jython-checkins at python.org (jeff.allen)
Date: Thu, 19 Jun 2014 23:33:30 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_superfluous_ANTLR_JA?=
	=?utf-8?q?Rs_from_extlibs=2E?=
Message-ID: <3gvbx631YXz7Lk0@mail.python.org>

http://hg.python.org/jython/rev/4de0eff57af5
changeset:   7305:4de0eff57af5
user:        Jeff Allen 
date:        Thu Jun 19 19:38:55 2014 +0100
summary:
  Remove superfluous ANTLR JARs from extlibs.
antlr-3.1.3.jar contains all the others. This also mitigates issue #2165.

files:
  extlibs/antlr-2.7.7.jar          |  Bin 
  extlibs/antlr-runtime-3.1.3.jar  |  Bin 
  extlibs/stringtemplate-3.2.1.jar |  Bin 
  3 files changed, 0 insertions(+), 0 deletions(-)


diff --git a/extlibs/antlr-2.7.7.jar b/extlibs/antlr-2.7.7.jar
deleted file mode 100644
index 5e5f14b35584eac2a9f0f888769f0ab93ca6d849..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/antlr-runtime-3.1.3.jar b/extlibs/antlr-runtime-3.1.3.jar
deleted file mode 100644
index b0a9ea69f5c29097145a48c26e127972920db440..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/stringtemplate-3.2.1.jar b/extlibs/stringtemplate-3.2.1.jar
deleted file mode 100644
index d0e11b7193734c5b2784127951a1a91372f2ab94..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Fri Jun 20 17:19:12 2014
From: jython-checkins at python.org (frank.wierzbicki)
Date: Fri, 20 Jun 2014 17:19:12 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_=231591=3A_Interactive_inte?=
 =?utf-8?b?cnByZXRlciBzdHVjayBvbiAnLi4uJyBsb29w?=
Message-ID: <3gw3Zm2hQ8z7Ljj@mail.python.org>

http://hg.python.org/jython/rev/a46979f3754d
changeset:   7306:a46979f3754d
user:        Peter Holloway 
date:        Fri Jun 20 15:19:05 2014 +0000
summary:
  #1591: Interactive interpreter stuck on '...' loop

files:
  ACKNOWLEDGMENTS         |  1 +
  Lib/test/test_codeop.py |  3 +++
  NEWS                    |  1 +
  grammar/PythonPartial.g |  2 +-
  4 files changed, 6 insertions(+), 1 deletions(-)


diff --git a/ACKNOWLEDGMENTS b/ACKNOWLEDGMENTS
--- a/ACKNOWLEDGMENTS
+++ b/ACKNOWLEDGMENTS
@@ -109,6 +109,7 @@
     Michael B?sch
     Richard Eckart de Castilho
     Timoth?e Lecomte
+    Peter Holloway
 
 Local Variables:
 mode: indented-text
diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py
--- a/Lib/test/test_codeop.py
+++ b/Lib/test/test_codeop.py
@@ -302,6 +302,9 @@
             ai("[i for i in range(10)] = (1, 2, 3)")
             ai("a = 1 and b = 2");
 
+        # Merge test cases below upstream.
+        ai("def x():\n pass\na=1\n")
+
     def test_filename(self):
         self.assertEquals(compile_command("a = 1\n", "abc").co_filename,
                           compile("a = 1\n", "abc", 'single').co_filename)
diff --git a/NEWS b/NEWS
--- a/NEWS
+++ b/NEWS
@@ -8,6 +8,7 @@
     - [ 1981 ] ast.And and ast.Or should be subclasses of ast.boolop
   New Features
     - [ 1896215 ] findResource(s) for SyspathJavaLoader
+    - [ 1591 ] Interactive interpreter stuck on '...' loop
 
 Jython 2.7b2
   Bugs Fixed
diff --git a/grammar/PythonPartial.g b/grammar/PythonPartial.g
--- a/grammar/PythonPartial.g
+++ b/grammar/PythonPartial.g
@@ -167,7 +167,7 @@
 
     : NEWLINE
     | simple_stmt
-    | compound_stmt NEWLINE?
+    | compound_stmt NEWLINE? EOF
     ;
 
 //eval_input: testlist NEWLINE* ENDMARKER

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 22 06:51:56 2014
From: jython-checkins at python.org (jim.baker)
Date: Sun, 22 Jun 2014 06:51:56 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_IntelliJ_project_fil?=
	=?utf-8?q?e?=
Message-ID: <3gx1Z475nCz7LjZ@mail.python.org>

http://hg.python.org/jython/rev/c7c2b677ac1b
changeset:   7307:c7c2b677ac1b
user:        Santoso Wijaya 
date:        Sat Jun 21 22:51:38 2014 -0600
summary:
  Remove IntelliJ project file

files:
  Jython27.iml |  22 ----------------------
  1 files changed, 0 insertions(+), 22 deletions(-)


diff --git a/Jython27.iml b/Jython27.iml
deleted file mode 100644
--- a/Jython27.iml
+++ /dev/null
@@ -1,22 +0,0 @@
-
-
-  
-    
-    
-      
-    
-    
-    
-    
-      
-        
-          
-        
-        
-        
-      
-    
-    
-  
-
-

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 22 07:59:44 2014
From: jython-checkins at python.org (jim.baker)
Date: Sun, 22 Jun 2014 07:59:44 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_xrange_compliance=2E?=
Message-ID: <3gx34J08Z5z7LjZ@mail.python.org>

http://hg.python.org/jython/rev/33ab610967ec
changeset:   7308:33ab610967ec
user:        Santoso Wijaya 
date:        Sat Jun 21 23:02:43 2014 -0600
summary:
  Fix xrange compliance.

Fixes http://bugs.jython.org/issue2146
Merged from https://bitbucket.org/jython/jython/pull-request/38/optimize-xrange-iterator/

files:
  Lib/test/test_enumerate.py        |   1 -
  src/org/python/core/PyXRange.java |  23 ++++++++++++++++++-
  2 files changed, 22 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_enumerate.py b/Lib/test/test_enumerate.py
--- a/Lib/test/test_enumerate.py
+++ b/Lib/test/test_enumerate.py
@@ -202,7 +202,6 @@
         # XXX: CPython implementation details
         del EnumerateTestCase.test_tuple_reuse
         del TestReversed.test_len
-        del TestReversed.test_xrange_optimization
     testclasses = (EnumerateTestCase, SubclassTestCase, TestEmpty, TestBig,
                    TestReversed)
     test_support.run_unittest(*testclasses)
diff --git a/src/org/python/core/PyXRange.java b/src/org/python/core/PyXRange.java
--- a/src/org/python/core/PyXRange.java
+++ b/src/org/python/core/PyXRange.java
@@ -115,9 +115,30 @@
         return ret;
     }
 
+    @Override
+    public PyObject __iter__() {
+        return xrange___iter__();
+    }
+
     @ExposedMethod(doc = BuiltinDocs.xrange___iter___doc)
     public PyObject xrange___iter__() {
-        return seq___iter__();
+        return range_iter();
+    }
+
+    @ExposedMethod(doc = BuiltinDocs.xrange___reversed___doc)
+    public PyObject xrange___reversed__() {
+        return range_reverse();
+    }
+
+    private final PyXRangeIter range_iter() {
+        return new PyXRangeIter(0, (long)start, (long)step, (long)len);
+    }
+
+    private final PyXRangeIter range_reverse() {
+        return new PyXRangeIter(0,
+                (start + (long)(len - 1) * step),   // start
+                (long)(0 - step),                   // step (negative value)
+                len);
     }
 
     @Override

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 22 07:59:45 2014
From: jython-checkins at python.org (jim.baker)
Date: Sun, 22 Jun 2014 07:59:45 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Ensure_that_bz2_decompressi?=
 =?utf-8?q?on_has_enough_data_to_decode=2E?=
Message-ID: <3gx34K1p3Lz7Ljl@mail.python.org>

http://hg.python.org/jython/rev/02a31eb57af2
changeset:   7309:02a31eb57af2
user:        Indra Talip 
date:        Sat Jun 21 23:05:02 2014 -0600
summary:
  Ensure that bz2 decompression has enough data to decode.

Fixes http://bugs.jython.org/issue2040
Merged from https://bitbucket.org/jython/jython/pull-request/24/the-pybz2decompressor-should-check-that/

files:
  src/org/python/modules/bz2/PyBZ2Decompressor.java |  3 ++-
  1 files changed, 2 insertions(+), 1 deletions(-)


diff --git a/src/org/python/modules/bz2/PyBZ2Decompressor.java b/src/org/python/modules/bz2/PyBZ2Decompressor.java
--- a/src/org/python/modules/bz2/PyBZ2Decompressor.java
+++ b/src/org/python/modules/bz2/PyBZ2Decompressor.java
@@ -69,7 +69,8 @@
         ByteArrayOutputStream decodedStream = new ByteArrayOutputStream();
         final byte[] buf = accumulator;
         for (int i = 0; i < buf.length; i++) {
-            if (((char) buf[i] == '\\') && ((char) buf[i + 1] == 'x')) {
+            if (((i + 3) < buf.length) &&
+                (((char) buf[i] == '\\') && ((char) buf[i + 1] == 'x'))) {
                 int decodedByte = ((Character.digit((char) buf[i + 2], 16) << 4) + Character
                         .digit((char) buf[i + 3], 16));
                 decodedStream.write(decodedByte);

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 22 08:24:17 2014
From: jython-checkins at python.org (jim.baker)
Date: Sun, 22 Jun 2014 08:24:17 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Enable_setting_the_mode_and?=
 =?utf-8?q?_buffersize_when_a_PyFile_wraps_InputStream?=
Message-ID: <3gx3cd4xPGz7LjV@mail.python.org>

http://hg.python.org/jython/rev/66800903dded
changeset:   7310:66800903dded
user:        Indra Talip 
date:        Sun Jun 22 00:24:07 2014 -0600
summary:
  Enable setting the mode and buffersize when a PyFile wraps InputStream
and OutputStream objects.

Merged from https://bitbucket.org/jython/jython/pull-request/7/additional-fileutilwrap-methods-that-allow/

files:
  Lib/test/test_java_integration.py      |  34 ++++++++++
  src/org/python/core/PyFile.java        |  23 +++++-
  src/org/python/core/util/FileUtil.java |  47 ++++++++++---
  3 files changed, 89 insertions(+), 15 deletions(-)


diff --git a/Lib/test/test_java_integration.py b/Lib/test/test_java_integration.py
--- a/Lib/test/test_java_integration.py
+++ b/Lib/test/test_java_integration.py
@@ -143,6 +143,40 @@
     def test_fileio_error(self):
         self.assertRaises(FileNotFoundException, FileInputStream, "garbage")
 
+    def fileutil_is_helper(self, mode, expected):
+        old_linesep = System.setProperty("line.separator", "\r\n")
+        try:
+            inputstream = ByteArrayInputStream(bytearray('1\r\n2\r\n3\r\n'))
+            inputfile = FileUtil.wrap(inputstream, mode)
+            actual = inputfile.readlines()
+            inputfile.close()
+            self.assertEquals(expected, actual)
+        finally:
+            System.setProperty("line.separator", old_linesep)
+
+    def test_fileutil_wrap_inputstream(self):
+        self.fileutil_is_helper('r', ['1\n', '2\n', '3\n'])
+
+    def test_fileutil_wrap_inputstream_binary(self):
+        self.fileutil_is_helper('rb', ['1\r\n', '2\r\n', '3\r\n'])
+
+    def fileutil_os_helper(self, mode, expected):
+        old_linesep = System.setProperty("line.separator", "\r\n")
+        try:
+            outputstream = ByteArrayOutputStream()
+            outputfile = FileUtil.wrap(outputstream, mode)
+            outputfile.writelines(["1\n", "2\n", "3\n"])
+            outputfile.close()
+            self.assertEquals(bytearray(outputstream.toByteArray()), expected)
+        finally:
+            System.setProperty("line.separator", old_linesep)
+
+    def test_fileutil_wrap_outputstream_default_textmode(self):
+        self.fileutil_os_helper("w", bytearray("1\r\n2\r\n3\r\n"))
+
+    def test_fileutil_wrap_outputstream_binary(self):
+        self.fileutil_os_helper("wb", bytearray("1\n2\n3\n"))
+
     def test_unsupported_tell(self):
         fp = FileUtil.wrap(System.out)
         self.assertRaises(IOError, fp.tell)
diff --git a/src/org/python/core/PyFile.java b/src/org/python/core/PyFile.java
--- a/src/org/python/core/PyFile.java
+++ b/src/org/python/core/PyFile.java
@@ -102,18 +102,28 @@
      * method file doesn't expose this functionality (open does
      * albeit deprecated) as it isn't available to regular Python code. To wrap an
      * InputStream in a file from Python, use
+     * {@link org.python.core.util.FileUtil#wrap(InputStream, String, int)}
+     * {@link org.python.core.util.FileUtil#wrap(InputStream, String)}
      * {@link org.python.core.util.FileUtil#wrap(InputStream, int)}
      * {@link org.python.core.util.FileUtil#wrap(InputStream)}
      */
+    public PyFile(InputStream istream, String mode, int bufsize) {
+        this(istream, "", mode, bufsize, true);
+    }
+
+    public PyFile(InputStream istream, String mode) {
+        this(istream, mode, -1);
+    }
+
     public PyFile(InputStream istream, int bufsize) {
-        this(istream, "", "r", bufsize, true);
+        this(istream, "r", bufsize);
     }
 
     public PyFile(InputStream istream) {
         this(istream, -1);
     }
 
-    PyFile(OutputStream ostream, String name, String mode, int bufsize, boolean closefd) {
+    public PyFile(OutputStream ostream, String name, String mode, int bufsize, boolean closefd) {
         parseMode(mode);
         file___init__(new StreamIO(ostream, closefd), name, mode, bufsize);
     }
@@ -123,11 +133,18 @@
      * method file doesn't expose this functionality (open does
      * albeit deprecated) as it isn't available to regular Python code. To wrap an
      * OutputStream in a file from Python, use
+     * {@link org.python.core.util.FileUtil#wrap(OutputStream, String, int)}
+     * {@link org.python.core.util.FileUtil#wrap(OutputStream, String)}
      * {@link org.python.core.util.FileUtil#wrap(OutputStream, int)}
      * {@link org.python.core.util.FileUtil#wrap(OutputStream)}
      */
+
+    public PyFile(OutputStream ostream, String mode, int bufsize) {
+         this(ostream, "", mode, bufsize, true);
+    }
+
     public PyFile(OutputStream ostream, int bufsize) {
-        this(ostream, "", "w", bufsize, true);
+        this(ostream, "w", bufsize);
     }
 
     public PyFile(OutputStream ostream) {
diff --git a/src/org/python/core/util/FileUtil.java b/src/org/python/core/util/FileUtil.java
--- a/src/org/python/core/util/FileUtil.java
+++ b/src/org/python/core/util/FileUtil.java
@@ -12,37 +12,60 @@
  * Utility methods for Java file handling.
  */
 public class FileUtil {
-
     /**
-     * Creates a PyFile that reads from the given InputStream with mode.
+     * Creates a PyFile with mode that reads from the given InputStream using bufsize.
      */
-    public static PyFile wrap(InputStream is, String mode) {
-        return new PyFile(is, "", mode, -1, true);    
+    public static PyFile wrap(InputStream is, String mode, int bufsize) {
+        return new PyFile(is, "", mode, bufsize, true);
     }
 
     /**
-     * Creates a PyFile that reads from the given InputStream with bufsize.
+     * Creates a PyFile with mode that reads from the InputStream.
      */
-    public static PyFile wrap(InputStream is, int bufsize) {
-        return new PyFile(is, bufsize);
+    public static PyFile wrap(InputStream is, String mode) {
+        return wrap(is, mode, -1);
     }
 
     /**
-     * Creates a PyFile that reads from the given InputStream.
+     * Creates a PyFile in text mode that reads from the given InputStream
+     * using bufsize.
+     */
+    public static PyFile wrap(InputStream is, int bufsize) {
+        return wrap(is, "r", bufsize);
+    }
+
+    /**
+     * Creates a PyFile in text mode that reads from the given InputStream.
      */
     public static PyFile wrap(InputStream is) {
         return wrap(is, -1);
     }
 
     /**
-     * Creates a PyFile that writes to the given OutputStream with bufsize.
+     * Creates a PyFile with mode that writes to the given OutputStream with the
+     * given bufsize.
      */
-    public static PyFile wrap(OutputStream os, int bufsize) {
-        return new PyFile(os, bufsize);
+    public static PyFile wrap(OutputStream os, String mode, int bufsize) {
+        return new PyFile(os, mode, bufsize);
     }
 
     /**
-     * Creates a PyFile that writes to the given OutputStream.
+     * Creates a PyFile with mode that writes to the given OutputStream
+     */
+    public static PyFile wrap(OutputStream os, String mode) {
+        return wrap(os, mode, -1);
+    }
+
+    /**
+     * Creates a PyFile in text mode that writes to the given OutputStream
+     * with bufsize.
+     */
+    public static PyFile wrap(OutputStream os, int bufsize) {
+        return wrap(os, "w", bufsize);
+    }
+
+    /**
+     * Creates a PyFile in text mode that writes to the given OutputStream.
      */
     public static PyFile wrap(OutputStream os) {
         return wrap(os, -1);

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 22 08:53:23 2014
From: jython-checkins at python.org (jim.baker)
Date: Sun, 22 Jun 2014 08:53:23 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Make_PyDictionary=23equals?=
 =?utf-8?q?=28=29_more_robust=2C_similar_to_PyTuple_and_PyList=2E?=
Message-ID: <3gx4GC14Bkz7LjN@mail.python.org>

http://hg.python.org/jython/rev/946090532a2c
changeset:   7311:946090532a2c
user:        Santoso Wijaya 
date:        Sun Jun 22 00:53:04 2014 -0600
summary:
  Make PyDictionary#equals() more robust, similar to PyTuple and PyList.

Fixes http://bugs.jython.org/issue1728
Merged from https://bitbucket.org/jython/jython/pull-request/50/fix-issue-1728-a-more-robust-pydictionary/

files:
  Lib/test/test_dict.py                 |   5 ++++
  src/org/python/core/PyDictionary.java |  18 +++++---------
  2 files changed, 12 insertions(+), 11 deletions(-)


diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py
--- a/Lib/test/test_dict.py
+++ b/Lib/test/test_dict.py
@@ -672,6 +672,11 @@
             pass
         self._tracked(MyDict())
 
+    def test_list_equality(self):
+        class A(dict): pass
+        for dtype in (A, dict):
+            self.assertEquals([dtype()], [dict()])
+
 
 from test import mapping_tests
 
diff --git a/src/org/python/core/PyDictionary.java b/src/org/python/core/PyDictionary.java
--- a/src/org/python/core/PyDictionary.java
+++ b/src/org/python/core/PyDictionary.java
@@ -732,19 +732,15 @@
 
     @Override
     public boolean equals(Object obj) {
-        if (obj == null) {
-            return false;
+        if (this == obj) {
+            return true;
         }
-        if (getClass() != obj.getClass()) {
-            return false;
+        if (obj instanceof PyDictionary) {
+            return ((PyDictionary) obj).getMap().equals(getMap());
+        } else if (obj instanceof Map) {
+            return getMap().equals((Map) obj);
         }
-        final PyDictionary other = (PyDictionary) obj;
-        ConcurrentMap map = getMap();
-        ConcurrentMap otherMap = other.getMap();
-        if (map != otherMap && (map == null || !map.equals(otherMap))) {
-            return false;
-        }
-        return true;
+        return false;
     }
 
     @ExposedMethod(doc = BuiltinDocs.dict___hash___doc)

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 22 21:12:33 2014
From: jython-checkins at python.org (jim.baker)
Date: Sun, 22 Jun 2014 21:12:33 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Missing_file_from_7311=3A94?=
	=?utf-8?q?6090532a2c?=
Message-ID: <3gxNg52xFMz7LjQ@mail.python.org>

http://hg.python.org/jython/rev/257b274ad39e
changeset:   7312:257b274ad39e
user:        Jim Baker 
date:        Sun Jun 22 13:12:24 2014 -0600
summary:
  Missing file from 7311:946090532a2c

files:
  src/org/python/core/PyXRangeIter.java |  45 +++++++++++++++
  1 files changed, 45 insertions(+), 0 deletions(-)


diff --git a/src/org/python/core/PyXRangeIter.java b/src/org/python/core/PyXRangeIter.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/PyXRangeIter.java
@@ -0,0 +1,45 @@
+/* Copyright (c) Jython Developers */
+package org.python.core;
+
+import org.python.expose.ExposedMethod;
+import org.python.expose.ExposedType;
+
+/**
+ * Specially optimized xrange iterator.
+ */
+ at ExposedType(name = "rangeiterator", base = PyObject.class, isBaseType = false)
+public class PyXRangeIter extends PyIterator {
+
+    public static final PyType TYPE = PyType.fromClass(PyXRangeIter.class);
+    static {
+        TYPE.setName("rangeiterator");
+    }
+
+    private long index;
+    private long start;
+    private long step;
+    private long len;
+
+    public PyXRangeIter(long index, long start, long step, long len) {
+        super(TYPE);
+        this.index = index;
+        this.start = start;
+        this.step = step;
+        this.len = len;
+    }
+
+    @ExposedMethod(doc = "x.next() -> the next value, or raise StopIteration")
+    final PyObject rangeiterator_next() {
+        return super.next();
+    }
+
+    @Override
+    public PyObject __iternext__() {
+        if (index < len) {
+            return Py.newInteger(start + index++ * step);
+        }
+
+        return null;
+    }
+
+}

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 22 22:28:35 2014
From: jython-checkins at python.org (jim.baker)
Date: Sun, 22 Jun 2014 22:28:35 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_test=5Fdumbdbm_so_that_?=
 =?utf-8?q?it_properly_determines_if_underlying_os_is_posix?=
Message-ID: <3gxQLq6sGvz7LjT@mail.python.org>

http://hg.python.org/jython/rev/9a58255792ac
changeset:   7313:9a58255792ac
user:        Jim Baker 
date:        Sun Jun 22 14:28:28 2014 -0600
summary:
  Fix test_dumbdbm so that it properly determines if underlying os is posix

files:
  Lib/test/test_dumbdbm.py |  8 +++++++-
  1 files changed, 7 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_dumbdbm.py b/Lib/test/test_dumbdbm.py
--- a/Lib/test/test_dumbdbm.py
+++ b/Lib/test/test_dumbdbm.py
@@ -51,7 +51,13 @@
             os.umask(old_umask)
 
         expected_mode = 0635
-        if os.name != 'posix':
+        is_posix = True
+        if os.name == 'java':
+            if os._name != 'posix':
+                is_posix = False
+        elif os.name != 'posix':
+            is_posix = False
+        if not is_posix:
             # Windows only supports setting the read-only attribute.
             # This shouldn't fail, but doesn't work like Unix either.
             expected_mode = 0666

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Tue Jun 24 16:58:35 2014
From: jython-checkins at python.org (frank.wierzbicki)
Date: Tue, 24 Jun 2014 16:58:35 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Ignore_lines_with_just_comm?=
 =?utf-8?q?ents=2C_rather_than_treating_as_newline=2E?=
Message-ID: <3gyVx710tGz7LkC@mail.python.org>

http://hg.python.org/jython/rev/a2ab729f5a16
changeset:   7314:a2ab729f5a16
user:        Peter Holloway 
date:        Tue Jun 24 14:58:29 2014 +0000
summary:
  Ignore lines with just comments, rather than treating as newline.

files:
  Lib/test/test_codeop.py                     |   4 ++
  src/org/python/antlr/PythonTokenSource.java |  16 ++++++++-
  2 files changed, 18 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py
--- a/Lib/test/test_codeop.py
+++ b/Lib/test/test_codeop.py
@@ -258,6 +258,10 @@
         ai("(x for x in")
         ai("(x for x in (")
 
+        # Merge test cases below upstream.
+        ai("def x():\n pass\n#comment")
+
+
     def test_invalid(self):
         ai = self.assertInvalid
         ai("a b")
diff --git a/src/org/python/antlr/PythonTokenSource.java b/src/org/python/antlr/PythonTokenSource.java
--- a/src/org/python/antlr/PythonTokenSource.java
+++ b/src/org/python/antlr/PythonTokenSource.java
@@ -252,15 +252,27 @@
     private List enqueueHiddens(Token t) {
         List newlines = new ArrayList();
         if (inSingle && t.getType() == Token.EOF) {
-            if (stream.size() > lastTokenAddedIndex + 1) {
-                Token hidden = stream.get(lastTokenAddedIndex + 1);
+            int k = 1;
+            while (stream.size() > lastTokenAddedIndex + k) {
+                Token hidden = stream.get(lastTokenAddedIndex + k);
                 if (hidden.getType() == PythonLexer.COMMENT) {
                     String text = hidden.getText();
                     int i = text.indexOf("\n");
+                    i = text.indexOf("\n", i + 1);
                     while(i != -1) {
                         newlines.add(hidden);
+                        lastTokenAddedIndex++;
                         i = text.indexOf("\n", i + 1);
                     }
+                    k++;
+                } else if (hidden.getType() == PythonLexer.NEWLINE) {
+                    generateNewline(hidden);
+                    lastTokenAddedIndex++;
+                    break;
+                } else if (hidden.getType() == PythonLexer.LEADING_WS) {
+                    k++;
+                } else {
+                    break;
                 }
             }
         }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Tue Jun 24 23:24:15 2014
From: jython-checkins at python.org (jeff.allen)
Date: Tue, 24 Jun 2014 23:24:15 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Preserve_original_name_of_c?=
 =?utf-8?q?onsole_encoding=2C_addresses_=232123=2E?=
Message-ID: <3gygV70PwDz7LjW@mail.python.org>

http://hg.python.org/jython/rev/44191dd20f5a
changeset:   7315:44191dd20f5a
user:        Jeff Allen 
date:        Tue Jun 24 21:56:43 2014 +0100
summary:
  Preserve original name of console encoding, addresses #2123.
Change interpreter and sys.stdin.encoding to use the encoding specified, or
implied, rather than the Java-canonocal name.

files:
  src/org/python/core/Console.java       |  8 ++++++++
  src/org/python/core/PlainConsole.java  |  5 +++++
  src/org/python/core/PySystemState.java |  2 +-
  src/org/python/util/jython.java        |  2 +-
  4 files changed, 15 insertions(+), 2 deletions(-)


diff --git a/src/org/python/core/Console.java b/src/org/python/core/Console.java
--- a/src/org/python/core/Console.java
+++ b/src/org/python/core/Console.java
@@ -31,6 +31,14 @@
     public void uninstall() throws UnsupportedOperationException;
 
     /**
+     * Name of the encoding, normally supplied during initialisation, and used for line input.
+     * This may not be the cononoical name of the codec returned by {@link #getEncodingCharset()}.
+     *
+     * @return name of the encoding in use.
+     */
+    public String getEncoding();
+
+    /**
      * Accessor for encoding to use for line input as a Charset.
      *
      * @return Charset of the encoding in use.
diff --git a/src/org/python/core/PlainConsole.java b/src/org/python/core/PlainConsole.java
--- a/src/org/python/core/PlainConsole.java
+++ b/src/org/python/core/PlainConsole.java
@@ -72,6 +72,11 @@
     }
 
     @Override
+    public String getEncoding() {
+        return encoding;
+    }
+
+    @Override
     public Charset getEncodingCharset() {
         return encodingCharset;
     }
diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java
--- a/src/org/python/core/PySystemState.java
+++ b/src/org/python/core/PySystemState.java
@@ -271,7 +271,7 @@
 
         if (encoding==null) {
             // We still don't have an explicit selection for this: match the console.
-            encoding = Py.getConsole().getEncodingCharset().name();
+            encoding = Py.getConsole().getEncoding();
         }
 
         ((PyFile)stdin).setEncoding(encoding, errors);
diff --git a/src/org/python/util/jython.java b/src/org/python/util/jython.java
--- a/src/org/python/util/jython.java
+++ b/src/org/python/util/jython.java
@@ -388,7 +388,7 @@
 
         if (opts.fixInteractive || (opts.filename == null && opts.command == null)) {
             // Go interactive with the console: the parser needs to know the encoding.
-            String encoding = Py.getConsole().getEncodingCharset().name();
+            String encoding = Py.getConsole().getEncoding();
 
             // Run the interpreter interactively
             try {

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:02 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:02 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_shutil_rmtree_by_making?=
 =?utf-8?q?_PosixModule=2Eunlink_correctly_remove_symlinks=2E?=
Message-ID: <3h0cm203t4z7LjQ@mail.python.org>

http://hg.python.org/jython/rev/0ccd93d2bb54
changeset:   7316:0ccd93d2bb54
parent:      7181:6e438088c0e3
user:        Indra Talip 
date:        Sat Mar 29 16:14:54 2014 +1100
summary:
  Fix shutil rmtree by making PosixModule.unlink correctly remove symlinks.

Addresses some of the test_glob regression tests. Uses nio Files/Path from
Java 7 rather than File.isDirectory which can distinguish between symlinks
to directories and plain directories.

files:
  src/org/python/modules/posix/PosixModule.java |  29 ++++++---
  1 files changed, 19 insertions(+), 10 deletions(-)


diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java
--- a/src/org/python/modules/posix/PosixModule.java
+++ b/src/org/python/modules/posix/PosixModule.java
@@ -13,6 +13,9 @@
 import java.nio.channels.Channel;
 import java.nio.channels.ClosedChannelException;
 import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
 import java.security.SecureRandom;
 import java.util.Iterator;
 import java.util.Map;
@@ -727,17 +730,23 @@
 
     public static void unlink(PyObject path) {
         String absolutePath = absolutePath(path);
-        File file = new File(absolutePath);
-        if (file.isDirectory()) {
-            throw Py.OSError(Errno.EISDIR, path);
-        } else if (!file.delete()) {
-            // Something went wrong, does stat raise an error?
-            posix.stat(absolutePath);
-            // It exists, do we not have permissions?
-            if (!file.canWrite()) {
-                throw Py.OSError(Errno.EPERM, path);
+        try {
+            Path nioPath = new File(absolutePath).toPath();
+            if (Files.isDirectory(nioPath, LinkOption.NOFOLLOW_LINKS)) {
+                throw Py.OSError(Errno.EISDIR, path);
+            } else if (!Files.deleteIfExists(nioPath)) {
+                // Something went wrong, does stat raise an error?
+                posix.stat(absolutePath);
+                // It exists, do we not have permissions?
+                if (!Files.isWritable(nioPath)) {
+                    throw Py.OSError(Errno.EPERM, path);
+                }
+                throw Py.OSError("unlink(): an unknown error occurred: " + absolutePath);
             }
-            throw Py.OSError("unlink(): an unknown error occurred: " + absolutePath);
+        } catch (IOException ex) {
+            PyException pyError = Py.OSError("unlink(): an unknown error occurred: " + absolutePath);
+            pyError.initCause(ex);
+            throw pyError;
         }
     }
 

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:03 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:03 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Make_os=2Elstat=28path=29_p?=
 =?utf-8?q?ass_trailing_slashes_in_the_path_to_the_native_posix?=
Message-ID: <3h0cm31krPz7Ljp@mail.python.org>

http://hg.python.org/jython/rev/58b54d33f50d
changeset:   7317:58b54d33f50d
user:        Indra Talip 
date:        Sat Mar 29 16:14:56 2014 +1100
summary:
  Make os.lstat(path) pass trailing slashes in the path to the native posix library.

This causes an exception to bubble up in the case that the path exists but is
not a directory thereby making it consistent with CPython. This also fixes two of
the test_glob regression tests.

files:
  src/org/python/modules/posix/PosixModule.java |  10 +++++++++-
  1 files changed, 9 insertions(+), 1 deletions(-)


diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java
--- a/src/org/python/modules/posix/PosixModule.java
+++ b/src/org/python/modules/posix/PosixModule.java
@@ -944,7 +944,15 @@
 
         @Override
         public PyObject __call__(PyObject path) {
-            return PyStatResult.fromFileStat(posix.lstat(absolutePath(path)));
+            String absolutePath = absolutePath(path);
+
+            // round tripping from a string to a file to a string loses
+            // trailing slashes so add them back back in to get correct posix.lstat
+            // behaviour if path is not a directory.
+            if (asPath(path).endsWith(File.separator)) {
+                absolutePath = absolutePath + File.separator;
+            }
+            return PyStatResult.fromFileStat(posix.lstat(absolutePath));
         }
     }
 

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:04 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:04 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython_=28merge_default_-=3E_default?=
 =?utf-8?q?=29=3A_Merged_in_indratalip/jython_=28pull_request_=2329=29?=
Message-ID: <3h0cm43BPbz7Ljm@mail.python.org>

http://hg.python.org/jython/rev/c66185c8ed12
changeset:   7318:c66185c8ed12
parent:      7239:d3bad92dfc5b
parent:      7317:58b54d33f50d
user:        Jim Baker 
date:        Sat May 10 18:27:44 2014 -0600
summary:
  Merged in indratalip/jython (pull request #29)

Fix regression tests for test_glob. This was actually merged as of 7207:7096b5a3f8e7, but closing out bitbucket.

files:



-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:05 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:05 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Do_not_cache_ThreadState_in?=
 =?utf-8?q?_a_ThreadLocal_=28at_least_without_cleanup=29?=
Message-ID: <3h0cm54xJZz7Lk1@mail.python.org>

http://hg.python.org/jython/rev/03565ebe5a8a
changeset:   7319:03565ebe5a8a
parent:      7315:44191dd20f5a
user:        Jim Baker 
date:        Tue Jun 24 19:14:13 2014 -0600
summary:
  Do not cache ThreadState in a ThreadLocal (at least without cleanup)

files:
  src/org/python/core/ThreadStateMapping.java |  16 ++++++---
  1 files changed, 10 insertions(+), 6 deletions(-)


diff --git a/src/org/python/core/ThreadStateMapping.java b/src/org/python/core/ThreadStateMapping.java
--- a/src/org/python/core/ThreadStateMapping.java
+++ b/src/org/python/core/ThreadStateMapping.java
@@ -1,16 +1,20 @@
 package org.python.core;
 
+import com.google.common.collect.MapMaker;
+
+import java.util.Map;
+
 class ThreadStateMapping {
-    private static final ThreadLocal cachedThreadState = new ThreadLocal();
+    private static final Map cachedThreadState =
+        new MapMaker().weakKeys().weakValues().makeMap();
 
     public ThreadState getThreadState(PySystemState newSystemState) {
-        ThreadState ts = cachedThreadState.get();
+        Thread currentThread = Thread.currentThread();
+        ThreadState ts = cachedThreadState.get(currentThread);
         if (ts != null) {
             return ts;
         }
 
-
-        Thread t = Thread.currentThread();
         if (newSystemState == null) {
             Py.writeDebug("threadstate", "no current system state");
             if (Py.defaultSystemState == null) {
@@ -19,8 +23,8 @@
             newSystemState = Py.defaultSystemState;
         }
 
-        ts = new ThreadState(t, newSystemState);
-        cachedThreadState.set(ts);
+        ts = new ThreadState(currentThread, newSystemState);
+        cachedThreadState.put(currentThread, ts);
         return ts;
     }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:06 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:06 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Use_Object=5B1=5D_indirecti?=
 =?utf-8?q?on_to_avoid_ClassLoader_issues_in_storing?=
Message-ID: <3h0cm66cJVz7Lk8@mail.python.org>

http://hg.python.org/jython/rev/c7c3e492da7c
changeset:   7320:c7c3e492da7c
user:        Jim Baker 
date:        Tue Jun 24 19:15:27 2014 -0600
summary:
  Use Object[1] indirection to avoid ClassLoader issues in storing org.python.core.ThreadContext into a ThreadLocal

files:
  src/org/python/core/Py.java                     |  2 +-
  src/org/python/core/PyObject.java               |  4 ++--
  src/org/python/core/PyReflectedConstructor.java |  4 ++--
  src/org/python/core/ThreadContext.java          |  9 ++++++---
  4 files changed, 11 insertions(+), 8 deletions(-)


diff --git a/src/org/python/core/Py.java b/src/org/python/core/Py.java
--- a/src/org/python/core/Py.java
+++ b/src/org/python/core/Py.java
@@ -938,7 +938,7 @@
     {
         if (proxy._getPyInstance() != null)
             return;
-        PyObject instance = ThreadContext.initializingProxy.get();
+        PyObject instance = (PyObject)(ThreadContext.initializingProxy.get()[0]);
         ThreadState ts = Py.getThreadState();
         if (instance != null) {
             if (instance.javaProxy != null) {
diff --git a/src/org/python/core/PyObject.java b/src/org/python/core/PyObject.java
--- a/src/org/python/core/PyObject.java
+++ b/src/org/python/core/PyObject.java
@@ -146,8 +146,8 @@
             throw Py.SystemError("Automatic proxy initialization should only occur on proxy classes");
         }
         PyProxy proxy;
-        PyObject previous = ThreadContext.initializingProxy.get();
-        ThreadContext.initializingProxy.set(this);
+        Object[] previous = ThreadContext.initializingProxy.get();
+        ThreadContext.initializingProxy.set(new Object[] { this });
         try {
             try {
                 proxy = (PyProxy)c.newInstance();
diff --git a/src/org/python/core/PyReflectedConstructor.java b/src/org/python/core/PyReflectedConstructor.java
--- a/src/org/python/core/PyReflectedConstructor.java
+++ b/src/org/python/core/PyReflectedConstructor.java
@@ -203,8 +203,8 @@
     protected void constructProxy(PyObject obj, Constructor ctor, Object[] args, Class proxy) {
         // Do the actual constructor call
         Object jself = null;
-        PyObject previous = ThreadContext.initializingProxy.get();
-        ThreadContext.initializingProxy.set(obj);
+        Object[] previous = ThreadContext.initializingProxy.get();
+        ThreadContext.initializingProxy.set(new Object[] { obj });
         try {
             try {
                 jself = ctor.newInstance(args);
diff --git a/src/org/python/core/ThreadContext.java b/src/org/python/core/ThreadContext.java
--- a/src/org/python/core/ThreadContext.java
+++ b/src/org/python/core/ThreadContext.java
@@ -2,7 +2,10 @@
 package org.python.core;
 
 class ThreadContext {
-    
-    static ThreadLocal initializingProxy = new ThreadLocal();
 
-}
\ No newline at end of file
+    static ThreadLocal initializingProxy = new ThreadLocal() {
+        protected Object[] initialValue() {
+            return new Object[1];
+        }
+    };
+}

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:08 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:08 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_start_parameter_to_enum?=
 =?utf-8?q?erate_for_PyEnumerateDerived?=
Message-ID: <3h0cm815GXz7Lk8@mail.python.org>

http://hg.python.org/jython/rev/798e2609c660
changeset:   7321:798e2609c660
user:        Jim Baker 
date:        Tue Jun 24 22:45:38 2014 -0600
summary:
  Add start parameter to enumerate for PyEnumerateDerived

files:
  src/templates/enumerate.derived |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/src/templates/enumerate.derived b/src/templates/enumerate.derived
--- a/src/templates/enumerate.derived
+++ b/src/templates/enumerate.derived
@@ -1,4 +1,4 @@
 base_class: PyEnumerate
 want_dict: true
-ctr: PyObject seq
+ctr: PyObject seq, PyObject start
 incl: object

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:09 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:09 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_recursion=5Fcount_ma?=
	=?utf-8?q?nagement?=
Message-ID: <3h0cm92mV5z7LjR@mail.python.org>

http://hg.python.org/jython/rev/0cdb9e4839e8
changeset:   7322:0cdb9e4839e8
user:        Jim Baker 
date:        Tue Jun 24 22:46:23 2014 -0600
summary:
  Remove recursion_count management

files:
  src/templates/object.derived |  16 +++++-----------
  1 files changed, 5 insertions(+), 11 deletions(-)


diff --git a/src/templates/object.derived b/src/templates/object.derived
--- a/src/templates/object.derived
+++ b/src/templates/object.derived
@@ -301,18 +301,12 @@
   }
 
   public PyObject __call__(PyObject args[], String keywords[]) {
-    ThreadState ts = Py.getThreadState();
-    if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-      throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-    try {
-      PyType self_type = getType();
-      PyObject impl = self_type.lookup("__call__");
-      if (impl != null)
-        return impl.__get__(this,self_type).__call__(args,keywords);
-      return super.__call__(args,keywords);
-    } finally {
-      --ts.recursion_depth;
+    PyType self_type = getType();
+    PyObject impl = self_type.lookup("__call__");
+    if (impl != null) {
+      return impl.__get__(this,self_type).__call__(args,keywords);
     }
+    return super.__call__(args,keywords);
   }
 
   public PyObject __findattr_ex__(String name) {

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:12 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:12 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Re-run_gderived=2Epy?=
Message-ID: <3h0cmD0jrrz7LjR@mail.python.org>

http://hg.python.org/jython/rev/f177ef0ee272
changeset:   7323:f177ef0ee272
user:        Jim Baker 
date:        Tue Jun 24 22:52:11 2014 -0600
summary:
  Re-run gderived.py

files:
  src/org/python/antlr/ast/AssertDerived.java                              |  16 +++------
  src/org/python/antlr/ast/AssignDerived.java                              |  16 +++------
  src/org/python/antlr/ast/AttributeDerived.java                           |  16 +++------
  src/org/python/antlr/ast/AugAssignDerived.java                           |  16 +++------
  src/org/python/antlr/ast/BinOpDerived.java                               |  16 +++------
  src/org/python/antlr/ast/BoolOpDerived.java                              |  16 +++------
  src/org/python/antlr/ast/BreakDerived.java                               |  16 +++------
  src/org/python/antlr/ast/CallDerived.java                                |  16 +++------
  src/org/python/antlr/ast/ClassDefDerived.java                            |  16 +++------
  src/org/python/antlr/ast/CompareDerived.java                             |  16 +++------
  src/org/python/antlr/ast/ContinueDerived.java                            |  16 +++------
  src/org/python/antlr/ast/DeleteDerived.java                              |  16 +++------
  src/org/python/antlr/ast/DictDerived.java                                |  16 +++------
  src/org/python/antlr/ast/EllipsisDerived.java                            |  16 +++------
  src/org/python/antlr/ast/ExceptHandlerDerived.java                       |  16 +++------
  src/org/python/antlr/ast/ExecDerived.java                                |  16 +++------
  src/org/python/antlr/ast/ExprDerived.java                                |  16 +++------
  src/org/python/antlr/ast/ExpressionDerived.java                          |  16 +++------
  src/org/python/antlr/ast/ExtSliceDerived.java                            |  16 +++------
  src/org/python/antlr/ast/ForDerived.java                                 |  16 +++------
  src/org/python/antlr/ast/FunctionDefDerived.java                         |  16 +++------
  src/org/python/antlr/ast/GeneratorExpDerived.java                        |  16 +++------
  src/org/python/antlr/ast/GlobalDerived.java                              |  16 +++------
  src/org/python/antlr/ast/IfDerived.java                                  |  16 +++------
  src/org/python/antlr/ast/IfExpDerived.java                               |  16 +++------
  src/org/python/antlr/ast/ImportDerived.java                              |  16 +++------
  src/org/python/antlr/ast/ImportFromDerived.java                          |  16 +++------
  src/org/python/antlr/ast/IndexDerived.java                               |  16 +++------
  src/org/python/antlr/ast/InteractiveDerived.java                         |  16 +++------
  src/org/python/antlr/ast/LambdaDerived.java                              |  16 +++------
  src/org/python/antlr/ast/ListCompDerived.java                            |  16 +++------
  src/org/python/antlr/ast/ListDerived.java                                |  16 +++------
  src/org/python/antlr/ast/ModuleDerived.java                              |  16 +++------
  src/org/python/antlr/ast/NameDerived.java                                |  16 +++------
  src/org/python/antlr/ast/NumDerived.java                                 |  16 +++------
  src/org/python/antlr/ast/PassDerived.java                                |  16 +++------
  src/org/python/antlr/ast/PrintDerived.java                               |  16 +++------
  src/org/python/antlr/ast/RaiseDerived.java                               |  16 +++------
  src/org/python/antlr/ast/ReprDerived.java                                |  16 +++------
  src/org/python/antlr/ast/ReturnDerived.java                              |  16 +++------
  src/org/python/antlr/ast/SliceDerived.java                               |  16 +++------
  src/org/python/antlr/ast/StrDerived.java                                 |  16 +++------
  src/org/python/antlr/ast/SubscriptDerived.java                           |  16 +++------
  src/org/python/antlr/ast/SuiteDerived.java                               |  16 +++------
  src/org/python/antlr/ast/TryExceptDerived.java                           |  16 +++------
  src/org/python/antlr/ast/TryFinallyDerived.java                          |  16 +++------
  src/org/python/antlr/ast/TupleDerived.java                               |  16 +++------
  src/org/python/antlr/ast/UnaryOpDerived.java                             |  16 +++------
  src/org/python/antlr/ast/WhileDerived.java                               |  16 +++------
  src/org/python/antlr/ast/WithDerived.java                                |  16 +++------
  src/org/python/antlr/ast/YieldDerived.java                               |  16 +++------
  src/org/python/antlr/ast/aliasDerived.java                               |  16 +++------
  src/org/python/antlr/ast/argumentsDerived.java                           |  16 +++------
  src/org/python/antlr/ast/comprehensionDerived.java                       |  16 +++------
  src/org/python/antlr/ast/keywordDerived.java                             |  16 +++------
  src/org/python/antlr/op/AddDerived.java                                  |  16 +++------
  src/org/python/antlr/op/AndDerived.java                                  |  16 +++------
  src/org/python/antlr/op/AugLoadDerived.java                              |  16 +++------
  src/org/python/antlr/op/AugStoreDerived.java                             |  16 +++------
  src/org/python/antlr/op/BitAndDerived.java                               |  16 +++------
  src/org/python/antlr/op/BitOrDerived.java                                |  16 +++------
  src/org/python/antlr/op/BitXorDerived.java                               |  16 +++------
  src/org/python/antlr/op/DelDerived.java                                  |  16 +++------
  src/org/python/antlr/op/DivDerived.java                                  |  16 +++------
  src/org/python/antlr/op/EqDerived.java                                   |  16 +++------
  src/org/python/antlr/op/FloorDivDerived.java                             |  16 +++------
  src/org/python/antlr/op/GtDerived.java                                   |  16 +++------
  src/org/python/antlr/op/GtEDerived.java                                  |  16 +++------
  src/org/python/antlr/op/InDerived.java                                   |  16 +++------
  src/org/python/antlr/op/InvertDerived.java                               |  16 +++------
  src/org/python/antlr/op/IsDerived.java                                   |  16 +++------
  src/org/python/antlr/op/IsNotDerived.java                                |  16 +++------
  src/org/python/antlr/op/LShiftDerived.java                               |  16 +++------
  src/org/python/antlr/op/LoadDerived.java                                 |  16 +++------
  src/org/python/antlr/op/LtDerived.java                                   |  16 +++------
  src/org/python/antlr/op/LtEDerived.java                                  |  16 +++------
  src/org/python/antlr/op/ModDerived.java                                  |  16 +++------
  src/org/python/antlr/op/MultDerived.java                                 |  16 +++------
  src/org/python/antlr/op/NotDerived.java                                  |  16 +++------
  src/org/python/antlr/op/NotEqDerived.java                                |  16 +++------
  src/org/python/antlr/op/NotInDerived.java                                |  16 +++------
  src/org/python/antlr/op/OrDerived.java                                   |  16 +++------
  src/org/python/antlr/op/ParamDerived.java                                |  16 +++------
  src/org/python/antlr/op/PowDerived.java                                  |  16 +++------
  src/org/python/antlr/op/RShiftDerived.java                               |  16 +++------
  src/org/python/antlr/op/StoreDerived.java                                |  16 +++------
  src/org/python/antlr/op/SubDerived.java                                  |  16 +++------
  src/org/python/antlr/op/UAddDerived.java                                 |  16 +++------
  src/org/python/antlr/op/USubDerived.java                                 |  16 +++------
  src/org/python/core/ClasspathPyImporterDerived.java                      |  16 +++------
  src/org/python/core/PyArrayDerived.java                                  |  16 +++------
  src/org/python/core/PyBaseExceptionDerived.java                          |  16 +++------
  src/org/python/core/PyByteArrayDerived.java                              |  16 +++------
  src/org/python/core/PyClassMethodDerived.java                            |  16 +++------
  src/org/python/core/PyComplexDerived.java                                |  16 +++------
  src/org/python/core/PyDictionaryDerived.java                             |  16 +++------
  src/org/python/core/PyEnumerateDerived.java                              |  16 +++------
  src/org/python/core/PyFileDerived.java                                   |  16 +++------
  src/org/python/core/PyFloatDerived.java                                  |  16 +++------
  src/org/python/core/PyFrozenSetDerived.java                              |  16 +++------
  src/org/python/core/PyIntegerDerived.java                                |  16 +++------
  src/org/python/core/PyListDerived.java                                   |  16 +++------
  src/org/python/core/PyLongDerived.java                                   |  16 +++------
  src/org/python/core/PyModuleDerived.java                                 |  16 +++------
  src/org/python/core/PyObjectDerived.java                                 |  16 +++------
  src/org/python/core/PyPropertyDerived.java                               |  16 +++------
  src/org/python/core/PySetDerived.java                                    |  16 +++------
  src/org/python/core/PyStringDerived.java                                 |  16 +++------
  src/org/python/core/PySuperDerived.java                                  |  16 +++------
  src/org/python/core/PyTupleDerived.java                                  |  16 +++------
  src/org/python/core/PyTypeDerived.java                                   |  16 +++------
  src/org/python/core/PyUnicodeDerived.java                                |  16 +++------
  src/org/python/modules/PyStructDerived.java                              |  16 +++------
  src/org/python/modules/_collections/PyDefaultDictDerived.java            |  16 +++------
  src/org/python/modules/_collections/PyDequeDerived.java                  |  16 +++------
  src/org/python/modules/_csv/PyDialectDerived.java                        |  16 +++------
  src/org/python/modules/_functools/PyPartialDerived.java                  |  16 +++------
  src/org/python/modules/_io/PyFileIODerived.java                          |  16 +++------
  src/org/python/modules/_io/PyIOBaseDerived.java                          |  16 +++------
  src/org/python/modules/_io/PyRawIOBaseDerived.java                       |  16 +++------
  src/org/python/modules/_weakref/ReferenceTypeDerived.java                |  16 +++------
  src/org/python/modules/bz2/PyBZ2CompressorDerived.java                   |  16 +++------
  src/org/python/modules/bz2/PyBZ2DecompressorDerived.java                 |  16 +++------
  src/org/python/modules/bz2/PyBZ2FileDerived.java                         |  16 +++------
  src/org/python/modules/itertools/PyTeeIteratorDerived.java               |  16 +++------
  src/org/python/modules/itertools/chainDerived.java                       |  16 +++------
  src/org/python/modules/itertools/combinationsDerived.java                |  16 +++------
  src/org/python/modules/itertools/combinationsWithReplacementDerived.java |  16 +++------
  src/org/python/modules/itertools/compressDerived.java                    |  16 +++------
  src/org/python/modules/itertools/countDerived.java                       |  16 +++------
  src/org/python/modules/itertools/cycleDerived.java                       |  16 +++------
  src/org/python/modules/itertools/dropwhileDerived.java                   |  16 +++------
  src/org/python/modules/itertools/groupbyDerived.java                     |  16 +++------
  src/org/python/modules/itertools/ifilterDerived.java                     |  16 +++------
  src/org/python/modules/itertools/ifilterfalseDerived.java                |  16 +++------
  src/org/python/modules/itertools/isliceDerived.java                      |  16 +++------
  src/org/python/modules/itertools/izipDerived.java                        |  16 +++------
  src/org/python/modules/itertools/izipLongestDerived.java                 |  16 +++------
  src/org/python/modules/itertools/permutationsDerived.java                |  16 +++------
  src/org/python/modules/itertools/productDerived.java                     |  16 +++------
  src/org/python/modules/itertools/repeatDerived.java                      |  16 +++------
  src/org/python/modules/itertools/starmapDerived.java                     |  16 +++------
  src/org/python/modules/itertools/takewhileDerived.java                   |  16 +++------
  src/org/python/modules/random/PyRandomDerived.java                       |  16 +++------
  src/org/python/modules/thread/PyLocalDerived.java                        |  16 +++------
  src/org/python/modules/zipimport/zipimporterDerived.java                 |  16 +++------
  146 files changed, 730 insertions(+), 1606 deletions(-)


diff --git a/src/org/python/antlr/ast/AssertDerived.java b/src/org/python/antlr/ast/AssertDerived.java
--- a/src/org/python/antlr/ast/AssertDerived.java
+++ b/src/org/python/antlr/ast/AssertDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/AssignDerived.java b/src/org/python/antlr/ast/AssignDerived.java
--- a/src/org/python/antlr/ast/AssignDerived.java
+++ b/src/org/python/antlr/ast/AssignDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/AttributeDerived.java b/src/org/python/antlr/ast/AttributeDerived.java
--- a/src/org/python/antlr/ast/AttributeDerived.java
+++ b/src/org/python/antlr/ast/AttributeDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/AugAssignDerived.java b/src/org/python/antlr/ast/AugAssignDerived.java
--- a/src/org/python/antlr/ast/AugAssignDerived.java
+++ b/src/org/python/antlr/ast/AugAssignDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/BinOpDerived.java b/src/org/python/antlr/ast/BinOpDerived.java
--- a/src/org/python/antlr/ast/BinOpDerived.java
+++ b/src/org/python/antlr/ast/BinOpDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/BoolOpDerived.java b/src/org/python/antlr/ast/BoolOpDerived.java
--- a/src/org/python/antlr/ast/BoolOpDerived.java
+++ b/src/org/python/antlr/ast/BoolOpDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/BreakDerived.java b/src/org/python/antlr/ast/BreakDerived.java
--- a/src/org/python/antlr/ast/BreakDerived.java
+++ b/src/org/python/antlr/ast/BreakDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/CallDerived.java b/src/org/python/antlr/ast/CallDerived.java
--- a/src/org/python/antlr/ast/CallDerived.java
+++ b/src/org/python/antlr/ast/CallDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ClassDefDerived.java b/src/org/python/antlr/ast/ClassDefDerived.java
--- a/src/org/python/antlr/ast/ClassDefDerived.java
+++ b/src/org/python/antlr/ast/ClassDefDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/CompareDerived.java b/src/org/python/antlr/ast/CompareDerived.java
--- a/src/org/python/antlr/ast/CompareDerived.java
+++ b/src/org/python/antlr/ast/CompareDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ContinueDerived.java b/src/org/python/antlr/ast/ContinueDerived.java
--- a/src/org/python/antlr/ast/ContinueDerived.java
+++ b/src/org/python/antlr/ast/ContinueDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/DeleteDerived.java b/src/org/python/antlr/ast/DeleteDerived.java
--- a/src/org/python/antlr/ast/DeleteDerived.java
+++ b/src/org/python/antlr/ast/DeleteDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/DictDerived.java b/src/org/python/antlr/ast/DictDerived.java
--- a/src/org/python/antlr/ast/DictDerived.java
+++ b/src/org/python/antlr/ast/DictDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/EllipsisDerived.java b/src/org/python/antlr/ast/EllipsisDerived.java
--- a/src/org/python/antlr/ast/EllipsisDerived.java
+++ b/src/org/python/antlr/ast/EllipsisDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ExceptHandlerDerived.java b/src/org/python/antlr/ast/ExceptHandlerDerived.java
--- a/src/org/python/antlr/ast/ExceptHandlerDerived.java
+++ b/src/org/python/antlr/ast/ExceptHandlerDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ExecDerived.java b/src/org/python/antlr/ast/ExecDerived.java
--- a/src/org/python/antlr/ast/ExecDerived.java
+++ b/src/org/python/antlr/ast/ExecDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ExprDerived.java b/src/org/python/antlr/ast/ExprDerived.java
--- a/src/org/python/antlr/ast/ExprDerived.java
+++ b/src/org/python/antlr/ast/ExprDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ExpressionDerived.java b/src/org/python/antlr/ast/ExpressionDerived.java
--- a/src/org/python/antlr/ast/ExpressionDerived.java
+++ b/src/org/python/antlr/ast/ExpressionDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ExtSliceDerived.java b/src/org/python/antlr/ast/ExtSliceDerived.java
--- a/src/org/python/antlr/ast/ExtSliceDerived.java
+++ b/src/org/python/antlr/ast/ExtSliceDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ForDerived.java b/src/org/python/antlr/ast/ForDerived.java
--- a/src/org/python/antlr/ast/ForDerived.java
+++ b/src/org/python/antlr/ast/ForDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/FunctionDefDerived.java b/src/org/python/antlr/ast/FunctionDefDerived.java
--- a/src/org/python/antlr/ast/FunctionDefDerived.java
+++ b/src/org/python/antlr/ast/FunctionDefDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/GeneratorExpDerived.java b/src/org/python/antlr/ast/GeneratorExpDerived.java
--- a/src/org/python/antlr/ast/GeneratorExpDerived.java
+++ b/src/org/python/antlr/ast/GeneratorExpDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/GlobalDerived.java b/src/org/python/antlr/ast/GlobalDerived.java
--- a/src/org/python/antlr/ast/GlobalDerived.java
+++ b/src/org/python/antlr/ast/GlobalDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/IfDerived.java b/src/org/python/antlr/ast/IfDerived.java
--- a/src/org/python/antlr/ast/IfDerived.java
+++ b/src/org/python/antlr/ast/IfDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/IfExpDerived.java b/src/org/python/antlr/ast/IfExpDerived.java
--- a/src/org/python/antlr/ast/IfExpDerived.java
+++ b/src/org/python/antlr/ast/IfExpDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ImportDerived.java b/src/org/python/antlr/ast/ImportDerived.java
--- a/src/org/python/antlr/ast/ImportDerived.java
+++ b/src/org/python/antlr/ast/ImportDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ImportFromDerived.java b/src/org/python/antlr/ast/ImportFromDerived.java
--- a/src/org/python/antlr/ast/ImportFromDerived.java
+++ b/src/org/python/antlr/ast/ImportFromDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/IndexDerived.java b/src/org/python/antlr/ast/IndexDerived.java
--- a/src/org/python/antlr/ast/IndexDerived.java
+++ b/src/org/python/antlr/ast/IndexDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/InteractiveDerived.java b/src/org/python/antlr/ast/InteractiveDerived.java
--- a/src/org/python/antlr/ast/InteractiveDerived.java
+++ b/src/org/python/antlr/ast/InteractiveDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/LambdaDerived.java b/src/org/python/antlr/ast/LambdaDerived.java
--- a/src/org/python/antlr/ast/LambdaDerived.java
+++ b/src/org/python/antlr/ast/LambdaDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ListCompDerived.java b/src/org/python/antlr/ast/ListCompDerived.java
--- a/src/org/python/antlr/ast/ListCompDerived.java
+++ b/src/org/python/antlr/ast/ListCompDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ListDerived.java b/src/org/python/antlr/ast/ListDerived.java
--- a/src/org/python/antlr/ast/ListDerived.java
+++ b/src/org/python/antlr/ast/ListDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ModuleDerived.java b/src/org/python/antlr/ast/ModuleDerived.java
--- a/src/org/python/antlr/ast/ModuleDerived.java
+++ b/src/org/python/antlr/ast/ModuleDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/NameDerived.java b/src/org/python/antlr/ast/NameDerived.java
--- a/src/org/python/antlr/ast/NameDerived.java
+++ b/src/org/python/antlr/ast/NameDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/NumDerived.java b/src/org/python/antlr/ast/NumDerived.java
--- a/src/org/python/antlr/ast/NumDerived.java
+++ b/src/org/python/antlr/ast/NumDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/PassDerived.java b/src/org/python/antlr/ast/PassDerived.java
--- a/src/org/python/antlr/ast/PassDerived.java
+++ b/src/org/python/antlr/ast/PassDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/PrintDerived.java b/src/org/python/antlr/ast/PrintDerived.java
--- a/src/org/python/antlr/ast/PrintDerived.java
+++ b/src/org/python/antlr/ast/PrintDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/RaiseDerived.java b/src/org/python/antlr/ast/RaiseDerived.java
--- a/src/org/python/antlr/ast/RaiseDerived.java
+++ b/src/org/python/antlr/ast/RaiseDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ReprDerived.java b/src/org/python/antlr/ast/ReprDerived.java
--- a/src/org/python/antlr/ast/ReprDerived.java
+++ b/src/org/python/antlr/ast/ReprDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/ReturnDerived.java b/src/org/python/antlr/ast/ReturnDerived.java
--- a/src/org/python/antlr/ast/ReturnDerived.java
+++ b/src/org/python/antlr/ast/ReturnDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/SliceDerived.java b/src/org/python/antlr/ast/SliceDerived.java
--- a/src/org/python/antlr/ast/SliceDerived.java
+++ b/src/org/python/antlr/ast/SliceDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/StrDerived.java b/src/org/python/antlr/ast/StrDerived.java
--- a/src/org/python/antlr/ast/StrDerived.java
+++ b/src/org/python/antlr/ast/StrDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/SubscriptDerived.java b/src/org/python/antlr/ast/SubscriptDerived.java
--- a/src/org/python/antlr/ast/SubscriptDerived.java
+++ b/src/org/python/antlr/ast/SubscriptDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/SuiteDerived.java b/src/org/python/antlr/ast/SuiteDerived.java
--- a/src/org/python/antlr/ast/SuiteDerived.java
+++ b/src/org/python/antlr/ast/SuiteDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/TryExceptDerived.java b/src/org/python/antlr/ast/TryExceptDerived.java
--- a/src/org/python/antlr/ast/TryExceptDerived.java
+++ b/src/org/python/antlr/ast/TryExceptDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/TryFinallyDerived.java b/src/org/python/antlr/ast/TryFinallyDerived.java
--- a/src/org/python/antlr/ast/TryFinallyDerived.java
+++ b/src/org/python/antlr/ast/TryFinallyDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/TupleDerived.java b/src/org/python/antlr/ast/TupleDerived.java
--- a/src/org/python/antlr/ast/TupleDerived.java
+++ b/src/org/python/antlr/ast/TupleDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/UnaryOpDerived.java b/src/org/python/antlr/ast/UnaryOpDerived.java
--- a/src/org/python/antlr/ast/UnaryOpDerived.java
+++ b/src/org/python/antlr/ast/UnaryOpDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/WhileDerived.java b/src/org/python/antlr/ast/WhileDerived.java
--- a/src/org/python/antlr/ast/WhileDerived.java
+++ b/src/org/python/antlr/ast/WhileDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/WithDerived.java b/src/org/python/antlr/ast/WithDerived.java
--- a/src/org/python/antlr/ast/WithDerived.java
+++ b/src/org/python/antlr/ast/WithDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/YieldDerived.java b/src/org/python/antlr/ast/YieldDerived.java
--- a/src/org/python/antlr/ast/YieldDerived.java
+++ b/src/org/python/antlr/ast/YieldDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/aliasDerived.java b/src/org/python/antlr/ast/aliasDerived.java
--- a/src/org/python/antlr/ast/aliasDerived.java
+++ b/src/org/python/antlr/ast/aliasDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/argumentsDerived.java b/src/org/python/antlr/ast/argumentsDerived.java
--- a/src/org/python/antlr/ast/argumentsDerived.java
+++ b/src/org/python/antlr/ast/argumentsDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/comprehensionDerived.java b/src/org/python/antlr/ast/comprehensionDerived.java
--- a/src/org/python/antlr/ast/comprehensionDerived.java
+++ b/src/org/python/antlr/ast/comprehensionDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/ast/keywordDerived.java b/src/org/python/antlr/ast/keywordDerived.java
--- a/src/org/python/antlr/ast/keywordDerived.java
+++ b/src/org/python/antlr/ast/keywordDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/AddDerived.java b/src/org/python/antlr/op/AddDerived.java
--- a/src/org/python/antlr/op/AddDerived.java
+++ b/src/org/python/antlr/op/AddDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/AndDerived.java b/src/org/python/antlr/op/AndDerived.java
--- a/src/org/python/antlr/op/AndDerived.java
+++ b/src/org/python/antlr/op/AndDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/AugLoadDerived.java b/src/org/python/antlr/op/AugLoadDerived.java
--- a/src/org/python/antlr/op/AugLoadDerived.java
+++ b/src/org/python/antlr/op/AugLoadDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/AugStoreDerived.java b/src/org/python/antlr/op/AugStoreDerived.java
--- a/src/org/python/antlr/op/AugStoreDerived.java
+++ b/src/org/python/antlr/op/AugStoreDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/BitAndDerived.java b/src/org/python/antlr/op/BitAndDerived.java
--- a/src/org/python/antlr/op/BitAndDerived.java
+++ b/src/org/python/antlr/op/BitAndDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/BitOrDerived.java b/src/org/python/antlr/op/BitOrDerived.java
--- a/src/org/python/antlr/op/BitOrDerived.java
+++ b/src/org/python/antlr/op/BitOrDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/BitXorDerived.java b/src/org/python/antlr/op/BitXorDerived.java
--- a/src/org/python/antlr/op/BitXorDerived.java
+++ b/src/org/python/antlr/op/BitXorDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/DelDerived.java b/src/org/python/antlr/op/DelDerived.java
--- a/src/org/python/antlr/op/DelDerived.java
+++ b/src/org/python/antlr/op/DelDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/DivDerived.java b/src/org/python/antlr/op/DivDerived.java
--- a/src/org/python/antlr/op/DivDerived.java
+++ b/src/org/python/antlr/op/DivDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/EqDerived.java b/src/org/python/antlr/op/EqDerived.java
--- a/src/org/python/antlr/op/EqDerived.java
+++ b/src/org/python/antlr/op/EqDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/FloorDivDerived.java b/src/org/python/antlr/op/FloorDivDerived.java
--- a/src/org/python/antlr/op/FloorDivDerived.java
+++ b/src/org/python/antlr/op/FloorDivDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/GtDerived.java b/src/org/python/antlr/op/GtDerived.java
--- a/src/org/python/antlr/op/GtDerived.java
+++ b/src/org/python/antlr/op/GtDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/GtEDerived.java b/src/org/python/antlr/op/GtEDerived.java
--- a/src/org/python/antlr/op/GtEDerived.java
+++ b/src/org/python/antlr/op/GtEDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/InDerived.java b/src/org/python/antlr/op/InDerived.java
--- a/src/org/python/antlr/op/InDerived.java
+++ b/src/org/python/antlr/op/InDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/InvertDerived.java b/src/org/python/antlr/op/InvertDerived.java
--- a/src/org/python/antlr/op/InvertDerived.java
+++ b/src/org/python/antlr/op/InvertDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/IsDerived.java b/src/org/python/antlr/op/IsDerived.java
--- a/src/org/python/antlr/op/IsDerived.java
+++ b/src/org/python/antlr/op/IsDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/IsNotDerived.java b/src/org/python/antlr/op/IsNotDerived.java
--- a/src/org/python/antlr/op/IsNotDerived.java
+++ b/src/org/python/antlr/op/IsNotDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/LShiftDerived.java b/src/org/python/antlr/op/LShiftDerived.java
--- a/src/org/python/antlr/op/LShiftDerived.java
+++ b/src/org/python/antlr/op/LShiftDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/LoadDerived.java b/src/org/python/antlr/op/LoadDerived.java
--- a/src/org/python/antlr/op/LoadDerived.java
+++ b/src/org/python/antlr/op/LoadDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/LtDerived.java b/src/org/python/antlr/op/LtDerived.java
--- a/src/org/python/antlr/op/LtDerived.java
+++ b/src/org/python/antlr/op/LtDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/LtEDerived.java b/src/org/python/antlr/op/LtEDerived.java
--- a/src/org/python/antlr/op/LtEDerived.java
+++ b/src/org/python/antlr/op/LtEDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/ModDerived.java b/src/org/python/antlr/op/ModDerived.java
--- a/src/org/python/antlr/op/ModDerived.java
+++ b/src/org/python/antlr/op/ModDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/MultDerived.java b/src/org/python/antlr/op/MultDerived.java
--- a/src/org/python/antlr/op/MultDerived.java
+++ b/src/org/python/antlr/op/MultDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/NotDerived.java b/src/org/python/antlr/op/NotDerived.java
--- a/src/org/python/antlr/op/NotDerived.java
+++ b/src/org/python/antlr/op/NotDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/NotEqDerived.java b/src/org/python/antlr/op/NotEqDerived.java
--- a/src/org/python/antlr/op/NotEqDerived.java
+++ b/src/org/python/antlr/op/NotEqDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/NotInDerived.java b/src/org/python/antlr/op/NotInDerived.java
--- a/src/org/python/antlr/op/NotInDerived.java
+++ b/src/org/python/antlr/op/NotInDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/OrDerived.java b/src/org/python/antlr/op/OrDerived.java
--- a/src/org/python/antlr/op/OrDerived.java
+++ b/src/org/python/antlr/op/OrDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/ParamDerived.java b/src/org/python/antlr/op/ParamDerived.java
--- a/src/org/python/antlr/op/ParamDerived.java
+++ b/src/org/python/antlr/op/ParamDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/PowDerived.java b/src/org/python/antlr/op/PowDerived.java
--- a/src/org/python/antlr/op/PowDerived.java
+++ b/src/org/python/antlr/op/PowDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/RShiftDerived.java b/src/org/python/antlr/op/RShiftDerived.java
--- a/src/org/python/antlr/op/RShiftDerived.java
+++ b/src/org/python/antlr/op/RShiftDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/StoreDerived.java b/src/org/python/antlr/op/StoreDerived.java
--- a/src/org/python/antlr/op/StoreDerived.java
+++ b/src/org/python/antlr/op/StoreDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/SubDerived.java b/src/org/python/antlr/op/SubDerived.java
--- a/src/org/python/antlr/op/SubDerived.java
+++ b/src/org/python/antlr/op/SubDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/UAddDerived.java b/src/org/python/antlr/op/UAddDerived.java
--- a/src/org/python/antlr/op/UAddDerived.java
+++ b/src/org/python/antlr/op/UAddDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/antlr/op/USubDerived.java b/src/org/python/antlr/op/USubDerived.java
--- a/src/org/python/antlr/op/USubDerived.java
+++ b/src/org/python/antlr/op/USubDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/ClasspathPyImporterDerived.java b/src/org/python/core/ClasspathPyImporterDerived.java
--- a/src/org/python/core/ClasspathPyImporterDerived.java
+++ b/src/org/python/core/ClasspathPyImporterDerived.java
@@ -961,18 +961,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyArrayDerived.java b/src/org/python/core/PyArrayDerived.java
--- a/src/org/python/core/PyArrayDerived.java
+++ b/src/org/python/core/PyArrayDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyBaseExceptionDerived.java b/src/org/python/core/PyBaseExceptionDerived.java
--- a/src/org/python/core/PyBaseExceptionDerived.java
+++ b/src/org/python/core/PyBaseExceptionDerived.java
@@ -961,18 +961,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyByteArrayDerived.java b/src/org/python/core/PyByteArrayDerived.java
--- a/src/org/python/core/PyByteArrayDerived.java
+++ b/src/org/python/core/PyByteArrayDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyClassMethodDerived.java b/src/org/python/core/PyClassMethodDerived.java
--- a/src/org/python/core/PyClassMethodDerived.java
+++ b/src/org/python/core/PyClassMethodDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyComplexDerived.java b/src/org/python/core/PyComplexDerived.java
--- a/src/org/python/core/PyComplexDerived.java
+++ b/src/org/python/core/PyComplexDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyDictionaryDerived.java b/src/org/python/core/PyDictionaryDerived.java
--- a/src/org/python/core/PyDictionaryDerived.java
+++ b/src/org/python/core/PyDictionaryDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyEnumerateDerived.java b/src/org/python/core/PyEnumerateDerived.java
--- a/src/org/python/core/PyEnumerateDerived.java
+++ b/src/org/python/core/PyEnumerateDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyFileDerived.java b/src/org/python/core/PyFileDerived.java
--- a/src/org/python/core/PyFileDerived.java
+++ b/src/org/python/core/PyFileDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyFloatDerived.java b/src/org/python/core/PyFloatDerived.java
--- a/src/org/python/core/PyFloatDerived.java
+++ b/src/org/python/core/PyFloatDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyFrozenSetDerived.java b/src/org/python/core/PyFrozenSetDerived.java
--- a/src/org/python/core/PyFrozenSetDerived.java
+++ b/src/org/python/core/PyFrozenSetDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyIntegerDerived.java b/src/org/python/core/PyIntegerDerived.java
--- a/src/org/python/core/PyIntegerDerived.java
+++ b/src/org/python/core/PyIntegerDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyListDerived.java b/src/org/python/core/PyListDerived.java
--- a/src/org/python/core/PyListDerived.java
+++ b/src/org/python/core/PyListDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyLongDerived.java b/src/org/python/core/PyLongDerived.java
--- a/src/org/python/core/PyLongDerived.java
+++ b/src/org/python/core/PyLongDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyModuleDerived.java b/src/org/python/core/PyModuleDerived.java
--- a/src/org/python/core/PyModuleDerived.java
+++ b/src/org/python/core/PyModuleDerived.java
@@ -961,18 +961,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyObjectDerived.java b/src/org/python/core/PyObjectDerived.java
--- a/src/org/python/core/PyObjectDerived.java
+++ b/src/org/python/core/PyObjectDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyPropertyDerived.java b/src/org/python/core/PyPropertyDerived.java
--- a/src/org/python/core/PyPropertyDerived.java
+++ b/src/org/python/core/PyPropertyDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PySetDerived.java b/src/org/python/core/PySetDerived.java
--- a/src/org/python/core/PySetDerived.java
+++ b/src/org/python/core/PySetDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyStringDerived.java b/src/org/python/core/PyStringDerived.java
--- a/src/org/python/core/PyStringDerived.java
+++ b/src/org/python/core/PyStringDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PySuperDerived.java b/src/org/python/core/PySuperDerived.java
--- a/src/org/python/core/PySuperDerived.java
+++ b/src/org/python/core/PySuperDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyTupleDerived.java b/src/org/python/core/PyTupleDerived.java
--- a/src/org/python/core/PyTupleDerived.java
+++ b/src/org/python/core/PyTupleDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyTypeDerived.java b/src/org/python/core/PyTypeDerived.java
--- a/src/org/python/core/PyTypeDerived.java
+++ b/src/org/python/core/PyTypeDerived.java
@@ -961,18 +961,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/core/PyUnicodeDerived.java b/src/org/python/core/PyUnicodeDerived.java
--- a/src/org/python/core/PyUnicodeDerived.java
+++ b/src/org/python/core/PyUnicodeDerived.java
@@ -985,18 +985,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/PyStructDerived.java b/src/org/python/modules/PyStructDerived.java
--- a/src/org/python/modules/PyStructDerived.java
+++ b/src/org/python/modules/PyStructDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/_collections/PyDefaultDictDerived.java b/src/org/python/modules/_collections/PyDefaultDictDerived.java
--- a/src/org/python/modules/_collections/PyDefaultDictDerived.java
+++ b/src/org/python/modules/_collections/PyDefaultDictDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/_collections/PyDequeDerived.java b/src/org/python/modules/_collections/PyDequeDerived.java
--- a/src/org/python/modules/_collections/PyDequeDerived.java
+++ b/src/org/python/modules/_collections/PyDequeDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/_csv/PyDialectDerived.java b/src/org/python/modules/_csv/PyDialectDerived.java
--- a/src/org/python/modules/_csv/PyDialectDerived.java
+++ b/src/org/python/modules/_csv/PyDialectDerived.java
@@ -962,18 +962,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/_functools/PyPartialDerived.java b/src/org/python/modules/_functools/PyPartialDerived.java
--- a/src/org/python/modules/_functools/PyPartialDerived.java
+++ b/src/org/python/modules/_functools/PyPartialDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/_io/PyFileIODerived.java b/src/org/python/modules/_io/PyFileIODerived.java
--- a/src/org/python/modules/_io/PyFileIODerived.java
+++ b/src/org/python/modules/_io/PyFileIODerived.java
@@ -962,18 +962,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/_io/PyIOBaseDerived.java b/src/org/python/modules/_io/PyIOBaseDerived.java
--- a/src/org/python/modules/_io/PyIOBaseDerived.java
+++ b/src/org/python/modules/_io/PyIOBaseDerived.java
@@ -962,18 +962,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/_io/PyRawIOBaseDerived.java b/src/org/python/modules/_io/PyRawIOBaseDerived.java
--- a/src/org/python/modules/_io/PyRawIOBaseDerived.java
+++ b/src/org/python/modules/_io/PyRawIOBaseDerived.java
@@ -962,18 +962,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/_weakref/ReferenceTypeDerived.java b/src/org/python/modules/_weakref/ReferenceTypeDerived.java
--- a/src/org/python/modules/_weakref/ReferenceTypeDerived.java
+++ b/src/org/python/modules/_weakref/ReferenceTypeDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/bz2/PyBZ2CompressorDerived.java b/src/org/python/modules/bz2/PyBZ2CompressorDerived.java
--- a/src/org/python/modules/bz2/PyBZ2CompressorDerived.java
+++ b/src/org/python/modules/bz2/PyBZ2CompressorDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/bz2/PyBZ2DecompressorDerived.java b/src/org/python/modules/bz2/PyBZ2DecompressorDerived.java
--- a/src/org/python/modules/bz2/PyBZ2DecompressorDerived.java
+++ b/src/org/python/modules/bz2/PyBZ2DecompressorDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/bz2/PyBZ2FileDerived.java b/src/org/python/modules/bz2/PyBZ2FileDerived.java
--- a/src/org/python/modules/bz2/PyBZ2FileDerived.java
+++ b/src/org/python/modules/bz2/PyBZ2FileDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/PyTeeIteratorDerived.java b/src/org/python/modules/itertools/PyTeeIteratorDerived.java
--- a/src/org/python/modules/itertools/PyTeeIteratorDerived.java
+++ b/src/org/python/modules/itertools/PyTeeIteratorDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/chainDerived.java b/src/org/python/modules/itertools/chainDerived.java
--- a/src/org/python/modules/itertools/chainDerived.java
+++ b/src/org/python/modules/itertools/chainDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/combinationsDerived.java b/src/org/python/modules/itertools/combinationsDerived.java
--- a/src/org/python/modules/itertools/combinationsDerived.java
+++ b/src/org/python/modules/itertools/combinationsDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/combinationsWithReplacementDerived.java b/src/org/python/modules/itertools/combinationsWithReplacementDerived.java
--- a/src/org/python/modules/itertools/combinationsWithReplacementDerived.java
+++ b/src/org/python/modules/itertools/combinationsWithReplacementDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/compressDerived.java b/src/org/python/modules/itertools/compressDerived.java
--- a/src/org/python/modules/itertools/compressDerived.java
+++ b/src/org/python/modules/itertools/compressDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/countDerived.java b/src/org/python/modules/itertools/countDerived.java
--- a/src/org/python/modules/itertools/countDerived.java
+++ b/src/org/python/modules/itertools/countDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/cycleDerived.java b/src/org/python/modules/itertools/cycleDerived.java
--- a/src/org/python/modules/itertools/cycleDerived.java
+++ b/src/org/python/modules/itertools/cycleDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/dropwhileDerived.java b/src/org/python/modules/itertools/dropwhileDerived.java
--- a/src/org/python/modules/itertools/dropwhileDerived.java
+++ b/src/org/python/modules/itertools/dropwhileDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/groupbyDerived.java b/src/org/python/modules/itertools/groupbyDerived.java
--- a/src/org/python/modules/itertools/groupbyDerived.java
+++ b/src/org/python/modules/itertools/groupbyDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/ifilterDerived.java b/src/org/python/modules/itertools/ifilterDerived.java
--- a/src/org/python/modules/itertools/ifilterDerived.java
+++ b/src/org/python/modules/itertools/ifilterDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/ifilterfalseDerived.java b/src/org/python/modules/itertools/ifilterfalseDerived.java
--- a/src/org/python/modules/itertools/ifilterfalseDerived.java
+++ b/src/org/python/modules/itertools/ifilterfalseDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/isliceDerived.java b/src/org/python/modules/itertools/isliceDerived.java
--- a/src/org/python/modules/itertools/isliceDerived.java
+++ b/src/org/python/modules/itertools/isliceDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/izipDerived.java b/src/org/python/modules/itertools/izipDerived.java
--- a/src/org/python/modules/itertools/izipDerived.java
+++ b/src/org/python/modules/itertools/izipDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/izipLongestDerived.java b/src/org/python/modules/itertools/izipLongestDerived.java
--- a/src/org/python/modules/itertools/izipLongestDerived.java
+++ b/src/org/python/modules/itertools/izipLongestDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/permutationsDerived.java b/src/org/python/modules/itertools/permutationsDerived.java
--- a/src/org/python/modules/itertools/permutationsDerived.java
+++ b/src/org/python/modules/itertools/permutationsDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/productDerived.java b/src/org/python/modules/itertools/productDerived.java
--- a/src/org/python/modules/itertools/productDerived.java
+++ b/src/org/python/modules/itertools/productDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/repeatDerived.java b/src/org/python/modules/itertools/repeatDerived.java
--- a/src/org/python/modules/itertools/repeatDerived.java
+++ b/src/org/python/modules/itertools/repeatDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/starmapDerived.java b/src/org/python/modules/itertools/starmapDerived.java
--- a/src/org/python/modules/itertools/starmapDerived.java
+++ b/src/org/python/modules/itertools/starmapDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/itertools/takewhileDerived.java b/src/org/python/modules/itertools/takewhileDerived.java
--- a/src/org/python/modules/itertools/takewhileDerived.java
+++ b/src/org/python/modules/itertools/takewhileDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/random/PyRandomDerived.java b/src/org/python/modules/random/PyRandomDerived.java
--- a/src/org/python/modules/random/PyRandomDerived.java
+++ b/src/org/python/modules/random/PyRandomDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/thread/PyLocalDerived.java b/src/org/python/modules/thread/PyLocalDerived.java
--- a/src/org/python/modules/thread/PyLocalDerived.java
+++ b/src/org/python/modules/thread/PyLocalDerived.java
@@ -962,18 +962,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {
diff --git a/src/org/python/modules/zipimport/zipimporterDerived.java b/src/org/python/modules/zipimport/zipimporterDerived.java
--- a/src/org/python/modules/zipimport/zipimporterDerived.java
+++ b/src/org/python/modules/zipimport/zipimporterDerived.java
@@ -986,18 +986,12 @@
     }
 
     public PyObject __call__(PyObject args[],String keywords[]) {
-        ThreadState ts=Py.getThreadState();
-        if (ts.recursion_depth++>ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            PyType self_type=getType();
-            PyObject impl=self_type.lookup("__call__");
-            if (impl!=null)
-                return impl.__get__(this,self_type).__call__(args,keywords);
-            return super.__call__(args,keywords);
-        } finally {
-            --ts.recursion_depth;
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__call__");
+        if (impl!=null) {
+            return impl.__get__(this,self_type).__call__(args,keywords);
         }
+        return super.__call__(args,keywords);
     }
 
     public PyObject __findattr_ex__(String name) {

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:14 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:14 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_recursion=5Fcount_ma?=
	=?utf-8?q?nagement?=
Message-ID: <3h0cmG01G3z7Lk7@mail.python.org>

http://hg.python.org/jython/rev/be1466db8edd
changeset:   7324:be1466db8edd
user:        Jim Baker 
date:        Tue Jun 24 22:53:10 2014 -0600
summary:
  Remove recursion_count management

files:
  src/org/python/core/Py.java          |   34 +-
  src/org/python/core/PyInstance.java  |  709 +++++---------
  src/org/python/core/ThreadState.java |   11 -
  3 files changed, 247 insertions(+), 507 deletions(-)


diff --git a/src/org/python/core/Py.java b/src/org/python/core/Py.java
--- a/src/org/python/core/Py.java
+++ b/src/org/python/core/Py.java
@@ -1987,16 +1987,10 @@
         }
 
         if (cls instanceof PyTuple) {
-            ThreadState threadState = Py.getThreadState();
-            threadState.enterRecursiveCall(" in __subclasscheck__");
-            try {
-                for (PyObject item : cls.asIterable()) {
-                    if (isInstance(inst, item)) {
-                        return true;
-                    }
+            for (PyObject item : cls.asIterable()) {
+                if (isInstance(inst, item)) {
+                    return true;
                 }
-            } finally {
-                threadState.leaveRecursiveCall();
             }
             return false;
         }
@@ -2047,16 +2041,10 @@
 
     public static boolean isSubClass(PyObject derived, PyObject cls) {
         if (cls instanceof PyTuple) {
-            ThreadState threadState = Py.getThreadState();
-            threadState.enterRecursiveCall(" in __subclasscheck__");
-            try {
-                for (PyObject item : cls.asIterable()) {
-                    if (isSubClass(derived, item)) {
-                        return true;
-                    }
+            for (PyObject item : cls.asIterable()) {
+                if (isSubClass(derived, item)) {
+                    return true;
                 }
-            } finally {
-                threadState.leaveRecursiveCall();
             }
             return false;
         }
@@ -2149,15 +2137,7 @@
             return null;
         }
 
-        PyObject result;
-        ThreadState threadState = Py.getThreadState();
-        threadState.enterRecursiveCall(" in " + checkerName);
-        try {
-            result = checker.__call__(checkerArg);
-        } finally {
-            threadState.leaveRecursiveCall();
-        }
-        return result;
+        return checker.__call__(checkerArg);
     }
 
     /**
diff --git a/src/org/python/core/PyInstance.java b/src/org/python/core/PyInstance.java
--- a/src/org/python/core/PyInstance.java
+++ b/src/org/python/core/PyInstance.java
@@ -17,7 +17,9 @@
     // xxx doc, final name
     public transient PyClass instclass;
 
-    /** The namespace of this instance.  Contains all instance attributes. */
+    /**
+     * The namespace of this instance.  Contains all instance attributes.
+     */
     public PyObject __dict__;
 
     public PyInstance() {
@@ -41,7 +43,7 @@
     public static PyObject instance___new__(PyNewWrapper new_, boolean init, PyType subtype,
                                             PyObject[] args, String[] keywords) {
         ArgParser ap = new ArgParser("instance", args, keywords, "name", "bases", "dict");
-        PyClass klass = (PyClass)ap.getPyObjectByType(0, PyClass.TYPE);
+        PyClass klass = (PyClass) ap.getPyObjectByType(0, PyClass.TYPE);
         PyObject dict = ap.getPyObject(1, Py.None);
         if (dict == Py.None) {
             dict = null;
@@ -57,8 +59,7 @@
 
     /* Override serialization behavior */
     private void readObject(java.io.ObjectInputStream in)
-        throws java.io.IOException, ClassNotFoundException
-    {
+            throws java.io.IOException, ClassNotFoundException {
         in.defaultReadObject();
 
         String module = in.readUTF();
@@ -67,20 +68,19 @@
         /* Check for types and missing members here */
         //System.out.println("module: "+module+", "+name);
         PyObject mod = imp.importName(module.intern(), false);
-        PyClass pyc = (PyClass)mod.__getattr__(name.intern());
+        PyClass pyc = (PyClass) mod.__getattr__(name.intern());
 
         instclass = pyc;
     }
 
     private void writeObject(java.io.ObjectOutputStream out)
-        throws java.io.IOException
-    {
+            throws java.io.IOException {
         //System.out.println("writing: "+getClass().getName());
         out.defaultWriteObject();
         PyObject name = instclass.__findattr__("__module__");
         if (!(name instanceof PyString) || name == Py.None) {
-            throw Py.ValueError("Can't find module for class: "+
-                                instclass.__name__);
+            throw Py.ValueError("Can't find module for class: " +
+                    instclass.__name__);
         }
         out.writeUTF(name.toString());
         name = instclass.__findattr__("__name__");
@@ -128,8 +128,7 @@
                     throw Py.TypeError("this constructor takes no arguments");
                 }
             }
-        }
-        else if (ret != Py.None) {
+        } else if (ret != Py.None) {
             throw Py.TypeError("__init__() should return None");
         }
     }
@@ -250,7 +249,7 @@
     @Override
     public void noAttributeError(String name) {
         throw Py.AttributeError(String.format("%.50s instance has no attribute '%.400s'",
-                                              instclass.__name__, name));
+                instclass.__name__, name));
     }
 
 
@@ -263,7 +262,7 @@
     final void instance___setattr__(String name, PyObject value) {
         if (name == "__class__") {
             if (value instanceof PyClass) {
-                instclass = (PyClass)value;
+                instclass = (PyClass) value;
             } else {
                 throw Py.TypeError("__class__ must be set to a class");
             }
@@ -305,13 +304,13 @@
             } catch (PyException exc) {
                 if (exc.match(Py.KeyError))
                     throw Py.AttributeError("class " + instclass.__name__ +
-                                        " has no attribute '" + name + "'");
-            };
+                            " has no attribute '" + name + "'");
+            }
+            ;
         }
     }
 
-    public PyObject invoke_ex(String name, PyObject[] args, String[] keywords)
-    {
+    public PyObject invoke_ex(String name, PyObject[] args, String[] keywords) {
         PyObject meth = __findattr__(name);
         if (meth == null)
             return null;
@@ -324,12 +323,14 @@
             return null;
         return meth.__call__();
     }
+
     public PyObject invoke_ex(String name, PyObject arg1) {
         PyObject meth = __findattr__(name);
         if (meth == null)
             return null;
         return meth.__call__(arg1);
     }
+
     public PyObject invoke_ex(String name, PyObject arg1, PyObject arg2) {
         PyObject meth = __findattr__(name);
         if (meth == null)
@@ -344,14 +345,7 @@
 
     @ExposedMethod
     final PyObject instance___call__(PyObject args[], String keywords[]) {
-        ThreadState ts = Py.getThreadState();
-        if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-            throw Py.RuntimeError("maximum __call__ recursion depth exceeded");
-        try {
-            return invoke("__call__", args, keywords);
-        } finally {
-            --ts.recursion_depth;
-        }
+        return invoke("__call__", args, keywords);
     }
 
     @Override
@@ -372,7 +366,7 @@
         }
         if (!(ret instanceof PyString))
             throw Py.TypeError("__repr__ method must return a string");
-        return (PyString)ret;
+        return (PyString) ret;
     }
 
     /**
@@ -383,7 +377,7 @@
         PyObject mod = instclass.__dict__.__finditem__("__module__");
         String modStr = (mod == null || !Py.isInstance(mod, PyString.TYPE)) ? "?" : mod.toString();
         return new PyString(String.format("<%s.%s instance at %s>", modStr, instclass.__name__,
-                                          Py.idstr(this)));
+                Py.idstr(this)));
     }
 
     @Override
@@ -398,7 +392,7 @@
             return __repr__();
         if (!(ret instanceof PyString))
             throw Py.TypeError("__str__ method must return a string");
-        return (PyString)ret;
+        return (PyString) ret;
     }
 
     @Override
@@ -409,12 +403,12 @@
     @ExposedMethod
     final PyUnicode instance___unicode__() {
         PyObject ret = invoke_ex("__unicode__");
-        if(ret == null) {
+        if (ret == null) {
             return super.__unicode__();
-        } else if(ret instanceof PyUnicode) {
-            return (PyUnicode)ret;
-        } else if(ret instanceof PyString) {
-            return new PyUnicode((PyString)ret);
+        } else if (ret instanceof PyUnicode) {
+            return (PyUnicode) ret;
+        } else if (ret instanceof PyString) {
+            return new PyUnicode((PyString) ret);
         } else {
             throw Py.TypeError("__unicode__ must return unicode or str");
         }
@@ -431,12 +425,11 @@
             return super.hashCode();
         }
         if (ret instanceof PyInteger) {
-            return ((PyInteger)ret).getValue();
+            return ((PyInteger) ret).getValue();
+        } else if (ret instanceof PyLong) {
+            return ((PyLong) ret).hashCode();
         }
-        else if (ret instanceof PyLong) {
-            return ((PyLong)ret).hashCode();
-        }
-        throw Py.TypeError("__hash__() must really return int" + ret.getType() );
+        throw Py.TypeError("__hash__() must really return int" + ret.getType());
     }
 
     @Override
@@ -462,20 +455,20 @@
             w = other;
         }
         if (v instanceof PyInstance) {
-            ret = ((PyInstance)v).invoke_ex("__cmp__",w);
+            ret = ((PyInstance) v).invoke_ex("__cmp__", w);
             if (ret != null) {
                 if (ret instanceof PyInteger) {
-                    int result = ((PyInteger)ret).getValue();
+                    int result = ((PyInteger) ret).getValue();
                     return result < 0 ? -1 : result > 0 ? 1 : 0;
                 }
                 throw Py.TypeError("__cmp__() must return int");
             }
         }
         if (w instanceof PyInstance) {
-            ret = ((PyInstance)w).invoke_ex("__cmp__",v);
+            ret = ((PyInstance) w).invoke_ex("__cmp__", v);
             if (ret != null) {
                 if (ret instanceof PyInteger) {
-                    int result = ((PyInteger)ret).getValue();
+                    int result = ((PyInteger) ret).getValue();
                     return -(result < 0 ? -1 : result > 0 ? 1 : 0);
                 }
                 throw Py.TypeError("__cmp__() must return int");
@@ -562,12 +555,14 @@
         PyObject meth = null;
         try {
             meth = __findattr__("__nonzero__");
-        } catch (PyException exc) { }
+        } catch (PyException exc) {
+        }
 
         if (meth == null) {
             try {
                 meth = __findattr__("__len__");
-            } catch (PyException exc) { }
+            } catch (PyException exc) {
+            }
             if (meth == null) {
                 return true;
             }
@@ -586,7 +581,7 @@
     final int instance___len__() {
         PyObject ret = invoke("__len__");
         if (ret instanceof PyInteger)
-            return ((PyInteger)ret).getValue();
+            return ((PyInteger) ret).getValue();
         throw Py.TypeError("__len__() should return an int");
     }
 
@@ -745,7 +740,7 @@
     final boolean instance___contains__(PyObject o) {
         PyObject func = __findattr__("__contains__");
         if (func == null)
-           return super.__contains__(o);
+            return super.__contains__(o);
         PyObject ret = func.__call__(o);
         return ret.__nonzero__();
     }
@@ -757,7 +752,7 @@
             return ret;
         if (!(ret instanceof PyTuple))
             throw Py.TypeError("coercion should return None or 2-tuple");
-        return ((PyTuple)ret).getArray();
+        return ((PyTuple) ret).getArray();
     }
 
     @Override
@@ -768,7 +763,7 @@
     /**
      * Implements the __index__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod
     final PyObject instance___index__() {
         PyObject ret;
@@ -784,7 +779,7 @@
             return ret;
         }
         throw Py.TypeError(String.format("__index__ returned non-(int,long) (type %s)",
-                                         ret.getType().fastGetName()));
+                ret.getType().fastGetName()));
     }
 
     @Override
@@ -796,7 +791,7 @@
     final PyObject instance___format__(PyObject formatSpec) {
         PyObject func = __findattr__("__format__");
         if (func == null) {
-           return super.__format__(formatSpec);
+            return super.__format__(formatSpec);
         }
         return func.__call__(formatSpec);
     }
@@ -813,12 +808,12 @@
     /**
      * Implements the __hex__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod
     final PyString instance___hex__() {
         PyObject ret = invoke("__hex__");
         if (ret instanceof PyString)
-            return (PyString)ret;
+            return (PyString) ret;
         throw Py.TypeError("__hex__() should return a string");
     }
 
@@ -830,12 +825,12 @@
     /**
      * Implements the __oct__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod
     final PyString instance___oct__() {
         PyObject ret = invoke("__oct__");
         if (ret instanceof PyString)
-            return (PyString)ret;
+            return (PyString) ret;
         throw Py.TypeError("__oct__() should return a string");
     }
 
@@ -847,7 +842,7 @@
     /**
      * Implements the __int__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod
     final PyObject instance___int__() {
         PyObject ret = invoke("__int__");
@@ -864,12 +859,12 @@
     /**
      * Implements the __float__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod
     final PyFloat instance___float__() {
         PyObject ret = invoke("__float__");
         if (ret instanceof PyFloat)
-            return (PyFloat)ret;
+            return (PyFloat) ret;
         throw Py.TypeError("__float__() should return a float");
     }
 
@@ -881,7 +876,7 @@
     /**
      * Implements the __long__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod
     final PyObject instance___long__() {
         PyObject ret = invoke("__long__");
@@ -898,12 +893,12 @@
     /**
      * Implements the __complex__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod
     final PyComplex instance___complex__() {
         PyObject ret = invoke("__complex__");
         if (ret instanceof PyComplex)
-            return (PyComplex)ret;
+            return (PyComplex) ret;
         throw Py.TypeError("__complex__() should return a complex");
     }
 
@@ -915,7 +910,7 @@
     /**
      * Implements the __pos__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod
     public PyObject instance___pos__() {
         return invoke("__pos__");
@@ -929,7 +924,7 @@
     /**
      * Implements the __neg__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod
     public PyObject instance___neg__() {
         return invoke("__neg__");
@@ -943,7 +938,7 @@
     /**
      * Implements the __abs__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod
     public PyObject instance___abs__() {
         return invoke("__abs__");
@@ -957,7 +952,7 @@
     /**
      * Implements the __invert__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod
     public PyObject instance___invert__() {
         return invoke("__invert__");
@@ -973,28 +968,20 @@
     /**
      * Implements the __add__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___add__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__add__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__add__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._add(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._add(o2);
             }
         }
     }
@@ -1007,28 +994,20 @@
     /**
      * Implements the __radd__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___radd__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__radd__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__radd__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._add(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._add(o1);
             }
         }
     }
@@ -1041,7 +1020,7 @@
     /**
      * Implements the __iadd__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___iadd__(PyObject o) {
         PyObject ret = invoke_ex("__iadd__", o);
@@ -1058,28 +1037,20 @@
     /**
      * Implements the __sub__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___sub__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__sub__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__sub__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._sub(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._sub(o2);
             }
         }
     }
@@ -1092,28 +1063,20 @@
     /**
      * Implements the __rsub__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rsub__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rsub__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rsub__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._sub(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._sub(o1);
             }
         }
     }
@@ -1126,7 +1089,7 @@
     /**
      * Implements the __isub__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___isub__(PyObject o) {
         PyObject ret = invoke_ex("__isub__", o);
@@ -1143,28 +1106,20 @@
     /**
      * Implements the __mul__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___mul__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__mul__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__mul__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._mul(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._mul(o2);
             }
         }
     }
@@ -1177,28 +1132,20 @@
     /**
      * Implements the __rmul__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rmul__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rmul__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rmul__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._mul(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._mul(o1);
             }
         }
     }
@@ -1211,7 +1158,7 @@
     /**
      * Implements the __imul__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___imul__(PyObject o) {
         PyObject ret = invoke_ex("__imul__", o);
@@ -1228,28 +1175,20 @@
     /**
      * Implements the __div__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___div__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__div__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__div__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._div(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._div(o2);
             }
         }
     }
@@ -1262,28 +1201,20 @@
     /**
      * Implements the __rdiv__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rdiv__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rdiv__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rdiv__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._div(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._div(o1);
             }
         }
     }
@@ -1296,7 +1227,7 @@
     /**
      * Implements the __idiv__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___idiv__(PyObject o) {
         PyObject ret = invoke_ex("__idiv__", o);
@@ -1313,28 +1244,20 @@
     /**
      * Implements the __floordiv__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___floordiv__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__floordiv__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__floordiv__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._floordiv(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._floordiv(o2);
             }
         }
     }
@@ -1347,28 +1270,20 @@
     /**
      * Implements the __rfloordiv__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rfloordiv__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rfloordiv__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rfloordiv__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._floordiv(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._floordiv(o1);
             }
         }
     }
@@ -1381,7 +1296,7 @@
     /**
      * Implements the __ifloordiv__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___ifloordiv__(PyObject o) {
         PyObject ret = invoke_ex("__ifloordiv__", o);
@@ -1398,28 +1313,20 @@
     /**
      * Implements the __truediv__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___truediv__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__truediv__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__truediv__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._truediv(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._truediv(o2);
             }
         }
     }
@@ -1432,28 +1339,20 @@
     /**
      * Implements the __rtruediv__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rtruediv__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rtruediv__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rtruediv__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._truediv(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._truediv(o1);
             }
         }
     }
@@ -1466,7 +1365,7 @@
     /**
      * Implements the __itruediv__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___itruediv__(PyObject o) {
         PyObject ret = invoke_ex("__itruediv__", o);
@@ -1483,28 +1382,20 @@
     /**
      * Implements the __mod__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___mod__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__mod__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__mod__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._mod(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._mod(o2);
             }
         }
     }
@@ -1517,28 +1408,20 @@
     /**
      * Implements the __rmod__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rmod__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rmod__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rmod__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._mod(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._mod(o1);
             }
         }
     }
@@ -1551,7 +1434,7 @@
     /**
      * Implements the __imod__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___imod__(PyObject o) {
         PyObject ret = invoke_ex("__imod__", o);
@@ -1568,28 +1451,20 @@
     /**
      * Implements the __divmod__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___divmod__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__divmod__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__divmod__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._divmod(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._divmod(o2);
             }
         }
     }
@@ -1602,28 +1477,20 @@
     /**
      * Implements the __rdivmod__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rdivmod__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rdivmod__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rdivmod__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._divmod(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._divmod(o1);
             }
         }
     }
@@ -1636,28 +1503,20 @@
     /**
      * Implements the __pow__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___pow__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__pow__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__pow__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._pow(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._pow(o2);
             }
         }
     }
@@ -1670,28 +1529,20 @@
     /**
      * Implements the __rpow__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rpow__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rpow__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rpow__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._pow(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._pow(o1);
             }
         }
     }
@@ -1704,7 +1555,7 @@
     /**
      * Implements the __ipow__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___ipow__(PyObject o) {
         PyObject ret = invoke_ex("__ipow__", o);
@@ -1721,28 +1572,20 @@
     /**
      * Implements the __lshift__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___lshift__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__lshift__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__lshift__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._lshift(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._lshift(o2);
             }
         }
     }
@@ -1755,28 +1598,20 @@
     /**
      * Implements the __rlshift__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rlshift__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rlshift__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rlshift__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._lshift(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._lshift(o1);
             }
         }
     }
@@ -1789,7 +1624,7 @@
     /**
      * Implements the __ilshift__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___ilshift__(PyObject o) {
         PyObject ret = invoke_ex("__ilshift__", o);
@@ -1806,28 +1641,20 @@
     /**
      * Implements the __rshift__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rshift__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rshift__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rshift__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._rshift(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._rshift(o2);
             }
         }
     }
@@ -1840,28 +1667,20 @@
     /**
      * Implements the __rrshift__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rrshift__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rrshift__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rrshift__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._rshift(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._rshift(o1);
             }
         }
     }
@@ -1874,7 +1693,7 @@
     /**
      * Implements the __irshift__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___irshift__(PyObject o) {
         PyObject ret = invoke_ex("__irshift__", o);
@@ -1891,28 +1710,20 @@
     /**
      * Implements the __and__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___and__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__and__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__and__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._and(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._and(o2);
             }
         }
     }
@@ -1925,28 +1736,20 @@
     /**
      * Implements the __rand__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rand__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rand__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rand__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._and(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._and(o1);
             }
         }
     }
@@ -1959,7 +1762,7 @@
     /**
      * Implements the __iand__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___iand__(PyObject o) {
         PyObject ret = invoke_ex("__iand__", o);
@@ -1976,28 +1779,20 @@
     /**
      * Implements the __or__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___or__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__or__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__or__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._or(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._or(o2);
             }
         }
     }
@@ -2010,28 +1805,20 @@
     /**
      * Implements the __ror__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___ror__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__ror__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__ror__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._or(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._or(o1);
             }
         }
     }
@@ -2044,7 +1831,7 @@
     /**
      * Implements the __ior__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___ior__(PyObject o) {
         PyObject ret = invoke_ex("__ior__", o);
@@ -2061,28 +1848,20 @@
     /**
      * Implements the __xor__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___xor__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__xor__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__xor__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o1._xor(o2);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o1._xor(o2);
             }
         }
     }
@@ -2095,28 +1874,20 @@
     /**
      * Implements the __rxor__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___rxor__(PyObject o) {
         Object ctmp = __coerce_ex__(o);
         if (ctmp == null || ctmp == Py.None)
             return invoke_ex("__rxor__", o);
         else {
-            PyObject o1 = ((PyObject[])ctmp)[0];
-            PyObject o2 = ((PyObject[])ctmp)[1];
+            PyObject o1 = ((PyObject[]) ctmp)[0];
+            PyObject o2 = ((PyObject[]) ctmp)[1];
             if (this == o1) {
-                // Prevent recusion if __coerce__ return self
+                // Prevent recursion if __coerce__ return self
                 return invoke_ex("__rxor__", o2);
-            }
-            else {
-                ThreadState ts = Py.getThreadState();
-                if (ts.recursion_depth++ > ts.systemState.getrecursionlimit())
-                    throw Py.RuntimeError("maximum recursion depth exceeded");
-                try {
-                    return o2._xor(o1);
-                } finally {
-                    --ts.recursion_depth;
-                }
+            } else {
+                return o2._xor(o1);
             }
         }
     }
@@ -2129,7 +1900,7 @@
     /**
      * Implements the __ixor__ method by looking it up
      * in the instance's dictionary and calling it if it is found.
-     **/
+     */
     @ExposedMethod(type = MethodType.BINARY)
     public PyObject instance___ixor__(PyObject o) {
         PyObject ret = invoke_ex("__ixor__", o);
diff --git a/src/org/python/core/ThreadState.java b/src/org/python/core/ThreadState.java
--- a/src/org/python/core/ThreadState.java
+++ b/src/org/python/core/ThreadState.java
@@ -19,8 +19,6 @@
 
     public int compareStateNesting;
 
-    public int recursion_depth;
-
     public TraceFunction tracefunc;
 
     public TraceFunction profilefunc;
@@ -64,13 +62,4 @@
         return compareStateDict;
     }
 
-    public void enterRecursiveCall(String where) {
-        if (recursion_depth++ > systemState.getrecursionlimit()) {
-            throw Py.RuntimeError("maximum recursion depth exceeded" + where);
-        }
-    }
-
-    public void leaveRecursiveCall() {
-        --recursion_depth;
-    }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:15 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:15 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Use_indirection_so_that_Thr?=
 =?utf-8?q?ead_does_not_hold_Jython_runtime_via_a_ThreadLocal?=
Message-ID: <3h0cmH1HKcz7LjR@mail.python.org>

http://hg.python.org/jython/rev/213d1b557f9a
changeset:   7325:213d1b557f9a
user:        Jim Baker 
date:        Tue Jun 24 23:39:40 2014 -0600
summary:
  Use indirection so that Thread does not hold Jython runtime via a ThreadLocal

files:
  src/org/python/modules/thread/PyLocal.java |  14 +++++++--
  1 files changed, 10 insertions(+), 4 deletions(-)


diff --git a/src/org/python/modules/thread/PyLocal.java b/src/org/python/modules/thread/PyLocal.java
--- a/src/org/python/modules/thread/PyLocal.java
+++ b/src/org/python/modules/thread/PyLocal.java
@@ -16,7 +16,12 @@
 
     public static final PyType TYPE = PyType.fromClass(PyLocal.class);
 
-    private ThreadLocal tdict = new ThreadLocal();
+    private ThreadLocal tdict = new ThreadLocal() {
+        @Override
+        protected Object initialValue() {
+            return new Object[1];
+        }
+    };
 
     private PyObject args[];
 
@@ -30,7 +35,7 @@
         super(subType);
         // Don't lazy load the underlying dict in the instantiating thread; that would
         // call __init__ a the second time
-        tdict.set(new PyDictionary());
+        tdict.set(new Object[] { new PyDictionary() });
     }
 
     @ExposedNew
@@ -71,10 +76,11 @@
 
     @Override
     public PyObject fastGetDict() {
-        PyDictionary ldict = tdict.get();
+        Object[] local = tdict.get();
+        PyDictionary ldict = (PyDictionary)(local[0]);
         if (ldict == null) {
             ldict = new PyDictionary();
-            tdict.set(ldict);
+            local[0] = ldict;
             dispatch__init__(args, keywords);
         }
         return ldict;

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:16 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:16 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_last_direct_ThreadLo?=
 =?utf-8?q?cal_to_Jython_runtime_classes?=
Message-ID: <3h0cmJ2fR4z7LkS@mail.python.org>

http://hg.python.org/jython/rev/68912d04f6f5
changeset:   7326:68912d04f6f5
user:        Jim Baker 
date:        Tue Jun 24 23:47:21 2014 -0600
summary:
  Remove last direct ThreadLocal to Jython runtime classes

files:
  src/org/python/util/PythonInterpreter.java |  37 ++++++---
  1 files changed, 24 insertions(+), 13 deletions(-)


diff --git a/src/org/python/util/PythonInterpreter.java b/src/org/python/util/PythonInterpreter.java
--- a/src/org/python/util/PythonInterpreter.java
+++ b/src/org/python/util/PythonInterpreter.java
@@ -31,7 +31,14 @@
     protected PySystemState systemState;
     PyObject globals;
 
-    protected ThreadLocal threadLocals;
+    protected final boolean useThreadLocalState;
+
+    protected static ThreadLocal threadLocals = new ThreadLocal() {
+        @Override
+        protected Object[] initialValue() {
+            return new Object[1];
+        }
+    };
 
     protected CompilerFlags cflags = new CompilerFlags();
 
@@ -103,9 +110,8 @@
         this.systemState = systemState;
         setSystemState();
 
-        if (useThreadLocalState) {
-            threadLocals = new ThreadLocal();
-        } else {
+        this.useThreadLocalState = useThreadLocalState;
+        if (!useThreadLocalState) {
             PyModule module = new PyModule("__main__", dict);
             systemState.modules.__setitem__("__main__", module);
         }
@@ -263,20 +269,24 @@
 
 
     public PyObject getLocals() {
-        if (threadLocals == null)
+        if (!useThreadLocalState) {
             return globals;
-
-        PyObject locals = threadLocals.get();
-        if (locals != null)
-            return locals;
-        return globals;
+        } else {
+            PyObject locals = (PyObject) threadLocals.get()[0];
+            if (locals != null) {
+                return locals;
+            }
+            return globals;
+        }
     }
 
     public void setLocals(PyObject d) {
-        if (threadLocals == null)
+        if (!useThreadLocalState) {
             globals = d;
-        else
-            threadLocals.set(d);
+        }
+        else {
+            threadLocals.get()[0] = d;
+        }
     }
 
     /**
@@ -352,6 +362,7 @@
         } catch (PyException pye) {
             // fall through
         }
+        threadLocals.remove();
         sys.cleanup();
     }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:17 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:17 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Make_RefReaperThread_a_Runn?=
 =?utf-8?q?able_and_ensure_it_terminates_to_avoid_ClassLoader?=
Message-ID: <3h0cmK5KrMz7Ljm@mail.python.org>

http://hg.python.org/jython/rev/4e538d0ebbd7
changeset:   7327:4e538d0ebbd7
user:        Indra Talip 
date:        Wed Jun 25 00:13:34 2014 -0600
summary:
  Make RefReaperThread a Runnable and ensure it terminates to avoid ClassLoader resource leaks

files:
  src/org/python/modules/_weakref/GlobalRef.java |  73 +++++++--
  1 files changed, 58 insertions(+), 15 deletions(-)


diff --git a/src/org/python/modules/_weakref/GlobalRef.java b/src/org/python/modules/_weakref/GlobalRef.java
--- a/src/org/python/modules/_weakref/GlobalRef.java
+++ b/src/org/python/modules/_weakref/GlobalRef.java
@@ -6,11 +6,14 @@
 import java.lang.ref.WeakReference;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.python.core.Py;
 import org.python.core.PyList;
 import org.python.core.PyObject;
+import org.python.core.PySystemState;
 import org.python.util.Generic;
 
 public class GlobalRef extends WeakReference {
@@ -34,14 +37,11 @@
 
     private static ReferenceQueue referenceQueue = new ReferenceQueue();
 
-    private static RefReaperThread reaperThread;
+    private static Thread reaperThread;
+    private static ReentrantReadWriteLock reaperLock = new ReentrantReadWriteLock();
 
     private static ConcurrentMap objects = Generic.concurrentMap();
 
-    static {
-        initReaperThread();
-    }
-
     public GlobalRef(PyObject object) {
         super(object, referenceQueue);
         hashCode = System.identityHashCode(object);
@@ -117,14 +117,36 @@
      * @return a new tracked GlobalRef
      */
     public static GlobalRef newInstance(PyObject object) {
-        GlobalRef ref = objects.get(new GlobalRef(object));
+        createReaperThreadIfAbsent();
+
+        GlobalRef newRef = new GlobalRef(object);
+        GlobalRef ref = objects.putIfAbsent(newRef, newRef);
         if (ref == null) {
-            ref = new GlobalRef(object);
-            objects.put(ref, ref);
+            ref = newRef;
         }
         return ref;
     }
 
+    private static void createReaperThreadIfAbsent() {
+        reaperLock.readLock().lock();
+        try {
+            if (reaperThread == null || !reaperThread.isAlive()) {
+                reaperLock.readLock().unlock();
+                reaperLock.writeLock().lock();
+                if (reaperThread == null || !reaperThread.isAlive()) {
+                    try {
+                        initReaperThread();
+                    } finally {
+                        reaperLock.readLock().lock();
+                        reaperLock.writeLock().unlock();
+                    }
+                }
+            }
+        } finally {
+            reaperLock.readLock().unlock();
+        }
+    }
+
     /**
      * Return the number of references to the specified PyObject.
      *
@@ -197,16 +219,18 @@
     }
 
     private static void initReaperThread() {
-        reaperThread = new RefReaperThread();
+        RefReaper reaper = new RefReaper();
+        PySystemState systemState = Py.getSystemState();
+        systemState.registerCloser(reaper);
+
+        reaperThread = new Thread(reaper, "weakref reaper");
         reaperThread.setDaemon(true);
         reaperThread.start();
     }
 
-    private static class RefReaperThread extends Thread {
-
-        RefReaperThread() {
-            super("weakref reaper");
-        }
+    private static class RefReaper implements Runnable, Callable {
+        private volatile boolean exit = false;
+        private Thread thread;
 
         public void collect() throws InterruptedException {
             GlobalRef gr = (GlobalRef)referenceQueue.remove();
@@ -216,13 +240,32 @@
         }
 
         public void run() {
+            // Store the actual reaper thread so that when PySystemState.cleanup()
+            // is called this thread can be interrupted and die.
+            this.thread = Thread.currentThread();
+
             while (true) {
                 try {
                     collect();
                 } catch (InterruptedException exc) {
-                    // ok
+                    // Is cleanup time so break out and die.
+                    if (exit) {
+                        break;
+                    }
                 }
             }
         }
+
+        @Override
+        public Void call() throws Exception {
+            this.exit = true;
+
+            if (thread != null && thread.isAlive()) {
+                this.thread.interrupt();
+                this.thread = null;
+            }
+
+            return null;
+        }
     }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:19 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:19 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Maintain_a_hybrid_strategy_?=
 =?utf-8?q?where_ThreadState_is_set_to_a_thread_local?=
Message-ID: <3h0cmM0VfHz7LkG@mail.python.org>

http://hg.python.org/jython/rev/06d196031fa1
changeset:   7328:06d196031fa1
user:        Jim Baker 
date:        Wed Jun 25 17:44:39 2014 -0600
summary:
  Maintain a hybrid strategy where ThreadState is set to a thread local
upon entry into Python context, reducing lookup cost. This also
restores recursionlimit support, although it's now only approximate,
since it doesn't take in account ops like __add__.

files:
  src/org/python/core/PyBaseCode.java                          |   5 +-
  src/org/python/core/PyTableCode.java                         |   3 +
  src/org/python/core/ThreadState.java                         |   5 +-
  src/org/python/core/ThreadStateMapping.java                  |  31 +++++++++-
  tests/java/org/python/expose/generate/MethodExposerTest.java |   2 +-
  5 files changed, 40 insertions(+), 6 deletions(-)


diff --git a/src/org/python/core/PyBaseCode.java b/src/org/python/core/PyBaseCode.java
--- a/src/org/python/core/PyBaseCode.java
+++ b/src/org/python/core/PyBaseCode.java
@@ -48,7 +48,7 @@
         }
         // nested scopes: setup env with closure
         // this should only be done once, so let the frame take care of it
-        frame.setupEnv((PyTuple)closure);
+        frame.setupEnv((PyTuple) closure);
 
         ts.frame = frame;
 
@@ -64,6 +64,7 @@
         }
 
         PyObject ret;
+        ThreadStateMapping.enterCall(ts);
         try {
             ret = interpret(frame, ts);
         } catch (Throwable t) {
@@ -84,6 +85,8 @@
             ts.exception = previous_exception;
             ts.frame = ts.frame.f_back;
             throw pye;
+        } finally {
+            ThreadStateMapping.exitCall(ts);
         }
 
         if (frame.tracefunc != null) {
diff --git a/src/org/python/core/PyTableCode.java b/src/org/python/core/PyTableCode.java
--- a/src/org/python/core/PyTableCode.java
+++ b/src/org/python/core/PyTableCode.java
@@ -161,6 +161,7 @@
         }
 
         PyObject ret;
+        ThreadStateMapping.enterCall(ts);
         try {
             ret = funcs.call_function(func_id, frame, ts);
         } catch (Throwable t) {
@@ -181,6 +182,8 @@
             ts.exception = previous_exception;
             ts.frame = ts.frame.f_back;
             throw pye;
+        } finally {
+            ThreadStateMapping.exitCall(ts);
         }
 
         if (frame.tracefunc != null) {
diff --git a/src/org/python/core/ThreadState.java b/src/org/python/core/ThreadState.java
--- a/src/org/python/core/ThreadState.java
+++ b/src/org/python/core/ThreadState.java
@@ -11,7 +11,7 @@
 
     public PyException exception;
 
-    public Thread thread;
+    public int call_depth;
 
     public boolean tracing;
 
@@ -25,9 +25,8 @@
 
     private PyDictionary compareStateDict;
 
-    public ThreadState(Thread t, PySystemState systemState) {
+    public ThreadState(PySystemState systemState) {
         this.systemState = systemState;
-        thread = t;
     }
 
     public boolean enterRepr(PyObject obj) {
diff --git a/src/org/python/core/ThreadStateMapping.java b/src/org/python/core/ThreadStateMapping.java
--- a/src/org/python/core/ThreadStateMapping.java
+++ b/src/org/python/core/ThreadStateMapping.java
@@ -8,7 +8,18 @@
     private static final Map cachedThreadState =
         new MapMaker().weakKeys().weakValues().makeMap();
 
+    private static ThreadLocal scopedThreadState= new ThreadLocal() {
+        @Override
+        protected Object[] initialValue() {
+            return new Object[1];
+        }
+    };
+
     public ThreadState getThreadState(PySystemState newSystemState) {
+        Object scoped = scopedThreadState.get()[0];
+        if (scoped != null) {
+            return (ThreadState)scoped;
+        }
         Thread currentThread = Thread.currentThread();
         ThreadState ts = cachedThreadState.get(currentThread);
         if (ts != null) {
@@ -23,8 +34,26 @@
             newSystemState = Py.defaultSystemState;
         }
 
-        ts = new ThreadState(currentThread, newSystemState);
+        ts = new ThreadState(newSystemState);
         cachedThreadState.put(currentThread, ts);
         return ts;
     }
+
+    public static void enterCall(ThreadState ts) {
+        if (ts.call_depth == 0) {
+            scopedThreadState.get()[0] = ts;
+//            Thread.currentThread().setContextClassLoader(imp.getSyspathJavaLoader());
+        } else if (ts.call_depth > ts.systemState.getrecursionlimit()) {
+            throw Py.RuntimeError("maximum recursion depth exceeded");
+        }
+        ts.call_depth++;
+    }
+
+    public static void exitCall(ThreadState ts) {
+        ts.call_depth--;
+        if (ts.call_depth == 0) {
+            scopedThreadState.get()[0] = null;
+//            Thread.currentThread().setContextClassLoader(null);
+        }
+    }
 }
diff --git a/tests/java/org/python/expose/generate/MethodExposerTest.java b/tests/java/org/python/expose/generate/MethodExposerTest.java
--- a/tests/java/org/python/expose/generate/MethodExposerTest.java
+++ b/tests/java/org/python/expose/generate/MethodExposerTest.java
@@ -234,7 +234,7 @@
         PyObject expected = Py.newString("foo got state " + ts.hashCode());
         assertEquals(expected, bound.__call__(Py.getThreadState(), Py.newString("foo")));
         assertEquals(expected, bound.__call__(Py.newString("foo")));
-        ts = new ThreadState(new Thread(), ts.systemState);
+        ts = new ThreadState(ts.systemState);
         assertEquals(Py.newString("foo got state " + ts.hashCode()),
                      bound.__call__(ts, Py.newString("foo")));
     }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:20 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:20 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_shadowing_of_mutable?=
 =?utf-8?q?_statics_in_PySystemState=2C_instead_make_them?=
Message-ID: <3h0cmN327bz7LjR@mail.python.org>

http://hg.python.org/jython/rev/0e29ad2e098c
changeset:   7329:0e29ad2e098c
user:        Jim Baker 
date:        Fri Jun 27 16:04:16 2014 -0600
summary:
  Remove shadowing of mutable statics in PySystemState, instead make them instance variables

files:
  src/org/python/core/PyBaseCode.java         |    2 +-
  src/org/python/core/PyFrame.java            |    2 +-
  src/org/python/core/PySystemState.java      |  103 ++-------
  src/org/python/core/PyTableCode.java        |    2 +-
  src/org/python/core/__builtin__.java        |    2 +-
  src/org/python/core/imp.java                |    2 +-
  src/org/python/util/InteractiveConsole.java |    2 +-
  7 files changed, 29 insertions(+), 86 deletions(-)


diff --git a/src/org/python/core/PyBaseCode.java b/src/org/python/core/PyBaseCode.java
--- a/src/org/python/core/PyBaseCode.java
+++ b/src/org/python/core/PyBaseCode.java
@@ -43,7 +43,7 @@
             } else {
                 //System.err.println("ts: "+ts);
                 //System.err.println("ss: "+ts.systemState);
-                frame.f_builtins = PySystemState.builtins;
+                frame.f_builtins = ts.systemState.builtins;
             }
         }
         // nested scopes: setup env with closure
diff --git a/src/org/python/core/PyFrame.java b/src/org/python/core/PyFrame.java
--- a/src/org/python/core/PyFrame.java
+++ b/src/org/python/core/PyFrame.java
@@ -273,7 +273,7 @@
 
         // Set up f_builtins if not already set
         if (f_builtins == null) {
-            f_builtins = PySystemState.builtins;
+            f_builtins = Py.getThreadState().systemState.builtins;
         }
         return f_builtins.__finditem__(index);
     }
diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java
--- a/src/org/python/core/PySystemState.java
+++ b/src/org/python/core/PySystemState.java
@@ -116,9 +116,6 @@
     private static PyList defaultArgv;
     private static PyObject defaultExecutable;
 
-    // XXX - from Jython code, these statics are immutable; we may wish to consider
-    // using the shadowing mechanism for them as well if in practice it makes
-    // sense for them to be changed
     public static Properties registry; // = init_registry();
     public static PyObject prefix;
     public static PyObject exec_prefix = Py.EmptyString;
@@ -135,10 +132,10 @@
     public PyObject modules;
     public PyList path;
 
-    // shadowed statics - don't use directly
-    public static PyList warnoptions = new PyList();
-    public static PyObject builtins;
-    public static PyObject platform = new PyString("java");
+    public PyList warnoptions = new PyList();
+    public PyObject builtins;
+    private static PyObject defaultPlatform = new PyString("java");
+    public PyObject platform = defaultPlatform;
 
     public PyList meta_path;
     public PyList path_hooks;
@@ -199,6 +196,8 @@
         path.append(Py.newString(JavaImporter.JAVA_IMPORT_PATH_ENTRY));
         path.append(Py.newString(ClasspathPyImporter.PYCLASSPATH_PREFIX));
         executable = defaultExecutable;
+        builtins = getDefaultBuiltins();
+        platform = defaultPlatform;
 
         meta_path = new PyList();
         path_hooks = new PyList();
@@ -279,14 +278,9 @@
         ((PyFile)stderr).setEncoding(encoding, "backslashreplace");
     }
 
-    // might be nice to have something general here, but for now these
-    // seem to be the only values that need to be explicitly shadowed
-    private Shadow shadowing;
-
-    public synchronized void shadow() {
-        if (shadowing == null) {
-            shadowing = new Shadow();
-        }
+    @Deprecated
+    public void shadow() {
+        // Now a no-op
     }
 
     private static class DefaultBuiltinsHolder {
@@ -304,53 +298,29 @@
         return DefaultBuiltinsHolder.builtins;
     }
 
-    public synchronized PyObject getBuiltins() {
-        if (shadowing == null) {
-            return getDefaultBuiltins();
-        } else {
-            return shadowing.builtins;
-        }
+    public PyObject getBuiltins() {
+        return builtins;
     }
 
-    public synchronized void setBuiltins(PyObject value) {
-        if (shadowing == null) {
-            builtins = value;
-        } else {
-            shadowing.builtins = value;
-        }
+    public void setBuiltins(PyObject value) {
+        builtins = value;
         modules.__setitem__("__builtin__", new PyModule("__builtin__", value));
     }
 
-    public synchronized PyObject getWarnoptions() {
-        if (shadowing == null) {
-            return warnoptions;
-        } else {
-            return shadowing.warnoptions;
-        }
+    public PyObject getWarnoptions() {
+        return warnoptions;
     }
 
-    public synchronized void setWarnoptions(PyObject value) {
-        if (shadowing == null) {
-            warnoptions = new PyList(value);
-        } else {
-            shadowing.warnoptions = new PyList(value);
-        }
+    public void setWarnoptions(PyObject value) {
+        warnoptions = new PyList(value);
     }
 
-    public synchronized PyObject getPlatform() {
-        if (shadowing == null) {
-            return platform;
-        } else {
-            return shadowing.platform;
-        }
+    public PyObject getPlatform() {
+        return platform;
     }
 
-    public synchronized void setPlatform(PyObject value) {
-        if (shadowing == null) {
-            platform = value;
-        } else {
-            shadowing.platform = value;
-        }
+    public void setPlatform(PyObject value) {
+        platform = value;
     }
 
     public synchronized codecs.CodecState getCodecState() {
@@ -388,12 +358,6 @@
                 return null;
             }
             return exc.traceback;
-        } else if (name == "warnoptions") {
-            return getWarnoptions();
-        } else if (name == "builtins") {
-            return getBuiltins();
-        } else if (name == "platform") {
-            return getPlatform();
         } else {
             PyObject ret = super.__findattr_ex__(name);
             if (ret != null) {
@@ -416,14 +380,7 @@
     public void __setattr__(String name, PyObject value) {
         checkReadOnly(name);
         if (name == "builtins") {
-            shadow();
             setBuiltins(value);
-        } else if (name == "warnoptions") {
-            shadow();
-            setWarnoptions(value);
-        } else if (name == "platform") {
-            shadow();
-            setPlatform(value);
         } else {
             PyObject ret = getType().lookup(name); // xxx fix fix fix
             if (ret != null && ret._doset(this, value)) {
@@ -761,7 +718,7 @@
         }
     }
 
-    public static void determinePlatform(Properties props) {
+    private static void determinePlatform(Properties props) {
         String version = props.getProperty("java.version");
         if (version == null) {
             version = "???";
@@ -776,7 +733,7 @@
         if (version.equals("12")) {
             version = "1.2";
         }
-        platform = new PyString("java" + version);
+        defaultPlatform = new PyString("java" + version);
     }
 
     private static void initRegistry(Properties preProperties, Properties postProperties,
@@ -1755,20 +1712,6 @@
 }
 
 
-class Shadow {
-
-    PyObject builtins;
-    PyList warnoptions;
-    PyObject platform;
-
-    Shadow() {
-        builtins = PySystemState.getDefaultBuiltins();
-        warnoptions = PySystemState.warnoptions;
-        platform = PySystemState.platform;
-    }
-}
-
-
 @ExposedType(name = "sys.float_info", isBaseType = false)
 class FloatInfo extends PyTuple {
 
diff --git a/src/org/python/core/PyTableCode.java b/src/org/python/core/PyTableCode.java
--- a/src/org/python/core/PyTableCode.java
+++ b/src/org/python/core/PyTableCode.java
@@ -140,7 +140,7 @@
             } else {
                 //System.err.println("ts: "+ts);
                 //System.err.println("ss: "+ts.systemState);
-                frame.f_builtins = PySystemState.builtins;
+                frame.f_builtins = ts.systemState.builtins;;
             }
         }
         // nested scopes: setup env with closure
diff --git a/src/org/python/core/__builtin__.java b/src/org/python/core/__builtin__.java
--- a/src/org/python/core/__builtin__.java
+++ b/src/org/python/core/__builtin__.java
@@ -1234,7 +1234,7 @@
         if (frame != null && frame.f_builtins != null) {
             builtins = frame.f_builtins;
         } else {
-            builtins = PySystemState.builtins;
+            builtins = Py.getSystemState().builtins;
         }
 
         PyObject __import__ = builtins.__finditem__("__import__");
diff --git a/src/org/python/core/imp.java b/src/org/python/core/imp.java
--- a/src/org/python/core/imp.java
+++ b/src/org/python/core/imp.java
@@ -493,7 +493,7 @@
         }
         if (name == "__builtin__") {
             Py.writeComment(IMPORT_LOG, "'" + name + "' as __builtin__ in builtin modules");
-            return new PyModule("__builtin__", PySystemState.builtins);
+            return new PyModule("__builtin__", Py.getSystemState().builtins);
         }
         String mod = PySystemState.getBuiltin(name);
         if (mod != null) {
diff --git a/src/org/python/util/InteractiveConsole.java b/src/org/python/util/InteractiveConsole.java
--- a/src/org/python/util/InteractiveConsole.java
+++ b/src/org/python/util/InteractiveConsole.java
@@ -62,7 +62,7 @@
     }
 
     public static String getDefaultBanner() {
-        return String.format("Jython %s on %s", PySystemState.version, PySystemState.platform);
+        return String.format("Jython %s on %s", PySystemState.version, Py.getSystemState().platform);
     }
 
     public void interact(String banner, PyObject file) {

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:21 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:21 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_PythonInterpreter_such_?=
 =?utf-8?q?that_it_supports_AutoCloseable=2C_Closeable?=
Message-ID: <3h0cmP4Kmvz7LkT@mail.python.org>

http://hg.python.org/jython/rev/3cf75a71b464
changeset:   7330:3cf75a71b464
user:        Jim Baker 
date:        Fri Jun 27 17:10:57 2014 -0600
summary:
  Fix PythonInterpreter such that it supports AutoCloseable, Closeable

files:
  src/org/python/jsr223/PyScriptEngine.java  |   6 ++++-
  src/org/python/util/PythonInterpreter.java |  12 +++++++++-
  2 files changed, 16 insertions(+), 2 deletions(-)


diff --git a/src/org/python/jsr223/PyScriptEngine.java b/src/org/python/jsr223/PyScriptEngine.java
--- a/src/org/python/jsr223/PyScriptEngine.java
+++ b/src/org/python/jsr223/PyScriptEngine.java
@@ -17,7 +17,7 @@
 import javax.script.SimpleBindings;
 import org.python.util.PythonInterpreter;
 
-public class PyScriptEngine extends AbstractScriptEngine implements Compilable, Invocable {
+public class PyScriptEngine extends AbstractScriptEngine implements Compilable, Invocable, AutoCloseable {
 
     private final PythonInterpreter interp;
     private final ScriptEngineFactory factory;
@@ -231,4 +231,8 @@
             return PyScriptEngine.this.eval(code, ctx);
         }
     }
+
+    public void close() {
+        interp.close();
+    }
 }
diff --git a/src/org/python/util/PythonInterpreter.java b/src/org/python/util/PythonInterpreter.java
--- a/src/org/python/util/PythonInterpreter.java
+++ b/src/org/python/util/PythonInterpreter.java
@@ -1,5 +1,6 @@
 package org.python.util;
 
+import java.io.Closeable;
 import java.io.Reader;
 import java.io.StringReader;
 import java.util.Properties;
@@ -25,7 +26,7 @@
  * The PythonInterpreter class is a standard wrapper for a Jython interpreter
  * for embedding in a Java application.
  */
-public class PythonInterpreter {
+public class PythonInterpreter implements AutoCloseable, Closeable {
 
     // Defaults if the interpreter uses thread-local state
     protected PySystemState systemState;
@@ -42,6 +43,8 @@
 
     protected CompilerFlags cflags = new CompilerFlags();
 
+    private volatile boolean closed = false;
+
     /**
      * Initializes the Jython runtime. This should only be called
      * once, before any other Python objects (including
@@ -365,4 +368,11 @@
         threadLocals.remove();
         sys.cleanup();
     }
+
+    public void close() {
+        if (!closed) {
+            closed = true;
+            cleanup();
+        }
+    }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:22 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:22 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_PySystemState_such_that?=
 =?utf-8?q?_it_supports_AutoCloseable=2C_Closeable?=
Message-ID: <3h0cmQ5j2xz7LkD@mail.python.org>

http://hg.python.org/jython/rev/d8233316cf09
changeset:   7331:d8233316cf09
user:        Jim Baker 
date:        Fri Jun 27 17:21:42 2014 -0600
summary:
  Fix PySystemState such that it supports AutoCloseable, Closeable

files:
  src/org/python/core/PySystemState.java |  5 ++++-
  1 files changed, 4 insertions(+), 1 deletions(-)


diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java
--- a/src/org/python/core/PySystemState.java
+++ b/src/org/python/core/PySystemState.java
@@ -2,6 +2,7 @@
 package org.python.core;
 
 import java.io.BufferedReader;
+import java.io.Closeable;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -48,7 +49,7 @@
  */
 // xxx Many have lamented, this should really be a module!
 // but it will require some refactoring to see this wish come true.
-public class PySystemState extends PyObject implements ClassDictInit {
+public class PySystemState extends PyObject implements AutoCloseable, ClassDictInit, Closeable {
 
     public static final String PYTHON_CACHEDIR = "python.cachedir";
     public static final String PYTHON_CACHEDIR_SKIP = "python.cachedir.skip";
@@ -1541,6 +1542,8 @@
         closer.cleanup();
     }
 
+    public void close() { cleanup(); }
+
     private static class PySystemStateCloser {
 
         private final Set> resourceClosers = new LinkedHashSet>();

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:23 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:23 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Ensure_that_the_site_module?=
 =?utf-8?q?_is_imported_when_using_PythonInterpreter?=
Message-ID: <3h0cmR6v96z7LjR@mail.python.org>

http://hg.python.org/jython/rev/51b28cc2c43d
changeset:   7332:51b28cc2c43d
user:        Timoth?e Lecomte 
date:        Fri Jun 27 17:23:51 2014 -0600
summary:
  Ensure that the site module is imported when using PythonInterpreter

files:
  src/org/python/util/PythonInterpreter.java |  7 +++++++
  1 files changed, 7 insertions(+), 0 deletions(-)


diff --git a/src/org/python/util/PythonInterpreter.java b/src/org/python/util/PythonInterpreter.java
--- a/src/org/python/util/PythonInterpreter.java
+++ b/src/org/python/util/PythonInterpreter.java
@@ -8,6 +8,8 @@
 import org.python.antlr.base.mod;
 import org.python.core.CompileMode;
 import org.python.core.CompilerFlags;
+import org.python.core.imp;
+import org.python.core.Options;
 import org.python.core.ParserFacade;
 import org.python.core.Py;
 import org.python.core.PyCode;
@@ -118,6 +120,11 @@
             PyModule module = new PyModule("__main__", dict);
             systemState.modules.__setitem__("__main__", module);
         }
+        
+        if (Options.importSite) {
+            // Ensure site-packages are available
+            imp.load("site");
+        }
     }
 
     public PySystemState getSystemState() {

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:25 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:25 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_test_verifying_site_imp?=
 =?utf-8?q?ort_by_default_for_PythonInterpreter=2E?=
Message-ID: <3h0cmT1Mfnz7LjR@mail.python.org>

http://hg.python.org/jython/rev/6d3e8e67af1d
changeset:   7333:6d3e8e67af1d
user:        Jim Baker 
date:        Fri Jun 27 17:44:54 2014 -0600
summary:
  Add test verifying site import by default for PythonInterpreter.

files:
  tests/java/org/python/jsr223/ScriptEngineTest.java |  21 ++++++++++
  1 files changed, 21 insertions(+), 0 deletions(-)


diff --git a/tests/java/org/python/jsr223/ScriptEngineTest.java b/tests/java/org/python/jsr223/ScriptEngineTest.java
--- a/tests/java/org/python/jsr223/ScriptEngineTest.java
+++ b/tests/java/org/python/jsr223/ScriptEngineTest.java
@@ -16,6 +16,7 @@
 import junit.framework.TestCase;
 
 import org.python.core.PyString;
+import org.python.core.Options;
 
 public class ScriptEngineTest extends TestCase {
 
@@ -271,5 +272,25 @@
         // Would previously fail
         pythonEngine.eval("warnings.warn('test')");
     }
+
+    public void testSiteImportedByDefault() throws ScriptException {
+        ScriptEngineManager manager = new ScriptEngineManager();
+        ScriptEngine pythonEngine = manager.getEngineByName("python");
+        pythonEngine.eval("import sys");
+        pythonEngine.eval("'site' in sys.modules");
+    }
+
+    public void testSiteCanBeNotImported() throws ScriptException {
+	try {
+	    Options.importSite = false;
+	    ScriptEngineManager manager = new ScriptEngineManager();
+	    ScriptEngine pythonEngine = manager.getEngineByName("python");
+
+	    pythonEngine.eval("import sys");
+	    pythonEngine.eval("'site' not in sys.modules");
+        } finally {
+	    Options.importSite = true;
+	}
+    }
     
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 03:28:26 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 03:28:26 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython_=28merge_default_-=3E_default?=
 =?utf-8?q?=29=3A_Merged_https=3A//bitbucket=2Eorg/jimbaker/jython-resourc?=
 =?utf-8?q?e-leaks?=
Message-ID: <3h0cmV2w6Nz7LkQ@mail.python.org>

http://hg.python.org/jython/rev/3ed687cdb18f
changeset:   7334:3ed687cdb18f
parent:      7333:6d3e8e67af1d
parent:      7318:c66185c8ed12
user:        Jim Baker 
date:        Fri Jun 27 19:27:56 2014 -0600
summary:
  Merged https://bitbucket.org/jimbaker/jython-resource-leaks

files:



-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 08:24:35 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 08:24:35 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Change_subprocess_construct?=
 =?utf-8?q?ion_such_that_handles_are_inherited_if_not_set=2E?=
Message-ID: <3h0lLC4n12z7LjR@mail.python.org>

http://hg.python.org/jython/rev/e9156cf6283f
changeset:   7335:e9156cf6283f
user:        Jim Baker 
date:        Sat Jun 28 00:21:57 2014 -0600
summary:
  Change subprocess construction such that handles are inherited if not set.
Support subprocess termination vs terminate/kill methods.

Fixes http://bugs.jython.org/issue1898 and http://bugs.jython.org/issue2096

files:
  Lib/subprocess.py |  16 +++++++++++++---
  1 files changed, 13 insertions(+), 3 deletions(-)


diff --git a/Lib/subprocess.py b/Lib/subprocess.py
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -832,7 +832,8 @@
                             startupinfo, creationflags, shell,
                             p2cread, p2cwrite,
                             c2pread, c2pwrite,
-                            errread, errwrite)
+                            errread, errwrite,
+                            stdin, stdout, stderr)
 
         if mswindows:
             if p2cwrite is not None:
@@ -1295,7 +1296,8 @@
                            startupinfo, creationflags, shell,
                            p2cread, p2cwrite,
                            c2pread, c2pwrite,
-                           errread, errwrite):
+                           errread, errwrite,
+                           stdin, stdout, stderr):
             """Execute program (Java version)"""
 
             if isinstance(args, types.StringTypes):
@@ -1316,6 +1318,14 @@
                 args[0] = executable
 
             builder = java.lang.ProcessBuilder(args)
+
+            if stdin is None:
+                builder.redirectInput(java.lang.ProcessBuilder.Redirect.INHERIT)
+            if stdout is None:
+                builder.redirectOutput(java.lang.ProcessBuilder.Redirect.INHERIT)
+            if stderr is None:
+                builder.redirectError(java.lang.ProcessBuilder.Redirect.INHERIT)
+
             # os.environ may be inherited for compatibility with CPython
             self._setup_env(dict(os.environ if env is None else env),
                             builder.environment())
@@ -1396,7 +1406,7 @@
         def terminate(self):
             """Terminates the process
             """
-            _subprocess.TerminateProcess(self._handle, 1)
+            self._process.destroy()
 
         kill = terminate
 

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 08:24:36 2014
From: jython-checkins at python.org (jim.baker)
Date: Sat, 28 Jun 2014 08:24:36 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Synchonize_runClosers_in_ca?=
 =?utf-8?q?se_other_files/resources_are_being?=
Message-ID: <3h0lLD64B5z7LjS@mail.python.org>

http://hg.python.org/jython/rev/b2c87cfba346
changeset:   7336:b2c87cfba346
user:        Jim Baker 
date:        Sat Jun 28 00:24:27 2014 -0600
summary:
  Synchonize runClosers in case other files/resources are being
opened/closed during shutdown.

Fixes http://bugs.jython.org/issue1747

files:
  src/org/python/core/PySystemState.java |  44 +++++++------
  1 files changed, 23 insertions(+), 21 deletions(-)


diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java
--- a/src/org/python/core/PySystemState.java
+++ b/src/org/python/core/PySystemState.java
@@ -1593,7 +1593,7 @@
             }
 
             // Close the listed resources (and clear the list)
-            runClosers(resourceClosers);
+            runClosers();
             resourceClosers.clear();
 
             // XXX Not sure this is ok, but it makes repeat testing possible.
@@ -1601,6 +1601,27 @@
             isCleanup = false;
         }
 
+        private synchronized void runClosers() {
+            // resourceClosers can be null in some strange cases
+            if (resourceClosers != null) {
+            /*
+             * Although a Set, the container iterates in the order closers were added. Make a Deque
+             * of it and deal from the top.
+             */
+                LinkedList> rc = new LinkedList>(resourceClosers);
+                Iterator> iter = rc.descendingIterator();
+
+                while (iter.hasNext()) {
+                    Callable callable = iter.next();
+                    try {
+                        callable.call();
+                    } catch (Exception e) {
+                        // just continue, nothing we can do
+                    }
+                }
+            }
+        }
+
         // Python scripts expect that files are closed upon an orderly cleanup of the VM.
         private Thread initShutdownCloser() {
             try {
@@ -1617,7 +1638,7 @@
 
         	@Override
             public synchronized void run() {
-                runClosers(resourceClosers);
+                runClosers();
                 resourceClosers.clear();
             }
         }
@@ -1632,26 +1653,7 @@
      *
      * @param resourceClosers to be called in turn
      */
-    private static void runClosers(Set> resourceClosers) {
-        // resourceClosers can be null in some strange cases
-        if (resourceClosers != null) {
-            /*
-             * Although a Set, the container iterates in the order closers were added. Make a Deque
-             * of it and deal from the top.
-             */
-            LinkedList> rc = new LinkedList>(resourceClosers);
-            Iterator> iter = rc.descendingIterator();
 
-            while (iter.hasNext()) {
-                Callable callable = iter.next();
-                try {
-                    callable.call();
-                } catch (Exception e) {
-                    // just continue, nothing we can do
-                }
-            }
-        }
-    }
 }
 
 

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 21:03:11 2014
From: jython-checkins at python.org (frank.wierzbicki)
Date: Sat, 28 Jun 2014 21:03:11 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_tag_v2=2E7b3_for_chan?=
	=?utf-8?q?geset_b2c87cfba346?=
Message-ID: <3h149W6ZRSz7LjS@mail.python.org>

http://hg.python.org/jython/rev/11d9da1192a5
changeset:   7337:11d9da1192a5
user:        Frank Wierzbicki 
date:        Sat Jun 28 19:02:57 2014 +0000
summary:
  Added tag v2.7b3 for changeset b2c87cfba346

files:
  .hgtags |  2 ++
  1 files changed, 2 insertions(+), 0 deletions(-)


diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -75,3 +75,5 @@
 2d4e0bbb87b5d92dea257ef8d26596935e8feb2d v2.7b3
 2d4e0bbb87b5d92dea257ef8d26596935e8feb2d v2.7b3
 5688f9c2b7439654684e3e35cbecaffe413385c5 v2.7b3
+5688f9c2b7439654684e3e35cbecaffe413385c5 v2.7b3
+b2c87cfba346a1b770f59ce92159892e356676ff v2.7b3

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 21:16:24 2014
From: jython-checkins at python.org (frank.wierzbicki)
Date: Sat, 28 Jun 2014 21:16:24 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Restore_antlr-runtime-3=2E1?=
	=?utf-8?b?LjMuamFyLg==?=
Message-ID: <3h14Sm0Flmz7LjQ@mail.python.org>

http://hg.python.org/jython/rev/5e7462875b63
changeset:   7338:5e7462875b63
tag:         v2.7b3
user:        Frank Wierzbicki 
date:        Sat Jun 28 19:15:56 2014 +0000
summary:
  Restore antlr-runtime-3.1.3.jar.

It is needed to create a final distribution, which doesn't need to bundle the
entire antlr distribution, just the runtime.

files:
  extlibs/antlr-runtime-3.1.3.jar |  Bin 
  1 files changed, 0 insertions(+), 0 deletions(-)


diff --git a/extlibs/antlr-runtime-3.1.3.jar b/extlibs/antlr-runtime-3.1.3.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b0a9ea69f5c29097145a48c26e127972920db440
GIT binary patch
[stripped]

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Jun 28 21:16:25 2014
From: jython-checkins at python.org (frank.wierzbicki)
Date: Sat, 28 Jun 2014 21:16:25 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_tag_v2=2E7b3_for_chan?=
	=?utf-8?q?geset_5e7462875b63?=
Message-ID: <3h14Sn1qsSz7LjW@mail.python.org>

http://hg.python.org/jython/rev/5efdcedc9817
changeset:   7339:5efdcedc9817
user:        Frank Wierzbicki 
date:        Sat Jun 28 19:16:10 2014 +0000
summary:
  Added tag v2.7b3 for changeset 5e7462875b63

files:
  .hgtags |  2 ++
  1 files changed, 2 insertions(+), 0 deletions(-)


diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -77,3 +77,5 @@
 5688f9c2b7439654684e3e35cbecaffe413385c5 v2.7b3
 5688f9c2b7439654684e3e35cbecaffe413385c5 v2.7b3
 b2c87cfba346a1b770f59ce92159892e356676ff v2.7b3
+b2c87cfba346a1b770f59ce92159892e356676ff v2.7b3
+5e7462875b633e4f13b77985a1ed5186a0b92cac v2.7b3

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 29 17:13:11 2014
From: jython-checkins at python.org (jeff.allen)
Date: Sun, 29 Jun 2014 17:13:11 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Comment_and_formatting_chan?=
 =?utf-8?q?ge_only_to_stringlib/MarkupIterator=2E?=
Message-ID: <3h1b1g0bv6z7LjW@mail.python.org>

http://hg.python.org/jython/rev/d45151484317
changeset:   7340:d45151484317
parent:      7315:44191dd20f5a
user:        Jeff Allen 
date:        Fri Jun 27 16:40:58 2014 +0100
summary:
  Comment and formatting change only to stringlib/MarkupIterator.

files:
  src/org/python/core/stringlib/MarkupIterator.java |  127 ++++++++-
  1 files changed, 113 insertions(+), 14 deletions(-)


diff --git a/src/org/python/core/stringlib/MarkupIterator.java b/src/org/python/core/stringlib/MarkupIterator.java
--- a/src/org/python/core/stringlib/MarkupIterator.java
+++ b/src/org/python/core/stringlib/MarkupIterator.java
@@ -9,21 +9,28 @@
 import org.python.expose.ExposedType;
 
 /**
- * Provides an implementation of str._formatter_parser()
+ * Provides an implementation of the object that str._formatter_parser() returns, which
+ * is an iterator returning successive 4-tuples, the sequence being equivalent to the original
+ * string.
  */
 @ExposedType(name = "formatteriterator", base = PyObject.class, isBaseType = false)
 public class MarkupIterator extends PyObject {
 
     public static final PyType TYPE = PyType.fromClass(MarkupIterator.class);
 
+    /** The string from which elements are being returned. */
     private final String markup;
+    /** How far along that string we are. */
     private int index;
+    /** A counter used to auto-number fields when not explicitly numbered in the format. */
     private final FieldNumbering numbering;
 
+    /** Constructor used at top-level to enumerate a format. */
     public MarkupIterator(String markup) {
         this(markup, null);
     }
 
+    /** Variant constructor used when formats are nested. */
     public MarkupIterator(String markup, MarkupIterator enclosingIterator) {
         this.markup = markup;
         if (enclosingIterator != null) {
@@ -48,6 +55,21 @@
         return formatteriterator___iternext__();
     }
 
+    /**
+     * Return the next "chunk" of the format (or return null if ended). A chunk is a 4-tuple
+     * describing
+     * 
    
+     * 
  1. the text leading up to the next format field,
    2. 
+     * 
  2. the field name or number (as a string) for accessing the value,
    3. 
+     * 
  3. the format specifier such as "#12x", and
    4. 
+     * 
  4. any conversion that should be applied (the 's' or 'r' codes for
+     * str() and repr())
    5. 
+     * 

+     * Elements 1-3 are None if this chunk contains no format specifier. Elements 0-2 are
+     * zero-length strings if missing from the format, while element 3 will be None if missing.
+     * 
+     * @return PyTuple chunk or null
+     */
     @ExposedMethod
     final PyObject formatteriterator___iternext__() {
         Chunk chunk;
@@ -60,16 +82,23 @@
             return null;
         }
         PyObject[] elements = new PyObject[4];
+
+        // Literal text is used verbatim.
         elements[0] = new PyString(chunk.literalText);
-        elements[1] = chunk.fieldName.length() == 0
-                    ? Py.None : new PyString(chunk.fieldName);
+
+        // A field name is empty only if there was no format at all.
+        elements[1] = chunk.fieldName.length() == 0 ? Py.None : new PyString(chunk.fieldName);
         if (chunk.fieldName.length() > 0) {
-            elements[2] = chunk.formatSpec == null
-                    ? Py.EmptyString : new PyString(chunk.formatSpec);
+            elements[2] =
+                    chunk.formatSpec == null ? Py.EmptyString : new PyString(chunk.formatSpec);
         } else {
             elements[2] = Py.None;
         }
+
+        // There may have been a conversion specifier.
         elements[3] = chunk.conversion == null ? Py.None : new PyString(chunk.conversion);
+
+        // And those make up the next answer.
         return new PyTuple(elements);
     }
 
@@ -78,36 +107,50 @@
             return null;
         }
         Chunk result = new Chunk();
+
+        // pos = index is the index of the first text not already chunked
         int pos = index;
+
+        // Advance pos to the first '{' that is not a "{{" (escaped brace), or pos<0 if none such.
         while (true) {
             pos = indexOfFirst(markup, pos, '{', '}');
             if (pos >= 0 && pos < markup.length() - 1
-                && markup.charAt(pos + 1) == markup.charAt(pos)) {
+                    && markup.charAt(pos + 1) == markup.charAt(pos)) {
                 // skip escaped bracket
                 pos += 2;
             } else if (pos >= 0 && markup.charAt(pos) == '}') {
+                // Un-escaped '}' is a syntax error
                 throw new IllegalArgumentException("Single '}' encountered in format string");
             } else {
+                // pos is at an un-escaped '{'
                 break;
             }
         }
+
+        // markup[index:pos] is the literal part of this chunk.
         if (pos < 0) {
+            // ... except pos<0, and there is no further format specifier, only literal text.
             result.literalText = unescapeBraces(markup.substring(index));
             result.fieldName = "";
             index = markup.length();
-        }
-        else {
+
+        } else {
+            // Grab the literal text, dealing with escaped braces.
             result.literalText = unescapeBraces(markup.substring(index, pos));
+            // Scan through the contents of the format spec, between the braces. Skip one '{'.
             pos++;
             int fieldStart = pos;
             int count = 1;
             while (pos < markup.length()) {
                 if (markup.charAt(pos) == '{') {
+                    // This means the spec we are gathering itself contains nested specifiers.
                     count++;
                     result.formatSpecNeedsExpanding = true;
                 } else if (markup.charAt(pos) == '}') {
+                    // And here is a '}' matching one we already counted.
                     count--;
                     if (count == 0) {
+                        // ... matching the one we began with: parse the replacement field.
                         parseField(result, markup.substring(fieldStart, pos));
                         pos++;
                         break;
@@ -116,6 +159,7 @@
                 pos++;
             }
             if (count > 0) {
+                // Must be end of string without matching '}'.
                 throw new IllegalArgumentException("Single '{' encountered in format string");
             }
             index = pos;
@@ -127,44 +171,75 @@
         return substring.replace("{{", "{").replace("}}", "}");
     }
 
+    /**
+     * Parse a "replacement field" consisting of a name, conversion and format specification.
+     * According to the Python Standard Library documentation, a replacement field has the
+     * structure:
+     * 
+     * 
+     * replacement_field ::=  "{" [field_name] ["!" conversion] [":" format_spec] "}"
+     * field_name        ::=  arg_name ("." attribute_name | "[" element_index "]")*
+     * arg_name          ::=  [identifier | integer]
+     * attribute_name    ::=  identifier
+     * element_index     ::=  integer | index_string
+     * 

+     * 
+     * except at this point, we have already discarded the outer braces.
+     * 
+     * @param result destination chunk
+     * @param fieldMarkup specifying a replacement field, possibly with nesting
+     */
     private void parseField(Chunk result, String fieldMarkup) {
         int pos = indexOfFirst(fieldMarkup, 0, '!', ':');
         if (pos >= 0) {
+            // There's a '!' or a ':', so what precedes the first of them is a field name.
             result.fieldName = fieldMarkup.substring(0, pos);
             if (fieldMarkup.charAt(pos) == '!') {
+                // There's a conversion specifier
                 if (pos == fieldMarkup.length() - 1) {
-                    throw new IllegalArgumentException("end of format while " +
-                            "looking for conversion specifier");
+                    throw new IllegalArgumentException("end of format while "
+                            + "looking for conversion specifier");
                 }
                 result.conversion = fieldMarkup.substring(pos + 1, pos + 2);
                 pos += 2;
+                // And if that's not the end, there ought to be a ':' now.
                 if (pos < fieldMarkup.length()) {
                     if (fieldMarkup.charAt(pos) != ':') {
-                        throw new IllegalArgumentException("expected ':' " +
-                                "after conversion specifier");
+                        throw new IllegalArgumentException("expected ':' "
+                                + "after conversion specifier");
                     }
+                    // So the format specifier is from the ':' to the end.
                     result.formatSpec = fieldMarkup.substring(pos + 1);
                 }
             } else {
+                // No '!', so the format specifier is from the ':' to the end. Or empty.
                 result.formatSpec = fieldMarkup.substring(pos + 1);
             }
         } else {
+            // Neither a '!' nor a ':', the whole thing is a name.
             result.fieldName = fieldMarkup;
         }
+
         if (result.fieldName.isEmpty()) {
+            // The field was empty, so generate a number automatically.
             result.fieldName = numbering.nextAutomaticFieldNumber();
             return;
         }
+
+        // Automatic numbers must also work when there is an .attribute or [index]
         char c = result.fieldName.charAt(0);
         if (c == '.' || c == '[') {
             result.fieldName = numbering.nextAutomaticFieldNumber() + result.fieldName;
             return;
         }
+
+        // Finally, remember the argument number was specified (perhaps complain of mixed use)
         if (Character.isDigit(c)) {
             numbering.useManualFieldNumbering();
         }
     }
 
+    /** Find the first of two characters, or return -1. */
     private int indexOfFirst(String s, int start, char c1, char c2) {
         int i1 = s.indexOf(c1, start);
         int i2 = s.indexOf(c2, start);
@@ -177,32 +252,56 @@
         return Math.min(i1, i2);
     }
 
+    /**
+     * Class used locally to assign indexes to the automatically-numbered arguments (see String
+     * Formatting section of the Python Standard Library).
+     */
     static final class FieldNumbering {
+
         private boolean manualFieldNumberSpecified;
         private int automaticFieldNumber = 0;
 
+        /**
+         * Generate a numeric argument index automatically, or raise an error if already started
+         * numbering manually.
+         * 
+         * @return index as string
+         */
         String nextAutomaticFieldNumber() {
             if (manualFieldNumberSpecified) {
-                throw new IllegalArgumentException("cannot switch from manual field specification to automatic field numbering");
+                throw new IllegalArgumentException(
+                        "cannot switch from manual field specification to automatic field numbering");
             }
             return Integer.toString(automaticFieldNumber++);
         }
+
+        /**
+         * Remember we are numbering manually, and raise an error if already started numbering
+         * automatically.
+         */
         void useManualFieldNumbering() {
             if (manualFieldNumberSpecified) {
                 return;
             }
             if (automaticFieldNumber != 0) {
-                throw new IllegalArgumentException("cannot switch from automatic field numbering to manual field specification");
+                throw new IllegalArgumentException(
+                        "cannot switch from automatic field numbering to manual field specification");
             }
             manualFieldNumberSpecified = true;
         }
     }
 
     public static final class Chunk {
+
+        /** The text leading up to the next format field. */
         public String literalText;
+        /** The field name or number (as a string) for accessing the value. */
         public String fieldName;
+        /** The format specifier such as "#12x". */
         public String formatSpec;
+        /** Conversion to be applied, e.g. 'r' for repr(). */
         public String conversion;
+        /** Signals the formatSpec needs expanding recursively. */
         public boolean formatSpecNeedsExpanding;
     }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 29 17:13:12 2014
From: jython-checkins at python.org (jeff.allen)
Date: Sun, 29 Jun 2014 17:13:12 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Implement_unicode=2E=5Fform?=
 =?utf-8?q?atter=5Fparser_and_unicode=2E=5Fformatter=5Fparser=2C_fix_=2320?=
 =?utf-8?b?Mjgu?=
Message-ID: <3h1b1h6LXsz7Lk3@mail.python.org>

http://hg.python.org/jython/rev/6cffc2f6a643
changeset:   7341:6cffc2f6a643
user:        Jeff Allen 
date:        Sun Jun 29 09:15:48 2014 +0100
summary:
  Implement unicode._formatter_parser and unicode._formatter_parser, fix #2028.
This is achieved by some making stringlib/MarkupIterator and FieldNameIterator
preserve the Unicode nature of their arguments, and some minor code change
elsewhere. Tests not in CPython were added by adapting those for str.

files:
  Lib/test/test_format_jy.py                           |   76 +-
  Lib/test/test_unicode_jy.py                          |  421 +++++++++-
  src/org/python/core/PyString.java                    |  101 +-
  src/org/python/core/PyUnicode.java                   |   21 +-
  src/org/python/core/stringlib/FieldNameIterator.java |  121 ++-
  src/org/python/core/stringlib/MarkupIterator.java    |  132 ++-
  tests/java/org/python/core/StringFormatTest.java     |   71 +-
  7 files changed, 826 insertions(+), 117 deletions(-)


diff --git a/Lib/test/test_format_jy.py b/Lib/test/test_format_jy.py
--- a/Lib/test/test_format_jy.py
+++ b/Lib/test/test_format_jy.py
@@ -1,4 +1,4 @@
-"""String foramtting tests
+"""String formatting tests
 
 Made for Jython.
 """
@@ -79,6 +79,80 @@
         self.assertEqual('%     ', '%-06%' % ())
         self.assertEqual('%   ', '%*%' % -4)
 
+    def test_formatter_parser(self):
+
+        def check_parse(fmt, expected):
+            fmt_list = list(fmt._formatter_parser())
+            #print repr(fmt_list)
+            self.assertListEqual(fmt_list, expected)
+            # Tuples elements are strings with type matching fmt or are None
+            t = (type(fmt), type(None))
+            for tup in fmt_list :
+                for s in tup :
+                    self.assertIsInstance(s, t)
+
+        # Verify str._formatter_parser()
+        check_parse('{a:8.2f}', [('', 'a', '8.2f', None)])
+        check_parse('{a!r}', [('', 'a', '', 'r')])
+        check_parse('{a.b[2]!r}', [('', 'a.b[2]', '', 'r')])
+        check_parse('A={a:#12x}', [('A=', 'a', '#12x', None)])
+        check_parse('Hello {2!r:9s} world!',
+                    [('Hello ', '2', '9s', 'r'), (' world!', None, None, None)])
+
+        # Verify unicode._formatter_parser()
+        check_parse(u'{a:8.2f}', [(u'', u'a', u'8.2f', None)])
+        check_parse(u'{a!r}', [(u'', u'a', u'', u'r')])
+        check_parse(u'{a.b[2]!r}', [(u'', u'a.b[2]', u'', u'r')])
+        check_parse(u'A={a:#12x}', [(u'A=', u'a', u'#12x', None)])
+        check_parse(u'Hello {2!r:9s} world!',
+                    [(u'Hello ', u'2', u'9s', u'r'), (u' world!', None, None, None)])
+
+        # Differs from CPython: Jython str._formatter_parser generates the
+        # automatic argument number, while CPython leaves it to the client.
+        check_parse('hello {:{}d} and {:{}.{}f}',
+                    [('hello ', '0', '{}d', None), (' and ', '1', '{}.{}f', None)] )
+        check_parse('hello {[2]:{}d} and {.xx:{}.{}f}',
+                    [('hello ', '0[2]', '{}d', None), (' and ', '1.xx', '{}.{}f', None)] )
+        # The result is the same, however, of:
+        self.assertEqual('hello {:{}d} and {:{}.{}f}'.format(20, 16, 12, 8, 4),
+                      'hello               20 and  12.0000' )
+
+    def test_formatter_field_name_split(self):
+
+        def check_split(name, xfirst, xrest):
+            first, r = name._formatter_field_name_split()
+            rest = list(r)
+            #print repr(first), repr(rest)
+            self.assertEqual(first, xfirst)
+            self.assertListEqual(rest, xrest)
+            # Types ought to match the original if not numeric
+            self.assertIsInstance(first, (type(name), int, long))
+            for is_attr, i in rest :
+                if is_attr :
+                    self.assertIsInstance(i, type(name))
+                else :
+                    self.assertIsInstance(i, (int, long))
+
+        # Verify str._formatter_field_name_split()
+        check_split('a', 'a', [])
+        check_split('2', 2, [])
+        check_split('.b', '', [(True, 'b')])
+        check_split('a.b[2]', 'a', [(True, 'b'), (False, 2)])
+        check_split('a.b[2].c[7]', 'a', [(True, 'b'), (False, 2), (True, 'c'), (False, 7)])
+        check_split('.b[2].c[7]', '', [(True, 'b'), (False, 2), (True, 'c'), (False, 7)])
+        check_split('[3].b[2].c[7]', '',
+                    [(False, 3), (True, 'b'), (False, 2), (True, 'c'), (False, 7)])
+
+        # Verify unicode._formatter_field_name_split()
+        check_split(u'a', 'a', [])
+        check_split(u'2', 2, [])
+        check_split(u'.b', '', [(True, 'b')])
+        check_split(u'a.b[2]', 'a', [(True, 'b'), (False, 2)])
+        check_split(u'a.b[2].c[7]', 'a', [(True, 'b'), (False, 2), (True, 'c'), (False, 7)])
+        check_split(u'.b[2].c[7]', '', [(True, 'b'), (False, 2), (True, 'c'), (False, 7)])
+        check_split(u'[3].b[2].c[7]', '',
+                    [(False, 3), (True, 'b'), (False, 2), (True, 'c'), (False, 7)])
+
 
 def test_main():
     test_support.run_unittest(
diff --git a/Lib/test/test_unicode_jy.py b/Lib/test/test_unicode_jy.py
--- a/Lib/test/test_unicode_jy.py
+++ b/Lib/test/test_unicode_jy.py
@@ -4,6 +4,7 @@
 Made for Jython.
 """
 import re
+import string
 import sys
 import unittest
 from StringIO import StringIO
@@ -182,10 +183,424 @@
         self.assertEqual(sys.stdout.getvalue(), msg)
 
 
+class UnicodeFormatStrTest(unittest.TestCase):
+    # Adapted from test_str StrTest by liberally adding u-prefixes.
+
+    def test__format__(self):
+        def test(value, format, expected):
+            r = value.__format__(format)
+            self.assertEqual(r, expected)
+            # note 'xyz'==u'xyz', so must check type separately
+            self.assertIsInstance(r, unicode)
+            # also test both with the trailing 's'
+            r = value.__format__(format + u's')
+            self.assertEqual(r, expected)
+            self.assertIsInstance(r, unicode)
+
+        test(u'', '', '')
+        test(u'abc', '', 'abc')
+        test(u'abc', '.3', 'abc')
+        test(u'ab', '.3', 'ab')
+        test(u'abcdef', '.3', 'abc')
+        test(u'abcdef', '.0', '')
+        test(u'abc', '3.3', 'abc')
+        test(u'abc', '2.3', 'abc')
+        test(u'abc', '2.2', 'ab')
+        test(u'abc', '3.2', 'ab ')
+        test(u'result', 'x<0', 'result')
+        test(u'result', 'x<5', 'result')
+        test(u'result', 'x<6', 'result')
+        test(u'result', 'x<7', 'resultx')
+        test(u'result', 'x<8', 'resultxx')
+        test(u'result', ' <7', 'result ')
+        test(u'result', '<7', 'result ')
+        test(u'result', '>7', ' result')
+        test(u'result', '>8', '  result')
+        test(u'result', '^8', ' result ')
+        test(u'result', '^9', ' result  ')
+        test(u'result', '^10', '  result  ')
+        test(u'a', '10000', 'a' + ' ' * 9999)
+        test(u'', '10000', ' ' * 10000)
+        test(u'', '10000000', ' ' * 10000000)
+
+    def test_format(self):
+        self.assertEqual(u''.format(), '')
+        self.assertEqual(u'a'.format(), 'a')
+        self.assertEqual(u'ab'.format(), 'ab')
+        self.assertEqual(u'a{{'.format(), 'a{')
+        self.assertEqual(u'a}}'.format(), 'a}')
+        self.assertEqual(u'{{b'.format(), '{b')
+        self.assertEqual(u'}}b'.format(), '}b')
+        self.assertEqual(u'a{{b'.format(), 'a{b')
+
+        # examples from the PEP:
+        import datetime
+        self.assertEqual(u"My name is {0}".format('Fred'), "My name is Fred")
+        self.assertIsInstance(u"My name is {0}".format('Fred'), unicode)
+        self.assertEqual(u"My name is {0[name]}".format(dict(name='Fred')),
+                         "My name is Fred")
+        self.assertEqual(u"My name is {0} :-{{}}".format('Fred'),
+                         "My name is Fred :-{}")
+
+        d = datetime.date(2007, 8, 18)
+        self.assertEqual(u"The year is {0.year}".format(d),
+                         "The year is 2007")
+
+        # classes we'll use for testing
+        class C:
+            def __init__(self, x=100):
+                self._x = x
+            def __format__(self, spec):
+                return spec
+
+        class D:
+            def __init__(self, x):
+                self.x = x
+            def __format__(self, spec):
+                return str(self.x)
+
+        # class with __str__, but no __format__
+        class E:
+            def __init__(self, x):
+                self.x = x
+            def __str__(self):
+                return 'E(' + self.x + ')'
+
+        # class with __repr__, but no __format__ or __str__
+        class F:
+            def __init__(self, x):
+                self.x = x
+            def __repr__(self):
+                return 'F(' + self.x + ')'
+
+        # class with __format__ that forwards to string, for some format_spec's
+        class G:
+            def __init__(self, x):
+                self.x = x
+            def __str__(self):
+                return "string is " + self.x
+            def __format__(self, format_spec):
+                if format_spec == 'd':
+                    return 'G(' + self.x + ')'
+                return object.__format__(self, format_spec)
+
+        # class that returns a bad type from __format__
+        class H:
+            def __format__(self, format_spec):
+                return 1.0
+
+        class I(datetime.date):
+            def __format__(self, format_spec):
+                return self.strftime(format_spec)
+
+        class J(int):
+            def __format__(self, format_spec):
+                return int.__format__(self * 2, format_spec)
+
+
+        self.assertEqual(u''.format(), '')
+        self.assertEqual(u'abc'.format(), 'abc')
+        self.assertEqual(u'{0}'.format('abc'), 'abc')
+        self.assertEqual(u'{0:}'.format('abc'), 'abc')
+        self.assertEqual(u'X{0}'.format('abc'), 'Xabc')
+        self.assertEqual(u'{0}X'.format('abc'), 'abcX')
+        self.assertEqual(u'X{0}Y'.format('abc'), 'XabcY')
+        self.assertEqual(u'{1}'.format(1, 'abc'), 'abc')
+        self.assertEqual(u'X{1}'.format(1, 'abc'), 'Xabc')
+        self.assertEqual(u'{1}X'.format(1, 'abc'), 'abcX')
+        self.assertEqual(u'X{1}Y'.format(1, 'abc'), 'XabcY')
+        self.assertEqual(u'{0}'.format(-15), '-15')
+        self.assertEqual(u'{0}{1}'.format(-15, 'abc'), '-15abc')
+        self.assertEqual(u'{0}X{1}'.format(-15, 'abc'), '-15Xabc')
+        self.assertEqual(u'{{'.format(), '{')
+        self.assertEqual(u'}}'.format(), '}')
+        self.assertEqual(u'{{}}'.format(), '{}')
+        self.assertEqual(u'{{x}}'.format(), '{x}')
+        self.assertEqual(u'{{{0}}}'.format(123), '{123}')
+        self.assertEqual(u'{{{{0}}}}'.format(), '{{0}}')
+        self.assertEqual(u'}}{{'.format(), '}{')
+        self.assertEqual(u'}}x{{'.format(), '}x{')
+
+        # weird field names
+        self.assertEqual(u"{0[foo-bar]}".format({'foo-bar':'baz'}), 'baz')
+        self.assertEqual(u"{0[foo bar]}".format({'foo bar':'baz'}), 'baz')
+        self.assertEqual(u"{0[ ]}".format({' ':3}), '3')
+
+        self.assertEqual(u'{foo._x}'.format(foo=C(20)), '20')
+        self.assertEqual(u'{1}{0}'.format(D(10), D(20)), '2010')
+        self.assertEqual(u'{0._x.x}'.format(C(D('abc'))), 'abc')
+        self.assertEqual(u'{0[0]}'.format(['abc', 'def']), 'abc')
+        self.assertEqual(u'{0[1]}'.format(['abc', 'def']), 'def')
+        self.assertEqual(u'{0[1][0]}'.format(['abc', ['def']]), 'def')
+        self.assertEqual(u'{0[1][0].x}'.format(['abc', [D('def')]]), 'def')
+
+        self.assertIsInstance(u'{0[1][0].x}'.format(['abc', [D('def')]]), unicode)
+
+        # strings
+        self.assertEqual(u'{0:.3s}'.format('abc'), 'abc')
+        self.assertEqual(u'{0:.3s}'.format('ab'), 'ab')
+        self.assertEqual(u'{0:.3s}'.format('abcdef'), 'abc')
+        self.assertEqual(u'{0:.0s}'.format('abcdef'), '')
+        self.assertEqual(u'{0:3.3s}'.format('abc'), 'abc')
+        self.assertEqual(u'{0:2.3s}'.format('abc'), 'abc')
+        self.assertEqual(u'{0:2.2s}'.format('abc'), 'ab')
+        self.assertEqual(u'{0:3.2s}'.format('abc'), 'ab ')
+        self.assertEqual(u'{0:x<0s}'.format('result'), 'result')
+        self.assertEqual(u'{0:x<5s}'.format('result'), 'result')
+        self.assertEqual(u'{0:x<6s}'.format('result'), 'result')
+        self.assertEqual(u'{0:x<7s}'.format('result'), 'resultx')
+        self.assertEqual(u'{0:x<8s}'.format('result'), 'resultxx')
+        self.assertEqual(u'{0: <7s}'.format('result'), 'result ')
+        self.assertEqual(u'{0:<7s}'.format('result'), 'result ')
+        self.assertEqual(u'{0:>7s}'.format('result'), ' result')
+        self.assertEqual(u'{0:>8s}'.format('result'), '  result')
+        self.assertEqual(u'{0:^8s}'.format('result'), ' result ')
+        self.assertEqual(u'{0:^9s}'.format('result'), ' result  ')
+        self.assertEqual(u'{0:^10s}'.format('result'), '  result  ')
+        self.assertEqual(u'{0:10000}'.format('a'), 'a' + ' ' * 9999)
+        self.assertEqual(u'{0:10000}'.format(''), ' ' * 10000)
+        self.assertEqual(u'{0:10000000}'.format(''), ' ' * 10000000)
+
+        # format specifiers for user defined type
+        self.assertEqual(u'{0:abc}'.format(C()), 'abc')
+
+        # !r and !s coercions
+        self.assertEqual(u'{0!s}'.format('Hello'), 'Hello')
+        self.assertEqual(u'{0!s:}'.format('Hello'), 'Hello')
+        self.assertEqual(u'{0!s:15}'.format('Hello'), 'Hello          ')
+        self.assertEqual(u'{0!s:15s}'.format('Hello'), 'Hello          ')
+        self.assertEqual(u'{0!r}'.format('Hello'), "'Hello'")
+        self.assertEqual(u'{0!r:}'.format('Hello'), "'Hello'")
+        self.assertEqual(u'{0!r}'.format(F('Hello')), 'F(Hello)')
+
+        # test fallback to object.__format__
+        self.assertEqual(u'{0}'.format({}), '{}')
+        self.assertEqual(u'{0}'.format([]), '[]')
+        self.assertEqual(u'{0}'.format([1]), '[1]')
+        self.assertEqual(u'{0}'.format(E('data')), 'E(data)')
+        self.assertEqual(u'{0:d}'.format(G('data')), 'G(data)')
+        self.assertEqual(u'{0!s}'.format(G('data')), 'string is data')
+
+        msg = 'object.__format__ with a non-empty format string is deprecated'
+        with test_support.check_warnings((msg, PendingDeprecationWarning)):
+            self.assertEqual(u'{0:^10}'.format(E('data')), ' E(data)  ')
+            self.assertEqual(u'{0:^10s}'.format(E('data')), ' E(data)  ')
+            self.assertEqual(u'{0:>15s}'.format(G('data')), ' string is data')
+
+        #FIXME: not supported in Jython yet:
+        if not test_support.is_jython:
+            self.assertEqual(u"{0:date: %Y-%m-%d}".format(I(year=2007,
+                                                           month=8,
+                                                           day=27)),
+                             "date: 2007-08-27")
+
+            # test deriving from a builtin type and overriding __format__
+            self.assertEqual(u"{0}".format(J(10)), "20")
+
+
+        # string format specifiers
+        self.assertEqual(u'{0:}'.format('a'), 'a')
+
+        # computed format specifiers
+        self.assertEqual(u"{0:.{1}}".format('hello world', 5), 'hello')
+        self.assertEqual(u"{0:.{1}s}".format('hello world', 5), 'hello')
+        self.assertEqual(u"{0:.{precision}s}".format('hello world', precision=5), 'hello')
+        self.assertEqual(u"{0:{width}.{precision}s}".format('hello world', width=10, precision=5), 'hello     ')
+        self.assertEqual(u"{0:{width}.{precision}s}".format('hello world', width='10', precision='5'), 'hello     ')
+
+        self.assertIsInstance(u"{0:{width}.{precision}s}".format('hello world', width='10', precision='5'), unicode)
+
+        # test various errors
+        self.assertRaises(ValueError, u'{'.format)
+        self.assertRaises(ValueError, u'}'.format)
+        self.assertRaises(ValueError, u'a{'.format)
+        self.assertRaises(ValueError, u'a}'.format)
+        self.assertRaises(ValueError, u'{a'.format)
+        self.assertRaises(ValueError, u'}a'.format)
+        self.assertRaises(IndexError, u'{0}'.format)
+        self.assertRaises(IndexError, u'{1}'.format, u'abc')
+        self.assertRaises(KeyError,   u'{x}'.format)
+        self.assertRaises(ValueError, u"}{".format)
+        self.assertRaises(ValueError, u"{".format)
+        self.assertRaises(ValueError, u"}".format)
+        self.assertRaises(ValueError, u"abc{0:{}".format)
+        self.assertRaises(ValueError, u"{0".format)
+        self.assertRaises(IndexError, u"{0.}".format)
+        self.assertRaises(ValueError, u"{0.}".format, 0)
+        self.assertRaises(IndexError, u"{0[}".format)
+        self.assertRaises(ValueError, u"{0[}".format, [])
+        self.assertRaises(KeyError,   u"{0]}".format)
+        self.assertRaises(ValueError, u"{0.[]}".format, 0)
+        self.assertRaises(ValueError, u"{0..foo}".format, 0)
+        self.assertRaises(ValueError, u"{0[0}".format, 0)
+        self.assertRaises(ValueError, u"{0[0:foo}".format, 0)
+        self.assertRaises(KeyError,   u"{c]}".format)
+        self.assertRaises(ValueError, u"{{ {{{0}}".format, 0)
+        self.assertRaises(ValueError, u"{0}}".format, 0)
+        self.assertRaises(KeyError,   u"{foo}".format, bar=3)
+        self.assertRaises(ValueError, u"{0!x}".format, 3)
+        self.assertRaises(ValueError, u"{0!}".format, 0)
+        self.assertRaises(ValueError, u"{0!rs}".format, 0)
+        self.assertRaises(ValueError, u"{!}".format)
+        self.assertRaises(IndexError, u"{:}".format)
+        self.assertRaises(IndexError, u"{:s}".format)
+        self.assertRaises(IndexError, u"{}".format)
+
+        # issue 6089
+        self.assertRaises(ValueError, u"{0[0]x}".format, [None])
+        self.assertRaises(ValueError, u"{0[0](10)}".format, [None])
+
+        # can't have a replacement on the field name portion
+        self.assertRaises(TypeError, u'{0[{1}]}'.format, 'abcdefg', 4)
+
+        # exceed maximum recursion depth
+        self.assertRaises(ValueError, u"{0:{1:{2}}}".format, 'abc', 's', '')
+        self.assertRaises(ValueError, u"{0:{1:{2:{3:{4:{5:{6}}}}}}}".format,
+                          0, 1, 2, 3, 4, 5, 6, 7)
+
+        # string format spec errors
+        self.assertRaises(ValueError, u"{0:-s}".format, '')
+        self.assertRaises(ValueError, format, "", u"-")
+        self.assertRaises(ValueError, u"{0:=s}".format, '')
+
+    def test_format_auto_numbering(self):
+        class C:
+            def __init__(self, x=100):
+                self._x = x
+            def __format__(self, spec):
+                return spec
+
+        self.assertEqual(u'{}'.format(10), '10')
+        self.assertEqual(u'{:5}'.format('s'), 's    ')
+        self.assertEqual(u'{!r}'.format('s'), "'s'")
+        self.assertEqual(u'{._x}'.format(C(10)), '10')
+        self.assertEqual(u'{[1]}'.format([1, 2]), '2')
+        self.assertEqual(u'{[a]}'.format({'a':4, 'b':2}), '4')
+        self.assertEqual(u'a{}b{}c'.format(0, 1), 'a0b1c')
+
+        self.assertEqual(u'a{:{}}b'.format('x', '^10'), 'a    x     b')
+        self.assertEqual(u'a{:{}x}b'.format(20, '#'), 'a0x14b')
+
+        # can't mix and match numbering and auto-numbering
+        self.assertRaises(ValueError, u'{}{1}'.format, 1, 2)
+        self.assertRaises(ValueError, u'{1}{}'.format, 1, 2)
+        self.assertRaises(ValueError, u'{:{1}}'.format, 1, 2)
+        self.assertRaises(ValueError, u'{0:{}}'.format, 1, 2)
+
+        # can mix and match auto-numbering and named
+        self.assertEqual(u'{f}{}'.format(4, f='test'), 'test4')
+        self.assertEqual(u'{}{f}'.format(4, f='test'), '4test')
+        self.assertEqual(u'{:{f}}{g}{}'.format(1, 3, g='g', f=2), ' 1g3')
+        self.assertEqual(u'{f:{}}{}{g}'.format(2, 4, f=1, g='g'), ' 14g')
+
+
+class StringModuleUnicodeTest(unittest.TestCase):
+    # Taken from test_string ModuleTest and converted for unicode
+
+    def test_formatter(self):
+
+        def assertEqualAndUnicode(r, exp):
+            self.assertEqual(r, exp)
+            self.assertIsInstance(r, unicode)
+
+        fmt = string.Formatter()
+        assertEqualAndUnicode(fmt.format(u"foo"), "foo")
+        assertEqualAndUnicode(fmt.format(u"foo{0}", "bar"), "foobar")
+        assertEqualAndUnicode(fmt.format(u"foo{1}{0}-{1}", "bar", 6), "foo6bar-6")
+        assertEqualAndUnicode(fmt.format(u"-{arg!r}-", arg='test'), "-'test'-")
+
+        # override get_value ############################################
+        class NamespaceFormatter(string.Formatter):
+            def __init__(self, namespace={}):
+                string.Formatter.__init__(self)
+                self.namespace = namespace
+
+            def get_value(self, key, args, kwds):
+                if isinstance(key, (str, unicode)):
+                    try:
+                        # Check explicitly passed arguments first
+                        return kwds[key]
+                    except KeyError:
+                        return self.namespace[key]
+                else:
+                    string.Formatter.get_value(key, args, kwds)
+
+        fmt = NamespaceFormatter({'greeting':'hello'})
+        assertEqualAndUnicode(fmt.format(u"{greeting}, world!"), 'hello, world!')
+
+
+        # override format_field #########################################
+        class CallFormatter(string.Formatter):
+            def format_field(self, value, format_spec):
+                return format(value(), format_spec)
+
+        fmt = CallFormatter()
+        assertEqualAndUnicode(fmt.format(u'*{0}*', lambda : 'result'), '*result*')
+
+
+        # override convert_field ########################################
+        class XFormatter(string.Formatter):
+            def convert_field(self, value, conversion):
+                if conversion == 'x':
+                    return None
+                return super(XFormatter, self).convert_field(value, conversion)
+
+        fmt = XFormatter()
+        assertEqualAndUnicode(fmt.format(u"{0!r}:{0!x}", 'foo', 'foo'), "'foo':None")
+
+
+        # override parse ################################################
+        class BarFormatter(string.Formatter):
+            # returns an iterable that contains tuples of the form:
+            # (literal_text, field_name, format_spec, conversion)
+            def parse(self, format_string):
+                for field in format_string.split('|'):
+                    if field[0] == '+':
+                        # it's markup
+                        field_name, _, format_spec = field[1:].partition(':')
+                        yield '', field_name, format_spec, None
+                    else:
+                        yield field, None, None, None
+
+        fmt = BarFormatter()
+        assertEqualAndUnicode(fmt.format(u'*|+0:^10s|*', 'foo'), '*   foo    *')
+
+        # test all parameters used
+        class CheckAllUsedFormatter(string.Formatter):
+            def check_unused_args(self, used_args, args, kwargs):
+                # Track which arguments actually got used
+                unused_args = set(kwargs.keys())
+                unused_args.update(range(0, len(args)))
+
+                for arg in used_args:
+                    unused_args.remove(arg)
+
+                if unused_args:
+                    raise ValueError("unused arguments")
+
+        fmt = CheckAllUsedFormatter()
+        # The next series should maybe also call assertEqualAndUnicode:
+        #assertEqualAndUnicode(fmt.format(u"{0}", 10), "10")
+        #assertEqualAndUnicode(fmt.format(u"{0}{i}", 10, i=100), "10100")
+        #assertEqualAndUnicode(fmt.format(u"{0}{i}{1}", 10, 20, i=100), "1010020")
+        # But string.Formatter.format returns bytes. See CPython Issue 15951.
+        self.assertEqual(fmt.format(u"{0}", 10), "10")
+        self.assertEqual(fmt.format(u"{0}{i}", 10, i=100), "10100")
+        self.assertEqual(fmt.format(u"{0}{i}{1}", 10, 20, i=100), "1010020")
+        self.assertRaises(ValueError, fmt.format, u"{0}{i}{1}", 10, 20, i=100, j=0)
+        self.assertRaises(ValueError, fmt.format, u"{0}", 10, 20)
+        self.assertRaises(ValueError, fmt.format, u"{0}", 10, 20, i=100)
+        self.assertRaises(ValueError, fmt.format, u"{i}", 10, 20, i=100)
+
+
 def test_main():
-    test_support.run_unittest(UnicodeTestCase,
-                              UnicodeFormatTestCase,
-                              UnicodeStdIOTestCase)
+    test_support.run_unittest(
+                UnicodeTestCase,
+                UnicodeFormatTestCase,
+                UnicodeStdIOTestCase,
+                UnicodeFormatStrTest,
+                StringModuleUnicodeTest,
+            )
 
 
 if __name__ == "__main__":
diff --git a/src/org/python/core/PyString.java b/src/org/python/core/PyString.java
--- a/src/org/python/core/PyString.java
+++ b/src/org/python/core/PyString.java
@@ -662,6 +662,13 @@
         }
     }
 
+    /**
+     * Create an instance of the same type as this object, from the Java String given as argument.
+     * This is to be overridden in a subclass to return its own type.
+     *
+     * @param str to wrap
+     * @return
+     */
     public PyString createInstance(String str) {
         return new PyString(str);
     }
@@ -3740,23 +3747,19 @@
 
     @ExposedMethod(doc = BuiltinDocs.str__formatter_parser_doc)
     final PyObject str__formatter_parser() {
-        return new MarkupIterator(getString());
+        return new MarkupIterator(this);
     }
 
     @ExposedMethod(doc = BuiltinDocs.str__formatter_field_name_split_doc)
     final PyObject str__formatter_field_name_split() {
-        FieldNameIterator iterator = new FieldNameIterator(getString());
-        Object headObj = iterator.head();
-        PyObject head =
-                headObj instanceof Integer ? new PyInteger((Integer)headObj) : new PyString(
-                        (String)headObj);
-        return new PyTuple(head, iterator);
+        FieldNameIterator iterator = new FieldNameIterator(this);
+        return new PyTuple(iterator.pyHead(), iterator);
     }
 
     @ExposedMethod(doc = BuiltinDocs.str_format_doc)
     final PyObject str_format(PyObject[] args, String[] keywords) {
         try {
-            return new PyString(buildFormattedString(getString(), args, keywords, null));
+            return new PyString(buildFormattedString(args, keywords, null, null));
         } catch (IllegalArgumentException e) {
             throw Py.ValueError(e.getMessage());
         }
@@ -3764,18 +3767,33 @@
 
     /**
      * Implements PEP-3101 {}-formatting methods str.format() and
-     * unicode.format().
+     * unicode.format(). When called with enclosingIterator == null, this
+     * method takes this object as its formatting string. The method is also called (calls itself)
+     * to deal with nested formatting sepecifications. In that case, enclosingIterator
+     * is a {@link MarkupIterator} on this object and value is a substring of this
+     * object needing recursive transaltion.
      *
-     * @param value the format string
      * @param args to be interpolated into the string
      * @param keywords for the trailing args
-     * @param enclosingIterator when used nested
+     * @param enclosingIterator when used nested, null if subject is this PyString
+     * @param value the format string when enclosingIterator is not null
      * @return the formatted string based on the arguments
      */
-    protected String buildFormattedString(String value, PyObject[] args, String[] keywords,
-            MarkupIterator enclosingIterator) {
+    protected String buildFormattedString(PyObject[] args, String[] keywords,
+            MarkupIterator enclosingIterator, String value) {
+
+        MarkupIterator it;
+        if (enclosingIterator == null) {
+            // Top-level call acts on this object.
+            it = new MarkupIterator(this);
+        } else {
+            // Nested call acts on the substring and some state from existing iterator.
+            it = new MarkupIterator(enclosingIterator, value);
+        }
+
+        // Result will be formed here
         StringBuilder result = new StringBuilder();
-        MarkupIterator it = new MarkupIterator(value, enclosingIterator);
+
         while (true) {
             MarkupIterator.Chunk chunk = it.nextChunk();
             if (chunk == null) {
@@ -3784,12 +3802,12 @@
             // A Chunk encapsulates a literal part ...
             result.append(chunk.literalText);
             // ... and the parsed form of the replacement field that followed it (if any)
-            if (chunk.fieldName.length() > 0) {
+            if (chunk.fieldName != null) {
                 // The grammar of the replacement field is:
                 // "{" [field_name] ["!" conversion] [":" format_spec] "}"
 
                 // Get the object referred to by the field name (which may be omitted).
-                PyObject fieldObj = getFieldObject(chunk.fieldName, args, keywords);
+                PyObject fieldObj = getFieldObject(chunk.fieldName, it.isBytes(), args, keywords);
                 if (fieldObj == null) {
                     continue;
                 }
@@ -3811,7 +3829,7 @@
                         throw Py.ValueError("Max string recursion exceeded");
                     }
                     // Recursively interpolate further args into chunk.formatSpec
-                    formatSpec = buildFormattedString(formatSpec, args, keywords, it);
+                    formatSpec = buildFormattedString(args, keywords, it, formatSpec);
                 }
                 renderField(fieldObj, formatSpec, result);
             }
@@ -3824,51 +3842,56 @@
      * argument list, containing positional and keyword arguments.
      *
      * @param fieldName to interpret.
+     * @param bytes true if the field name is from a PyString, false for PyUnicode.
      * @param args argument list (positional then keyword arguments).
      * @param keywords naming the keyword arguments.
      * @return the object designated or null.
      */
-    private PyObject getFieldObject(String fieldName, PyObject[] args, String[] keywords) {
-        FieldNameIterator iterator = new FieldNameIterator(fieldName);
-        Object head = iterator.head();
+    private PyObject getFieldObject(String fieldName, boolean bytes, PyObject[] args, String[] keywords) {
+        FieldNameIterator iterator = new FieldNameIterator(fieldName, bytes);
+        PyObject head = iterator.pyHead();
         PyObject obj = null;
         int positionalCount = args.length - keywords.length;
 
-        if (head instanceof Integer) {
-            int index = (Integer)head;
+        if (head.isIndex()) {
+            // The field name begins with an integer argument index (not a [n]-type index).
+            int index = head.asIndex();
             if (index >= positionalCount) {
                 throw Py.IndexError("tuple index out of range");
             }
             obj = args[index];
 
         } else {
+            // The field name begins with keyword.
             for (int i = 0; i < keywords.length; i++) {
-                if (keywords[i].equals(head)) {
+                if (keywords[i].equals(head.asString())) {
                     obj = args[positionalCount + i];
                     break;
                 }
             }
+            // And if we don't find it, that's an error
             if (obj == null) {
-                throw Py.KeyError((String)head);
+                throw Py.KeyError(head);
             }
         }
 
-        if (obj != null) {
-            while (true) {
-                FieldNameIterator.Chunk chunk = iterator.nextChunk();
-                if (chunk == null) {
-                    break;
-                }
-                if (chunk.is_attr) {
-                    obj = obj.__getattr__((String)chunk.value);
+        // Now deal with the iterated sub-fields
+        while (obj != null) {
+            FieldNameIterator.Chunk chunk = iterator.nextChunk();
+            if (chunk == null) {
+                // End of iterator
+                break;
+            }
+            Object key = chunk.value;
+            if (chunk.is_attr) {
+                // key must be a String
+                obj = obj.__getattr__((String)key);
+            } else {
+                if (key instanceof Integer) {
+                    // Can this happen?
+                    obj = obj.__getitem__(((Integer)key).intValue());
                 } else {
-                    PyObject key =
-                            chunk.value instanceof String ? new PyString((String)chunk.value)
-                                    : new PyInteger((Integer)chunk.value);
-                    obj = obj.__getitem__(key);
-                }
-                if (obj == null) {
-                    break;
+                    obj = obj.__getitem__(new PyString(key.toString()));
                 }
             }
         }
diff --git a/src/org/python/core/PyUnicode.java b/src/org/python/core/PyUnicode.java
--- a/src/org/python/core/PyUnicode.java
+++ b/src/org/python/core/PyUnicode.java
@@ -7,6 +7,8 @@
 import java.util.List;
 import java.util.Set;
 
+import org.python.core.stringlib.FieldNameIterator;
+import org.python.core.stringlib.MarkupIterator;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedNew;
 import org.python.expose.ExposedType;
@@ -1431,13 +1433,30 @@
 
     @Override
     public PyObject __format__(PyObject formatSpec) {
+        return unicode___format__(formatSpec);
+    }
+
+    @ExposedMethod(doc = BuiltinDocs.unicode___format___doc)
+    final PyObject unicode___format__(PyObject formatSpec) {
+        // Re-use the str implementation, which adapts itself to unicode.
         return str___format__(formatSpec);
     }
 
+    @ExposedMethod(doc = BuiltinDocs.unicode__formatter_parser_doc)
+    final PyObject unicode__formatter_parser() {
+        return new MarkupIterator(this);
+    }
+
+    @ExposedMethod(doc = BuiltinDocs.unicode__formatter_field_name_split_doc)
+    final PyObject unicode__formatter_field_name_split() {
+        FieldNameIterator iterator = new FieldNameIterator(this);
+        return new PyTuple(iterator.pyHead(), iterator);
+    }
+
     @ExposedMethod(doc = BuiltinDocs.unicode_format_doc)
     final PyObject unicode_format(PyObject[] args, String[] keywords) {
         try {
-            return new PyUnicode(buildFormattedString(getString(), args, keywords, null));
+            return new PyUnicode(buildFormattedString(args, keywords, null, null));
         } catch (IllegalArgumentException e) {
             throw Py.ValueError(e.getMessage());
         }
diff --git a/src/org/python/core/stringlib/FieldNameIterator.java b/src/org/python/core/stringlib/FieldNameIterator.java
--- a/src/org/python/core/stringlib/FieldNameIterator.java
+++ b/src/org/python/core/stringlib/FieldNameIterator.java
@@ -1,30 +1,61 @@
 package org.python.core.stringlib;
 
+import org.python.core.Py;
 import org.python.core.PyBoolean;
 import org.python.core.PyInteger;
 import org.python.core.PyObject;
 import org.python.core.PyString;
 import org.python.core.PyTuple;
 import org.python.core.PyType;
+import org.python.core.PyUnicode;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedType;
 
 /**
- * Provides an implementation of str._formatter_field_name_split()
+ * This class is an implementation of the iterator object returned by
+ * str._formatter_field_name_split() and
+ * unicode._formatter_field_name_split(). The function
+ * _formatter_field_name_split() returns a pair (tuple) consisting of a head element
+ * and an instance of this iterator. The constructor of this class effectively implements that
+ * function, since as well as "being" the iterator, the object has an extra method {@link #head()}
+ * to return the required first member of the pair.
  */
 @ExposedType(name = "fieldnameiterator", base = PyObject.class, isBaseType = false)
 public class FieldNameIterator extends PyObject {
 
     public static final PyType TYPE = PyType.fromClass(FieldNameIterator.class);
 
-    private String markup;
+    /** The UTF-16 string from which elements are being returned. */
+    private final String markup;
+    /** True if originally given a PyString (so must return PyString not PyUnicode). */
+    private final boolean bytes;
+    /** How far along that string we are. */
+    private int index;
     private Object head;
-    private int index;
 
-    public FieldNameIterator(String markup) {
-        this.markup = markup;
-        this.index = nextDotOrBracket(markup);
-        String headStr = markup.substring(0, index);
+    /**
+     * Create an iterator for the parts of this field name (and extract the head name field, which
+     * may be an empty string). According to the Python Standard Library documentation, a
+     * replacement field name has the structure:
+     *
+     * 
+     * field_name        ::=  arg_name ("." attribute_name | "[" element_index "]")*
+     * arg_name          ::=  [identifier | integer]
+     * attribute_name    ::=  identifier
+     * element_index     ::=  integer | index_string
+     * 

+     *
+     * The object is used from PyUnicode and from PyString, and we have to signal which it is, so
+     * that returned values may match in type.
+     *
+     * @param fieldName the field name as UTF-16
+     * @param bytes true if elements returned should be PyString, else PyUnicode
+     */
+    public FieldNameIterator(String fieldName, boolean bytes) {
+        this.markup = fieldName;
+        this.bytes = bytes;
+        this.index = nextDotOrBracket(fieldName);
+        String headStr = fieldName.substring(0, index);
         try {
             this.head = Integer.parseInt(headStr);
         } catch (NumberFormatException e) {
@@ -32,6 +63,17 @@
         }
     }
 
+    /**
+     * Create an iterator for the parts of this field name (and extract the head name field, which
+     * may be an empty string).
+     *
+     * @param fieldNameObject
+     */
+    public FieldNameIterator(PyString fieldNameObject) {
+        // Extract UTF-16 string but remember whether PyString or PyUnicode shouyld result.
+        this(fieldNameObject.getString(), !(fieldNameObject instanceof PyUnicode));
+    }
+
     @Override
     public PyObject __iter__() {
         return fieldnameiterator___iter__();
@@ -53,14 +95,31 @@
         if (chunk == null) {
             return null;
         }
-        PyObject[] elements = new PyObject[2];
-        elements [0] = new PyBoolean(chunk.is_attr);
-        if (chunk.value instanceof Integer) {
-            elements [1] = new PyInteger((Integer) chunk.value);
+        return new PyTuple(Py.newBoolean(chunk.is_attr), wrap(chunk.value));
+    }
+
+    /**
+     * Convenience method to wrap a value as a PyInteger, if it is an Integer, or as
+     * PyString or PyUnicode according to the type of the original field
+     * name string. These objects are being used as field specifiers in navigating arguments to a
+     * format statement.
+     *
+     * @param value to wrap as a PyObject.
+     * @return PyObject equivalent field specifier
+     */
+    private PyObject wrap(Object value) {
+        if (value instanceof Integer) {
+            return Py.newInteger(((Integer)value).intValue());
         } else {
-            elements [1] = new PyString((String) chunk.value);
+            // It can only be a String (but if not, at least we see it).
+            String s = value.toString();
+            if (s.length() == 0) {
+                // This is frequent so avoid the constructor
+                return bytes ? Py.EmptyString : Py.EmptyUnicode;
+            } else {
+                return bytes ? Py.newString(s) : Py.newUnicode(s);
+            }
         }
-        return new PyTuple(elements);
     }
 
     private int nextDotOrBracket(String markup) {
@@ -75,10 +134,40 @@
         return Math.min(dotPos, bracketPos);
     }
 
+    /** @return the isolated head object from the field name. */
     public Object head() {
         return head;
     }
 
+    /**
+     * Return the head object from the field name, as PyInteger, PyString
+     * or PyUnicode.
+     *
+     * @return the isolated head object from the field name.
+     */
+    public PyObject pyHead() {
+        return wrap(head());
+    }
+
+    /**
+     * If originally given a PyString, the iterator must return PyString not PyUnicode.
+     *
+     * @return true if originally given a PyString
+     */
+    public final boolean isBytes() {
+        return bytes;
+    }
+
+    /**
+     * Return the next "chunk" of the field name (or return null if ended). A chunk is a 2-tuple
+     * describing:
+     * 
    
+     * 
  1. whether the chunk is an attribute name,
    2. 
+     * 
  2. the name or number (as a String or Integer) for accessing the value.
    3. 
+     * 

+     *
+     * @return next element of the field name
+     */
     public Chunk nextChunk() {
         if (index == markup.length()) {
             return null;
@@ -89,14 +178,15 @@
         } else if (markup.charAt(index) == '.') {
             parseAttrChunk(chunk);
         } else {
-            throw new IllegalArgumentException("Only '.' or '[' may follow ']' in format field specifier");
+            throw new IllegalArgumentException(
+                    "Only '.' or '[' may follow ']' in format field specifier");
         }
         return chunk;
     }
 
     private void parseItemChunk(Chunk chunk) {
         chunk.is_attr = false;
-        int endBracket = markup.indexOf(']', index+1);
+        int endBracket = markup.indexOf(']', index + 1);
         if (endBracket < 0) {
             throw new IllegalArgumentException("Missing ']' in format string");
         }
@@ -124,6 +214,7 @@
     }
 
     public static class Chunk {
+
         public boolean is_attr;
         /** Integer or String. */
         public Object value;
diff --git a/src/org/python/core/stringlib/MarkupIterator.java b/src/org/python/core/stringlib/MarkupIterator.java
--- a/src/org/python/core/stringlib/MarkupIterator.java
+++ b/src/org/python/core/stringlib/MarkupIterator.java
@@ -5,6 +5,7 @@
 import org.python.core.PyString;
 import org.python.core.PyTuple;
 import org.python.core.PyType;
+import org.python.core.PyUnicode;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedType;
 
@@ -18,26 +19,27 @@
 
     public static final PyType TYPE = PyType.fromClass(MarkupIterator.class);
 
-    /** The string from which elements are being returned. */
+    /** The UTF-16 string from which elements are being returned. */
     private final String markup;
+    /** True if originally given a PyString (so must return PyString not PyUnicode). */
+    private final boolean bytes;
     /** How far along that string we are. */
     private int index;
     /** A counter used to auto-number fields when not explicitly numbered in the format. */
     private final FieldNumbering numbering;
 
     /** Constructor used at top-level to enumerate a format. */
-    public MarkupIterator(String markup) {
-        this(markup, null);
+    public MarkupIterator(PyString markupObject) {
+        markup = markupObject.getString();
+        bytes = !(markupObject instanceof PyUnicode);
+        numbering = new FieldNumbering();
     }
 
     /** Variant constructor used when formats are nested. */
-    public MarkupIterator(String markup, MarkupIterator enclosingIterator) {
-        this.markup = markup;
-        if (enclosingIterator != null) {
-            numbering = enclosingIterator.numbering;
-        } else {
-            numbering = new FieldNumbering();
-        }
+    public MarkupIterator(MarkupIterator enclosingIterator, String subMarkup) {
+        markup = subMarkup;
+        bytes = enclosingIterator.bytes;
+        numbering = enclosingIterator.numbering;
     }
 
     @Override
@@ -67,41 +69,82 @@
      * 
      * Elements 1-3 are None if this chunk contains no format specifier. Elements 0-2 are
      * zero-length strings if missing from the format, while element 3 will be None if missing.
-     * 
+     *
      * @return PyTuple chunk or null
      */
     @ExposedMethod
     final PyObject formatteriterator___iternext__() {
-        Chunk chunk;
         try {
-            chunk = nextChunk();
+            // Parse off the next literal text and replacement field
+            Chunk chunk = nextChunk();
+
+            if (chunk != null) {
+                // Result will be built here
+                PyObject[] elements = new PyObject[4];
+
+                // Literal text is used verbatim.
+                elements[0] = wrap(chunk.literalText, "");
+
+                if (chunk.fieldName == null) {
+                    // A field name is null only if there was no replacement field at all.
+                    for (int i = 1; i < elements.length; i++) {
+                        elements[i] = Py.None;
+                    }
+
+                } else {
+                    // Otherwise, this is the field name
+                    elements[1] = wrap(chunk.fieldName, "");
+                    // The format spec may be blank
+                    elements[2] = wrap(chunk.formatSpec, "");
+                    // There may have been a conversion specifier (if not, None is signalled).
+                    elements[3] = wrap(chunk.conversion, null);
+                }
+
+                // And those make up the next answer.
+                return new PyTuple(elements);
+
+            } else {
+                // End of format: end of iteration
+                return null;
+            }
+
         } catch (IllegalArgumentException e) {
             throw Py.ValueError(e.getMessage());
         }
-        if (chunk == null) {
-            return null;
-        }
-        PyObject[] elements = new PyObject[4];
-
-        // Literal text is used verbatim.
-        elements[0] = new PyString(chunk.literalText);
-
-        // A field name is empty only if there was no format at all.
-        elements[1] = chunk.fieldName.length() == 0 ? Py.None : new PyString(chunk.fieldName);
-        if (chunk.fieldName.length() > 0) {
-            elements[2] =
-                    chunk.formatSpec == null ? Py.EmptyString : new PyString(chunk.formatSpec);
-        } else {
-            elements[2] = Py.None;
-        }
-
-        // There may have been a conversion specifier.
-        elements[3] = chunk.conversion == null ? Py.None : new PyString(chunk.conversion);
-
-        // And those make up the next answer.
-        return new PyTuple(elements);
     }
 
+    /**
+     * Convenience method for populating the return tuple, returning a PyString or
+     * PyUnicode according to the type of the original markup string, or
+     * Py.None if both arguments are null.
+     *
+     * @param value to wrap as a PyObject or null if defaultValue should be wrapped.
+     * @param defaultValue to return or null if default return is None.
+     * @return object for tuple
+     */
+    private PyObject wrap(String value, String defaultValue) {
+        if (value == null) {
+            value = defaultValue;
+        }
+        if (value == null) {
+            // It's still null, we want a None
+            return Py.None;
+        } else if (value.length() == 0) {
+            // This is frequent so avoid the constructor
+            return bytes ? Py.EmptyString : Py.EmptyUnicode;
+        } else {
+            return bytes ? Py.newString(value) : Py.newUnicode(value);
+        }
+    }
+
+    /**
+     * Return the next {@link Chunk} from the iterator, which is a structure containing parsed
+     * elements of the replacement field (if any), and its preceding text. This is the Java
+     * equivalent of the tuple returned by {@link __iternext__()}. This finds use in the
+     * implementation of str.format and unicode.format.
+     *
+     * @return the chunk
+     */
     public Chunk nextChunk() {
         if (index == markup.length()) {
             return null;
@@ -131,7 +174,6 @@
         if (pos < 0) {
             // ... except pos<0, and there is no further format specifier, only literal text.
             result.literalText = unescapeBraces(markup.substring(index));
-            result.fieldName = "";
             index = markup.length();
 
         } else {
@@ -167,6 +209,16 @@
         return result;
     }
 
+    /**
+     * If originally given a PyString, string elements in the returned tuples must be PyString not
+     * PyUnicode.
+     *
+     * @return true if originally given a PyString
+     */
+    public final boolean isBytes() {
+        return bytes;
+    }
+
     private String unescapeBraces(String substring) {
         return substring.replace("{{", "{").replace("}}", "}");
     }
@@ -175,7 +227,7 @@
      * Parse a "replacement field" consisting of a name, conversion and format specification.
      * According to the Python Standard Library documentation, a replacement field has the
      * structure:
-     * 
+     *
      * 
      * replacement_field ::=  "{" [field_name] ["!" conversion] [":" format_spec] "}"
      * field_name        ::=  arg_name ("." attribute_name | "[" element_index "]")*
@@ -183,9 +235,9 @@
      * attribute_name    ::=  identifier
      * element_index     ::=  integer | index_string
      * 

-     * 
+     *
      * except at this point, we have already discarded the outer braces.
-     * 
+     *
      * @param result destination chunk
      * @param fieldMarkup specifying a replacement field, possibly with nesting
      */
@@ -264,7 +316,7 @@
         /**
          * Generate a numeric argument index automatically, or raise an error if already started
          * numbering manually.
-         * 
+         *
          * @return index as string
          */
         String nextAutomaticFieldNumber() {
diff --git a/tests/java/org/python/core/StringFormatTest.java b/tests/java/org/python/core/StringFormatTest.java
--- a/tests/java/org/python/core/StringFormatTest.java
+++ b/tests/java/org/python/core/StringFormatTest.java
@@ -20,6 +20,9 @@
     /** Exception-raising seems to need the interpreter to be initialised **/
     PythonInterpreter interp = new PythonInterpreter();
 
+    /** Switches mode in tests that have a shared implementation for bytes and Unicode modes. */
+    private boolean useBytes = true;
+
     public void testInternalFormatSpec() {
         InternalFormat.Spec spec;
         spec = InternalFormat.fromText("x");
@@ -76,8 +79,9 @@
     }
 
     /**
-     * Test the IntegerFormatter returned by {@link PyInteger#prepareFormat}. This is based on the original
-     * testFormatIntOrLong which tested PyInteger.formatIntOrLong.
+     * Test the IntegerFormatter returned by {@link PyInteger#prepareFormat}. This is based on the
+     * original testFormatIntOrLong which tested PyInteger.formatIntOrLong
+     * .
      */
     public void testPrepareFormatter() {
         int v = 123;
@@ -137,8 +141,9 @@
     }
 
     /**
-     * Test the IntegerFormatter returned by {@link PyInteger#prepareFormat}. This is based on the original
-     * testFormatIntOrLong which tested PyInteger.formatIntOrLong.
+     * Test the IntegerFormatter returned by {@link PyInteger#prepareFormat}. This is based on the
+     * original testFormatIntOrLong which tested PyInteger.formatIntOrLong
+     * .
      */
     public void testPrepareFormatterLong() {
         BigInteger v = BigInteger.valueOf(123);
@@ -238,34 +243,34 @@
         assertEquals("abc   ", f.format(v).pad().getResult());
     }
 
-    public void testMarkupIterator() {
-        MarkupIterator iterator = new MarkupIterator("abc");
+    public void implTestMarkupIterator() {
+        MarkupIterator iterator = newMarkupIterator("abc");
         assertEquals("abc", iterator.nextChunk().literalText);
         assertNull(iterator.nextChunk());
 
-        iterator = new MarkupIterator("First, thou shalt count to {0}");
+        iterator = newMarkupIterator("First, thou shalt count to {0}");
         MarkupIterator.Chunk chunk = iterator.nextChunk();
         assertEquals("First, thou shalt count to ", chunk.literalText);
         assertEquals("0", chunk.fieldName);
         assertNull(iterator.nextChunk());
 
-        iterator = new MarkupIterator("Weight in tons {0.weight!r:s}");
+        iterator = newMarkupIterator("Weight in tons {0.weight!r:s}");
         chunk = iterator.nextChunk();
         assertEquals("Weight in tons ", chunk.literalText);
         assertEquals("0.weight", chunk.fieldName);
         assertEquals("r", chunk.conversion);
         assertEquals("s", chunk.formatSpec);
 
-        chunk = new MarkupIterator("{{").nextChunk();
+        chunk = newMarkupIterator("{{").nextChunk();
         assertEquals("{", chunk.literalText);
 
-        chunk = new MarkupIterator("}}").nextChunk();
+        chunk = newMarkupIterator("}}").nextChunk();
         assertEquals("}", chunk.literalText);
 
-        chunk = new MarkupIterator("{{}}").nextChunk();
+        chunk = newMarkupIterator("{{}}").nextChunk();
         assertEquals("{}", chunk.literalText);
 
-        chunk = new MarkupIterator("{0:.{1}}").nextChunk();
+        chunk = newMarkupIterator("{0:.{1}}").nextChunk();
         assertEquals("0", chunk.fieldName);
         assertEquals(".{1}", chunk.formatSpec);
         assertTrue(chunk.formatSpecNeedsExpanding);
@@ -276,8 +281,23 @@
         assertMarkupError("}", "Single '}' encountered in format string");
     }
 
+    public void testMarkupIteratorBytes() {
+        useBytes = true;
+        implTestMarkupIterator();
+    }
+
+    public void testMarkupIteratorUnicode() {
+        useBytes = false;
+        implTestMarkupIterator();
+    }
+
+    private MarkupIterator newMarkupIterator(String markup) {
+        PyString markupObject = useBytes ? Py.newString(markup) : Py.newUnicode(markup);
+        return new MarkupIterator(markupObject);
+    }
+
     private void assertMarkupError(String markup, String expected) {
-        MarkupIterator iterator = new MarkupIterator(markup);
+        MarkupIterator iterator = newMarkupIterator(markup);
         String error = null;
         try {
             iterator.nextChunk();
@@ -287,27 +307,42 @@
         assertEquals(expected, error);
     }
 
-    public void testFieldNameIterator() {
-        FieldNameIterator it = new FieldNameIterator("abc");
+    public void implTestFieldNameIterator() {
+        FieldNameIterator it = newFieldNameIterator("abc");
         assertEquals("abc", it.head());
         assertNull(it.nextChunk());
 
-        it = new FieldNameIterator("3");
+        it = newFieldNameIterator("3");
         assertEquals(3, it.head());
         assertNull(it.nextChunk());
 
-        it = new FieldNameIterator("abc[0]");
+        it = newFieldNameIterator("abc[0]");
         assertEquals("abc", it.head());
         FieldNameIterator.Chunk chunk = it.nextChunk();
         assertEquals(0, chunk.value);
         assertFalse(chunk.is_attr);
         assertNull(it.nextChunk());
 
-        it = new FieldNameIterator("abc.def");
+        it = newFieldNameIterator("abc.def");
         assertEquals("abc", it.head());
         chunk = it.nextChunk();
         assertEquals("def", chunk.value);
         assertTrue(chunk.is_attr);
         assertNull(it.nextChunk());
     }
+
+    public void testFieldNameIteratorBytes() {
+        useBytes = true;
+        implTestFieldNameIterator();
+    }
+
+    public void testFieldNameIteratorUnicode() {
+        useBytes = false;
+        implTestFieldNameIterator();
+    }
+
+    private FieldNameIterator newFieldNameIterator(String field) {
+        PyString fieldObject = useBytes ? Py.newString(field) : Py.newUnicode(field);
+        return new FieldNameIterator(fieldObject);
+    }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sun Jun 29 17:13:14 2014
From: jython-checkins at python.org (jeff.allen)
Date: Sun, 29 Jun 2014 17:13:14 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython_=28merge_default_-=3E_default?=
 =?utf-8?q?=29=3A_Merge_fix_for_=232028_to_trunk?=
Message-ID: <3h1b1k51rgz7Ljq@mail.python.org>

http://hg.python.org/jython/rev/4d451f2d023c
changeset:   7342:4d451f2d023c
parent:      7339:5efdcedc9817
parent:      7341:6cffc2f6a643
user:        Jeff Allen 
date:        Sun Jun 29 16:12:55 2014 +0100
summary:
  Merge fix for #2028 to trunk

files:
  Lib/test/test_format_jy.py                           |   76 +-
  Lib/test/test_unicode_jy.py                          |  421 +++++++++-
  src/org/python/core/PyString.java                    |  101 +-
  src/org/python/core/PyUnicode.java                   |   21 +-
  src/org/python/core/stringlib/FieldNameIterator.java |  121 ++-
  src/org/python/core/stringlib/MarkupIterator.java    |  225 ++++-
  tests/java/org/python/core/StringFormatTest.java     |   71 +-
  7 files changed, 922 insertions(+), 114 deletions(-)


diff --git a/Lib/test/test_format_jy.py b/Lib/test/test_format_jy.py
--- a/Lib/test/test_format_jy.py
+++ b/Lib/test/test_format_jy.py
@@ -1,4 +1,4 @@
-"""String foramtting tests
+"""String formatting tests
 
 Made for Jython.
 """
@@ -79,6 +79,80 @@
         self.assertEqual('%     ', '%-06%' % ())
         self.assertEqual('%   ', '%*%' % -4)
 
+    def test_formatter_parser(self):
+
+        def check_parse(fmt, expected):
+            fmt_list = list(fmt._formatter_parser())
+            #print repr(fmt_list)
+            self.assertListEqual(fmt_list, expected)
+            # Tuples elements are strings with type matching fmt or are None
+            t = (type(fmt), type(None))
+            for tup in fmt_list :
+                for s in tup :
+                    self.assertIsInstance(s, t)
+
+        # Verify str._formatter_parser()
+        check_parse('{a:8.2f}', [('', 'a', '8.2f', None)])
+        check_parse('{a!r}', [('', 'a', '', 'r')])
+        check_parse('{a.b[2]!r}', [('', 'a.b[2]', '', 'r')])
+        check_parse('A={a:#12x}', [('A=', 'a', '#12x', None)])
+        check_parse('Hello {2!r:9s} world!',
+                    [('Hello ', '2', '9s', 'r'), (' world!', None, None, None)])
+
+        # Verify unicode._formatter_parser()
+        check_parse(u'{a:8.2f}', [(u'', u'a', u'8.2f', None)])
+        check_parse(u'{a!r}', [(u'', u'a', u'', u'r')])
+        check_parse(u'{a.b[2]!r}', [(u'', u'a.b[2]', u'', u'r')])
+        check_parse(u'A={a:#12x}', [(u'A=', u'a', u'#12x', None)])
+        check_parse(u'Hello {2!r:9s} world!',
+                    [(u'Hello ', u'2', u'9s', u'r'), (u' world!', None, None, None)])
+
+        # Differs from CPython: Jython str._formatter_parser generates the
+        # automatic argument number, while CPython leaves it to the client.
+        check_parse('hello {:{}d} and {:{}.{}f}',
+                    [('hello ', '0', '{}d', None), (' and ', '1', '{}.{}f', None)] )
+        check_parse('hello {[2]:{}d} and {.xx:{}.{}f}',
+                    [('hello ', '0[2]', '{}d', None), (' and ', '1.xx', '{}.{}f', None)] )
+        # The result is the same, however, of:
+        self.assertEqual('hello {:{}d} and {:{}.{}f}'.format(20, 16, 12, 8, 4),
+                      'hello               20 and  12.0000' )
+
+    def test_formatter_field_name_split(self):
+
+        def check_split(name, xfirst, xrest):
+            first, r = name._formatter_field_name_split()
+            rest = list(r)
+            #print repr(first), repr(rest)
+            self.assertEqual(first, xfirst)
+            self.assertListEqual(rest, xrest)
+            # Types ought to match the original if not numeric
+            self.assertIsInstance(first, (type(name), int, long))
+            for is_attr, i in rest :
+                if is_attr :
+                    self.assertIsInstance(i, type(name))
+                else :
+                    self.assertIsInstance(i, (int, long))
+
+        # Verify str._formatter_field_name_split()
+        check_split('a', 'a', [])
+        check_split('2', 2, [])
+        check_split('.b', '', [(True, 'b')])
+        check_split('a.b[2]', 'a', [(True, 'b'), (False, 2)])
+        check_split('a.b[2].c[7]', 'a', [(True, 'b'), (False, 2), (True, 'c'), (False, 7)])
+        check_split('.b[2].c[7]', '', [(True, 'b'), (False, 2), (True, 'c'), (False, 7)])
+        check_split('[3].b[2].c[7]', '',
+                    [(False, 3), (True, 'b'), (False, 2), (True, 'c'), (False, 7)])
+
+        # Verify unicode._formatter_field_name_split()
+        check_split(u'a', 'a', [])
+        check_split(u'2', 2, [])
+        check_split(u'.b', '', [(True, 'b')])
+        check_split(u'a.b[2]', 'a', [(True, 'b'), (False, 2)])
+        check_split(u'a.b[2].c[7]', 'a', [(True, 'b'), (False, 2), (True, 'c'), (False, 7)])
+        check_split(u'.b[2].c[7]', '', [(True, 'b'), (False, 2), (True, 'c'), (False, 7)])
+        check_split(u'[3].b[2].c[7]', '',
+                    [(False, 3), (True, 'b'), (False, 2), (True, 'c'), (False, 7)])
+
 
 def test_main():
     test_support.run_unittest(
diff --git a/Lib/test/test_unicode_jy.py b/Lib/test/test_unicode_jy.py
--- a/Lib/test/test_unicode_jy.py
+++ b/Lib/test/test_unicode_jy.py
@@ -4,6 +4,7 @@
 Made for Jython.
 """
 import re
+import string
 import sys
 import unittest
 from StringIO import StringIO
@@ -182,10 +183,424 @@
         self.assertEqual(sys.stdout.getvalue(), msg)
 
 
+class UnicodeFormatStrTest(unittest.TestCase):
+    # Adapted from test_str StrTest by liberally adding u-prefixes.
+
+    def test__format__(self):
+        def test(value, format, expected):
+            r = value.__format__(format)
+            self.assertEqual(r, expected)
+            # note 'xyz'==u'xyz', so must check type separately
+            self.assertIsInstance(r, unicode)
+            # also test both with the trailing 's'
+            r = value.__format__(format + u's')
+            self.assertEqual(r, expected)
+            self.assertIsInstance(r, unicode)
+
+        test(u'', '', '')
+        test(u'abc', '', 'abc')
+        test(u'abc', '.3', 'abc')
+        test(u'ab', '.3', 'ab')
+        test(u'abcdef', '.3', 'abc')
+        test(u'abcdef', '.0', '')
+        test(u'abc', '3.3', 'abc')
+        test(u'abc', '2.3', 'abc')
+        test(u'abc', '2.2', 'ab')
+        test(u'abc', '3.2', 'ab ')
+        test(u'result', 'x<0', 'result')
+        test(u'result', 'x<5', 'result')
+        test(u'result', 'x<6', 'result')
+        test(u'result', 'x<7', 'resultx')
+        test(u'result', 'x<8', 'resultxx')
+        test(u'result', ' <7', 'result ')
+        test(u'result', '<7', 'result ')
+        test(u'result', '>7', ' result')
+        test(u'result', '>8', '  result')
+        test(u'result', '^8', ' result ')
+        test(u'result', '^9', ' result  ')
+        test(u'result', '^10', '  result  ')
+        test(u'a', '10000', 'a' + ' ' * 9999)
+        test(u'', '10000', ' ' * 10000)
+        test(u'', '10000000', ' ' * 10000000)
+
+    def test_format(self):
+        self.assertEqual(u''.format(), '')
+        self.assertEqual(u'a'.format(), 'a')
+        self.assertEqual(u'ab'.format(), 'ab')
+        self.assertEqual(u'a{{'.format(), 'a{')
+        self.assertEqual(u'a}}'.format(), 'a}')
+        self.assertEqual(u'{{b'.format(), '{b')
+        self.assertEqual(u'}}b'.format(), '}b')
+        self.assertEqual(u'a{{b'.format(), 'a{b')
+
+        # examples from the PEP:
+        import datetime
+        self.assertEqual(u"My name is {0}".format('Fred'), "My name is Fred")
+        self.assertIsInstance(u"My name is {0}".format('Fred'), unicode)
+        self.assertEqual(u"My name is {0[name]}".format(dict(name='Fred')),
+                         "My name is Fred")
+        self.assertEqual(u"My name is {0} :-{{}}".format('Fred'),
+                         "My name is Fred :-{}")
+
+        d = datetime.date(2007, 8, 18)
+        self.assertEqual(u"The year is {0.year}".format(d),
+                         "The year is 2007")
+
+        # classes we'll use for testing
+        class C:
+            def __init__(self, x=100):
+                self._x = x
+            def __format__(self, spec):
+                return spec
+
+        class D:
+            def __init__(self, x):
+                self.x = x
+            def __format__(self, spec):
+                return str(self.x)
+
+        # class with __str__, but no __format__
+        class E:
+            def __init__(self, x):
+                self.x = x
+            def __str__(self):
+                return 'E(' + self.x + ')'
+
+        # class with __repr__, but no __format__ or __str__
+        class F:
+            def __init__(self, x):
+                self.x = x
+            def __repr__(self):
+                return 'F(' + self.x + ')'
+
+        # class with __format__ that forwards to string, for some format_spec's
+        class G:
+            def __init__(self, x):
+                self.x = x
+            def __str__(self):
+                return "string is " + self.x
+            def __format__(self, format_spec):
+                if format_spec == 'd':
+                    return 'G(' + self.x + ')'
+                return object.__format__(self, format_spec)
+
+        # class that returns a bad type from __format__
+        class H:
+            def __format__(self, format_spec):
+                return 1.0
+
+        class I(datetime.date):
+            def __format__(self, format_spec):
+                return self.strftime(format_spec)
+
+        class J(int):
+            def __format__(self, format_spec):
+                return int.__format__(self * 2, format_spec)
+
+
+        self.assertEqual(u''.format(), '')
+        self.assertEqual(u'abc'.format(), 'abc')
+        self.assertEqual(u'{0}'.format('abc'), 'abc')
+        self.assertEqual(u'{0:}'.format('abc'), 'abc')
+        self.assertEqual(u'X{0}'.format('abc'), 'Xabc')
+        self.assertEqual(u'{0}X'.format('abc'), 'abcX')
+        self.assertEqual(u'X{0}Y'.format('abc'), 'XabcY')
+        self.assertEqual(u'{1}'.format(1, 'abc'), 'abc')
+        self.assertEqual(u'X{1}'.format(1, 'abc'), 'Xabc')
+        self.assertEqual(u'{1}X'.format(1, 'abc'), 'abcX')
+        self.assertEqual(u'X{1}Y'.format(1, 'abc'), 'XabcY')
+        self.assertEqual(u'{0}'.format(-15), '-15')
+        self.assertEqual(u'{0}{1}'.format(-15, 'abc'), '-15abc')
+        self.assertEqual(u'{0}X{1}'.format(-15, 'abc'), '-15Xabc')
+        self.assertEqual(u'{{'.format(), '{')
+        self.assertEqual(u'}}'.format(), '}')
+        self.assertEqual(u'{{}}'.format(), '{}')
+        self.assertEqual(u'{{x}}'.format(), '{x}')
+        self.assertEqual(u'{{{0}}}'.format(123), '{123}')
+        self.assertEqual(u'{{{{0}}}}'.format(), '{{0}}')
+        self.assertEqual(u'}}{{'.format(), '}{')
+        self.assertEqual(u'}}x{{'.format(), '}x{')
+
+        # weird field names
+        self.assertEqual(u"{0[foo-bar]}".format({'foo-bar':'baz'}), 'baz')
+        self.assertEqual(u"{0[foo bar]}".format({'foo bar':'baz'}), 'baz')
+        self.assertEqual(u"{0[ ]}".format({' ':3}), '3')
+
+        self.assertEqual(u'{foo._x}'.format(foo=C(20)), '20')
+        self.assertEqual(u'{1}{0}'.format(D(10), D(20)), '2010')
+        self.assertEqual(u'{0._x.x}'.format(C(D('abc'))), 'abc')
+        self.assertEqual(u'{0[0]}'.format(['abc', 'def']), 'abc')
+        self.assertEqual(u'{0[1]}'.format(['abc', 'def']), 'def')
+        self.assertEqual(u'{0[1][0]}'.format(['abc', ['def']]), 'def')
+        self.assertEqual(u'{0[1][0].x}'.format(['abc', [D('def')]]), 'def')
+
+        self.assertIsInstance(u'{0[1][0].x}'.format(['abc', [D('def')]]), unicode)
+
+        # strings
+        self.assertEqual(u'{0:.3s}'.format('abc'), 'abc')
+        self.assertEqual(u'{0:.3s}'.format('ab'), 'ab')
+        self.assertEqual(u'{0:.3s}'.format('abcdef'), 'abc')
+        self.assertEqual(u'{0:.0s}'.format('abcdef'), '')
+        self.assertEqual(u'{0:3.3s}'.format('abc'), 'abc')
+        self.assertEqual(u'{0:2.3s}'.format('abc'), 'abc')
+        self.assertEqual(u'{0:2.2s}'.format('abc'), 'ab')
+        self.assertEqual(u'{0:3.2s}'.format('abc'), 'ab ')
+        self.assertEqual(u'{0:x<0s}'.format('result'), 'result')
+        self.assertEqual(u'{0:x<5s}'.format('result'), 'result')
+        self.assertEqual(u'{0:x<6s}'.format('result'), 'result')
+        self.assertEqual(u'{0:x<7s}'.format('result'), 'resultx')
+        self.assertEqual(u'{0:x<8s}'.format('result'), 'resultxx')
+        self.assertEqual(u'{0: <7s}'.format('result'), 'result ')
+        self.assertEqual(u'{0:<7s}'.format('result'), 'result ')
+        self.assertEqual(u'{0:>7s}'.format('result'), ' result')
+        self.assertEqual(u'{0:>8s}'.format('result'), '  result')
+        self.assertEqual(u'{0:^8s}'.format('result'), ' result ')
+        self.assertEqual(u'{0:^9s}'.format('result'), ' result  ')
+        self.assertEqual(u'{0:^10s}'.format('result'), '  result  ')
+        self.assertEqual(u'{0:10000}'.format('a'), 'a' + ' ' * 9999)
+        self.assertEqual(u'{0:10000}'.format(''), ' ' * 10000)
+        self.assertEqual(u'{0:10000000}'.format(''), ' ' * 10000000)
+
+        # format specifiers for user defined type
+        self.assertEqual(u'{0:abc}'.format(C()), 'abc')
+
+        # !r and !s coercions
+        self.assertEqual(u'{0!s}'.format('Hello'), 'Hello')
+        self.assertEqual(u'{0!s:}'.format('Hello'), 'Hello')
+        self.assertEqual(u'{0!s:15}'.format('Hello'), 'Hello          ')
+        self.assertEqual(u'{0!s:15s}'.format('Hello'), 'Hello          ')
+        self.assertEqual(u'{0!r}'.format('Hello'), "'Hello'")
+        self.assertEqual(u'{0!r:}'.format('Hello'), "'Hello'")
+        self.assertEqual(u'{0!r}'.format(F('Hello')), 'F(Hello)')
+
+        # test fallback to object.__format__
+        self.assertEqual(u'{0}'.format({}), '{}')
+        self.assertEqual(u'{0}'.format([]), '[]')
+        self.assertEqual(u'{0}'.format([1]), '[1]')
+        self.assertEqual(u'{0}'.format(E('data')), 'E(data)')
+        self.assertEqual(u'{0:d}'.format(G('data')), 'G(data)')
+        self.assertEqual(u'{0!s}'.format(G('data')), 'string is data')
+
+        msg = 'object.__format__ with a non-empty format string is deprecated'
+        with test_support.check_warnings((msg, PendingDeprecationWarning)):
+            self.assertEqual(u'{0:^10}'.format(E('data')), ' E(data)  ')
+            self.assertEqual(u'{0:^10s}'.format(E('data')), ' E(data)  ')
+            self.assertEqual(u'{0:>15s}'.format(G('data')), ' string is data')
+
+        #FIXME: not supported in Jython yet:
+        if not test_support.is_jython:
+            self.assertEqual(u"{0:date: %Y-%m-%d}".format(I(year=2007,
+                                                           month=8,
+                                                           day=27)),
+                             "date: 2007-08-27")
+
+            # test deriving from a builtin type and overriding __format__
+            self.assertEqual(u"{0}".format(J(10)), "20")
+
+
+        # string format specifiers
+        self.assertEqual(u'{0:}'.format('a'), 'a')
+
+        # computed format specifiers
+        self.assertEqual(u"{0:.{1}}".format('hello world', 5), 'hello')
+        self.assertEqual(u"{0:.{1}s}".format('hello world', 5), 'hello')
+        self.assertEqual(u"{0:.{precision}s}".format('hello world', precision=5), 'hello')
+        self.assertEqual(u"{0:{width}.{precision}s}".format('hello world', width=10, precision=5), 'hello     ')
+        self.assertEqual(u"{0:{width}.{precision}s}".format('hello world', width='10', precision='5'), 'hello     ')
+
+        self.assertIsInstance(u"{0:{width}.{precision}s}".format('hello world', width='10', precision='5'), unicode)
+
+        # test various errors
+        self.assertRaises(ValueError, u'{'.format)
+        self.assertRaises(ValueError, u'}'.format)
+        self.assertRaises(ValueError, u'a{'.format)
+        self.assertRaises(ValueError, u'a}'.format)
+        self.assertRaises(ValueError, u'{a'.format)
+        self.assertRaises(ValueError, u'}a'.format)
+        self.assertRaises(IndexError, u'{0}'.format)
+        self.assertRaises(IndexError, u'{1}'.format, u'abc')
+        self.assertRaises(KeyError,   u'{x}'.format)
+        self.assertRaises(ValueError, u"}{".format)
+        self.assertRaises(ValueError, u"{".format)
+        self.assertRaises(ValueError, u"}".format)
+        self.assertRaises(ValueError, u"abc{0:{}".format)
+        self.assertRaises(ValueError, u"{0".format)
+        self.assertRaises(IndexError, u"{0.}".format)
+        self.assertRaises(ValueError, u"{0.}".format, 0)
+        self.assertRaises(IndexError, u"{0[}".format)
+        self.assertRaises(ValueError, u"{0[}".format, [])
+        self.assertRaises(KeyError,   u"{0]}".format)
+        self.assertRaises(ValueError, u"{0.[]}".format, 0)
+        self.assertRaises(ValueError, u"{0..foo}".format, 0)
+        self.assertRaises(ValueError, u"{0[0}".format, 0)
+        self.assertRaises(ValueError, u"{0[0:foo}".format, 0)
+        self.assertRaises(KeyError,   u"{c]}".format)
+        self.assertRaises(ValueError, u"{{ {{{0}}".format, 0)
+        self.assertRaises(ValueError, u"{0}}".format, 0)
+        self.assertRaises(KeyError,   u"{foo}".format, bar=3)
+        self.assertRaises(ValueError, u"{0!x}".format, 3)
+        self.assertRaises(ValueError, u"{0!}".format, 0)
+        self.assertRaises(ValueError, u"{0!rs}".format, 0)
+        self.assertRaises(ValueError, u"{!}".format)
+        self.assertRaises(IndexError, u"{:}".format)
+        self.assertRaises(IndexError, u"{:s}".format)
+        self.assertRaises(IndexError, u"{}".format)
+
+        # issue 6089
+        self.assertRaises(ValueError, u"{0[0]x}".format, [None])
+        self.assertRaises(ValueError, u"{0[0](10)}".format, [None])
+
+        # can't have a replacement on the field name portion
+        self.assertRaises(TypeError, u'{0[{1}]}'.format, 'abcdefg', 4)
+
+        # exceed maximum recursion depth
+        self.assertRaises(ValueError, u"{0:{1:{2}}}".format, 'abc', 's', '')
+        self.assertRaises(ValueError, u"{0:{1:{2:{3:{4:{5:{6}}}}}}}".format,
+                          0, 1, 2, 3, 4, 5, 6, 7)
+
+        # string format spec errors
+        self.assertRaises(ValueError, u"{0:-s}".format, '')
+        self.assertRaises(ValueError, format, "", u"-")
+        self.assertRaises(ValueError, u"{0:=s}".format, '')
+
+    def test_format_auto_numbering(self):
+        class C:
+            def __init__(self, x=100):
+                self._x = x
+            def __format__(self, spec):
+                return spec
+
+        self.assertEqual(u'{}'.format(10), '10')
+        self.assertEqual(u'{:5}'.format('s'), 's    ')
+        self.assertEqual(u'{!r}'.format('s'), "'s'")
+        self.assertEqual(u'{._x}'.format(C(10)), '10')
+        self.assertEqual(u'{[1]}'.format([1, 2]), '2')
+        self.assertEqual(u'{[a]}'.format({'a':4, 'b':2}), '4')
+        self.assertEqual(u'a{}b{}c'.format(0, 1), 'a0b1c')
+
+        self.assertEqual(u'a{:{}}b'.format('x', '^10'), 'a    x     b')
+        self.assertEqual(u'a{:{}x}b'.format(20, '#'), 'a0x14b')
+
+        # can't mix and match numbering and auto-numbering
+        self.assertRaises(ValueError, u'{}{1}'.format, 1, 2)
+        self.assertRaises(ValueError, u'{1}{}'.format, 1, 2)
+        self.assertRaises(ValueError, u'{:{1}}'.format, 1, 2)
+        self.assertRaises(ValueError, u'{0:{}}'.format, 1, 2)
+
+        # can mix and match auto-numbering and named
+        self.assertEqual(u'{f}{}'.format(4, f='test'), 'test4')
+        self.assertEqual(u'{}{f}'.format(4, f='test'), '4test')
+        self.assertEqual(u'{:{f}}{g}{}'.format(1, 3, g='g', f=2), ' 1g3')
+        self.assertEqual(u'{f:{}}{}{g}'.format(2, 4, f=1, g='g'), ' 14g')
+
+
+class StringModuleUnicodeTest(unittest.TestCase):
+    # Taken from test_string ModuleTest and converted for unicode
+
+    def test_formatter(self):
+
+        def assertEqualAndUnicode(r, exp):
+            self.assertEqual(r, exp)
+            self.assertIsInstance(r, unicode)
+
+        fmt = string.Formatter()
+        assertEqualAndUnicode(fmt.format(u"foo"), "foo")
+        assertEqualAndUnicode(fmt.format(u"foo{0}", "bar"), "foobar")
+        assertEqualAndUnicode(fmt.format(u"foo{1}{0}-{1}", "bar", 6), "foo6bar-6")
+        assertEqualAndUnicode(fmt.format(u"-{arg!r}-", arg='test'), "-'test'-")
+
+        # override get_value ############################################
+        class NamespaceFormatter(string.Formatter):
+            def __init__(self, namespace={}):
+                string.Formatter.__init__(self)
+                self.namespace = namespace
+
+            def get_value(self, key, args, kwds):
+                if isinstance(key, (str, unicode)):
+                    try:
+                        # Check explicitly passed arguments first
+                        return kwds[key]
+                    except KeyError:
+                        return self.namespace[key]
+                else:
+                    string.Formatter.get_value(key, args, kwds)
+
+        fmt = NamespaceFormatter({'greeting':'hello'})
+        assertEqualAndUnicode(fmt.format(u"{greeting}, world!"), 'hello, world!')
+
+
+        # override format_field #########################################
+        class CallFormatter(string.Formatter):
+            def format_field(self, value, format_spec):
+                return format(value(), format_spec)
+
+        fmt = CallFormatter()
+        assertEqualAndUnicode(fmt.format(u'*{0}*', lambda : 'result'), '*result*')
+
+
+        # override convert_field ########################################
+        class XFormatter(string.Formatter):
+            def convert_field(self, value, conversion):
+                if conversion == 'x':
+                    return None
+                return super(XFormatter, self).convert_field(value, conversion)
+
+        fmt = XFormatter()
+        assertEqualAndUnicode(fmt.format(u"{0!r}:{0!x}", 'foo', 'foo'), "'foo':None")
+
+
+        # override parse ################################################
+        class BarFormatter(string.Formatter):
+            # returns an iterable that contains tuples of the form:
+            # (literal_text, field_name, format_spec, conversion)
+            def parse(self, format_string):
+                for field in format_string.split('|'):
+                    if field[0] == '+':
+                        # it's markup
+                        field_name, _, format_spec = field[1:].partition(':')
+                        yield '', field_name, format_spec, None
+                    else:
+                        yield field, None, None, None
+
+        fmt = BarFormatter()
+        assertEqualAndUnicode(fmt.format(u'*|+0:^10s|*', 'foo'), '*   foo    *')
+
+        # test all parameters used
+        class CheckAllUsedFormatter(string.Formatter):
+            def check_unused_args(self, used_args, args, kwargs):
+                # Track which arguments actually got used
+                unused_args = set(kwargs.keys())
+                unused_args.update(range(0, len(args)))
+
+                for arg in used_args:
+                    unused_args.remove(arg)
+
+                if unused_args:
+                    raise ValueError("unused arguments")
+
+        fmt = CheckAllUsedFormatter()
+        # The next series should maybe also call assertEqualAndUnicode:
+        #assertEqualAndUnicode(fmt.format(u"{0}", 10), "10")
+        #assertEqualAndUnicode(fmt.format(u"{0}{i}", 10, i=100), "10100")
+        #assertEqualAndUnicode(fmt.format(u"{0}{i}{1}", 10, 20, i=100), "1010020")
+        # But string.Formatter.format returns bytes. See CPython Issue 15951.
+        self.assertEqual(fmt.format(u"{0}", 10), "10")
+        self.assertEqual(fmt.format(u"{0}{i}", 10, i=100), "10100")
+        self.assertEqual(fmt.format(u"{0}{i}{1}", 10, 20, i=100), "1010020")
+        self.assertRaises(ValueError, fmt.format, u"{0}{i}{1}", 10, 20, i=100, j=0)
+        self.assertRaises(ValueError, fmt.format, u"{0}", 10, 20)
+        self.assertRaises(ValueError, fmt.format, u"{0}", 10, 20, i=100)
+        self.assertRaises(ValueError, fmt.format, u"{i}", 10, 20, i=100)
+
+
 def test_main():
-    test_support.run_unittest(UnicodeTestCase,
-                              UnicodeFormatTestCase,
-                              UnicodeStdIOTestCase)
+    test_support.run_unittest(
+                UnicodeTestCase,
+                UnicodeFormatTestCase,
+                UnicodeStdIOTestCase,
+                UnicodeFormatStrTest,
+                StringModuleUnicodeTest,
+            )
 
 
 if __name__ == "__main__":
diff --git a/src/org/python/core/PyString.java b/src/org/python/core/PyString.java
--- a/src/org/python/core/PyString.java
+++ b/src/org/python/core/PyString.java
@@ -662,6 +662,13 @@
         }
     }
 
+    /**
+     * Create an instance of the same type as this object, from the Java String given as argument.
+     * This is to be overridden in a subclass to return its own type.
+     *
+     * @param str to wrap
+     * @return
+     */
     public PyString createInstance(String str) {
         return new PyString(str);
     }
@@ -3740,23 +3747,19 @@
 
     @ExposedMethod(doc = BuiltinDocs.str__formatter_parser_doc)
     final PyObject str__formatter_parser() {
-        return new MarkupIterator(getString());
+        return new MarkupIterator(this);
     }
 
     @ExposedMethod(doc = BuiltinDocs.str__formatter_field_name_split_doc)
     final PyObject str__formatter_field_name_split() {
-        FieldNameIterator iterator = new FieldNameIterator(getString());
-        Object headObj = iterator.head();
-        PyObject head =
-                headObj instanceof Integer ? new PyInteger((Integer)headObj) : new PyString(
-                        (String)headObj);
-        return new PyTuple(head, iterator);
+        FieldNameIterator iterator = new FieldNameIterator(this);
+        return new PyTuple(iterator.pyHead(), iterator);
     }
 
     @ExposedMethod(doc = BuiltinDocs.str_format_doc)
     final PyObject str_format(PyObject[] args, String[] keywords) {
         try {
-            return new PyString(buildFormattedString(getString(), args, keywords, null));
+            return new PyString(buildFormattedString(args, keywords, null, null));
         } catch (IllegalArgumentException e) {
             throw Py.ValueError(e.getMessage());
         }
@@ -3764,18 +3767,33 @@
 
     /**
      * Implements PEP-3101 {}-formatting methods str.format() and
-     * unicode.format().
+     * unicode.format(). When called with enclosingIterator == null, this
+     * method takes this object as its formatting string. The method is also called (calls itself)
+     * to deal with nested formatting sepecifications. In that case, enclosingIterator
+     * is a {@link MarkupIterator} on this object and value is a substring of this
+     * object needing recursive transaltion.
      *
-     * @param value the format string
      * @param args to be interpolated into the string
      * @param keywords for the trailing args
-     * @param enclosingIterator when used nested
+     * @param enclosingIterator when used nested, null if subject is this PyString
+     * @param value the format string when enclosingIterator is not null
      * @return the formatted string based on the arguments
      */
-    protected String buildFormattedString(String value, PyObject[] args, String[] keywords,
-            MarkupIterator enclosingIterator) {
+    protected String buildFormattedString(PyObject[] args, String[] keywords,
+            MarkupIterator enclosingIterator, String value) {
+
+        MarkupIterator it;
+        if (enclosingIterator == null) {
+            // Top-level call acts on this object.
+            it = new MarkupIterator(this);
+        } else {
+            // Nested call acts on the substring and some state from existing iterator.
+            it = new MarkupIterator(enclosingIterator, value);
+        }
+
+        // Result will be formed here
         StringBuilder result = new StringBuilder();
-        MarkupIterator it = new MarkupIterator(value, enclosingIterator);
+
         while (true) {
             MarkupIterator.Chunk chunk = it.nextChunk();
             if (chunk == null) {
@@ -3784,12 +3802,12 @@
             // A Chunk encapsulates a literal part ...
             result.append(chunk.literalText);
             // ... and the parsed form of the replacement field that followed it (if any)
-            if (chunk.fieldName.length() > 0) {
+            if (chunk.fieldName != null) {
                 // The grammar of the replacement field is:
                 // "{" [field_name] ["!" conversion] [":" format_spec] "}"
 
                 // Get the object referred to by the field name (which may be omitted).
-                PyObject fieldObj = getFieldObject(chunk.fieldName, args, keywords);
+                PyObject fieldObj = getFieldObject(chunk.fieldName, it.isBytes(), args, keywords);
                 if (fieldObj == null) {
                     continue;
                 }
@@ -3811,7 +3829,7 @@
                         throw Py.ValueError("Max string recursion exceeded");
                     }
                     // Recursively interpolate further args into chunk.formatSpec
-                    formatSpec = buildFormattedString(formatSpec, args, keywords, it);
+                    formatSpec = buildFormattedString(args, keywords, it, formatSpec);
                 }
                 renderField(fieldObj, formatSpec, result);
             }
@@ -3824,51 +3842,56 @@
      * argument list, containing positional and keyword arguments.
      *
      * @param fieldName to interpret.
+     * @param bytes true if the field name is from a PyString, false for PyUnicode.
      * @param args argument list (positional then keyword arguments).
      * @param keywords naming the keyword arguments.
      * @return the object designated or null.
      */
-    private PyObject getFieldObject(String fieldName, PyObject[] args, String[] keywords) {
-        FieldNameIterator iterator = new FieldNameIterator(fieldName);
-        Object head = iterator.head();
+    private PyObject getFieldObject(String fieldName, boolean bytes, PyObject[] args, String[] keywords) {
+        FieldNameIterator iterator = new FieldNameIterator(fieldName, bytes);
+        PyObject head = iterator.pyHead();
         PyObject obj = null;
         int positionalCount = args.length - keywords.length;
 
-        if (head instanceof Integer) {
-            int index = (Integer)head;
+        if (head.isIndex()) {
+            // The field name begins with an integer argument index (not a [n]-type index).
+            int index = head.asIndex();
             if (index >= positionalCount) {
                 throw Py.IndexError("tuple index out of range");
             }
             obj = args[index];
 
         } else {
+            // The field name begins with keyword.
             for (int i = 0; i < keywords.length; i++) {
-                if (keywords[i].equals(head)) {
+                if (keywords[i].equals(head.asString())) {
                     obj = args[positionalCount + i];
                     break;
                 }
             }
+            // And if we don't find it, that's an error
             if (obj == null) {
-                throw Py.KeyError((String)head);
+                throw Py.KeyError(head);
             }
         }
 
-        if (obj != null) {
-            while (true) {
-                FieldNameIterator.Chunk chunk = iterator.nextChunk();
-                if (chunk == null) {
-                    break;
-                }
-                if (chunk.is_attr) {
-                    obj = obj.__getattr__((String)chunk.value);
+        // Now deal with the iterated sub-fields
+        while (obj != null) {
+            FieldNameIterator.Chunk chunk = iterator.nextChunk();
+            if (chunk == null) {
+                // End of iterator
+                break;
+            }
+            Object key = chunk.value;
+            if (chunk.is_attr) {
+                // key must be a String
+                obj = obj.__getattr__((String)key);
+            } else {
+                if (key instanceof Integer) {
+                    // Can this happen?
+                    obj = obj.__getitem__(((Integer)key).intValue());
                 } else {
-                    PyObject key =
-                            chunk.value instanceof String ? new PyString((String)chunk.value)
-                                    : new PyInteger((Integer)chunk.value);
-                    obj = obj.__getitem__(key);
-                }
-                if (obj == null) {
-                    break;
+                    obj = obj.__getitem__(new PyString(key.toString()));
                 }
             }
         }
diff --git a/src/org/python/core/PyUnicode.java b/src/org/python/core/PyUnicode.java
--- a/src/org/python/core/PyUnicode.java
+++ b/src/org/python/core/PyUnicode.java
@@ -7,6 +7,8 @@
 import java.util.List;
 import java.util.Set;
 
+import org.python.core.stringlib.FieldNameIterator;
+import org.python.core.stringlib.MarkupIterator;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedNew;
 import org.python.expose.ExposedType;
@@ -1431,13 +1433,30 @@
 
     @Override
     public PyObject __format__(PyObject formatSpec) {
+        return unicode___format__(formatSpec);
+    }
+
+    @ExposedMethod(doc = BuiltinDocs.unicode___format___doc)
+    final PyObject unicode___format__(PyObject formatSpec) {
+        // Re-use the str implementation, which adapts itself to unicode.
         return str___format__(formatSpec);
     }
 
+    @ExposedMethod(doc = BuiltinDocs.unicode__formatter_parser_doc)
+    final PyObject unicode__formatter_parser() {
+        return new MarkupIterator(this);
+    }
+
+    @ExposedMethod(doc = BuiltinDocs.unicode__formatter_field_name_split_doc)
+    final PyObject unicode__formatter_field_name_split() {
+        FieldNameIterator iterator = new FieldNameIterator(this);
+        return new PyTuple(iterator.pyHead(), iterator);
+    }
+
     @ExposedMethod(doc = BuiltinDocs.unicode_format_doc)
     final PyObject unicode_format(PyObject[] args, String[] keywords) {
         try {
-            return new PyUnicode(buildFormattedString(getString(), args, keywords, null));
+            return new PyUnicode(buildFormattedString(args, keywords, null, null));
         } catch (IllegalArgumentException e) {
             throw Py.ValueError(e.getMessage());
         }
diff --git a/src/org/python/core/stringlib/FieldNameIterator.java b/src/org/python/core/stringlib/FieldNameIterator.java
--- a/src/org/python/core/stringlib/FieldNameIterator.java
+++ b/src/org/python/core/stringlib/FieldNameIterator.java
@@ -1,30 +1,61 @@
 package org.python.core.stringlib;
 
+import org.python.core.Py;
 import org.python.core.PyBoolean;
 import org.python.core.PyInteger;
 import org.python.core.PyObject;
 import org.python.core.PyString;
 import org.python.core.PyTuple;
 import org.python.core.PyType;
+import org.python.core.PyUnicode;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedType;
 
 /**
- * Provides an implementation of str._formatter_field_name_split()
+ * This class is an implementation of the iterator object returned by
+ * str._formatter_field_name_split() and
+ * unicode._formatter_field_name_split(). The function
+ * _formatter_field_name_split() returns a pair (tuple) consisting of a head element
+ * and an instance of this iterator. The constructor of this class effectively implements that
+ * function, since as well as "being" the iterator, the object has an extra method {@link #head()}
+ * to return the required first member of the pair.
  */
 @ExposedType(name = "fieldnameiterator", base = PyObject.class, isBaseType = false)
 public class FieldNameIterator extends PyObject {
 
     public static final PyType TYPE = PyType.fromClass(FieldNameIterator.class);
 
-    private String markup;
+    /** The UTF-16 string from which elements are being returned. */
+    private final String markup;
+    /** True if originally given a PyString (so must return PyString not PyUnicode). */
+    private final boolean bytes;
+    /** How far along that string we are. */
+    private int index;
     private Object head;
-    private int index;
 
-    public FieldNameIterator(String markup) {
-        this.markup = markup;
-        this.index = nextDotOrBracket(markup);
-        String headStr = markup.substring(0, index);
+    /**
+     * Create an iterator for the parts of this field name (and extract the head name field, which
+     * may be an empty string). According to the Python Standard Library documentation, a
+     * replacement field name has the structure:
+     *
+     * 
+     * field_name        ::=  arg_name ("." attribute_name | "[" element_index "]")*
+     * arg_name          ::=  [identifier | integer]
+     * attribute_name    ::=  identifier
+     * element_index     ::=  integer | index_string
+     * 

+     *
+     * The object is used from PyUnicode and from PyString, and we have to signal which it is, so
+     * that returned values may match in type.
+     *
+     * @param fieldName the field name as UTF-16
+     * @param bytes true if elements returned should be PyString, else PyUnicode
+     */
+    public FieldNameIterator(String fieldName, boolean bytes) {
+        this.markup = fieldName;
+        this.bytes = bytes;
+        this.index = nextDotOrBracket(fieldName);
+        String headStr = fieldName.substring(0, index);
         try {
             this.head = Integer.parseInt(headStr);
         } catch (NumberFormatException e) {
@@ -32,6 +63,17 @@
         }
     }
 
+    /**
+     * Create an iterator for the parts of this field name (and extract the head name field, which
+     * may be an empty string).
+     *
+     * @param fieldNameObject
+     */
+    public FieldNameIterator(PyString fieldNameObject) {
+        // Extract UTF-16 string but remember whether PyString or PyUnicode shouyld result.
+        this(fieldNameObject.getString(), !(fieldNameObject instanceof PyUnicode));
+    }
+
     @Override
     public PyObject __iter__() {
         return fieldnameiterator___iter__();
@@ -53,14 +95,31 @@
         if (chunk == null) {
             return null;
         }
-        PyObject[] elements = new PyObject[2];
-        elements [0] = new PyBoolean(chunk.is_attr);
-        if (chunk.value instanceof Integer) {
-            elements [1] = new PyInteger((Integer) chunk.value);
+        return new PyTuple(Py.newBoolean(chunk.is_attr), wrap(chunk.value));
+    }
+
+    /**
+     * Convenience method to wrap a value as a PyInteger, if it is an Integer, or as
+     * PyString or PyUnicode according to the type of the original field
+     * name string. These objects are being used as field specifiers in navigating arguments to a
+     * format statement.
+     *
+     * @param value to wrap as a PyObject.
+     * @return PyObject equivalent field specifier
+     */
+    private PyObject wrap(Object value) {
+        if (value instanceof Integer) {
+            return Py.newInteger(((Integer)value).intValue());
         } else {
-            elements [1] = new PyString((String) chunk.value);
+            // It can only be a String (but if not, at least we see it).
+            String s = value.toString();
+            if (s.length() == 0) {
+                // This is frequent so avoid the constructor
+                return bytes ? Py.EmptyString : Py.EmptyUnicode;
+            } else {
+                return bytes ? Py.newString(s) : Py.newUnicode(s);
+            }
         }
-        return new PyTuple(elements);
     }
 
     private int nextDotOrBracket(String markup) {
@@ -75,10 +134,40 @@
         return Math.min(dotPos, bracketPos);
     }
 
+    /** @return the isolated head object from the field name. */
     public Object head() {
         return head;
     }
 
+    /**
+     * Return the head object from the field name, as PyInteger, PyString
+     * or PyUnicode.
+     *
+     * @return the isolated head object from the field name.
+     */
+    public PyObject pyHead() {
+        return wrap(head());
+    }
+
+    /**
+     * If originally given a PyString, the iterator must return PyString not PyUnicode.
+     *
+     * @return true if originally given a PyString
+     */
+    public final boolean isBytes() {
+        return bytes;
+    }
+
+    /**
+     * Return the next "chunk" of the field name (or return null if ended). A chunk is a 2-tuple
+     * describing:
+     * 
    
+     * 
  1. whether the chunk is an attribute name,
    2. 
+     * 
  2. the name or number (as a String or Integer) for accessing the value.
    3. 
+     * 

+     *
+     * @return next element of the field name
+     */
     public Chunk nextChunk() {
         if (index == markup.length()) {
             return null;
@@ -89,14 +178,15 @@
         } else if (markup.charAt(index) == '.') {
             parseAttrChunk(chunk);
         } else {
-            throw new IllegalArgumentException("Only '.' or '[' may follow ']' in format field specifier");
+            throw new IllegalArgumentException(
+                    "Only '.' or '[' may follow ']' in format field specifier");
         }
         return chunk;
     }
 
     private void parseItemChunk(Chunk chunk) {
         chunk.is_attr = false;
-        int endBracket = markup.indexOf(']', index+1);
+        int endBracket = markup.indexOf(']', index + 1);
         if (endBracket < 0) {
             throw new IllegalArgumentException("Missing ']' in format string");
         }
@@ -124,6 +214,7 @@
     }
 
     public static class Chunk {
+
         public boolean is_attr;
         /** Integer or String. */
         public Object value;
diff --git a/src/org/python/core/stringlib/MarkupIterator.java b/src/org/python/core/stringlib/MarkupIterator.java
--- a/src/org/python/core/stringlib/MarkupIterator.java
+++ b/src/org/python/core/stringlib/MarkupIterator.java
@@ -5,32 +5,41 @@
 import org.python.core.PyString;
 import org.python.core.PyTuple;
 import org.python.core.PyType;
+import org.python.core.PyUnicode;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedType;
 
 /**
- * Provides an implementation of str._formatter_parser()
+ * Provides an implementation of the object that str._formatter_parser() returns, which
+ * is an iterator returning successive 4-tuples, the sequence being equivalent to the original
+ * string.
  */
 @ExposedType(name = "formatteriterator", base = PyObject.class, isBaseType = false)
 public class MarkupIterator extends PyObject {
 
     public static final PyType TYPE = PyType.fromClass(MarkupIterator.class);
 
+    /** The UTF-16 string from which elements are being returned. */
     private final String markup;
+    /** True if originally given a PyString (so must return PyString not PyUnicode). */
+    private final boolean bytes;
+    /** How far along that string we are. */
     private int index;
+    /** A counter used to auto-number fields when not explicitly numbered in the format. */
     private final FieldNumbering numbering;
 
-    public MarkupIterator(String markup) {
-        this(markup, null);
+    /** Constructor used at top-level to enumerate a format. */
+    public MarkupIterator(PyString markupObject) {
+        markup = markupObject.getString();
+        bytes = !(markupObject instanceof PyUnicode);
+        numbering = new FieldNumbering();
     }
 
-    public MarkupIterator(String markup, MarkupIterator enclosingIterator) {
-        this.markup = markup;
-        if (enclosingIterator != null) {
-            numbering = enclosingIterator.numbering;
-        } else {
-            numbering = new FieldNumbering();
-        }
+    /** Variant constructor used when formats are nested. */
+    public MarkupIterator(MarkupIterator enclosingIterator, String subMarkup) {
+        markup = subMarkup;
+        bytes = enclosingIterator.bytes;
+        numbering = enclosingIterator.numbering;
     }
 
     @Override
@@ -48,66 +57,142 @@
         return formatteriterator___iternext__();
     }
 
+    /**
+     * Return the next "chunk" of the format (or return null if ended). A chunk is a 4-tuple
+     * describing
+     * 
    
+     * 
  1. the text leading up to the next format field,
    2. 
+     * 
  2. the field name or number (as a string) for accessing the value,
    3. 
+     * 
  3. the format specifier such as "#12x", and
    4. 
+     * 
  4. any conversion that should be applied (the 's' or 'r' codes for
+     * str() and repr())
    5. 
+     * 

+     * Elements 1-3 are None if this chunk contains no format specifier. Elements 0-2 are
+     * zero-length strings if missing from the format, while element 3 will be None if missing.
+     *
+     * @return PyTuple chunk or null
+     */
     @ExposedMethod
     final PyObject formatteriterator___iternext__() {
-        Chunk chunk;
         try {
-            chunk = nextChunk();
+            // Parse off the next literal text and replacement field
+            Chunk chunk = nextChunk();
+
+            if (chunk != null) {
+                // Result will be built here
+                PyObject[] elements = new PyObject[4];
+
+                // Literal text is used verbatim.
+                elements[0] = wrap(chunk.literalText, "");
+
+                if (chunk.fieldName == null) {
+                    // A field name is null only if there was no replacement field at all.
+                    for (int i = 1; i < elements.length; i++) {
+                        elements[i] = Py.None;
+                    }
+
+                } else {
+                    // Otherwise, this is the field name
+                    elements[1] = wrap(chunk.fieldName, "");
+                    // The format spec may be blank
+                    elements[2] = wrap(chunk.formatSpec, "");
+                    // There may have been a conversion specifier (if not, None is signalled).
+                    elements[3] = wrap(chunk.conversion, null);
+                }
+
+                // And those make up the next answer.
+                return new PyTuple(elements);
+
+            } else {
+                // End of format: end of iteration
+                return null;
+            }
+
         } catch (IllegalArgumentException e) {
             throw Py.ValueError(e.getMessage());
         }
-        if (chunk == null) {
-            return null;
-        }
-        PyObject[] elements = new PyObject[4];
-        elements[0] = new PyString(chunk.literalText);
-        elements[1] = chunk.fieldName.length() == 0
-                    ? Py.None : new PyString(chunk.fieldName);
-        if (chunk.fieldName.length() > 0) {
-            elements[2] = chunk.formatSpec == null
-                    ? Py.EmptyString : new PyString(chunk.formatSpec);
-        } else {
-            elements[2] = Py.None;
-        }
-        elements[3] = chunk.conversion == null ? Py.None : new PyString(chunk.conversion);
-        return new PyTuple(elements);
     }
 
+    /**
+     * Convenience method for populating the return tuple, returning a PyString or
+     * PyUnicode according to the type of the original markup string, or
+     * Py.None if both arguments are null.
+     *
+     * @param value to wrap as a PyObject or null if defaultValue should be wrapped.
+     * @param defaultValue to return or null if default return is None.
+     * @return object for tuple
+     */
+    private PyObject wrap(String value, String defaultValue) {
+        if (value == null) {
+            value = defaultValue;
+        }
+        if (value == null) {
+            // It's still null, we want a None
+            return Py.None;
+        } else if (value.length() == 0) {
+            // This is frequent so avoid the constructor
+            return bytes ? Py.EmptyString : Py.EmptyUnicode;
+        } else {
+            return bytes ? Py.newString(value) : Py.newUnicode(value);
+        }
+    }
+
+    /**
+     * Return the next {@link Chunk} from the iterator, which is a structure containing parsed
+     * elements of the replacement field (if any), and its preceding text. This is the Java
+     * equivalent of the tuple returned by {@link __iternext__()}. This finds use in the
+     * implementation of str.format and unicode.format.
+     *
+     * @return the chunk
+     */
     public Chunk nextChunk() {
         if (index == markup.length()) {
             return null;
         }
         Chunk result = new Chunk();
+
+        // pos = index is the index of the first text not already chunked
         int pos = index;
+
+        // Advance pos to the first '{' that is not a "{{" (escaped brace), or pos<0 if none such.
         while (true) {
             pos = indexOfFirst(markup, pos, '{', '}');
             if (pos >= 0 && pos < markup.length() - 1
-                && markup.charAt(pos + 1) == markup.charAt(pos)) {
+                    && markup.charAt(pos + 1) == markup.charAt(pos)) {
                 // skip escaped bracket
                 pos += 2;
             } else if (pos >= 0 && markup.charAt(pos) == '}') {
+                // Un-escaped '}' is a syntax error
                 throw new IllegalArgumentException("Single '}' encountered in format string");
             } else {
+                // pos is at an un-escaped '{'
                 break;
             }
         }
+
+        // markup[index:pos] is the literal part of this chunk.
         if (pos < 0) {
+            // ... except pos<0, and there is no further format specifier, only literal text.
             result.literalText = unescapeBraces(markup.substring(index));
-            result.fieldName = "";
             index = markup.length();
-        }
-        else {
+
+        } else {
+            // Grab the literal text, dealing with escaped braces.
             result.literalText = unescapeBraces(markup.substring(index, pos));
+            // Scan through the contents of the format spec, between the braces. Skip one '{'.
             pos++;
             int fieldStart = pos;
             int count = 1;
             while (pos < markup.length()) {
                 if (markup.charAt(pos) == '{') {
+                    // This means the spec we are gathering itself contains nested specifiers.
                     count++;
                     result.formatSpecNeedsExpanding = true;
                 } else if (markup.charAt(pos) == '}') {
+                    // And here is a '}' matching one we already counted.
                     count--;
                     if (count == 0) {
+                        // ... matching the one we began with: parse the replacement field.
                         parseField(result, markup.substring(fieldStart, pos));
                         pos++;
                         break;
@@ -116,6 +201,7 @@
                 pos++;
             }
             if (count > 0) {
+                // Must be end of string without matching '}'.
                 throw new IllegalArgumentException("Single '{' encountered in format string");
             }
             index = pos;
@@ -123,48 +209,89 @@
         return result;
     }
 
+    /**
+     * If originally given a PyString, string elements in the returned tuples must be PyString not
+     * PyUnicode.
+     *
+     * @return true if originally given a PyString
+     */
+    public final boolean isBytes() {
+        return bytes;
+    }
+
     private String unescapeBraces(String substring) {
         return substring.replace("{{", "{").replace("}}", "}");
     }
 
+    /**
+     * Parse a "replacement field" consisting of a name, conversion and format specification.
+     * According to the Python Standard Library documentation, a replacement field has the
+     * structure:
+     *
+     * 
+     * replacement_field ::=  "{" [field_name] ["!" conversion] [":" format_spec] "}"
+     * field_name        ::=  arg_name ("." attribute_name | "[" element_index "]")*
+     * arg_name          ::=  [identifier | integer]
+     * attribute_name    ::=  identifier
+     * element_index     ::=  integer | index_string
+     * 

+     *
+     * except at this point, we have already discarded the outer braces.
+     *
+     * @param result destination chunk
+     * @param fieldMarkup specifying a replacement field, possibly with nesting
+     */
     private void parseField(Chunk result, String fieldMarkup) {
         int pos = indexOfFirst(fieldMarkup, 0, '!', ':');
         if (pos >= 0) {
+            // There's a '!' or a ':', so what precedes the first of them is a field name.
             result.fieldName = fieldMarkup.substring(0, pos);
             if (fieldMarkup.charAt(pos) == '!') {
+                // There's a conversion specifier
                 if (pos == fieldMarkup.length() - 1) {
-                    throw new IllegalArgumentException("end of format while " +
-                            "looking for conversion specifier");
+                    throw new IllegalArgumentException("end of format while "
+                            + "looking for conversion specifier");
                 }
                 result.conversion = fieldMarkup.substring(pos + 1, pos + 2);
                 pos += 2;
+                // And if that's not the end, there ought to be a ':' now.
                 if (pos < fieldMarkup.length()) {
                     if (fieldMarkup.charAt(pos) != ':') {
-                        throw new IllegalArgumentException("expected ':' " +
-                                "after conversion specifier");
+                        throw new IllegalArgumentException("expected ':' "
+                                + "after conversion specifier");
                     }
+                    // So the format specifier is from the ':' to the end.
                     result.formatSpec = fieldMarkup.substring(pos + 1);
                 }
             } else {
+                // No '!', so the format specifier is from the ':' to the end. Or empty.
                 result.formatSpec = fieldMarkup.substring(pos + 1);
             }
         } else {
+            // Neither a '!' nor a ':', the whole thing is a name.
             result.fieldName = fieldMarkup;
         }
+
         if (result.fieldName.isEmpty()) {
+            // The field was empty, so generate a number automatically.
             result.fieldName = numbering.nextAutomaticFieldNumber();
             return;
         }
+
+        // Automatic numbers must also work when there is an .attribute or [index]
         char c = result.fieldName.charAt(0);
         if (c == '.' || c == '[') {
             result.fieldName = numbering.nextAutomaticFieldNumber() + result.fieldName;
             return;
         }
+
+        // Finally, remember the argument number was specified (perhaps complain of mixed use)
         if (Character.isDigit(c)) {
             numbering.useManualFieldNumbering();
         }
     }
 
+    /** Find the first of two characters, or return -1. */
     private int indexOfFirst(String s, int start, char c1, char c2) {
         int i1 = s.indexOf(c1, start);
         int i2 = s.indexOf(c2, start);
@@ -177,32 +304,56 @@
         return Math.min(i1, i2);
     }
 
+    /**
+     * Class used locally to assign indexes to the automatically-numbered arguments (see String
+     * Formatting section of the Python Standard Library).
+     */
     static final class FieldNumbering {
+
         private boolean manualFieldNumberSpecified;
         private int automaticFieldNumber = 0;
 
+        /**
+         * Generate a numeric argument index automatically, or raise an error if already started
+         * numbering manually.
+         *
+         * @return index as string
+         */
         String nextAutomaticFieldNumber() {
             if (manualFieldNumberSpecified) {
-                throw new IllegalArgumentException("cannot switch from manual field specification to automatic field numbering");
+                throw new IllegalArgumentException(
+                        "cannot switch from manual field specification to automatic field numbering");
             }
             return Integer.toString(automaticFieldNumber++);
         }
+
+        /**
+         * Remember we are numbering manually, and raise an error if already started numbering
+         * automatically.
+         */
         void useManualFieldNumbering() {
             if (manualFieldNumberSpecified) {
                 return;
             }
             if (automaticFieldNumber != 0) {
-                throw new IllegalArgumentException("cannot switch from automatic field numbering to manual field specification");
+                throw new IllegalArgumentException(
+                        "cannot switch from automatic field numbering to manual field specification");
             }
             manualFieldNumberSpecified = true;
         }
     }
 
     public static final class Chunk {
+
+        /** The text leading up to the next format field. */
         public String literalText;
+        /** The field name or number (as a string) for accessing the value. */
         public String fieldName;
+        /** The format specifier such as "#12x". */
         public String formatSpec;
+        /** Conversion to be applied, e.g. 'r' for repr(). */
         public String conversion;
+        /** Signals the formatSpec needs expanding recursively. */
         public boolean formatSpecNeedsExpanding;
     }
 }
diff --git a/tests/java/org/python/core/StringFormatTest.java b/tests/java/org/python/core/StringFormatTest.java
--- a/tests/java/org/python/core/StringFormatTest.java
+++ b/tests/java/org/python/core/StringFormatTest.java
@@ -20,6 +20,9 @@
     /** Exception-raising seems to need the interpreter to be initialised **/
     PythonInterpreter interp = new PythonInterpreter();
 
+    /** Switches mode in tests that have a shared implementation for bytes and Unicode modes. */
+    private boolean useBytes = true;
+
     public void testInternalFormatSpec() {
         InternalFormat.Spec spec;
         spec = InternalFormat.fromText("x");
@@ -76,8 +79,9 @@
     }
 
     /**
-     * Test the IntegerFormatter returned by {@link PyInteger#prepareFormat}. This is based on the original
-     * testFormatIntOrLong which tested PyInteger.formatIntOrLong.
+     * Test the IntegerFormatter returned by {@link PyInteger#prepareFormat}. This is based on the
+     * original testFormatIntOrLong which tested PyInteger.formatIntOrLong
+     * .
      */
     public void testPrepareFormatter() {
         int v = 123;
@@ -137,8 +141,9 @@
     }
 
     /**
-     * Test the IntegerFormatter returned by {@link PyInteger#prepareFormat}. This is based on the original
-     * testFormatIntOrLong which tested PyInteger.formatIntOrLong.
+     * Test the IntegerFormatter returned by {@link PyInteger#prepareFormat}. This is based on the
+     * original testFormatIntOrLong which tested PyInteger.formatIntOrLong
+     * .
      */
     public void testPrepareFormatterLong() {
         BigInteger v = BigInteger.valueOf(123);
@@ -238,34 +243,34 @@
         assertEquals("abc   ", f.format(v).pad().getResult());
     }
 
-    public void testMarkupIterator() {
-        MarkupIterator iterator = new MarkupIterator("abc");
+    public void implTestMarkupIterator() {
+        MarkupIterator iterator = newMarkupIterator("abc");
         assertEquals("abc", iterator.nextChunk().literalText);
         assertNull(iterator.nextChunk());
 
-        iterator = new MarkupIterator("First, thou shalt count to {0}");
+        iterator = newMarkupIterator("First, thou shalt count to {0}");
         MarkupIterator.Chunk chunk = iterator.nextChunk();
         assertEquals("First, thou shalt count to ", chunk.literalText);
         assertEquals("0", chunk.fieldName);
         assertNull(iterator.nextChunk());
 
-        iterator = new MarkupIterator("Weight in tons {0.weight!r:s}");
+        iterator = newMarkupIterator("Weight in tons {0.weight!r:s}");
         chunk = iterator.nextChunk();
         assertEquals("Weight in tons ", chunk.literalText);
         assertEquals("0.weight", chunk.fieldName);
         assertEquals("r", chunk.conversion);
         assertEquals("s", chunk.formatSpec);
 
-        chunk = new MarkupIterator("{{").nextChunk();
+        chunk = newMarkupIterator("{{").nextChunk();
         assertEquals("{", chunk.literalText);
 
-        chunk = new MarkupIterator("}}").nextChunk();
+        chunk = newMarkupIterator("}}").nextChunk();
         assertEquals("}", chunk.literalText);
 
-        chunk = new MarkupIterator("{{}}").nextChunk();
+        chunk = newMarkupIterator("{{}}").nextChunk();
         assertEquals("{}", chunk.literalText);
 
-        chunk = new MarkupIterator("{0:.{1}}").nextChunk();
+        chunk = newMarkupIterator("{0:.{1}}").nextChunk();
         assertEquals("0", chunk.fieldName);
         assertEquals(".{1}", chunk.formatSpec);
         assertTrue(chunk.formatSpecNeedsExpanding);
@@ -276,8 +281,23 @@
         assertMarkupError("}", "Single '}' encountered in format string");
     }
 
+    public void testMarkupIteratorBytes() {
+        useBytes = true;
+        implTestMarkupIterator();
+    }
+
+    public void testMarkupIteratorUnicode() {
+        useBytes = false;
+        implTestMarkupIterator();
+    }
+
+    private MarkupIterator newMarkupIterator(String markup) {
+        PyString markupObject = useBytes ? Py.newString(markup) : Py.newUnicode(markup);
+        return new MarkupIterator(markupObject);
+    }
+
     private void assertMarkupError(String markup, String expected) {
-        MarkupIterator iterator = new MarkupIterator(markup);
+        MarkupIterator iterator = newMarkupIterator(markup);
         String error = null;
         try {
             iterator.nextChunk();
@@ -287,27 +307,42 @@
         assertEquals(expected, error);
     }
 
-    public void testFieldNameIterator() {
-        FieldNameIterator it = new FieldNameIterator("abc");
+    public void implTestFieldNameIterator() {
+        FieldNameIterator it = newFieldNameIterator("abc");
         assertEquals("abc", it.head());
         assertNull(it.nextChunk());
 
-        it = new FieldNameIterator("3");
+        it = newFieldNameIterator("3");
         assertEquals(3, it.head());
         assertNull(it.nextChunk());
 
-        it = new FieldNameIterator("abc[0]");
+        it = newFieldNameIterator("abc[0]");
         assertEquals("abc", it.head());
         FieldNameIterator.Chunk chunk = it.nextChunk();
         assertEquals(0, chunk.value);
         assertFalse(chunk.is_attr);
         assertNull(it.nextChunk());
 
-        it = new FieldNameIterator("abc.def");
+        it = newFieldNameIterator("abc.def");
         assertEquals("abc", it.head());
         chunk = it.nextChunk();
         assertEquals("def", chunk.value);
         assertTrue(chunk.is_attr);
         assertNull(it.nextChunk());
     }
+
+    public void testFieldNameIteratorBytes() {
+        useBytes = true;
+        implTestFieldNameIterator();
+    }
+
+    public void testFieldNameIteratorUnicode() {
+        useBytes = false;
+        implTestFieldNameIterator();
+    }
+
+    private FieldNameIterator newFieldNameIterator(String field) {
+        PyString fieldObject = useBytes ? Py.newString(field) : Py.newUnicode(field);
+        return new FieldNameIterator(fieldObject);
+    }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Mon Jun 30 20:26:37 2014
From: jython-checkins at python.org (jim.baker)
Date: Mon, 30 Jun 2014 20:26:37 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Update_Netty_extlibs_to_4?=
	=?utf-8?b?LjAuMjA=?=
Message-ID: <3h2HGP3HFdz7Ljn@mail.python.org>

http://hg.python.org/jython/rev/9dd90cab04a9
changeset:   7343:9dd90cab04a9
user:        Jim Baker 
date:        Mon Jun 30 12:26:28 2014 -0600
summary:
  Update Netty extlibs to 4.0.20

files:
  build.xml                                |   10 +++++-----
  extlibs/netty-buffer-4.0.18.Final.jar    |  Bin 
  extlibs/netty-buffer-4.0.20.Final.jar    |  Bin 
  extlibs/netty-codec-4.0.18.Final.jar     |  Bin 
  extlibs/netty-codec-4.0.20.Final.jar     |  Bin 
  extlibs/netty-common-4.0.18.Final.jar    |  Bin 
  extlibs/netty-common-4.0.20.Final.jar    |  Bin 
  extlibs/netty-handler-4.0.18.Final.jar   |  Bin 
  extlibs/netty-handler-4.0.20.Final.jar   |  Bin 
  extlibs/netty-transport-4.0.18.Final.jar |  Bin 
  extlibs/netty-transport-4.0.20.Final.jar |  Bin 
  11 files changed, 5 insertions(+), 5 deletions(-)


diff --git a/build.xml b/build.xml
--- a/build.xml
+++ b/build.xml
@@ -583,15 +583,15 @@
             
             
             
-            
+            
             
-            
+            
             
-            
+            
             
-            
+            
             
-            
+            
             
             
             
diff --git a/extlibs/netty-buffer-4.0.18.Final.jar b/extlibs/netty-buffer-4.0.18.Final.jar
deleted file mode 100644
index 4ef63f516ba9c0dbb951c85a7b1382f9d3c1f23f..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-buffer-4.0.20.Final.jar b/extlibs/netty-buffer-4.0.20.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0ac294bfdbcda2224d025ddb146b58284bfc0525
GIT binary patch
[stripped]
diff --git a/extlibs/netty-codec-4.0.18.Final.jar b/extlibs/netty-codec-4.0.18.Final.jar
deleted file mode 100644
index 248d26950900a16fa5612b06aaaf0a58d6b000fc..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-codec-4.0.20.Final.jar b/extlibs/netty-codec-4.0.20.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..4d4d3a99aca2b16b2df6558d4051fdc53219e8cc
GIT binary patch
[stripped]
diff --git a/extlibs/netty-common-4.0.18.Final.jar b/extlibs/netty-common-4.0.18.Final.jar
deleted file mode 100644
index ee7491c13be9228430aecbe358297f2aada76db1..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-common-4.0.20.Final.jar b/extlibs/netty-common-4.0.20.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..421956e0a7fbc901d50ac8ef03eadb639235d986
GIT binary patch
[stripped]
diff --git a/extlibs/netty-handler-4.0.18.Final.jar b/extlibs/netty-handler-4.0.18.Final.jar
deleted file mode 100644
index 22eb114f569f9c6f5ca05cf4a935c9913f5c29d8..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-handler-4.0.20.Final.jar b/extlibs/netty-handler-4.0.20.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..636e0c957874f623954b06c6992d7188f21de0a2
GIT binary patch
[stripped]
diff --git a/extlibs/netty-transport-4.0.18.Final.jar b/extlibs/netty-transport-4.0.18.Final.jar
deleted file mode 100644
index 133c0fa65c71555ed90c960bf95233167aacc4c1..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-transport-4.0.20.Final.jar b/extlibs/netty-transport-4.0.20.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0aeebc30bbc977bb212c1c2e8b3dce7838117f7b
GIT binary patch
[stripped]

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Mon Jun 30 20:39:18 2014
From: jython-checkins at python.org (jim.baker)
Date: Mon, 30 Jun 2014 20:39:18 +0200 (CEST)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Workaround_SSL_handshaking_?=
 =?utf-8?q?problem_in_http=3A//bugs=2Ejython=2Eorg/issue2174?=
Message-ID: <3h2HY20DShz7Ljl@mail.python.org>

http://hg.python.org/jython/rev/f6c9712832d0
changeset:   7344:f6c9712832d0
user:        Jim Baker 
date:        Mon Jun 30 12:39:09 2014 -0600
summary:
  Workaround SSL handshaking problem in http://bugs.jython.org/issue2174

files:
  Lib/ssl.py |  2 ++
  1 files changed, 2 insertions(+), 0 deletions(-)


diff --git a/Lib/ssl.py b/Lib/ssl.py
--- a/Lib/ssl.py
+++ b/Lib/ssl.py
@@ -1,4 +1,5 @@
 import logging
+import time
 
 try:
     # jarjar-ed version
@@ -112,6 +113,7 @@
 
         self.ssl_handler.handshakeFuture().addListener(handshake_step)
         if self.do_handshake_on_connect and self.already_handshaked:
+            time.sleep(0.1)  # FIXME do we need this sleep
             self.ssl_handler.handshakeFuture().sync()
             log.debug("SSL handshaking completed", extra={"sock": self._sock})
 

-- 
Repository URL: http://hg.python.org/jython