From python-checkins at python.org Sun Sep 1 00:13:56 2013 From: python-checkins at python.org (eli.bendersky) Date: Sun, 1 Sep 2013 00:13:56 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Switch_the_AF=5F*_and_SOCK?= =?utf-8?q?=5F*_constants_in_the_socket_module_to_IntEnum=2E?= Message-ID: <3cSBfX3PyCzRlV@mail.python.org> http://hg.python.org/cpython/rev/038543d34166 changeset: 85481:038543d34166 user: Eli Bendersky date: Sat Aug 31 15:13:30 2013 -0700 summary: Switch the AF_* and SOCK_* constants in the socket module to IntEnum. Closes #18720. files: Lib/socket.py | 66 ++++++++++++++++++++++++++++- Lib/test/test_socket.py | 28 +++++++++++- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/Lib/socket.py b/Lib/socket.py --- a/Lib/socket.py +++ b/Lib/socket.py @@ -48,6 +48,7 @@ from _socket import * import os, sys, io +from enum import IntEnum try: import errno @@ -60,6 +61,30 @@ __all__ = ["getfqdn", "create_connection"] __all__.extend(os._get_exports_list(_socket)) +# Set up the socket.AF_* socket.SOCK_* constants as members of IntEnums for +# nicer string representations. +# Note that _socket only knows about the integer values. The public interface +# in this module understands the enums and translates them back from integers +# where needed (e.g. .family property of a socket object). +AddressFamily = IntEnum('AddressFamily', + {name: value for name, value in globals().items() + if name.isupper() and name.startswith('AF_')}) +globals().update(AddressFamily.__members__) + +SocketType = IntEnum('SocketType', + {name: value for name, value in globals().items() + if name.isupper() and name.startswith('SOCK_')}) +globals().update(SocketType.__members__) + +def _intenum_converter(value, enum_klass): + """Convert a numeric family value to an IntEnum member. + + If it's not a known member, return the numeric value itself. + """ + try: + return enum_klass(value) + except ValueError: + return value _realsocket = socket @@ -91,6 +116,10 @@ __slots__ = ["__weakref__", "_io_refs", "_closed"] def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None): + # For user code address family and type values are IntEnum members, but + # for the underlying _socket.socket they're just integers. The + # constructor of _socket.socket converts the given argument to an + # integer automatically. _socket.socket.__init__(self, family, type, proto, fileno) self._io_refs = 0 self._closed = False @@ -230,6 +259,18 @@ self._closed = True return super().detach() + @property + def family(self): + """Read-only access to the address family for this socket. + """ + return _intenum_converter(super().family, AddressFamily) + + @property + def type(self): + """Read-only access to the socket type. + """ + return _intenum_converter(super().type, SocketType) + if os.name == 'nt': def get_inheritable(self): return os.get_handle_inheritable(self.fileno()) @@ -243,7 +284,6 @@ get_inheritable.__doc__ = "Get the inheritable flag of the socket" set_inheritable.__doc__ = "Set the inheritable flag of the socket" - def fromfd(fd, family, type, proto=0): """ fromfd(fd, family, type[, proto]) -> socket object @@ -469,3 +509,27 @@ raise err else: raise error("getaddrinfo returns an empty list") + +def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0): + """Resolve host and port into list of address info entries. + + Translate the host/port argument into a sequence of 5-tuples that contain + all the necessary arguments for creating a socket connected to that service. + host is a domain name, a string representation of an IPv4/v6 address or + None. port is a string service name such as 'http', a numeric port number or + None. By passing None as the value of host and port, you can pass NULL to + the underlying C API. + + The family, type and proto arguments can be optionally specified in order to + narrow the list of addresses returned. Passing zero as a value for each of + these arguments selects the full range of results. + """ + # We override this function since we want to translate the numeric family + # and socket type values to enum constants. + addrlist = [] + for res in _socket.getaddrinfo(host, port, family, type, proto, flags): + af, socktype, proto, canonname, sa = res + addrlist.append((_intenum_converter(af, AddressFamily), + _intenum_converter(socktype, SocketType), + proto, canonname, sa)) + return addrlist diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1161,9 +1161,12 @@ socket.getaddrinfo(HOST, 80) socket.getaddrinfo(HOST, None) # test family and socktype filters - infos = socket.getaddrinfo(HOST, None, socket.AF_INET) - for family, _, _, _, _ in infos: + infos = socket.getaddrinfo(HOST, 80, socket.AF_INET, socket.SOCK_STREAM) + for family, type, _, _, _ in infos: self.assertEqual(family, socket.AF_INET) + self.assertEqual(str(family), 'AddressFamily.AF_INET') + self.assertEqual(type, socket.SOCK_STREAM) + self.assertEqual(str(type), 'SocketType.SOCK_STREAM') infos = socket.getaddrinfo(HOST, None, 0, socket.SOCK_STREAM) for _, socktype, _, _, _ in infos: self.assertEqual(socktype, socket.SOCK_STREAM) @@ -1321,6 +1324,27 @@ with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: self.assertRaises(OverflowError, s.bind, (support.HOSTv6, 0, -10)) + def test_str_for_enums(self): + # Make sure that the AF_* and SOCK_* constants have enum-like string + # reprs. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + self.assertEqual(str(s.family), 'AddressFamily.AF_INET') + self.assertEqual(str(s.type), 'SocketType.SOCK_STREAM') + + @unittest.skipIf(os.name == 'nt', 'Will not work on Windows') + def test_uknown_socket_family_repr(self): + # Test that when created with a family that's not one of the known + # AF_*/SOCK_* constants, socket.family just returns the number. + # + # To do this we fool socket.socket into believing it already has an + # open fd because on this path it doesn't actually verify the family and + # type and populates the socket object. + # + # On Windows this trick won't work, so the test is skipped. + fd, _ = tempfile.mkstemp() + with socket.socket(family=42424, type=13331, fileno=fd) as s: + self.assertEqual(s.family, 42424) + self.assertEqual(s.type, 13331) @unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') class BasicCANTest(unittest.TestCase): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 1 00:19:14 2013 From: python-checkins at python.org (eli.bendersky) Date: Sun, 1 Sep 2013 00:19:14 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Update_whatsnew/3=2E4=2Ers?= =?utf-8?q?t_wrt=2E_the_socket_constants_switch_to_IntEnum?= Message-ID: <3cSBmf31XfzPkT@mail.python.org> http://hg.python.org/cpython/rev/4d604f1f0219 changeset: 85482:4d604f1f0219 user: Eli Bendersky date: Sat Aug 31 15:18:48 2013 -0700 summary: Update whatsnew/3.4.rst wrt. the socket constants switch to IntEnum [issue #18730] files: Doc/whatsnew/3.4.rst | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -320,6 +320,9 @@ * :meth:`socket.socket.get_inheritable`, :meth:`socket.socket.set_inheritable` +The ``socket.AF_*`` and ``socket.SOCK_*`` constants are enumeration values, +using the new :mod:`enum` module. This allows descriptive reporting during +debugging, instead of seeing integer "magic numbers". ssl --- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 1 02:37:40 2013 From: python-checkins at python.org (terry.reedy) Date: Sun, 1 Sep 2013 02:37:40 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Silence_deprec?= =?utf-8?q?ation_warning_in_test=5Funittest_for_=27None_=3E_1=27=2E?= Message-ID: <3cSFrN4Kn3zPw6@mail.python.org> http://hg.python.org/cpython/rev/7b9b85505db9 changeset: 85483:7b9b85505db9 branch: 2.7 parent: 85476:4179e2312089 user: Terry Jan Reedy date: Sat Aug 31 20:37:21 2013 -0400 summary: Silence deprecation warning in test_unittest for 'None > 1'. This is the same change that was made in 3.x when this became an error. files: Lib/unittest/test/test_runner.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -159,7 +159,7 @@ # This used to raise an exception due to TextTestResult not passing # on arguments in its __init__ super call - ATextResult(None, None, None) + ATextResult(None, None, 1) def testBufferAndFailfast(self): -- Repository URL: http://hg.python.org/cpython From tjreedy at udel.edu Sun Sep 1 01:52:17 2013 From: tjreedy at udel.edu (Terry Reedy) Date: Sat, 31 Aug 2013 19:52:17 -0400 Subject: [Python-checkins] cpython: Update whatsnew/3.4.rst wrt. the socket constants switch to IntEnum In-Reply-To: <3cSBmf31XfzPkT@mail.python.org> References: <3cSBmf31XfzPkT@mail.python.org> Message-ID: <522281B1.50004@udel.edu> On 8/31/2013 6:19 PM, eli.bendersky wrote: > http://hg.python.org/cpython/rev/4d604f1f0219 > changeset: 85482:4d604f1f0219 > user: Eli Bendersky > date: Sat Aug 31 15:18:48 2013 -0700 > summary: > Update whatsnew/3.4.rst wrt. the socket constants switch to IntEnum > > [issue #18730] Wrong issue number I think. From python-checkins at python.org Sun Sep 1 03:14:21 2013 From: python-checkins at python.org (terry.reedy) Date: Sun, 1 Sep 2013 03:14:21 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Silence_deprec?= =?utf-8?q?ation_warning_in_test=5Fmmap_for_ValueError=2Emessage=2E?= Message-ID: <3cSGfj660RzR7L@mail.python.org> http://hg.python.org/cpython/rev/869d50357a71 changeset: 85484:869d50357a71 branch: 2.7 user: Terry Jan Reedy date: Sat Aug 31 21:14:00 2013 -0400 summary: Silence deprecation warning in test_mmap for ValueError.message. Replacement is from 3.3, with 'p' added to make '...Regexp'. files: Lib/test/test_mmap.py | 12 ++++-------- 1 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -470,14 +470,10 @@ f = open (TESTFN, 'w+b') f.close() with open(TESTFN, "rb") as f : - try: - m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) - m.close() - self.fail("should not have been able to mmap empty file") - except ValueError as e: - self.assertEqual(e.message, "cannot mmap an empty file") - except: - self.fail("unexpected exception: " + str(e)) + self.assertRaisesRegexp(ValueError, + "cannot mmap an empty file", + mmap.mmap, f.fileno(), 0, + access=mmap.ACCESS_READ) def test_offset (self): f = open (TESTFN, 'w+b') -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 1 04:18:13 2013 From: python-checkins at python.org (ethan.furman) Date: Sun, 1 Sep 2013 04:18:13 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Close_=2318738=3A__Route_?= =?utf-8?q?=5F=5Fformat=5F=5F_calls_to_mixed-in_type_for_mixed_Enums_=28su?= =?utf-8?q?ch_as?= Message-ID: <3cSJ4P72lGz7LjM@mail.python.org> http://hg.python.org/cpython/rev/058cb219b3b5 changeset: 85485:058cb219b3b5 parent: 85482:4d604f1f0219 user: Ethan Furman date: Sat Aug 31 19:17:41 2013 -0700 summary: Close #18738: Route __format__ calls to mixed-in type for mixed Enums (such as IntEnum). files: Doc/library/enum.rst | 6 + Lib/enum.py | 18 ++++- Lib/test/test_enum.py | 102 +++++++++++++++++++++++++++++- 3 files changed, 122 insertions(+), 4 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -463,6 +463,12 @@ 3. When another data type is mixed in, the :attr:`value` attribute is *not the same* as the enum member itself, although it is equivalant and will compare equal. +4. %-style formatting: `%s` and `%r` call :class:`Enum`'s :meth:`__str__` and + :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for + IntEnum) treat the enum member as its mixed-in type. +5. :class:`str`.:meth:`__format__` (or :func:`format`) will use the mixed-in + type's :meth:`__format__`. If the :class:`Enum`'s :func:`str` or + :func:`repr` is desired use the `!s` or `!r` :class:`str` format codes. Interesting examples diff --git a/Lib/enum.py b/Lib/enum.py --- a/Lib/enum.py +++ b/Lib/enum.py @@ -50,7 +50,6 @@ cls.__reduce__ = _break_on_call_reduce cls.__module__ = '' - class _EnumDict(dict): """Keeps track of definition order of the enum items. @@ -182,7 +181,7 @@ # double check that repr and friends are not the mixin's or various # things break (such as pickle) - for name in ('__repr__', '__str__', '__getnewargs__'): + for name in ('__repr__', '__str__', '__format__', '__getnewargs__'): class_method = getattr(enum_class, name) obj_method = getattr(member_type, name, None) enum_method = getattr(first_enum, name, None) @@ -441,6 +440,21 @@ return self is other return NotImplemented + def __format__(self, format_spec): + # mixed-in Enums should use the mixed-in type's __format__, otherwise + # we can get strange results with the Enum name showing up instead of + # the value + + # pure Enum branch + if self._member_type_ is object: + cls = str + val = str(self) + # mix-in branch + else: + cls = self._member_type_ + val = self.value + return cls.__format__(val, format_spec) + def __getnewargs__(self): return (self._value_, ) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -67,6 +67,33 @@ WINTER = 4 self.Season = Season + class Konstants(float, Enum): + E = 2.7182818 + PI = 3.1415926 + TAU = 2 * PI + self.Konstants = Konstants + + class Grades(IntEnum): + A = 5 + B = 4 + C = 3 + D = 2 + F = 0 + self.Grades = Grades + + class Directional(str, Enum): + EAST = 'east' + WEST = 'west' + NORTH = 'north' + SOUTH = 'south' + self.Directional = Directional + + from datetime import date + class Holiday(date, Enum): + NEW_YEAR = 2013, 1, 1 + IDES_OF_MARCH = 2013, 3, 15 + self.Holiday = Holiday + def test_dir_on_class(self): Season = self.Season self.assertEqual( @@ -207,6 +234,77 @@ self.assertIs(type(Huh.name), Huh) self.assertEqual(Huh.name.name, 'name') self.assertEqual(Huh.name.value, 1) + + def test_format_enum(self): + Season = self.Season + self.assertEqual('{}'.format(Season.SPRING), + '{}'.format(str(Season.SPRING))) + self.assertEqual( '{:}'.format(Season.SPRING), + '{:}'.format(str(Season.SPRING))) + self.assertEqual('{:20}'.format(Season.SPRING), + '{:20}'.format(str(Season.SPRING))) + self.assertEqual('{:^20}'.format(Season.SPRING), + '{:^20}'.format(str(Season.SPRING))) + self.assertEqual('{:>20}'.format(Season.SPRING), + '{:>20}'.format(str(Season.SPRING))) + self.assertEqual('{:<20}'.format(Season.SPRING), + '{:<20}'.format(str(Season.SPRING))) + + def test_format_enum_custom(self): + class TestFloat(float, Enum): + one = 1.0 + two = 2.0 + def __format__(self, spec): + return 'TestFloat success!' + self.assertEqual('{}'.format(TestFloat.one), 'TestFloat success!') + + def assertFormatIsValue(self, spec, member): + self.assertEqual(spec.format(member), spec.format(member.value)) + + def test_format_enum_date(self): + Holiday = self.Holiday + self.assertFormatIsValue('{}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:^20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:>20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:<20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:%Y %m}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:%Y %m %M:00}', Holiday.IDES_OF_MARCH) + + def test_format_enum_float(self): + Konstants = self.Konstants + self.assertFormatIsValue('{}', Konstants.TAU) + self.assertFormatIsValue('{:}', Konstants.TAU) + self.assertFormatIsValue('{:20}', Konstants.TAU) + self.assertFormatIsValue('{:^20}', Konstants.TAU) + self.assertFormatIsValue('{:>20}', Konstants.TAU) + self.assertFormatIsValue('{:<20}', Konstants.TAU) + self.assertFormatIsValue('{:n}', Konstants.TAU) + self.assertFormatIsValue('{:5.2}', Konstants.TAU) + self.assertFormatIsValue('{:f}', Konstants.TAU) + + def test_format_enum_int(self): + Grades = self.Grades + self.assertFormatIsValue('{}', Grades.C) + self.assertFormatIsValue('{:}', Grades.C) + self.assertFormatIsValue('{:20}', Grades.C) + self.assertFormatIsValue('{:^20}', Grades.C) + self.assertFormatIsValue('{:>20}', Grades.C) + self.assertFormatIsValue('{:<20}', Grades.C) + self.assertFormatIsValue('{:+}', Grades.C) + self.assertFormatIsValue('{:08X}', Grades.C) + self.assertFormatIsValue('{:b}', Grades.C) + + def test_format_enum_str(self): + Directional = self.Directional + self.assertFormatIsValue('{}', Directional.WEST) + self.assertFormatIsValue('{:}', Directional.WEST) + self.assertFormatIsValue('{:20}', Directional.WEST) + self.assertFormatIsValue('{:^20}', Directional.WEST) + self.assertFormatIsValue('{:>20}', Directional.WEST) + self.assertFormatIsValue('{:<20}', Directional.WEST) + def test_hash(self): Season = self.Season dates = {} @@ -232,7 +330,7 @@ def test_floatenum_from_scratch(self): class phy(float, Enum): - pi = 3.141596 + pi = 3.1415926 tau = 2 * pi self.assertTrue(phy.pi < phy.tau) @@ -240,7 +338,7 @@ class FloatEnum(float, Enum): pass class phy(FloatEnum): - pi = 3.141596 + pi = 3.1415926 tau = 2 * pi self.assertTrue(phy.pi < phy.tau) -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Sun Sep 1 06:16:52 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sun, 01 Sep 2013 06:16:52 +0200 Subject: [Python-checkins] Daily reference leaks (4d604f1f0219): sum=0 Message-ID: results for 4d604f1f0219 on branch "default" -------------------------------------------- Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogcD6_Ao', '-x'] From python-checkins at python.org Sun Sep 1 06:29:23 2013 From: python-checkins at python.org (raymond.hettinger) Date: Sun, 1 Sep 2013 06:29:23 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Further_reduce_the_cost_of?= =?utf-8?q?_hash_collisions_by_inspecting_an_additional_nearby?= Message-ID: <3cSLzl490rzSc9@mail.python.org> http://hg.python.org/cpython/rev/d40a65658ff0 changeset: 85486:d40a65658ff0 parent: 85482:4d604f1f0219 user: Raymond Hettinger date: Sat Aug 31 21:27:08 2013 -0700 summary: Further reduce the cost of hash collisions by inspecting an additional nearby entry. files: Objects/setobject.c | 43 +++++++++++++++++++++++++++++--- 1 files changed, 39 insertions(+), 4 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -65,10 +65,11 @@ The initial probe index is computed as hash mod the table size. Subsequent probe indices are computed as explained in Objects/dictobject.c. -To improve cache locality, each probe is done in pairs. -After the probe is examined, an adjacent entry is then examined as well. -The likelihood is that an adjacent entry is in the same cache line and -can be examined more cheaply than another probe elsewhere in memory. +To improve cache locality, each probe inspects nearby entries before +moving on to probes elsewhere in memory. Depending on alignment and the +size of a cache line, the nearby entries are cheaper to inspect than +other probes elsewhere in memory. This probe strategy reduces the cost +of hash collisions. All arithmetic on hash should ignore overflow. @@ -130,6 +131,26 @@ if (entry->key == dummy && freeslot == NULL) freeslot = entry; + entry = &table[j ^ 2]; + if (entry->key == NULL) + break; + if (entry->key == key) + return entry; + if (entry->hash == hash && entry->key != dummy) { + PyObject *startkey = entry->key; + Py_INCREF(startkey); + cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) + return NULL; + if (table != so->table || entry->key != startkey) + return set_lookkey(so, key, hash); + if (cmp > 0) + return entry; + } + if (entry->key == dummy && freeslot == NULL) + freeslot = entry; + i = i * 5 + perturb + 1; j = i & mask; perturb >>= PERTURB_SHIFT; @@ -190,6 +211,17 @@ if (entry->key == dummy && freeslot == NULL) freeslot = entry; + entry = &table[j ^ 2]; + if (entry->key == NULL) + break; + if (entry->key == key + || (entry->hash == hash + && entry->key != dummy + && unicode_eq(entry->key, key))) + return entry; + if (entry->key == dummy && freeslot == NULL) + freeslot = entry; + i = i * 5 + perturb + 1; j = i & mask; perturb >>= PERTURB_SHIFT; @@ -258,6 +290,9 @@ entry = &table[j ^ 1]; if (entry->key == NULL) break; + entry = &table[j ^ 2]; + if (entry->key == NULL) + break; i = i * 5 + perturb + 1; j = i & mask; perturb >>= PERTURB_SHIFT; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 1 06:29:25 2013 From: python-checkins at python.org (raymond.hettinger) Date: Sun, 1 Sep 2013 06:29:25 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?= =?utf-8?q?=29=3A_merge?= Message-ID: <3cSLzn03Mcz7LjM@mail.python.org> http://hg.python.org/cpython/rev/47ccca4b27ce changeset: 85487:47ccca4b27ce parent: 85486:d40a65658ff0 parent: 85485:058cb219b3b5 user: Raymond Hettinger date: Sat Aug 31 21:28:58 2013 -0700 summary: merge files: Doc/library/enum.rst | 6 + Lib/enum.py | 18 ++++- Lib/test/test_enum.py | 102 +++++++++++++++++++++++++++++- 3 files changed, 122 insertions(+), 4 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -463,6 +463,12 @@ 3. When another data type is mixed in, the :attr:`value` attribute is *not the same* as the enum member itself, although it is equivalant and will compare equal. +4. %-style formatting: `%s` and `%r` call :class:`Enum`'s :meth:`__str__` and + :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for + IntEnum) treat the enum member as its mixed-in type. +5. :class:`str`.:meth:`__format__` (or :func:`format`) will use the mixed-in + type's :meth:`__format__`. If the :class:`Enum`'s :func:`str` or + :func:`repr` is desired use the `!s` or `!r` :class:`str` format codes. Interesting examples diff --git a/Lib/enum.py b/Lib/enum.py --- a/Lib/enum.py +++ b/Lib/enum.py @@ -50,7 +50,6 @@ cls.__reduce__ = _break_on_call_reduce cls.__module__ = '' - class _EnumDict(dict): """Keeps track of definition order of the enum items. @@ -182,7 +181,7 @@ # double check that repr and friends are not the mixin's or various # things break (such as pickle) - for name in ('__repr__', '__str__', '__getnewargs__'): + for name in ('__repr__', '__str__', '__format__', '__getnewargs__'): class_method = getattr(enum_class, name) obj_method = getattr(member_type, name, None) enum_method = getattr(first_enum, name, None) @@ -441,6 +440,21 @@ return self is other return NotImplemented + def __format__(self, format_spec): + # mixed-in Enums should use the mixed-in type's __format__, otherwise + # we can get strange results with the Enum name showing up instead of + # the value + + # pure Enum branch + if self._member_type_ is object: + cls = str + val = str(self) + # mix-in branch + else: + cls = self._member_type_ + val = self.value + return cls.__format__(val, format_spec) + def __getnewargs__(self): return (self._value_, ) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -67,6 +67,33 @@ WINTER = 4 self.Season = Season + class Konstants(float, Enum): + E = 2.7182818 + PI = 3.1415926 + TAU = 2 * PI + self.Konstants = Konstants + + class Grades(IntEnum): + A = 5 + B = 4 + C = 3 + D = 2 + F = 0 + self.Grades = Grades + + class Directional(str, Enum): + EAST = 'east' + WEST = 'west' + NORTH = 'north' + SOUTH = 'south' + self.Directional = Directional + + from datetime import date + class Holiday(date, Enum): + NEW_YEAR = 2013, 1, 1 + IDES_OF_MARCH = 2013, 3, 15 + self.Holiday = Holiday + def test_dir_on_class(self): Season = self.Season self.assertEqual( @@ -207,6 +234,77 @@ self.assertIs(type(Huh.name), Huh) self.assertEqual(Huh.name.name, 'name') self.assertEqual(Huh.name.value, 1) + + def test_format_enum(self): + Season = self.Season + self.assertEqual('{}'.format(Season.SPRING), + '{}'.format(str(Season.SPRING))) + self.assertEqual( '{:}'.format(Season.SPRING), + '{:}'.format(str(Season.SPRING))) + self.assertEqual('{:20}'.format(Season.SPRING), + '{:20}'.format(str(Season.SPRING))) + self.assertEqual('{:^20}'.format(Season.SPRING), + '{:^20}'.format(str(Season.SPRING))) + self.assertEqual('{:>20}'.format(Season.SPRING), + '{:>20}'.format(str(Season.SPRING))) + self.assertEqual('{:<20}'.format(Season.SPRING), + '{:<20}'.format(str(Season.SPRING))) + + def test_format_enum_custom(self): + class TestFloat(float, Enum): + one = 1.0 + two = 2.0 + def __format__(self, spec): + return 'TestFloat success!' + self.assertEqual('{}'.format(TestFloat.one), 'TestFloat success!') + + def assertFormatIsValue(self, spec, member): + self.assertEqual(spec.format(member), spec.format(member.value)) + + def test_format_enum_date(self): + Holiday = self.Holiday + self.assertFormatIsValue('{}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:^20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:>20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:<20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:%Y %m}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{:%Y %m %M:00}', Holiday.IDES_OF_MARCH) + + def test_format_enum_float(self): + Konstants = self.Konstants + self.assertFormatIsValue('{}', Konstants.TAU) + self.assertFormatIsValue('{:}', Konstants.TAU) + self.assertFormatIsValue('{:20}', Konstants.TAU) + self.assertFormatIsValue('{:^20}', Konstants.TAU) + self.assertFormatIsValue('{:>20}', Konstants.TAU) + self.assertFormatIsValue('{:<20}', Konstants.TAU) + self.assertFormatIsValue('{:n}', Konstants.TAU) + self.assertFormatIsValue('{:5.2}', Konstants.TAU) + self.assertFormatIsValue('{:f}', Konstants.TAU) + + def test_format_enum_int(self): + Grades = self.Grades + self.assertFormatIsValue('{}', Grades.C) + self.assertFormatIsValue('{:}', Grades.C) + self.assertFormatIsValue('{:20}', Grades.C) + self.assertFormatIsValue('{:^20}', Grades.C) + self.assertFormatIsValue('{:>20}', Grades.C) + self.assertFormatIsValue('{:<20}', Grades.C) + self.assertFormatIsValue('{:+}', Grades.C) + self.assertFormatIsValue('{:08X}', Grades.C) + self.assertFormatIsValue('{:b}', Grades.C) + + def test_format_enum_str(self): + Directional = self.Directional + self.assertFormatIsValue('{}', Directional.WEST) + self.assertFormatIsValue('{:}', Directional.WEST) + self.assertFormatIsValue('{:20}', Directional.WEST) + self.assertFormatIsValue('{:^20}', Directional.WEST) + self.assertFormatIsValue('{:>20}', Directional.WEST) + self.assertFormatIsValue('{:<20}', Directional.WEST) + def test_hash(self): Season = self.Season dates = {} @@ -232,7 +330,7 @@ def test_floatenum_from_scratch(self): class phy(float, Enum): - pi = 3.141596 + pi = 3.1415926 tau = 2 * pi self.assertTrue(phy.pi < phy.tau) @@ -240,7 +338,7 @@ class FloatEnum(float, Enum): pass class phy(FloatEnum): - pi = 3.141596 + pi = 3.1415926 tau = 2 * pi self.assertTrue(phy.pi < phy.tau) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 1 06:34:39 2013 From: python-checkins at python.org (raymond.hettinger) Date: Sun, 1 Sep 2013 06:34:39 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Update_copyright=2E?= Message-ID: <3cSM5q0tcCz7LjM@mail.python.org> http://hg.python.org/cpython/rev/47e7f18cd3a1 changeset: 85488:47e7f18cd3a1 user: Raymond Hettinger date: Sat Aug 31 21:34:24 2013 -0700 summary: Update copyright. files: Objects/setobject.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -3,7 +3,7 @@ Written and maintained by Raymond D. Hettinger Derived from Lib/sets.py and Objects/dictobject.c. - Copyright (c) 2003-2008 Python Software Foundation. + Copyright (c) 2003-2013 Python Software Foundation. All rights reserved. */ -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 1 06:45:01 2013 From: python-checkins at python.org (tim.peters) Date: Sun, 1 Sep 2013 06:45:01 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Back_out_868ad6fa8e68_-_it?= =?utf-8?q?_left_all_the_buildbots_failing=2E?= Message-ID: <3cSMKn0ZTWz7LjX@mail.python.org> http://hg.python.org/cpython/rev/7035b5d8fc0f changeset: 85489:7035b5d8fc0f user: Tim Peters date: Sat Aug 31 23:44:34 2013 -0500 summary: Back out 868ad6fa8e68 - it left all the buildbots failing. Unclear to me why it was pushed to begin with. See issue 11798. Perhaps it's because regrtest with -R was failing? Fine, but that's better than regrtest _always_ failing ;-) files: Lib/unittest/suite.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/Lib/unittest/suite.py b/Lib/unittest/suite.py --- a/Lib/unittest/suite.py +++ b/Lib/unittest/suite.py @@ -66,7 +66,6 @@ def _removeTestAtIndex(self, index): """Stop holding a reference to the TestCase at index.""" - return try: self._tests[index] = None except TypeError: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 1 06:58:57 2013 From: python-checkins at python.org (andrew.svetlov) Date: Sun, 1 Sep 2013 06:58:57 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2311798=3A_fix_test?= =?utf-8?q?s_for_regrtest_-R_=3A?= Message-ID: <3cSMds1ZCzzPrk@mail.python.org> http://hg.python.org/cpython/rev/39781c3737f8 changeset: 85490:39781c3737f8 user: Andrew Svetlov date: Sun Sep 01 07:58:41 2013 +0300 summary: Issue #11798: fix tests for regrtest -R : files: Lib/test/regrtest.py | 5 +++++ Lib/unittest/suite.py | 8 ++++++-- Lib/unittest/test/test_suite.py | 8 ++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -496,6 +496,8 @@ if ns.slaveargs is not None: args, kwargs = json.loads(ns.slaveargs) + if kwargs.get('huntrleaks'): + unittest.BaseTestSuite._cleanup = False try: result = runtest(*args, **kwargs) except KeyboardInterrupt: @@ -528,6 +530,9 @@ #gc.set_debug(gc.DEBUG_SAVEALL) found_garbage = [] + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + if ns.single: filename = os.path.join(TEMPDIR, 'pynexttest') try: diff --git a/Lib/unittest/suite.py b/Lib/unittest/suite.py --- a/Lib/unittest/suite.py +++ b/Lib/unittest/suite.py @@ -16,6 +16,8 @@ class BaseTestSuite(object): """A simple test suite that doesn't provide class or module shared fixtures. """ + _cleanup = True + def __init__(self, tests=()): self._tests = [] self.addTests(tests) @@ -61,7 +63,8 @@ if result.shouldStop: break test(result) - self._removeTestAtIndex(index) + if self._cleanup: + self._removeTestAtIndex(index) return result def _removeTestAtIndex(self, index): @@ -115,7 +118,8 @@ else: test.debug() - self._removeTestAtIndex(index) + if self._cleanup: + self._removeTestAtIndex(index) if topLevel: self._tearDownPreviousClass(None, result) diff --git a/Lib/unittest/test/test_suite.py b/Lib/unittest/test/test_suite.py --- a/Lib/unittest/test/test_suite.py +++ b/Lib/unittest/test/test_suite.py @@ -303,6 +303,9 @@ suite.run(unittest.TestResult()) def test_remove_test_at_index(self): + if not unittest.BaseTestSuite._cleanup: + raise unittest.SkipTest("Suite cleanup is disabled") + suite = unittest.TestSuite() suite._tests = [1, 2, 3] @@ -311,6 +314,9 @@ self.assertEqual([1, None, 3], suite._tests) def test_remove_test_at_index_not_indexable(self): + if not unittest.BaseTestSuite._cleanup: + raise unittest.SkipTest("Suite cleanup is disabled") + suite = unittest.TestSuite() suite._tests = None @@ -318,6 +324,8 @@ suite._removeTestAtIndex(2) def assert_garbage_collect_test_after_run(self, TestSuiteClass): + if not unittest.BaseTestSuite._cleanup: + raise unittest.SkipTest("Suite cleanup is disabled") class Foo(unittest.TestCase): def test_nothing(self): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 1 10:22:52 2013 From: python-checkins at python.org (victor.stinner) Date: Sun, 1 Sep 2013 10:22:52 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318571=3A_Merge_du?= =?utf-8?q?plicate_test_code?= Message-ID: <3cSS9823jwzRDt@mail.python.org> http://hg.python.org/cpython/rev/c27527dce71e changeset: 85491:c27527dce71e user: Victor Stinner date: Sun Sep 01 10:22:41 2013 +0200 summary: Issue #18571: Merge duplicate test code Merge test/subprocessdata/inherited.py into test/subprocessdata/fd_status.py files: Lib/test/subprocessdata/fd_status.py | 23 +++++++++++---- Lib/test/subprocessdata/inherited.py | 22 --------------- Lib/test/test_subprocess.py | 4 +- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/Lib/test/subprocessdata/fd_status.py b/Lib/test/subprocessdata/fd_status.py --- a/Lib/test/subprocessdata/fd_status.py +++ b/Lib/test/subprocessdata/fd_status.py @@ -1,18 +1,27 @@ """When called as a script, print a comma-separated list of the open -file descriptors on stdout.""" +file descriptors on stdout. + +Usage: +fd_stats.py: check all file descriptors +fd_status.py fd1 fd2 ...: check only specified file descriptors +""" import errno import os import stat - -try: - _MAXFD = os.sysconf("SC_OPEN_MAX") -except: - _MAXFD = 256 +import sys if __name__ == "__main__": fds = [] - for fd in range(0, _MAXFD): + if len(sys.argv) == 1: + try: + _MAXFD = os.sysconf("SC_OPEN_MAX") + except: + _MAXFD = 256 + test_fds = range(0, _MAXFD) + else: + test_fds = map(int, sys.argv[1:]) + for fd in test_fds: try: st = os.fstat(fd) except OSError as e: diff --git a/Lib/test/subprocessdata/inherited.py b/Lib/test/subprocessdata/inherited.py deleted file mode 100644 --- a/Lib/test/subprocessdata/inherited.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Similar to fd_status.py, but only checks file descriptors passed on the -command line.""" - -import errno -import os -import sys -import stat - -if __name__ == "__main__": - fds = map(int, sys.argv[1:]) - inherited = [] - for fd in fds: - try: - st = os.fstat(fd) - except OSError as e: - if e.errno == errno.EBADF: - continue - raise - # Ignore Solaris door files - if not stat.S_ISDOOR(st.st_mode): - inherited.append(fd) - print(','.join(map(str, inherited))) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1926,7 +1926,7 @@ self.assertIn('overriding close_fds', str(context.warning)) def test_pass_fds_inheritable(self): - script = support.findfile("inherited.py", subdir="subprocessdata") + script = support.findfile("fd_status.py", subdir="subprocessdata") inheritable, non_inheritable = os.pipe() self.addCleanup(os.close, inheritable) @@ -1945,7 +1945,7 @@ # the inheritable file descriptor must be inherited, so its inheritable # flag must be set in the child process after fork() and before exec() - self.assertEqual(fds, set(pass_fds)) + self.assertEqual(fds, set(pass_fds), "output=%a" % output) # inheritable flag must not be changed in the parent process self.assertEqual(os.get_inheritable(inheritable), True) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 1 11:20:47 2013 From: python-checkins at python.org (donald.stufft) Date: Sun, 1 Sep 2013 11:20:47 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Accept_PEP449?= Message-ID: <3cSTRz0s62zRQq@mail.python.org> http://hg.python.org/peps/rev/dd176eb8a1af changeset: 5088:dd176eb8a1af user: Donald Stufft date: Sun Sep 01 05:20:36 2013 -0400 summary: Accept PEP449 files: pep-0449.txt | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/pep-0449.txt b/pep-0449.txt --- a/pep-0449.txt +++ b/pep-0449.txt @@ -5,12 +5,13 @@ Author: Donald Stufft BDFL-Delegate: Richard Jones Discussions-To: distutils-sig at python.org -Status: Draft +Status: Accepted Type: Process Content-Type: text/x-rst Created: 04-Aug-2013 Post-History: 04-Aug-2013 Replaces: 381 +Resolution: http://mail.python.org/pipermail/distutils-sig/2013-August/022518.html Abstract -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sun Sep 1 19:52:29 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sun, 1 Sep 2013 19:52:29 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogdGVzdF90aHJlYWRp?= =?utf-8?q?ng_isn=27t_rudimentary_anymore?= Message-ID: <3cShpP0Vz5zSGS@mail.python.org> http://hg.python.org/cpython/rev/764ceb41192b changeset: 85492:764ceb41192b branch: 3.3 parent: 85479:f0eedca4b2a2 user: Antoine Pitrou date: Sun Sep 01 19:51:49 2013 +0200 summary: test_threading isn't rudimentary anymore files: Lib/test/test_threading.py | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1,4 +1,6 @@ -# Very rudimentary test of threading module +""" +Tests for the threading module. +""" import test.support from test.support import verbose, strip_python_stderr, import_module -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 1 19:52:30 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sun, 1 Sep 2013 19:52:30 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_test=5Fthreading_isn=27t_rudimentary_anymore?= Message-ID: <3cShpQ2SNdzSKR@mail.python.org> http://hg.python.org/cpython/rev/267e09700978 changeset: 85493:267e09700978 parent: 85491:c27527dce71e parent: 85492:764ceb41192b user: Antoine Pitrou date: Sun Sep 01 19:52:08 2013 +0200 summary: test_threading isn't rudimentary anymore files: Lib/test/test_threading.py | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1,4 +1,6 @@ -# Very rudimentary test of threading module +""" +Tests for the threading module. +""" import test.support from test.support import verbose, strip_python_stderr, import_module -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 1 20:00:15 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sun, 1 Sep 2013 20:00:15 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?devguide=3A_Add_a_benchmarks_category?= =?utf-8?q?_in_the_=22experts=22_section=2C_and_put_Brett_and_myself?= Message-ID: <3cShzM1Llzz7Ljg@mail.python.org> http://hg.python.org/devguide/rev/a9d8795af3b3 changeset: 639:a9d8795af3b3 user: Antoine Pitrou date: Sun Sep 01 20:00:03 2013 +0200 summary: Add a benchmarks category in the "experts" section, and put Brett and myself inside it files: experts.rst | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/experts.rst b/experts.rst --- a/experts.rst +++ b/experts.rst @@ -299,6 +299,7 @@ ast/compiler ncoghlan, benjamin.peterson, brett.cannon, georg.brandl autoconf/makefiles bsd +benchmarks pitrou, brett.cannon bug tracker ezio.melotti buildbots pitrou bytecode pitrou, georg.brandl -- Repository URL: http://hg.python.org/devguide From python-checkins at python.org Sun Sep 1 23:02:16 2013 From: python-checkins at python.org (tim.peters) Date: Sun, 1 Sep 2013 23:02:16 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Fix_issue_1888?= =?utf-8?q?9=3A_test=5Fsax=3A_multiple_failures_on_Windows_desktop=2E?= Message-ID: <3cSn1N51LZzQkJ@mail.python.org> http://hg.python.org/cpython/rev/8efcf3c823f9 changeset: 85494:8efcf3c823f9 branch: 3.3 parent: 85492:764ceb41192b user: Tim Peters date: Sun Sep 01 15:56:22 2013 -0500 summary: Fix issue 18889: test_sax: multiple failures on Windows desktop. "The fix" is to tell Mercurial that the test files are binary. Windows developers: to get the correct line endings in your checkout, delete Lib\test\xmltestdata, and then "hg revert" that directory. Why the Windows buildbots didn't fail test_sax remains a mystery :-( files: .hgeol | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/.hgeol b/.hgeol --- a/.hgeol +++ b/.hgeol @@ -31,6 +31,7 @@ Lib/test/decimaltestdata/*.decTest = BIN Lib/test/sndhdrdata/sndhdr.* = BIN Lib/test/test_email/data/msg_26.txt = BIN +Lib/test/xmltestdata/* = BIN Lib/venv/scripts/nt/* = BIN -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 1 23:02:17 2013 From: python-checkins at python.org (tim.peters) Date: Sun, 1 Sep 2013 23:02:17 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Merge_fix_from_3=2E3_into_default=2E?= Message-ID: <3cSn1P710Hz7LjX@mail.python.org> http://hg.python.org/cpython/rev/25211a22228b changeset: 85495:25211a22228b parent: 85493:267e09700978 parent: 85494:8efcf3c823f9 user: Tim Peters date: Sun Sep 01 16:01:46 2013 -0500 summary: Merge fix from 3.3 into default. Fix issue 18889: test_sax: multiple failures on Windows desktop. "The fix" is to tell Mercurial that the test files are binary. Windows developers: to get the correct line endings in your checkout, delete Lib\test\xmltestdata, and then "hg revert" that directory. Why the Windows buildbots didn't fail test_sax remains a mystery :-( files: .hgeol | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/.hgeol b/.hgeol --- a/.hgeol +++ b/.hgeol @@ -31,6 +31,7 @@ Lib/test/decimaltestdata/*.decTest = BIN Lib/test/sndhdrdata/sndhdr.* = BIN Lib/test/test_email/data/msg_26.txt = BIN +Lib/test/xmltestdata/* = BIN Lib/venv/scripts/nt/* = BIN -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 2 01:04:43 2013 From: python-checkins at python.org (terry.reedy) Date: Mon, 2 Sep 2013 01:04:43 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Remove_obsolet?= =?utf-8?q?e_=2Ehgeol_entry_pointing_to_file_moved_elsewhere=2E?= Message-ID: <3cSqkg6y1sz7Lk8@mail.python.org> http://hg.python.org/cpython/rev/43f27e69bc29 changeset: 85496:43f27e69bc29 branch: 3.3 parent: 85494:8efcf3c823f9 user: Terry Jan Reedy date: Sun Sep 01 19:03:41 2013 -0400 summary: Remove obsolete .hgeol entry pointing to file moved elsewhere. This kine was already replaced by Lib/test/test_email/data/msg_26.txt = BIN which is just below the last line in the patch context. files: .hgeol | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/.hgeol b/.hgeol --- a/.hgeol +++ b/.hgeol @@ -26,7 +26,6 @@ **.xar = BIN **.zip = BIN -Lib/email/test/data/msg_26.txt = BIN Lib/test/cjkencodings/* = BIN Lib/test/decimaltestdata/*.decTest = BIN Lib/test/sndhdrdata/sndhdr.* = BIN -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 2 01:04:45 2013 From: python-checkins at python.org (terry.reedy) Date: Mon, 2 Sep 2013 01:04:45 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Merge_with_3=2E3?= Message-ID: <3cSqkj1Zc7z7Ljj@mail.python.org> http://hg.python.org/cpython/rev/4a0ecfe0a907 changeset: 85497:4a0ecfe0a907 parent: 85495:25211a22228b parent: 85496:43f27e69bc29 user: Terry Jan Reedy date: Sun Sep 01 19:04:18 2013 -0400 summary: Merge with 3.3 files: .hgeol | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/.hgeol b/.hgeol --- a/.hgeol +++ b/.hgeol @@ -26,7 +26,6 @@ **.xar = BIN **.zip = BIN -Lib/email/test/data/msg_26.txt = BIN Lib/test/cjkencodings/* = BIN Lib/test/decimaltestdata/*.decTest = BIN Lib/test/sndhdrdata/sndhdr.* = BIN -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 2 01:07:00 2013 From: python-checkins at python.org (benjamin.peterson) Date: Mon, 2 Sep 2013 01:07:00 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_document_that_?= =?utf-8?q?various_functions_that_parse_from_source_will_interpret_things_?= =?utf-8?q?as?= Message-ID: <3cSqnJ59t1z7LjS@mail.python.org> http://hg.python.org/cpython/rev/869cbcabb934 changeset: 85498:869cbcabb934 branch: 2.7 parent: 85484:869d50357a71 user: Benjamin Peterson date: Sun Sep 01 19:06:35 2013 -0400 summary: document that various functions that parse from source will interpret things as latin-1 (closes #18870) files: Doc/library/ast.rst | 8 ++++---- Doc/library/functions.rst | 12 +++++++----- Doc/reference/simple_stmts.rst | 19 ++++++++++--------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -131,10 +131,10 @@ .. function:: literal_eval(node_or_string) - Safely evaluate an expression node or a string containing a Python - expression. The string or node provided may only consist of the following - Python literal structures: strings, numbers, tuples, lists, dicts, booleans, - and ``None``. + Safely evaluate an expression node or a Unicode or *Latin-1* encoded string + containing a Python expression. The string or node provided may only consist + of the following Python literal structures: strings, numbers, tuples, lists, + dicts, booleans, and ``None``. This can be used for safely evaluating strings containing Python expressions from untrusted sources without the need to parse the values oneself. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -199,8 +199,10 @@ Compile the *source* into a code or AST object. Code objects can be executed by an :keyword:`exec` statement or evaluated by a call to :func:`eval`. - *source* can either be a string or an AST object. Refer to the :mod:`ast` - module documentation for information on how to work with AST objects. + *source* can either be a Unicode string, a *Latin-1* encoded string or an + AST object. + Refer to the :mod:`ast` module documentation for information on how to work + with AST objects. The *filename* argument should give the file from which the code was read; pass some recognizable value if it wasn't read from a file (``''`` is @@ -388,9 +390,9 @@ .. function:: eval(expression[, globals[, locals]]) - The arguments are a string and optional globals and locals. If provided, - *globals* must be a dictionary. If provided, *locals* can be any mapping - object. + The arguments are a Unicode or *Latin-1* encoded string and optional + globals and locals. If provided, *globals* must be a dictionary. + If provided, *locals* can be any mapping object. .. versionchanged:: 2.4 formerly *locals* was required to be a dictionary. diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -981,15 +981,16 @@ exec_stmt: "exec" `or_expr` ["in" `expression` ["," `expression`]] This statement supports dynamic execution of Python code. The first expression -should evaluate to either a string, an open file object, a code object, or a -tuple. If it is a string, the string is parsed as a suite of Python statements -which is then executed (unless a syntax error occurs). [#]_ If it is an open -file, the file is parsed until EOF and executed. If it is a code object, it is -simply executed. For the interpretation of a tuple, see below. In all cases, -the code that's executed is expected to be valid as file input (see section -:ref:`file-input`). Be aware that the :keyword:`return` and :keyword:`yield` -statements may not be used outside of function definitions even within the -context of code passed to the :keyword:`exec` statement. +should evaluate to either a Unicode string, a *Latin-1* encoded string, an open +file object, a code object, or a tuple. If it is a string, the string is parsed +as a suite of Python statements which is then executed (unless a syntax error +occurs). [#]_ If it is an open file, the file is parsed until EOF and executed. +If it is a code object, it is simply executed. For the interpretation of a +tuple, see below. In all cases, the code that's executed is expected to be +valid as file input (see section :ref:`file-input`). Be aware that the +:keyword:`return` and :keyword:`yield` statements may not be used outside of +function definitions even within the context of code passed to the +:keyword:`exec` statement. In all cases, if the optional parts are omitted, the code is executed in the current scope. If only the first expression after ``in`` is specified, -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Mon Sep 2 06:19:58 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Mon, 02 Sep 2013 06:19:58 +0200 Subject: [Python-checkins] Daily reference leaks (4a0ecfe0a907): sum=0 Message-ID: results for 4a0ecfe0a907 on branch "default" -------------------------------------------- Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogFcJ6NO', '-x'] From python-checkins at python.org Mon Sep 2 10:15:23 2013 From: python-checkins at python.org (ethan.furman) Date: Mon, 2 Sep 2013 10:15:23 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Close_=2318745=3A_Improve_?= =?utf-8?q?enum_tests_in_test=5Fjson_for_infinities_and_NaN=2E?= Message-ID: <3cT3y35jVnzNH3@mail.python.org> http://hg.python.org/cpython/rev/50e583f20d78 changeset: 85499:50e583f20d78 parent: 85497:4a0ecfe0a907 user: Ethan Furman date: Mon Sep 02 01:14:56 2013 -0700 summary: Close #18745: Improve enum tests in test_json for infinities and NaN. files: Lib/test/test_json/test_enum.py | 49 ++++++++++++++++++-- 1 files changed, 44 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_json/test_enum.py b/Lib/test/test_json/test_enum.py --- a/Lib/test/test_json/test_enum.py +++ b/Lib/test/test_json/test_enum.py @@ -1,4 +1,5 @@ from enum import Enum, IntEnum +from math import isnan from test.test_json import PyTest, CTest SMALL = 1 @@ -21,6 +22,15 @@ pi = PI tau = TAU +INF = float('inf') +NEG_INF = float('-inf') +NAN = float('nan') + +class WierdNum(float, Enum): + inf = INF + neg_inf = NEG_INF + nan = NAN + class TestEnum: def test_floats(self): @@ -29,6 +39,16 @@ self.assertEqual(float(self.dumps(enum)), enum) self.assertEqual(self.loads(self.dumps(enum)), enum) + def test_weird_floats(self): + for enum, expected in zip(WierdNum, ('Infinity', '-Infinity', 'NaN')): + self.assertEqual(self.dumps(enum), expected) + if not isnan(enum): + self.assertEqual(float(self.dumps(enum)), enum) + self.assertEqual(self.loads(self.dumps(enum)), enum) + else: + self.assertTrue(isnan(float(self.dumps(enum)))) + self.assertTrue(isnan(self.loads(self.dumps(enum)))) + def test_ints(self): for enum in BigNum: self.assertEqual(self.dumps(enum), str(enum.value)) @@ -36,18 +56,28 @@ self.assertEqual(self.loads(self.dumps(enum)), enum) def test_list(self): - self.assertEqual( - self.dumps(list(BigNum)), - str([SMALL, BIG, HUGE, REALLY_HUGE]), - ) - self.assertEqual(self.dumps(list(FloatNum)), str([E, PI, TAU])) + self.assertEqual(self.dumps(list(BigNum)), + str([SMALL, BIG, HUGE, REALLY_HUGE])) + self.assertEqual(self.loads(self.dumps(list(BigNum))), + list(BigNum)) + self.assertEqual(self.dumps(list(FloatNum)), + str([E, PI, TAU])) + self.assertEqual(self.loads(self.dumps(list(FloatNum))), + list(FloatNum)) + self.assertEqual(self.dumps(list(WierdNum)), + '[Infinity, -Infinity, NaN]') + self.assertEqual(self.loads(self.dumps(list(WierdNum)))[:2], + list(WierdNum)[:2]) + self.assertTrue(isnan(self.loads(self.dumps(list(WierdNum)))[2])) def test_dict_keys(self): s, b, h, r = BigNum e, p, t = FloatNum + i, j, n = WierdNum d = { s:'tiny', b:'large', h:'larger', r:'largest', e:"Euler's number", p:'pi', t:'tau', + i:'Infinity', j:'-Infinity', n:'NaN', } nd = self.loads(self.dumps(d)) self.assertEqual(nd[str(SMALL)], 'tiny') @@ -57,6 +87,9 @@ self.assertEqual(nd[repr(E)], "Euler's number") self.assertEqual(nd[repr(PI)], 'pi') self.assertEqual(nd[repr(TAU)], 'tau') + self.assertEqual(nd['Infinity'], 'Infinity') + self.assertEqual(nd['-Infinity'], '-Infinity') + self.assertEqual(nd['NaN'], 'NaN') def test_dict_values(self): d = dict( @@ -67,6 +100,9 @@ e=FloatNum.e, pi=FloatNum.pi, tau=FloatNum.tau, + i=WierdNum.inf, + j=WierdNum.neg_inf, + n=WierdNum.nan, ) nd = self.loads(self.dumps(d)) self.assertEqual(nd['tiny'], SMALL) @@ -76,6 +112,9 @@ self.assertEqual(nd['e'], E) self.assertEqual(nd['pi'], PI) self.assertEqual(nd['tau'], TAU) + self.assertEqual(nd['i'], INF) + self.assertEqual(nd['j'], NEG_INF) + self.assertTrue(isnan(nd['n'])) class TestPyEnum(TestEnum, PyTest): pass class TestCEnum(TestEnum, CTest): pass -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 2 12:23:30 2013 From: python-checkins at python.org (raymond.hettinger) Date: Mon, 2 Sep 2013 12:23:30 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Instead_of_XORed_indicies?= =?utf-8?q?=2C_switch_to_a_hybrid_of_linear_probing_and_open?= Message-ID: <3cT6nt4HXjzMx4@mail.python.org> http://hg.python.org/cpython/rev/4a02212064ce changeset: 85500:4a02212064ce user: Raymond Hettinger date: Mon Sep 02 03:23:21 2013 -0700 summary: Instead of XORed indicies, switch to a hybrid of linear probing and open addressing. Modern processors tend to make consecutive memory accesses cheaper than random probes into memory. Small sets can fit into L1 cache, so they get less benefit. But they do come out ahead because the consecutive probes don't probe the same key more than once and because the randomization step occurs less frequently (or not at all). For the open addressing step, putting the perturb shift before the index calculation gets the upper bits into play sooner. files: Objects/setobject.c | 163 +++++++++++++------------------ 1 files changed, 70 insertions(+), 93 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -27,6 +27,7 @@ /* This must be >= 1. */ #define PERTURB_SHIFT 5 +#define LINEAR_PROBES 9 /* Object used as dummy key to fill deleted entries */ static PyObject _dummy_struct; @@ -59,17 +60,17 @@ /* The basic lookup function used by all operations. This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. -Open addressing is preferred over chaining since the link overhead for -chaining would be substantial (100% with typical malloc overhead). -The initial probe index is computed as hash mod the table size. Subsequent -probe indices are computed as explained in Objects/dictobject.c. +The initial probe index is computed as hash mod the table size. +Subsequent probe indices are computed as explained in Objects/dictobject.c. -To improve cache locality, each probe inspects nearby entries before -moving on to probes elsewhere in memory. Depending on alignment and the -size of a cache line, the nearby entries are cheaper to inspect than -other probes elsewhere in memory. This probe strategy reduces the cost -of hash collisions. +To improve cache locality, each probe inspects a series of consecutive +nearby entries before moving on to probes elsewhere in memory. This leaves +us with a hybrid of linear probing and open addressing. The linear probing +reduces the cost of hash collisions because consecutive memory accesses +tend to be much cheaper than scattered probes. After LINEAR_PROBES steps, +we then use open addressing with the upper bits from the hash value. This +helps break-up long chains of collisions. All arithmetic on hash should ignore overflow. @@ -83,13 +84,14 @@ setentry *table = so->table; setentry *freeslot = NULL; setentry *entry; + setentry *limit; size_t perturb = hash; size_t mask = so->mask; - size_t i = (size_t)hash & mask; /* Unsigned for defined overflow behavior. */ - size_t j = i; + size_t i = (size_t)hash; /* Unsigned for defined overflow behavior. */ + size_t j; int cmp; - entry = &table[i]; + entry = &table[i & mask]; if (entry->key == NULL) return entry; @@ -111,54 +113,37 @@ if (entry->key == dummy && freeslot == NULL) freeslot = entry; - entry = &table[j ^ 1]; - if (entry->key == NULL) - break; - if (entry->key == key) - return entry; - if (entry->hash == hash && entry->key != dummy) { - PyObject *startkey = entry->key; - Py_INCREF(startkey); - cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); - Py_DECREF(startkey); - if (cmp < 0) - return NULL; - if (table != so->table || entry->key != startkey) - return set_lookkey(so, key, hash); - if (cmp > 0) + limit = &table[mask]; + for (j = 0 ; j < LINEAR_PROBES ; j++) { + entry = (entry == limit) ? &table[0] : entry + 1; + if (entry->key == NULL) + goto found_null; + if (entry->key == key) return entry; + if (entry->hash == hash && entry->key != dummy) { + PyObject *startkey = entry->key; + Py_INCREF(startkey); + cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) + return NULL; + if (table != so->table || entry->key != startkey) + return set_lookkey(so, key, hash); + if (cmp > 0) + return entry; + } + if (entry->key == dummy && freeslot == NULL) + freeslot = entry; } - if (entry->key == dummy && freeslot == NULL) - freeslot = entry; - entry = &table[j ^ 2]; - if (entry->key == NULL) - break; - if (entry->key == key) - return entry; - if (entry->hash == hash && entry->key != dummy) { - PyObject *startkey = entry->key; - Py_INCREF(startkey); - cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); - Py_DECREF(startkey); - if (cmp < 0) - return NULL; - if (table != so->table || entry->key != startkey) - return set_lookkey(so, key, hash); - if (cmp > 0) - return entry; - } - if (entry->key == dummy && freeslot == NULL) - freeslot = entry; + perturb >>= PERTURB_SHIFT; + i = i * 5 + perturb + 1; - i = i * 5 + perturb + 1; - j = i & mask; - perturb >>= PERTURB_SHIFT; - - entry = &table[j]; + entry = &table[i & mask]; if (entry->key == NULL) break; } + found_null: return freeslot == NULL ? entry : freeslot; } @@ -173,10 +158,11 @@ setentry *table = so->table; setentry *freeslot = NULL; setentry *entry; + setentry *limit; size_t perturb = hash; size_t mask = so->mask; - size_t i = (size_t)hash & mask; - size_t j = i; + size_t i = (size_t)hash; + size_t j; /* Make sure this function doesn't have to handle non-unicode keys, including subclasses of str; e.g., one reason to subclass @@ -187,7 +173,7 @@ return set_lookkey(so, key, hash); } - entry = &table[i]; + entry = &table[i & mask]; if (entry->key == NULL) return entry; @@ -200,36 +186,28 @@ if (entry->key == dummy && freeslot == NULL) freeslot = entry; - entry = &table[j ^ 1]; - if (entry->key == NULL) - break; - if (entry->key == key - || (entry->hash == hash - && entry->key != dummy - && unicode_eq(entry->key, key))) - return entry; - if (entry->key == dummy && freeslot == NULL) - freeslot = entry; + limit = &table[mask]; + for (j = 0 ; j < LINEAR_PROBES ; j++) { + entry = (entry == limit) ? &table[0] : entry + 1; + if (entry->key == NULL) + goto found_null; + if (entry->key == key + || (entry->hash == hash + && entry->key != dummy + && unicode_eq(entry->key, key))) + return entry; + if (entry->key == dummy && freeslot == NULL) + freeslot = entry; + } - entry = &table[j ^ 2]; - if (entry->key == NULL) - break; - if (entry->key == key - || (entry->hash == hash - && entry->key != dummy - && unicode_eq(entry->key, key))) - return entry; - if (entry->key == dummy && freeslot == NULL) - freeslot = entry; + perturb >>= PERTURB_SHIFT; + i = i * 5 + perturb + 1; - i = i * 5 + perturb + 1; - j = i & mask; - perturb >>= PERTURB_SHIFT; - - entry = &table[j]; + entry = &table[i & mask]; if (entry->key == NULL) break; } + found_null: return freeslot == NULL ? entry : freeslot; } @@ -280,23 +258,22 @@ setentry *entry; size_t perturb = hash; size_t mask = (size_t)so->mask; - size_t i, j; + size_t i = (size_t)hash; + size_t j; - i = j = (size_t)hash & mask; while (1) { - entry = &table[j]; + entry = &table[i & mask]; if (entry->key == NULL) - break; - entry = &table[j ^ 1]; - if (entry->key == NULL) - break; - entry = &table[j ^ 2]; - if (entry->key == NULL) - break; + goto found_null; + for (j = 1 ; j <= LINEAR_PROBES ; j++) { + entry = &table[(i + j) & mask]; + if (entry->key == NULL) + goto found_null; + } + perturb >>= PERTURB_SHIFT; i = i * 5 + perturb + 1; - j = i & mask; - perturb >>= PERTURB_SHIFT; } + found_null: so->fill++; entry->key = key; entry->hash = hash; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 2 17:57:49 2013 From: python-checkins at python.org (eli.bendersky) Date: Mon, 2 Sep 2013 17:57:49 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Refactor_the_main_function?= =?utf-8?q?_of_regrtest_a_bit=2E?= Message-ID: <3cTGCd5M1rz7LjN@mail.python.org> http://hg.python.org/cpython/rev/5c7697f28ee4 changeset: 85501:5c7697f28ee4 user: Eli Bendersky date: Mon Sep 02 08:57:21 2013 -0700 summary: Refactor the main function of regrtest a bit. Moving subprocess execution of tests into a function. files: Lib/test/regrtest.py | 56 ++++++++++++++++++------------- 1 files changed, 33 insertions(+), 23 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -426,6 +426,38 @@ return ns +def run_test_in_subprocess(testname, ns): + """Run the given test in a subprocess with --slaveargs. + + ns is the option Namespace parsed from command-line arguments. regrtest + is invoked in a subprocess with the --slaveargs argument; when the + subprocess exits, its return code, stdout and stderr are returned as a + 3-tuple. + """ + from subprocess import Popen, PIPE + base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + + ['-X', 'faulthandler', '-m', 'test.regrtest']) + + slaveargs = ( + (testname, ns.verbose, ns.quiet), + dict(huntrleaks=ns.huntrleaks, + use_resources=ns.use_resources, + debug=ns.debug, output_on_failure=ns.verbose3, + timeout=ns.timeout, failfast=ns.failfast, + match_tests=ns.match_tests)) + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], + stdout=PIPE, stderr=PIPE, + universal_newlines=True, + close_fds=(os.name != 'nt'), + cwd=support.SAVEDCWD) + stdout, stderr = popen.communicate() + retcode = popen.wait() + return retcode, stdout, stderr + + def main(tests=None, **kwargs): """Execute a test suite. @@ -648,13 +680,9 @@ print("Multiprocess option requires thread support") sys.exit(2) from queue import Queue - from subprocess import Popen, PIPE debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") output = Queue() pending = MultiprocessTests(tests) - opt_args = support.args_from_interpreter_flags() - base_cmd = [sys.executable] + opt_args - base_cmd += ['-X', 'faulthandler', '-m', 'test.regrtest'] def work(): # A worker thread. try: @@ -664,25 +692,7 @@ except StopIteration: output.put((None, None, None, None)) return - args_tuple = ( - (test, ns.verbose, ns.quiet), - dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - debug=ns.debug, output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests) - ) - # -E is needed by some tests, e.g. test_import - # Running the child from the same working directory ensures - # that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(args_tuple)], - stdout=PIPE, stderr=PIPE, - universal_newlines=True, - close_fds=(os.name != 'nt'), - cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() + retcode, stdout, stderr = run_test_in_subprocess(test, ns) # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 3 00:36:12 2013 From: python-checkins at python.org (victor.stinner) Date: Tue, 3 Sep 2013 00:36:12 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454=3A_Add_a_new_tracemal?= =?utf-8?q?loc_module_to_trace_Python_memory_allocations?= Message-ID: <3cTR3J2C6Pz7Ljm@mail.python.org> http://hg.python.org/peps/rev/d864f728f5da changeset: 5089:d864f728f5da user: Victor Stinner date: Tue Sep 03 00:35:54 2013 +0200 summary: PEP 454: Add a new tracemalloc module to trace Python memory allocations files: pep-0454.txt | 257 +++++++++++++++++++++++++++++++++++++++ 1 files changed, 257 insertions(+), 0 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt new file mode 100644 --- /dev/null +++ b/pep-0454.txt @@ -0,0 +1,257 @@ +PEP: 454 +Title: Add a new tracemalloc module to trace Python memory allocations +Version: $Revision$ +Last-Modified: $Date$ +Author: Victor Stinner +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 3-September-2013 +Python-Version: 3.4 + + +Abstract +======== + +Add a new tracemalloc module to trace Python memory allocations. + + + +Rationale +========= + +This PEP proposes to a new ``tracemalloc`` module, a debug tool to trace +memory allocations made by Python. The module provides the following +information: + +* Statistics on allocations per Python line number (file and line): + size, number, and average size of allocations +* Compute delta between two "snapshots" +* Location of a memory allocation: Python filename and line number + +To trace the most Python memory allocations, the module should be +enabled as early as possible in your application by calling +``tracemalloc.enable()`` function, by setting the ``PYTHONTRACEMALLOC`` +environment variable to ``1``, or by using ``-X tracemalloc`` command +line option. + + +API +=== + +Functions +--------- + +enable(): + + Start tracing Python memory allocations. + +disable(): + + Stop tracing Python memory allocations and stop the timer started by + ``start_timer()``. + +is_enabled(): + + ``True`` if the module is enabled, ``False`` otherwise. + +get_object_trace(obj): + + Get the trace of a Python object *obj* as a namedtuple with 3 attributes: + + - ``size``: size in bytes of the object + - ``filename``: name of the Python script where the object was allocated + - ``lineno``: line number where the object was allocated + + Return ``None`` if tracemalloc did not save the location where the object + was allocated, for example if tracemalloc was disabled. + +get_process_memory(): + + Get the memory usage of the current process as a meminfo namedtuple with + two attributes: + + * ``rss``: Resident Set Size in bytes + * ``vms``: size of the virtual memory in bytes + + Return ``None`` if the platform is not supported. + + Use the ``psutil`` module if available. + +start_timer(delay: int, func: callable, args: tuple=(), kwargs: dict={}): + + Start a timer calling ``func(*args, **kwargs)`` every *delay* + seconds. + + The timer is based on the Python memory allocator, it is not real + time. *func* is called after at least *delay* seconds, it is not + called exactly after *delay* seconds if no Python memory allocation + occurred. + + If ``start_timer()`` is called twice, previous parameters are + replaced. The timer has a resolution of 1 second. + + `start_timer()`` is used by ``DisplayTop`` and ``TakeSnapshot`` + to run regulary a task. + +stop_timer(): + + Stop the timer started by ``start_timer()``. + + +DisplayTop +---------- + +DisplayTop(count: int, file=sys.stdout) class: + + Display the list of the N biggest memory allocations. + +display(): + + Display the top + +start(delay: int): + + Start a task using tracemalloc timer to display the top every delay seconds. + +stop(): + + Stop the task started by the ``DisplayTop.start()`` method + +color attribute: + + Use color if True (bool, default: stream.isatty()). + +compare_with_previous attribute: + + If ``True``, compare with the previous top if ``True``. If ``False``, + compare with the first one (bool, default: ``True``): . + +filename_parts attribute: + + Number of displayed filename parts (int, default: ``3``). + +show_average attribute: + + If ``True``, show the average size of allocations (bool, default: ``True``). + +show_count attribute: + + If ``True``, show the number of allocations (bool, default: ``True``). + +show_lineno attribute: + + If ``True``, use also the line number, not only the filename (bool, default: + ``True``). If ``False``, only show the filename. + +show_size attribute: + + if ``True``, show the size of allocations (bool, default: ``True``). + +user_data_callback attribute: + + Optional callback collecting user data (callable, default: + ``None``). See ``Snapshot.create()``. + + +Snapshot +-------- + +Snapshot() class: + + Snapshot of Python memory allocations. Use ``TakeSnapshot`` to regulary + take snapshots. + +create(user_data_callback=None) + + Take a snapshot. If user_data_callback is specified, it must be a callable + object returning a list of (title: str, format: str, value: int). format + must be "size". The list must always have the same length and the same order + to be able to compute differences between values. + + Example: ``[('Video memory', 'size', 234902)]``. + +filter_filenames(patterns: str|list, include: bool) + + Remove filenames not matching any pattern if include is True, or remove + filenames matching a pattern if include is False (exclude). See + fnmatch.fnmatch() for the syntax of patterns. + +write(filename) + + Write the snapshot into a file. + +pid attribute: + + Identifier of the process which created the snapshot (int). + +stats attribute: + + Raw memory allocation statistics (dict). + +timestamp attribute: + + Date and time of the creation of the snapshot (str). + + +TakeSnapshot +------------ + +TakeSnapshot class: + + Task taking snapshots of Python memory allocations: write them into files. + +start(delay: int) + + Start a task taking a snapshot every delay seconds. + +stop(): + + Stop the task started by the ``TakeSnapshot.start()`` method. + +take_snapshot(): + + Take a snapshot. + +filename_template attribute: + + Template (str) to create a filename. "Variables" can be used in the + template: + + * ``$pid``: identifier of the current process + * ``$timestamp``: current date and time + * ``$counter``: counter starting at 1 and incremented at each snapshot + +user_data_callback attribute: + + Optional callback collecting user data (callable, default: None). + See ``Snapshot.create()``. + + +Links +===== + +* `Python issue #18874: Add a new tracemalloc module to trace Python + memory allocations `_ + +Similar projects: + +* `Meliae: Python Memory Usage Analyzer + `_ +* `Issue #3329: API for setting the memory allocator used by Python + `_ +* `Guppy-PE: umbrella package combining Heapy and GSL + `_ +* `PySizer `_: developed for Python 2.4 +* `memory_profiler `_ +* `pympler `_ +* `Dozer `_: WSGI Middleware version of + the CherryPy memory leak debugger +* `objgraph `_ +* `caulk `_ + +Copyright +========= + +This document has been placed into the public domain. + -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Tue Sep 3 00:41:32 2013 From: python-checkins at python.org (victor.stinner) Date: Tue, 3 Sep 2013 00:41:32 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454=3A_add_Snapshot=2Eloa?= =?utf-8?q?d=28=29_method?= Message-ID: <3cTR9S3m96z7LjY@mail.python.org> http://hg.python.org/peps/rev/8575d152ab03 changeset: 5090:8575d152ab03 user: Victor Stinner date: Tue Sep 03 00:41:20 2013 +0200 summary: PEP 454: add Snapshot.load() method files: pep-0454.txt | 38 ++++++++++++++++++++++---------------- 1 files changed, 22 insertions(+), 16 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -99,22 +99,22 @@ Stop the timer started by ``start_timer()``. -DisplayTop ----------- +DisplayTop class +---------------- DisplayTop(count: int, file=sys.stdout) class: Display the list of the N biggest memory allocations. -display(): +display() method: Display the top -start(delay: int): +start(delay: int) method: Start a task using tracemalloc timer to display the top every delay seconds. -stop(): +stop() method: Stop the task started by the ``DisplayTop.start()`` method @@ -154,15 +154,15 @@ ``None``). See ``Snapshot.create()``. -Snapshot --------- +Snapshot class +-------------- Snapshot() class: Snapshot of Python memory allocations. Use ``TakeSnapshot`` to regulary take snapshots. -create(user_data_callback=None) +create(user_data_callback=None) method: Take a snapshot. If user_data_callback is specified, it must be a callable object returning a list of (title: str, format: str, value: int). format @@ -171,16 +171,20 @@ Example: ``[('Video memory', 'size', 234902)]``. -filter_filenames(patterns: str|list, include: bool) +filter_filenames(patterns: str|list, include: bool) method: Remove filenames not matching any pattern if include is True, or remove filenames matching a pattern if include is False (exclude). See fnmatch.fnmatch() for the syntax of patterns. -write(filename) +write(filename) method: Write the snapshot into a file. +load(filename), classmethod: + + Load a snapshot from a file. + pid attribute: Identifier of the process which created the snapshot (int). @@ -194,22 +198,22 @@ Date and time of the creation of the snapshot (str). -TakeSnapshot ------------- +TakeSnapshot class +------------------ TakeSnapshot class: Task taking snapshots of Python memory allocations: write them into files. -start(delay: int) +start(delay: int) method: Start a task taking a snapshot every delay seconds. -stop(): +stop() method: Stop the task started by the ``TakeSnapshot.start()`` method. -take_snapshot(): +take_snapshot() method: Take a snapshot. @@ -231,7 +235,9 @@ Links ===== -* `Python issue #18874: Add a new tracemalloc module to trace Python +Python issues: + +* `#18874: Add a new tracemalloc module to trace Python memory allocations `_ Similar projects: -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Tue Sep 3 00:53:26 2013 From: python-checkins at python.org (victor.stinner) Date: Tue, 3 Sep 2013 00:53:26 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_EP_454=3A_reformat?= Message-ID: <3cTRRB179TzPkT@mail.python.org> http://hg.python.org/peps/rev/4587a24fba7b changeset: 5091:4587a24fba7b user: Victor Stinner date: Tue Sep 03 00:53:13 2013 +0200 summary: EP 454: reformat files: pep-0454.txt | 215 +++++++++++++++++++++----------------- 1 files changed, 118 insertions(+), 97 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -13,7 +13,7 @@ Abstract ======== -Add a new tracemalloc module to trace Python memory allocations. +Add a new ``tracemalloc`` module to trace Python memory allocations. @@ -29,6 +29,14 @@ * Compute delta between two "snapshots" * Location of a memory allocation: Python filename and line number +The ``tracemalloc`` module is different than other third-party modules +like ``Heapy`` or ``PySizer``, because it is focused on the location of +a memory allocation rather that the object type or object content. + + +API +=== + To trace the most Python memory allocations, the module should be enabled as early as possible in your application by calling ``tracemalloc.enable()`` function, by setting the ``PYTHONTRACEMALLOC`` @@ -36,200 +44,213 @@ line option. -API -=== - Functions --------- enable(): - Start tracing Python memory allocations. + Start tracing Python memory allocations. disable(): - Stop tracing Python memory allocations and stop the timer started by - ``start_timer()``. + Stop tracing Python memory allocations and stop the timer started by + ``start_timer()``. is_enabled(): - ``True`` if the module is enabled, ``False`` otherwise. + Get the status of the module: ``True`` if it is enabled, ``False`` + otherwise. get_object_trace(obj): - Get the trace of a Python object *obj* as a namedtuple with 3 attributes: + Get the trace of a Python object *obj* as a namedtuple with 3 attributes: - - ``size``: size in bytes of the object - - ``filename``: name of the Python script where the object was allocated - - ``lineno``: line number where the object was allocated + - ``size``: size in bytes of the object + - ``filename``: name of the Python script where the object was allocated + - ``lineno``: line number where the object was allocated - Return ``None`` if tracemalloc did not save the location where the object - was allocated, for example if tracemalloc was disabled. + Return ``None`` if tracemalloc did not save the location where the object + was allocated, for example if tracemalloc was disabled. get_process_memory(): - Get the memory usage of the current process as a meminfo namedtuple with - two attributes: + Get the memory usage of the current process as a meminfo namedtuple + with two attributes: - * ``rss``: Resident Set Size in bytes - * ``vms``: size of the virtual memory in bytes + * ``rss``: Resident Set Size in bytes + * ``vms``: size of the virtual memory in bytes - Return ``None`` if the platform is not supported. + Return ``None`` if the platform is not supported. - Use the ``psutil`` module if available. + Use the ``psutil`` module if available. start_timer(delay: int, func: callable, args: tuple=(), kwargs: dict={}): - Start a timer calling ``func(*args, **kwargs)`` every *delay* - seconds. + Start a timer calling ``func(*args, **kwargs)`` every *delay* + seconds. - The timer is based on the Python memory allocator, it is not real - time. *func* is called after at least *delay* seconds, it is not - called exactly after *delay* seconds if no Python memory allocation - occurred. + The timer is based on the Python memory allocator, it is not real + time. *func* is called after at least *delay* seconds, it is not + called exactly after *delay* seconds if no Python memory allocation + occurred. - If ``start_timer()`` is called twice, previous parameters are - replaced. The timer has a resolution of 1 second. + If ``start_timer()`` is called twice, previous parameters are + replaced. The timer has a resolution of 1 second. - `start_timer()`` is used by ``DisplayTop`` and ``TakeSnapshot`` - to run regulary a task. + `start_timer()`` is used by ``DisplayTop`` and ``TakeSnapshot`` to + run regulary a task. stop_timer(): - Stop the timer started by ``start_timer()``. + Stop the timer started by ``start_timer()``. DisplayTop class ---------------- -DisplayTop(count: int, file=sys.stdout) class: +``DisplayTop(count: int, file=sys.stdout)`` class: - Display the list of the N biggest memory allocations. + Display the list of the N biggest memory allocations. -display() method: +``display()`` method: - Display the top + Display the top -start(delay: int) method: +``start(delay: int)`` method: - Start a task using tracemalloc timer to display the top every delay seconds. + Start a task using tracemalloc timer to display the top every delay seconds. -stop() method: +``stop()`` method: - Stop the task started by the ``DisplayTop.start()`` method + Stop the task started by the ``DisplayTop.start()`` method -color attribute: +``color`` attribute: - Use color if True (bool, default: stream.isatty()). + ``display()`` uses colors if ``True`` (bool, + default: ``stream.isatty()``). -compare_with_previous attribute: +``compare_with_previous`` attribute: - If ``True``, compare with the previous top if ``True``. If ``False``, - compare with the first one (bool, default: ``True``): . + If ``True``, ``display()`` compares with the previous top if + ``True``. If ``False``, compare with the first one (bool, default: + ``True``): . -filename_parts attribute: +``filename_parts`` attribute: Number of displayed filename parts (int, default: ``3``). -show_average attribute: +``show_average`` attribute: - If ``True``, show the average size of allocations (bool, default: ``True``). + If ``True``, ``display()`` shows the average size of allocations + (bool, default: ``True``). -show_count attribute: +``show_count`` attribute: - If ``True``, show the number of allocations (bool, default: ``True``). + If ``True``, ``display()`` shows the number of allocations (bool, default: ``True``). -show_lineno attribute: +``show_lineno`` attribute: - If ``True``, use also the line number, not only the filename (bool, default: - ``True``). If ``False``, only show the filename. + If ``True``, use also the line number, not only the filename (bool, + default: ``True``). If ``False``, only show the filename. -show_size attribute: +``show_size`` attribute: - if ``True``, show the size of allocations (bool, default: ``True``). + If ``True``, ``display()`` shows the size of allocations (bool, + default: ``True``). -user_data_callback attribute: +``user_data_callback`` attribute: - Optional callback collecting user data (callable, default: - ``None``). See ``Snapshot.create()``. + Optional callback collecting user data (callable, default: + ``None``). See ``Snapshot.create()``. Snapshot class -------------- -Snapshot() class: +``Snapshot()`` class: - Snapshot of Python memory allocations. Use ``TakeSnapshot`` to regulary - take snapshots. + Snapshot of Python memory allocations. Use ``TakeSnapshot`` to + regulary take snapshots. -create(user_data_callback=None) method: +``create(user_data_callback=None)`` method: - Take a snapshot. If user_data_callback is specified, it must be a callable - object returning a list of (title: str, format: str, value: int). format - must be "size". The list must always have the same length and the same order - to be able to compute differences between values. + Take a snapshot. If user_data_callback is specified, it must be a + callable object returning a list of (title: str, format: str, value: + int). format must be "size". The list must always have the same + length and the same order to be able to compute differences between + values. - Example: ``[('Video memory', 'size', 234902)]``. + Example: ``[('Video memory', 'size', 234902)]``. -filter_filenames(patterns: str|list, include: bool) method: +``filter_filenames(patterns: str|list, include: bool)`` method: - Remove filenames not matching any pattern if include is True, or remove - filenames matching a pattern if include is False (exclude). See - fnmatch.fnmatch() for the syntax of patterns. + Remove filenames not matching any pattern if include is True, or + remove filenames matching a pattern if include is False (exclude). + See fnmatch.fnmatch() for the syntax of patterns. -write(filename) method: +``write(filename)`` method: - Write the snapshot into a file. + Write the snapshot into a file. -load(filename), classmethod: +``load(filename)`` classmethod: - Load a snapshot from a file. + Load a snapshot from a file. -pid attribute: +``process_memory`` attribute: - Identifier of the process which created the snapshot (int). + Memory usage of the process, result of ``get_process_memory()``. + It can be ``None``. -stats attribute: +``user_data`` attribute: - Raw memory allocation statistics (dict). + Optional list of user data, result of *user_data_callback* in + ``Snapshot.create()`` (default: None). -timestamp attribute: +``pid`` attribute: - Date and time of the creation of the snapshot (str). + Identifier of the process which created the snapshot (int). + +``stats`` attribute: + + Raw memory allocation statistics (dict). + +``timestamp`` attribute: + + Date and time of the creation of the snapshot (str). TakeSnapshot class ------------------ -TakeSnapshot class: +``TakeSnapshot`` class: - Task taking snapshots of Python memory allocations: write them into files. + Task taking snapshots of Python memory allocations: write them into files. -start(delay: int) method: +``start(delay: int)`` method: - Start a task taking a snapshot every delay seconds. + Start a task taking a snapshot every delay seconds. -stop() method: +``stop()`` method: - Stop the task started by the ``TakeSnapshot.start()`` method. + Stop the task started by the ``TakeSnapshot.start()`` method. -take_snapshot() method: +``take_snapshot()`` method: - Take a snapshot. + Take a snapshot. -filename_template attribute: +``filename_template`` attribute: - Template (str) to create a filename. "Variables" can be used in the - template: + Template (str) to create a filename. "Variables" can be used in the + template: - * ``$pid``: identifier of the current process - * ``$timestamp``: current date and time - * ``$counter``: counter starting at 1 and incremented at each snapshot + * ``$pid``: identifier of the current process + * ``$timestamp``: current date and time + * ``$counter``: counter starting at 1 and incremented at each snapshot -user_data_callback attribute: +``user_data_callback`` attribute: - Optional callback collecting user data (callable, default: None). - See ``Snapshot.create()``. + Optional callback collecting user data (callable, default: None). + See ``Snapshot.create()``. Links -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Tue Sep 3 00:59:34 2013 From: python-checkins at python.org (raymond.hettinger) Date: Tue, 3 Sep 2013 00:59:34 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Factor-out_the_common_code?= =?utf-8?q?_for_setting_a_KeyError=2E?= Message-ID: <3cTRZG6RF3z7LjN@mail.python.org> http://hg.python.org/cpython/rev/bfaf3a2907bd changeset: 85502:bfaf3a2907bd user: Raymond Hettinger date: Mon Sep 02 15:59:26 2013 -0700 summary: Factor-out the common code for setting a KeyError. files: Include/pyerrors.h | 1 + Objects/dictobject.c | 22 ++++------------------ Objects/setobject.c | 16 +--------------- Python/errors.c | 14 ++++++++++++++ 4 files changed, 20 insertions(+), 33 deletions(-) diff --git a/Include/pyerrors.h b/Include/pyerrors.h --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -75,6 +75,7 @@ PyAPI_FUNC(void) PyErr_SetNone(PyObject *); PyAPI_FUNC(void) PyErr_SetObject(PyObject *, PyObject *); +PyAPI_FUNC(void) _PyErr_SetKeyError(PyObject *); PyAPI_FUNC(void) PyErr_SetString( PyObject *exception, const char *string /* decoded from utf-8 */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -95,20 +95,6 @@ it's USABLE_FRACTION (currently two-thirds) full. */ -/* Set a key error with the specified argument, wrapping it in a - * tuple automatically so that tuple keys are not unpacked as the - * exception arguments. */ -static void -set_key_error(PyObject *arg) -{ - PyObject *tup; - tup = PyTuple_Pack(1, arg); - if (!tup) - return; /* caller will expect error to be set anyway */ - PyErr_SetObject(PyExc_KeyError, tup); - Py_DECREF(tup); -} - #define PERTURB_SHIFT 5 /* @@ -1241,7 +1227,7 @@ if (ep == NULL) return -1; if (*value_addr == NULL) { - set_key_error(key); + _PyErr_SetKeyError(key); return -1; } old_value = *value_addr; @@ -1530,7 +1516,7 @@ else if (PyErr_Occurred()) return NULL; } - set_key_error(key); + _PyErr_SetKeyError(key); return NULL; } else @@ -2302,7 +2288,7 @@ Py_INCREF(deflt); return deflt; } - set_key_error(key); + _PyErr_SetKeyError(key); return NULL; } if (!PyUnicode_CheckExact(key) || @@ -2320,7 +2306,7 @@ Py_INCREF(deflt); return deflt; } - set_key_error(key); + _PyErr_SetKeyError(key); return NULL; } *value_addr = NULL; diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -11,20 +11,6 @@ #include "structmember.h" #include "stringlib/eq.h" -/* Set a key error with the specified argument, wrapping it in a - * tuple automatically so that tuple keys are not unpacked as the - * exception arguments. */ -static void -set_key_error(PyObject *arg) -{ - PyObject *tup; - tup = PyTuple_Pack(1, arg); - if (!tup) - return; /* caller will expect error to be set anyway */ - PyErr_SetObject(PyExc_KeyError, tup); - Py_DECREF(tup); -} - /* This must be >= 1. */ #define PERTURB_SHIFT 5 #define LINEAR_PROBES 9 @@ -1948,7 +1934,7 @@ } if (rv == DISCARD_NOTFOUND) { - set_key_error(key); + _PyErr_SetKeyError(key); return NULL; } Py_RETURN_NONE; diff --git a/Python/errors.c b/Python/errors.c --- a/Python/errors.c +++ b/Python/errors.c @@ -117,6 +117,20 @@ PyErr_Restore(exception, value, tb); } +/* Set a key error with the specified argument, wrapping it in a + * tuple automatically so that tuple keys are not unpacked as the + * exception arguments. */ +void +_PyErr_SetKeyError(PyObject *arg) +{ + PyObject *tup; + tup = PyTuple_Pack(1, arg); + if (!tup) + return; /* caller will expect error to be set anyway */ + PyErr_SetObject(PyExc_KeyError, tup); + Py_DECREF(tup); +} + void PyErr_SetNone(PyObject *exception) { -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 3 01:32:34 2013 From: python-checkins at python.org (raymond.hettinger) Date: Tue, 3 Sep 2013 01:32:34 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Minor_touchups=2E?= Message-ID: <3cTSJL30L0z7LjP@mail.python.org> http://hg.python.org/cpython/rev/035c062af722 changeset: 85503:035c062af722 user: Raymond Hettinger date: Mon Sep 02 16:32:27 2013 -0700 summary: Minor touchups. files: Objects/setobject.c | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -11,8 +11,10 @@ #include "structmember.h" #include "stringlib/eq.h" -/* This must be >= 1. */ +/* This must be >= 1 */ #define PERTURB_SHIFT 5 + +/* This should be >= PySet_MINSIZE - 1 */ #define LINEAR_PROBES 9 /* Object used as dummy key to fill deleted entries */ @@ -123,7 +125,7 @@ } perturb >>= PERTURB_SHIFT; - i = i * 5 + perturb + 1; + i = i * 5 + 1 + perturb; entry = &table[i & mask]; if (entry->key == NULL) @@ -187,7 +189,7 @@ } perturb >>= PERTURB_SHIFT; - i = i * 5 + perturb + 1; + i = i * 5 + 1 + perturb; entry = &table[i & mask]; if (entry->key == NULL) @@ -257,7 +259,7 @@ goto found_null; } perturb >>= PERTURB_SHIFT; - i = i * 5 + perturb + 1; + i = i * 5 + 1 + perturb; } found_null: so->fill++; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 3 01:52:55 2013 From: python-checkins at python.org (eli.bendersky) Date: Tue, 3 Sep 2013 01:52:55 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Fix_docstring_and_some_ind?= =?utf-8?q?entation?= Message-ID: <3cTSlq1Jv8z7LjY@mail.python.org> http://hg.python.org/cpython/rev/a3a5411bf9bf changeset: 85504:a3a5411bf9bf user: Eli Bendersky date: Mon Sep 02 16:52:25 2013 -0700 summary: Fix docstring and some indentation files: Lib/test/regrtest.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -918,14 +918,15 @@ test -- the name of the test verbose -- if true, print more messages quiet -- if true, don't print 'skipped' messages (probably redundant) - test_times -- a list of (time, test_name) pairs huntrleaks -- run multiple times to test for leaks; requires a debug build; a triple corresponding to -R's three arguments + use_resources -- list of extra resources to use output_on_failure -- if true, display test output on failure timeout -- dump the traceback and exit if a test takes more than timeout seconds + failfast, match_tests -- See regrtest command-line flags for these. - Returns one of the test result constants: + Returns the tuple result, test_time, where result is one of the constants: INTERRUPTED KeyboardInterrupt when run under -j RESOURCE_DENIED test skipped because resource denied SKIPPED test skipped for some other reason @@ -1276,8 +1277,7 @@ test_runner = lambda: support.run_unittest(tests) test_runner() if huntrleaks: - refleak = dash_R(the_module, test, test_runner, - huntrleaks) + refleak = dash_R(the_module, test, test_runner, huntrleaks) test_time = time.time() - start_time except support.ResourceDenied as msg: if not quiet: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 3 02:01:40 2013 From: python-checkins at python.org (eli.bendersky) Date: Tue, 3 Sep 2013 02:01:40 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Remove_unused_--debug_opti?= =?utf-8?q?on_of_regrtest=2E?= Message-ID: <3cTSxw0NNQz7LjN@mail.python.org> http://hg.python.org/cpython/rev/aeb3faaf4754 changeset: 85505:aeb3faaf4754 user: Eli Bendersky date: Mon Sep 02 17:01:10 2013 -0700 summary: Remove unused --debug option of regrtest. If bots fail due to using this flag, the buildbot scripts have to be modified to omit it. Regrtest ignores it anyway. files: Lib/test/regrtest.py | 16 +++++++--------- Lib/test/test_regrtest.py | 6 ------ 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -252,8 +252,6 @@ help='re-run failed tests in verbose mode') group.add_argument('-W', '--verbose3', action='store_true', help='display test output on failure') - group.add_argument('-d', '--debug', action='store_true', - help='print traceback for failed tests') group.add_argument('-q', '--quiet', action='store_true', help='no output unless one or more tests fail') group.add_argument('-o', '--slow', action='store_true', dest='print_slow', @@ -442,7 +440,7 @@ (testname, ns.verbose, ns.quiet), dict(huntrleaks=ns.huntrleaks, use_resources=ns.use_resources, - debug=ns.debug, output_on_failure=ns.verbose3, + output_on_failure=ns.verbose3, timeout=ns.timeout, failfast=ns.failfast, match_tests=ns.match_tests)) # Running the child from the same working directory as regrtest's original @@ -757,7 +755,7 @@ else: try: result = runtest(test, ns.verbose, ns.quiet, - ns.huntrleaks, ns.debug, + ns.huntrleaks, output_on_failure=ns.verbose3, timeout=ns.timeout, failfast=ns.failfast, match_tests=ns.match_tests) @@ -817,7 +815,7 @@ sys.stdout.flush() try: ns.verbose = True - ok = runtest(test, True, ns.quiet, ns.huntrleaks, ns.debug, + ok = runtest(test, True, ns.quiet, ns.huntrleaks, timeout=ns.timeout) except KeyboardInterrupt: # print a newline separate from the ^C @@ -910,7 +908,7 @@ atexit.register(restore_stdout) def runtest(test, verbose, quiet, - huntrleaks=False, debug=False, use_resources=None, + huntrleaks=False, use_resources=None, output_on_failure=False, failfast=False, match_tests=None, timeout=None): """Run a single test. @@ -964,7 +962,7 @@ sys.stdout = stream sys.stderr = stream result = runtest_inner(test, verbose, quiet, huntrleaks, - debug, display_failure=False) + display_failure=False) if result[0] == FAILED: output = stream.getvalue() orig_stderr.write(output) @@ -974,7 +972,7 @@ sys.stderr = orig_stderr else: support.verbose = verbose # Tell tests to be moderately quiet - result = runtest_inner(test, verbose, quiet, huntrleaks, debug, + result = runtest_inner(test, verbose, quiet, huntrleaks, display_failure=not verbose) return result finally: @@ -1255,7 +1253,7 @@ def runtest_inner(test, verbose, quiet, - huntrleaks=False, debug=False, display_failure=True): + huntrleaks=False, display_failure=True): support.unload(test) test_time = 0.0 diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -71,12 +71,6 @@ ns = regrtest._parse_args([opt]) self.assertTrue(ns.verbose3) - def test_debug(self): - for opt in '-d', '--debug': - with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) - self.assertTrue(ns.debug) - def test_quiet(self): for opt in '-q', '--quiet': with self.subTest(opt=opt): -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Tue Sep 3 06:22:06 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Tue, 03 Sep 2013 06:22:06 +0200 Subject: [Python-checkins] Daily reference leaks (aeb3faaf4754): sum=4 Message-ID: results for aeb3faaf4754 on branch "default" -------------------------------------------- test_support leaked [-1, 1, 0] references, sum=0 test_support leaked [-1, 3, 2] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogfyuk80', '-x'] From python-checkins at python.org Tue Sep 3 13:18:46 2013 From: python-checkins at python.org (victor.stinner) Date: Tue, 3 Sep 2013 13:18:46 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454?= Message-ID: <3cTlzB6tzpz7Ljc@mail.python.org> http://hg.python.org/peps/rev/9879b34ef73a changeset: 5092:9879b34ef73a user: Victor Stinner date: Tue Sep 03 13:18:48 2013 +0200 summary: PEP 454 files: pep-0454.txt | 95 ++++++++++++++++++++++----------------- 1 files changed, 53 insertions(+), 42 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -20,14 +20,15 @@ Rationale ========= -This PEP proposes to a new ``tracemalloc`` module, a debug tool to trace -memory allocations made by Python. The module provides the following -information: +This PEP proposes to a new ``tracemalloc`` module. It is a debug tool to +trace memory allocations made by Python based on API added by the PEP +445. The module provides the following information: -* Statistics on allocations per Python line number (file and line): - size, number, and average size of allocations -* Compute delta between two "snapshots" -* Location of a memory allocation: Python filename and line number +* Statistics on Python memory allocations per Python filename and line + number: size, number, and average size of allocations +* Compute differences between two snapshots of Python memory allocations +* Location of a Python memory allocation: size in bytes, Python filename + and line number The ``tracemalloc`` module is different than other third-party modules like ``Heapy`` or ``PySizer``, because it is focused on the location of @@ -47,32 +48,35 @@ Functions --------- -enable(): +``enable()`` function: Start tracing Python memory allocations. -disable(): +``disable()`` function: Stop tracing Python memory allocations and stop the timer started by ``start_timer()``. -is_enabled(): +``is_enabled()`` function: Get the status of the module: ``True`` if it is enabled, ``False`` otherwise. -get_object_trace(obj): +``get_object_trace(obj)`` function: - Get the trace of a Python object *obj* as a namedtuple with 3 attributes: + Get the trace of the memory allocation of the Python object *obj*. + Return a namedtuple with 3 attributes if the memory allocation was + traced: - ``size``: size in bytes of the object - ``filename``: name of the Python script where the object was allocated - ``lineno``: line number where the object was allocated - Return ``None`` if tracemalloc did not save the location where the object - was allocated, for example if tracemalloc was disabled. + Return ``None`` if ``tracemalloc`` did not trace the memory + allocation, for example if ``tracemalloc`` was disabled when the + object was created. -get_process_memory(): +``get_process_memory()`` function: Get the memory usage of the current process as a meminfo namedtuple with two attributes: @@ -84,7 +88,7 @@ Use the ``psutil`` module if available. -start_timer(delay: int, func: callable, args: tuple=(), kwargs: dict={}): +``start_timer(delay: int, func: callable, args: tuple=(), kwargs: dict={})`` function: Start a timer calling ``func(*args, **kwargs)`` every *delay* seconds. @@ -97,10 +101,10 @@ If ``start_timer()`` is called twice, previous parameters are replaced. The timer has a resolution of 1 second. - `start_timer()`` is used by ``DisplayTop`` and ``TakeSnapshot`` to + ``start_timer()`` is used by ``DisplayTop`` and ``TakeSnapshot`` to run regulary a task. -stop_timer(): +``stop_timer()`` function: Stop the timer started by ``start_timer()``. @@ -110,15 +114,17 @@ ``DisplayTop(count: int, file=sys.stdout)`` class: - Display the list of the N biggest memory allocations. + Display the list of the *count* biggest memory allocations into + *file*. ``display()`` method: - Display the top + Display the top once. ``start(delay: int)`` method: - Start a task using tracemalloc timer to display the top every delay seconds. + Start a task using ``tracemalloc`` timer to display the top every + *delay* seconds. ``stop()`` method: @@ -132,21 +138,22 @@ ``compare_with_previous`` attribute: If ``True``, ``display()`` compares with the previous top if - ``True``. If ``False``, compare with the first one (bool, default: - ``True``): . + ``True``. If ``False``, compare with the first top (bool, default: + ``True``). ``filename_parts`` attribute: - Number of displayed filename parts (int, default: ``3``). + Number of displayed filename parts (int, default: ``3``). ``show_average`` attribute: - If ``True``, ``display()`` shows the average size of allocations - (bool, default: ``True``). + If ``True``, ``display()`` shows the average size of allocations + (bool, default: ``True``). ``show_count`` attribute: - If ``True``, ``display()`` shows the number of allocations (bool, default: ``True``). + If ``True``, ``display()`` shows the number of allocations (bool, + default: ``True``). ``show_lineno`` attribute: @@ -169,24 +176,27 @@ ``Snapshot()`` class: - Snapshot of Python memory allocations. Use ``TakeSnapshot`` to - regulary take snapshots. + Snapshot of Python memory allocations. + + Use ``TakeSnapshot`` to take regulary snapshots. ``create(user_data_callback=None)`` method: - Take a snapshot. If user_data_callback is specified, it must be a + Take a snapshot. If *user_data_callback* is specified, it must be a callable object returning a list of (title: str, format: str, value: - int). format must be "size". The list must always have the same + int). format must be ``'size'``. The list must always have the same length and the same order to be able to compute differences between values. Example: ``[('Video memory', 'size', 234902)]``. -``filter_filenames(patterns: str|list, include: bool)`` method: +``filter_filenames(patterns: list, include: bool)`` method: - Remove filenames not matching any pattern if include is True, or - remove filenames matching a pattern if include is False (exclude). - See fnmatch.fnmatch() for the syntax of patterns. + Remove filenames not matching any pattern of *patterns* if *include* + is ``True``, or remove filenames matching a pattern of *patterns* if + *include* is ``False`` (exclude). + + See ``fnmatch.fnmatch()`` for the syntax of a pattern. ``write(filename)`` method: @@ -224,7 +234,8 @@ ``TakeSnapshot`` class: - Task taking snapshots of Python memory allocations: write them into files. + Task taking snapshots of Python memory allocations: write them into + files. By default, snapshots are written in the current directory. ``start(delay: int)`` method: @@ -240,17 +251,19 @@ ``filename_template`` attribute: - Template (str) to create a filename. "Variables" can be used in the - template: + Template (``str``) used to create a filename. The following + variables can be used in the template: * ``$pid``: identifier of the current process * ``$timestamp``: current date and time * ``$counter``: counter starting at 1 and incremented at each snapshot + The default pattern is ``'tracemalloc-$counter.pickle'``. + ``user_data_callback`` attribute: - Optional callback collecting user data (callable, default: None). - See ``Snapshot.create()``. + Optional callback collecting user data (callable, default: + ``None``). See ``Snapshot.create()``. Links @@ -265,8 +278,6 @@ * `Meliae: Python Memory Usage Analyzer `_ -* `Issue #3329: API for setting the memory allocator used by Python - `_ * `Guppy-PE: umbrella package combining Heapy and GSL `_ * `PySizer `_: developed for Python 2.4 -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Tue Sep 3 14:47:09 2013 From: python-checkins at python.org (christian.heimes) Date: Tue, 3 Sep 2013 14:47:09 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi42KTogUHl0aG9uIDIuNidz?= =?utf-8?q?_ssl_module_has_neither_OPENSSL=5FVERSION=5FINFO_nor?= Message-ID: <3cTnx91B8fz7Ljs@mail.python.org> http://hg.python.org/cpython/rev/25683ceaf341 changeset: 85506:25683ceaf341 branch: 2.6 parent: 85377:50803d881a92 user: Christian Heimes date: Tue Sep 03 14:47:00 2013 +0200 summary: Python 2.6's ssl module has neither OPENSSL_VERSION_INFO nor _OPENSSL_API_VERSION files: Lib/test/test_ssl.py | 19 +++++-------------- 1 files changed, 5 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -102,21 +102,12 @@ (('emailAddress', 'python-dev at python.org'),)) self.assertEqual(p['subject'], subject) self.assertEqual(p['issuer'], subject) - if ssl._OPENSSL_API_VERSION >= (0, 9, 8): - san = (('DNS', 'altnull.python.org\x00example.com'), - ('email', 'null at python.org\x00user at example.org'), - ('URI', 'http://null.python.org\x00http://example.org'), - ('IP Address', '192.0.2.1'), - ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) - else: - # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName - san = (('DNS', 'altnull.python.org\x00example.com'), - ('email', 'null at python.org\x00user at example.org'), - ('URI', 'http://null.python.org\x00http://example.org'), - ('IP Address', '192.0.2.1'), - ('IP Address', '')) + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null at python.org\x00user at example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1')) - self.assertEqual(p['subjectAltName'], san) + self.assertEqual(p['subjectAltName'][:4], san) def test_DER_to_PEM(self): with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 3 15:39:27 2013 From: python-checkins at python.org (eli.bendersky) Date: Tue, 3 Sep 2013 15:39:27 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4OTEy?= =?utf-8?q?=3A_Fix_indentation_in_docstring?= Message-ID: <3cTq5W517wz7Ljg@mail.python.org> http://hg.python.org/cpython/rev/8e174ee0575a changeset: 85507:8e174ee0575a branch: 3.3 parent: 85496:43f27e69bc29 user: Eli Bendersky date: Tue Sep 03 06:37:19 2013 -0700 summary: Issue #18912: Fix indentation in docstring Contributed by Jeroen Van Goey files: Modules/itertoolsmodule.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -3978,10 +3978,10 @@ Return a count object whose .__next__() method returns consecutive values.\n\ Equivalent to:\n\n\ def count(firstval=0, step=1):\n\ - x = firstval\n\ - while 1:\n\ - yield x\n\ - x += step\n"); + x = firstval\n\ + while 1:\n\ + yield x\n\ + x += step\n"); static PyTypeObject count_type = { PyVarObject_HEAD_INIT(NULL, 0) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 3 15:39:28 2013 From: python-checkins at python.org (eli.bendersky) Date: Tue, 3 Sep 2013 15:39:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2318912=3A_Fix_indentation_in_docstring?= Message-ID: <3cTq5X6tzhz7Lks@mail.python.org> http://hg.python.org/cpython/rev/31ef590a0d2f changeset: 85508:31ef590a0d2f parent: 85505:aeb3faaf4754 parent: 85507:8e174ee0575a user: Eli Bendersky date: Tue Sep 03 06:38:55 2013 -0700 summary: Issue #18912: Fix indentation in docstring Contributed by Jeroen Van Goey files: Misc/ACKS | 1 + Modules/itertoolsmodule.c | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -445,6 +445,7 @@ Matt Giuca Wim Glenn Michael Goderbauer +Jeroen Van Goey Christoph Gohlke Tim Golden Guilherme Gon?alves diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -3978,10 +3978,10 @@ Return a count object whose .__next__() method returns consecutive values.\n\ Equivalent to:\n\n\ def count(firstval=0, step=1):\n\ - x = firstval\n\ - while 1:\n\ - yield x\n\ - x += step\n"); + x = firstval\n\ + while 1:\n\ + yield x\n\ + x += step\n"); static PyTypeObject count_type = { PyVarObject_HEAD_INIT(NULL, 0) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 3 15:42:30 2013 From: python-checkins at python.org (eli.bendersky) Date: Tue, 3 Sep 2013 15:42:30 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogQ2xvc2UgIzE4OTEy?= =?utf-8?q?=3A_Fix_indentation_in_docstring?= Message-ID: <3cTq924BpMz7Lkh@mail.python.org> http://hg.python.org/cpython/rev/a559cda6a498 changeset: 85509:a559cda6a498 branch: 2.7 parent: 85498:869cbcabb934 user: Eli Bendersky date: Tue Sep 03 06:41:58 2013 -0700 summary: Close #18912: Fix indentation in docstring Contributed by Jeroen Van Goey files: Modules/itertoolsmodule.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -3426,10 +3426,10 @@ Return a count object whose .next() method returns consecutive values.\n\ Equivalent to:\n\n\ def count(firstval=0, step=1):\n\ - x = firstval\n\ - while 1:\n\ - yield x\n\ - x += step\n"); + x = firstval\n\ + while 1:\n\ + yield x\n\ + x += step\n"); static PyTypeObject count_type = { PyVarObject_HEAD_INIT(NULL, 0) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 3 18:39:24 2013 From: python-checkins at python.org (tim.peters) Date: Tue, 3 Sep 2013 18:39:24 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMi42IC0+IDIuNyk6?= =?utf-8?q?_Null_merge_of_2=2E6_into_2=2E7=2E?= Message-ID: <3cTv58409Qz7LjY@mail.python.org> http://hg.python.org/cpython/rev/2a38df26e009 changeset: 85510:2a38df26e009 branch: 2.7 parent: 85509:a559cda6a498 parent: 85506:25683ceaf341 user: Tim Peters date: Tue Sep 03 11:39:06 2013 -0500 summary: Null merge of 2.6 into 2.7. Changeset 25683ceaf341 left a new head on 2.6. files: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 3 19:35:50 2013 From: python-checkins at python.org (tim.peters) Date: Tue, 3 Sep 2013 19:35:50 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogY3dyX25leHQoKTog?= =?utf-8?q?_move_invariants_out_of_loops=2E?= Message-ID: <3cTwLG0CSbz7Ljc@mail.python.org> http://hg.python.org/cpython/rev/4bbaa3d7bcbf changeset: 85511:4bbaa3d7bcbf branch: 3.3 parent: 85507:8e174ee0575a user: Tim Peters date: Tue Sep 03 11:49:31 2013 -0500 summary: cwr_next(): move invariants out of loops. This simplifies and clarifies the code, and gives a small speedup. files: Modules/itertoolsmodule.c | 28 +++++++++++--------------- 1 files changed, 12 insertions(+), 16 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -2713,20 +2713,20 @@ PyObject *result = co->result; Py_ssize_t n = PyTuple_GET_SIZE(pool); Py_ssize_t r = co->r; - Py_ssize_t i, j, index; + Py_ssize_t i, index; if (co->stopped) return NULL; if (result == NULL) { - /* On the first pass, initialize result tuple using the indices */ + /* On the first pass, initialize result tuple with pool[0] */ result = PyTuple_New(r); if (result == NULL) goto empty; co->result = result; + elem = PyTuple_GET_ITEM(pool, 0); for (i=0; i= 0 && indices[i] == n-1; i--) ; /* If i is negative, then the indices are all at - their maximum value and we're done. */ + their maximum value and we're done. */ if (i < 0) goto empty; /* Increment the current index which we know is not at its - maximum. Then set all to the right to the same value. */ - indices[i]++; - for (j=i+1 ; j http://hg.python.org/cpython/rev/4fd9407ad112 changeset: 85512:4fd9407ad112 parent: 85508:31ef590a0d2f parent: 85511:4bbaa3d7bcbf user: Tim Peters date: Tue Sep 03 11:52:59 2013 -0500 summary: Merge 3.3 into default. cwr_next(): move invariants out of loops. This simplifies and clarifies the code, and gives a small speedup. files: Modules/itertoolsmodule.c | 28 +++++++++++--------------- 1 files changed, 12 insertions(+), 16 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -2713,20 +2713,20 @@ PyObject *result = co->result; Py_ssize_t n = PyTuple_GET_SIZE(pool); Py_ssize_t r = co->r; - Py_ssize_t i, j, index; + Py_ssize_t i, index; if (co->stopped) return NULL; if (result == NULL) { - /* On the first pass, initialize result tuple using the indices */ + /* On the first pass, initialize result tuple with pool[0] */ result = PyTuple_New(r); if (result == NULL) goto empty; co->result = result; + elem = PyTuple_GET_ITEM(pool, 0); for (i=0; i= 0 && indices[i] == n-1; i--) ; /* If i is negative, then the indices are all at - their maximum value and we're done. */ + their maximum value and we're done. */ if (i < 0) goto empty; /* Increment the current index which we know is not at its - maximum. Then set all to the right to the same value. */ - indices[i]++; - for (j=i+1 ; j http://hg.python.org/cpython/rev/a14ec46de0a4 changeset: 85513:a14ec46de0a4 user: Serhiy Storchaka date: Wed Sep 04 00:28:43 2013 +0300 summary: Issue #17487: The result of the wave getparams method now is pickleable again. Patch by Claudiu Popa. files: Lib/test/test_wave.py | 13 +++++++++++++ Lib/wave.py | 6 +++--- Misc/NEWS | 3 +++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -1,5 +1,6 @@ from test.support import TESTFN, unlink import wave +import pickle import unittest nchannels = 2 @@ -69,6 +70,18 @@ self.assertEqual(params.comptype, self.f.getcomptype()) self.assertEqual(params.compname, self.f.getcompname()) + def test_getparams_picklable(self): + self.f = wave.open(TESTFN, 'wb') + self.f.setnchannels(nchannels) + self.f.setsampwidth(sampwidth) + self.f.setframerate(framerate) + self.f.close() + + self.f = wave.open(TESTFN, 'rb') + params = self.f.getparams() + dump = pickle.dumps(params) + self.assertEqual(pickle.loads(dump), params) + def test_wave_write_context_manager_calls_close(self): # Close checks for a minimum header and will raise an error # if it is not set, so this proves that close is called. diff --git a/Lib/wave.py b/Lib/wave.py --- a/Lib/wave.py +++ b/Lib/wave.py @@ -87,7 +87,7 @@ from chunk import Chunk from collections import namedtuple -_result = namedtuple('params', +_wave_params = namedtuple('_wave_params', 'nchannels sampwidth framerate nframes comptype compname') class Wave_read: @@ -212,7 +212,7 @@ return self._compname def getparams(self): - return _result(self.getnchannels(), self.getsampwidth(), + return _wave_params(self.getnchannels(), self.getsampwidth(), self.getframerate(), self.getnframes(), self.getcomptype(), self.getcompname()) @@ -410,7 +410,7 @@ def getparams(self): if not self._nchannels or not self._sampwidth or not self._framerate: raise Error('not all parameters set') - return _result(self._nchannels, self._sampwidth, self._framerate, + return _wave_params(self._nchannels, self._sampwidth, self._framerate, self._nframes, self._comptype, self._compname) def setmark(self, id, pos, name): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,6 +54,9 @@ Library ------- +- Issue #17487: The result of the wave getparams method now is pickleable again. + Patch by Claudiu Popa. + - Issue #18756: os.urandom() now uses a lazily-opened persistent file descriptor, so as to avoid using many file descriptors when run in parallel from multiple threads. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 3 23:43:40 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Tue, 3 Sep 2013 23:43:40 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318901=3A_The_suna?= =?utf-8?q?u_getparams_method_now_returns_a_namedtuple_rather_than?= Message-ID: <3cV1rD6Ylxz7Lkb@mail.python.org> http://hg.python.org/cpython/rev/99d26f32aed3 changeset: 85514:99d26f32aed3 user: Serhiy Storchaka date: Wed Sep 04 00:43:03 2013 +0300 summary: Issue #18901: The sunau getparams method now returns a namedtuple rather than a plain tuple. Patch by Claudiu Popa. files: Doc/library/sunau.rst | 5 +++-- Doc/whatsnew/3.4.rst | 7 +++++++ Lib/sunau.py | 19 ++++++++++++------- Lib/test/test_sunau.py | 22 ++++++++++++++++++++++ Misc/NEWS | 3 +++ 5 files changed, 47 insertions(+), 9 deletions(-) diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst --- a/Doc/library/sunau.rst +++ b/Doc/library/sunau.rst @@ -150,8 +150,9 @@ .. method:: AU_read.getparams() - Returns a tuple ``(nchannels, sampwidth, framerate, nframes, comptype, - compname)``, equivalent to output of the :meth:`get\*` methods. + Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, + framerate, nframes, comptype, compname)``, equivalent to output of the + :meth:`get\*` methods. .. method:: AU_read.readframes(n) diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -365,6 +365,13 @@ (Contributed by Antoine Pitrou in :issue:`17804`.) +sunau +----- + +The :meth:`~sunau.getparams` method now returns a namedtuple rather than a +plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) + + urllib ------ diff --git a/Lib/sunau.py b/Lib/sunau.py --- a/Lib/sunau.py +++ b/Lib/sunau.py @@ -51,7 +51,7 @@ getcomptype() -- returns compression type ('NONE' or 'ULAW') getcompname() -- returns human-readable version of compression type ('not compressed' matches 'NONE') - getparams() -- returns a tuple consisting of all of the + getparams() -- returns a namedtuple consisting of all of the above in the above order getmarkers() -- returns None (for compatibility with the aifc module) @@ -103,6 +103,11 @@ is destroyed. """ +from collections import namedtuple + +_sunau_params = namedtuple('_sunau_params', + 'nchannels sampwidth framerate nframes comptype compname') + # from AUDIO_FILE_MAGIC = 0x2e736e64 AUDIO_FILE_ENCODING_MULAW_8 = 1 @@ -242,9 +247,9 @@ return 'not compressed' def getparams(self): - return self.getnchannels(), self.getsampwidth(), \ - self.getframerate(), self.getnframes(), \ - self.getcomptype(), self.getcompname() + return _sunau_params(self.getnchannels(), self.getsampwidth(), + self.getframerate(), self.getnframes(), + self.getcomptype(), self.getcompname()) def getmarkers(self): return None @@ -381,9 +386,9 @@ self.setcomptype(comptype, compname) def getparams(self): - return self.getnchannels(), self.getsampwidth(), \ - self.getframerate(), self.getnframes(), \ - self.getcomptype(), self.getcompname() + return _sunau_getparams(self.getnchannels(), self.getsampwidth(), + self.getframerate(), self.getnframes(), + self.getcomptype(), self.getcompname()) def tell(self): return self._nframeswritten diff --git a/Lib/test/test_sunau.py b/Lib/test/test_sunau.py --- a/Lib/test/test_sunau.py +++ b/Lib/test/test_sunau.py @@ -1,5 +1,6 @@ from test.support import run_unittest, TESTFN import unittest +import pickle import os import sunau @@ -62,6 +63,27 @@ self.assertEqual(self.f.readframes(nframes), output) self.f.close() + def test_getparams(self): + self.f = sunau.open(TESTFN, 'w') + self.f.setnchannels(nchannels) + self.f.setsampwidth(sampwidth) + self.f.setframerate(framerate) + self.f.setcomptype('ULAW', '') + output = b'\0' * nframes * nchannels * sampwidth + self.f.writeframes(output) + self.f.close() + + self.f = sunau.open(TESTFN, 'rb') + params = self.f.getparams() + self.assertEqual(params.nchannels, nchannels) + self.assertEqual(params.sampwidth, sampwidth) + self.assertEqual(params.framerate, framerate) + self.assertEqual(params.nframes, nframes) + self.assertEqual(params.comptype, 'ULAW') + + dump = pickle.dumps(params) + self.assertEqual(pickle.loads(dump), params) + def test_main(): run_unittest(SunAUTest) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,6 +54,9 @@ Library ------- +- Issue #18901: The sunau getparams method now returns a namedtuple rather than + a plain tuple. Patch by Claudiu Popa. + - Issue #17487: The result of the wave getparams method now is pickleable again. Patch by Claudiu Popa. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 00:49:39 2013 From: python-checkins at python.org (meador.inge) Date: Wed, 4 Sep 2013 00:49:39 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE2ODI2?= =?utf-8?q?=3A_Don=27t_check_for_PYTHONCASEOK_when_using_-E=2E?= Message-ID: <3cV3JM4SBfz7Ljc@mail.python.org> http://hg.python.org/cpython/rev/934e650abc4d changeset: 85515:934e650abc4d branch: 3.3 parent: 85511:4bbaa3d7bcbf user: Meador Inge date: Tue Sep 03 16:37:26 2013 -0500 summary: Issue #16826: Don't check for PYTHONCASEOK when using -E. This commit fixes a regression that sneaked into Python 3.3 where importlib was not respecting -E when checking for the PYTHONCASEOK environment variable. files: Lib/importlib/_bootstrap.py | 5 +- Lib/test/test_importlib/extension/test_case_sensitivity.py | 37 +- Lib/test/test_importlib/source/test_case_sensitivity.py | 67 +- Misc/NEWS | 2 + Python/importlib.h | 8461 +++++---- 5 files changed, 4325 insertions(+), 4247 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -33,7 +33,10 @@ if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): def _relax_case(): """True if filenames must be checked case-insensitively.""" - return b'PYTHONCASEOK' in _os.environ + if sys.flags.ignore_environment: + return False + else: + return b'PYTHONCASEOK' in _os.environ else: def _relax_case(): """True if filenames must be checked case-insensitively.""" diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -5,7 +5,8 @@ from importlib import _bootstrap from .. import util from . import util as ext_util - +import os +import subprocess @util.case_insensitive_tests class ExtensionModuleCaseSensitivityTest(unittest.TestCase): @@ -29,14 +30,34 @@ self.assertIsNone(loader) def test_case_insensitivity(self): - with support.EnvironmentVarGuard() as env: - env.set('PYTHONCASEOK', '1') - if b'PYTHONCASEOK' not in _bootstrap._os.environ: - self.skipTest('os.environ changes not reflected in ' - '_os.environ') - loader = self.find_module() - self.assertTrue(hasattr(loader, 'load_module')) + find_snippet = """if True: + from importlib import _bootstrap + import sys + finder = _bootstrap.FileFinder('{path}', + (_bootstrap.ExtensionFileLoader, + _bootstrap.EXTENSION_SUFFIXES)) + loader = finder.find_module('{bad_name}') + print(str(hasattr(loader, 'load_module'))) + """.format(bad_name=ext_util.NAME.upper(), path=ext_util.PATH) + newenv = os.environ.copy() + newenv["PYTHONCASEOK"] = "1" + + def check_output(expected, extra_arg=None): + args = [sys.executable] + if extra_arg: + args.append(extra_arg) + args.extend(["-c", find_snippet]) + p = subprocess.Popen(args, stdout=subprocess.PIPE, env=newenv) + actual = p.communicate()[0].decode().strip() + self.assertEqual(expected, actual) + self.assertEqual(p.wait(), 0) + + # Test with PYTHONCASEOK=1. + check_output("True") + + # Test with PYTHONCASEOK=1 ignored because of -E. + check_output("False", "-E") diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -8,6 +8,7 @@ import sys from test import support as test_support import unittest +import subprocess @util.case_insensitive_tests @@ -50,16 +51,62 @@ self.assertIsNone(insensitive) def test_insensitive(self): - with test_support.EnvironmentVarGuard() as env: - env.set('PYTHONCASEOK', '1') - if b'PYTHONCASEOK' not in _bootstrap._os.environ: - self.skipTest('os.environ changes not reflected in ' - '_os.environ') - sensitive, insensitive = self.sensitivity_test() - self.assertTrue(hasattr(sensitive, 'load_module')) - self.assertIn(self.name, sensitive.get_filename(self.name)) - self.assertTrue(hasattr(insensitive, 'load_module')) - self.assertIn(self.name, insensitive.get_filename(self.name)) + sensitive_pkg = 'sensitive.{0}'.format(self.name) + insensitive_pkg = 'insensitive.{0}'.format(self.name.lower()) + context = source_util.create_modules(insensitive_pkg, sensitive_pkg) + with context as mapping: + sensitive_path = os.path.join(mapping['.root'], 'sensitive') + insensitive_path = os.path.join(mapping['.root'], 'insensitive') + find_snippet = """if True: + import sys + from importlib import machinery + + def find(path): + f = machinery.FileFinder(path, + (machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES), + (machinery.SourcelessFileLoader, + machinery.BYTECODE_SUFFIXES)) + return f.find_module('{name}') + + sensitive = find('{sensitive_path}') + insensitive = find('{insensitive_path}') + print(str(hasattr(sensitive, 'load_module'))) + if hasattr(sensitive, 'load_module'): + print(sensitive.get_filename('{name}')) + else: + print('None') + print(str(hasattr(insensitive, 'load_module'))) + if hasattr(insensitive, 'load_module'): + print(insensitive.get_filename('{name}')) + else: + print('None') + """.format(sensitive_path=sensitive_path, + insensitive_path=insensitive_path, + name=self.name) + + newenv = os.environ.copy() + newenv["PYTHONCASEOK"] = "1" + + def check_output(expected, extra_arg=None): + args = [sys.executable] + if extra_arg: + args.append(extra_arg) + args.extend(["-c", find_snippet]) + p = subprocess.Popen(args, stdout=subprocess.PIPE, + env=newenv) + actual = p.communicate()[0].decode().split() + self.assertEqual(expected[0], actual[0]) + self.assertIn(expected[1], actual[1]) + self.assertEqual(expected[2], actual[2]) + self.assertIn(expected[3], actual[3]) + self.assertEqual(p.wait(), 0) + + # Test with PYTHONCASEOK=1. + check_output(["True", self.name, "True", self.name]) + + # Test with PYTHONCASEOK=1 ignored because of -E. + check_output(["True", self.name, "False", "None"], "-E") def test_main(): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -66,6 +66,8 @@ Library ------- +- Issue #16826: Don't check for PYTHONCASEOK if interpreter started with -E. + - Issue #18418: After fork(), reinit all threads states, not only active ones. Patch by A. Jesse Jiryu Davis. diff --git a/Python/importlib.h b/Python/importlib.h --- a/Python/importlib.h +++ b/Python/importlib.h [stripped] -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 00:49:41 2013 From: python-checkins at python.org (meador.inge) Date: Wed, 4 Sep 2013 00:49:41 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2316826=3A_Don=27t_check_for_PYTHONCASEOK_when_us?= =?utf-8?q?ing_-E=2E?= Message-ID: <3cV3JP0bDsz7Lkr@mail.python.org> http://hg.python.org/cpython/rev/ba850a78cbbc changeset: 85516:ba850a78cbbc parent: 85512:4fd9407ad112 parent: 85515:934e650abc4d user: Meador Inge date: Tue Sep 03 16:53:22 2013 -0500 summary: Issue #16826: Don't check for PYTHONCASEOK when using -E. This commit fixes a regression that sneaked into Python 3.3 where importlib was not respecting -E when checking for the PYTHONCASEOK environment variable. files: Lib/importlib/_bootstrap.py | 5 +- Lib/test/test_importlib/extension/test_case_sensitivity.py | 37 +- Lib/test/test_importlib/source/test_case_sensitivity.py | 67 +- Misc/NEWS | 2 + Python/importlib.h | 6236 +++++---- 5 files changed, 3212 insertions(+), 3135 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -29,7 +29,10 @@ if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): def _relax_case(): """True if filenames must be checked case-insensitively.""" - return b'PYTHONCASEOK' in _os.environ + if sys.flags.ignore_environment: + return False + else: + return b'PYTHONCASEOK' in _os.environ else: def _relax_case(): """True if filenames must be checked case-insensitively.""" diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -6,7 +6,8 @@ from importlib import machinery from .. import util from . import util as ext_util - +import os +import subprocess @util.case_insensitive_tests class ExtensionModuleCaseSensitivityTest(unittest.TestCase): @@ -30,14 +31,34 @@ self.assertIsNone(loader) def test_case_insensitivity(self): - with support.EnvironmentVarGuard() as env: - env.set('PYTHONCASEOK', '1') - if b'PYTHONCASEOK' not in _bootstrap._os.environ: - self.skipTest('os.environ changes not reflected in ' - '_os.environ') - loader = self.find_module() - self.assertTrue(hasattr(loader, 'load_module')) + find_snippet = """if True: + from importlib import _bootstrap + import sys + finder = _bootstrap.FileFinder('{path}', + (_bootstrap.ExtensionFileLoader, + _bootstrap.EXTENSION_SUFFIXES)) + loader = finder.find_module('{bad_name}') + print(str(hasattr(loader, 'load_module'))) + """.format(bad_name=ext_util.NAME.upper(), path=ext_util.PATH) + newenv = os.environ.copy() + newenv["PYTHONCASEOK"] = "1" + + def check_output(expected, extra_arg=None): + args = [sys.executable] + if extra_arg: + args.append(extra_arg) + args.extend(["-c", find_snippet]) + p = subprocess.Popen(args, stdout=subprocess.PIPE, env=newenv) + actual = p.communicate()[0].decode().strip() + self.assertEqual(expected, actual) + self.assertEqual(p.wait(), 0) + + # Test with PYTHONCASEOK=1. + check_output("True") + + # Test with PYTHONCASEOK=1 ignored because of -E. + check_output("False", "-E") diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -8,6 +8,7 @@ import sys from test import support as test_support import unittest +import subprocess @util.case_insensitive_tests @@ -50,16 +51,62 @@ self.assertIsNone(insensitive) def test_insensitive(self): - with test_support.EnvironmentVarGuard() as env: - env.set('PYTHONCASEOK', '1') - if b'PYTHONCASEOK' not in _bootstrap._os.environ: - self.skipTest('os.environ changes not reflected in ' - '_os.environ') - sensitive, insensitive = self.sensitivity_test() - self.assertTrue(hasattr(sensitive, 'load_module')) - self.assertIn(self.name, sensitive.get_filename(self.name)) - self.assertTrue(hasattr(insensitive, 'load_module')) - self.assertIn(self.name, insensitive.get_filename(self.name)) + sensitive_pkg = 'sensitive.{0}'.format(self.name) + insensitive_pkg = 'insensitive.{0}'.format(self.name.lower()) + context = source_util.create_modules(insensitive_pkg, sensitive_pkg) + with context as mapping: + sensitive_path = os.path.join(mapping['.root'], 'sensitive') + insensitive_path = os.path.join(mapping['.root'], 'insensitive') + find_snippet = """if True: + import sys + from importlib import machinery + + def find(path): + f = machinery.FileFinder(path, + (machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES), + (machinery.SourcelessFileLoader, + machinery.BYTECODE_SUFFIXES)) + return f.find_module('{name}') + + sensitive = find('{sensitive_path}') + insensitive = find('{insensitive_path}') + print(str(hasattr(sensitive, 'load_module'))) + if hasattr(sensitive, 'load_module'): + print(sensitive.get_filename('{name}')) + else: + print('None') + print(str(hasattr(insensitive, 'load_module'))) + if hasattr(insensitive, 'load_module'): + print(insensitive.get_filename('{name}')) + else: + print('None') + """.format(sensitive_path=sensitive_path, + insensitive_path=insensitive_path, + name=self.name) + + newenv = os.environ.copy() + newenv["PYTHONCASEOK"] = "1" + + def check_output(expected, extra_arg=None): + args = [sys.executable] + if extra_arg: + args.append(extra_arg) + args.extend(["-c", find_snippet]) + p = subprocess.Popen(args, stdout=subprocess.PIPE, + env=newenv) + actual = p.communicate()[0].decode().split() + self.assertEqual(expected[0], actual[0]) + self.assertIn(expected[1], actual[1]) + self.assertEqual(expected[2], actual[2]) + self.assertIn(expected[3], actual[3]) + self.assertEqual(p.wait(), 0) + + # Test with PYTHONCASEOK=1. + check_output(["True", self.name, "True", self.name]) + + # Test with PYTHONCASEOK=1 ignored because of -E. + check_output(["True", self.name, "False", "None"], "-E") def test_main(): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,6 +54,8 @@ Library ------- +- Issue #16826: Don't check for PYTHONCASEOK if interpreter started with -E. + - Issue #18756: os.urandom() now uses a lazily-opened persistent file descriptor, so as to avoid using many file descriptors when run in parallel from multiple threads. diff --git a/Python/importlib.h b/Python/importlib.h --- a/Python/importlib.h +++ b/Python/importlib.h [stripped] -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 00:49:42 2013 From: python-checkins at python.org (meador.inge) Date: Wed, 4 Sep 2013 00:49:42 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?= =?utf-8?b?KTogTWVyZ2UgaGVhZHMu?= Message-ID: <3cV3JQ3ggDz7LlZ@mail.python.org> http://hg.python.org/cpython/rev/7694e888286d changeset: 85517:7694e888286d parent: 85514:99d26f32aed3 parent: 85516:ba850a78cbbc user: Meador Inge date: Tue Sep 03 17:32:13 2013 -0500 summary: Merge heads. files: Lib/importlib/_bootstrap.py | 5 +- Lib/test/test_importlib/extension/test_case_sensitivity.py | 37 +- Lib/test/test_importlib/source/test_case_sensitivity.py | 67 +- Misc/NEWS | 2 + Python/importlib.h | 6236 +++++---- 5 files changed, 3212 insertions(+), 3135 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -29,7 +29,10 @@ if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): def _relax_case(): """True if filenames must be checked case-insensitively.""" - return b'PYTHONCASEOK' in _os.environ + if sys.flags.ignore_environment: + return False + else: + return b'PYTHONCASEOK' in _os.environ else: def _relax_case(): """True if filenames must be checked case-insensitively.""" diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -6,7 +6,8 @@ from importlib import machinery from .. import util from . import util as ext_util - +import os +import subprocess @util.case_insensitive_tests class ExtensionModuleCaseSensitivityTest(unittest.TestCase): @@ -30,14 +31,34 @@ self.assertIsNone(loader) def test_case_insensitivity(self): - with support.EnvironmentVarGuard() as env: - env.set('PYTHONCASEOK', '1') - if b'PYTHONCASEOK' not in _bootstrap._os.environ: - self.skipTest('os.environ changes not reflected in ' - '_os.environ') - loader = self.find_module() - self.assertTrue(hasattr(loader, 'load_module')) + find_snippet = """if True: + from importlib import _bootstrap + import sys + finder = _bootstrap.FileFinder('{path}', + (_bootstrap.ExtensionFileLoader, + _bootstrap.EXTENSION_SUFFIXES)) + loader = finder.find_module('{bad_name}') + print(str(hasattr(loader, 'load_module'))) + """.format(bad_name=ext_util.NAME.upper(), path=ext_util.PATH) + newenv = os.environ.copy() + newenv["PYTHONCASEOK"] = "1" + + def check_output(expected, extra_arg=None): + args = [sys.executable] + if extra_arg: + args.append(extra_arg) + args.extend(["-c", find_snippet]) + p = subprocess.Popen(args, stdout=subprocess.PIPE, env=newenv) + actual = p.communicate()[0].decode().strip() + self.assertEqual(expected, actual) + self.assertEqual(p.wait(), 0) + + # Test with PYTHONCASEOK=1. + check_output("True") + + # Test with PYTHONCASEOK=1 ignored because of -E. + check_output("False", "-E") diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -8,6 +8,7 @@ import sys from test import support as test_support import unittest +import subprocess @util.case_insensitive_tests @@ -50,16 +51,62 @@ self.assertIsNone(insensitive) def test_insensitive(self): - with test_support.EnvironmentVarGuard() as env: - env.set('PYTHONCASEOK', '1') - if b'PYTHONCASEOK' not in _bootstrap._os.environ: - self.skipTest('os.environ changes not reflected in ' - '_os.environ') - sensitive, insensitive = self.sensitivity_test() - self.assertTrue(hasattr(sensitive, 'load_module')) - self.assertIn(self.name, sensitive.get_filename(self.name)) - self.assertTrue(hasattr(insensitive, 'load_module')) - self.assertIn(self.name, insensitive.get_filename(self.name)) + sensitive_pkg = 'sensitive.{0}'.format(self.name) + insensitive_pkg = 'insensitive.{0}'.format(self.name.lower()) + context = source_util.create_modules(insensitive_pkg, sensitive_pkg) + with context as mapping: + sensitive_path = os.path.join(mapping['.root'], 'sensitive') + insensitive_path = os.path.join(mapping['.root'], 'insensitive') + find_snippet = """if True: + import sys + from importlib import machinery + + def find(path): + f = machinery.FileFinder(path, + (machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES), + (machinery.SourcelessFileLoader, + machinery.BYTECODE_SUFFIXES)) + return f.find_module('{name}') + + sensitive = find('{sensitive_path}') + insensitive = find('{insensitive_path}') + print(str(hasattr(sensitive, 'load_module'))) + if hasattr(sensitive, 'load_module'): + print(sensitive.get_filename('{name}')) + else: + print('None') + print(str(hasattr(insensitive, 'load_module'))) + if hasattr(insensitive, 'load_module'): + print(insensitive.get_filename('{name}')) + else: + print('None') + """.format(sensitive_path=sensitive_path, + insensitive_path=insensitive_path, + name=self.name) + + newenv = os.environ.copy() + newenv["PYTHONCASEOK"] = "1" + + def check_output(expected, extra_arg=None): + args = [sys.executable] + if extra_arg: + args.append(extra_arg) + args.extend(["-c", find_snippet]) + p = subprocess.Popen(args, stdout=subprocess.PIPE, + env=newenv) + actual = p.communicate()[0].decode().split() + self.assertEqual(expected[0], actual[0]) + self.assertIn(expected[1], actual[1]) + self.assertEqual(expected[2], actual[2]) + self.assertIn(expected[3], actual[3]) + self.assertEqual(p.wait(), 0) + + # Test with PYTHONCASEOK=1. + check_output(["True", self.name, "True", self.name]) + + # Test with PYTHONCASEOK=1 ignored because of -E. + check_output(["True", self.name, "False", "None"], "-E") def test_main(): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,6 +54,8 @@ Library ------- +- Issue #16826: Don't check for PYTHONCASEOK if interpreter started with -E. + - Issue #18901: The sunau getparams method now returns a namedtuple rather than a plain tuple. Patch by Claudiu Popa. diff --git a/Python/importlib.h b/Python/importlib.h --- a/Python/importlib.h +++ b/Python/importlib.h [stripped] -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 01:19:40 2013 From: python-checkins at python.org (victor.stinner) Date: Wed, 4 Sep 2013 01:19:40 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454?= Message-ID: <3cV3z03zJGz7LjQ@mail.python.org> http://hg.python.org/peps/rev/e8b2a7afeb4c changeset: 5093:e8b2a7afeb4c user: Victor Stinner date: Wed Sep 04 01:19:30 2013 +0200 summary: PEP 454 files: pep-0454.txt | 243 ++++++++++++++++++++++++++++++-------- 1 files changed, 191 insertions(+), 52 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -20,9 +20,34 @@ Rationale ========= -This PEP proposes to a new ``tracemalloc`` module. It is a debug tool to -trace memory allocations made by Python based on API added by the PEP -445. The module provides the following information: +Common debug tools tracing memory allocations read the C filename and +number. Using such tool to analyze Python memory allocations does not +help because most memory allocations are done in the same C function, +``PyMem_Malloc()`` for example. + +There are debug tools dedicated to the Python languages like ``Heapy`` +and ``PySizer``. These projects analyze objects type and/or content. +These tools are useful when the most memory leak are instances of the +same type and this type in allocated only in a few functions. The +problem is when the object type is very common like ``str`` or +``tuple``, and it is hard to identify where these objects are allocated. + +Finding reference cycles is also a difficult task. There are different +tools to draw a diagram of all references. These tools cannot be used +huge on large applications with thousands of objects because the diagram +is too huge to be analyzed manually. + + +Proposal +======== + +Using the PEP 445, it becomes easy to setup an hook on Python memory +allocators. The hook can inspect the current Python frame to get the +Python filename and line number. + +This PEP proposes to add a new ``tracemalloc`` module. It is a debug +tool to trace memory allocations made by Python. The module provides the +following information: * Statistics on Python memory allocations per Python filename and line number: size, number, and average size of allocations @@ -30,9 +55,64 @@ * Location of a Python memory allocation: size in bytes, Python filename and line number -The ``tracemalloc`` module is different than other third-party modules -like ``Heapy`` or ``PySizer``, because it is focused on the location of -a memory allocation rather that the object type or object content. + +Command line options +==================== + +The ``python -m tracemalloc`` command can be used to analyze and compare +snapshots. The command takes a list of snapshot filenames and has the +following options. + +``-g``, ``--group-per-file`` + + Group allocations per filename, instead of grouping per line number. + +``-n NTRACES``, ``--number NTRACES`` + + Number of traces displayed per top (default: 10). + +``--first`` + + Compare with the first snapshot, instead of comparing with the + previous snapshot. + +``--include PATTERN`` + + Only include filenames matching pattern *PATTERN*. The option can be + specified multiple times. + + See ``fnmatch.fnmatch()`` for the syntax of patterns. + +``--exclude PATTERN`` + + Exclude filenames matching pattern *PATTERN*. The option can be + specified multiple times. + + See ``fnmatch.fnmatch()`` for the syntax of patterns. + +``-S``, ``--hide-size`` + + Hide the size of allocations. + +``-C``, ``--hide-count`` + + Hide the number of allocations. + +``-A``, ``--hide-average`` + + Hide the average size of allocations. + +``-P PARTS``, ``--filename-parts=PARTS`` + + Number of displayed filename parts (default: 3). + +``--color`` + + Force usage of colors even if ``sys.stdout`` is not a TTY device. + +``--no-color`` + + Disable colors if ``sys.stdout`` is a TTY device. API @@ -62,19 +142,17 @@ Get the status of the module: ``True`` if it is enabled, ``False`` otherwise. +``get_object_address(obj)`` function: + + Get the address of the memory block of the specified Python object. + ``get_object_trace(obj)`` function: - Get the trace of the memory allocation of the Python object *obj*. - Return a namedtuple with 3 attributes if the memory allocation was - traced: + Get the trace of a Python object *obj* as a ``trace`` instance. - - ``size``: size in bytes of the object - - ``filename``: name of the Python script where the object was allocated - - ``lineno``: line number where the object was allocated - - Return ``None`` if ``tracemalloc`` did not trace the memory - allocation, for example if ``tracemalloc`` was disabled when the - object was created. + Return ``None`` if the tracemalloc module did not save the location + when the object was allocated, for example if the module was + disabled. ``get_process_memory()`` function: @@ -88,6 +166,28 @@ Use the ``psutil`` module if available. +``get_stats()`` function: + + Get statistics on Python memory allocations per Python filename and + per Python line number. + + Return a dictionary + ``{filename: str -> {line_number: int -> stats: line_stat}}`` + where *stats* in a ``line_stat`` instance. *filename* and + *line_number* can be ``None``. + + Return an empty dictionary if the tracemalloc module is disabled. + + +``get_traces(obj)`` function: + + Get all traces of a Python memory allocations. + Return a dictionary ``{pointer: int -> trace}`` where *trace* + is a ``trace`` instance. + + Return an empty dictionary if the ``tracemalloc`` module is disabled. + + ``start_timer(delay: int, func: callable, args: tuple=(), kwargs: dict={})`` function: Start a timer calling ``func(*args, **kwargs)`` every *delay* @@ -109,10 +209,48 @@ Stop the timer started by ``start_timer()``. +trace class +----------- + +``trace`` class: + + This class represents debug information of an allocated memory block. + +``size`` attribute: + + Size in bytes of the memory block. + +``filename`` attribute: + + Name of the Python script where the memory block was allocated, + ``None`` if unknown. + +``lineno`` attribute: + + Line number where the memory block was allocated, ``None`` if + unknown. + + +line_stat class +---------------- + +``line_stat`` class: + + Statistics on Python memory allocations of a specific line number. + +``size`` attribute: + + Total size in bytes of all memory blocks allocated on the line. + +``count`` attribute: + + Number of memory blocks allocated on the line. + + DisplayTop class ---------------- -``DisplayTop(count: int, file=sys.stdout)`` class: +``DisplayTop(count: int=10, file=sys.stdout)`` class: Display the list of the *count* biggest memory allocations into *file*. @@ -132,38 +270,38 @@ ``color`` attribute: - ``display()`` uses colors if ``True`` (bool, - default: ``stream.isatty()``). + If ``True``, ``display()`` uses color. The default value is + ``file.isatty()``. ``compare_with_previous`` attribute: - If ``True``, ``display()`` compares with the previous top if - ``True``. If ``False``, compare with the first top (bool, default: - ``True``). + If ``True`` (default value), ``display()`` compares with the + previous snapshot. If ``False``, compare with the first snapshot. ``filename_parts`` attribute: - Number of displayed filename parts (int, default: ``3``). + Number of displayed filename parts (int, default: ``3``). Extra + parts are replaced with ``"..."``. + +``group_per_file`` attribute: + + If ``True``, group memory allocations per Python filename. If + ``False`` (default value), group allocation per Python line number. ``show_average`` attribute: - If ``True``, ``display()`` shows the average size of allocations - (bool, default: ``True``). + If ``True`` (default value), ``display()`` shows the average size + of allocations. ``show_count`` attribute: - If ``True``, ``display()`` shows the number of allocations (bool, - default: ``True``). - -``show_lineno`` attribute: - - If ``True``, use also the line number, not only the filename (bool, - default: ``True``). If ``False``, only show the filename. + If ``True`` (default value), ``display()`` shows the number of + allocations. ``show_size`` attribute: - If ``True``, ``display()`` shows the size of allocations (bool, - default: ``True``). + If ``True`` (default value), ``display()`` shows the size of + allocations. ``user_data_callback`` attribute: @@ -183,8 +321,9 @@ ``create(user_data_callback=None)`` method: Take a snapshot. If *user_data_callback* is specified, it must be a - callable object returning a list of (title: str, format: str, value: - int). format must be ``'size'``. The list must always have the same + callable object returning a list of + ``(title: str, format: str, value: int)``. + *format* must be ``'size'``. The list must always have the same length and the same order to be able to compute differences between values. @@ -198,36 +337,36 @@ See ``fnmatch.fnmatch()`` for the syntax of a pattern. +``load(filename)`` classmethod: + + Load a snapshot from a file. + ``write(filename)`` method: Write the snapshot into a file. -``load(filename)`` classmethod: +``pid`` attribute: - Load a snapshot from a file. + Identifier of the process which created the snapshot (int). ``process_memory`` attribute: - Memory usage of the process, result of ``get_process_memory()``. - It can be ``None``. + Result of the ``get_process_memory()`` function, can be ``None``. + +``stats`` attribute: + + Result of the ``get_stats()`` function (dict). + +``timestamp`` attribute: + + Creation date and time of the snapshot, ``datetime.datetime`` + instance. ``user_data`` attribute: Optional list of user data, result of *user_data_callback* in ``Snapshot.create()`` (default: None). -``pid`` attribute: - - Identifier of the process which created the snapshot (int). - -``stats`` attribute: - - Raw memory allocation statistics (dict). - -``timestamp`` attribute: - - Date and time of the creation of the snapshot (str). - TakeSnapshot class ------------------ -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Wed Sep 4 02:03:04 2013 From: python-checkins at python.org (victor.stinner) Date: Wed, 4 Sep 2013 02:03:04 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454=3A_uh_ho=2C_fix_many_?= =?utf-8?q?typos_in_the_Rationale?= Message-ID: <3cV4x41d5zz7Ljb@mail.python.org> http://hg.python.org/peps/rev/05052537468d changeset: 5094:05052537468d user: Victor Stinner date: Wed Sep 04 02:02:50 2013 +0200 summary: PEP 454: uh ho, fix many typos in the Rationale files: pep-0454.txt | 21 +++++++++++---------- 1 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -22,20 +22,21 @@ Common debug tools tracing memory allocations read the C filename and number. Using such tool to analyze Python memory allocations does not -help because most memory allocations are done in the same C function, -``PyMem_Malloc()`` for example. +help because most memory block are allocated in the same C function, +in ``PyMem_Malloc()`` for example. -There are debug tools dedicated to the Python languages like ``Heapy`` +There are debug tools dedicated to the Python language like ``Heapy`` and ``PySizer``. These projects analyze objects type and/or content. -These tools are useful when the most memory leak are instances of the -same type and this type in allocated only in a few functions. The +These tools are useful when most memory leaks are instances of the +same type and this type is only instancied in a few functions. The problem is when the object type is very common like ``str`` or -``tuple``, and it is hard to identify where these objects are allocated. +``tuple``, and it is hard to identify where these objects are +instancied. -Finding reference cycles is also a difficult task. There are different -tools to draw a diagram of all references. These tools cannot be used -huge on large applications with thousands of objects because the diagram -is too huge to be analyzed manually. +Finding reference cycles is also a difficult problem. There are +different tools to draw a diagram of all references. These tools cannot +be used on large applications with thousands of objects because the +diagram is too huge to be analyzed manually. Proposal -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Wed Sep 4 02:54:59 2013 From: python-checkins at python.org (meador.inge) Date: Wed, 4 Sep 2013 02:54:59 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE2ODI2?= =?utf-8?q?=3A_Revert_fix_while_Windows_issues_are_being_worked_out=2E?= Message-ID: <3cV64z15nYz7LjW@mail.python.org> http://hg.python.org/cpython/rev/7801ef4a4ce3 changeset: 85518:7801ef4a4ce3 branch: 3.3 parent: 85515:934e650abc4d user: Meador Inge date: Tue Sep 03 19:43:49 2013 -0500 summary: Issue #16826: Revert fix while Windows issues are being worked out. files: Lib/importlib/_bootstrap.py | 5 +- Lib/test/test_importlib/extension/test_case_sensitivity.py | 37 +- Lib/test/test_importlib/source/test_case_sensitivity.py | 67 +- Misc/NEWS | 2 - Python/importlib.h | 8457 ++++----- 5 files changed, 4245 insertions(+), 4323 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -33,10 +33,7 @@ if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): def _relax_case(): """True if filenames must be checked case-insensitively.""" - if sys.flags.ignore_environment: - return False - else: - return b'PYTHONCASEOK' in _os.environ + return b'PYTHONCASEOK' in _os.environ else: def _relax_case(): """True if filenames must be checked case-insensitively.""" diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -5,8 +5,7 @@ from importlib import _bootstrap from .. import util from . import util as ext_util -import os -import subprocess + @util.case_insensitive_tests class ExtensionModuleCaseSensitivityTest(unittest.TestCase): @@ -30,34 +29,14 @@ self.assertIsNone(loader) def test_case_insensitivity(self): - find_snippet = """if True: - from importlib import _bootstrap - import sys - finder = _bootstrap.FileFinder('{path}', - (_bootstrap.ExtensionFileLoader, - _bootstrap.EXTENSION_SUFFIXES)) - loader = finder.find_module('{bad_name}') - print(str(hasattr(loader, 'load_module'))) - """.format(bad_name=ext_util.NAME.upper(), path=ext_util.PATH) + with support.EnvironmentVarGuard() as env: + env.set('PYTHONCASEOK', '1') + if b'PYTHONCASEOK' not in _bootstrap._os.environ: + self.skipTest('os.environ changes not reflected in ' + '_os.environ') + loader = self.find_module() + self.assertTrue(hasattr(loader, 'load_module')) - newenv = os.environ.copy() - newenv["PYTHONCASEOK"] = "1" - - def check_output(expected, extra_arg=None): - args = [sys.executable] - if extra_arg: - args.append(extra_arg) - args.extend(["-c", find_snippet]) - p = subprocess.Popen(args, stdout=subprocess.PIPE, env=newenv) - actual = p.communicate()[0].decode().strip() - self.assertEqual(expected, actual) - self.assertEqual(p.wait(), 0) - - # Test with PYTHONCASEOK=1. - check_output("True") - - # Test with PYTHONCASEOK=1 ignored because of -E. - check_output("False", "-E") diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -8,7 +8,6 @@ import sys from test import support as test_support import unittest -import subprocess @util.case_insensitive_tests @@ -51,62 +50,16 @@ self.assertIsNone(insensitive) def test_insensitive(self): - sensitive_pkg = 'sensitive.{0}'.format(self.name) - insensitive_pkg = 'insensitive.{0}'.format(self.name.lower()) - context = source_util.create_modules(insensitive_pkg, sensitive_pkg) - with context as mapping: - sensitive_path = os.path.join(mapping['.root'], 'sensitive') - insensitive_path = os.path.join(mapping['.root'], 'insensitive') - find_snippet = """if True: - import sys - from importlib import machinery - - def find(path): - f = machinery.FileFinder(path, - (machinery.SourceFileLoader, - machinery.SOURCE_SUFFIXES), - (machinery.SourcelessFileLoader, - machinery.BYTECODE_SUFFIXES)) - return f.find_module('{name}') - - sensitive = find('{sensitive_path}') - insensitive = find('{insensitive_path}') - print(str(hasattr(sensitive, 'load_module'))) - if hasattr(sensitive, 'load_module'): - print(sensitive.get_filename('{name}')) - else: - print('None') - print(str(hasattr(insensitive, 'load_module'))) - if hasattr(insensitive, 'load_module'): - print(insensitive.get_filename('{name}')) - else: - print('None') - """.format(sensitive_path=sensitive_path, - insensitive_path=insensitive_path, - name=self.name) - - newenv = os.environ.copy() - newenv["PYTHONCASEOK"] = "1" - - def check_output(expected, extra_arg=None): - args = [sys.executable] - if extra_arg: - args.append(extra_arg) - args.extend(["-c", find_snippet]) - p = subprocess.Popen(args, stdout=subprocess.PIPE, - env=newenv) - actual = p.communicate()[0].decode().split() - self.assertEqual(expected[0], actual[0]) - self.assertIn(expected[1], actual[1]) - self.assertEqual(expected[2], actual[2]) - self.assertIn(expected[3], actual[3]) - self.assertEqual(p.wait(), 0) - - # Test with PYTHONCASEOK=1. - check_output(["True", self.name, "True", self.name]) - - # Test with PYTHONCASEOK=1 ignored because of -E. - check_output(["True", self.name, "False", "None"], "-E") + with test_support.EnvironmentVarGuard() as env: + env.set('PYTHONCASEOK', '1') + if b'PYTHONCASEOK' not in _bootstrap._os.environ: + self.skipTest('os.environ changes not reflected in ' + '_os.environ') + sensitive, insensitive = self.sensitivity_test() + self.assertTrue(hasattr(sensitive, 'load_module')) + self.assertIn(self.name, sensitive.get_filename(self.name)) + self.assertTrue(hasattr(insensitive, 'load_module')) + self.assertIn(self.name, insensitive.get_filename(self.name)) def test_main(): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -66,8 +66,6 @@ Library ------- -- Issue #16826: Don't check for PYTHONCASEOK if interpreter started with -E. - - Issue #18418: After fork(), reinit all threads states, not only active ones. Patch by A. Jesse Jiryu Davis. diff --git a/Python/importlib.h b/Python/importlib.h --- a/Python/importlib.h +++ b/Python/importlib.h [stripped] -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 02:55:00 2013 From: python-checkins at python.org (meador.inge) Date: Wed, 4 Sep 2013 02:55:00 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2316826=3A_Revert_fix_while_Windows_issues_are_be?= =?utf-8?q?ing_worked_out=2E?= Message-ID: <3cV6504GhTz7Ljb@mail.python.org> http://hg.python.org/cpython/rev/a1282b67b4cf changeset: 85519:a1282b67b4cf parent: 85517:7694e888286d parent: 85518:7801ef4a4ce3 user: Meador Inge date: Tue Sep 03 19:54:40 2013 -0500 summary: Issue #16826: Revert fix while Windows issues are being worked out. files: Lib/importlib/_bootstrap.py | 5 +- Lib/test/test_importlib/extension/test_case_sensitivity.py | 37 +- Lib/test/test_importlib/source/test_case_sensitivity.py | 67 +- Misc/NEWS | 2 - Python/importlib.h | 6250 ++++----- 5 files changed, 3142 insertions(+), 3219 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -29,10 +29,7 @@ if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): def _relax_case(): """True if filenames must be checked case-insensitively.""" - if sys.flags.ignore_environment: - return False - else: - return b'PYTHONCASEOK' in _os.environ + return b'PYTHONCASEOK' in _os.environ else: def _relax_case(): """True if filenames must be checked case-insensitively.""" diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -6,8 +6,7 @@ from importlib import machinery from .. import util from . import util as ext_util -import os -import subprocess + @util.case_insensitive_tests class ExtensionModuleCaseSensitivityTest(unittest.TestCase): @@ -31,34 +30,14 @@ self.assertIsNone(loader) def test_case_insensitivity(self): - find_snippet = """if True: - from importlib import _bootstrap - import sys - finder = _bootstrap.FileFinder('{path}', - (_bootstrap.ExtensionFileLoader, - _bootstrap.EXTENSION_SUFFIXES)) - loader = finder.find_module('{bad_name}') - print(str(hasattr(loader, 'load_module'))) - """.format(bad_name=ext_util.NAME.upper(), path=ext_util.PATH) + with support.EnvironmentVarGuard() as env: + env.set('PYTHONCASEOK', '1') + if b'PYTHONCASEOK' not in _bootstrap._os.environ: + self.skipTest('os.environ changes not reflected in ' + '_os.environ') + loader = self.find_module() + self.assertTrue(hasattr(loader, 'load_module')) - newenv = os.environ.copy() - newenv["PYTHONCASEOK"] = "1" - - def check_output(expected, extra_arg=None): - args = [sys.executable] - if extra_arg: - args.append(extra_arg) - args.extend(["-c", find_snippet]) - p = subprocess.Popen(args, stdout=subprocess.PIPE, env=newenv) - actual = p.communicate()[0].decode().strip() - self.assertEqual(expected, actual) - self.assertEqual(p.wait(), 0) - - # Test with PYTHONCASEOK=1. - check_output("True") - - # Test with PYTHONCASEOK=1 ignored because of -E. - check_output("False", "-E") diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -8,7 +8,6 @@ import sys from test import support as test_support import unittest -import subprocess @util.case_insensitive_tests @@ -51,62 +50,16 @@ self.assertIsNone(insensitive) def test_insensitive(self): - sensitive_pkg = 'sensitive.{0}'.format(self.name) - insensitive_pkg = 'insensitive.{0}'.format(self.name.lower()) - context = source_util.create_modules(insensitive_pkg, sensitive_pkg) - with context as mapping: - sensitive_path = os.path.join(mapping['.root'], 'sensitive') - insensitive_path = os.path.join(mapping['.root'], 'insensitive') - find_snippet = """if True: - import sys - from importlib import machinery - - def find(path): - f = machinery.FileFinder(path, - (machinery.SourceFileLoader, - machinery.SOURCE_SUFFIXES), - (machinery.SourcelessFileLoader, - machinery.BYTECODE_SUFFIXES)) - return f.find_module('{name}') - - sensitive = find('{sensitive_path}') - insensitive = find('{insensitive_path}') - print(str(hasattr(sensitive, 'load_module'))) - if hasattr(sensitive, 'load_module'): - print(sensitive.get_filename('{name}')) - else: - print('None') - print(str(hasattr(insensitive, 'load_module'))) - if hasattr(insensitive, 'load_module'): - print(insensitive.get_filename('{name}')) - else: - print('None') - """.format(sensitive_path=sensitive_path, - insensitive_path=insensitive_path, - name=self.name) - - newenv = os.environ.copy() - newenv["PYTHONCASEOK"] = "1" - - def check_output(expected, extra_arg=None): - args = [sys.executable] - if extra_arg: - args.append(extra_arg) - args.extend(["-c", find_snippet]) - p = subprocess.Popen(args, stdout=subprocess.PIPE, - env=newenv) - actual = p.communicate()[0].decode().split() - self.assertEqual(expected[0], actual[0]) - self.assertIn(expected[1], actual[1]) - self.assertEqual(expected[2], actual[2]) - self.assertIn(expected[3], actual[3]) - self.assertEqual(p.wait(), 0) - - # Test with PYTHONCASEOK=1. - check_output(["True", self.name, "True", self.name]) - - # Test with PYTHONCASEOK=1 ignored because of -E. - check_output(["True", self.name, "False", "None"], "-E") + with test_support.EnvironmentVarGuard() as env: + env.set('PYTHONCASEOK', '1') + if b'PYTHONCASEOK' not in _bootstrap._os.environ: + self.skipTest('os.environ changes not reflected in ' + '_os.environ') + sensitive, insensitive = self.sensitivity_test() + self.assertTrue(hasattr(sensitive, 'load_module')) + self.assertIn(self.name, sensitive.get_filename(self.name)) + self.assertTrue(hasattr(insensitive, 'load_module')) + self.assertIn(self.name, insensitive.get_filename(self.name)) def test_main(): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,8 +54,6 @@ Library ------- -- Issue #16826: Don't check for PYTHONCASEOK if interpreter started with -E. - - Issue #18901: The sunau getparams method now returns a namedtuple rather than a plain tuple. Patch by Claudiu Popa. diff --git a/Python/importlib.h b/Python/importlib.h --- a/Python/importlib.h +++ b/Python/importlib.h [stripped] -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 06:01:26 2013 From: python-checkins at python.org (andrew.svetlov) Date: Wed, 4 Sep 2013 06:01:26 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318882=3A_Add_thre?= =?utf-8?q?ading=2Emain=5Fthread=28=29_function=2E?= Message-ID: <3cVBD63mn0z7Ljg@mail.python.org> http://hg.python.org/cpython/rev/96e55a1a0de7 changeset: 85520:96e55a1a0de7 user: Andrew Svetlov date: Wed Sep 04 07:01:07 2013 +0300 summary: Issue #18882: Add threading.main_thread() function. files: Doc/library/threading.rst | 9 +++ Lib/test/test_threading.py | 78 +++++++++++++++++++++++-- Lib/threading.py | 36 ++++++----- Misc/NEWS | 2 + 4 files changed, 102 insertions(+), 23 deletions(-) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -57,6 +57,15 @@ and threads that have not yet been started. +.. function:: main_thread() + + Return the main :class:`Thread` object. In normal conditions, the + main thread is the thread from which the Python interpreter was + started. + + .. versionadded:: 3.4 + + .. function:: settrace(func) .. index:: single: trace function diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -21,6 +21,15 @@ from test import lock_tests + +# Between fork() and exec(), only async-safe functions are allowed (issues +# #12316 and #11870), and fork() from a worker thread is known to trigger +# problems with some operating systems (issue #3863): skip problematic tests +# on platforms known to behave badly. +platforms_to_skip = ('freebsd4', 'freebsd5', 'freebsd6', 'netbsd5', + 'hp-ux11') + + # A trivial mutable counter. class Counter(object): def __init__(self): @@ -468,16 +477,71 @@ pid, status = os.waitpid(pid, 0) self.assertEqual(0, status) + def test_main_thread(self): + main = threading.main_thread() + self.assertEqual(main.name, 'MainThread') + self.assertEqual(main.ident, threading.current_thread().ident) + self.assertEqual(main.ident, threading.get_ident()) + + def f(): + self.assertNotEqual(threading.main_thread().ident, + threading.current_thread().ident) + th = threading.Thread(target=f) + th.start() + th.join() + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork(self): + code = """if 1: + import os, threading + + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + else: + os.waitpid(pid, 0) + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "MainThread\nTrue\nTrue\n") + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork_from_nonmain_thread(self): + code = """if 1: + import os, threading, sys + + def f(): + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + else: + os.waitpid(pid, 0) + + th = threading.Thread(target=f) + th.start() + th.join() + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "Thread-1\nTrue\nTrue\n") + class ThreadJoinOnShutdown(BaseTestCase): - # Between fork() and exec(), only async-safe functions are allowed (issues - # #12316 and #11870), and fork() from a worker thread is known to trigger - # problems with some operating systems (issue #3863): skip problematic tests - # on platforms known to behave badly. - platforms_to_skip = ('freebsd4', 'freebsd5', 'freebsd6', 'netbsd5', - 'hp-ux11') - def _run_and_join(self, script): script = """if 1: import sys, os, time, threading diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -840,20 +840,6 @@ with _active_limbo_lock: _active[self._ident] = self - def _exitfunc(self): - self._stop() - t = _pickSomeNonDaemonThread() - while t: - t.join() - t = _pickSomeNonDaemonThread() - self._delete() - -def _pickSomeNonDaemonThread(): - for t in enumerate(): - if not t.daemon and t.is_alive(): - return t - return None - # Dummy thread class to represent threads not started here. # These aren't garbage collected when they die, nor can they be waited for. @@ -915,7 +901,24 @@ # and make it available for the interpreter # (Py_Main) as threading._shutdown. -_shutdown = _MainThread()._exitfunc +_main_thread = _MainThread() + +def _shutdown(): + _main_thread._stop() + t = _pickSomeNonDaemonThread() + while t: + t.join() + t = _pickSomeNonDaemonThread() + _main_thread._delete() + +def _pickSomeNonDaemonThread(): + for t in enumerate(): + if not t.daemon and t.is_alive(): + return t + return None + +def main_thread(): + return _main_thread # get thread-local implementation, either from the thread # module, or from the python fallback @@ -933,12 +936,13 @@ # Reset _active_limbo_lock, in case we forked while the lock was held # by another (non-forked) thread. http://bugs.python.org/issue874900 - global _active_limbo_lock + global _active_limbo_lock, _main_thread _active_limbo_lock = _allocate_lock() # fork() only copied the current thread; clear references to others. new_active = {} current = current_thread() + _main_thread = current with _active_limbo_lock: for thread in _enumerate(): # Any lock/condition variable may be currently locked or in an diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,6 +54,8 @@ Library ------- +- Issue #18882: Add threading.main_thread() function. + - Issue #18901: The sunau getparams method now returns a namedtuple rather than a plain tuple. Patch by Claudiu Popa. -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Wed Sep 4 06:18:18 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Wed, 04 Sep 2013 06:18:18 +0200 Subject: [Python-checkins] Daily reference leaks (a1282b67b4cf): sum=4 Message-ID: results for a1282b67b4cf on branch "default" -------------------------------------------- test_support leaked [-1, 1, 0] references, sum=0 test_support leaked [-1, 3, 2] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogpHIV28', '-x'] From python-checkins at python.org Wed Sep 4 09:33:42 2013 From: python-checkins at python.org (andrew.svetlov) Date: Wed, 4 Sep 2013 09:33:42 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Add_docstring_for_threadin?= =?utf-8?b?Zy5tYWluX3RocmVhZCgpLg==?= Message-ID: <3cVGx274LLz7LjQ@mail.python.org> http://hg.python.org/cpython/rev/f18fb7f39c24 changeset: 85521:f18fb7f39c24 user: Andrew Svetlov date: Wed Sep 04 10:33:11 2013 +0300 summary: Add docstring for threading.main_thread(). files: Lib/threading.py | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -918,6 +918,11 @@ return None def main_thread(): + """Return the main thread object. + + In normal conditions, the main thread is the thread from which the + Python interpreter was started. + """ return _main_thread # get thread-local implementation, either from the thread -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 13:21:30 2013 From: python-checkins at python.org (victor.stinner) Date: Wed, 4 Sep 2013 13:21:30 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454=3A_add_frame_class?= Message-ID: <3cVMzt4j9Gz7Lmf@mail.python.org> http://hg.python.org/peps/rev/644bdd93fba2 changeset: 5095:644bdd93fba2 user: Victor Stinner date: Wed Sep 04 13:19:17 2013 +0200 summary: PEP 454: add frame class Other changes: * add the type and default value of almost all attributes * move the "Command line options" section to the end files: pep-0454.txt | 591 +++++++++++++++++++------------------- 1 files changed, 301 insertions(+), 290 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -57,6 +57,307 @@ and line number +API +=== + +To trace the most Python memory allocations, the module should be +enabled as early as possible in your application by calling +``tracemalloc.enable()`` function, by setting the ``PYTHONTRACEMALLOC`` +environment variable to ``1``, or by using ``-X tracemalloc`` command +line option. + + +Functions +--------- + +``enable()`` function: + + Start tracing Python memory allocations. + +``disable()`` function: + + Stop tracing Python memory allocations and stop the timer started by + ``start_timer()``. + +``is_enabled()`` function: + + Get the status of the module: ``True`` if it is enabled, ``False`` + otherwise. + +``get_object_address(obj)`` function: + + Get the address of the memory block of the specified Python object. + +``get_object_trace(obj)`` function: + + Get the trace of a Python object *obj* as a ``trace`` instance. + + Return ``None`` if the tracemalloc module did not save the location + when the object was allocated, for example if the module was + disabled. + +``get_process_memory()`` function: + + Get the memory usage of the current process as a meminfo namedtuple + with two attributes: + + * ``rss``: Resident Set Size in bytes + * ``vms``: size of the virtual memory in bytes + + Return ``None`` if the platform is not supported. + + Use the ``psutil`` module if available. + +``get_stats()`` function: + + Get statistics on Python memory allocations per Python filename and + per Python line number. + + Return a dictionary + ``{filename: str -> {line_number: int -> stats: line_stat}}`` + where *stats* in a ``line_stat`` instance. *filename* and + *line_number* can be ``None``. + + Return an empty dictionary if the tracemalloc module is disabled. + + +``get_traces(obj)`` function: + + Get all traces of a Python memory allocations. + Return a dictionary ``{pointer: int -> trace}`` where *trace* + is a ``trace`` instance. + + Return an empty dictionary if the ``tracemalloc`` module is disabled. + + +``start_timer(delay: int, func: callable, args: tuple=(), kwargs: dict={})`` function: + + Start a timer calling ``func(*args, **kwargs)`` every *delay* + seconds. + + The timer is based on the Python memory allocator, it is not real + time. *func* is called after at least *delay* seconds, it is not + called exactly after *delay* seconds if no Python memory allocation + occurred. + + If ``start_timer()`` is called twice, previous parameters are + replaced. The timer has a resolution of 1 second. + + ``start_timer()`` is used by ``DisplayTop`` and ``TakeSnapshot`` to + run regulary a task. + +``stop_timer()`` function: + + Stop the timer started by ``start_timer()``. + + +frame class +----------- + +``frame`` class: + + Trace of a Python frame. + +``filename`` attribute (``str``): + + Python filename, ``None`` if unknown. + +``lineno`` attribute (``int``): + + Python line number, ``None`` if unknown. + + +trace class +----------- + +``trace`` class: + + This class represents debug information of an allocated memory block. + +``size`` attribute (``int``): + + Size in bytes of the memory block. + +``frames`` attribute (``list``): + + Traceback where the memory block was allocated as a list of + ``frame`` instances (most recent first). + + The list can be empty or incomplete if the tracemalloc module was + unable to retrieve the full traceback. + + For efficiency, the traceback is truncated to 10 frames. + + +line_stat class +---------------- + +``line_stat`` class: + + Statistics on Python memory allocations of a specific line number. + +``size`` attribute (``int``): + + Total size in bytes of all memory blocks allocated on the line. + +``count`` attribute (``int``): + + Number of memory blocks allocated on the line. + + +DisplayTop class +---------------- + +``DisplayTop(count: int=10, file=sys.stdout)`` class: + + Display the list of the *count* biggest memory allocations into + *file*. + +``display()`` method: + + Display the top once. + +``start(delay: int)`` method: + + Start a task using ``tracemalloc`` timer to display the top every + *delay* seconds. + +``stop()`` method: + + Stop the task started by the ``DisplayTop.start()`` method + +``color`` attribute (``bool``, default: ``file.isatty()``): + + If ``True``, ``display()`` uses color. + +``compare_with_previous`` attribute (``bool``, default: ``True``): + + If ``True``, ``display()`` compares with the + previous snapshot. If ``False``, compare with the first snapshot. + +``filename_parts`` attribute (``int``, default: ``3``): + + Number of displayed filename parts. Extra parts are replaced + with ``"..."``. + +``group_per_file`` attribute (``bool``, default: ``False``): + + If ``True``, group memory allocations per Python filename. If + ``False``, group allocation per Python line number. + +``show_average`` attribute (``bool``, default: ``True``): + + If ``True``, ``display()`` shows the average size + of allocations. + +``show_count`` attribute (``bool``, default: ``True``): + + If ``True``, ``display()`` shows the number of + allocations. + +``show_size`` attribute (``bool``, default: ``True``): + + If ``True``, ``display()`` shows the size of + allocations. + +``user_data_callback`` attribute (``callable``, default: ``None``): + + Optional callback collecting user data. See ``Snapshot.create()``. + + +Snapshot class +-------------- + +``Snapshot()`` class: + + Snapshot of Python memory allocations. + + Use ``TakeSnapshot`` to take regulary snapshots. + +``create(user_data_callback=None)`` method: + + Take a snapshot. If *user_data_callback* is specified, it must be a + callable object returning a list of + ``(title: str, format: str, value: int)``. + *format* must be ``'size'``. The list must always have the same + length and the same order to be able to compute differences between + values. + + Example: ``[('Video memory', 'size', 234902)]``. + +``filter_filenames(patterns: list, include: bool)`` method: + + Remove filenames not matching any pattern of *patterns* if *include* + is ``True``, or remove filenames matching a pattern of *patterns* if + *include* is ``False`` (exclude). + + See ``fnmatch.fnmatch()`` for the syntax of a pattern. + +``load(filename)`` classmethod: + + Load a snapshot from a file. + +``write(filename)`` method: + + Write the snapshot into a file. + +``pid`` attribute (``int``): + + Identifier of the process which created the snapshot. + +``process_memory`` attribute: + + Result of the ``get_process_memory()`` function, can be ``None``. + +``stats`` attribute (``dict``): + + Result of the ``get_stats()`` function. + +``timestamp`` attribute (``datetime.datetime``): + + Creation date and time of the snapshot. + +``user_data`` attribute (``list``, default: ``None``): + + Optional list of user data, result of *user_data_callback* in + ``Snapshot.create()``. + + +TakeSnapshot class +------------------ + +``TakeSnapshot`` class: + + Task taking snapshots of Python memory allocations: write them into + files. By default, snapshots are written in the current directory. + +``start(delay: int)`` method: + + Start a task taking a snapshot every delay seconds. + +``stop()`` method: + + Stop the task started by the ``TakeSnapshot.start()`` method. + +``take_snapshot()`` method: + + Take a snapshot. + +``filename_template`` attribute (``str``, +default: ``'tracemalloc-$counter.pickle'``): + + Template used to create a filename. The following variables can be + used in the template: + + * ``$pid``: identifier of the current process + * ``$timestamp``: current date and time + * ``$counter``: counter starting at 1 and incremented at each snapshot + +``user_data_callback`` attribute (``callable``, default: ``None``): + + Optional callback collecting user data. See ``Snapshot.create()``. + + Command line options ==================== @@ -116,296 +417,6 @@ Disable colors if ``sys.stdout`` is a TTY device. -API -=== - -To trace the most Python memory allocations, the module should be -enabled as early as possible in your application by calling -``tracemalloc.enable()`` function, by setting the ``PYTHONTRACEMALLOC`` -environment variable to ``1``, or by using ``-X tracemalloc`` command -line option. - - -Functions ---------- - -``enable()`` function: - - Start tracing Python memory allocations. - -``disable()`` function: - - Stop tracing Python memory allocations and stop the timer started by - ``start_timer()``. - -``is_enabled()`` function: - - Get the status of the module: ``True`` if it is enabled, ``False`` - otherwise. - -``get_object_address(obj)`` function: - - Get the address of the memory block of the specified Python object. - -``get_object_trace(obj)`` function: - - Get the trace of a Python object *obj* as a ``trace`` instance. - - Return ``None`` if the tracemalloc module did not save the location - when the object was allocated, for example if the module was - disabled. - -``get_process_memory()`` function: - - Get the memory usage of the current process as a meminfo namedtuple - with two attributes: - - * ``rss``: Resident Set Size in bytes - * ``vms``: size of the virtual memory in bytes - - Return ``None`` if the platform is not supported. - - Use the ``psutil`` module if available. - -``get_stats()`` function: - - Get statistics on Python memory allocations per Python filename and - per Python line number. - - Return a dictionary - ``{filename: str -> {line_number: int -> stats: line_stat}}`` - where *stats* in a ``line_stat`` instance. *filename* and - *line_number* can be ``None``. - - Return an empty dictionary if the tracemalloc module is disabled. - - -``get_traces(obj)`` function: - - Get all traces of a Python memory allocations. - Return a dictionary ``{pointer: int -> trace}`` where *trace* - is a ``trace`` instance. - - Return an empty dictionary if the ``tracemalloc`` module is disabled. - - -``start_timer(delay: int, func: callable, args: tuple=(), kwargs: dict={})`` function: - - Start a timer calling ``func(*args, **kwargs)`` every *delay* - seconds. - - The timer is based on the Python memory allocator, it is not real - time. *func* is called after at least *delay* seconds, it is not - called exactly after *delay* seconds if no Python memory allocation - occurred. - - If ``start_timer()`` is called twice, previous parameters are - replaced. The timer has a resolution of 1 second. - - ``start_timer()`` is used by ``DisplayTop`` and ``TakeSnapshot`` to - run regulary a task. - -``stop_timer()`` function: - - Stop the timer started by ``start_timer()``. - - -trace class ------------ - -``trace`` class: - - This class represents debug information of an allocated memory block. - -``size`` attribute: - - Size in bytes of the memory block. - -``filename`` attribute: - - Name of the Python script where the memory block was allocated, - ``None`` if unknown. - -``lineno`` attribute: - - Line number where the memory block was allocated, ``None`` if - unknown. - - -line_stat class ----------------- - -``line_stat`` class: - - Statistics on Python memory allocations of a specific line number. - -``size`` attribute: - - Total size in bytes of all memory blocks allocated on the line. - -``count`` attribute: - - Number of memory blocks allocated on the line. - - -DisplayTop class ----------------- - -``DisplayTop(count: int=10, file=sys.stdout)`` class: - - Display the list of the *count* biggest memory allocations into - *file*. - -``display()`` method: - - Display the top once. - -``start(delay: int)`` method: - - Start a task using ``tracemalloc`` timer to display the top every - *delay* seconds. - -``stop()`` method: - - Stop the task started by the ``DisplayTop.start()`` method - -``color`` attribute: - - If ``True``, ``display()`` uses color. The default value is - ``file.isatty()``. - -``compare_with_previous`` attribute: - - If ``True`` (default value), ``display()`` compares with the - previous snapshot. If ``False``, compare with the first snapshot. - -``filename_parts`` attribute: - - Number of displayed filename parts (int, default: ``3``). Extra - parts are replaced with ``"..."``. - -``group_per_file`` attribute: - - If ``True``, group memory allocations per Python filename. If - ``False`` (default value), group allocation per Python line number. - -``show_average`` attribute: - - If ``True`` (default value), ``display()`` shows the average size - of allocations. - -``show_count`` attribute: - - If ``True`` (default value), ``display()`` shows the number of - allocations. - -``show_size`` attribute: - - If ``True`` (default value), ``display()`` shows the size of - allocations. - -``user_data_callback`` attribute: - - Optional callback collecting user data (callable, default: - ``None``). See ``Snapshot.create()``. - - -Snapshot class --------------- - -``Snapshot()`` class: - - Snapshot of Python memory allocations. - - Use ``TakeSnapshot`` to take regulary snapshots. - -``create(user_data_callback=None)`` method: - - Take a snapshot. If *user_data_callback* is specified, it must be a - callable object returning a list of - ``(title: str, format: str, value: int)``. - *format* must be ``'size'``. The list must always have the same - length and the same order to be able to compute differences between - values. - - Example: ``[('Video memory', 'size', 234902)]``. - -``filter_filenames(patterns: list, include: bool)`` method: - - Remove filenames not matching any pattern of *patterns* if *include* - is ``True``, or remove filenames matching a pattern of *patterns* if - *include* is ``False`` (exclude). - - See ``fnmatch.fnmatch()`` for the syntax of a pattern. - -``load(filename)`` classmethod: - - Load a snapshot from a file. - -``write(filename)`` method: - - Write the snapshot into a file. - -``pid`` attribute: - - Identifier of the process which created the snapshot (int). - -``process_memory`` attribute: - - Result of the ``get_process_memory()`` function, can be ``None``. - -``stats`` attribute: - - Result of the ``get_stats()`` function (dict). - -``timestamp`` attribute: - - Creation date and time of the snapshot, ``datetime.datetime`` - instance. - -``user_data`` attribute: - - Optional list of user data, result of *user_data_callback* in - ``Snapshot.create()`` (default: None). - - -TakeSnapshot class ------------------- - -``TakeSnapshot`` class: - - Task taking snapshots of Python memory allocations: write them into - files. By default, snapshots are written in the current directory. - -``start(delay: int)`` method: - - Start a task taking a snapshot every delay seconds. - -``stop()`` method: - - Stop the task started by the ``TakeSnapshot.start()`` method. - -``take_snapshot()`` method: - - Take a snapshot. - -``filename_template`` attribute: - - Template (``str``) used to create a filename. The following - variables can be used in the template: - - * ``$pid``: identifier of the current process - * ``$timestamp``: current date and time - * ``$counter``: counter starting at 1 and incremented at each snapshot - - The default pattern is ``'tracemalloc-$counter.pickle'``. - -``user_data_callback`` attribute: - - Optional callback collecting user data (callable, default: - ``None``). See ``Snapshot.create()``. - - Links ===== -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Wed Sep 4 13:30:37 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Wed, 4 Sep 2013 13:30:37 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issues_=2318901=2C_=231891?= =?utf-8?q?9=3A_Fix_a_typo_in_the_=5Fsunau=5Fparams_name=2E?= Message-ID: <3cVNBP4gPFz7Llb@mail.python.org> http://hg.python.org/cpython/rev/61ca4732399b changeset: 85522:61ca4732399b user: Serhiy Storchaka date: Wed Sep 04 14:30:16 2013 +0300 summary: Issues #18901, #18919: Fix a typo in the _sunau_params name. files: Lib/sunau.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/sunau.py b/Lib/sunau.py --- a/Lib/sunau.py +++ b/Lib/sunau.py @@ -386,7 +386,7 @@ self.setcomptype(comptype, compname) def getparams(self): - return _sunau_getparams(self.getnchannels(), self.getsampwidth(), + return _sunau_params(self.getnchannels(), self.getsampwidth(), self.getframerate(), self.getnframes(), self.getcomptype(), self.getcompname()) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 14:01:49 2013 From: python-checkins at python.org (victor.stinner) Date: Wed, 4 Sep 2013 14:01:49 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454=3A_add_link_to_the_pr?= =?utf-8?q?oject_on_PyPI?= Message-ID: <3cVNtP1Xgrz7Llj@mail.python.org> http://hg.python.org/peps/rev/5e0175ae7ae0 changeset: 5096:5e0175ae7ae0 user: Victor Stinner date: Wed Sep 04 14:01:56 2013 +0200 summary: PEP 454: add link to the project on PyPI files: pep-0454.txt | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -420,10 +420,12 @@ Links ===== -Python issues: +tracemalloc: * `#18874: Add a new tracemalloc module to trace Python memory allocations `_ +* `pytracemalloc on PyPI + `_ Similar projects: -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Wed Sep 4 19:05:46 2013 From: python-checkins at python.org (charles-francois.natali) Date: Wed, 4 Sep 2013 19:05:46 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2316853=3A_Add_new_?= =?utf-8?q?selectors_module=2E?= Message-ID: <3cVWd7003Mz7Lnp@mail.python.org> http://hg.python.org/cpython/rev/e4d45315c38c changeset: 85523:e4d45315c38c user: Charles-Fran?ois Natali date: Wed Sep 04 19:02:49 2013 +0200 summary: Issue #16853: Add new selectors module. files: Doc/library/concurrency.rst | 1 + Doc/library/selectors.rst | 231 +++++++++++++ Doc/whatsnew/3.4.rst | 7 +- Lib/selectors.py | 405 ++++++++++++++++++++++++ Lib/test/test_selectors.py | 390 +++++++++++++++++++++++ Misc/NEWS | 2 + 6 files changed, 1033 insertions(+), 3 deletions(-) diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -21,6 +21,7 @@ sched.rst queue.rst select.rst + selectors.rst The following are support modules for some of the above services: diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst new file mode 100644 --- /dev/null +++ b/Doc/library/selectors.rst @@ -0,0 +1,231 @@ +:mod:`selectors` -- High-level I/O multiplexing +=============================================== + +.. module:: selectors + :synopsis: High-level I/O multiplexing. + +.. versionadded:: 3.4 + + +Introduction +------------ + +This module allows high-level and efficient I/O multiplexing, built upon the +:mod:`select` module primitives. Users are encouraged to use this module +instead, unless they want precise control over the OS-level primitives used. + +It defines a :class:`BaseSelector` abstract base class, along with several +concrete implementations (:class:`KqueueSelector`, :class:`EpollSelector`...), +that can be used to wait for I/O readiness notification on multiple file +objects. In the following, "file object" refers to any object with a +:meth:`fileno()` method, or a raw file descriptor. See :term:`file object`. + +:class:`DefaultSelector` is an alias to the most efficient implementation +available on the current platform: this should be the default choice for most +users. + +.. note:: + The type of file objects supported depends on the platform: on Windows, + sockets are supported, but not pipes, whereas on Unix, both are supported + (some other types may be supported as well, such as fifos or special file + devices). + +.. seealso:: + + :mod:`select` + Low-level I/O multiplexing module. + + +Classes +------- + +Classes hierarchy:: + + BaseSelector + +-- SelectSelector + +-- PollSelector + +-- EpollSelector + +-- KqueueSelector + + +In the following, *events* is a bitwise mask indicating which I/O events should +be waited for on a given file object. It can be a combination of the constants +below: + + +-----------------------+-----------------------------------------------+ + | Constant | Meaning | + +=======================+===============================================+ + | :const:`EVENT_READ` | Available for read | + +-----------------------+-----------------------------------------------+ + | :const:`EVENT_WRITE` | Available for write | + +-----------------------+-----------------------------------------------+ + + +.. class:: SelectorKey + + A :class:`SelectorKey` is a :class:`~collections.namedtuple` used to + associate a file object to its underlying file decriptor, selected event + mask and attached data. It is returned by several :class:`BaseSelector` + methods. + + .. attribute:: fileobj + + File object registered. + + .. attribute:: fd + + Underlying file descriptor. + + .. attribute:: events + + Events that must be waited for this file object. + + .. attribute:: data + + Optional opaque data associated to this file object: for example, this + could be used to store per-client session. + + +.. class:: BaseSelector + + A :class:`BaseSelector` is used to wait for I/O event readiness on multiple + file objects. It supports file stream registration, unregistration, and a + method to wait for I/O events on those streams, with an optional timeout. + It's an abstract base class, so cannot be instantiated. Use + :class:`DefaultSelector` instead, or one of :class:`SelectSelector`, + :class:`KqueueSelector` etc. if you want to specifically use an + implementation, and your platform supports it. + :class:`BaseSelector` and its concrete implementations support the + :term:`context manager` protocol. + + .. method:: register(fileobj, events, data=None) + + Register a file object for selection, monitoring it for I/O events. + + *fileobj* is the file object to monitor. + *events* is a bitwise mask of events to monitor. + *data* is an opaque object. + + This returns a new :class:`SelectorKey` instance, or raises a + :exc:`ValueError` in case of invalid event mask or file descriptor, or + :exc:`KeyError` if the file object is already registered. + + .. method:: unregister(fileobj) + + Unregister a file object from selection, removing it from monitoring. A + file object shall be unregistered prior to being closed. + + *fileobj* must be a file object previously registered. + + This returns the associated :class:`SelectorKey` instance, or raises a + :exc:`KeyError` if the file object is not registered. + + .. method:: modify(fileobj, events, data=None) + + Change a registered file object monitored events or attached data. + + This is equivalent to :meth:`BaseSelector.unregister(fileobj)` followed + by :meth:`BaseSelector.register(fileobj, events, data)`, except that it + can be implemented more efficiently. + + This returns a new :class:`SelectorKey` instance, or raises a + :exc:`ValueError` in case of invalid event mask or file descriptor, or + :exc:`KeyError` if the file object is not registered. + + .. method:: select(timeout=None) + + Wait until some registered file objects become ready, or the timeout + expires. + + If ``timeout > 0``, this specifies the maximum wait time, in seconds. + If ``timeout <= 0``, the call won't block, and will report the currently + ready file objects. + If *timeout* is ``None``, the call will block until a monitored file object + becomes ready. + + This returns a list of ``(key, events)`` tuple, one for each ready file + object. + + *key* is the :class:`SelectorKey` instance corresponding to a ready file + object. + *events* is a bitmask of events ready on this file object. + + .. method:: close() + + Close the selector. + + This must be called to make sure that any underlying resource is freed. + The selector shall not be used once it has been closed. + + .. method:: get_key(fileobj) + + Return the key associated to a registered file object. + + This returns the :class:`SelectorKey` instance associated to this file + object, or raises :exc:`KeyError` if the file object is not registered. + + +.. class:: DefaultSelector() + + The default selector class, using the most efficient implementation + available on the current platform. This should be the default choice for + most users. + + +.. class:: SelectSelector() + + :func:`select.select`-based selector. + + +.. class:: PollSelector() + + :func:`select.poll`-based selector. + + +.. class:: EpollSelector() + + :func:`select.epoll`-based selector. + + .. method:: fileno() + + This returns the file descriptor used by the underlying + :func:`select.epoll` object. + + +.. class:: KqueueSelector() + + :func:`select.kqueue`-based selector. + + .. method:: fileno() + + This returns the file descriptor used by the underlying + :func:`select.kqueue` object. + + +Examples of selector usage:: + + >>> import selectors + >>> import socket + >>> + >>> s = selectors.DefaultSelector() + >>> r, w = socket.socketpair() + >>> + >>> s.register(r, selectors.EVENT_READ) + SelectorKey(fileobj=, fd=4, events=1, data=None) + >>> s.register(w, selectors.EVENT_WRITE) + SelectorKey(fileobj=, fd=5, events=2, data=None) + >>> + >>> print(s.select()) + [(SelectorKey(fileobj=, fd=5, events=2, data=None), 2)] + >>> + >>> for key, events in s.select(): + ... if events & selectors.EVENT_WRITE: + ... key.fileobj.send(b'spam') + ... + 4 + >>> for key, events in s.select(): + ... if events & selectors.EVENT_READ: + ... print(key.fileobj.recv(1024)) + ... + b'spam' + >>> s.close() diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -174,10 +174,11 @@ New Modules =========== -.. module name -.. ----------- +selectors +--------- -* None yet. +The new :mod:`selectors` module allows high-level and efficient I/O +multiplexing, built upon the :mod:`select` module primitives. Improved Modules diff --git a/Lib/selectors.py b/Lib/selectors.py new file mode 100644 --- /dev/null +++ b/Lib/selectors.py @@ -0,0 +1,405 @@ +"""Selectors module. + +This module allows high-level and efficient I/O multiplexing, built upon the +`select` module primitives. +""" + + +from abc import ABCMeta, abstractmethod +from collections import namedtuple +import functools +import select +import sys + + +# generic events, that must be mapped to implementation-specific ones +EVENT_READ = (1 << 0) +EVENT_WRITE = (1 << 1) + + +def _fileobj_to_fd(fileobj): + """Return a file descriptor from a file object. + + Parameters: + fileobj -- file object or file descriptor + + Returns: + corresponding file descriptor + """ + if isinstance(fileobj, int): + fd = fileobj + else: + try: + fd = int(fileobj.fileno()) + except (AttributeError, TypeError, ValueError): + raise ValueError("Invalid file object: " + "{!r}".format(fileobj)) from None + if fd < 0: + raise ValueError("Invalid file descriptor: {}".format(fd)) + return fd + + +SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) +"""Object used to associate a file object to its backing file descriptor, +selected event mask and attached data.""" + + +class BaseSelector(metaclass=ABCMeta): + """Base selector class. + + A selector supports registering file objects to be monitored for specific + I/O events. + + A file object is a file descriptor or any object with a `fileno()` method. + An arbitrary object can be attached to the file object, which can be used + for example to store context information, a callback, etc. + + A selector can use various implementations (select(), poll(), epoll()...) + depending on the platform. The default `Selector` class uses the most + performant implementation on the current platform. + """ + + def __init__(self): + # this maps file descriptors to keys + self._fd_to_key = {} + + def register(self, fileobj, events, data=None): + """Register a file object. + + Parameters: + fileobj -- file object or file descriptor + events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE) + data -- attached data + + Returns: + SelectorKey instance + """ + if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)): + raise ValueError("Invalid events: {!r}".format(events)) + + key = SelectorKey(fileobj, _fileobj_to_fd(fileobj), events, data) + + if key.fd in self._fd_to_key: + raise KeyError("{!r} (FD {}) is already " + "registered".format(fileobj, key.fd)) + + self._fd_to_key[key.fd] = key + return key + + def unregister(self, fileobj): + """Unregister a file object. + + Parameters: + fileobj -- file object or file descriptor + + Returns: + SelectorKey instance + """ + try: + key = self._fd_to_key.pop(_fileobj_to_fd(fileobj)) + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + return key + + def modify(self, fileobj, events, data=None): + """Change a registered file object monitored events or attached data. + + Parameters: + fileobj -- file object or file descriptor + events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE) + data -- attached data + + Returns: + SelectorKey instance + """ + # TODO: Subclasses can probably optimize this even further. + try: + key = self._fd_to_key[_fileobj_to_fd(fileobj)] + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + if events != key.events or data != key.data: + # TODO: If only the data changed, use a shortcut that only + # updates the data. + self.unregister(fileobj) + return self.register(fileobj, events, data) + else: + return key + + @abstractmethod + def select(self, timeout=None): + """Perform the actual selection, until some monitored file objects are + ready or a timeout expires. + + Parameters: + timeout -- if timeout > 0, this specifies the maximum wait time, in + seconds + if timeout <= 0, the select() call won't block, and will + report the currently ready file objects + if timeout is None, select() will block until a monitored + file object becomes ready + + Returns: + list of (key, events) for ready file objects + `events` is a bitwise mask of EVENT_READ|EVENT_WRITE + """ + raise NotImplementedError() + + def close(self): + """Close the selector. + + This must be called to make sure that any underlying resource is freed. + """ + self._fd_to_key.clear() + + def get_key(self, fileobj): + """Return the key associated to a registered file object. + + Returns: + SelectorKey for this file object + """ + try: + return self._fd_to_key[_fileobj_to_fd(fileobj)] + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def _key_from_fd(self, fd): + """Return the key associated to a given file descriptor. + + Parameters: + fd -- file descriptor + + Returns: + corresponding key, or None if not found + """ + try: + return self._fd_to_key[fd] + except KeyError: + return None + + +class SelectSelector(BaseSelector): + """Select-based selector.""" + + def __init__(self): + super().__init__() + self._readers = set() + self._writers = set() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + if events & EVENT_READ: + self._readers.add(key.fd) + if events & EVENT_WRITE: + self._writers.add(key.fd) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + self._readers.discard(key.fd) + self._writers.discard(key.fd) + return key + + if sys.platform == 'win32': + def _select(self, r, w, _, timeout=None): + r, w, x = select.select(r, w, w, timeout) + return r, w + x, [] + else: + _select = select.select + + def select(self, timeout=None): + timeout = None if timeout is None else max(timeout, 0) + ready = [] + try: + r, w, _ = self._select(self._readers, self._writers, [], timeout) + except InterruptedError: + return ready + r = set(r) + w = set(w) + for fd in r | w: + events = 0 + if fd in r: + events |= EVENT_READ + if fd in w: + events |= EVENT_WRITE + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + +if hasattr(select, 'poll'): + + class PollSelector(BaseSelector): + """Poll-based selector.""" + + def __init__(self): + super().__init__() + self._poll = select.poll() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + poll_events = 0 + if events & EVENT_READ: + poll_events |= select.POLLIN + if events & EVENT_WRITE: + poll_events |= select.POLLOUT + self._poll.register(key.fd, poll_events) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + self._poll.unregister(key.fd) + return key + + def select(self, timeout=None): + timeout = None if timeout is None else max(int(1000 * timeout), 0) + ready = [] + try: + fd_event_list = self._poll.poll(timeout) + except InterruptedError: + return ready + for fd, event in fd_event_list: + events = 0 + if event & ~select.POLLIN: + events |= EVENT_WRITE + if event & ~select.POLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + +if hasattr(select, 'epoll'): + + class EpollSelector(BaseSelector): + """Epoll-based selector.""" + + def __init__(self): + super().__init__() + self._epoll = select.epoll() + + def fileno(self): + return self._epoll.fileno() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + epoll_events = 0 + if events & EVENT_READ: + epoll_events |= select.EPOLLIN + if events & EVENT_WRITE: + epoll_events |= select.EPOLLOUT + self._epoll.register(key.fd, epoll_events) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + self._epoll.unregister(key.fd) + return key + + def select(self, timeout=None): + timeout = -1 if timeout is None else max(timeout, 0) + max_ev = len(self._fd_to_key) + ready = [] + try: + fd_event_list = self._epoll.poll(timeout, max_ev) + except InterruptedError: + return ready + for fd, event in fd_event_list: + events = 0 + if event & ~select.EPOLLIN: + events |= EVENT_WRITE + if event & ~select.EPOLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + def close(self): + super().close() + self._epoll.close() + + +if hasattr(select, 'kqueue'): + + class KqueueSelector(BaseSelector): + """Kqueue-based selector.""" + + def __init__(self): + super().__init__() + self._kqueue = select.kqueue() + + def fileno(self): + return self._kqueue.fileno() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + if events & EVENT_READ: + kev = select.kevent(key.fd, select.KQ_FILTER_READ, + select.KQ_EV_ADD) + self._kqueue.control([kev], 0, 0) + if events & EVENT_WRITE: + kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, + select.KQ_EV_ADD) + self._kqueue.control([kev], 0, 0) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + if key.events & EVENT_READ: + kev = select.kevent(key.fd, select.KQ_FILTER_READ, + select.KQ_EV_DELETE) + self._kqueue.control([kev], 0, 0) + if key.events & EVENT_WRITE: + kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, + select.KQ_EV_DELETE) + self._kqueue.control([kev], 0, 0) + return key + + def select(self, timeout=None): + timeout = None if timeout is None else max(timeout, 0) + max_ev = len(self._fd_to_key) + ready = [] + try: + kev_list = self._kqueue.control(None, max_ev, timeout) + except InterruptedError: + return ready + for kev in kev_list: + fd = kev.ident + flag = kev.filter + events = 0 + if flag == select.KQ_FILTER_READ: + events |= EVENT_READ + if flag == select.KQ_FILTER_WRITE: + events |= EVENT_WRITE + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + def close(self): + super().close() + self._kqueue.close() + + +# Choose the best implementation: roughly, epoll|kqueue > poll > select. +# select() also can't accept a FD > FD_SETSIZE (usually around 1024) +if 'KqueueSelector' in globals(): + DefaultSelector = KqueueSelector +elif 'EpollSelector' in globals(): + DefaultSelector = EpollSelector +elif 'PollSelector' in globals(): + DefaultSelector = PollSelector +else: + DefaultSelector = SelectSelector diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py new file mode 100644 --- /dev/null +++ b/Lib/test/test_selectors.py @@ -0,0 +1,390 @@ +import errno +import random +import selectors +import signal +import socket +from test import support +from time import sleep +import unittest +try: + from time import monotonic as time +except ImportError: + from time import time as time +try: + import resource +except ImportError: + resource = None + + +if hasattr(socket, 'socketpair'): + socketpair = socket.socketpair +else: + def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): + with socket.socket(family, type, proto) as l: + l.bind((support.HOST, 0)) + l.listen(3) + c = socket.socket(family, type, proto) + try: + c.connect(l.getsockname()) + caddr = c.getsockname() + while True: + a, addr = l.accept() + # check that we've got the correct client + if addr == caddr: + return c, a + a.close() + except OSError: + c.close() + raise + + +def find_ready_matching(ready, flag): + match = [] + for key, events in ready: + if events & flag: + match.append(key.fileobj) + return match + + +class BaseSelectorTestCase(unittest.TestCase): + + def test_register(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertIsInstance(key, selectors.SelectorKey) + self.assertEqual(key.fileobj, rd) + self.assertEqual(key.fd, rd.fileno()) + self.assertEqual(key.events, selectors.EVENT_READ) + self.assertEqual(key.data, "data") + + # register an unknown event + self.assertRaises(ValueError, s.register, 0, 999999) + + # register an invalid FD + self.assertRaises(ValueError, s.register, -10, selectors.EVENT_READ) + + # register twice + self.assertRaises(KeyError, s.register, rd, selectors.EVENT_READ) + + # register the same FD, but with a different object + self.assertRaises(KeyError, s.register, rd.fileno(), + selectors.EVENT_READ) + + def test_unregister(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + s.register(rd, selectors.EVENT_READ) + s.unregister(rd) + + # unregister an unknown file obj + self.assertRaises(KeyError, s.unregister, 999999) + + # unregister twice + self.assertRaises(KeyError, s.unregister, rd) + + def test_modify(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + key = s.register(rd, selectors.EVENT_READ) + + # modify events + key2 = s.modify(rd, selectors.EVENT_WRITE) + self.assertNotEqual(key.events, key2.events) + self.assertEqual(key2, s.get_key(rd)) + + s.unregister(rd) + + # modify data + d1 = object() + d2 = object() + + key = s.register(rd, selectors.EVENT_READ, d1) + key2 = s.modify(rd, selectors.EVENT_READ, d2) + self.assertEqual(key.events, key2.events) + self.assertNotEqual(key.data, key2.data) + self.assertEqual(key2, s.get_key(rd)) + self.assertEqual(key2.data, d2) + + # modify unknown file obj + self.assertRaises(KeyError, s.modify, 999999, selectors.EVENT_READ) + + def test_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + + s.close() + self.assertRaises(KeyError, s.get_key, rd) + self.assertRaises(KeyError, s.get_key, wr) + + def test_get_key(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertEqual(key, s.get_key(rd)) + + # unknown file obj + self.assertRaises(KeyError, s.get_key, 999999) + + def test_select(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + s.register(rd, selectors.EVENT_READ) + wr_key = s.register(wr, selectors.EVENT_WRITE) + + result = s.select() + for key, events in result: + self.assertTrue(isinstance(key, selectors.SelectorKey)) + self.assertTrue(events) + self.assertFalse(events & ~(selectors.EVENT_READ | + selectors.EVENT_WRITE)) + + self.assertEqual([(wr_key, selectors.EVENT_WRITE)], result) + + def test_context_manager(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + with s as sel: + sel.register(rd, selectors.EVENT_READ) + sel.register(wr, selectors.EVENT_WRITE) + + self.assertRaises(KeyError, s.get_key, rd) + self.assertRaises(KeyError, s.get_key, wr) + + def test_fileno(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + if hasattr(s, 'fileno'): + fd = s.fileno() + self.assertTrue(isinstance(fd, int)) + self.assertGreaterEqual(fd, 0) + + def test_selector(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + NUM_SOCKETS = 12 + MSG = b" This is a test." + MSG_LEN = len(MSG) + readers = [] + writers = [] + r2w = {} + w2r = {} + + for i in range(NUM_SOCKETS): + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + readers.append(rd) + writers.append(wr) + r2w[rd] = wr + w2r[wr] = rd + + bufs = [] + + while writers: + ready = s.select() + ready_writers = find_ready_matching(ready, selectors.EVENT_WRITE) + if not ready_writers: + self.fail("no sockets ready for writing") + wr = random.choice(ready_writers) + wr.send(MSG) + + for i in range(10): + ready = s.select() + ready_readers = find_ready_matching(ready, + selectors.EVENT_READ) + if ready_readers: + break + # there might be a delay between the write to the write end and + # the read end is reported ready + sleep(0.1) + else: + self.fail("no sockets ready for reading") + self.assertEqual([w2r[wr]], ready_readers) + rd = ready_readers[0] + buf = rd.recv(MSG_LEN) + self.assertEqual(len(buf), MSG_LEN) + bufs.append(buf) + s.unregister(r2w[rd]) + s.unregister(rd) + writers.remove(r2w[rd]) + + self.assertEqual(bufs, [MSG] * NUM_SOCKETS) + + def test_timeout(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + s.register(wr, selectors.EVENT_WRITE) + t = time() + self.assertEqual(1, len(s.select(0))) + self.assertEqual(1, len(s.select(-1))) + self.assertTrue(time() - t < 0.5) + + s.unregister(wr) + s.register(rd, selectors.EVENT_READ) + t = time() + self.assertFalse(s.select(0)) + self.assertFalse(s.select(-1)) + self.assertTrue(time() - t < 0.5) + + t = time() + self.assertFalse(s.select(1)) + self.assertTrue(0.5 < time() - t < 1.5) + + @unittest.skipUnless(hasattr(signal, "alarm"), + "signal.alarm() required for this test") + def test_interrupted_retry(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + orig_alrm_handler = signal.signal(signal.SIGALRM, lambda *args: None) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(signal.alarm, 0) + + signal.alarm(1) + + s.register(rd, selectors.EVENT_READ) + t = time() + self.assertFalse(s.select(2)) + self.assertLess(time() - t, 2.5) + + +class ScalableSelectorMixIn: + + @support.requires_mac_ver(10, 5) + @unittest.skipUnless(resource, "Test needs resource module") + def test_above_fd_setsize(self): + # A scalable implementation should have no problem with more than + # FD_SETSIZE file descriptors. Since we don't know the value, we just + # try to set the soft RLIMIT_NOFILE to the hard RLIMIT_NOFILE ceiling. + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) + self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, + (soft, hard)) + NUM_FDS = hard + except OSError: + NUM_FDS = soft + + # guard for already allocated FDs (stdin, stdout...) + NUM_FDS -= 32 + + s = self.SELECTOR() + self.addCleanup(s.close) + + for i in range(NUM_FDS // 2): + try: + rd, wr = socketpair() + except OSError: + # too many FDs, skip - note that we should only catch EMFILE + # here, but apparently *BSD and Solaris can fail upon connect() + # or bind() with EADDRNOTAVAIL, so let's be safe + self.skipTest("FD limit reached") + + self.addCleanup(rd.close) + self.addCleanup(wr.close) + + try: + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + except OSError as e: + if e.errno == errno.ENOSPC: + # this can be raised by epoll if we go over + # fs.epoll.max_user_watches sysctl + self.skipTest("FD limit reached") + raise + + self.assertEqual(NUM_FDS // 2, len(s.select())) + + +class DefaultSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = selectors.DefaultSelector + + +class SelectSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = selectors.SelectSelector + + + at unittest.skipUnless(hasattr(selectors, 'PollSelector'), + "Test needs selectors.PollSelector") +class PollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'PollSelector', None) + + + at unittest.skipUnless(hasattr(selectors, 'EpollSelector'), + "Test needs selectors.EpollSelector") +class EpollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'EpollSelector', None) + + + at unittest.skipUnless(hasattr(selectors, 'KqueueSelector'), + "Test needs selectors.KqueueSelector)") +class KqueueSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'KqueueSelector', None) + + +def test_main(): + tests = [DefaultSelectorTestCase, SelectSelectorTestCase, + PollSelectorTestCase, EpollSelectorTestCase, + KqueueSelectorTestCase] + support.run_unittest(*tests) + support.reap_children() + + +if __name__ == "__main__": + test_main() diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,6 +54,8 @@ Library ------- +- Issue #16853: Add new selectors module. + - Issue #18882: Add threading.main_thread() function. - Issue #18901: The sunau getparams method now returns a namedtuple rather than -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 20:30:49 2013 From: python-checkins at python.org (victor.stinner) Date: Wed, 4 Sep 2013 20:30:49 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_test?= Message-ID: <3cVYWF3GvBz7LpY@mail.python.org> http://hg.python.org/cpython/rev/8ff1173ca831 changeset: 85524:8ff1173ca831 user: Victor Stinner date: Wed Sep 04 20:24:32 2013 +0200 summary: test files: README | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/README b/README --- a/README +++ b/README @@ -1,3 +1,4 @@ +test This is Python version 3.4.0 alpha 1 ==================================== -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 20:30:50 2013 From: python-checkins at python.org (victor.stinner) Date: Wed, 4 Sep 2013 20:30:50 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2316853=3A_Mention_?= =?utf-8?q?the_new_selectors_module_in_What=27s_New_in_Python_3=2E4?= Message-ID: <3cVYWG5Bkhz7Lm8@mail.python.org> http://hg.python.org/cpython/rev/6b14ebe0f7ac changeset: 85525:6b14ebe0f7ac user: Victor Stinner date: Wed Sep 04 20:30:34 2013 +0200 summary: Issue #16853: Mention the new selectors module in What's New in Python 3.4 files: Doc/whatsnew/3.4.rst | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -91,6 +91,8 @@ New library modules: * :mod:`enum`: Implementation of the :pep:`435`. +* :mod:`selectors`: High-level and efficient I/O multiplexing, built upon the + :mod:`select` module primitives. New built-in features: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 20:35:09 2013 From: python-checkins at python.org (victor.stinner) Date: Wed, 4 Sep 2013 20:35:09 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_oops=2C_revert_test_commit?= Message-ID: <3cVYcF03khz7LmJ@mail.python.org> http://hg.python.org/cpython/rev/b76229620487 changeset: 85526:b76229620487 user: Victor Stinner date: Wed Sep 04 20:34:52 2013 +0200 summary: oops, revert test commit files: README | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/README b/README --- a/README +++ b/README @@ -1,4 +1,3 @@ -test This is Python version 3.4.0 alpha 1 ==================================== -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 20:40:28 2013 From: python-checkins at python.org (victor.stinner) Date: Wed, 4 Sep 2013 20:40:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2316853=3A_Mention_?= =?utf-8?q?the_new_selectors_module_in_the_select_module?= Message-ID: <3cVYkN2dSyzSDD@mail.python.org> http://hg.python.org/cpython/rev/142ff216e4b4 changeset: 85527:142ff216e4b4 user: Victor Stinner date: Wed Sep 04 20:40:13 2013 +0200 summary: Issue #16853: Mention the new selectors module in the select module files: Doc/library/select.rst | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diff --git a/Doc/library/select.rst b/Doc/library/select.rst --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -14,6 +14,14 @@ It cannot be used on regular files to determine whether a file has grown since it was last read. +.. note:: + + The :mod:`selectors` module allows high-level and efficient I/O + multiplexing, built upon the :mod:`select` module primitives. Users are + encouraged to use the :mod:`selectors` module instead, unless they want + precise control over the OS-level primitives used. + + The module defines the following: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 20:58:07 2013 From: python-checkins at python.org (antoine.pitrou) Date: Wed, 4 Sep 2013 20:58:07 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4ODc2?= =?utf-8?q?=3A_The_FileIO=2Emode_attribute_now_better_reflects_the_actual_?= =?utf-8?q?mode?= Message-ID: <3cVZ6l0pTfz7Lk9@mail.python.org> http://hg.python.org/cpython/rev/33bd39b67cc1 changeset: 85528:33bd39b67cc1 branch: 3.3 parent: 85518:7801ef4a4ce3 user: Antoine Pitrou date: Wed Sep 04 20:46:33 2013 +0200 summary: Issue #18876: The FileIO.mode attribute now better reflects the actual mode under which the file was opened. Patch by Erik Bray. files: Lib/test/test_fileio.py | 17 ++++++++++++++++- Misc/NEWS | 3 +++ Modules/_io/fileio.c | 23 +++++++++++++---------- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -304,7 +304,7 @@ finally: os.unlink(TESTFN) - def testModeStrings(self): + def testInvalidModeStrings(self): # check invalid mode strings for mode in ("", "aU", "wU+", "rw", "rt"): try: @@ -315,6 +315,21 @@ f.close() self.fail('%r is an invalid file mode' % mode) + def testModeStrings(self): + # test that the mode attribute is correct for various mode strings + # given as init args + try: + for modes in [('w', 'wb'), ('wb', 'wb'), ('wb+', 'rb+'), + ('w+b', 'rb+'), ('a', 'ab'), ('ab', 'ab'), + ('ab+', 'ab+'), ('a+b', 'ab+'), ('r', 'rb'), + ('rb', 'rb'), ('rb+', 'rb+'), ('r+b', 'rb+')]: + # read modes are last so that TESTFN will exist first + with _FileIO(TESTFN, modes[0]) as f: + self.assertEqual(f.mode, modes[1]) + finally: + if os.path.exists(TESTFN): + os.unlink(TESTFN) + def testUnicodeOpen(self): # verify repr works for unicode too f = _FileIO(str(TESTFN), "w") diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -66,6 +66,9 @@ Library ------- +- Issue #18876: The FileIO.mode attribute now better reflects the actual mode + under which the file was opened. Patch by Erik Bray. + - Issue #18418: After fork(), reinit all threads states, not only active ones. Patch by A. Jesse Jiryu Davis. diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -49,6 +49,7 @@ unsigned int created : 1; unsigned int readable : 1; unsigned int writable : 1; + unsigned int appending : 1; signed int seekable : 2; /* -1 means unknown */ unsigned int closefd : 1; unsigned int deallocating: 1; @@ -156,6 +157,7 @@ self->created = 0; self->readable = 0; self->writable = 0; + self->appending = 0; self->seekable = -1; self->closefd = 1; self->weakreflist = NULL; @@ -216,7 +218,7 @@ Py_UNICODE *widename = NULL; #endif int ret = 0; - int rwa = 0, plus = 0, append = 0; + int rwa = 0, plus = 0; int flags = 0; int fd = -1; int closefd = 1; @@ -309,8 +311,8 @@ goto bad_mode; rwa = 1; self->writable = 1; - flags |= O_CREAT; - append = 1; + self->appending = 1; + flags |= O_APPEND | O_CREAT; break; case 'b': break; @@ -341,11 +343,6 @@ flags |= O_BINARY; #endif -#ifdef O_APPEND - if (append) - flags |= O_APPEND; -#endif - if (fd >= 0) { if (check_fd(fd)) goto error; @@ -411,7 +408,7 @@ if (PyObject_SetAttrString((PyObject *)self, "name", nameobj) < 0) goto error; - if (append) { + if (self->appending) { /* For consistent behaviour, we explicitly seek to the end of file (otherwise, it might be done only on the first write()). */ @@ -1012,7 +1009,13 @@ else return "xb"; } - if (self->readable) { + if (self->appending) { + if (self->readable) + return "ab+"; + else + return "ab"; + } + else if (self->readable) { if (self->writable) return "rb+"; else -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 4 20:58:09 2013 From: python-checkins at python.org (antoine.pitrou) Date: Wed, 4 Sep 2013 20:58:09 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2318876=3A_The_FileIO=2Emode_attribute_now_better?= =?utf-8?q?_reflects_the_actual_mode?= Message-ID: <3cVZ6n6HMmz7Lk9@mail.python.org> http://hg.python.org/cpython/rev/b5530669ef70 changeset: 85529:b5530669ef70 parent: 85527:142ff216e4b4 parent: 85528:33bd39b67cc1 user: Antoine Pitrou date: Wed Sep 04 20:52:14 2013 +0200 summary: Issue #18876: The FileIO.mode attribute now better reflects the actual mode under which the file was opened. Patch by Erik Bray. files: Lib/test/test_fileio.py | 17 ++++++++++++++++- Misc/NEWS | 3 +++ Modules/_io/fileio.c | 22 +++++++++++++--------- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -304,7 +304,7 @@ finally: os.unlink(TESTFN) - def testModeStrings(self): + def testInvalidModeStrings(self): # check invalid mode strings for mode in ("", "aU", "wU+", "rw", "rt"): try: @@ -315,6 +315,21 @@ f.close() self.fail('%r is an invalid file mode' % mode) + def testModeStrings(self): + # test that the mode attribute is correct for various mode strings + # given as init args + try: + for modes in [('w', 'wb'), ('wb', 'wb'), ('wb+', 'rb+'), + ('w+b', 'rb+'), ('a', 'ab'), ('ab', 'ab'), + ('ab+', 'ab+'), ('a+b', 'ab+'), ('r', 'rb'), + ('rb', 'rb'), ('rb+', 'rb+'), ('r+b', 'rb+')]: + # read modes are last so that TESTFN will exist first + with _FileIO(TESTFN, modes[0]) as f: + self.assertEqual(f.mode, modes[1]) + finally: + if os.path.exists(TESTFN): + os.unlink(TESTFN) + def testUnicodeOpen(self): # verify repr works for unicode too f = _FileIO(str(TESTFN), "w") diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,6 +54,9 @@ Library ------- +- Issue #18876: The FileIO.mode attribute now better reflects the actual mode + under which the file was opened. Patch by Erik Bray. + - Issue #16853: Add new selectors module. - Issue #18882: Add threading.main_thread() function. diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -49,6 +49,7 @@ unsigned int created : 1; unsigned int readable : 1; unsigned int writable : 1; + unsigned int appending : 1; signed int seekable : 2; /* -1 means unknown */ unsigned int closefd : 1; char finalizing; @@ -156,6 +157,7 @@ self->created = 0; self->readable = 0; self->writable = 0; + self->appending = 0; self->seekable = -1; self->closefd = 1; self->weakreflist = NULL; @@ -219,7 +221,7 @@ Py_UNICODE *widename = NULL; #endif int ret = 0; - int rwa = 0, plus = 0, append = 0; + int rwa = 0, plus = 0; int flags = 0; int fd = -1; int closefd = 1; @@ -317,8 +319,8 @@ goto bad_mode; rwa = 1; self->writable = 1; - flags |= O_CREAT; - append = 1; + self->appending = 1; + flags |= O_APPEND | O_CREAT; break; case 'b': break; @@ -349,10 +351,6 @@ flags |= O_BINARY; #endif -#ifdef O_APPEND - if (append) - flags |= O_APPEND; -#endif #ifdef MS_WINDOWS flags |= O_NOINHERIT; #elif defined(O_CLOEXEC) @@ -432,7 +430,7 @@ if (PyObject_SetAttrString((PyObject *)self, "name", nameobj) < 0) goto error; - if (append) { + if (self->appending) { /* For consistent behaviour, we explicitly seek to the end of file (otherwise, it might be done only on the first write()). */ @@ -1019,7 +1017,13 @@ else return "xb"; } - if (self->readable) { + if (self->appending) { + if (self->readable) + return "ab+"; + else + return "ab"; + } + else if (self->readable) { if (self->writable) return "rb+"; else -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 00:03:50 2013 From: python-checkins at python.org (antoine.pitrou) Date: Thu, 5 Sep 2013 00:03:50 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4ODc2?= =?utf-8?q?=3A_The_FileIO=2Emode_attribute_now_better_reflects_the_actual_?= =?utf-8?q?mode?= Message-ID: <3cVfF24Bhpz7Ljk@mail.python.org> http://hg.python.org/cpython/rev/fc2b88a27fa1 changeset: 85530:fc2b88a27fa1 branch: 2.7 parent: 85510:2a38df26e009 user: Antoine Pitrou date: Wed Sep 04 20:46:33 2013 +0200 summary: Issue #18876: The FileIO.mode attribute now better reflects the actual mode under which the file was opened. Patch by Erik Bray. files: Lib/test/test_fileio.py | 17 ++++++++++++++++- Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/_io/fileio.c | 23 +++++++++++++---------- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -305,7 +305,7 @@ finally: os.unlink(TESTFN) - def testModeStrings(self): + def testInvalidModeStrings(self): # check invalid mode strings for mode in ("", "aU", "wU+", "rw", "rt"): try: @@ -316,6 +316,21 @@ f.close() self.fail('%r is an invalid file mode' % mode) + def testModeStrings(self): + # test that the mode attribute is correct for various mode strings + # given as init args + try: + for modes in [('w', 'wb'), ('wb', 'wb'), ('wb+', 'rb+'), + ('w+b', 'rb+'), ('a', 'ab'), ('ab', 'ab'), + ('ab+', 'ab+'), ('a+b', 'ab+'), ('r', 'rb'), + ('rb', 'rb'), ('rb+', 'rb+'), ('r+b', 'rb+')]: + # read modes are last so that TESTFN will exist first + with _FileIO(TESTFN, modes[0]) as f: + self.assertEqual(f.mode, modes[1]) + finally: + if os.path.exists(TESTFN): + os.unlink(TESTFN) + def testUnicodeOpen(self): # verify repr works for unicode too f = _FileIO(str(TESTFN), "w") diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -124,6 +124,7 @@ Georg Brandl Christopher Brannon Terrence Brannon +Erik Bray Brian Brazil Dave Brennan Tom Bridgman diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Library ------- +- Issue #18876: The FileIO.mode attribute now better reflects the actual mode + under which the file was opened. Patch by Erik Bray. + - Issue #18851: Avoid a double close of subprocess pipes when the child process fails starting. diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -47,6 +47,7 @@ int fd; unsigned int readable : 1; unsigned int writable : 1; + unsigned int appending : 1; signed int seekable : 2; /* -1 means unknown */ unsigned int closefd : 1; PyObject *weakreflist; @@ -124,6 +125,7 @@ self->fd = -1; self->readable = 0; self->writable = 0; + self->appending = 0; self->seekable = -1; self->closefd = 1; self->weakreflist = NULL; @@ -184,7 +186,7 @@ Py_UNICODE *widename = NULL; #endif int ret = 0; - int rwa = 0, plus = 0, append = 0; + int rwa = 0, plus = 0; int flags = 0; int fd = -1; int closefd = 1; @@ -279,8 +281,8 @@ goto bad_mode; rwa = 1; self->writable = 1; - flags |= O_CREAT; - append = 1; + self->appending = 1; + flags |= O_APPEND | O_CREAT; break; case 'b': break; @@ -311,11 +313,6 @@ flags |= O_BINARY; #endif -#ifdef O_APPEND - if (append) - flags |= O_APPEND; -#endif - if (fd >= 0) { if (check_fd(fd)) goto error; @@ -356,7 +353,7 @@ if (PyObject_SetAttrString((PyObject *)self, "name", nameobj) < 0) goto error; - if (append) { + if (self->appending) { /* For consistent behaviour, we explicitly seek to the end of file (otherwise, it might be done only on the first write()). */ @@ -898,7 +895,13 @@ static char * mode_string(fileio *self) { - if (self->readable) { + if (self->appending) { + if (self->readable) + return "ab+"; + else + return "ab"; + } + else if (self->readable) { if (self->writable) return "rb+"; else -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 00:28:27 2013 From: python-checkins at python.org (victor.stinner) Date: Thu, 5 Sep 2013 00:28:27 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4OTA5?= =?utf-8?q?=3A_Fix_=5Ftkinter=2Etkapp=2Einterpaddr=28=29_on_Windows_64-bit?= =?utf-8?q?=2C_don=27t_cast?= Message-ID: <3cVfnR1RdMz7Ljb@mail.python.org> http://hg.python.org/cpython/rev/f01e06d26b41 changeset: 85531:f01e06d26b41 branch: 3.3 parent: 85528:33bd39b67cc1 user: Victor Stinner date: Thu Sep 05 00:22:24 2013 +0200 summary: Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast 64-bit pointer to long (32 bits). files: Misc/NEWS | 3 +++ Modules/_tkinter.c | 2 +- 2 files changed, 4 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -66,6 +66,9 @@ Library ------- +- Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast + 64-bit pointer to long (32 bits). + - Issue #18876: The FileIO.mode attribute now better reflects the actual mode under which the file was opened. Patch by Erik Bray. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -2688,7 +2688,7 @@ if (!PyArg_ParseTuple(args, ":interpaddr")) return NULL; - return PyLong_FromLong((long)Tkapp_Interp(self)); + return PyLong_FromVoidPtr(Tkapp_Interp(self)); } static PyObject * -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 00:28:28 2013 From: python-checkins at python.org (victor.stinner) Date: Thu, 5 Sep 2013 00:28:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?b?KTogKE1lcmdlIDMuMykgSXNzdWUgIzE4OTA5OiBGaXggX3RraW50ZXIudGth?= =?utf-8?q?pp=2Einterpaddr=28=29_on_Windows_64-bit=2C?= Message-ID: <3cVfnS3Nkkz7Ljk@mail.python.org> http://hg.python.org/cpython/rev/ac27d979078a changeset: 85532:ac27d979078a parent: 85529:b5530669ef70 parent: 85531:f01e06d26b41 user: Victor Stinner date: Thu Sep 05 00:23:08 2013 +0200 summary: (Merge 3.3) Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast 64-bit pointer to long (32 bits). files: Misc/NEWS | 3 +++ Modules/_tkinter.c | 2 +- 2 files changed, 4 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,6 +54,9 @@ Library ------- +- Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast + 64-bit pointer to long (32 bits). + - Issue #18876: The FileIO.mode attribute now better reflects the actual mode under which the file was opened. Patch by Erik Bray. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -2462,7 +2462,7 @@ if (!PyArg_ParseTuple(args, ":interpaddr")) return NULL; - return PyLong_FromLong((long)Tkapp_Interp(self)); + return PyLong_FromVoidPtr(Tkapp_Interp(self)); } static PyObject * -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 00:28:29 2013 From: python-checkins at python.org (victor.stinner) Date: Thu, 5 Sep 2013 00:28:29 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4OTA5?= =?utf-8?q?=3A_Fix_=5Ftkinter=2Etkapp=2Einterpaddr=28=29_on_Windows_64-bit?= =?utf-8?q?=2C_don=27t_cast?= Message-ID: <3cVfnT5Hc8z7Lkf@mail.python.org> http://hg.python.org/cpython/rev/1a65bb15dedf changeset: 85533:1a65bb15dedf branch: 2.7 parent: 85530:fc2b88a27fa1 user: Victor Stinner date: Thu Sep 05 00:26:15 2013 +0200 summary: Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast 64-bit pointer to long (32 bits). files: Misc/NEWS | 3 +++ Modules/_tkinter.c | 2 +- 2 files changed, 4 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Library ------- +- Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast + 64-bit pointer to long (32 bits). + - Issue #18876: The FileIO.mode attribute now better reflects the actual mode under which the file was opened. Patch by Erik Bray. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -2814,7 +2814,7 @@ if (!PyArg_ParseTuple(args, ":interpaddr")) return NULL; - return PyInt_FromLong((long)Tkapp_Interp(self)); + return PyLong_FromVoidPtr(Tkapp_Interp(self)); } static PyObject * -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Thu Sep 5 06:17:33 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Thu, 05 Sep 2013 06:17:33 +0200 Subject: [Python-checkins] Daily reference leaks (ac27d979078a): sum=0 Message-ID: results for ac27d979078a on branch "default" -------------------------------------------- Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogHh74of', '-x'] From python-checkins at python.org Thu Sep 5 16:04:05 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Thu, 5 Sep 2013 16:04:05 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318878=3A_sunau=2E?= =?utf-8?q?open_now_supports_the_context_manager_protocol=2E__Based_on?= Message-ID: <3cW3Y157ZSz7LjQ@mail.python.org> http://hg.python.org/cpython/rev/a62f59667c9e changeset: 85534:a62f59667c9e parent: 85532:ac27d979078a user: Serhiy Storchaka date: Thu Sep 05 17:01:53 2013 +0300 summary: Issue #18878: sunau.open now supports the context manager protocol. Based on patches by Claudiu Popa and R. David Murray. files: Doc/whatsnew/3.4.rst | 2 + Lib/sunau.py | 31 +++++++++++++---- Lib/test/test_sunau.py | 51 +++++++++++++++++++++++++---- Misc/NEWS | 3 + 4 files changed, 72 insertions(+), 15 deletions(-) diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -374,6 +374,8 @@ The :meth:`~sunau.getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) +:meth:`sunau.open` now supports the context manager protocol (:issue:`18878`). + urllib ------ diff --git a/Lib/sunau.py b/Lib/sunau.py --- a/Lib/sunau.py +++ b/Lib/sunau.py @@ -168,6 +168,12 @@ if self._file: self.close() + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + def initfp(self, file): self._file = file self._soundpos = 0 @@ -303,6 +309,12 @@ self.close() self._file = None + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + def initfp(self, file): self._file = file self._framerate = 0 @@ -410,14 +422,17 @@ self._patchheader() def close(self): - self._ensure_header_written() - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten: - self._patchheader() - self._file.flush() - if self._opened and self._file: - self._file.close() - self._file = None + if self._file: + try: + self._ensure_header_written() + if self._nframeswritten != self._nframes or \ + self._datalength != self._datawritten: + self._patchheader() + self._file.flush() + finally: + if self._opened and self._file: + self._file.close() + self._file = None # # private methods diff --git a/Lib/test/test_sunau.py b/Lib/test/test_sunau.py --- a/Lib/test/test_sunau.py +++ b/Lib/test/test_sunau.py @@ -1,4 +1,4 @@ -from test.support import run_unittest, TESTFN +from test.support import TESTFN, unlink import unittest import pickle import os @@ -18,10 +18,7 @@ def tearDown(self): if self.f is not None: self.f.close() - try: - os.remove(TESTFN) - except OSError: - pass + unlink(TESTFN) def test_lin(self): self.f = sunau.open(TESTFN, 'w') @@ -84,9 +81,49 @@ dump = pickle.dumps(params) self.assertEqual(pickle.loads(dump), params) + def test_write_context_manager_calls_close(self): + # Close checks for a minimum header and will raise an error + # if it is not set, so this proves that close is called. + with self.assertRaises(sunau.Error): + with sunau.open(TESTFN, 'wb') as f: + pass + with self.assertRaises(sunau.Error): + with open(TESTFN, 'wb') as testfile: + with sunau.open(testfile): + pass -def test_main(): - run_unittest(SunAUTest) + def test_context_manager_with_open_file(self): + with open(TESTFN, 'wb') as testfile: + with sunau.open(testfile) as f: + f.setnchannels(nchannels) + f.setsampwidth(sampwidth) + f.setframerate(framerate) + self.assertFalse(testfile.closed) + with open(TESTFN, 'rb') as testfile: + with sunau.open(testfile) as f: + self.assertFalse(f.getfp().closed) + params = f.getparams() + self.assertEqual(params[0], nchannels) + self.assertEqual(params[1], sampwidth) + self.assertEqual(params[2], framerate) + self.assertIsNone(f.getfp()) + self.assertFalse(testfile.closed) + + def test_context_manager_with_filename(self): + # If the file doesn't get closed, this test won't fail, but it will + # produce a resource leak warning. + with sunau.open(TESTFN, 'wb') as f: + f.setnchannels(nchannels) + f.setsampwidth(sampwidth) + f.setframerate(framerate) + with sunau.open(TESTFN) as f: + self.assertFalse(f.getfp().closed) + params = f.getparams() + self.assertEqual(params[0], nchannels) + self.assertEqual(params[1], sampwidth) + self.assertEqual(params[2], framerate) + self.assertIsNone(f.getfp()) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,6 +54,9 @@ Library ------- +- Issue #18878: sunau.open now supports the context manager protocol. Based on + patches by Claudiu Popa and R. David Murray. + - Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast 64-bit pointer to long (32 bits). -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:07:00 2013 From: python-checkins at python.org (christian.heimes) Date: Thu, 5 Sep 2013 16:07:00 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4NzA5?= =?utf-8?q?=3A_GCC_4=2E6_complains_that_=27v=27_may_be_used_uninitialized_?= =?utf-8?q?in?= Message-ID: <3cW3cN4MsYz7Lks@mail.python.org> http://hg.python.org/cpython/rev/90040e560527 changeset: 85535:90040e560527 branch: 3.3 parent: 85531:f01e06d26b41 user: Christian Heimes date: Thu Sep 05 16:04:35 2013 +0200 summary: Issue #18709: GCC 4.6 complains that 'v' may be used uninitialized in GEN_EMAIL/GEN_URI/GEN_DNS case files: Modules/_ssl.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -721,7 +721,7 @@ int i, j; PyObject *peer_alt_names = Py_None; - PyObject *v, *t; + PyObject *v = NULL, *t; X509_EXTENSION *ext = NULL; GENERAL_NAMES *names = NULL; GENERAL_NAME *name; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:07:01 2013 From: python-checkins at python.org (christian.heimes) Date: Thu, 5 Sep 2013 16:07:01 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2318709=3A_GCC_4=2E6_complains_that_=27v=27_may_b?= =?utf-8?q?e_used_uninitialized_in?= Message-ID: <3cW3cP6trrz7Lkm@mail.python.org> http://hg.python.org/cpython/rev/4e93f32176fb changeset: 85536:4e93f32176fb parent: 85532:ac27d979078a parent: 85535:90040e560527 user: Christian Heimes date: Thu Sep 05 16:04:50 2013 +0200 summary: Issue #18709: GCC 4.6 complains that 'v' may be used uninitialized in GEN_EMAIL/GEN_URI/GEN_DNS case files: Modules/_ssl.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -755,7 +755,7 @@ int i, j; PyObject *peer_alt_names = Py_None; - PyObject *v, *t; + PyObject *v = NULL, *t; X509_EXTENSION *ext = NULL; GENERAL_NAMES *names = NULL; GENERAL_NAME *name; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:07:03 2013 From: python-checkins at python.org (christian.heimes) Date: Thu, 5 Sep 2013 16:07:03 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi42KTogSXNzdWUgIzE4NzA5?= =?utf-8?q?=3A_GCC_4=2E6_complains_that_=27v=27_may_be_used_uninitialized_?= =?utf-8?q?in?= Message-ID: <3cW3cR1zX5z7Lk4@mail.python.org> http://hg.python.org/cpython/rev/07ee48ce4513 changeset: 85537:07ee48ce4513 branch: 2.6 parent: 85506:25683ceaf341 user: Christian Heimes date: Thu Sep 05 16:04:35 2013 +0200 summary: Issue #18709: GCC 4.6 complains that 'v' may be used uninitialized in GEN_EMAIL/GEN_URI/GEN_DNS case files: Modules/_ssl.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -670,7 +670,7 @@ int i, j; PyObject *peer_alt_names = Py_None; - PyObject *v, *t; + PyObject *v = NULL, *t; X509_EXTENSION *ext = NULL; GENERAL_NAMES *names = NULL; GENERAL_NAME *name; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:07:04 2013 From: python-checkins at python.org (christian.heimes) Date: Thu, 5 Sep 2013 16:07:04 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMi42IC0+IDIuNyk6?= =?utf-8?q?_Issue_=2318709=3A_GCC_4=2E6_complains_that_=27v=27_may_be_used?= =?utf-8?q?_uninitialized_in?= Message-ID: <3cW3cS4mVfz7Lly@mail.python.org> http://hg.python.org/cpython/rev/a7d5b86ffb95 changeset: 85538:a7d5b86ffb95 branch: 2.7 parent: 85533:1a65bb15dedf parent: 85537:07ee48ce4513 user: Christian Heimes date: Thu Sep 05 16:05:50 2013 +0200 summary: Issue #18709: GCC 4.6 complains that 'v' may be used uninitialized in GEN_EMAIL/GEN_URI/GEN_DNS case files: Modules/_ssl.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -691,7 +691,7 @@ int i, j; PyObject *peer_alt_names = Py_None; - PyObject *v, *t; + PyObject *v = NULL, *t; X509_EXTENSION *ext = NULL; GENERAL_NAMES *names = NULL; GENERAL_NAME *name; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:07:06 2013 From: python-checkins at python.org (christian.heimes) Date: Thu, 5 Sep 2013 16:07:06 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?= =?utf-8?q?=29=3A_merge?= Message-ID: <3cW3cV1Hqsz7LkY@mail.python.org> http://hg.python.org/cpython/rev/cecec75c2250 changeset: 85539:cecec75c2250 parent: 85536:4e93f32176fb parent: 85534:a62f59667c9e user: Christian Heimes date: Thu Sep 05 16:06:46 2013 +0200 summary: merge files: Doc/whatsnew/3.4.rst | 2 + Lib/sunau.py | 31 +++++++++++++---- Lib/test/test_sunau.py | 51 +++++++++++++++++++++++++---- Misc/NEWS | 3 + 4 files changed, 72 insertions(+), 15 deletions(-) diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -374,6 +374,8 @@ The :meth:`~sunau.getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) +:meth:`sunau.open` now supports the context manager protocol (:issue:`18878`). + urllib ------ diff --git a/Lib/sunau.py b/Lib/sunau.py --- a/Lib/sunau.py +++ b/Lib/sunau.py @@ -168,6 +168,12 @@ if self._file: self.close() + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + def initfp(self, file): self._file = file self._soundpos = 0 @@ -303,6 +309,12 @@ self.close() self._file = None + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + def initfp(self, file): self._file = file self._framerate = 0 @@ -410,14 +422,17 @@ self._patchheader() def close(self): - self._ensure_header_written() - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten: - self._patchheader() - self._file.flush() - if self._opened and self._file: - self._file.close() - self._file = None + if self._file: + try: + self._ensure_header_written() + if self._nframeswritten != self._nframes or \ + self._datalength != self._datawritten: + self._patchheader() + self._file.flush() + finally: + if self._opened and self._file: + self._file.close() + self._file = None # # private methods diff --git a/Lib/test/test_sunau.py b/Lib/test/test_sunau.py --- a/Lib/test/test_sunau.py +++ b/Lib/test/test_sunau.py @@ -1,4 +1,4 @@ -from test.support import run_unittest, TESTFN +from test.support import TESTFN, unlink import unittest import pickle import os @@ -18,10 +18,7 @@ def tearDown(self): if self.f is not None: self.f.close() - try: - os.remove(TESTFN) - except OSError: - pass + unlink(TESTFN) def test_lin(self): self.f = sunau.open(TESTFN, 'w') @@ -84,9 +81,49 @@ dump = pickle.dumps(params) self.assertEqual(pickle.loads(dump), params) + def test_write_context_manager_calls_close(self): + # Close checks for a minimum header and will raise an error + # if it is not set, so this proves that close is called. + with self.assertRaises(sunau.Error): + with sunau.open(TESTFN, 'wb') as f: + pass + with self.assertRaises(sunau.Error): + with open(TESTFN, 'wb') as testfile: + with sunau.open(testfile): + pass -def test_main(): - run_unittest(SunAUTest) + def test_context_manager_with_open_file(self): + with open(TESTFN, 'wb') as testfile: + with sunau.open(testfile) as f: + f.setnchannels(nchannels) + f.setsampwidth(sampwidth) + f.setframerate(framerate) + self.assertFalse(testfile.closed) + with open(TESTFN, 'rb') as testfile: + with sunau.open(testfile) as f: + self.assertFalse(f.getfp().closed) + params = f.getparams() + self.assertEqual(params[0], nchannels) + self.assertEqual(params[1], sampwidth) + self.assertEqual(params[2], framerate) + self.assertIsNone(f.getfp()) + self.assertFalse(testfile.closed) + + def test_context_manager_with_filename(self): + # If the file doesn't get closed, this test won't fail, but it will + # produce a resource leak warning. + with sunau.open(TESTFN, 'wb') as f: + f.setnchannels(nchannels) + f.setsampwidth(sampwidth) + f.setframerate(framerate) + with sunau.open(TESTFN) as f: + self.assertFalse(f.getfp().closed) + params = f.getparams() + self.assertEqual(params[0], nchannels) + self.assertEqual(params[1], sampwidth) + self.assertEqual(params[2], framerate) + self.assertIsNone(f.getfp()) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,6 +54,9 @@ Library ------- +- Issue #18878: sunau.open now supports the context manager protocol. Based on + patches by Claudiu Popa and R. David Murray. + - Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast 64-bit pointer to long (32 bits). -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:37:49 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Thu, 5 Sep 2013 16:37:49 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4ODMw?= =?utf-8?q?=3A_inspect=2Egetclasstree=28=29_no_more_produces_duplicated_en?= =?utf-8?q?tries_even?= Message-ID: <3cW4Hx2Z6Yz7LjQ@mail.python.org> http://hg.python.org/cpython/rev/39d0dfa5808c changeset: 85540:39d0dfa5808c branch: 3.3 parent: 85531:f01e06d26b41 user: Serhiy Storchaka date: Thu Sep 05 17:14:32 2013 +0300 summary: Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. files: Lib/inspect.py | 3 ++- Lib/test/inspect_fodder.py | 2 ++ Lib/test/test_inspect.py | 21 +++++++++++++++++++-- Misc/NEWS | 3 +++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -753,7 +753,8 @@ for parent in c.__bases__: if not parent in children: children[parent] = [] - children[parent].append(c) + if c not in children[parent]: + children[parent].append(c) if unique and parent in classes: break elif c not in roots: roots.append(c) diff --git a/Lib/test/inspect_fodder.py b/Lib/test/inspect_fodder.py --- a/Lib/test/inspect_fodder.py +++ b/Lib/test/inspect_fodder.py @@ -49,6 +49,8 @@ class MalodorousPervert(StupidGit): pass +Tit = MalodorousPervert + class ParrotDroppings: pass diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -224,8 +224,25 @@ [('FesteringGob', mod.FesteringGob), ('MalodorousPervert', mod.MalodorousPervert), ('ParrotDroppings', mod.ParrotDroppings), - ('StupidGit', mod.StupidGit)]) - tree = inspect.getclasstree([cls[1] for cls in classes], 1) + ('StupidGit', mod.StupidGit), + ('Tit', mod.MalodorousPervert), + ]) + tree = inspect.getclasstree([cls[1] for cls in classes]) + self.assertEqual(tree, + [(object, ()), + [(mod.ParrotDroppings, (object,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ], + (mod.StupidGit, (object,)), + [(mod.MalodorousPervert, (mod.StupidGit,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ] + ] + ] + ]) + tree = inspect.getclasstree([cls[1] for cls in classes], True) self.assertEqual(tree, [(object, ()), [(mod.ParrotDroppings, (object,)), diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -66,6 +66,9 @@ Library ------- +- Issue #18830: inspect.getclasstree() no more produces duplicated entries even + when input list contains duplicates. + - Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast 64-bit pointer to long (32 bits). -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:37:50 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Thu, 5 Sep 2013 16:37:50 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2318830=3A_inspect=2Egetclasstree=28=29_no_more_p?= =?utf-8?q?roduces_duplicated_entries_even?= Message-ID: <3cW4Hy4w36z7LjS@mail.python.org> http://hg.python.org/cpython/rev/86ab7b7c173e changeset: 85541:86ab7b7c173e parent: 85534:a62f59667c9e parent: 85540:39d0dfa5808c user: Serhiy Storchaka date: Thu Sep 05 17:16:12 2013 +0300 summary: Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. files: Lib/inspect.py | 3 ++- Lib/test/inspect_fodder.py | 2 ++ Lib/test/test_inspect.py | 21 +++++++++++++++++++-- Misc/NEWS | 3 +++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -789,7 +789,8 @@ for parent in c.__bases__: if not parent in children: children[parent] = [] - children[parent].append(c) + if c not in children[parent]: + children[parent].append(c) if unique and parent in classes: break elif c not in roots: roots.append(c) diff --git a/Lib/test/inspect_fodder.py b/Lib/test/inspect_fodder.py --- a/Lib/test/inspect_fodder.py +++ b/Lib/test/inspect_fodder.py @@ -49,6 +49,8 @@ class MalodorousPervert(StupidGit): pass +Tit = MalodorousPervert + class ParrotDroppings: pass diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -225,8 +225,25 @@ [('FesteringGob', mod.FesteringGob), ('MalodorousPervert', mod.MalodorousPervert), ('ParrotDroppings', mod.ParrotDroppings), - ('StupidGit', mod.StupidGit)]) - tree = inspect.getclasstree([cls[1] for cls in classes], 1) + ('StupidGit', mod.StupidGit), + ('Tit', mod.MalodorousPervert), + ]) + tree = inspect.getclasstree([cls[1] for cls in classes]) + self.assertEqual(tree, + [(object, ()), + [(mod.ParrotDroppings, (object,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ], + (mod.StupidGit, (object,)), + [(mod.MalodorousPervert, (mod.StupidGit,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ] + ] + ] + ]) + tree = inspect.getclasstree([cls[1] for cls in classes], True) self.assertEqual(tree, [(object, ()), [(mod.ParrotDroppings, (object,)), diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,6 +54,9 @@ Library ------- +- Issue #18830: inspect.getclasstree() no more produces duplicated entries even + when input list contains duplicates. + - Issue #18878: sunau.open now supports the context manager protocol. Based on patches by Claudiu Popa and R. David Murray. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:37:52 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Thu, 5 Sep 2013 16:37:52 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4ODMw?= =?utf-8?q?=3A_inspect=2Egetclasstree=28=29_no_more_produces_duplicated_en?= =?utf-8?q?tries_even?= Message-ID: <3cW4J00sRDz7Ljc@mail.python.org> http://hg.python.org/cpython/rev/408165aced01 changeset: 85542:408165aced01 branch: 2.7 parent: 85533:1a65bb15dedf user: Serhiy Storchaka date: Thu Sep 05 17:28:10 2013 +0300 summary: Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. files: Lib/inspect.py | 3 ++- Lib/test/inspect_fodder.py | 2 ++ Lib/test/test_inspect.py | 19 +++++++++++++++++-- Misc/NEWS | 3 +++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -728,7 +728,8 @@ for parent in c.__bases__: if not parent in children: children[parent] = [] - children[parent].append(c) + if c not in children[parent]: + children[parent].append(c) if unique and parent in classes: break elif c not in roots: roots.append(c) diff --git a/Lib/test/inspect_fodder.py b/Lib/test/inspect_fodder.py --- a/Lib/test/inspect_fodder.py +++ b/Lib/test/inspect_fodder.py @@ -49,6 +49,8 @@ class MalodorousPervert(StupidGit): pass +Tit = MalodorousPervert + class ParrotDroppings: pass diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -220,8 +220,23 @@ [('FesteringGob', mod.FesteringGob), ('MalodorousPervert', mod.MalodorousPervert), ('ParrotDroppings', mod.ParrotDroppings), - ('StupidGit', mod.StupidGit)]) - tree = inspect.getclasstree([cls[1] for cls in classes], 1) + ('StupidGit', mod.StupidGit), + ('Tit', mod.MalodorousPervert), + ]) + tree = inspect.getclasstree([cls[1] for cls in classes]) + self.assertEqual(tree, + [(mod.ParrotDroppings, ()), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ], + (mod.StupidGit, ()), + [(mod.MalodorousPervert, (mod.StupidGit,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ] + ] + ]) + tree = inspect.getclasstree([cls[1] for cls in classes], True) self.assertEqual(tree, [(mod.ParrotDroppings, ()), (mod.StupidGit, ()), diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Library ------- +- Issue #18830: inspect.getclasstree() no more produces duplicated entries even + when input list contains duplicates. + - Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast 64-bit pointer to long (32 bits). -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:37:53 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Thu, 5 Sep 2013 16:37:53 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMi43IC0+IDIuNyk6?= =?utf-8?q?_Merge_heads?= Message-ID: <3cW4J13YBjz7LjS@mail.python.org> http://hg.python.org/cpython/rev/3163a01ac074 changeset: 85543:3163a01ac074 branch: 2.7 parent: 85542:408165aced01 parent: 85538:a7d5b86ffb95 user: Serhiy Storchaka date: Thu Sep 05 17:30:56 2013 +0300 summary: Merge heads files: Modules/_ssl.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -691,7 +691,7 @@ int i, j; PyObject *peer_alt_names = Py_None; - PyObject *v, *t; + PyObject *v = NULL, *t; X509_EXTENSION *ext = NULL; GENERAL_NAMES *names = NULL; GENERAL_NAME *name; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:37:54 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Thu, 5 Sep 2013 16:37:54 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy4zIC0+IDMuMyk6?= =?utf-8?q?_Merge_heads?= Message-ID: <3cW4J26PVJz7LkF@mail.python.org> http://hg.python.org/cpython/rev/9439e720b12e changeset: 85544:9439e720b12e branch: 3.3 parent: 85540:39d0dfa5808c parent: 85535:90040e560527 user: Serhiy Storchaka date: Thu Sep 05 17:31:37 2013 +0300 summary: Merge heads files: Modules/_ssl.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -721,7 +721,7 @@ int i, j; PyObject *peer_alt_names = Py_None; - PyObject *v, *t; + PyObject *v = NULL, *t; X509_EXTENSION *ext = NULL; GENERAL_NAMES *names = NULL; GENERAL_NAME *name; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:37:56 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Thu, 5 Sep 2013 16:37:56 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?= =?utf-8?q?=29=3A_Merge_heads?= Message-ID: <3cW4J42FsQz7LkF@mail.python.org> http://hg.python.org/cpython/rev/60007e358557 changeset: 85545:60007e358557 parent: 85541:86ab7b7c173e parent: 85539:cecec75c2250 user: Serhiy Storchaka date: Thu Sep 05 17:32:15 2013 +0300 summary: Merge heads files: Modules/_ssl.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -755,7 +755,7 @@ int i, j; PyObject *peer_alt_names = Py_None; - PyObject *v, *t; + PyObject *v = NULL, *t; X509_EXTENSION *ext = NULL; GENERAL_NAMES *names = NULL; GENERAL_NAME *name; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:37:57 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Thu, 5 Sep 2013 16:37:57 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Null_merge?= Message-ID: <3cW4J55913z7Lkc@mail.python.org> http://hg.python.org/cpython/rev/922a98a5f0a8 changeset: 85546:922a98a5f0a8 parent: 85545:60007e358557 parent: 85544:9439e720b12e user: Serhiy Storchaka date: Thu Sep 05 17:33:04 2013 +0300 summary: Null merge files: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 16:45:24 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Thu, 5 Sep 2013 16:45:24 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318922=3A_Now_The_?= =?utf-8?q?Lib/smtpd=2Epy_and_Tools/i18n/msgfmt=2Epy_scripts_write?= Message-ID: <3cW4Sh08Gpz7LjV@mail.python.org> http://hg.python.org/cpython/rev/e81699a6390c changeset: 85547:e81699a6390c user: Serhiy Storchaka date: Thu Sep 05 17:44:53 2013 +0300 summary: Issue #18922: Now The Lib/smtpd.py and Tools/i18n/msgfmt.py scripts write their version strings to stdout, and not to sderr. files: Lib/smtpd.py | 2 +- Misc/NEWS | 6 ++++++ Tools/i18n/msgfmt.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/smtpd.py b/Lib/smtpd.py --- a/Lib/smtpd.py +++ b/Lib/smtpd.py @@ -780,7 +780,7 @@ if opt in ('-h', '--help'): usage(0) elif opt in ('-V', '--version'): - print(__version__, file=sys.stderr) + print(__version__) sys.exit(0) elif opt in ('-n', '--nosetuid'): options.setuid = 0 diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -214,6 +214,12 @@ - Issue #18783: Removed existing mentions of Python long type in docstrings, error messages and comments. +Tools/Demos +----------- + +- Issue #18922: Now The Lib/smtpd.py and Tools/i18n/msgfmt.py scripts write + their version strings to stdout, and not to sderr. + What's New in Python 3.4.0 Alpha 1? =================================== diff --git a/Tools/i18n/msgfmt.py b/Tools/i18n/msgfmt.py --- a/Tools/i18n/msgfmt.py +++ b/Tools/i18n/msgfmt.py @@ -218,7 +218,7 @@ if opt in ('-h', '--help'): usage(0) elif opt in ('-V', '--version'): - print("msgfmt.py", __version__, file=sys.stderr) + print("msgfmt.py", __version__) sys.exit(0) elif opt in ('-o', '--output-file'): outfile = arg -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 17:05:40 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Thu, 5 Sep 2013 17:05:40 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4Njcy?= =?utf-8?q?=3A_Fixed_format_specifiers_for_Py=5Fssize=5Ft_in_debugging_out?= =?utf-8?q?put_in?= Message-ID: <3cW4w45zzgz7Lk6@mail.python.org> http://hg.python.org/cpython/rev/99310e5e1b1c changeset: 85548:99310e5e1b1c branch: 3.3 parent: 85544:9439e720b12e user: Serhiy Storchaka date: Thu Sep 05 18:01:15 2013 +0300 summary: Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in the _sre moduel. files: Misc/NEWS | 3 +++ Modules/_sre.c | 37 +++++++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -66,6 +66,9 @@ Library ------- +- Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in + the _sre moduel. + - Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. diff --git a/Modules/_sre.c b/Modules/_sre.c --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -242,7 +242,7 @@ if (cursize < minsize) { void* stack; cursize = minsize+minsize/4+1024; - TRACE(("allocate/grow stack %d\n", cursize)); + TRACE(("allocate/grow stack %" PY_FORMAT_SIZE_T "d\n", cursize)); stack = PyMem_REALLOC(state->data_stack, cursize); if (!stack) { data_stack_dealloc(state); @@ -561,12 +561,13 @@ if (!i) break; } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, + TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr, ((char*)state->ptr - ptr)/state->charsize)); return ((char*)state->ptr - ptr)/state->charsize; } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, (ptr - (char*) state->ptr)/state->charsize)); + TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr, + (ptr - (char*) state->ptr)/state->charsize)); return (ptr - (char*) state->ptr)/state->charsize; } @@ -653,7 +654,8 @@ #define DATA_STACK_ALLOC(state, type, ptr) \ do { \ alloc_pos = state->data_stack_base; \ - TRACE(("allocating %s in %d (%d)\n", \ + TRACE(("allocating %s in %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ SFY(type), alloc_pos, sizeof(type))); \ if (sizeof(type) > state->data_stack_size - alloc_pos) { \ int j = data_stack_grow(state, sizeof(type)); \ @@ -667,13 +669,14 @@ #define DATA_STACK_LOOKUP_AT(state, type, ptr, pos) \ do { \ - TRACE(("looking up %s at %d\n", SFY(type), pos)); \ + TRACE(("looking up %s at %" PY_FORMAT_SIZE_T "d\n", SFY(type), pos)); \ ptr = (type*)(state->data_stack+pos); \ } while (0) #define DATA_STACK_PUSH(state, data, size) \ do { \ - TRACE(("copy data in %p to %d (%d)\n", \ + TRACE(("copy data in %p to %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ data, state->data_stack_base, size)); \ if (size > state->data_stack_size - state->data_stack_base) { \ int j = data_stack_grow(state, size); \ @@ -687,7 +690,8 @@ #define DATA_STACK_POP(state, data, size, discard) \ do { \ - TRACE(("copy data to %p from %d (%d)\n", \ + TRACE(("copy data to %p from %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ data, state->data_stack_base-size, size)); \ memcpy(data, state->data_stack+state->data_stack_base-size, size); \ if (discard) \ @@ -696,7 +700,8 @@ #define DATA_STACK_POP_DISCARD(state, size) \ do { \ - TRACE(("discard data from %d (%d)\n", \ + TRACE(("discard data from %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ state->data_stack_base-size, size)); \ state->data_stack_base -= size; \ } while(0) @@ -800,8 +805,10 @@ /* optimization info block */ /* <1=skip> <2=flags> <3=min> ... */ if (ctx->pattern[3] && (end - ctx->ptr)/state->charsize < ctx->pattern[3]) { - TRACE(("reject (got %d chars, need %d)\n", - (end - ctx->ptr)/state->charsize, ctx->pattern[3])); + TRACE(("reject (got %" PY_FORMAT_SIZE_T "d chars, " + "need %" PY_FORMAT_SIZE_T "d)\n", + (end - ctx->ptr)/state->charsize, + (Py_ssize_t) ctx->pattern[3])); RETURN_FAILURE; } ctx->pattern += ctx->pattern[1] + 1; @@ -1177,7 +1184,7 @@ ctx->count = ctx->u.rep->count+1; - TRACE(("|%p|%p|MAX_UNTIL %d\n", ctx->pattern, + TRACE(("|%p|%p|MAX_UNTIL %" PY_FORMAT_SIZE_T "d\n", ctx->pattern, ctx->ptr, ctx->count)); if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { @@ -1240,7 +1247,7 @@ ctx->count = ctx->u.rep->count+1; - TRACE(("|%p|%p|MIN_UNTIL %d %p\n", ctx->pattern, + TRACE(("|%p|%p|MIN_UNTIL %" PY_FORMAT_SIZE_T "d %p\n", ctx->pattern, ctx->ptr, ctx->count, ctx->u.rep->pattern)); if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { @@ -1457,7 +1464,8 @@ TRACE(("|%p|%p|JUMP_ASSERT_NOT\n", ctx->pattern, ctx->ptr)); goto jump_assert_not; case JUMP_NONE: - TRACE(("|%p|%p|RETURN %d\n", ctx->pattern, ctx->ptr, ret)); + TRACE(("|%p|%p|RETURN %" PY_FORMAT_SIZE_T "d\n", ctx->pattern, + ctx->ptr, ret)); break; } @@ -1506,7 +1514,8 @@ pattern += 1 + pattern[1]; } - TRACE(("prefix = %p %d %d\n", prefix, prefix_len, prefix_skip)); + TRACE(("prefix = %p %" PY_FORMAT_SIZE_T "d %" PY_FORMAT_SIZE_T "d\n", + prefix, prefix_len, prefix_skip)); TRACE(("charset = %p\n", charset)); #if defined(USE_FAST_SEARCH) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 17:05:42 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Thu, 5 Sep 2013 17:05:42 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2318672=3A_Fixed_format_specifiers_for_Py=5Fssize?= =?utf-8?q?=5Ft_in_debugging_output_in?= Message-ID: <3cW4w62NYNz7Ljm@mail.python.org> http://hg.python.org/cpython/rev/c41c68a18bb6 changeset: 85549:c41c68a18bb6 parent: 85547:e81699a6390c parent: 85548:99310e5e1b1c user: Serhiy Storchaka date: Thu Sep 05 18:02:31 2013 +0300 summary: Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in the _sre moduel. files: Misc/NEWS | 3 +++ Modules/_sre.c | 37 +++++++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -54,6 +54,9 @@ Library ------- +- Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in + the _sre moduel. + - Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. diff --git a/Modules/_sre.c b/Modules/_sre.c --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -238,7 +238,7 @@ if (cursize < minsize) { void* stack; cursize = minsize+minsize/4+1024; - TRACE(("allocate/grow stack %d\n", cursize)); + TRACE(("allocate/grow stack %" PY_FORMAT_SIZE_T "d\n", cursize)); stack = PyMem_REALLOC(state->data_stack, cursize); if (!stack) { data_stack_dealloc(state); @@ -557,12 +557,13 @@ if (!i) break; } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, + TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr, ((char*)state->ptr - ptr)/state->charsize)); return ((char*)state->ptr - ptr)/state->charsize; } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, (ptr - (char*) state->ptr)/state->charsize)); + TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr, + (ptr - (char*) state->ptr)/state->charsize)); return (ptr - (char*) state->ptr)/state->charsize; } @@ -649,7 +650,8 @@ #define DATA_STACK_ALLOC(state, type, ptr) \ do { \ alloc_pos = state->data_stack_base; \ - TRACE(("allocating %s in %d (%d)\n", \ + TRACE(("allocating %s in %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ SFY(type), alloc_pos, sizeof(type))); \ if (sizeof(type) > state->data_stack_size - alloc_pos) { \ int j = data_stack_grow(state, sizeof(type)); \ @@ -663,13 +665,14 @@ #define DATA_STACK_LOOKUP_AT(state, type, ptr, pos) \ do { \ - TRACE(("looking up %s at %d\n", SFY(type), pos)); \ + TRACE(("looking up %s at %" PY_FORMAT_SIZE_T "d\n", SFY(type), pos)); \ ptr = (type*)(state->data_stack+pos); \ } while (0) #define DATA_STACK_PUSH(state, data, size) \ do { \ - TRACE(("copy data in %p to %d (%d)\n", \ + TRACE(("copy data in %p to %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ data, state->data_stack_base, size)); \ if (size > state->data_stack_size - state->data_stack_base) { \ int j = data_stack_grow(state, size); \ @@ -683,7 +686,8 @@ #define DATA_STACK_POP(state, data, size, discard) \ do { \ - TRACE(("copy data to %p from %d (%d)\n", \ + TRACE(("copy data to %p from %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ data, state->data_stack_base-size, size)); \ memcpy(data, state->data_stack+state->data_stack_base-size, size); \ if (discard) \ @@ -692,7 +696,8 @@ #define DATA_STACK_POP_DISCARD(state, size) \ do { \ - TRACE(("discard data from %d (%d)\n", \ + TRACE(("discard data from %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ state->data_stack_base-size, size)); \ state->data_stack_base -= size; \ } while(0) @@ -796,8 +801,10 @@ /* optimization info block */ /* <1=skip> <2=flags> <3=min> ... */ if (ctx->pattern[3] && (Py_uintptr_t)(end - ctx->ptr)/state->charsize < ctx->pattern[3]) { - TRACE(("reject (got %d chars, need %d)\n", - (end - ctx->ptr)/state->charsize, ctx->pattern[3])); + TRACE(("reject (got %" PY_FORMAT_SIZE_T "d chars, " + "need %" PY_FORMAT_SIZE_T "d)\n", + (end - ctx->ptr)/state->charsize, + (Py_ssize_t) ctx->pattern[3])); RETURN_FAILURE; } ctx->pattern += ctx->pattern[1] + 1; @@ -1173,7 +1180,7 @@ ctx->count = ctx->u.rep->count+1; - TRACE(("|%p|%p|MAX_UNTIL %d\n", ctx->pattern, + TRACE(("|%p|%p|MAX_UNTIL %" PY_FORMAT_SIZE_T "d\n", ctx->pattern, ctx->ptr, ctx->count)); if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { @@ -1236,7 +1243,7 @@ ctx->count = ctx->u.rep->count+1; - TRACE(("|%p|%p|MIN_UNTIL %d %p\n", ctx->pattern, + TRACE(("|%p|%p|MIN_UNTIL %" PY_FORMAT_SIZE_T "d %p\n", ctx->pattern, ctx->ptr, ctx->count, ctx->u.rep->pattern)); if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { @@ -1453,7 +1460,8 @@ TRACE(("|%p|%p|JUMP_ASSERT_NOT\n", ctx->pattern, ctx->ptr)); goto jump_assert_not; case JUMP_NONE: - TRACE(("|%p|%p|RETURN %d\n", ctx->pattern, ctx->ptr, ret)); + TRACE(("|%p|%p|RETURN %" PY_FORMAT_SIZE_T "d\n", ctx->pattern, + ctx->ptr, ret)); break; } @@ -1502,7 +1510,8 @@ pattern += 1 + pattern[1]; } - TRACE(("prefix = %p %d %d\n", prefix, prefix_len, prefix_skip)); + TRACE(("prefix = %p %" PY_FORMAT_SIZE_T "d %" PY_FORMAT_SIZE_T "d\n", + prefix, prefix_len, prefix_skip)); TRACE(("charset = %p\n", charset)); #if defined(USE_FAST_SEARCH) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 17:05:43 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Thu, 5 Sep 2013 17:05:43 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4Njcy?= =?utf-8?q?=3A_Fixed_format_specifiers_for_Py=5Fssize=5Ft_in_debugging_out?= =?utf-8?q?put_in?= Message-ID: <3cW4w761XDz7LkH@mail.python.org> http://hg.python.org/cpython/rev/603b4d593758 changeset: 85550:603b4d593758 branch: 2.7 parent: 85543:3163a01ac074 user: Serhiy Storchaka date: Thu Sep 05 18:02:57 2013 +0300 summary: Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in the _sre moduel. files: Misc/NEWS | 3 +++ Modules/_sre.c | 36 ++++++++++++++++++++++-------------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Library ------- +- Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in + the _sre moduel. + - Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. diff --git a/Modules/_sre.c b/Modules/_sre.c --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -272,7 +272,7 @@ if (cursize < minsize) { void* stack; cursize = minsize+minsize/4+1024; - TRACE(("allocate/grow stack %d\n", cursize)); + TRACE(("allocate/grow stack %" PY_FORMAT_SIZE_T "d\n", cursize)); stack = PyMem_REALLOC(state->data_stack, cursize); if (!stack) { data_stack_dealloc(state); @@ -592,12 +592,13 @@ if (!i) break; } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, + TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr, (SRE_CHAR*) state->ptr - ptr)); return (SRE_CHAR*) state->ptr - ptr; } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, ptr - (SRE_CHAR*) state->ptr)); + TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr, + ptr - (SRE_CHAR*) state->ptr)); return ptr - (SRE_CHAR*) state->ptr; } @@ -684,7 +685,8 @@ #define DATA_STACK_ALLOC(state, type, ptr) \ do { \ alloc_pos = state->data_stack_base; \ - TRACE(("allocating %s in %d (%d)\n", \ + TRACE(("allocating %s in %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ SFY(type), alloc_pos, sizeof(type))); \ if (sizeof(type) > state->data_stack_size - alloc_pos) { \ int j = data_stack_grow(state, sizeof(type)); \ @@ -698,13 +700,14 @@ #define DATA_STACK_LOOKUP_AT(state, type, ptr, pos) \ do { \ - TRACE(("looking up %s at %d\n", SFY(type), pos)); \ + TRACE(("looking up %s at %" PY_FORMAT_SIZE_T "d\n", SFY(type), pos)); \ ptr = (type*)(state->data_stack+pos); \ } while (0) #define DATA_STACK_PUSH(state, data, size) \ do { \ - TRACE(("copy data in %p to %d (%d)\n", \ + TRACE(("copy data in %p to %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ data, state->data_stack_base, size)); \ if (size > state->data_stack_size - state->data_stack_base) { \ int j = data_stack_grow(state, size); \ @@ -718,7 +721,8 @@ #define DATA_STACK_POP(state, data, size, discard) \ do { \ - TRACE(("copy data to %p from %d (%d)\n", \ + TRACE(("copy data to %p from %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ data, state->data_stack_base-size, size)); \ memcpy(data, state->data_stack+state->data_stack_base-size, size); \ if (discard) \ @@ -727,7 +731,8 @@ #define DATA_STACK_POP_DISCARD(state, size) \ do { \ - TRACE(("discard data from %d (%d)\n", \ + TRACE(("discard data from %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ state->data_stack_base-size, size)); \ state->data_stack_base -= size; \ } while(0) @@ -831,8 +836,9 @@ /* optimization info block */ /* <1=skip> <2=flags> <3=min> ... */ if (ctx->pattern[3] && (end - ctx->ptr) < ctx->pattern[3]) { - TRACE(("reject (got %d chars, need %d)\n", - (end - ctx->ptr), ctx->pattern[3])); + TRACE(("reject (got %" PY_FORMAT_SIZE_T "d chars, " + "need %" PY_FORMAT_SIZE_T "d)\n", + (end - ctx->ptr), (Py_ssize_t) ctx->pattern[3])); RETURN_FAILURE; } ctx->pattern += ctx->pattern[1] + 1; @@ -1207,7 +1213,7 @@ ctx->count = ctx->u.rep->count+1; - TRACE(("|%p|%p|MAX_UNTIL %d\n", ctx->pattern, + TRACE(("|%p|%p|MAX_UNTIL %" PY_FORMAT_SIZE_T "d\n", ctx->pattern, ctx->ptr, ctx->count)); if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { @@ -1270,7 +1276,7 @@ ctx->count = ctx->u.rep->count+1; - TRACE(("|%p|%p|MIN_UNTIL %d %p\n", ctx->pattern, + TRACE(("|%p|%p|MIN_UNTIL %" PY_FORMAT_SIZE_T "d %p\n", ctx->pattern, ctx->ptr, ctx->count, ctx->u.rep->pattern)); if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { @@ -1483,7 +1489,8 @@ TRACE(("|%p|%p|JUMP_ASSERT_NOT\n", ctx->pattern, ctx->ptr)); goto jump_assert_not; case JUMP_NONE: - TRACE(("|%p|%p|RETURN %d\n", ctx->pattern, ctx->ptr, ret)); + TRACE(("|%p|%p|RETURN %" PY_FORMAT_SIZE_T "d\n", ctx->pattern, + ctx->ptr, ret)); break; } @@ -1532,7 +1539,8 @@ pattern += 1 + pattern[1]; } - TRACE(("prefix = %p %d %d\n", prefix, prefix_len, prefix_skip)); + TRACE(("prefix = %p %" PY_FORMAT_SIZE_T "d %" PY_FORMAT_SIZE_T "d\n", + prefix, prefix_len, prefix_skip)); TRACE(("charset = %p\n", charset)); #if defined(USE_FAST_SEARCH) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 20:47:12 2013 From: python-checkins at python.org (charles-francois.natali) Date: Thu, 5 Sep 2013 20:47:12 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318934=3A_multipro?= =?utf-8?q?cessing=3A_use_selectors_module=2E?= Message-ID: <3cW9qh0g7Sz7Lkb@mail.python.org> http://hg.python.org/cpython/rev/81f0c6358a5f changeset: 85551:81f0c6358a5f parent: 85549:c41c68a18bb6 user: Charles-Fran?ois Natali date: Thu Sep 05 20:46:49 2013 +0200 summary: Issue #18934: multiprocessing: use selectors module. files: Lib/multiprocessing/connection.py | 51 +++++------------- Lib/multiprocessing/forkserver.py | 14 +++- 2 files changed, 26 insertions(+), 39 deletions(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -12,7 +12,6 @@ import io import os import sys -import select import socket import struct import errno @@ -877,28 +876,7 @@ else: - if hasattr(select, 'poll'): - def _poll(fds, timeout): - if timeout is not None: - timeout = int(timeout * 1000) # timeout is in milliseconds - fd_map = {} - pollster = select.poll() - for fd in fds: - pollster.register(fd, select.POLLIN) - if hasattr(fd, 'fileno'): - fd_map[fd.fileno()] = fd - else: - fd_map[fd] = fd - ls = [] - for fd, event in pollster.poll(timeout): - if event & select.POLLNVAL: - raise ValueError('invalid file descriptor %i' % fd) - ls.append(fd_map[fd]) - return ls - else: - def _poll(fds, timeout): - return select.select(fds, [], [], timeout)[0] - + import selectors def wait(object_list, timeout=None): ''' @@ -906,19 +884,22 @@ Returns list of those objects in object_list which are ready/readable. ''' - if timeout is not None: - if timeout <= 0: - return _poll(object_list, 0) - else: + with selectors.DefaultSelector() as selector: + for obj in object_list: + selector.register(obj, selectors.EVENT_READ) + + if timeout is not None: deadline = time.time() + timeout - while True: - try: - return _poll(object_list, timeout) - except OSError as e: - if e.errno != errno.EINTR: - raise - if timeout is not None: - timeout = deadline - time.time() + + while True: + ready = selector.select(timeout) + if ready: + return [key.fileobj for (key, events) in ready] + else: + if timeout is not None: + timeout = deadline - time.time() + if timeout < 0: + return ready # # Make connection and socket objects sharable if possible diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -1,6 +1,6 @@ import errno import os -import select +import selectors import signal import socket import struct @@ -149,14 +149,20 @@ # ignoring SIGCHLD means no need to reap zombie processes handler = signal.signal(signal.SIGCHLD, signal.SIG_IGN) - with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener: + with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \ + selectors.DefaultSelector() as selector: global _forkserver_address _forkserver_address = listener.getsockname() - readers = [listener, alive_r] + + selector.register(listener, selectors.EVENT_READ) + selector.register(alive_r, selectors.EVENT_READ) while True: try: - rfds, wfds, xfds = select.select(readers, [], []) + while True: + rfds = [key.fileobj for (key, events) in selector.select()] + if rfds: + break if alive_r in rfds: # EOF because no more client processes left -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 5 23:17:53 2013 From: python-checkins at python.org (victor.stinner) Date: Thu, 5 Sep 2013 23:17:53 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454=3A_add_get/set=5Fnumb?= =?utf-8?q?er=5Fframe=28=29?= Message-ID: <3cWF9Y1wySz7LjS@mail.python.org> http://hg.python.org/peps/rev/63d477127448 changeset: 5097:63d477127448 user: Victor Stinner date: Thu Sep 05 23:15:45 2013 +0200 summary: PEP 454: add get/set_number_frame() files: pep-0454.txt | 37 +++++++++++++++++++++++++++---------- 1 files changed, 27 insertions(+), 10 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -66,23 +66,30 @@ environment variable to ``1``, or by using ``-X tracemalloc`` command line option. +By default, tracemalloc only stores one ``frame`` instance per memory +allocation. Use ``tracemalloc.set_number_frame()`` to store more frames. + Functions --------- -``enable()`` function: +``clear_traces()`` function: - Start tracing Python memory allocations. + Clear all traces and statistics of memory allocations. ``disable()`` function: Stop tracing Python memory allocations and stop the timer started by ``start_timer()``. -``is_enabled()`` function: +``enable()`` function: - Get the status of the module: ``True`` if it is enabled, ``False`` - otherwise. + Start tracing Python memory allocations. + +``get_number_frame()`` function: + + Get the maximum number of frames stored in a trace of a memory + allocation. ``get_object_address(obj)`` function: @@ -120,15 +127,25 @@ Return an empty dictionary if the tracemalloc module is disabled. - ``get_traces(obj)`` function: - Get all traces of a Python memory allocations. - Return a dictionary ``{pointer: int -> trace}`` where *trace* - is a ``trace`` instance. + Get all traces of a Python memory allocations. + Return a dictionary ``{pointer: int -> trace}`` where *trace* + is a ``trace`` instance. - Return an empty dictionary if the ``tracemalloc`` module is disabled. + Return an empty dictionary if the ``tracemalloc`` module is disabled. +``is_enabled()`` function: + + Get the status of the module: ``True`` if it is enabled, ``False`` + otherwise. + +``set_number_frame(nframe: int)`` function: + + Set the maximum number of frames stored in a trace of a memory + allocation. + + All traces and statistics of memory allocations are cleared. ``start_timer(delay: int, func: callable, args: tuple=(), kwargs: dict={})`` function: -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Thu Sep 5 23:27:07 2013 From: python-checkins at python.org (victor.stinner) Date: Thu, 5 Sep 2013 23:27:07 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454=3A_API_inspired_by_th?= =?utf-8?q?e_faulthandler_module?= Message-ID: <3cWFNC0yzrz7LjS@mail.python.org> http://hg.python.org/peps/rev/ac5c29a04951 changeset: 5098:ac5c29a04951 user: Victor Stinner date: Thu Sep 05 23:23:35 2013 +0200 summary: PEP 454: API inspired by the faulthandler module files: pep-0454.txt | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -56,6 +56,12 @@ * Location of a Python memory allocation: size in bytes, Python filename and line number +The tracemalloc module has an API close the the faulthandler module: +``enable()``, ``disable()`` and ``is_enabled()`` functions, an +environment variable (``PYTHONFAULTHANDLER`` and ``PYTHONTRACEMALLOC``), +a ``-X`` command line option (``-X faulthandler`` and ``-X +tracemalloc``). + API === -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Thu Sep 5 23:27:08 2013 From: python-checkins at python.org (victor.stinner) Date: Thu, 5 Sep 2013 23:27:08 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454=3A_tracemalloc_has_be?= =?utf-8?q?en_written_for_CPython?= Message-ID: <3cWFND2cnZz7LjS@mail.python.org> http://hg.python.org/peps/rev/606568983a1a changeset: 5099:606568983a1a user: Victor Stinner date: Thu Sep 05 23:26:55 2013 +0200 summary: PEP 454: tracemalloc has been written for CPython files: pep-0454.txt | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -62,6 +62,9 @@ a ``-X`` command line option (``-X faulthandler`` and ``-X tracemalloc``). +The tracemalloc module has been written for CPython. Other +implementations of Python may not provide it. + API === -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Thu Sep 5 23:29:51 2013 From: python-checkins at python.org (victor.stinner) Date: Thu, 5 Sep 2013 23:29:51 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454=3A_add_link_to_faulth?= =?utf-8?q?andler_doc?= Message-ID: <3cWFRM03z4z7LjW@mail.python.org> http://hg.python.org/peps/rev/82d582042509 changeset: 5100:82d582042509 user: Victor Stinner date: Thu Sep 05 23:29:40 2013 +0200 summary: PEP 454: add link to faulthandler doc files: pep-0454.txt | 12 +++++++----- 1 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -56,11 +56,13 @@ * Location of a Python memory allocation: size in bytes, Python filename and line number -The tracemalloc module has an API close the the faulthandler module: -``enable()``, ``disable()`` and ``is_enabled()`` functions, an -environment variable (``PYTHONFAULTHANDLER`` and ``PYTHONTRACEMALLOC``), -a ``-X`` command line option (``-X faulthandler`` and ``-X -tracemalloc``). +The API of the tracemalloc module is similar to the API of the +faulthandler module: ``enable()``, ``disable()`` and ``is_enabled()`` +functions, an environment variable (``PYTHONFAULTHANDLER`` and +``PYTHONTRACEMALLOC``), a ``-X`` command line option (``-X +faulthandler`` and ``-X tracemalloc``). See the +`documentation of the faulthandler module +`_. The tracemalloc module has been written for CPython. Other implementations of Python may not provide it. -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Fri Sep 6 00:02:58 2013 From: python-checkins at python.org (vinay.sajip) Date: Fri, 6 Sep 2013 00:02:58 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4OTMz?= =?utf-8?q?=3A_Added_links_to_source_code=2E?= Message-ID: <3cWG9Z6rBRz7LjT@mail.python.org> http://hg.python.org/cpython/rev/dc4e6b48c321 changeset: 85552:dc4e6b48c321 branch: 2.7 parent: 85550:603b4d593758 user: Vinay Sajip date: Thu Sep 05 22:57:20 2013 +0100 summary: Issue #18933: Added links to source code. files: Doc/library/logging.config.rst | 4 ++++ Doc/library/logging.handlers.rst | 4 ++++ Doc/library/logging.rst | 3 +++ 3 files changed, 11 insertions(+), 0 deletions(-) diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -17,6 +17,10 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/config.py` + +-------------- + This section describes the API for configuring the logging module. .. _logging-config-api: diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -17,6 +17,10 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/handlers.py` + +-------------- + .. currentmodule:: logging The following useful handlers are provided in the package. Note that three of diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -20,6 +20,9 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/__init__.py` + +-------------- .. versionadded:: 2.3 -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 00:03:00 2013 From: python-checkins at python.org (vinay.sajip) Date: Fri, 6 Sep 2013 00:03:00 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4OTMz?= =?utf-8?q?=3A_Added_links_to_source_code=2E?= Message-ID: <3cWG9c1dQ6z7Ljk@mail.python.org> http://hg.python.org/cpython/rev/34e515f2fdfe changeset: 85553:34e515f2fdfe branch: 3.3 parent: 85548:99310e5e1b1c user: Vinay Sajip date: Thu Sep 05 23:01:07 2013 +0100 summary: Issue #18933: Added links to source code. files: Doc/library/logging.config.rst | 4 ++++ Doc/library/logging.handlers.rst | 4 ++++ Doc/library/logging.rst | 4 ++++ 3 files changed, 12 insertions(+), 0 deletions(-) diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -17,6 +17,10 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/config.py` + +-------------- + This section describes the API for configuring the logging module. .. _logging-config-api: diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -17,6 +17,10 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/handlers.py` + +-------------- + .. currentmodule:: logging The following useful handlers are provided in the package. Note that three of diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -21,6 +21,10 @@ * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/__init__.py` + +-------------- + This module defines functions and classes which implement a flexible event logging system for applications and libraries. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 00:03:01 2013 From: python-checkins at python.org (vinay.sajip) Date: Fri, 6 Sep 2013 00:03:01 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Closes_=2318933=3A_Merged_update_from_3=2E3=2E?= Message-ID: <3cWG9d3Tvkz7LkQ@mail.python.org> http://hg.python.org/cpython/rev/c5924523747e changeset: 85554:c5924523747e parent: 85551:81f0c6358a5f parent: 85553:34e515f2fdfe user: Vinay Sajip date: Thu Sep 05 23:02:45 2013 +0100 summary: Closes #18933: Merged update from 3.3. files: Doc/library/logging.config.rst | 4 ++++ Doc/library/logging.handlers.rst | 4 ++++ Doc/library/logging.rst | 4 ++++ 3 files changed, 12 insertions(+), 0 deletions(-) diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -17,6 +17,10 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/config.py` + +-------------- + This section describes the API for configuring the logging module. .. _logging-config-api: diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -17,6 +17,10 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/handlers.py` + +-------------- + .. currentmodule:: logging The following useful handlers are provided in the package. Note that three of diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -21,6 +21,10 @@ * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/__init__.py` + +-------------- + This module defines functions and classes which implement a flexible event logging system for applications and libraries. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 06:11:27 2013 From: python-checkins at python.org (tim.peters) Date: Fri, 6 Sep 2013 06:11:27 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4OTQy?= =?utf-8?q?=3A_sys=2E=5Fdebugmallocstats=28=29_output_was_damaged_on_Windo?= =?utf-8?b?d3Mu?= Message-ID: <3cWQLl3x30z7LjS@mail.python.org> http://hg.python.org/cpython/rev/d95cc29ea94e changeset: 85555:d95cc29ea94e branch: 3.3 parent: 85553:34e515f2fdfe user: Tim Peters date: Thu Sep 05 22:57:04 2013 -0500 summary: Issue #18942: sys._debugmallocstats() output was damaged on Windows. _PyDebugAllocatorStats() called PyOS_snprintf() with a %zd format code, but MS doesn't support that code. Interpolated PY_FORMAT_SIZE_T in place of the "z". files: Misc/NEWS | 2 ++ Objects/obmalloc.c | 2 +- 2 files changed, 3 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,8 @@ Core and Builtins ----------------- +- Issue #18942: sys._debugmallocstats() output was damaged on Windows. + - Issue #18667: Add missing "HAVE_FCHOWNAT" symbol to posix._have_functions. - Issue #18368: PyOS_StdioReadline() no longer leaks memory when realloc() diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1761,7 +1761,7 @@ char buf1[128]; char buf2[128]; PyOS_snprintf(buf1, sizeof(buf1), - "%d %ss * %zd bytes each", + "%d %ss * %" PY_FORMAT_SIZE_T "d bytes each", num_blocks, block_name, sizeof_block); PyOS_snprintf(buf2, sizeof(buf2), "%48s ", buf1); -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 06:11:28 2013 From: python-checkins at python.org (tim.peters) Date: Fri, 6 Sep 2013 06:11:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Nerge_3=2E3_into_default=2E?= Message-ID: <3cWQLm5pkZz7Ljk@mail.python.org> http://hg.python.org/cpython/rev/43f772554872 changeset: 85556:43f772554872 parent: 85554:c5924523747e parent: 85555:d95cc29ea94e user: Tim Peters date: Thu Sep 05 23:04:26 2013 -0500 summary: Nerge 3.3 into default. Issue #18942: sys._debugmallocstats() output was damaged on Windows. _PyDebugAllocatorStats() called PyOS_snprintf() with a %zd format code, but MS doesn't support that code. Interpolated PY_FORMAT_SIZE_T in place of the "z". files: Misc/NEWS | 2 ++ Objects/obmalloc.c | 2 +- 2 files changed, 3 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Core and Builtins ----------------- +- Issue #18942: sys._debugmallocstats() output was damaged on Windows. + - Issue #18571: Implementation of the PEP 446: file descriptors and file handles are now created non-inheritable; add functions os.get/set_inheritable(), os.get/set_handle_inheritable() and diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2035,7 +2035,7 @@ char buf1[128]; char buf2[128]; PyOS_snprintf(buf1, sizeof(buf1), - "%d %ss * %zd bytes each", + "%d %ss * %" PY_FORMAT_SIZE_T "d bytes each", num_blocks, block_name, sizeof_block); PyOS_snprintf(buf2, sizeof(buf2), "%48s ", buf1); -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Fri Sep 6 06:17:55 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Fri, 06 Sep 2013 06:17:55 +0200 Subject: [Python-checkins] Daily reference leaks (c5924523747e): sum=-1 Message-ID: results for c5924523747e on branch "default" -------------------------------------------- test_support leaked [0, 0, -1] references, sum=-1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogtzm1wN', '-x'] From python-checkins at python.org Fri Sep 6 06:44:02 2013 From: python-checkins at python.org (senthil.kumaran) Date: Fri, 6 Sep 2013 06:44:02 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Improve_urlenc?= =?utf-8?q?ode_docstring=2E_Patch_by_Brian_Brazil=2E?= Message-ID: <3cWR4L4MD7z7LjR@mail.python.org> http://hg.python.org/cpython/rev/6e76cfeee777 changeset: 85557:6e76cfeee777 branch: 3.3 parent: 85555:d95cc29ea94e user: Senthil Kumaran date: Thu Sep 05 21:42:38 2013 -0700 summary: Improve urlencode docstring. Patch by Brian Brazil. files: Lib/urllib/parse.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -728,7 +728,7 @@ return ''.join([quoter(char) for char in bs]) def urlencode(query, doseq=False, safe='', encoding=None, errors=None): - """Encode a sequence of two-element tuples or dictionary into a URL query string. + """Encode a dict or sequence of two-element tuples into a URL query string. If any values in the query arg are sequences and doseq is true, each sequence element is converted to a separate parameter. @@ -737,9 +737,9 @@ parameters in the output will match the order of parameters in the input. - The query arg may be either a string or a bytes type. When query arg is a - string, the safe, encoding and error parameters are sent the quote_plus for - encoding. + The components of a query arg may each be either a string or a bytes type. + When a component is a string, the safe, encoding and error parameters are + sent to the quote_plus function for encoding. """ if hasattr(query, "items"): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 06:44:03 2013 From: python-checkins at python.org (senthil.kumaran) Date: Fri, 6 Sep 2013 06:44:03 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_merge_from_3=2E3?= Message-ID: <3cWR4M66dZz7Ljk@mail.python.org> http://hg.python.org/cpython/rev/975d1e180689 changeset: 85558:975d1e180689 parent: 85556:43f772554872 parent: 85557:6e76cfeee777 user: Senthil Kumaran date: Thu Sep 05 21:43:53 2013 -0700 summary: merge from 3.3 Improve urlencode docstring. Patch by Brian Brazil. Closes issue #15350 files: Lib/urllib/parse.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -728,7 +728,7 @@ return ''.join([quoter(char) for char in bs]) def urlencode(query, doseq=False, safe='', encoding=None, errors=None): - """Encode a sequence of two-element tuples or dictionary into a URL query string. + """Encode a dict or sequence of two-element tuples into a URL query string. If any values in the query arg are sequences and doseq is true, each sequence element is converted to a separate parameter. @@ -737,9 +737,9 @@ parameters in the output will match the order of parameters in the input. - The query arg may be either a string or a bytes type. When query arg is a - string, the safe, encoding and error parameters are sent the quote_plus for - encoding. + The components of a query arg may each be either a string or a bytes type. + When a component is a string, the safe, encoding and error parameters are + sent to the quote_plus function for encoding. """ if hasattr(query, "items"): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 10:08:10 2013 From: python-checkins at python.org (ned.deily) Date: Fri, 6 Sep 2013 10:08:10 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=231584=3A_Provide_o?= =?utf-8?q?ptions_to_override_default_search_paths_for_Tcl_and_Tk?= Message-ID: <3cWWbt1t1Zz7LjN@mail.python.org> http://hg.python.org/cpython/rev/0986e4f5750d changeset: 85559:0986e4f5750d user: Ned Deily date: Fri Sep 06 01:07:05 2013 -0700 summary: Issue #1584: Provide options to override default search paths for Tcl and Tk when building _tkinter. configure has two new options; if used, both must be specified: ./configure \ --with-tcltk-includes="-I/opt/local/include" \ --with-tcltk-libs="-L/opt/local/lib -ltcl8.5 -ltk8.5" In addition, the options can be overridden with make: make \ TCLTK_INCLUDES="-I/opt/local/include" \ TCLTK_LIBS="-L/opt/local/lib -ltcl8.6 -ltk8.6" files: Makefile.pre.in | 5 +++ Misc/NEWS | 6 ++++ configure | 48 +++++++++++++++++++++++++++++++++++++ configure.ac | 28 +++++++++++++++++++++ setup.py | 43 ++++++++++++++++++++++++++++++++- 5 files changed, 129 insertions(+), 1 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -207,6 +207,10 @@ BUILD_GNU_TYPE= @build@ HOST_GNU_TYPE= @host@ +# Tcl and Tk config info from --with-tcltk-includes and -libs options +TCLTK_INCLUDES= @TCLTK_INCLUDES@ +TCLTK_LIBS= @TCLTK_LIBS@ + # The task to run while instrument when building the profile-opt target PROFILE_TASK= $(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck #PROFILE_TASK= $(srcdir)/Lib/test/regrtest.py @@ -535,6 +539,7 @@ *) quiet="";; \ esac; \ $(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' \ + _TCLTK_INCLUDES='$(TCLTK_INCLUDES)' _TCLTK_LIBS='$(TCLTK_LIBS)' \ $(PYTHON_FOR_BUILD) $(srcdir)/setup.py $$quiet build # Build static library diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -219,6 +219,12 @@ - Issue #18783: Removed existing mentions of Python long type in docstrings, error messages and comments. +Build +----- + +- Issue #1584: Provide configure options to override default search paths for + Tcl and Tk when building _tkinter. + Tools/Demos ----------- diff --git a/configure b/configure --- a/configure +++ b/configure @@ -646,6 +646,8 @@ USE_THREAD_MODULE SIGNAL_OBJS USE_SIGNAL_MODULE +TCLTK_LIBS +TCLTK_INCLUDES LIBFFI_INCLUDEDIR PKG_CONFIG SHLIBS @@ -795,6 +797,8 @@ with_system_ffi with_system_libmpdec enable_loadable_sqlite_extensions +with_tcltk_includes +with_tcltk_libs with_dbmliborder with_signal_module with_threads @@ -1467,6 +1471,10 @@ --with-system-ffi build _ctypes module using an installed ffi library --with-system-libmpdec build _decimal module using an installed libmpdec library + --with-tcltk-includes='-I...' + override search for Tcl and Tk include files + --with-tcltk-libs='-L...' + override search for Tcl and Tk libs --with-dbmliborder=db1:db2:... order to check db backends for dbm. Valid value is a colon separated string with the backend names @@ -9237,6 +9245,46 @@ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_loadable_sqlite_extensions" >&5 $as_echo "$enable_loadable_sqlite_extensions" >&6; } +# Check for --with-tcltk-includes=path and --with-tcltk-libs=path + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-tcltk-includes" >&5 +$as_echo_n "checking for --with-tcltk-includes... " >&6; } + +# Check whether --with-tcltk-includes was given. +if test "${with_tcltk_includes+set}" = set; then : + withval=$with_tcltk_includes; +else + with_tcltk_includes="default" +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_tcltk_includes" >&5 +$as_echo "$with_tcltk_includes" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-tcltk-libs" >&5 +$as_echo_n "checking for --with-tcltk-libs... " >&6; } + +# Check whether --with-tcltk-libs was given. +if test "${with_tcltk_libs+set}" = set; then : + withval=$with_tcltk_libs; +else + with_tcltk_libs="default" +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_tcltk_libs" >&5 +$as_echo "$with_tcltk_libs" >&6; } +if test "x$with_tcltk_includes" = xdefault || test "x$with_tcltk_libs" = xdefault +then + if test "x$with_tcltk_includes" != "x$with_tcltk_libs" + then + as_fn_error $? "use both --with-tcltk-includes='...' and --with-tcltk-libs='...' or neither" "$LINENO" 5 + fi + TCLTK_INCLUDES="" + TCLTK_LIBS="" +else + TCLTK_INCLUDES="$with_tcltk_includes" + TCLTK_LIBS="$with_tcltk_libs" +fi + # Check for --with-dbmliborder { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-dbmliborder" >&5 $as_echo_n "checking for --with-dbmliborder... " >&6; } diff --git a/configure.ac b/configure.ac --- a/configure.ac +++ b/configure.ac @@ -2282,6 +2282,34 @@ AC_MSG_RESULT($enable_loadable_sqlite_extensions) +# Check for --with-tcltk-includes=path and --with-tcltk-libs=path +AC_SUBST(TCLTK_INCLUDES) +AC_SUBST(TCLTK_LIBS) +AC_MSG_CHECKING(for --with-tcltk-includes) +AC_ARG_WITH(tcltk-includes, + AS_HELP_STRING([--with-tcltk-includes='-I...'], [override search for Tcl and Tk include files]), + [], + [with_tcltk_includes="default"]) +AC_MSG_RESULT($with_tcltk_includes) +AC_MSG_CHECKING(for --with-tcltk-libs) +AC_ARG_WITH(tcltk-libs, + AS_HELP_STRING([--with-tcltk-libs='-L...'], [override search for Tcl and Tk libs]), + [], + [with_tcltk_libs="default"]) +AC_MSG_RESULT($with_tcltk_libs) +if test "x$with_tcltk_includes" = xdefault || test "x$with_tcltk_libs" = xdefault +then + if test "x$with_tcltk_includes" != "x$with_tcltk_libs" + then + AC_MSG_ERROR([use both --with-tcltk-includes='...' and --with-tcltk-libs='...' or neither]) + fi + TCLTK_INCLUDES="" + TCLTK_LIBS="" +else + TCLTK_INCLUDES="$with_tcltk_includes" + TCLTK_LIBS="$with_tcltk_libs" +fi + # Check for --with-dbmliborder AC_MSG_CHECKING(for --with-dbmliborder) AC_ARG_WITH(dbmliborder, diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -1545,6 +1545,41 @@ return missing + def detect_tkinter_explicitly(self): + # Build _tkinter using explicit locations for Tcl/Tk. + # + # This is enabled when both arguments are given to ./configure: + # + # --with-tcltk-includes="-I/path/to/tclincludes \ + # -I/path/to/tkincludes" + # --with-tcltk-libs="-L/path/to/tcllibs -ltclm.n \ + # -L/path/to/tklibs -ltkm.n" + # + # These values can also be specified or overriden via make: + # make TCLTK_INCLUDES="..." TCLTK_LIBS="..." + # + # This can be useful for building and testing tkinter with multiple + # versions of Tcl/Tk. Note that a build of Tk depends on a particular + # build of Tcl so you need to specify both arguments and use care when + # overriding. + + # The _TCLTK variables are created in the Makefile sharedmods target. + tcltk_includes = os.environ.get('_TCLTK_INCLUDES') + tcltk_libs = os.environ.get('_TCLTK_LIBS') + if not (tcltk_includes and tcltk_libs): + # Resume default configuration search. + return 0 + + extra_compile_args = tcltk_includes.split() + extra_link_args = tcltk_libs.split() + ext = Extension('_tkinter', ['_tkinter.c', 'tkappinit.c'], + define_macros=[('WITH_APPINIT', 1)], + extra_compile_args = extra_compile_args, + extra_link_args = extra_link_args, + ) + self.extensions.append(ext) + return 1 + def detect_tkinter_darwin(self, inc_dirs, lib_dirs): # The _tkinter module, using frameworks. Since frameworks are quite # different the UNIX search logic is not sharable. @@ -1634,10 +1669,16 @@ self.extensions.append(ext) return 1 - def detect_tkinter(self, inc_dirs, lib_dirs): # The _tkinter module. + # Check whether --with-tcltk-includes and --with-tcltk-libs were + # configured or passed into the make target. If so, use these values + # to build tkinter and bypass the searches for Tcl and TK in standard + # locations. + if self.detect_tkinter_explicitly(): + return + # Rather than complicate the code below, detecting and building # AquaTk is a separate method. Only one Tkinter will be built on # Darwin - either AquaTk, if it is found, or X11 based Tk. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 10:22:05 2013 From: python-checkins at python.org (ned.deily) Date: Fri, 6 Sep 2013 10:22:05 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2315663=3A_Tcl/Tk_8?= =?utf-8?q?=2E5=2E14_is_now_included_with_the_OS_X_10=2E6+?= Message-ID: <3cWWvx59zLz7LjS@mail.python.org> http://hg.python.org/cpython/rev/985384cd6365 changeset: 85560:985384cd6365 user: Ned Deily date: Fri Sep 06 01:18:36 2013 -0700 summary: Issue #15663: Tcl/Tk 8.5.14 is now included with the OS X 10.6+ 64-bit/32-bit installer for 10.6+. It is no longer necessary to install a third-party version of Tcl/Tk 8.5 to work around the problems in the Apple-supplied Tcl/Tk 8.5 shipped in OS X 10.6 and later releases. files: Mac/BuildScript/README.txt | 33 ++++++- Mac/BuildScript/build-installer.py | 77 ++++++++++++++- Mac/BuildScript/resources/ReadMe.txt | 51 +++++++-- Mac/BuildScript/resources/Welcome.rtf | 16 +- Misc/NEWS | 5 + 5 files changed, 155 insertions(+), 27 deletions(-) diff --git a/Mac/BuildScript/README.txt b/Mac/BuildScript/README.txt --- a/Mac/BuildScript/README.txt +++ b/Mac/BuildScript/README.txt @@ -57,13 +57,42 @@ * NCurses 5.9 (http://bugs.python.org/issue15037) * SQLite 3.7.13 + * Tcl 8.5.14 + * Tk 8.5.14 * XZ 5.0.3 - uses system-supplied versions of third-party libraries * readline module links with Apple BSD editline (libedit) - - requires ActiveState Tcl/Tk 8.5.9 (or later) to be installed for building + - requires ActiveState Tcl/Tk 8.5.14 (or later) to be installed for building + + * Beginning with Python 3.4 alpha2, this installer now includes its own + private copy of Tcl and Tk 8.5.14 libraries and thus is no longer + dependent on the buggy releases of Aqua Cocoa Tk 8.5 shipped with + OS X 10.6 or on installing a newer third-party version of Tcl/Tk + in /Library/Frameworks, such as from ActiveState. Because this + is a new feature, it should be considered somewhat experimental and + subject to change prior to the final release of Python 3.4. If it + is necessary to fallback to using a third-party Tcl/Tk because of + a problem with the private Tcl/Tk, there is a backup version of + the _tkinter extension included which will dynamically link to + Tcl and Tk frameworks in /Library/Frameworks as in previous releases. + To enable (for all users of this Python 3.4):: + + sudo bash + cd /Library/Frameworks/Python.framework/Versions/3.4 + cd ./lib/python3.4/lib-dynload + cp -p _tkinter.so.framework _tkinter.so + exit + + To restore using Python's private versions of Tcl and Tk:: + + sudo bash + cd /Library/Frameworks/Python.framework/Versions/3.4 + cd ./lib/python3.4/lib-dynload + cp -p _tkinter.so.private _tkinter.so + exit - recommended build environment: @@ -82,7 +111,7 @@ considered a migration aid by Apple and is not likely to be fixed, its use should be avoided. The other compiler, ``clang``, has been undergoing rapid development. While it appears to have become - production-ready in the most recent Xcode 4 releases (Xcode 4.4.1 + production-ready in the most recent Xcode 4 releases (Xcode 4.6.3 as of this writing), there are still some open issues when building Python and there has not yet been the level of exposure in production environments that the Xcode 3 gcc-4.2 compiler has had. diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -192,6 +192,44 @@ LT_10_5 = bool(DEPTARGET < '10.5') + if DEPTARGET > '10.5': + result.extend([ + dict( + name="Tcl 8.5.14", + url="ftp://ftp.tcl.tk/pub/tcl//tcl8_5/tcl8.5.14-src.tar.gz", + checksum='44b50e58ab45dd272f6714dce2129123', + buildDir="unix", + configure_pre=[ + '--enable-shared', + '--enable-threads', + '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), + ], + useLDFlags=False, + install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ + "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), + "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.5'%(getVersion())), + }, + ), + dict( + name="Tk 8.5.14", + url="ftp://ftp.tcl.tk/pub/tcl//tcl8_5/tk8.5.14-src.tar.gz", + checksum='a9c48921b3688020470cd3a6dd4e867d', + buildDir="unix", + configure_pre=[ + '--enable-aqua', + '--enable-shared', + '--enable-threads', + '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), + ], + useLDFlags=False, + install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ + "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), + "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.5'%(getVersion())), + "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.5'%(getVersion())), + }, + ), + ]) + if getVersionTuple() >= (3, 3): result.extend([ dict( @@ -525,6 +563,20 @@ % frameworks['Tk'], ] + # For 10.6+ builds, we build two versions of _tkinter: + # - the traditional version (renamed to _tkinter.so.framework) linked + # with /Library/Frameworks/{Tcl,Tk}.framework + # - the default version linked with our private copies of Tcl and Tk + if DEPTARGET > '10.5': + EXPECTED_SHARED_LIBS['_tkinter.so.framework'] = \ + EXPECTED_SHARED_LIBS['_tkinter.so'] + EXPECTED_SHARED_LIBS['_tkinter.so'] = [ + "/Library/Frameworks/Python.framework/Versions/%s/lib/libtcl%s.dylib" + % (getVersion(), frameworks['Tcl']), + "/Library/Frameworks/Python.framework/Versions/%s/lib/libtk%s.dylib" + % (getVersion(), frameworks['Tk']), + ] + # Remove inherited environment variables which might influence build environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_', 'LD_', 'LIBRARY_', 'PATH', 'PYTHON'] @@ -633,13 +685,19 @@ XXX: This function assumes that archives contain a toplevel directory that is has the same name as the basename of the archive. This is - save enough for anything we use. + safe enough for almost anything we use. Unfortunately, it does not + work for current Tcl and Tk source releases where the basename of + the archive ends with "-src" but the uncompressed directory does not. + For now, just special case Tcl and Tk tar.gz downloads. """ curdir = os.getcwd() try: os.chdir(builddir) if archiveName.endswith('.tar.gz'): retval = os.path.basename(archiveName[:-7]) + if ((retval.startswith('tcl') or retval.startswith('tk')) + and retval.endswith('-src')): + retval = retval[:-4] if os.path.exists(retval): shutil.rmtree(retval) fp = os.popen("tar zxf %s 2>&1"%(shellQuote(archiveName),), 'r') @@ -903,6 +961,23 @@ print("Running make") runCommand("make") + # For deployment targets of 10.6 and higher, we build our own version + # of Tcl and Cocoa Aqua Tk libs because the Apple-supplied Tk 8.5 is + # out-of-date and has critical bugs. Save the _tkinter.so that was + # linked with /Library/Frameworks/{Tck,Tk}.framework and build + # another _tkinter.so linked with our private Tcl and Tk libs. + if DEPTARGET > '10.5': + runCommand("find build -name '_tkinter.so' " + " -execdir mv '{}' '{}'.framework \;") + print("Running make to rebuild _tkinter") + runCommand("make TCLTK_INCLUDES='-I%s/libraries/usr/local/include' " + "TCLTK_LIBS='-L%s/libraries/usr/local/lib -ltcl8.5 -ltk8.5'"%( + shellQuote(WORKDIR)[1:-1], + shellQuote(WORKDIR)[1:-1])) + # make a backup copy, just in case + runCommand("find build -name '_tkinter.so' " + " -execdir cp -p '{}' '{}'.private \;") + print("Running make install") runCommand("make install DESTDIR=%s"%( shellQuote(rootDir))) diff --git a/Mac/BuildScript/resources/ReadMe.txt b/Mac/BuildScript/resources/ReadMe.txt --- a/Mac/BuildScript/resources/ReadMe.txt +++ b/Mac/BuildScript/resources/ReadMe.txt @@ -2,10 +2,12 @@ $MACOSX_DEPLOYMENT_TARGET for the following architecture(s): $ARCHITECTURES. -Installation requires approximately $INSTALL_SIZE MB of disk space, -ignore the message that it will take zero bytes. + **** IMPORTANT **** -If you are attempting to install on an OS X 10.8 system, you may +Installing on OS X 10.8 (Mountain Lion) or later systems +======================================================== + +If you are attempting to install on an OS X 10.8+ system, you may see a message that Python can't be installed because it is from an unidentified developer. This is because this Python installer package is not yet compatible with the Gatekeeper security feature @@ -15,22 +17,40 @@ installer package icon. Then select "Open using ... Installer" from the contextual menu that appears. + **** IMPORTANT changes if you use IDLE and Tkinter **** + +Installing a third-party version of Tcl/Tk is no longer required +================================================================ + +Beginning with Python 3.4 alpha2, the 10.6+ 64-bit installer now +comes with its own private copy of Tcl and Tk 8.5 libraries. For +this version of Python, it is no longer necessary to install +a third-party version of Tcl/Tk 8.5, such as those from ActiveState, +to work around the problematic versions of Tcl/Tk 8.5 shipped by +Apple in OS X 10.6 and later. (This does not change the requirements +for older versions of Python installed from python.org.) By default, +this version of Python will always use its own private version, +regardless of whether a third-party Tcl/Tk is installed. +The 10.5+ 32-bit-only installer continues to use Tcl/Tk 8.4, +either a third-party or system-supplied version. +Since this is a new feature, it should be considered somewhat +experimental and subject to change prior to the final release of +Python 3.4. Please report any problems found to the Python bug +tracker at http://bugs.python.org. + +Visit http://www.python.org/download/mac/tcltk/ +for current information about supported and recommended versions of +Tcl/Tk for this version of Python and of Mac OS X. + +Using this version of Python on OS X +==================================== + Python consists of the Python programming language interpreter, plus a set of programs to allow easy access to it for Mac users including an integrated development environment, IDLE, plus a set of pre-built extension modules that open up specific Macintosh technologies to Python programs. - **** IMPORTANT **** - -To use IDLE or other programs that use the tkinter graphical user -interface toolkit, you may need to install a third-party version of -the Tcl/Tk frameworks. Visit http://www.python.org/download/mac/tcltk/ -for current information about supported and recommended versions of -Tcl/Tk for this version of Python and of Mac OS X. - - ******************* - The installer puts applications, an "Update Shell Profile" command, and a link to the optionally installed Python Documentation into the "Python $VERSION" subfolder of the system Applications folder, @@ -41,12 +61,15 @@ "bin" directory inside the framework to your shell's search path. You must install onto your current boot disk, even though the -installer does not enforce this, otherwise things will not work. +installer may not enforce this, otherwise things will not work. You can verify the integrity of the disk image file containing the installer package and this ReadMe file by comparing its md5 checksum and size with the values published on the release page linked at http://www.python.org/download/ +Installation requires approximately $INSTALL_SIZE MB of disk space, +ignore the message that it will take zero bytes. + More information on Python in general can be found at http://www.python.org. diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,7 +1,7 @@ -{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\rtf1\ansi\ansicpg1252\cocoartf1187\cocoasubrtf390 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} -\paperw11904\paperh16836\margl1440\margr1440\vieww9640\viewh10620\viewkind0 +\paperw11905\paperh16837\margl1440\margr1440\vieww9640\viewh10620\viewkind0 \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640 \f0\fs24 \cf0 This package will install @@ -25,11 +25,7 @@ \b0 at any time to make $FULL_VERSION the default Python 3 version. This version can co-exist with other installed versions of Python 3 and Python 2.\ \ -\b IMPORTANT: -\b0 -\b IDLE -\b0 and other programs using the -\b tkinter -\b0 graphical user interface toolkit require specific versions of the +\b IMPORTANT for users of IDLE and tkinter: +\b0 Beginning with Python 3.4 alpha 2, it is no longer necessary to install third-party versions of the \b Tcl/Tk -\b0 platform independent windowing toolkit. Visit {\field{\*\fldinst{HYPERLINK "http://www.python.org/download/mac/tcltk/"}}{\fldrslt http://www.python.org/download/mac/tcltk/}} for current information on supported and recommended versions of Tcl/Tk for this version of Python and Mac OS X.} \ No newline at end of file +\b0 platform independent windowing toolkit. Please read the ReadMe file and visit {\field{\*\fldinst{HYPERLINK "http://www.python.org/download/mac/tcltk/"}}{\fldrslt http://www.python.org/download/mac/tcltk/}} for more information on supported and recommended versions of Tcl/Tk for this version of Python and Mac OS X.} \ No newline at end of file diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -225,6 +225,11 @@ - Issue #1584: Provide configure options to override default search paths for Tcl and Tk when building _tkinter. +- Issue #15663: Tcl/Tk 8.5.14 is now included with the OS X 10.6+ 64-/32-bit + installer. It is no longer necessary to install a third-party version of + Tcl/Tk 8.5 to work around the problems in the Apple-supplied Tcl/Tk 8.5 + shipped in OS X 10.6 and later releases. + Tools/Demos ----------- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 10:51:56 2013 From: python-checkins at python.org (vinay.sajip) Date: Fri, 6 Sep 2013 10:51:56 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4OTM5?= =?utf-8?q?=3A_Updated_venv_documentation_with_some_clarifications=2E?= Message-ID: <3cWXZN0qq4z7LjR@mail.python.org> http://hg.python.org/cpython/rev/ad09332f856f changeset: 85561:ad09332f856f branch: 3.3 parent: 85557:6e76cfeee777 user: Vinay Sajip date: Fri Sep 06 09:50:43 2013 +0100 summary: Issue #18939: Updated venv documentation with some clarifications. files: Doc/library/venv.rst | 14 ++++++++++++++ 1 files changed, 14 insertions(+), 0 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -21,6 +21,7 @@ creation of environments with various Python versions) and can have its own independent set of installed Python packages in its site directories. +See :pep:`405` for more information about Python virtual environments. Creating virtual environments ----------------------------- @@ -61,6 +62,19 @@ ignored from all distutils configuration files to prevent projects being inadvertently installed outside of the virtual environment. + When working in a command shell, users can make a venv active by running an + ``activate`` script in the venv's executables directory (the precise filename + is shell-dependent), which prepends the venv's directory for executables to + the ``PATH`` environment variable for the running shell. There should be no + need in other circumstances to activate a venv -- scripts installed into + venvs have a shebang line which points to the venv's Python interpreter. This + means that the script will run with that interpreter regardless of the value + of ``PATH``. On Windows, shebang line processing is supported if you have the + Python Launcher for Windows installed (this was added to Python in 3.3 - see + :pep:`397` for more details). Thus, double-clicking an installed script in + a Windows Explorer window should run the script with the correct interpreter + without there needing to be any reference to its venv in ``PATH``. + API --- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 10:51:57 2013 From: python-checkins at python.org (vinay.sajip) Date: Fri, 6 Sep 2013 10:51:57 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Closes_=2318939=3A_Merged_documentation_update_from_3=2E?= =?utf-8?q?3=2E?= Message-ID: <3cWXZP2d8zz7Ljn@mail.python.org> http://hg.python.org/cpython/rev/408b6b3dcf9a changeset: 85562:408b6b3dcf9a parent: 85560:985384cd6365 parent: 85561:ad09332f856f user: Vinay Sajip date: Fri Sep 06 09:51:27 2013 +0100 summary: Closes #18939: Merged documentation update from 3.3. files: Doc/library/venv.rst | 14 ++++++++++++++ 1 files changed, 14 insertions(+), 0 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -21,6 +21,7 @@ creation of environments with various Python versions) and can have its own independent set of installed Python packages in its site directories. +See :pep:`405` for more information about Python virtual environments. Creating virtual environments ----------------------------- @@ -61,6 +62,19 @@ ignored from all distutils configuration files to prevent projects being inadvertently installed outside of the virtual environment. + When working in a command shell, users can make a venv active by running an + ``activate`` script in the venv's executables directory (the precise filename + is shell-dependent), which prepends the venv's directory for executables to + the ``PATH`` environment variable for the running shell. There should be no + need in other circumstances to activate a venv -- scripts installed into + venvs have a shebang line which points to the venv's Python interpreter. This + means that the script will run with that interpreter regardless of the value + of ``PATH``. On Windows, shebang line processing is supported if you have the + Python Launcher for Windows installed (this was added to Python in 3.3 - see + :pep:`397` for more details). Thus, double-clicking an installed script in + a Windows Explorer window should run the script with the correct interpreter + without there needing to be any reference to its venv in ``PATH``. + API --- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 11:11:51 2013 From: python-checkins at python.org (vinay.sajip) Date: Fri, 6 Sep 2013 11:11:51 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4OTQw?= =?utf-8?q?=3A_Handled_low-volume_logging_when_delay_is_True=2E?= Message-ID: <3cWY1M4T7fz7LjW@mail.python.org> http://hg.python.org/cpython/rev/6a591870017c changeset: 85563:6a591870017c branch: 2.7 parent: 85552:dc4e6b48c321 user: Vinay Sajip date: Fri Sep 06 10:09:45 2013 +0100 summary: Issue #18940: Handled low-volume logging when delay is True. files: Lib/logging/handlers.py | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -137,7 +137,9 @@ dfn = self.baseFilename + ".1" if os.path.exists(dfn): os.remove(dfn) - os.rename(self.baseFilename, dfn) + # Issue 18940: A file may not have been created if delay is True. + if os.path.exists(self.baseFilename): + os.rename(self.baseFilename, dfn) #print "%s -> %s" % (self.baseFilename, dfn) self.stream = self._open() @@ -343,7 +345,9 @@ dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple) if os.path.exists(dfn): os.remove(dfn) - os.rename(self.baseFilename, dfn) + # Issue 18940: A file may not have been created if delay is True. + if os.path.exists(self.baseFilename): + os.rename(self.baseFilename, dfn) if self.backupCount > 0: # find the oldest log file and delete it #s = glob.glob(self.baseFilename + ".20*") -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 11:11:52 2013 From: python-checkins at python.org (vinay.sajip) Date: Fri, 6 Sep 2013 11:11:52 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4OTQw?= =?utf-8?q?=3A_Handled_low-volume_logging_when_delay_is_True=2E?= Message-ID: <3cWY1N732dz7Ljn@mail.python.org> http://hg.python.org/cpython/rev/324774a59256 changeset: 85564:324774a59256 branch: 3.3 parent: 85561:ad09332f856f user: Vinay Sajip date: Fri Sep 06 10:10:22 2013 +0100 summary: Issue #18940: Handled low-volume logging when delay is True. files: Lib/logging/handlers.py | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -111,7 +111,9 @@ what the source is rotated to, e.g. 'test.log.1'. """ if not callable(self.rotator): - os.rename(source, dest) + # Issue 18940: A file may not have been created if delay is True. + if os.path.exists(source): + os.rename(source, dest) else: self.rotator(source, dest) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 11:11:54 2013 From: python-checkins at python.org (vinay.sajip) Date: Fri, 6 Sep 2013 11:11:54 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Closes_=2318940=3A_Merged_fix_from_3=2E3=2E?= Message-ID: <3cWY1Q2DJJz7Ljr@mail.python.org> http://hg.python.org/cpython/rev/8002aee72837 changeset: 85565:8002aee72837 parent: 85562:408b6b3dcf9a parent: 85564:324774a59256 user: Vinay Sajip date: Fri Sep 06 10:11:37 2013 +0100 summary: Closes #18940: Merged fix from 3.3. files: Lib/logging/handlers.py | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -109,7 +109,9 @@ what the source is rotated to, e.g. 'test.log.1'. """ if not callable(self.rotator): - os.rename(source, dest) + # Issue 18940: A file may not have been created if delay is True. + if os.path.exists(source): + os.rename(source, dest) else: self.rotator(source, dest) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 11:27:03 2013 From: python-checkins at python.org (vinay.sajip) Date: Fri, 6 Sep 2013 11:27:03 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4OTQx?= =?utf-8?q?=3A_Respected_delay_when_doing_rollover=2E?= Message-ID: <3cWYLv6wMqz7LjR@mail.python.org> http://hg.python.org/cpython/rev/4d45f1ed1179 changeset: 85566:4d45f1ed1179 branch: 2.7 parent: 85563:6a591870017c user: Vinay Sajip date: Fri Sep 06 10:24:08 2013 +0100 summary: Issue #18941: Respected delay when doing rollover. files: Lib/logging/__init__.py | 1 + Lib/logging/handlers.py | 13 ++++--------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -893,6 +893,7 @@ self.baseFilename = os.path.abspath(filename) self.mode = mode self.encoding = encoding + self.delay = delay if delay: #We don't open the stream, but we still need to call the #Handler constructor to set level, formatter, lock etc. diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -140,8 +140,8 @@ # Issue 18940: A file may not have been created if delay is True. if os.path.exists(self.baseFilename): os.rename(self.baseFilename, dfn) - #print "%s -> %s" % (self.baseFilename, dfn) - self.stream = self._open() + if not self.delay: + self.stream = self._open() def shouldRollover(self, record): """ @@ -349,15 +349,10 @@ if os.path.exists(self.baseFilename): os.rename(self.baseFilename, dfn) if self.backupCount > 0: - # find the oldest log file and delete it - #s = glob.glob(self.baseFilename + ".20*") - #if len(s) > self.backupCount: - # s.sort() - # os.remove(s[0]) for s in self.getFilesToDelete(): os.remove(s) - #print "%s -> %s" % (self.baseFilename, dfn) - self.stream = self._open() + if not self.delay: + self.stream = self._open() newRolloverAt = self.computeRollover(currentTime) while newRolloverAt <= currentTime: newRolloverAt = newRolloverAt + self.interval -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 11:27:05 2013 From: python-checkins at python.org (vinay.sajip) Date: Fri, 6 Sep 2013 11:27:05 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4OTQx?= =?utf-8?q?=3A_Respected_delay_when_doing_rollover=2E?= Message-ID: <3cWYLx1rhQz7Ljm@mail.python.org> http://hg.python.org/cpython/rev/0577c9a82c0a changeset: 85567:0577c9a82c0a branch: 3.3 parent: 85564:324774a59256 user: Vinay Sajip date: Fri Sep 06 10:25:31 2013 +0100 summary: Issue #18941: Respected delay when doing rollover. files: Lib/logging/__init__.py | 5 +++-- Lib/logging/handlers.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2001-2012 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2013 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -18,7 +18,7 @@ Logging package for Python. Based on PEP 282 and comments thereto in comp.lang.python. -Copyright (C) 2001-2012 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2013 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ @@ -957,6 +957,7 @@ self.baseFilename = os.path.abspath(filename) self.mode = mode self.encoding = encoding + self.delay = delay if delay: #We don't open the stream, but we still need to call the #Handler constructor to set level, formatter, lock etc. diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -174,7 +174,8 @@ if os.path.exists(dfn): os.remove(dfn) self.rotate(self.baseFilename, dfn) - self.stream = self._open() + if not self.delay: + self.stream = self._open() def shouldRollover(self, record): """ @@ -382,7 +383,8 @@ if self.backupCount > 0: for s in self.getFilesToDelete(): os.remove(s) - self.stream = self._open() + if not self.delay: + self.stream = self._open() newRolloverAt = self.computeRollover(currentTime) while newRolloverAt <= currentTime: newRolloverAt = newRolloverAt + self.interval -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 11:27:06 2013 From: python-checkins at python.org (vinay.sajip) Date: Fri, 6 Sep 2013 11:27:06 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Closes_=2318941=3A_Merged_fix_from_3=2E3=2E?= Message-ID: <3cWYLy454xz7LkX@mail.python.org> http://hg.python.org/cpython/rev/7627fea85a6d changeset: 85568:7627fea85a6d parent: 85565:8002aee72837 parent: 85567:0577c9a82c0a user: Vinay Sajip date: Fri Sep 06 10:26:48 2013 +0100 summary: Closes #18941: Merged fix from 3.3. files: Lib/logging/__init__.py | 5 +++-- Lib/logging/handlers.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2001-2012 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2013 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -18,7 +18,7 @@ Logging package for Python. Based on PEP 282 and comments thereto in comp.lang.python. -Copyright (C) 2001-2012 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2013 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ @@ -971,6 +971,7 @@ self.baseFilename = os.path.abspath(filename) self.mode = mode self.encoding = encoding + self.delay = delay if delay: #We don't open the stream, but we still need to call the #Handler constructor to set level, formatter, lock etc. diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -172,7 +172,8 @@ if os.path.exists(dfn): os.remove(dfn) self.rotate(self.baseFilename, dfn) - self.stream = self._open() + if not self.delay: + self.stream = self._open() def shouldRollover(self, record): """ @@ -394,7 +395,8 @@ if self.backupCount > 0: for s in self.getFilesToDelete(): os.remove(s) - self.stream = self._open() + if not self.delay: + self.stream = self._open() newRolloverAt = self.computeRollover(currentTime) while newRolloverAt <= currentTime: newRolloverAt = newRolloverAt + self.interval -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 15:14:57 2013 From: python-checkins at python.org (eli.bendersky) Date: Fri, 6 Sep 2013 15:14:57 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4ODQ5?= =?utf-8?q?=3A_Fixed_a_Windows-specific_tempfile_bug_where_collision_with_?= =?utf-8?q?an?= Message-ID: <3cWfPs2YFvz7LjW@mail.python.org> http://hg.python.org/cpython/rev/7611e7244bdd changeset: 85569:7611e7244bdd branch: 3.3 parent: 85567:0577c9a82c0a user: Eli Bendersky date: Fri Sep 06 06:11:19 2013 -0700 summary: Issue #18849: Fixed a Windows-specific tempfile bug where collision with an existing directory caused mkstemp and related APIs to fail instead of retrying. Report and fix by Vlad Shcherbina. files: Lib/tempfile.py | 7 +++++++ Lib/test/test_tempfile.py | 26 ++++++++++++++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 4 ++++ 4 files changed, 38 insertions(+), 0 deletions(-) diff --git a/Lib/tempfile.py b/Lib/tempfile.py --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -219,6 +219,13 @@ return (fd, _os.path.abspath(file)) except FileExistsError: continue # try again + except PermissionError: + # This exception is thrown when a directory with the chosen name + # already exists on windows. + if _os.name == 'nt': + continue + else: + raise raise FileExistsError(_errno.EEXIST, "No usable temporary file name found") diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -372,6 +372,32 @@ os.lseek(f.fd, 0, os.SEEK_SET) self.assertEqual(os.read(f.fd, 20), b"blat") + def test_collision_with_existing_directory(self): + # _mkstemp_inner tries another name when a directory with + # the chosen name already exists + container_dir = tempfile.mkdtemp() + try: + def mock_get_candidate_names(): + return iter(['aaa', 'aaa', 'bbb']) + with support.swap_attr(tempfile, + '_get_candidate_names', + mock_get_candidate_names): + dir = tempfile.mkdtemp(dir=container_dir) + self.assertTrue(dir.endswith('aaa')) + + flags = tempfile._bin_openflags + (fd, name) = tempfile._mkstemp_inner(container_dir, + tempfile.template, + '', + flags) + try: + self.assertTrue(name.endswith('bbb')) + finally: + os.close(fd) + os.unlink(name) + finally: + support.rmtree(container_dir) + class TestGetTempPrefix(BaseTestCase): """Test gettempprefix().""" diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1121,6 +1121,7 @@ Ha Shao Mark Shannon Richard Shapiro +Vlad Shcherbina Justin Sheehy Charlie Shepherd Bruce Sherwood diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -280,6 +280,10 @@ - Issue #18113: Fixed a refcount leak in the curses.panel module's set_userptr() method. Reported by Atsuo Ishimoto. +- Issue #18849: Fixed a Windows-specific tempfile bug where collision with an + existing directory caused mkstemp and related APIs to fail instead of + retrying. Report and fix by Vlad Shcherbina. + C API ----- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 15:14:58 2013 From: python-checkins at python.org (eli.bendersky) Date: Fri, 6 Sep 2013 15:14:58 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2318849=3A_Fixed_a_Windows-specific_tempfile_bug_?= =?utf-8?q?where_collision_with_an?= Message-ID: <3cWfPt6T0vz7Ljv@mail.python.org> http://hg.python.org/cpython/rev/035b61b52caa changeset: 85570:035b61b52caa parent: 85568:7627fea85a6d parent: 85569:7611e7244bdd user: Eli Bendersky date: Fri Sep 06 06:14:16 2013 -0700 summary: Issue #18849: Fixed a Windows-specific tempfile bug where collision with an existing directory caused mkstemp and related APIs to fail instead of retrying. Report and fix by Vlad Shcherbina. files: Lib/tempfile.py | 7 +++++++ Lib/test/test_tempfile.py | 26 ++++++++++++++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 4 ++++ 4 files changed, 38 insertions(+), 0 deletions(-) diff --git a/Lib/tempfile.py b/Lib/tempfile.py --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -199,6 +199,13 @@ return (fd, _os.path.abspath(file)) except FileExistsError: continue # try again + except PermissionError: + # This exception is thrown when a directory with the chosen name + # already exists on windows. + if _os.name == 'nt': + continue + else: + raise raise FileExistsError(_errno.EEXIST, "No usable temporary file name found") diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -373,6 +373,32 @@ os.lseek(f.fd, 0, os.SEEK_SET) self.assertEqual(os.read(f.fd, 20), b"blat") + def test_collision_with_existing_directory(self): + # _mkstemp_inner tries another name when a directory with + # the chosen name already exists + container_dir = tempfile.mkdtemp() + try: + def mock_get_candidate_names(): + return iter(['aaa', 'aaa', 'bbb']) + with support.swap_attr(tempfile, + '_get_candidate_names', + mock_get_candidate_names): + dir = tempfile.mkdtemp(dir=container_dir) + self.assertTrue(dir.endswith('aaa')) + + flags = tempfile._bin_openflags + (fd, name) = tempfile._mkstemp_inner(container_dir, + tempfile.template, + '', + flags) + try: + self.assertTrue(name.endswith('bbb')) + finally: + os.close(fd) + os.unlink(name) + finally: + support.rmtree(container_dir) + class TestGetTempPrefix(BaseTestCase): """Test gettempprefix().""" diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1157,6 +1157,7 @@ Ha Shao Mark Shannon Richard Shapiro +Vlad Shcherbina Justin Sheehy Charlie Shepherd Bruce Sherwood diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -189,6 +189,10 @@ - Issue #8860: Fixed rounding in timedelta constructor. +- Issue #18849: Fixed a Windows-specific tempfile bug where collision with an + existing directory caused mkstemp and related APIs to fail instead of + retrying. Report and fix by Vlad Shcherbina. + Tests ----- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 15:17:56 2013 From: python-checkins at python.org (eli.bendersky) Date: Fri, 6 Sep 2013 15:17:56 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogQ2xvc2UgIzE4ODQ5?= =?utf-8?q?=3A_Fixed_a_Windows-specific_tempfile_bug_where_collision_with_?= =?utf-8?q?an?= Message-ID: <3cWfTJ44wWz7LlB@mail.python.org> http://hg.python.org/cpython/rev/e0037f266d45 changeset: 85571:e0037f266d45 branch: 2.7 parent: 85566:4d45f1ed1179 user: Eli Bendersky date: Fri Sep 06 06:17:15 2013 -0700 summary: Close #18849: Fixed a Windows-specific tempfile bug where collision with an existing directory caused mkstemp and related APIs to fail instead of retrying. Report and fix by Vlad Shcherbina. files: Lib/tempfile.py | 4 ++++ Lib/test/test_tempfile.py | 26 ++++++++++++++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 5 +++++ 4 files changed, 36 insertions(+), 0 deletions(-) diff --git a/Lib/tempfile.py b/Lib/tempfile.py --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -242,6 +242,10 @@ except OSError, e: if e.errno == _errno.EEXIST: continue # try again + if _os.name == 'nt' and e.errno == _errno.EACCES: + # On windows, when a directory with the chosen name already + # exists, EACCES error code is returned instead of EEXIST. + continue raise raise IOError, (_errno.EEXIST, "No usable temporary file name found") diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -386,6 +386,32 @@ self.do_create(bin=0).write("blat\n") # XXX should test that the file really is a text file + def test_collision_with_existing_directory(self): + # _mkstemp_inner tries another name when a directory with + # the chosen name already exists + container_dir = tempfile.mkdtemp() + try: + def mock_get_candidate_names(): + return iter(['aaa', 'aaa', 'bbb']) + with support.swap_attr(tempfile, + '_get_candidate_names', + mock_get_candidate_names): + dir = tempfile.mkdtemp(dir=container_dir) + self.assertTrue(dir.endswith('aaa')) + + flags = tempfile._bin_openflags + (fd, name) = tempfile._mkstemp_inner(container_dir, + tempfile.template, + '', + flags) + try: + self.assertTrue(name.endswith('bbb')) + finally: + os.close(fd) + os.unlink(name) + finally: + support.rmtree(container_dir) + test_classes.append(test__mkstemp_inner) diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -936,6 +936,7 @@ Ha Shao Mark Shannon Richard Shapiro +Vlad Shcherbina Justin Sheehy Charlie Shepherd Bruce Sherwood diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -156,6 +156,11 @@ - Issue #18113: Fixed a refcount leak in the curses.panel module's set_userptr() method. Reported by Atsuo Ishimoto. +- Issue #18849: Fixed a Windows-specific tempfile bug where collision with an + existing directory caused mkstemp and related APIs to fail instead of + retrying. Report and fix by Vlad Shcherbina. + + Tools/Demos ----------- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 15:49:55 2013 From: python-checkins at python.org (eli.bendersky) Date: Fri, 6 Sep 2013 15:49:55 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318920=3A_argparse?= =?utf-8?q?=27s_default_version_action_=28for_-v=2C_--version=29_should?= Message-ID: <3cWgBC4lvZz7LjM@mail.python.org> http://hg.python.org/cpython/rev/ec9a4b77f37b changeset: 85572:ec9a4b77f37b parent: 85570:035b61b52caa user: Eli Bendersky date: Fri Sep 06 06:49:15 2013 -0700 summary: Issue #18920: argparse's default version action (for -v, --version) should output to stdout, matching the 'python -v' Reported by Wolfgang Maier files: Lib/argparse.py | 3 ++- Lib/test/test_argparse.py | 4 ++-- Misc/NEWS | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/argparse.py b/Lib/argparse.py --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1037,7 +1037,8 @@ version = parser.version formatter = parser._get_formatter() formatter.add_text(version) - parser.exit(message=formatter.format_help()) + parser._print_message(formatter.format_help(), _sys.stdout) + parser.exit() class _SubParsersAction(Action): diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -4359,7 +4359,7 @@ def test_version_format(self): parser = ErrorRaisingArgumentParser(prog='PPP') parser.add_argument('-v', '--version', action='version', version='%(prog)s 3.5') - msg = self._get_error(parser.parse_args, ['-v']).stderr + msg = self._get_error(parser.parse_args, ['-v']).stdout self.assertEqual('PPP 3.5\n', msg) def test_version_no_help(self): @@ -4372,7 +4372,7 @@ def test_version_action(self): parser = ErrorRaisingArgumentParser(prog='XXX') parser.add_argument('-V', action='version', version='%(prog)s 3.7') - msg = self._get_error(parser.parse_args, ['-V']).stderr + msg = self._get_error(parser.parse_args, ['-V']).stdout self.assertEqual('XXX 3.7\n', msg) def test_no_help(self): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -193,6 +193,9 @@ existing directory caused mkstemp and related APIs to fail instead of retrying. Report and fix by Vlad Shcherbina. +- Issue #18920: argparse's default destination for the version action (-v, + --version) has also been changed to stdout, to match the Python executable. + Tests ----- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 15:56:39 2013 From: python-checkins at python.org (eli.bendersky) Date: Fri, 6 Sep 2013 15:56:39 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Update_whatsnew/3=2E4_wrt?= =?utf-8?q?=2E_--version_going_to_stdout=2E_=2318338=2C_=2318920=2C_=23189?= =?utf-8?q?22?= Message-ID: <3cWgKz5zbGz7LjZ@mail.python.org> http://hg.python.org/cpython/rev/587bdb940524 changeset: 85573:587bdb940524 user: Eli Bendersky date: Fri Sep 06 06:55:58 2013 -0700 summary: Update whatsnew/3.4 wrt. --version going to stdout. #18338, #18920, #18922 files: Doc/whatsnew/3.4.rst | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -428,6 +428,13 @@ (Contributed by Antoine Pitrou and ?ric Araujo in :issue:`5845`.) +Python invocation changes +========================= + +Invoking the Python interpreter with ``--version`` now outputs the version to +standard output instead of standard error (:issue:`18338`). Similar changes +were made to :mod:`argparse` (:issue:`18920`) and other modules that have +script-like invocation capabilities (:issue:`18922`). Optimizations ============= -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 16:17:19 2013 From: python-checkins at python.org (ethan.furman) Date: Fri, 6 Sep 2013 16:17:19 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Close_=2318924=3A__Block_n?= =?utf-8?q?aive_attempts_to_change_an_Enum_member=2E?= Message-ID: <3cWgnq4x1vz7Lkc@mail.python.org> http://hg.python.org/cpython/rev/1d88d04aade2 changeset: 85574:1d88d04aade2 user: Ethan Furman date: Fri Sep 06 07:16:48 2013 -0700 summary: Close #18924: Block naive attempts to change an Enum member. files: Lib/enum.py | 13 +++++++++++++ Lib/test/test_enum.py | 5 +++++ 2 files changed, 18 insertions(+), 0 deletions(-) diff --git a/Lib/enum.py b/Lib/enum.py --- a/Lib/enum.py +++ b/Lib/enum.py @@ -263,6 +263,19 @@ def __repr__(cls): return "" % cls.__name__ + def __setattr__(cls, name, value): + """Block attempts to reassign Enum members. + + A simple assignment to the class namespace only changes one of the + several possible ways to get an Enum member from the Enum class, + resulting in an inconsistent Enumeration. + + """ + member_map = cls.__dict__.get('_member_map_', {}) + if name in member_map: + raise AttributeError('Cannot reassign members.') + super().__setattr__(name, value) + def _create_(cls, class_name, names=None, *, module=None, type=None): """Convenience method to create a new Enum class. diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -152,6 +152,11 @@ with self.assertRaises(AttributeError): Season.SPRING.value = 2 + def test_changing_member(self): + Season = self.Season + with self.assertRaises(AttributeError): + Season.WINTER = 'really cold' + def test_invalid_names(self): with self.assertRaises(ValueError): class Wrong(Enum): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 19:09:15 2013 From: python-checkins at python.org (r.david.murray) Date: Fri, 6 Sep 2013 19:09:15 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_=2318852=3A_Handle_readlin?= =?utf-8?q?e=2E=5F=5Fdoc=5F=5F_being_None_in_site=2Epy_readline_activation?= =?utf-8?q?=2E?= Message-ID: <3cWlcC0J9hz7Lky@mail.python.org> http://hg.python.org/cpython/rev/3070fdd58645 changeset: 85575:3070fdd58645 user: R David Murray date: Fri Sep 06 13:08:08 2013 -0400 summary: #18852: Handle readline.__doc__ being None in site.py readline activation. Patch by Berker Peksag. files: Lib/site.py | 5 +++-- Misc/NEWS | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/site.py b/Lib/site.py --- a/Lib/site.py +++ b/Lib/site.py @@ -388,8 +388,9 @@ return # Reading the initialization (config) file may not be enough to set a - # completion key, so we set one first and then read the file - if 'libedit' in getattr(readline, '__doc__', ''): + # completion key, so we set one first and then read the file. + readline_doc = getattr(readline, '__doc__', '') + if readline_doc is not None and 'libedit' in readline_doc: readline.parse_and_bind('bind ^I rl_complete') else: readline.parse_and_bind('tab: complete') diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -56,6 +56,9 @@ Library ------- +- Issue #18852: Handle case of ``readline.__doc__`` being ``None`` in the new + readline activation code in ``site.py``. + - Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in the _sre moduel. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 20:50:13 2013 From: python-checkins at python.org (antoine.pitrou) Date: Fri, 6 Sep 2013 20:50:13 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318623=3A_Factor_o?= =?utf-8?q?ut_the_=5FSuppressCoreFiles_context_manager_into?= Message-ID: <3cWnrj0v3Nz7LjW@mail.python.org> http://hg.python.org/cpython/rev/28a1843c0fc1 changeset: 85576:28a1843c0fc1 user: Antoine Pitrou date: Fri Sep 06 20:50:00 2013 +0200 summary: Issue #18623: Factor out the _SuppressCoreFiles context manager into test.support. Patch by Valerie Lambert. files: Lib/test/support/__init__.py | 45 ++++++++++++++++++++++ Lib/test/test_subprocess.py | 47 +----------------------- Lib/test/test_support.py | 1 + Misc/NEWS | 3 + 4 files changed, 50 insertions(+), 46 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -57,6 +57,11 @@ except ImportError: lzma = None +try: + import resource +except ImportError: + resource = None + __all__ = [ "Error", "TestFailed", "ResourceDenied", "import_module", "verbose", "use_resources", "max_memuse", "record_original_stdout", @@ -77,6 +82,7 @@ "skip_unless_xattr", "import_fresh_module", "requires_zlib", "PIPE_MAX_SIZE", "failfast", "anticipate_failure", "run_with_tz", "requires_gzip", "requires_bz2", "requires_lzma", "suppress_crash_popup", + "SuppressCoreFiles", ] class Error(Exception): @@ -2055,3 +2061,42 @@ # actually override the attribute setattr(object_to_patch, attr_name, new_value) + + +class SuppressCoreFiles(object): + + """Try to prevent core files from being created.""" + old_limit = None + + def __enter__(self): + """Try to save previous ulimit, then set the soft limit to 0.""" + if resource is not None: + try: + self.old_limit = resource.getrlimit(resource.RLIMIT_CORE) + resource.setrlimit(resource.RLIMIT_CORE, (0, self.old_limit[1])) + except (ValueError, OSError): + pass + if sys.platform == 'darwin': + # Check if the 'Crash Reporter' on OSX was configured + # in 'Developer' mode and warn that it will get triggered + # when it is. + # + # This assumes that this context manager is used in tests + # that might trigger the next manager. + value = subprocess.Popen(['/usr/bin/defaults', 'read', + 'com.apple.CrashReporter', 'DialogType'], + stdout=subprocess.PIPE).communicate()[0] + if value.strip() == b'developer': + print("this test triggers the Crash Reporter, " + "that is intentional", end='') + sys.stdout.flush() + + def __exit__(self, *ignore_exc): + """Return core file behavior to default.""" + if self.old_limit is None: + return + if resource is not None: + try: + resource.setrlimit(resource.RLIMIT_CORE, self.old_limit) + except (ValueError, OSError): + pass diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -19,10 +19,6 @@ import textwrap try: - import resource -except ImportError: - resource = None -try: import threading except ImportError: threading = None @@ -1147,47 +1143,6 @@ fds_after_exception = os.listdir(fd_directory) self.assertEqual(fds_before_popen, fds_after_exception) - -# context manager -class _SuppressCoreFiles(object): - """Try to prevent core files from being created.""" - old_limit = None - - def __enter__(self): - """Try to save previous ulimit, then set it to (0, 0).""" - if resource is not None: - try: - self.old_limit = resource.getrlimit(resource.RLIMIT_CORE) - resource.setrlimit(resource.RLIMIT_CORE, (0, self.old_limit[1])) - except (ValueError, resource.error): - pass - - if sys.platform == 'darwin': - # Check if the 'Crash Reporter' on OSX was configured - # in 'Developer' mode and warn that it will get triggered - # when it is. - # - # This assumes that this context manager is used in tests - # that might trigger the next manager. - value = subprocess.Popen(['/usr/bin/defaults', 'read', - 'com.apple.CrashReporter', 'DialogType'], - stdout=subprocess.PIPE).communicate()[0] - if value.strip() == b'developer': - print("this tests triggers the Crash Reporter, " - "that is intentional", end='') - sys.stdout.flush() - - def __exit__(self, *args): - """Return core file behavior to default.""" - if self.old_limit is None: - return - if resource is not None: - try: - resource.setrlimit(resource.RLIMIT_CORE, self.old_limit) - except (ValueError, resource.error): - pass - - @unittest.skipIf(mswindows, "POSIX specific tests") class POSIXProcessTestCase(BaseTestCase): @@ -1276,7 +1231,7 @@ def test_run_abort(self): # returncode handles signal termination - with _SuppressCoreFiles(): + with support.SuppressCoreFiles(): p = subprocess.Popen([sys.executable, "-c", 'import os; os.abort()']) p.wait() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -306,6 +306,7 @@ # args_from_interpreter_flags # can_symlink # skip_unless_symlink + # SuppressCoreFiles def test_main(): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -202,6 +202,9 @@ Tests ----- +- Issue #18623: Factor out the _SuppressCoreFiles context manager into + test.support. Patch by Valerie Lambert. + - Issue #12037: Fix test_email for desktop Windows. - Issue #15507: test_subprocess's test_send_signal could fail if the test -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 21:17:30 2013 From: python-checkins at python.org (charles-francois.natali) Date: Fri, 6 Sep 2013 21:17:30 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318934=3A_Relax_te?= =?utf-8?q?st=5Fmultiprocessing=2Etest=5Finvalid=5Fhandles_a_bit=3A_we_jus?= =?utf-8?q?t?= Message-ID: <3cWpSB5FZ0z7Lnq@mail.python.org> http://hg.python.org/cpython/rev/14972c999e80 changeset: 85577:14972c999e80 user: Charles-Fran?ois Natali date: Fri Sep 06 21:12:22 2013 +0200 summary: Issue #18934: Relax test_multiprocessing.test_invalid_handles a bit: we just want to check that Connection.poll() doesn't crash. files: Lib/test/_test_multiprocessing.py | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2958,8 +2958,11 @@ @unittest.skipIf(WIN32, "skipped on Windows") def test_invalid_handles(self): conn = multiprocessing.connection.Connection(44977608) + # check that poll() doesn't crash try: - self.assertRaises((ValueError, OSError), conn.poll) + conn.poll() + except (ValueError, OSError): + pass finally: # Hack private attribute _handle to avoid printing an error # in conn.__del__ -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 21:19:22 2013 From: python-checkins at python.org (antoine.pitrou) Date: Fri, 6 Sep 2013 21:19:22 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Remove_old-school_inherita?= =?utf-8?q?nce?= Message-ID: <3cWpVL3hwVz7Lks@mail.python.org> http://hg.python.org/cpython/rev/cefb76baa331 changeset: 85578:cefb76baa331 user: Antoine Pitrou date: Fri Sep 06 21:18:25 2013 +0200 summary: Remove old-school inheritance files: Lib/test/support/__init__.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2063,7 +2063,7 @@ setattr(object_to_patch, attr_name, new_value) -class SuppressCoreFiles(object): +class SuppressCoreFiles: """Try to prevent core files from being created.""" old_limit = None -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 22:43:27 2013 From: python-checkins at python.org (tim.peters) Date: Fri, 6 Sep 2013 22:43:27 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgMTg5NDQ6?= =?utf-8?q?__fix_a_1-character_typo_in_test=5Fset=2Epy=2E?= Message-ID: <3cWrMM30z0z7LnY@mail.python.org> http://hg.python.org/cpython/rev/b6059bac8a9c changeset: 85579:b6059bac8a9c branch: 3.3 parent: 85569:7611e7244bdd user: Tim Peters date: Fri Sep 06 15:41:30 2013 -0500 summary: Issue 18944: fix a 1-character typo in test_set.py. The error caused test_inline_methods() to test much less than intended. Caught (& fixed) by Armin Rigo. files: Lib/test/test_set.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -1674,7 +1674,7 @@ for meth in (s.union, s.intersection, s.difference, s.symmetric_difference, s.isdisjoint): for g in (G, I, Ig, L, R): expected = meth(data) - actual = meth(G(data)) + actual = meth(g(data)) if isinstance(expected, bool): self.assertEqual(actual, expected) else: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 6 22:43:28 2013 From: python-checkins at python.org (tim.peters) Date: Fri, 6 Sep 2013 22:43:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Merge_3=2E3_into_default=2E?= Message-ID: <3cWrMN4x3zz7Ljw@mail.python.org> http://hg.python.org/cpython/rev/4b64166d5abb changeset: 85580:4b64166d5abb parent: 85578:cefb76baa331 parent: 85579:b6059bac8a9c user: Tim Peters date: Fri Sep 06 15:42:47 2013 -0500 summary: Merge 3.3 into default. Issue 18944: fix a 1-character typo in test_set.py. The error caused test_inline_methods() to test much less than intended. Caught (& fixed) by Armin Rigo. files: Lib/test/test_set.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -1672,7 +1672,7 @@ for meth in (s.union, s.intersection, s.difference, s.symmetric_difference, s.isdisjoint): for g in (G, I, Ig, L, R): expected = meth(data) - actual = meth(G(data)) + actual = meth(g(data)) if isinstance(expected, bool): self.assertEqual(actual, expected) else: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 00:17:07 2013 From: python-checkins at python.org (ned.deily) Date: Sat, 7 Sep 2013 00:17:07 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318458=3A_Prevent_?= =?utf-8?q?crashes_with_newer_versions_of_libedit=2E__Its_readline?= Message-ID: <3cWtRR1Sd5z7LjM@mail.python.org> http://hg.python.org/cpython/rev/b6b8a2171aa3 changeset: 85581:b6b8a2171aa3 user: Ned Deily date: Fri Sep 06 15:16:19 2013 -0700 summary: Issue #18458: Prevent crashes with newer versions of libedit. Its readline emulation has changed from 0-based indexing to 1-based like gnu readline. Original patch by Ronald Oussoren. files: Misc/NEWS | 3 +++ Modules/readline.c | 34 ++++++++++++++++++++++------------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -56,6 +56,9 @@ Library ------- +- Issue #18458: Prevent crashes with newer versions of libedit. Its readline + emulation has changed from 0-based indexing to 1-based like gnu readline. + - Issue #18852: Handle case of ``readline.__doc__`` being ``None`` in the new readline activation code in ``site.py``. diff --git a/Modules/readline.c b/Modules/readline.c --- a/Modules/readline.c +++ b/Modules/readline.c @@ -63,6 +63,8 @@ */ static int using_libedit_emulation = 0; static const char libedit_version_tag[] = "EditLine wrapper"; + +static int libedit_history_start = 0; #endif /* __APPLE__ */ #ifdef HAVE_RL_COMPLETION_DISPLAY_MATCHES_HOOK @@ -627,21 +629,21 @@ return NULL; #ifdef __APPLE__ if (using_libedit_emulation) { - /* Libedit emulation uses 0-based indexes, - * the real one uses 1-based indexes, - * adjust the index to ensure that Python - * code doesn't have to worry about the - * difference. + /* Older versions of libedit's readline emulation + * use 0-based indexes, while readline and newer + * versions of libedit use 1-based indexes. */ int length = _py_get_history_length(); - idx --; + + idx = idx - 1 + libedit_history_start; /* * Apple's readline emulation crashes when * the index is out of range, therefore * test for that and fail gracefully. */ - if (idx < 0 || idx >= length) { + if (idx < (0 + libedit_history_start) + || idx >= (length + libedit_history_start)) { Py_RETURN_NONE; } } @@ -974,6 +976,17 @@ */ if (using_libedit_emulation) rl_initialize(); + + /* Detect if libedit's readline emulation uses 0-based + * indexing or 1-based indexing. + */ + add_history("1"); + if (history_get(1) == NULL) { + libedit_history_start = 0; + } else { + libedit_history_start = 1; + } + clear_history(); #endif /* __APPLE__ */ using_history(); @@ -1178,11 +1191,8 @@ if (length > 0) #ifdef __APPLE__ if (using_libedit_emulation) { - /* - * Libedit's emulation uses 0-based indexes, - * the real readline uses 1-based indexes. - */ - line = (const char *)history_get(length - 1)->line; + /* handle older 0-based or newer 1-based indexing */ + line = (const char *)history_get(length + libedit_history_start - 1)->line; } else #endif /* __APPLE__ */ line = (const char *)history_get(length)->line; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 04:53:42 2013 From: python-checkins at python.org (ethan.furman) Date: Sat, 7 Sep 2013 04:53:42 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Close_=2318908=3A_Keep_Enu?= =?utf-8?q?m_docs_in_their_own_section=2E__Patch_by_Elazar_Gershuni=2E?= Message-ID: <3cX0ZZ64jnz7Ljb@mail.python.org> http://hg.python.org/cpython/rev/a55cbaf9a581 changeset: 85582:a55cbaf9a581 user: Ethan Furman date: Fri Sep 06 19:53:30 2013 -0700 summary: Close #18908: Keep Enum docs in their own section. Patch by Elazar Gershuni. files: Doc/library/enum.rst | 125 ++++++++++++++++-------------- 1 files changed, 67 insertions(+), 58 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -36,11 +36,15 @@ ... red = 1 ... green = 2 ... blue = 3 + ... -**A note on nomenclature**: we call :class:`Color` an *enumeration* (or *enum*) -and :attr:`Color.red`, :attr:`Color.green` are *enumeration members* (or -*enum members*). Enumeration members also have *values* (the value of -:attr:`Color.red` is ``1``, etc.) +..note: Nomenclature + - The class :class:`Color` is an *enumeration* (or *enum*) + - The attributes :attr:`Color.red`, :attr:`Color.green`, etc., are + *enumeration members* (or *enum members*). + - The enum members have *names* and *values* (the name of + :attr:`Color.red` is ``red``, the value of :attr:`Color.blue` is + ``3``, etc.) Enumeration members have human readable string representations:: @@ -68,13 +72,13 @@ Enumerations support iteration, in definition order:: >>> class Shake(Enum): - ... vanilla = 7 - ... chocolate = 4 - ... cookies = 9 - ... mint = 3 + ... vanilla = 7 + ... chocolate = 4 + ... cookies = 9 + ... mint = 3 ... >>> for shake in Shake: - ... print(shake) + ... print(shake) ... Shake.vanilla Shake.chocolate @@ -124,8 +128,8 @@ Having two enum members with the same name is invalid:: >>> class Shape(Enum): - ... square = 2 - ... square = 3 + ... square = 2 + ... square = 3 ... Traceback (most recent call last): ... @@ -137,10 +141,10 @@ return A:: >>> class Shape(Enum): - ... square = 2 - ... diamond = 1 - ... circle = 3 - ... alias_for_square = 2 + ... square = 2 + ... diamond = 1 + ... circle = 3 + ... alias_for_square = 2 ... >>> Shape.square @@ -151,7 +155,7 @@ Ensuring unique enumeration values -================================== +---------------------------------- By default, enumerations allow multiple names as aliases for the same value. When this behavior isn't desired, the following decorator can be used to @@ -166,17 +170,18 @@ >>> from enum import Enum, unique >>> @unique ... class Mistake(Enum): - ... one = 1 - ... two = 2 - ... three = 3 - ... four = 3 + ... one = 1 + ... two = 2 + ... three = 3 + ... four = 3 + ... Traceback (most recent call last): ... ValueError: duplicate values found in : four -> three Iteration -========= +--------- Iterating over the members of an enum does not provide the aliases:: @@ -188,7 +193,7 @@ aliases:: >>> for name, member in Shape.__members__.items(): - ... name, member + ... name, member ... ('square', ) ('diamond', ) @@ -252,20 +257,21 @@ usual. If we have this enumeration:: >>> class Mood(Enum): - ... funky = 1 - ... happy = 3 + ... funky = 1 + ... happy = 3 ... - ... def describe(self): - ... # self is the member here - ... return self.name, self.value + ... def describe(self): + ... # self is the member here + ... return self.name, self.value ... - ... def __str__(self): - ... return 'my custom str! {0}'.format(self.value) + ... def __str__(self): + ... return 'my custom str! {0}'.format(self.value) ... - ... @classmethod - ... def favorite_mood(cls): - ... # cls here is the enumeration - ... return cls.happy + ... @classmethod + ... def favorite_mood(cls): + ... # cls here is the enumeration + ... return cls.happy + ... Then:: @@ -294,7 +300,8 @@ any members. So this is forbidden:: >>> class MoreColor(Color): - ... pink = 17 + ... pink = 17 + ... Traceback (most recent call last): ... TypeError: Cannot extend enumerations @@ -302,12 +309,12 @@ But this is allowed:: >>> class Foo(Enum): - ... def some_behavior(self): - ... pass + ... def some_behavior(self): + ... pass ... >>> class Bar(Foo): - ... happy = 1 - ... sad = 2 + ... happy = 1 + ... sad = 2 ... Allowing subclassing of enums that define members would lead to a violation of @@ -363,10 +370,11 @@ assignment to :class:`Animal` is equivalent to:: >>> class Animals(Enum): - ... ant = 1 - ... bee = 2 - ... cat = 3 - ... dog = 4 + ... ant = 1 + ... bee = 2 + ... cat = 3 + ... dog = 4 + ... The reason for defaulting to ``1`` as the starting number and not ``0`` is that ``0`` is ``False`` in a boolean sense, but enum members all evaluate @@ -381,10 +389,10 @@ >>> Animals = Enum('Animals', 'ant bee cat dog', module=__name__) Derived Enumerations -==================== +-------------------- IntEnum -------- +^^^^^^^ A variation of :class:`Enum` is provided which is also a subclass of :class:`int`. Members of an :class:`IntEnum` can be compared to integers; @@ -393,12 +401,12 @@ >>> from enum import IntEnum >>> class Shape(IntEnum): - ... circle = 1 - ... square = 2 + ... circle = 1 + ... square = 2 ... >>> class Request(IntEnum): - ... post = 1 - ... get = 2 + ... post = 1 + ... get = 2 ... >>> Shape == 1 False @@ -410,12 +418,12 @@ However, they still can't be compared to standard :class:`Enum` enumerations:: >>> class Shape(IntEnum): - ... circle = 1 - ... square = 2 + ... circle = 1 + ... square = 2 ... >>> class Color(Enum): - ... red = 1 - ... green = 2 + ... red = 1 + ... green = 2 ... >>> Shape.circle == Color.red False @@ -439,7 +447,7 @@ Others ------- +^^^^^^ While :class:`IntEnum` is part of the :mod:`enum` module, it would be very simple to implement independently:: @@ -472,7 +480,7 @@ Interesting examples -==================== +-------------------- While :class:`Enum` and :class:`IntEnum` are expected to cover the majority of use-cases, they cannot cover them all. Here are recipes for some different @@ -481,7 +489,7 @@ AutoNumber ----------- +^^^^^^^^^^ Avoids having to specify the value for each enumeration member:: @@ -502,7 +510,7 @@ OrderedEnum ------------ +^^^^^^^^^^^ An ordered enumeration that is not based on :class:`IntEnum` and so maintains the normal :class:`Enum` invariants (such as not being comparable to other @@ -538,7 +546,7 @@ DuplicateFreeEnum ------------------ +^^^^^^^^^^^^^^^^^ Raises an error if a duplicate member name is found instead of creating an alias:: @@ -558,6 +566,7 @@ ... green = 2 ... blue = 3 ... grene = 2 + ... Traceback (most recent call last): ... ValueError: aliases not allowed in DuplicateFreeEnum: 'grene' --> 'green' @@ -570,7 +579,7 @@ Planet ------- +^^^^^^ If :meth:`__new__` or :meth:`__init__` is defined the value of the enum member will be passed to those methods:: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 04:58:27 2013 From: python-checkins at python.org (ethan.furman) Date: Sat, 7 Sep 2013 04:58:27 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Added_Elazar_to_Misc/ACKS?= =?utf-8?q?=2E?= Message-ID: <3cX0h30LH4z7LjM@mail.python.org> http://hg.python.org/cpython/rev/4e2a1b656c1f changeset: 85583:4e2a1b656c1f user: Ethan Furman date: Fri Sep 06 19:58:01 2013 -0700 summary: Added Elazar to Misc/ACKS. files: Misc/ACKS | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -435,6 +435,7 @@ Thomas Gellekum Gabriel Genellina Christos Georgiou +Elazar Gershuni Ben Gertzfield Nadim Ghaznavi Dinu Gherman -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Sat Sep 7 06:19:38 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sat, 07 Sep 2013 06:19:38 +0200 Subject: [Python-checkins] Daily reference leaks (b6b8a2171aa3): sum=0 Message-ID: results for b6b8a2171aa3 on branch "default" -------------------------------------------- Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogDFtFIK', '-x'] From python-checkins at python.org Sat Sep 7 10:37:57 2013 From: python-checkins at python.org (victor.stinner) Date: Sat, 7 Sep 2013 10:37:57 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Close_=2318954=3A_Fix_some?= =?utf-8?q?_typo_in_fileutils=2Ec_comments?= Message-ID: <3cX8Cn0qmYzQph@mail.python.org> http://hg.python.org/cpython/rev/98f82b124c7d changeset: 85584:98f82b124c7d user: Victor Stinner date: Sat Sep 07 10:36:04 2013 +0200 summary: Close #18954: Fix some typo in fileutils.c comments Patch written by Vajrasky Kok. files: Python/fileutils.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/fileutils.c b/Python/fileutils.c --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -22,7 +22,7 @@ #endif #ifdef O_CLOEXEC -/* Does open() supports the O_CLOEXEC flag? Possible values: +/* Does open() support the O_CLOEXEC flag? Possible values: -1: unknown 0: open() ignores O_CLOEXEC flag, ex: Linux kernel older than 2.6.23 @@ -607,7 +607,7 @@ } /* Get the inheritable flag of the specified file descriptor. - Return 1 if it the file descriptor can be inherited, 0 if it cannot, + Return 1 if the file descriptor can be inherited, 0 if it cannot, raise an exception and return -1 on error. */ int _Py_get_inheritable(int fd) @@ -706,7 +706,7 @@ } /* Make the file descriptor non-inheritable. - Return 0 success, set errno and return -1 on error. */ + Return 0 on success, set errno and return -1 on error. */ static int make_non_inheritable(int fd) { -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 13:33:36 2013 From: python-checkins at python.org (larry.hastings) Date: Sat, 7 Sep 2013 13:33:36 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Twice_now_I=27ve_assumed_that?= =?utf-8?q?_release=2Epy_--bump_would_check_in_its?= Message-ID: <3cXD6S2GvGz7LjN@mail.python.org> http://hg.python.org/peps/rev/8880eba377c2 changeset: 5101:8880eba377c2 user: Larry Hastings date: Sat Sep 07 23:33:30 2013 +1200 summary: Twice now I've assumed that release.py --bump would check in its changes, and scotched the release. Changing PEP 101 to hopefully prevent this for 3.4.0a3. files: pep-0101.txt | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/pep-0101.txt b/pep-0101.txt --- a/pep-0101.txt +++ b/pep-0101.txt @@ -192,7 +192,8 @@ content of this file. You should only need to review the text for sanity, and update the release date with today's date. - Commit the changes once you have reviewed them. + ___ Make sure all changes have been committed. (``release.py --bump`` + doesn't check in its changes for you.) ___ Check the years on the copyright notice. If the last release was some time last year, add the current year to the copyright -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sat Sep 7 14:20:30 2013 From: python-checkins at python.org (ezio.melotti) Date: Sat, 7 Sep 2013 14:20:30 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogIzE4ODk0OiByZW1v?= =?utf-8?q?ve_mention_of_deprecated_fail*_methods=2E?= Message-ID: <3cXF8Z5hXjz7LkT@mail.python.org> http://hg.python.org/cpython/rev/c391c30fee99 changeset: 85585:c391c30fee99 branch: 2.7 parent: 85571:e0037f266d45 user: Ezio Melotti date: Sat Sep 07 15:19:30 2013 +0300 summary: #18894: remove mention of deprecated fail* methods. files: Doc/library/unittest.rst | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1603,8 +1603,7 @@ A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.fail\*` or - :meth:`TestCase.assert\*` methods. + was explicitly signalled using the :meth:`TestCase.assert\*` methods. .. versionchanged:: 2.2 Contains formatted tracebacks instead of :func:`sys.exc_info` results. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 14:20:32 2013 From: python-checkins at python.org (ezio.melotti) Date: Sat, 7 Sep 2013 14:20:32 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE4ODk0OiByZW1v?= =?utf-8?q?ve_mention_of_deprecated_fail*_methods=2E?= Message-ID: <3cXF8c0Jlqz7Lkx@mail.python.org> http://hg.python.org/cpython/rev/c3a2c0504960 changeset: 85586:c3a2c0504960 branch: 3.3 parent: 85579:b6059bac8a9c user: Ezio Melotti date: Sat Sep 07 15:19:30 2013 +0300 summary: #18894: remove mention of deprecated fail* methods. files: Doc/library/unittest.rst | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1577,8 +1577,7 @@ A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.fail\*` or - :meth:`TestCase.assert\*` methods. + was explicitly signalled using the :meth:`TestCase.assert\*` methods. .. attribute:: skipped -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 14:20:33 2013 From: python-checkins at python.org (ezio.melotti) Date: Sat, 7 Sep 2013 14:20:33 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?b?KTogIzE4ODk0OiBtZXJnZSB3aXRoIDMuMy4=?= Message-ID: <3cXF8d27p7z7LmJ@mail.python.org> http://hg.python.org/cpython/rev/5fbb01033a08 changeset: 85587:5fbb01033a08 parent: 85584:98f82b124c7d parent: 85586:c3a2c0504960 user: Ezio Melotti date: Sat Sep 07 15:20:03 2013 +0300 summary: #18894: merge with 3.3. files: Doc/library/unittest.rst | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1674,8 +1674,7 @@ A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.fail\*` or - :meth:`TestCase.assert\*` methods. + was explicitly signalled using the :meth:`TestCase.assert\*` methods. .. attribute:: skipped -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 14:24:28 2013 From: python-checkins at python.org (ezio.melotti) Date: Sat, 7 Sep 2013 14:24:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogIzE4ODk1OiBzcGxp?= =?utf-8?q?t_a_sentence_in_unittest_docs=2E?= Message-ID: <3cXFF82Yz4z7LkG@mail.python.org> http://hg.python.org/cpython/rev/523cfc78847c changeset: 85588:523cfc78847c branch: 2.7 parent: 85585:c391c30fee99 user: Ezio Melotti date: Sat Sep 07 15:23:36 2013 +0300 summary: #18895: split a sentence in unittest docs. files: Doc/library/unittest.rst | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1703,7 +1703,7 @@ .. method:: addError(test, err) - Called when the test case *test* raises an unexpected exception *err* is a + Called when the test case *test* raises an unexpected exception. *err* is a tuple of the form returned by :func:`sys.exc_info`: ``(type, value, traceback)``. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 14:24:29 2013 From: python-checkins at python.org (ezio.melotti) Date: Sat, 7 Sep 2013 14:24:29 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE4ODk1OiBzcGxp?= =?utf-8?q?t_a_sentence_in_unittest_docs=2E?= Message-ID: <3cXFF94Ggqz7Lmk@mail.python.org> http://hg.python.org/cpython/rev/dd669daad7de changeset: 85589:dd669daad7de branch: 3.3 parent: 85586:c3a2c0504960 user: Ezio Melotti date: Sat Sep 07 15:23:36 2013 +0300 summary: #18895: split a sentence in unittest docs. files: Doc/library/unittest.rst | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1674,7 +1674,7 @@ .. method:: addError(test, err) - Called when the test case *test* raises an unexpected exception *err* is a + Called when the test case *test* raises an unexpected exception. *err* is a tuple of the form returned by :func:`sys.exc_info`: ``(type, value, traceback)``. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 14:24:30 2013 From: python-checkins at python.org (ezio.melotti) Date: Sat, 7 Sep 2013 14:24:30 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?b?KTogIzE4ODk1OiBtZXJnZSB3aXRoIDMuMy4=?= Message-ID: <3cXFFB6BSXz7LnG@mail.python.org> http://hg.python.org/cpython/rev/1043cc2cb0ff changeset: 85590:1043cc2cb0ff parent: 85587:5fbb01033a08 parent: 85589:dd669daad7de user: Ezio Melotti date: Sat Sep 07 15:24:01 2013 +0300 summary: #18895: merge with 3.3. files: Doc/library/unittest.rst | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1771,7 +1771,7 @@ .. method:: addError(test, err) - Called when the test case *test* raises an unexpected exception *err* is a + Called when the test case *test* raises an unexpected exception. *err* is a tuple of the form returned by :func:`sys.exc_info`: ``(type, value, traceback)``. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 18:42:43 2013 From: python-checkins at python.org (richard.oudkerk) Date: Sat, 7 Sep 2013 18:42:43 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Fix_conversion_from_Py=5Fs?= =?utf-8?q?size=5Ft_to_int=2E?= Message-ID: <3cXLz71K19z7Lm4@mail.python.org> http://hg.python.org/cpython/rev/c116b658aede changeset: 85591:c116b658aede user: Richard Oudkerk date: Sat Sep 07 17:40:45 2013 +0100 summary: Fix conversion from Py_ssize_t to int. files: Modules/_multiprocessing/multiprocessing.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -99,13 +99,15 @@ { HANDLE handle; Py_buffer buf; - int ret; + int ret, length; if (!PyArg_ParseTuple(args, F_HANDLE "y*:send" , &handle, &buf)) return NULL; + length = (int)Py_MIN(buf.len, INT_MAX); + Py_BEGIN_ALLOW_THREADS - ret = send((SOCKET) handle, buf.buf, buf.len, 0); + ret = send((SOCKET) handle, buf.buf, length, 0); Py_END_ALLOW_THREADS PyBuffer_Release(&buf); -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 20:30:15 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sat, 7 Sep 2013 20:30:15 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogb3MuaXNhdHR5IGlz?= =?utf-8?q?_not_Unix_only=2E_Correct_the_wrong_documentation=2E?= Message-ID: <3cXPMC69fvz7LjX@mail.python.org> http://hg.python.org/cpython/rev/d5c5ac33b9a1 changeset: 85592:d5c5ac33b9a1 branch: 2.7 parent: 85588:523cfc78847c user: Senthil Kumaran date: Sat Sep 07 11:27:45 2013 -0700 summary: os.isatty is not Unix only. Correct the wrong documentation. Addresses issue #18553 files: Doc/library/os.rst | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -737,8 +737,6 @@ Return ``True`` if the file descriptor *fd* is open and connected to a tty(-like) device, else ``False``. - Availability: Unix. - .. function:: lseek(fd, pos, how) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 20:30:17 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sat, 7 Sep 2013 20:30:17 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Removing_the_m?= =?utf-8?q?ention_of_os=2Eisatty_mention_as_Unix_only?= Message-ID: <3cXPMF0yWXz7Ljm@mail.python.org> http://hg.python.org/cpython/rev/14ba90816930 changeset: 85593:14ba90816930 branch: 3.3 parent: 85589:dd669daad7de user: Senthil Kumaran date: Sat Sep 07 11:28:58 2013 -0700 summary: Removing the mention of os.isatty mention as Unix only Correct the wrong documentation. files: Doc/library/os.rst | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -744,8 +744,6 @@ As of Python 3.3, this is equivalent to ``os.pathconf(fd, name)``. - Availability: Unix. - .. function:: fstat(fd) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 20:30:18 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sat, 7 Sep 2013 20:30:18 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_merge_from_3=2E3?= Message-ID: <3cXPMG2qGpz7LkS@mail.python.org> http://hg.python.org/cpython/rev/678e3c0d2d99 changeset: 85594:678e3c0d2d99 parent: 85591:c116b658aede parent: 85593:14ba90816930 user: Senthil Kumaran date: Sat Sep 07 11:30:04 2013 -0700 summary: merge from 3.3 Removing the mention of os.isatty mention as Unix only Correct the wrong documentation. files: Doc/library/os.rst | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -757,8 +757,6 @@ As of Python 3.3, this is equivalent to ``os.pathconf(fd, name)``. - Availability: Unix. - .. function:: fstat(fd) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 23:13:21 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sat, 7 Sep 2013 23:13:21 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Fix_License_UR?= =?utf-8?q?L_display_and_add_test_to_check_for_license_url_presence=2E?= Message-ID: <3cXSzP0RnVz7LjW@mail.python.org> http://hg.python.org/cpython/rev/a723adeb1f47 changeset: 85595:a723adeb1f47 branch: 3.3 parent: 85593:14ba90816930 user: Senthil Kumaran date: Sat Sep 07 13:59:17 2013 -0700 summary: Fix License URL display and add test to check for license url presence. Fixes issue #18206 Patch contributed by Berker Peksag and py.user files: Lib/site.py | 3 ++- Lib/test/test_site.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/Lib/site.py b/Lib/site.py --- a/Lib/site.py +++ b/Lib/site.py @@ -451,7 +451,8 @@ for supporting Python development. See www.python.org for more information.""") here = os.path.dirname(os.__file__) builtins.license = _Printer( - "license", "See http://www.python.org/%.3s/license.html" % sys.version, + "license", + "See http://www.python.org/download/releases/%.5s/license/" % sys.version, ["LICENSE.txt", "LICENSE"], [os.path.join(here, os.pardir), here, os.curdir]) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -5,6 +5,7 @@ """ import unittest +import test.support from test.support import run_unittest, TESTFN, EnvironmentVarGuard from test.support import captured_stderr import builtins @@ -377,9 +378,10 @@ self.assertTrue(hasattr(builtins, "exit")) def test_setting_copyright(self): - # 'copyright' and 'credits' should be in builtins + # 'copyright', 'credits', and 'license' should be in builtins self.assertTrue(hasattr(builtins, "copyright")) self.assertTrue(hasattr(builtins, "credits")) + self.assertTrue(hasattr(builtins, "license")) def test_setting_help(self): # 'help' should be set in builtins @@ -405,8 +407,30 @@ else: self.fail("sitecustomize not imported automatically") + +class LicenseURL(unittest.TestCase): + """Test accessibility of the license.""" + + @unittest.skipUnless(str(license).startswith('See http://'), + 'license is available as a file') + def test_license_page(self): + """urlopen should return the license page""" + pat = r'^See (http://www\.python\.org/download/releases/[^/]+/license/)$' + mo = re.search(pat, str(license)) + self.assertIsNotNone(mo, msg='can\'t find appropriate url in license') + if mo is not None: + url = mo.group(1) + with test.support.transient_internet(url): + import urllib.request, urllib.error + try: + with urllib.request.urlopen(url) as data: + code = data.getcode() + except urllib.error.HTTPError as e: + code = e.code + self.assertEqual(code, 200, msg=url) + def test_main(): - run_unittest(HelperFunctionsTests, ImportSideEffectTests) + run_unittest(HelperFunctionsTests, ImportSideEffectTests, LicenseURL) if __name__ == "__main__": test_main() -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 23:13:22 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sat, 7 Sep 2013 23:13:22 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_merge_from_3=2E3?= Message-ID: <3cXSzQ26Skz7Ll2@mail.python.org> http://hg.python.org/cpython/rev/12aaf64feca2 changeset: 85596:12aaf64feca2 parent: 85594:678e3c0d2d99 parent: 85595:a723adeb1f47 user: Senthil Kumaran date: Sat Sep 07 14:09:48 2013 -0700 summary: merge from 3.3 Fix License URL display and add test to check for license url presence. Fixes issue #18206 Patch contributed by Berker Peksag and py.user files: Lib/site.py | 6 ++++++ Lib/test/test_site.py | 26 +++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletions(-) diff --git a/Lib/site.py b/Lib/site.py --- a/Lib/site.py +++ b/Lib/site.py @@ -361,8 +361,14 @@ Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands for supporting Python development. See www.python.org for more information.""") here = os.path.dirname(os.__file__) +<<<<<<< local builtins.license = _sitebuiltins._Printer( "license", "See http://www.python.org/%.3s/license.html" % sys.version, +======= + builtins.license = _Printer( + "license", + "See http://www.python.org/download/releases/%.5s/license/" % sys.version, +>>>>>>> other ["LICENSE.txt", "LICENSE"], [os.path.join(here, os.pardir), here, os.curdir]) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -5,6 +5,7 @@ """ import unittest +import test.support from test.support import run_unittest, TESTFN, EnvironmentVarGuard from test.support import captured_stderr import builtins @@ -373,9 +374,10 @@ self.assertTrue(hasattr(builtins, "exit")) def test_setting_copyright(self): - # 'copyright' and 'credits' should be in builtins + # 'copyright', 'credits', and 'license' should be in builtins self.assertTrue(hasattr(builtins, "copyright")) self.assertTrue(hasattr(builtins, "credits")) + self.assertTrue(hasattr(builtins, "license")) def test_setting_help(self): # 'help' should be set in builtins @@ -402,5 +404,27 @@ self.fail("sitecustomize not imported automatically") +class LicenseURL(unittest.TestCase): + """Test accessibility of the license.""" + + @unittest.skipUnless(str(license).startswith('See http://'), + 'license is available as a file') + def test_license_page(self): + """urlopen should return the license page""" + pat = r'^See (http://www\.python\.org/download/releases/[^/]+/license/)$' + mo = re.search(pat, str(license)) + self.assertIsNotNone(mo, msg='can\'t find appropriate url in license') + if mo is not None: + url = mo.group(1) + with test.support.transient_internet(url): + import urllib.request, urllib.error + try: + with urllib.request.urlopen(url) as data: + code = data.getcode() + except urllib.error.HTTPError as e: + code = e.code + self.assertEqual(code, 200, msg=url) + + if __name__ == "__main__": unittest.main() -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 23:13:23 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sat, 7 Sep 2013 23:13:23 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Fix_the_merge_conflict?= Message-ID: <3cXSzR3mLbz7Lk2@mail.python.org> http://hg.python.org/cpython/rev/6ecdf2b3192b changeset: 85597:6ecdf2b3192b user: Senthil Kumaran date: Sat Sep 07 14:12:55 2013 -0700 summary: Fix the merge conflict files: Lib/site.py | 6 ------ 1 files changed, 0 insertions(+), 6 deletions(-) diff --git a/Lib/site.py b/Lib/site.py --- a/Lib/site.py +++ b/Lib/site.py @@ -361,14 +361,8 @@ Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands for supporting Python development. See www.python.org for more information.""") here = os.path.dirname(os.__file__) -<<<<<<< local builtins.license = _sitebuiltins._Printer( "license", "See http://www.python.org/%.3s/license.html" % sys.version, -======= - builtins.license = _Printer( - "license", - "See http://www.python.org/download/releases/%.5s/license/" % sys.version, ->>>>>>> other ["LICENSE.txt", "LICENSE"], [os.path.join(here, os.pardir), here, os.curdir]) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 7 23:39:17 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sat, 7 Sep 2013 23:39:17 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318808=3A_Thread?= =?utf-8?q?=2Ejoin=28=29_now_waits_for_the_underlying_thread_state_to_be?= Message-ID: <3cXTYK5jbBz7Ljl@mail.python.org> http://hg.python.org/cpython/rev/d52b68edbca6 changeset: 85598:d52b68edbca6 user: Antoine Pitrou date: Sat Sep 07 23:38:37 2013 +0200 summary: Issue #18808: Thread.join() now waits for the underlying thread state to be destroyed before returning. This prevents unpredictable aborts in Py_EndInterpreter() when some non-daemon threads are still running. files: Include/pystate.h | 26 +++++++ Lib/_dummy_thread.py | 4 + Lib/test/test_threading.py | 70 ++++++++++++++++++++- Lib/threading.py | 85 ++++++++++++++++--------- Misc/NEWS | 4 + Modules/_threadmodule.c | 62 ++++++++++++++++++ Python/pystate.c | 5 + 7 files changed, 223 insertions(+), 33 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h --- a/Include/pystate.h +++ b/Include/pystate.h @@ -118,6 +118,32 @@ int trash_delete_nesting; PyObject *trash_delete_later; + /* Called when a thread state is deleted normally, but not when it + * is destroyed after fork(). + * Pain: to prevent rare but fatal shutdown errors (issue 18808), + * Thread.join() must wait for the join'ed thread's tstate to be unlinked + * from the tstate chain. That happens at the end of a thread's life, + * in pystate.c. + * The obvious way doesn't quite work: create a lock which the tstate + * unlinking code releases, and have Thread.join() wait to acquire that + * lock. The problem is that we _are_ at the end of the thread's life: + * if the thread holds the last reference to the lock, decref'ing the + * lock will delete the lock, and that may trigger arbitrary Python code + * if there's a weakref, with a callback, to the lock. But by this time + * _PyThreadState_Current is already NULL, so only the simplest of C code + * can be allowed to run (in particular it must not be possible to + * release the GIL). + * So instead of holding the lock directly, the tstate holds a weakref to + * the lock: that's the value of on_delete_data below. Decref'ing a + * weakref is harmless. + * on_delete points to _threadmodule.c's static release_sentinel() function. + * After the tstate is unlinked, release_sentinel is called with the + * weakref-to-lock (on_delete_data) argument, and release_sentinel releases + * the indirectly held lock. + */ + void (*on_delete)(void *); + void *on_delete_data; + /* XXX signal handlers should also be here */ } PyThreadState; diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py --- a/Lib/_dummy_thread.py +++ b/Lib/_dummy_thread.py @@ -81,6 +81,10 @@ raise error("setting thread stack size not supported") return 0 +def _set_sentinel(): + """Dummy implementation of _thread._set_sentinel().""" + return LockType() + class LockType(object): """Class implementing dummy implementation of _thread.LockType. diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -539,6 +539,40 @@ self.assertEqual(err, b"") self.assertEqual(data, "Thread-1\nTrue\nTrue\n") + def test_tstate_lock(self): + # Test an implementation detail of Thread objects. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + time.sleep(0.01) + # The tstate lock is None until the thread is started + t = threading.Thread(target=f) + self.assertIs(t._tstate_lock, None) + t.start() + started.acquire() + self.assertTrue(t.is_alive()) + # The tstate lock can't be acquired when the thread is running + # (or suspended). + tstate_lock = t._tstate_lock + self.assertFalse(tstate_lock.acquire(timeout=0), False) + finish.release() + # When the thread ends, the state_lock can be successfully + # acquired. + self.assertTrue(tstate_lock.acquire(timeout=5), False) + # But is_alive() is still True: we hold _tstate_lock now, which + # prevents is_alive() from knowing the thread's end-of-life C code + # is done. + self.assertTrue(t.is_alive()) + # Let is_alive() find out the C code is done. + tstate_lock.release() + self.assertFalse(t.is_alive()) + # And verify the thread disposed of _tstate_lock. + self.assertTrue(t._tstate_lock is None) + class ThreadJoinOnShutdown(BaseTestCase): @@ -669,7 +703,7 @@ # someone else tries to fix this test case by acquiring this lock # before forking instead of resetting it, the test case will # deadlock when it shouldn't. - condition = w._block + condition = w._stopped._cond orig_acquire = condition.acquire call_count_lock = threading.Lock() call_count = 0 @@ -733,7 +767,7 @@ # causes the worker to fork. At this point, the problematic waiter # lock has been acquired once by the waiter and has been put onto # the waiters list. - condition = w._block + condition = w._stopped._cond orig_release_save = condition._release_save def my_release_save(): global start_fork @@ -867,6 +901,38 @@ # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") + def test_threads_join_2(self): + # Same as above, but a delay gets introduced after the thread's + # Python code returned but before the thread state is deleted. + # To achieve this, we register a thread-local object which sleeps + # a bit when deallocated. + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + class Sleeper: + def __del__(self): + time.sleep(0.05) + + tls = threading.local() + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + tls.x = Sleeper() + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = _testcapi.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + def test_daemon_threads_fatal_error(self): subinterp_code = r"""if 1: import os diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -33,6 +33,7 @@ # Rename some stuff so "from threading import *" is safe _start_new_thread = _thread.start_new_thread _allocate_lock = _thread.allocate_lock +_set_sentinel = _thread._set_sentinel get_ident = _thread.get_ident ThreadError = _thread.error try: @@ -548,28 +549,33 @@ else: self._daemonic = current_thread().daemon self._ident = None + self._tstate_lock = None self._started = Event() - self._stopped = False - self._block = Condition(Lock()) + self._stopped = Event() self._initialized = True # sys.stderr is not stored in the class like # sys.exc_info since it can be changed between instances self._stderr = _sys.stderr _dangling.add(self) - def _reset_internal_locks(self): + def _reset_internal_locks(self, is_alive): # private! Called by _after_fork() to reset our internal locks as # they may be in an invalid state leading to a deadlock or crash. - if hasattr(self, '_block'): # DummyThread deletes _block - self._block.__init__() self._started._reset_internal_locks() + self._stopped._reset_internal_locks() + if is_alive: + self._set_tstate_lock() + else: + # The thread isn't alive after fork: it doesn't have a tstate + # anymore. + self._tstate_lock = None def __repr__(self): assert self._initialized, "Thread.__init__() was not called" status = "initial" if self._started.is_set(): status = "started" - if self._stopped: + if self._stopped.is_set(): status = "stopped" if self._daemonic: status += " daemon" @@ -625,9 +631,18 @@ def _set_ident(self): self._ident = get_ident() + def _set_tstate_lock(self): + """ + Set a lock object which will be released by the interpreter when + the underlying thread state (see pystate.h) gets deleted. + """ + self._tstate_lock = _set_sentinel() + self._tstate_lock.acquire() + def _bootstrap_inner(self): try: self._set_ident() + self._set_tstate_lock() self._started.set() with _active_limbo_lock: _active[self._ident] = self @@ -691,10 +706,7 @@ pass def _stop(self): - self._block.acquire() - self._stopped = True - self._block.notify_all() - self._block.release() + self._stopped.set() def _delete(self): "Remove current thread from the dict of currently running threads." @@ -738,21 +750,29 @@ raise RuntimeError("cannot join thread before it is started") if self is current_thread(): raise RuntimeError("cannot join current thread") + if not self.is_alive(): + return + self._stopped.wait(timeout) + if self._stopped.is_set(): + self._wait_for_tstate_lock(timeout is None) - self._block.acquire() - try: - if timeout is None: - while not self._stopped: - self._block.wait() - else: - deadline = _time() + timeout - while not self._stopped: - delay = deadline - _time() - if delay <= 0: - break - self._block.wait(delay) - finally: - self._block.release() + def _wait_for_tstate_lock(self, block): + # Issue #18808: wait for the thread state to be gone. + # When self._stopped is set, the Python part of the thread is done, + # but the thread's tstate has not yet been destroyed. The C code + # releases self._tstate_lock when the C part of the thread is done + # (the code at the end of the thread's life to remove all knowledge + # of the thread from the C data structures). + # This method waits to acquire _tstate_lock if `block` is True, or + # sees whether it can be acquired immediately if `block` is False. + # If it does acquire the lock, the C code is done, and _tstate_lock + # is set to None. + lock = self._tstate_lock + if lock is None: + return # already determined that the C code is done + if lock.acquire(block): + lock.release() + self._tstate_lock = None @property def name(self): @@ -771,7 +791,14 @@ def is_alive(self): assert self._initialized, "Thread.__init__() not called" - return self._started.is_set() and not self._stopped + if not self._started.is_set(): + return False + if not self._stopped.is_set(): + return True + # The Python part of the thread is done, but the C part may still be + # waiting to run. + self._wait_for_tstate_lock(False) + return self._tstate_lock is not None isAlive = is_alive @@ -854,11 +881,6 @@ def __init__(self): Thread.__init__(self, name=_newname("Dummy-%d"), daemon=True) - # Thread._block consumes an OS-level locking primitive, which - # can never be used by a _DummyThread. Since a _DummyThread - # instance is immortal, that's bad, so release this resource. - del self._block - self._started.set() self._set_ident() with _active_limbo_lock: @@ -952,15 +974,16 @@ for thread in _enumerate(): # Any lock/condition variable may be currently locked or in an # invalid state, so we reinitialize them. - thread._reset_internal_locks() if thread is current: # There is only one active thread. We reset the ident to # its new value since it can have changed. + thread._reset_internal_locks(True) ident = get_ident() thread._ident = ident new_active[ident] = thread else: # All the others are already stopped. + thread._reset_internal_locks(False) thread._stop() _limbo.clear() diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -56,6 +56,10 @@ Library ------- +- Issue #18808: Thread.join() now waits for the underlying thread state to + be destroyed before returning. This prevents unpredictable aborts in + Py_EndInterpreter() when some non-daemon threads are still running. + - Issue #18458: Prevent crashes with newer versions of libedit. Its readline emulation has changed from 0-based indexing to 1-based like gnu readline. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1172,6 +1172,66 @@ This function is meant for internal and specialized purposes only.\n\ In most applications `threading.enumerate()` should be used instead."); +static void +release_sentinel(void *wr) +{ + /* Tricky: this function is called when the current thread state + is being deleted. Therefore, only simple C code can safely + execute here. */ + PyObject *obj = PyWeakref_GET_OBJECT(wr); + lockobject *lock; + if (obj != Py_None) { + assert(Py_TYPE(obj) == &Locktype); + lock = (lockobject *) obj; + if (lock->locked) { + PyThread_release_lock(lock->lock_lock); + lock->locked = 0; + } + } + /* Deallocating a weakref with a NULL callback only calls + PyObject_GC_Del(), which can't call any Python code. */ + Py_DECREF(wr); +} + +static PyObject * +thread__set_sentinel(PyObject *self) +{ + PyObject *wr; + PyThreadState *tstate = PyThreadState_Get(); + lockobject *lock; + + if (tstate->on_delete_data != NULL) { + /* We must support the re-creation of the lock from a + fork()ed child. */ + assert(tstate->on_delete == &release_sentinel); + wr = (PyObject *) tstate->on_delete_data; + tstate->on_delete = NULL; + tstate->on_delete_data = NULL; + Py_DECREF(wr); + } + lock = newlockobject(); + if (lock == NULL) + return NULL; + /* The lock is owned by whoever called _set_sentinel(), but the weakref + hangs to the thread state. */ + wr = PyWeakref_NewRef((PyObject *) lock, NULL); + if (wr == NULL) { + Py_DECREF(lock); + return NULL; + } + tstate->on_delete_data = (void *) wr; + tstate->on_delete = &release_sentinel; + return (PyObject *) lock; +} + +PyDoc_STRVAR(_set_sentinel_doc, +"_set_sentinel() -> lock\n\ +\n\ +Set a sentinel lock that will be released when the current thread\n\ +state is finalized (after it is untied from the interpreter).\n\ +\n\ +This is a private API for the threading module."); + static PyObject * thread_stack_size(PyObject *self, PyObject *args) { @@ -1247,6 +1307,8 @@ METH_NOARGS, _count_doc}, {"stack_size", (PyCFunction)thread_stack_size, METH_VARARGS, stack_size_doc}, + {"_set_sentinel", (PyCFunction)thread__set_sentinel, + METH_NOARGS, _set_sentinel_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/pystate.c b/Python/pystate.c --- a/Python/pystate.c +++ b/Python/pystate.c @@ -208,6 +208,8 @@ tstate->trash_delete_nesting = 0; tstate->trash_delete_later = NULL; + tstate->on_delete = NULL; + tstate->on_delete_data = NULL; if (init) _PyThreadState_Init(tstate); @@ -390,6 +392,9 @@ if (tstate->next) tstate->next->prev = tstate->prev; HEAD_UNLOCK(); + if (tstate->on_delete != NULL) { + tstate->on_delete(tstate->on_delete_data); + } PyMem_RawFree(tstate); } -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 00:05:15 2013 From: python-checkins at python.org (raymond.hettinger) Date: Sun, 8 Sep 2013 00:05:15 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Move_the_overview_comment_?= =?utf-8?q?to_the_top_of_the_file=2E?= Message-ID: <3cXV7H6r0nz7LjW@mail.python.org> http://hg.python.org/cpython/rev/a4696f805088 changeset: 85599:a4696f805088 user: Raymond Hettinger date: Sat Sep 07 15:05:00 2013 -0700 summary: Move the overview comment to the top of the file. files: Objects/setobject.c | 42 +++++++++++++++----------------- 1 files changed, 20 insertions(+), 22 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1,10 +1,30 @@ /* set object implementation + Written and maintained by Raymond D. Hettinger Derived from Lib/sets.py and Objects/dictobject.c. Copyright (c) 2003-2013 Python Software Foundation. All rights reserved. + + The basic lookup function used by all operations. + This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. + + The initial probe index is computed as hash mod the table size. + Subsequent probe indices are computed as explained in Objects/dictobject.c. + + To improve cache locality, each probe inspects a series of consecutive + nearby entries before moving on to probes elsewhere in memory. This leaves + us with a hybrid of linear probing and open addressing. The linear probing + reduces the cost of hash collisions because consecutive memory accesses + tend to be much cheaper than scattered probes. After LINEAR_PROBES steps, + we then use open addressing with the upper bits from the hash value. This + helps break-up long chains of collisions. + + All arithmetic on hash should ignore overflow. + + Unlike the dictionary implementation, the lookkey functions can return + NULL if the rich comparison returns an error. */ #include "Python.h" @@ -44,28 +64,6 @@ static PySetObject *free_list[PySet_MAXFREELIST]; static int numfree = 0; - -/* -The basic lookup function used by all operations. -This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. - -The initial probe index is computed as hash mod the table size. -Subsequent probe indices are computed as explained in Objects/dictobject.c. - -To improve cache locality, each probe inspects a series of consecutive -nearby entries before moving on to probes elsewhere in memory. This leaves -us with a hybrid of linear probing and open addressing. The linear probing -reduces the cost of hash collisions because consecutive memory accesses -tend to be much cheaper than scattered probes. After LINEAR_PROBES steps, -we then use open addressing with the upper bits from the hash value. This -helps break-up long chains of collisions. - -All arithmetic on hash should ignore overflow. - -Unlike the dictionary implementation, the lookkey functions can return -NULL if the rich comparison returns an error. -*/ - static setentry * set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) { -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 02:41:35 2013 From: python-checkins at python.org (raymond.hettinger) Date: Sun, 8 Sep 2013 02:41:35 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Small_rearrangement_to_bri?= =?utf-8?q?ng_together_the_three_functions_for_probing_the_hash?= Message-ID: <3cXYbg706Qz7LjW@mail.python.org> http://hg.python.org/cpython/rev/dea4c70cc8d1 changeset: 85600:dea4c70cc8d1 user: Raymond Hettinger date: Sat Sep 07 17:41:01 2013 -0700 summary: Small rearrangement to bring together the three functions for probing the hash table. files: Objects/setobject.c | 71 ++++++++++++++++++-------------- 1 files changed, 39 insertions(+), 32 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -64,6 +64,9 @@ static PySetObject *free_list[PySet_MAXFREELIST]; static int numfree = 0; +/* ======================================================================== */ +/* ======= Begin logic for probing the hash table ========================= */ + static setentry * set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) { @@ -198,38 +201,6 @@ } /* -Internal routine to insert a new key into the table. -Used by the public insert routine. -Eats a reference to key. -*/ -static int -set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) -{ - setentry *entry; - - assert(so->lookup != NULL); - entry = so->lookup(so, key, hash); - if (entry == NULL) - return -1; - if (entry->key == NULL) { - /* UNUSED */ - so->fill++; - entry->key = key; - entry->hash = hash; - so->used++; - } else if (entry->key == dummy) { - /* DUMMY */ - entry->key = key; - entry->hash = hash; - so->used++; - } else { - /* ACTIVE */ - Py_DECREF(key); - } - return 0; -} - -/* Internal routine used by set_table_resize() to insert an item which is known to be absent from the set. This routine also assumes that the set contains no deleted entries. Besides the performance benefit, @@ -266,6 +237,42 @@ so->used++; } +/* ======== End logic for probing the hash table ========================== */ +/* ======================================================================== */ + + +/* +Internal routine to insert a new key into the table. +Used by the public insert routine. +Eats a reference to key. +*/ +static int +set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) +{ + setentry *entry; + + assert(so->lookup != NULL); + entry = so->lookup(so, key, hash); + if (entry == NULL) + return -1; + if (entry->key == NULL) { + /* UNUSED */ + so->fill++; + entry->key = key; + entry->hash = hash; + so->used++; + } else if (entry->key == dummy) { + /* DUMMY */ + entry->key = key; + entry->hash = hash; + so->used++; + } else { + /* ACTIVE */ + Py_DECREF(key); + } + return 0; +} + /* Restructure the table by allocating a new table and reinserting all keys again. When entries have been deleted, the new table may -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 02:52:46 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sun, 8 Sep 2013 02:52:46 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Correct_Profil?= =?utf-8?q?e_class_usage_example=2E_Addresses_issue_=2318033_=2E?= Message-ID: <3cXYrZ1PDcz7LjR@mail.python.org> http://hg.python.org/cpython/rev/93018d47793f changeset: 85601:93018d47793f branch: 2.7 parent: 85592:d5c5ac33b9a1 user: Senthil Kumaran date: Sat Sep 07 17:50:35 2013 -0700 summary: Correct Profile class usage example. Addresses issue #18033 . Patch contributed by Olivier Hervieu and Dmi Baranov. files: Doc/library/profile.rst | 12 +++++++----- 1 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -267,14 +267,16 @@ Directly using the :class:`Profile` class allows formatting profile results without writing the profile data to a file:: - import cProfile, pstats, io + import cProfile, pstats, StringIO pr = cProfile.Profile() pr.enable() - ... do something ... + # ... do something ... pr.disable() - s = io.StringIO() - ps = pstats.Stats(pr, stream=s) - ps.print_results() + s = StringIO.StringIO() + sortby = 'cumulative' + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats() + print s.getvalue() .. method:: enable() -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 02:52:47 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sun, 8 Sep 2013 02:52:47 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Correct_Profil?= =?utf-8?q?e_class_usage_example=2E_Addresses_issue_=2318033=2E?= Message-ID: <3cXYrb33T4z7LjX@mail.python.org> http://hg.python.org/cpython/rev/ab4d3ccb92e6 changeset: 85602:ab4d3ccb92e6 branch: 3.3 parent: 85595:a723adeb1f47 user: Senthil Kumaran date: Sat Sep 07 17:51:58 2013 -0700 summary: Correct Profile class usage example. Addresses issue #18033. Patch contributed by Olivier Hervieu and Dmi Baranov. files: Doc/library/profile.rst | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -247,11 +247,13 @@ import cProfile, pstats, io pr = cProfile.Profile() pr.enable() - ... do something ... + # ... do something ... pr.disable() s = io.StringIO() - ps = pstats.Stats(pr, stream=s) - ps.print_results() + sortby = 'cumulative' + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats() + print(s.getvalue()) .. method:: enable() -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 02:52:48 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sun, 8 Sep 2013 02:52:48 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_merge_from_3=2E3?= Message-ID: <3cXYrc4pMlz7Lk2@mail.python.org> http://hg.python.org/cpython/rev/88182b388bae changeset: 85603:88182b388bae parent: 85600:dea4c70cc8d1 parent: 85602:ab4d3ccb92e6 user: Senthil Kumaran date: Sat Sep 07 17:52:38 2013 -0700 summary: merge from 3.3 Correct Profile class usage example. Addresses issue #18033. Patch contributed by Olivier Hervieu and Dmi Baranov. files: Doc/library/profile.rst | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -247,11 +247,13 @@ import cProfile, pstats, io pr = cProfile.Profile() pr.enable() - ... do something ... + # ... do something ... pr.disable() s = io.StringIO() - ps = pstats.Stats(pr, stream=s) - ps.print_results() + sortby = 'cumulative' + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats() + print(s.getvalue()) .. method:: enable() -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 04:23:24 2013 From: python-checkins at python.org (tim.peters) Date: Sun, 8 Sep 2013 04:23:24 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_18808=3A__blind_atte?= =?utf-8?q?mpt_to_repair_some_buildbot_failures=2E?= Message-ID: <3cXbs84ppBz7LjW@mail.python.org> http://hg.python.org/cpython/rev/5cfd7b2eb994 changeset: 85604:5cfd7b2eb994 user: Tim Peters date: Sat Sep 07 21:23:03 2013 -0500 summary: Issue 18808: blind attempt to repair some buildbot failures. test_is_alive_after_fork is failing on some old Linux kernels, but passing on all newer ones. Since virtually anything can go wrong with locks when mixing threads with fork, replace the most likely cause with a redundant simple data member. files: Lib/threading.py | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -552,6 +552,10 @@ self._tstate_lock = None self._started = Event() self._stopped = Event() + # _is_stopped should be the same as _stopped.is_set(). The bizarre + # duplication is to allow test_is_alive_after_fork to pass on old + # Linux kernels. See issue 18808. + self._is_stopped = False self._initialized = True # sys.stderr is not stored in the class like # sys.exc_info since it can be changed between instances @@ -707,6 +711,7 @@ def _stop(self): self._stopped.set() + self._is_stopped = True def _delete(self): "Remove current thread from the dict of currently running threads." @@ -793,7 +798,7 @@ assert self._initialized, "Thread.__init__() not called" if not self._started.is_set(): return False - if not self._stopped.is_set(): + if not self._is_stopped: return True # The Python part of the thread is done, but the C part may still be # waiting to run. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 04:51:23 2013 From: python-checkins at python.org (nick.coghlan) Date: Sun, 8 Sep 2013 04:51:23 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogQ2xvc2UgIzE4OTUy?= =?utf-8?q?=3A_correctly_download_test_support_data?= Message-ID: <3cXcTR3qrFz7LkN@mail.python.org> http://hg.python.org/cpython/rev/23770d446c73 changeset: 85605:23770d446c73 branch: 3.3 parent: 85602:ab4d3ccb92e6 user: Nick Coghlan date: Sun Sep 08 11:40:34 2013 +1000 summary: Close #18952: correctly download test support data When test.support was converted to a package, it started silently skipping the tests which needed to download support data to run. This change refactors the affected code, and also tidies up test.support.findfile to remove the unused *here* parameter, document the *subdir* parameter and rename the *filename* parameter to avoid shadowing the file builtin and be consistent with the documentation. The unexpected skips were noticed and reported by Zachary Ware files: Doc/library/test.rst | 5 +++- Lib/test/support/__init__.py | 31 ++++++++++++++--------- Misc/NEWS | 4 +++ 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/Doc/library/test.rst b/Doc/library/test.rst --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -263,12 +263,15 @@ Used when tests are executed by :mod:`test.regrtest`. -.. function:: findfile(filename) +.. function:: findfile(filename, subdir=None) Return the path to the file named *filename*. If no match is found *filename* is returned. This does not equal a failure since it could be the path to the file. + Setting *subdir* indicates a relative path to use to find the file + rather than looking directly in the path directories. + .. function:: run_unittest(\*classes) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -853,24 +853,31 @@ finally: os.umask(oldmask) -# TEST_HOME refers to the top level directory of the "test" package +# TEST_HOME_DIR refers to the top level directory of the "test" package # that contains Python's regression test suite -TEST_HOME = os.path.dirname(os.path.abspath(__file__)) +TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR) -def findfile(file, here=TEST_HOME, subdir=None): +# TEST_DATA_DIR is used as a target download location for remote resources +TEST_DATA_DIR = os.path.join(TEST_HOME_DIR, "data") + +def findfile(filename, subdir=None): """Try to find a file on sys.path or in the test directory. If it is not found the argument passed to the function is returned (this does not - necessarily signal failure; could still be the legitimate path).""" - if os.path.isabs(file): - return file + necessarily signal failure; could still be the legitimate path). + + Setting *subdir* indicates a relative path to use to find the file + rather than looking directly in the path directories. + """ + if os.path.isabs(filename): + return filename if subdir is not None: - file = os.path.join(subdir, file) - path = sys.path - path = [os.path.dirname(here)] + path + filename = os.path.join(subdir, filename) + path = [TEST_HOME_DIR] + sys.path for dn in path: - fn = os.path.join(dn, file) + fn = os.path.join(dn, filename) if os.path.exists(fn): return fn - return file + return filename def create_empty_file(filename): """Create an empty file. If the file already exists, truncate it.""" @@ -907,7 +914,7 @@ filename = urllib.parse.urlparse(url)[2].split('/')[-1] # '/': it's URL! - fn = os.path.join(os.path.dirname(__file__), "data", filename) + fn = os.path.join(TEST_DATA_DIR, filename) def check_valid_file(fn): f = open(fn, *args, **kw) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -332,6 +332,10 @@ Tests ----- +- Issue #18952: Fix regression in support data downloads introduced when + test.support was converted to a package. Regression noticed by Zachary + Ware. + - Issue #12037: Fix test_email for desktop Windows. - Issue #15507: test_subprocess's test_send_signal could fail if the test -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 04:51:24 2013 From: python-checkins at python.org (nick.coghlan) Date: Sun, 8 Sep 2013 04:51:24 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Merge_=2318952_fix_from_3=2E3?= Message-ID: <3cXcTS5fSPz7Ljf@mail.python.org> http://hg.python.org/cpython/rev/cb8dabc113b8 changeset: 85606:cb8dabc113b8 parent: 85604:5cfd7b2eb994 parent: 85605:23770d446c73 user: Nick Coghlan date: Sun Sep 08 12:49:53 2013 +1000 summary: Merge #18952 fix from 3.3 files: Doc/library/test.rst | 5 +++- Lib/test/support/__init__.py | 31 ++++++++++++++--------- Misc/NEWS | 13 ++++++++++ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/Doc/library/test.rst b/Doc/library/test.rst --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -263,12 +263,15 @@ Used when tests are executed by :mod:`test.regrtest`. -.. function:: findfile(filename) +.. function:: findfile(filename, subdir=None) Return the path to the file named *filename*. If no match is found *filename* is returned. This does not equal a failure since it could be the path to the file. + Setting *subdir* indicates a relative path to use to find the file + rather than looking directly in the path directories. + .. function:: run_unittest(\*classes) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -860,24 +860,31 @@ finally: os.umask(oldmask) -# TEST_HOME refers to the top level directory of the "test" package +# TEST_HOME_DIR refers to the top level directory of the "test" package # that contains Python's regression test suite -TEST_HOME = os.path.dirname(os.path.abspath(__file__)) +TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR) -def findfile(file, here=TEST_HOME, subdir=None): +# TEST_DATA_DIR is used as a target download location for remote resources +TEST_DATA_DIR = os.path.join(TEST_HOME_DIR, "data") + +def findfile(filename, subdir=None): """Try to find a file on sys.path or in the test directory. If it is not found the argument passed to the function is returned (this does not - necessarily signal failure; could still be the legitimate path).""" - if os.path.isabs(file): - return file + necessarily signal failure; could still be the legitimate path). + + Setting *subdir* indicates a relative path to use to find the file + rather than looking directly in the path directories. + """ + if os.path.isabs(filename): + return filename if subdir is not None: - file = os.path.join(subdir, file) - path = sys.path - path = [os.path.dirname(here)] + path + filename = os.path.join(subdir, filename) + path = [TEST_HOME_DIR] + sys.path for dn in path: - fn = os.path.join(dn, file) + fn = os.path.join(dn, filename) if os.path.exists(fn): return fn - return file + return filename def create_empty_file(filename): """Create an empty file. If the file already exists, truncate it.""" @@ -914,7 +921,7 @@ filename = urllib.parse.urlparse(url)[2].split('/')[-1] # '/': it's URL! - fn = os.path.join(os.path.dirname(__file__), "data", filename) + fn = os.path.join(TEST_DATA_DIR, filename) def check_valid_file(fn): f = open(fn, *args, **kw) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,19 @@ Python News +++++++++++ +What's New in Python 3.4.0 Alpha 3? +=================================== + +Projected Release date: 2013-10-XX + +Tests +----- + +- Issue #18952: Fix regression in support data downloads introduced when + test.support was converted to a package. Regression noticed by Zachary + Ware. + + What's New in Python 3.4.0 Alpha 2? =================================== -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 05:27:02 2013 From: python-checkins at python.org (raymond.hettinger) Date: Sun, 8 Sep 2013 05:27:02 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Remove_the_freelist_scheme?= =?utf-8?q?_for_setobjects=2E?= Message-ID: <3cXdGZ0n5nz7LjR@mail.python.org> http://hg.python.org/cpython/rev/36f3f58fddce changeset: 85607:36f3f58fddce user: Raymond Hettinger date: Sat Sep 07 20:26:50 2013 -0700 summary: Remove the freelist scheme for setobjects. The setobject freelist was consuming memory but not providing much value. Even when a freelisted setobject was available, most of the setobject fields still needed to be initialized and the small table still required a memset(). This meant that the custom freelisting scheme for sets was providing almost no incremental benefit over the default Python freelist scheme used by _PyObject_Malloc() in Objects/obmalloc.c. files: Include/setobject.h | 1 - Objects/object.c | 1 - Objects/setobject.c | 55 ++++---------------------------- 3 files changed, 8 insertions(+), 49 deletions(-) diff --git a/Include/setobject.h b/Include/setobject.h --- a/Include/setobject.h +++ b/Include/setobject.h @@ -105,7 +105,6 @@ PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); PyAPI_FUNC(int) PySet_ClearFreeList(void); -PyAPI_FUNC(void) _PySet_DebugMallocStats(FILE *out); #endif #ifdef __cplusplus diff --git a/Objects/object.c b/Objects/object.c --- a/Objects/object.c +++ b/Objects/object.c @@ -1955,7 +1955,6 @@ _PyFrame_DebugMallocStats(out); _PyList_DebugMallocStats(out); _PyMethod_DebugMallocStats(out); - _PySet_DebugMallocStats(out); _PyTuple_DebugMallocStats(out); } diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -57,13 +57,6 @@ INIT_NONZERO_SET_SLOTS(so); \ } while(0) -/* Reuse scheme to save calls to malloc, free, and memset */ -#ifndef PySet_MAXFREELIST -#define PySet_MAXFREELIST 80 -#endif -static PySetObject *free_list[PySet_MAXFREELIST]; -static int numfree = 0; - /* ======================================================================== */ /* ======= Begin logic for probing the hash table ========================= */ @@ -565,10 +558,7 @@ } if (so->table != so->smalltable) PyMem_DEL(so->table); - if (numfree < PySet_MAXFREELIST && PyAnySet_CheckExact(so)) - free_list[numfree++] = so; - else - Py_TYPE(so)->tp_free(so); + Py_TYPE(so)->tp_free(so); Py_TRASHCAN_SAFE_END(so) } @@ -1023,22 +1013,12 @@ PySetObject *so = NULL; /* create PySetObject structure */ - if (numfree && - (type == &PySet_Type || type == &PyFrozenSet_Type)) { - so = free_list[--numfree]; - assert (so != NULL && PyAnySet_CheckExact(so)); - Py_TYPE(so) = type; - _Py_NewReference((PyObject *)so); - EMPTY_TO_MINSIZE(so); - PyObject_GC_Track(so); - } else { - so = (PySetObject *)type->tp_alloc(type, 0); - if (so == NULL) - return NULL; - /* tp_alloc has already zeroed the structure */ - assert(so->table == NULL && so->fill == 0 && so->used == 0); - INIT_NONZERO_SET_SLOTS(so); - } + so = (PySetObject *)type->tp_alloc(type, 0); + if (so == NULL) + return NULL; + /* tp_alloc has already zeroed the structure */ + assert(so->table == NULL && so->fill == 0 && so->used == 0); + INIT_NONZERO_SET_SLOTS(so); so->lookup = set_lookkey_unicode; so->weakreflist = NULL; @@ -1103,34 +1083,15 @@ int PySet_ClearFreeList(void) { - int freelist_size = numfree; - PySetObject *so; - - while (numfree) { - numfree--; - so = free_list[numfree]; - PyObject_GC_Del(so); - } - return freelist_size; + return 0; } void PySet_Fini(void) { - PySet_ClearFreeList(); Py_CLEAR(emptyfrozenset); } -/* Print summary info about the state of the optimized allocator */ -void -_PySet_DebugMallocStats(FILE *out) -{ - _PyDebugAllocatorStats(out, - "free PySetObject", - numfree, sizeof(PySetObject)); -} - - static PyObject * set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 05:34:24 2013 From: python-checkins at python.org (michael.foord) Date: Sun, 8 Sep 2013 05:34:24 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Closes_issue_1?= =?utf-8?q?4971=2E?= Message-ID: <3cXdR429Lqz7Ljc@mail.python.org> http://hg.python.org/cpython/rev/78a5de507f19 changeset: 85608:78a5de507f19 branch: 2.7 parent: 85601:93018d47793f user: Michael Foord date: Sun Sep 08 15:34:27 2013 +1200 summary: Closes issue 14971. unittest test discovery no longer gets confused when a function has a different __name__ than its name in the TestCase class dictionary. files: Lib/unittest/loader.py | 4 +++- Lib/unittest/test/test_loader.py | 15 +++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 21 insertions(+), 1 deletions(-) diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -106,7 +106,9 @@ elif (isinstance(obj, types.UnboundMethodType) and isinstance(parent, type) and issubclass(parent, case.TestCase)): - return self.suiteClass([parent(obj.__name__)]) + name = parts[-1] + inst = parent(name) + return self.suiteClass([inst]) elif isinstance(obj, suite.TestSuite): return obj elif hasattr(obj, '__call__'): diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py --- a/Lib/unittest/test/test_loader.py +++ b/Lib/unittest/test/test_loader.py @@ -1281,6 +1281,21 @@ loader = unittest.TestLoader() self.assertTrue(loader.suiteClass is unittest.TestSuite) + # Make sure the dotted name resolution works even if the actual + # function doesn't have the same name as is used to find it. + def test_loadTestsFromName__function_with_different_name_than_method(self): + # lambdas have the name ''. + m = types.ModuleType('m') + class MyTestCase(unittest.TestCase): + test = lambda: 1 + m.testcase_1 = MyTestCase + + loader = unittest.TestLoader() + suite = loader.loadTestsFromNames(['testcase_1.test'], m) + self.assertIsInstance(suite, loader.suiteClass) + + ref_suite = unittest.TestSuite([MyTestCase('test')]) + self.assertEqual(list(suite), [ref_suite]) if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Library ------- +- Issue #14971: unittest test discovery no longer gets confused when a function + has a different __name__ than its name in the TestCase class dictionary. + - Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in the _sre moduel. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 06:01:47 2013 From: python-checkins at python.org (raymond.hettinger) Date: Sun, 8 Sep 2013 06:01:47 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Improve_code_clarity_by_re?= =?utf-8?q?moving_two_unattractive_macros=2E?= Message-ID: <3cXf2g0j9qz7LjR@mail.python.org> http://hg.python.org/cpython/rev/cfcc85773ecd changeset: 85609:cfcc85773ecd parent: 85607:36f3f58fddce user: Raymond Hettinger date: Sat Sep 07 21:01:29 2013 -0700 summary: Improve code clarity by removing two unattractive macros. files: Objects/setobject.c | 34 +++++++++++++++++--------------- 1 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -45,17 +45,6 @@ /* Exported for the gdb plugin's benefit. */ PyObject *_PySet_Dummy = dummy; -#define INIT_NONZERO_SET_SLOTS(so) do { \ - (so)->table = (so)->smalltable; \ - (so)->mask = PySet_MINSIZE - 1; \ - (so)->hash = -1; \ - } while(0) - -#define EMPTY_TO_MINSIZE(so) do { \ - memset((so)->smalltable, 0, sizeof((so)->smalltable)); \ - (so)->used = (so)->fill = 0; \ - INIT_NONZERO_SET_SLOTS(so); \ - } while(0) /* ======================================================================== */ /* ======= Begin logic for probing the hash table ========================= */ @@ -439,6 +428,17 @@ return DISCARD_FOUND; } +static void +set_empty_to_minsize(PySetObject *so) +{ + memset(so->smalltable, 0, sizeof(so->smalltable)); + so->fill = 0; + so->used = 0; + so->mask = PySet_MINSIZE - 1; + so->table = so->smalltable; + so->hash = -1; +} + static int set_clear_internal(PySetObject *so) { @@ -466,7 +466,7 @@ */ fill = so->fill; if (table_is_malloced) - EMPTY_TO_MINSIZE(so); + set_empty_to_minsize(so); else if (fill > 0) { /* It's a small table with something that needs to be cleared. @@ -475,7 +475,7 @@ */ memcpy(small_copy, table, sizeof(small_copy)); table = small_copy; - EMPTY_TO_MINSIZE(so); + set_empty_to_minsize(so); } /* else it's a small table that's already empty */ @@ -1016,11 +1016,13 @@ so = (PySetObject *)type->tp_alloc(type, 0); if (so == NULL) return NULL; - /* tp_alloc has already zeroed the structure */ - assert(so->table == NULL && so->fill == 0 && so->used == 0); - INIT_NONZERO_SET_SLOTS(so); + so->fill = 0; + so->used = 0; + so->mask = PySet_MINSIZE - 1; + so->table = so->smalltable; so->lookup = set_lookkey_unicode; + so->hash = -1; so->weakreflist = NULL; if (iterable != NULL) { -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Sun Sep 8 06:16:46 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sun, 08 Sep 2013 06:16:46 +0200 Subject: [Python-checkins] Daily reference leaks (88182b388bae): sum=0 Message-ID: results for 88182b388bae on branch "default" -------------------------------------------- Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogoGzV9L', '-x'] From python-checkins at python.org Sun Sep 8 07:06:44 2013 From: python-checkins at python.org (raymond.hettinger) Date: Sun, 8 Sep 2013 07:06:44 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Minor_code_beautification?= =?utf-8?q?=2E?= Message-ID: <3cXgTc1mWQz7LjY@mail.python.org> http://hg.python.org/cpython/rev/7d9d55fbc19c changeset: 85610:7d9d55fbc19c user: Raymond Hettinger date: Sat Sep 07 22:06:35 2013 -0700 summary: Minor code beautification. files: Objects/setobject.c | 11 +++++------ 1 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -446,14 +446,13 @@ int table_is_malloced; Py_ssize_t fill; setentry small_copy[PySet_MINSIZE]; + #ifdef Py_DEBUG - Py_ssize_t i, n; - assert (PyAnySet_Check(so)); - - n = so->mask + 1; - i = 0; + Py_ssize_t i = 0; + Py_ssize_t n = so->mask + 1; #endif + assert (PyAnySet_Check(so)); table = so->table; assert(table != NULL); table_is_malloced = table != so->smalltable; @@ -2366,7 +2365,7 @@ Py_ssize_t count; char *s; Py_ssize_t i; - PyObject *elem=NULL, *dup=NULL, *t, *f, *dup2, *x; + PyObject *elem=NULL, *dup=NULL, *t, *f, *dup2, *x=NULL; PyObject *ob = (PyObject *)so; Py_hash_t hash; PyObject *str; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 08:20:15 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sun, 8 Sep 2013 08:20:15 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Fix_the_dead_l?= =?utf-8?q?ink_of_IEEE=5F854-1987_standard_with_the_Wikipedia_entry=2E?= Message-ID: <3cXj6R55bnz7Ljd@mail.python.org> http://hg.python.org/cpython/rev/44ed0cd3dc6d changeset: 85611:44ed0cd3dc6d branch: 2.7 parent: 85608:78a5de507f19 user: Senthil Kumaran date: Sat Sep 07 23:18:53 2013 -0700 summary: Fix the dead link of IEEE_854-1987 standard with the Wikipedia entry. Addresses issue #18438 files: Lib/decimal.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/decimal.py b/Lib/decimal.py --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -25,7 +25,7 @@ and IEEE standard 854-1987: - www.cs.berkeley.edu/~ejr/projects/754/private/drafts/854-1987/dir.html + http://en.wikipedia.org/wiki/IEEE_854-1987 Decimal floating point has finite precision with arbitrarily large bounds. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 08:20:16 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sun, 8 Sep 2013 08:20:16 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Fix_the_dead_l?= =?utf-8?q?ink_of_IEEE=5F854-1987_standard_with_the_Wikipedia_entry=2E?= Message-ID: <3cXj6S73nfz7Ljd@mail.python.org> http://hg.python.org/cpython/rev/0eef1670f316 changeset: 85612:0eef1670f316 branch: 3.3 parent: 85605:23770d446c73 user: Senthil Kumaran date: Sat Sep 07 23:19:29 2013 -0700 summary: Fix the dead link of IEEE_854-1987 standard with the Wikipedia entry. Addresses issue #18438 files: Lib/decimal.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/decimal.py b/Lib/decimal.py --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -21,7 +21,7 @@ and IEEE standard 854-1987: - www.cs.berkeley.edu/~ejr/projects/754/private/drafts/854-1987/dir.html + http://en.wikipedia.org/wiki/IEEE_854-1987 Decimal floating point has finite precision with arbitrarily large bounds. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 08:20:18 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sun, 8 Sep 2013 08:20:18 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_merge_from_3=2E3?= Message-ID: <3cXj6V206Sz7LkW@mail.python.org> http://hg.python.org/cpython/rev/44fa59286012 changeset: 85613:44fa59286012 parent: 85610:7d9d55fbc19c parent: 85612:0eef1670f316 user: Senthil Kumaran date: Sat Sep 07 23:20:06 2013 -0700 summary: merge from 3.3 Fix the dead link of IEEE_854-1987 standard with the Wikipedia entry. Addresses issue #18438 files: Lib/decimal.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/decimal.py b/Lib/decimal.py --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -21,7 +21,7 @@ and IEEE standard 854-1987: - www.cs.berkeley.edu/~ejr/projects/754/private/drafts/854-1987/dir.html + http://en.wikipedia.org/wiki/IEEE_854-1987 Decimal floating point has finite precision with arbitrarily large bounds. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 09:26:04 2013 From: python-checkins at python.org (raymond.hettinger) Date: Sun, 8 Sep 2013 09:26:04 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Put_the_defines_in_the_log?= =?utf-8?q?ical_section_and_fix_indentation=2E?= Message-ID: <3cXkZN6sf1z7LjR@mail.python.org> http://hg.python.org/cpython/rev/b03f98c1052f changeset: 85614:b03f98c1052f user: Raymond Hettinger date: Sun Sep 08 00:25:57 2013 -0700 summary: Put the defines in the logical section and fix indentation. files: Objects/setobject.c | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -31,12 +31,6 @@ #include "structmember.h" #include "stringlib/eq.h" -/* This must be >= 1 */ -#define PERTURB_SHIFT 5 - -/* This should be >= PySet_MINSIZE - 1 */ -#define LINEAR_PROBES 9 - /* Object used as dummy key to fill deleted entries */ static PyObject _dummy_struct; @@ -49,6 +43,12 @@ /* ======================================================================== */ /* ======= Begin logic for probing the hash table ========================= */ +/* This should be >= PySet_MINSIZE - 1 */ +#define LINEAR_PROBES 9 + +/* This must be >= 1 */ +#define PERTURB_SHIFT 5 + static setentry * set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) { @@ -151,8 +151,8 @@ while (1) { if (entry->key == key || (entry->hash == hash - && entry->key != dummy - && unicode_eq(entry->key, key))) + && entry->key != dummy + && unicode_eq(entry->key, key))) return entry; if (entry->key == dummy && freeslot == NULL) freeslot = entry; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 11:35:11 2013 From: python-checkins at python.org (charles-francois.natali) Date: Sun, 8 Sep 2013 11:35:11 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318934=3A_Use_poll?= =?utf-8?q?/select-based_selectors_for_multiprocessing=2EConnection=2C?= Message-ID: <3cXnRM6JQ7z7LjR@mail.python.org> http://hg.python.org/cpython/rev/0e52b9f77dbf changeset: 85615:0e52b9f77dbf user: Charles-Fran?ois Natali date: Sun Sep 08 11:30:53 2013 +0200 summary: Issue #18934: Use poll/select-based selectors for multiprocessing.Connection, to avoid one extra FD per Connection. files: Lib/multiprocessing/connection.py | 10 +++++++++- 1 files changed, 9 insertions(+), 1 deletions(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -878,13 +878,21 @@ import selectors + # poll/select have the advantage of not requiring any extra file + # descriptor, contrarily to epoll/kqueue (also, they require a single + # syscall). + if hasattr(selectors, 'PollSelector'): + _WaitSelector = selectors.PollSelector + else: + _WaitSelector = selectors.SelectSelector + def wait(object_list, timeout=None): ''' Wait till an object in object_list is ready/readable. Returns list of those objects in object_list which are ready/readable. ''' - with selectors.DefaultSelector() as selector: + with _WaitSelector() as selector: for obj in object_list: selector.register(obj, selectors.EVENT_READ) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 11:35:13 2013 From: python-checkins at python.org (charles-francois.natali) Date: Sun, 8 Sep 2013 11:35:13 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318963=3A_Fix_test?= =?utf-8?q?=5Fselectors=2Etest=5Fabove=5Ffd=5Fsetsize_on_OS_X=2C_where_the?= Message-ID: <3cXnRP15wZz7Ljl@mail.python.org> http://hg.python.org/cpython/rev/9ba1432fdc5a changeset: 85616:9ba1432fdc5a user: Charles-Fran?ois Natali date: Sun Sep 08 11:34:42 2013 +0200 summary: Issue #18963: Fix test_selectors.test_above_fd_setsize on OS X, where the default RLIMIT_NOFILE hard limit can be RLIMIT_INFINITY. files: Lib/test/test_selectors.py | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -301,7 +301,6 @@ class ScalableSelectorMixIn: - @support.requires_mac_ver(10, 5) @unittest.skipUnless(resource, "Test needs resource module") def test_above_fd_setsize(self): # A scalable implementation should have no problem with more than @@ -313,7 +312,7 @@ self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, (soft, hard)) NUM_FDS = hard - except OSError: + except (OSError, ValueError): NUM_FDS = soft # guard for already allocated FDs (stdin, stdout...) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 11:36:48 2013 From: python-checkins at python.org (victor.stinner) Date: Sun, 8 Sep 2013 11:36:48 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Close_=2318957=3A_The_PYTH?= =?utf-8?q?ONFAULTHANDLER_environment_variable_now_only_enables_the?= Message-ID: <3cXnTD1SXbz7LjR@mail.python.org> http://hg.python.org/cpython/rev/5c2cf4349adc changeset: 85617:5c2cf4349adc user: Victor Stinner date: Sun Sep 08 11:36:23 2013 +0200 summary: Close #18957: The PYTHONFAULTHANDLER environment variable now only enables the faulthandler module if the variable is non-empty. Same behaviour than other variables like PYTHONDONTWRITEBYTECODE. files: Doc/using/cmdline.rst | 16 ++++++------ Lib/test/test_faulthandler.py | 30 +++++++++++++++++----- Misc/NEWS | 7 +++++ Modules/faulthandler.c | 5 +++- 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -511,9 +511,9 @@ .. envvar:: PYTHONDONTWRITEBYTECODE - If this is set, Python won't try to write ``.pyc`` or ``.pyo`` files on the - import of source modules. This is equivalent to specifying the :option:`-B` - option. + If this is set to a non-empty string, Python won't try to write ``.pyc`` or + ``.pyo`` files on the import of source modules. This is equivalent to + specifying the :option:`-B` option. .. envvar:: PYTHONHASHSEED @@ -582,11 +582,11 @@ .. envvar:: PYTHONFAULTHANDLER - If this environment variable is set, :func:`faulthandler.enable` is called - at startup: install a handler for :const:`SIGSEGV`, :const:`SIGFPE`, - :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` signals to dump the - Python traceback. This is equivalent to :option:`-X` ``faulthandler`` - option. + If this environment variable is set to a non-empty string, + :func:`faulthandler.enable` is called at startup: install a handler for + :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and + :const:`SIGILL` signals to dump the Python traceback. This is equivalent to + :option:`-X` ``faulthandler`` option. .. versionadded:: 3.3 diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -265,17 +265,33 @@ # By default, the module should be disabled code = "import faulthandler; print(faulthandler.is_enabled())" args = (sys.executable, '-E', '-c', code) - # use subprocess module directly because test.script_helper adds - # "-X faulthandler" to the command line - stdout = subprocess.check_output(args) - self.assertEqual(stdout.rstrip(), b"False") + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args) + self.assertEqual(output.rstrip(), b"False") def test_sys_xoptions(self): # Test python -X faulthandler code = "import faulthandler; print(faulthandler.is_enabled())" - rc, stdout, stderr = assert_python_ok("-X", "faulthandler", "-c", code) - stdout = (stdout + stderr).strip() - self.assertEqual(stdout, b"True") + args = (sys.executable, "-E", "-X", "faulthandler", "-c", code) + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args) + self.assertEqual(output.rstrip(), b"True") + + def test_env_var(self): + # empty env var + code = "import faulthandler; print(faulthandler.is_enabled())" + args = (sys.executable, "-c", code) + env = os.environ.copy() + env['PYTHONFAULTHANDLER'] = '' + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args, env=env) + self.assertEqual(output.rstrip(), b"False") + + # non-empty env var + env = os.environ.copy() + env['PYTHONFAULTHANDLER'] = '1' + output = subprocess.check_output(args, env=env) + self.assertEqual(output.rstrip(), b"True") def check_dump_traceback(self, filename): """ diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -7,6 +7,13 @@ Projected Release date: 2013-10-XX +Library +------- + +- The :envvar:`PYTHONFAULTHANDLER` environment variable now only enables the + faulthandler module if the variable is non-empty. Same behaviour than other + variables like :envvar:`PYTHONDONTWRITEBYTECODE`. + Tests ----- diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1048,8 +1048,11 @@ { PyObject *xoptions, *key, *module, *res; _Py_IDENTIFIER(enable); + char *p; - if (!Py_GETENV("PYTHONFAULTHANDLER")) { + if (!((p = Py_GETENV("PYTHONFAULTHANDLER")) && *p != '\0')) { + /* PYTHONFAULTHANDLER environment variable is missing + or an empty string */ int has_key; xoptions = PySys_GetXOptions(); -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 11:48:11 2013 From: python-checkins at python.org (victor.stinner) Date: Sun, 8 Sep 2013 11:48:11 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318904=3A_Improve_?= =?utf-8?q?os=2Eget/set=5Finheritable=28=29_tests?= Message-ID: <3cXnkM0X7vz7LjR@mail.python.org> http://hg.python.org/cpython/rev/b13cec63b495 changeset: 85618:b13cec63b495 user: Victor Stinner date: Sun Sep 08 11:47:54 2013 +0200 summary: Issue #18904: Improve os.get/set_inheritable() tests files: Lib/test/test_os.py | 36 ++++++++++++++++++++++++++------ 1 files changed, 29 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -34,6 +34,10 @@ import resource except ImportError: resource = None +try: + import fcntl +except ImportError: + fcntl = None from test.script_helper import assert_python_ok @@ -2300,19 +2304,37 @@ class FDInheritanceTests(unittest.TestCase): - def test_get_inheritable(self): + def test_get_set_inheritable(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) - for inheritable in (False, True): - os.set_inheritable(fd, inheritable) - self.assertEqual(os.get_inheritable(fd), inheritable) + self.assertEqual(os.get_inheritable(fd), False) - def test_set_inheritable(self): - fd = os.open(__file__, os.O_RDONLY) - self.addCleanup(os.close, fd) os.set_inheritable(fd, True) self.assertEqual(os.get_inheritable(fd), True) + if fcntl: + def test_get_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(os.get_inheritable(fd), True) + + def test_set_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + os.set_inheritable(fd, True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + def test_open(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 11:54:00 2013 From: python-checkins at python.org (victor.stinner) Date: Sun, 8 Sep 2013 11:54:00 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318904=3A_test=5Fs?= =?utf-8?q?ocket=3A_add_inheritance_tests_using_fcntl_and_FD=5FCLOEXEC?= Message-ID: <3cXns46D3sz7LjR@mail.python.org> http://hg.python.org/cpython/rev/b7f6f6f59e91 changeset: 85619:b7f6f6f59e91 user: Victor Stinner date: Sun Sep 08 11:53:09 2013 +0200 summary: Issue #18904: test_socket: add inheritance tests using fcntl and FD_CLOEXEC files: Lib/test/test_socket.py | 30 +++++++++++++++++++++++++++++ 1 files changed, 30 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -26,6 +26,10 @@ import multiprocessing except ImportError: multiprocessing = False +try: + import fcntl +except ImportError: + fcntl = None HOST = support.HOST MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return @@ -4804,6 +4808,32 @@ sock.set_inheritable(False) self.assertEqual(sock.get_inheritable(), False) + if fcntl: + def test_get_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(sock.get_inheritable(), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(sock.get_inheritable(), True) + + def test_set_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + sock.set_inheritable(True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + + @unittest.skipUnless(hasattr(socket, "socketpair"), "need socket.socketpair()") def test_socketpair(self): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 12:36:15 2013 From: python-checkins at python.org (charles-francois.natali) Date: Sun, 8 Sep 2013 12:36:15 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318935=3A_Fix_test?= =?utf-8?q?=5Fregrtest=2Etest=5Ftimeout_when_built_--without-threads_=28th?= =?utf-8?q?e?= Message-ID: <3cXpnq6R77z7LjM@mail.python.org> http://hg.python.org/cpython/rev/2982ac8b45bc changeset: 85620:2982ac8b45bc parent: 85616:9ba1432fdc5a user: Charles-Fran?ois Natali date: Sun Sep 08 12:27:33 2013 +0200 summary: Issue #18935: Fix test_regrtest.test_timeout when built --without-threads (the '--timeout' option requires faulthandler.dump_traceback_later). files: Lib/test/test_regrtest.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -3,6 +3,7 @@ """ import argparse +import faulthandler import getopt import os.path import unittest @@ -25,6 +26,8 @@ regrtest._parse_args([opt]) self.assertIn('Run Python regression tests.', out.getvalue()) + @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), + "faulthandler.dump_traceback_later() required") def test_timeout(self): ns = regrtest._parse_args(['--timeout', '4.2']) self.assertEqual(ns.timeout, 4.2) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 12:36:17 2013 From: python-checkins at python.org (charles-francois.natali) Date: Sun, 8 Sep 2013 12:36:17 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318963=3A_skip_tes?= =?utf-8?q?t=5Fselectors=2Etest=5Fabove=5Ffd=5Fsetsize_on_older_OS_X_versi?= =?utf-8?q?ons=2E?= Message-ID: <3cXpns121dz7Ljv@mail.python.org> http://hg.python.org/cpython/rev/fa735675e485 changeset: 85621:fa735675e485 user: Charles-Fran?ois Natali date: Sun Sep 08 12:31:32 2013 +0200 summary: Issue #18963: skip test_selectors.test_above_fd_setsize on older OS X versions. files: Lib/test/test_selectors.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -301,6 +301,8 @@ class ScalableSelectorMixIn: + # see issue #18963 for why it's skipped on older OS X versions + @support.requires_mac_ver(10, 5) @unittest.skipUnless(resource, "Test needs resource module") def test_above_fd_setsize(self): # A scalable implementation should have no problem with more than -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 12:36:18 2013 From: python-checkins at python.org (charles-francois.natali) Date: Sun, 8 Sep 2013 12:36:18 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?= =?utf-8?b?KTogTWVyZ2Uu?= Message-ID: <3cXpnt4K18z7Lkk@mail.python.org> http://hg.python.org/cpython/rev/59b15f5194dc changeset: 85622:59b15f5194dc parent: 85621:fa735675e485 parent: 85619:b7f6f6f59e91 user: Charles-Fran?ois Natali date: Sun Sep 08 12:35:53 2013 +0200 summary: Merge. files: Doc/using/cmdline.rst | 16 +++++----- Lib/test/test_faulthandler.py | 30 ++++++++++++++---- Lib/test/test_os.py | 36 ++++++++++++++++++---- Lib/test/test_socket.py | 30 +++++++++++++++++++ Misc/NEWS | 7 ++++ Modules/faulthandler.c | 5 ++- 6 files changed, 101 insertions(+), 23 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -511,9 +511,9 @@ .. envvar:: PYTHONDONTWRITEBYTECODE - If this is set, Python won't try to write ``.pyc`` or ``.pyo`` files on the - import of source modules. This is equivalent to specifying the :option:`-B` - option. + If this is set to a non-empty string, Python won't try to write ``.pyc`` or + ``.pyo`` files on the import of source modules. This is equivalent to + specifying the :option:`-B` option. .. envvar:: PYTHONHASHSEED @@ -582,11 +582,11 @@ .. envvar:: PYTHONFAULTHANDLER - If this environment variable is set, :func:`faulthandler.enable` is called - at startup: install a handler for :const:`SIGSEGV`, :const:`SIGFPE`, - :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` signals to dump the - Python traceback. This is equivalent to :option:`-X` ``faulthandler`` - option. + If this environment variable is set to a non-empty string, + :func:`faulthandler.enable` is called at startup: install a handler for + :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and + :const:`SIGILL` signals to dump the Python traceback. This is equivalent to + :option:`-X` ``faulthandler`` option. .. versionadded:: 3.3 diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -265,17 +265,33 @@ # By default, the module should be disabled code = "import faulthandler; print(faulthandler.is_enabled())" args = (sys.executable, '-E', '-c', code) - # use subprocess module directly because test.script_helper adds - # "-X faulthandler" to the command line - stdout = subprocess.check_output(args) - self.assertEqual(stdout.rstrip(), b"False") + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args) + self.assertEqual(output.rstrip(), b"False") def test_sys_xoptions(self): # Test python -X faulthandler code = "import faulthandler; print(faulthandler.is_enabled())" - rc, stdout, stderr = assert_python_ok("-X", "faulthandler", "-c", code) - stdout = (stdout + stderr).strip() - self.assertEqual(stdout, b"True") + args = (sys.executable, "-E", "-X", "faulthandler", "-c", code) + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args) + self.assertEqual(output.rstrip(), b"True") + + def test_env_var(self): + # empty env var + code = "import faulthandler; print(faulthandler.is_enabled())" + args = (sys.executable, "-c", code) + env = os.environ.copy() + env['PYTHONFAULTHANDLER'] = '' + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args, env=env) + self.assertEqual(output.rstrip(), b"False") + + # non-empty env var + env = os.environ.copy() + env['PYTHONFAULTHANDLER'] = '1' + output = subprocess.check_output(args, env=env) + self.assertEqual(output.rstrip(), b"True") def check_dump_traceback(self, filename): """ diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -34,6 +34,10 @@ import resource except ImportError: resource = None +try: + import fcntl +except ImportError: + fcntl = None from test.script_helper import assert_python_ok @@ -2300,19 +2304,37 @@ class FDInheritanceTests(unittest.TestCase): - def test_get_inheritable(self): + def test_get_set_inheritable(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) - for inheritable in (False, True): - os.set_inheritable(fd, inheritable) - self.assertEqual(os.get_inheritable(fd), inheritable) + self.assertEqual(os.get_inheritable(fd), False) - def test_set_inheritable(self): - fd = os.open(__file__, os.O_RDONLY) - self.addCleanup(os.close, fd) os.set_inheritable(fd, True) self.assertEqual(os.get_inheritable(fd), True) + if fcntl: + def test_get_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(os.get_inheritable(fd), True) + + def test_set_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + os.set_inheritable(fd, True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + def test_open(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -26,6 +26,10 @@ import multiprocessing except ImportError: multiprocessing = False +try: + import fcntl +except ImportError: + fcntl = None HOST = support.HOST MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return @@ -4804,6 +4808,32 @@ sock.set_inheritable(False) self.assertEqual(sock.get_inheritable(), False) + if fcntl: + def test_get_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(sock.get_inheritable(), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(sock.get_inheritable(), True) + + def test_set_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + sock.set_inheritable(True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + + @unittest.skipUnless(hasattr(socket, "socketpair"), "need socket.socketpair()") def test_socketpair(self): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -7,6 +7,13 @@ Projected Release date: 2013-10-XX +Library +------- + +- The :envvar:`PYTHONFAULTHANDLER` environment variable now only enables the + faulthandler module if the variable is non-empty. Same behaviour than other + variables like :envvar:`PYTHONDONTWRITEBYTECODE`. + Tests ----- diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1048,8 +1048,11 @@ { PyObject *xoptions, *key, *module, *res; _Py_IDENTIFIER(enable); + char *p; - if (!Py_GETENV("PYTHONFAULTHANDLER")) { + if (!((p = Py_GETENV("PYTHONFAULTHANDLER")) && *p != '\0')) { + /* PYTHONFAULTHANDLER environment variable is missing + or an empty string */ int has_key; xoptions = PySys_GetXOptions(); -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 13:19:14 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sun, 8 Sep 2013 13:19:14 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318808_again=3A_fi?= =?utf-8?q?x_the_after-fork_logic_for_not-yet-started_or?= Message-ID: <3cXqlQ00w3z7LjR@mail.python.org> http://hg.python.org/cpython/rev/74dc664ad699 changeset: 85623:74dc664ad699 user: Antoine Pitrou date: Sun Sep 08 13:19:06 2013 +0200 summary: Issue #18808 again: fix the after-fork logic for not-yet-started or already-stopped threads. (AFAICT, in theory, we must reset all the locks, not just those in use) files: Lib/test/test_threading.py | 2 +- Lib/threading.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -109,7 +109,7 @@ if verbose: print('waiting for all tasks to complete') for t in threads: - t.join(NUMTASKS) + t.join() self.assertTrue(not t.is_alive()) self.assertNotEqual(t.ident, 0) self.assertFalse(t.ident is None) diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -517,8 +517,6 @@ _active_limbo_lock = _allocate_lock() _active = {} # maps thread id to Thread object _limbo = {} - -# For debug and leak testing _dangling = WeakSet() # Main class for threads @@ -552,14 +550,11 @@ self._tstate_lock = None self._started = Event() self._stopped = Event() - # _is_stopped should be the same as _stopped.is_set(). The bizarre - # duplication is to allow test_is_alive_after_fork to pass on old - # Linux kernels. See issue 18808. - self._is_stopped = False self._initialized = True # sys.stderr is not stored in the class like # sys.exc_info since it can be changed between instances self._stderr = _sys.stderr + # For debugging and _after_fork() _dangling.add(self) def _reset_internal_locks(self, is_alive): @@ -711,7 +706,6 @@ def _stop(self): self._stopped.set() - self._is_stopped = True def _delete(self): "Remove current thread from the dict of currently running threads." @@ -798,7 +792,7 @@ assert self._initialized, "Thread.__init__() not called" if not self._started.is_set(): return False - if not self._is_stopped: + if not self._stopped.is_set(): return True # The Python part of the thread is done, but the C part may still be # waiting to run. @@ -976,7 +970,11 @@ current = current_thread() _main_thread = current with _active_limbo_lock: - for thread in _enumerate(): + # Dangling thread instances must still have their locks reset, + # because someone may join() them. + threads = set(_enumerate()) + threads.update(_dangling) + for thread in threads: # Any lock/condition variable may be currently locked or in an # invalid state, so we reinitialize them. if thread is current: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 14:14:53 2013 From: python-checkins at python.org (victor.stinner) Date: Sun, 8 Sep 2013 14:14:53 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318904=3A_test=5Fo?= =?utf-8?q?s_and_test=5Fsocket_use_unittest=2EskipIf=28=29_to_check_if_fcn?= =?utf-8?q?tl?= Message-ID: <3cXrzd1GxQz7Ljt@mail.python.org> http://hg.python.org/cpython/rev/aea58e1cae75 changeset: 85624:aea58e1cae75 user: Victor Stinner date: Sun Sep 08 14:14:38 2013 +0200 summary: Issue #18904: test_os and test_socket use unittest.skipIf() to check if fcntl module is present (to record skipped tests) files: Lib/test/test_os.py | 37 +++++++++++---------- Lib/test/test_socket.py | 49 ++++++++++++++-------------- 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2312,28 +2312,29 @@ os.set_inheritable(fd, True) self.assertEqual(os.get_inheritable(fd), True) - if fcntl: - def test_get_inheritable_cloexec(self): - fd = os.open(__file__, os.O_RDONLY) - self.addCleanup(os.close, fd) - self.assertEqual(os.get_inheritable(fd), False) + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) - # clear FD_CLOEXEC flag - flags = fcntl.fcntl(fd, fcntl.F_GETFD) - flags &= ~fcntl.FD_CLOEXEC - fcntl.fcntl(fd, fcntl.F_SETFD, flags) + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) - self.assertEqual(os.get_inheritable(fd), True) + self.assertEqual(os.get_inheritable(fd), True) - def test_set_inheritable_cloexec(self): - fd = os.open(__file__, os.O_RDONLY) - self.addCleanup(os.close, fd) - self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, - fcntl.FD_CLOEXEC) + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) - os.set_inheritable(fd, True) - self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, - 0) + os.set_inheritable(fd, True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) def test_open(self): fd = os.open(__file__, os.O_RDONLY) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -4808,30 +4808,31 @@ sock.set_inheritable(False) self.assertEqual(sock.get_inheritable(), False) - if fcntl: - def test_get_inheritable_cloexec(self): - sock = socket.socket() - with sock: - fd = sock.fileno() - self.assertEqual(sock.get_inheritable(), False) - - # clear FD_CLOEXEC flag - flags = fcntl.fcntl(fd, fcntl.F_GETFD) - flags &= ~fcntl.FD_CLOEXEC - fcntl.fcntl(fd, fcntl.F_SETFD, flags) - - self.assertEqual(sock.get_inheritable(), True) - - def test_set_inheritable_cloexec(self): - sock = socket.socket() - with sock: - fd = sock.fileno() - self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, - fcntl.FD_CLOEXEC) - - sock.set_inheritable(True) - self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, - 0) + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(sock.get_inheritable(), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(sock.get_inheritable(), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + sock.set_inheritable(True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) @unittest.skipUnless(hasattr(socket, "socketpair"), -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 15:15:40 2013 From: python-checkins at python.org (victor.stinner) Date: Sun, 8 Sep 2013 15:15:40 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454=3A_add_clear=5Ffilter?= =?utf-8?b?cygpLCBmaWx0ZXIoKSBhbmQgZ2V0X2ZpbHRlcnMoKSBmdW5jdGlvbnM=?= Message-ID: <3cXtKm47bSz7LjM@mail.python.org> http://hg.python.org/peps/rev/47234db47d7b changeset: 5102:47234db47d7b user: Victor Stinner date: Sun Sep 08 15:15:26 2013 +0200 summary: PEP 454: add clear_filters(), filter() and get_filters() functions files: pep-0454.txt | 24 ++++++++++++++++++++++++ 1 files changed, 24 insertions(+), 0 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -84,6 +84,10 @@ Functions --------- +``clear_filters()`` function: + + Reset the filter list. + ``clear_traces()`` function: Clear all traces and statistics of memory allocations. @@ -97,6 +101,26 @@ Start tracing Python memory allocations. +``filter(include: bool, filename: str, lineno: int=None)`` function: + + Add a filter. If *include* is ``True``, only trace memory blocks + allocated in a file with a name matching *filename*. If + *include* is ``False``, don't trace memory blocks allocated in a + file with a name matching *filename*. + + The match is done using *filename* as a prefix. For example, + ``'/usr/bin/'`` only matchs files the ``/usr/bin`` directories. + + *lineno* is a line number. If *lineno* is ``None`` or lesser than + ``1``, it matches any line number. + +``get_filters()`` function: + + Get the filters as list of + ``(include: bool, filename: str, lineno: int)`` tuples. + + If *lineno* is ``None``, a filter matchs any line number. + ``get_number_frame()`` function: Get the maximum number of frames stored in a trace of a memory -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sun Sep 8 15:58:13 2013 From: python-checkins at python.org (victor.stinner) Date: Sun, 8 Sep 2013 15:58:13 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_454=3A_rename_filter=28?= =?utf-8?q?=29_to_add=5Ffilter=28=29=3B_update_the_PEP_to_the_last?= Message-ID: <3cXvGs4PZTz7LjR@mail.python.org> http://hg.python.org/peps/rev/e39c05e32466 changeset: 5103:e39c05e32466 user: Victor Stinner date: Sun Sep 08 15:58:01 2013 +0200 summary: PEP 454: rename filter() to add_filter(); update the PEP to the last implementation files: pep-0454.txt | 40 +++++++++++++++++++++++++++------------- 1 files changed, 27 insertions(+), 13 deletions(-) diff --git a/pep-0454.txt b/pep-0454.txt --- a/pep-0454.txt +++ b/pep-0454.txt @@ -84,6 +84,21 @@ Functions --------- +``add_filter(include: bool, filename: str, lineno: int=None)`` function: + + Add a filter. If *include* is ``True``, only trace memory blocks + allocated in a file with a name matching *filename*. If + *include* is ``False``, don't trace memory blocks allocated in a + file with a name matching *filename*. + + The match is done using *filename* as a prefix. For example, + ``'/usr/bin/'`` only matchs files the ``/usr/bin`` directories. The + ``.pyc`` and ``.pyo`` suffixes are automatically replaced with + ``.py`` when matching the filename. + + *lineno* is a line number. If *lineno* is ``None`` or lesser than + ``1``, it matches any line number. + ``clear_filters()`` function: Reset the filter list. @@ -101,19 +116,6 @@ Start tracing Python memory allocations. -``filter(include: bool, filename: str, lineno: int=None)`` function: - - Add a filter. If *include* is ``True``, only trace memory blocks - allocated in a file with a name matching *filename*. If - *include* is ``False``, don't trace memory blocks allocated in a - file with a name matching *filename*. - - The match is done using *filename* as a prefix. For example, - ``'/usr/bin/'`` only matchs files the ``/usr/bin`` directories. - - *lineno* is a line number. If *lineno* is ``None`` or lesser than - ``1``, it matches any line number. - ``get_filters()`` function: Get the filters as list of @@ -121,6 +123,9 @@ If *lineno* is ``None``, a filter matchs any line number. + By default, the filename of the Python tracemalloc module + (``tracemalloc.py``) is excluded. + ``get_number_frame()`` function: Get the maximum number of frames stored in a trace of a memory @@ -162,6 +167,10 @@ Return an empty dictionary if the tracemalloc module is disabled. +``get_tracemalloc_size()`` function: + + Get the memory usage in bytes of the ``tracemalloc`` module. + ``get_traces(obj)`` function: Get all traces of a Python memory allocations. @@ -365,6 +374,11 @@ Result of the ``get_stats()`` function. +``tracemalloc_size`` attribute (``int``): + + The memory usage in bytes of the ``tracemalloc`` module, + result of the ``get_tracemalloc_size()`` function. + ``timestamp`` attribute (``datetime.datetime``): Creation date and time of the snapshot. -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sun Sep 8 19:34:46 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Sun, 8 Sep 2013 19:34:46 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Fixed_tests_wi?= =?utf-8?q?th_Tcl/Tk_=3C8=2E5_=28closes_=2318964=29=2E?= Message-ID: <3cY04k5rsgz7Ljx@mail.python.org> http://hg.python.org/cpython/rev/03ee22236465 changeset: 85625:03ee22236465 branch: 3.3 parent: 85612:0eef1670f316 user: Serhiy Storchaka date: Sun Sep 08 20:29:37 2013 +0300 summary: Fixed tests with Tcl/Tk <8.5 (closes #18964). files: Lib/test/test_tcl.py | 22 ++++++++++++++++++---- 1 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -15,6 +15,14 @@ from tkinter import Tcl from _tkinter import TclError +tcl_version = _tkinter.TCL_VERSION.split('.') +try: + for i in range(len(tcl_version)): + tcl_version[i] = int(tcl_version[i]) +except ValueError: + pass +tcl_version = tuple(tcl_version) + class TkinterTest(unittest.TestCase): @@ -200,9 +208,12 @@ (('a', 3.4), ('a', 3.4)), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), - (call('dict', 'create', 1, '\u20ac', b'\xe2\x82\xac', (3.4,)), - (1, '\u20ac', '\u20ac', (3.4,))), ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 1, '\u20ac', b'\xe2\x82\xac', (3.4,)), + (1, '\u20ac', '\u20ac', (3.4,))), + ] for arg, res in testcases: self.assertEqual(splitlist(arg), res, msg=arg) self.assertRaises(TclError, splitlist, '{') @@ -234,9 +245,12 @@ (('a', (2, 3.4)), ('a', (2, 3.4))), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), - (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), - (12, '\u20ac', '\u20ac', (3.4,))), ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), + (12, '\u20ac', '\u20ac', (3.4,))), + ] for arg, res in testcases: self.assertEqual(split(arg), res, msg=arg) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 19:34:48 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Sun, 8 Sep 2013 19:34:48 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Fixed_tests_with_Tcl/Tk_=3C8=2E5_=28closes_=2318964=29?= =?utf-8?q?=2E?= Message-ID: <3cY04m0WY2z7LkW@mail.python.org> http://hg.python.org/cpython/rev/138e086e187d changeset: 85626:138e086e187d parent: 85624:aea58e1cae75 parent: 85625:03ee22236465 user: Serhiy Storchaka date: Sun Sep 08 20:31:20 2013 +0300 summary: Fixed tests with Tcl/Tk <8.5 (closes #18964). files: Lib/test/test_tcl.py | 22 ++++++++++++++++++---- 1 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -15,6 +15,14 @@ from tkinter import Tcl from _tkinter import TclError +tcl_version = _tkinter.TCL_VERSION.split('.') +try: + for i in range(len(tcl_version)): + tcl_version[i] = int(tcl_version[i]) +except ValueError: + pass +tcl_version = tuple(tcl_version) + class TkinterTest(unittest.TestCase): @@ -200,9 +208,12 @@ (('a', 3.4), ('a', 3.4)), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), - (call('dict', 'create', 1, '\u20ac', b'\xe2\x82\xac', (3.4,)), - (1, '\u20ac', '\u20ac', (3.4,))), ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 1, '\u20ac', b'\xe2\x82\xac', (3.4,)), + (1, '\u20ac', '\u20ac', (3.4,))), + ] for arg, res in testcases: self.assertEqual(splitlist(arg), res, msg=arg) self.assertRaises(TclError, splitlist, '{') @@ -234,9 +245,12 @@ (('a', (2, 3.4)), ('a', (2, 3.4))), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), - (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), - (12, '\u20ac', '\u20ac', (3.4,))), ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), + (12, '\u20ac', '\u20ac', (3.4,))), + ] for arg, res in testcases: self.assertEqual(split(arg), res, msg=arg) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 19:34:49 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Sun, 8 Sep 2013 19:34:49 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Fixed_tests_wi?= =?utf-8?q?th_Tcl/Tk_=3C8=2E5_=28closes_=2318964=29=2E?= Message-ID: <3cY04n2qm5z7Lkh@mail.python.org> http://hg.python.org/cpython/rev/a22cfd0bdc9a changeset: 85627:a22cfd0bdc9a branch: 2.7 parent: 85611:44ed0cd3dc6d user: Serhiy Storchaka date: Sun Sep 08 20:32:56 2013 +0300 summary: Fixed tests with Tcl/Tk <8.5 (closes #18964). files: Lib/test/test_tcl.py | 22 ++++++++++++++++++---- 1 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -13,6 +13,14 @@ from Tkinter import Tcl from _tkinter import TclError +tcl_version = _tkinter.TCL_VERSION.split('.') +try: + for i in range(len(tcl_version)): + tcl_version[i] = int(tcl_version[i]) +except ValueError: + pass +tcl_version = tuple(tcl_version) + class TkinterTest(unittest.TestCase): @@ -209,9 +217,12 @@ (('a', 3.4), ('a', 3.4)), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), - (call('dict', 'create', 1, u'\u20ac', '\xe2\x82\xac', (3.4,)), - (1, u'\u20ac', u'\u20ac', (3.4,))), ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 1, u'\u20ac', '\xe2\x82\xac', (3.4,)), + (1, u'\u20ac', u'\u20ac', (3.4,))), + ] for arg, res in testcases: self.assertEqual(splitlist(arg), res) self.assertRaises(TclError, splitlist, '{') @@ -243,9 +254,12 @@ (('a', (2, 3.4)), ('a', (2, 3.4))), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), - (call('dict', 'create', 12, u'\u20ac', '\xe2\x82\xac', (3.4,)), - (12, u'\u20ac', u'\u20ac', (3.4,))), ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 12, u'\u20ac', '\xe2\x82\xac', (3.4,)), + (12, u'\u20ac', u'\u20ac', (3.4,))), + ] for arg, res in testcases: self.assertEqual(split(arg), res) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 19:45:42 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Sun, 8 Sep 2013 19:45:42 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogRml4IGEgdHlwby4g?= =?utf-8?b?KGNsb3NlcyAjMTg5NTMp?= Message-ID: <3cY0KL1LMlzSYh@mail.python.org> http://hg.python.org/cpython/rev/c08e92529f62 changeset: 85628:c08e92529f62 branch: 3.3 parent: 85625:03ee22236465 user: Serhiy Storchaka date: Sun Sep 08 20:42:13 2013 +0300 summary: Fix a typo. (closes #18953) files: Misc/NEWS | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -69,7 +69,7 @@ ------- - Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in - the _sre moduel. + the _sre module. - Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 19:45:43 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Sun, 8 Sep 2013 19:45:43 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?b?KTogRml4IGEgdHlwby4gKGNsb3NlcyAjMTg5NTMp?= Message-ID: <3cY0KM3Rkjz7Lkl@mail.python.org> http://hg.python.org/cpython/rev/9dc5bdab157e changeset: 85629:9dc5bdab157e parent: 85626:138e086e187d parent: 85628:c08e92529f62 user: Serhiy Storchaka date: Sun Sep 08 20:43:02 2013 +0300 summary: Fix a typo. (closes #18953) files: Misc/NEWS | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -87,7 +87,7 @@ readline activation code in ``site.py``. - Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in - the _sre moduel. + the _sre module. - Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 8 19:45:44 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Sun, 8 Sep 2013 19:45:44 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogRml4IGEgdHlwby4g?= =?utf-8?b?KGNsb3NlcyAjMTg5NTMp?= Message-ID: <3cY0KN5hfZz7LlL@mail.python.org> http://hg.python.org/cpython/rev/6ecdf5de6252 changeset: 85630:6ecdf5de6252 branch: 2.7 parent: 85627:a22cfd0bdc9a user: Serhiy Storchaka date: Sun Sep 08 20:43:16 2013 +0300 summary: Fix a typo. (closes #18953) files: Misc/NEWS | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,7 +36,7 @@ has a different __name__ than its name in the TestCase class dictionary. - Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in - the _sre moduel. + the _sre module. - Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. -- Repository URL: http://hg.python.org/cpython From root at python.org Sun Sep 8 22:05:23 2013 From: root at python.org (Cron Daemon) Date: Sun, 08 Sep 2013 22:05:23 +0200 Subject: [Python-checkins] Cron /home/docs/build-devguide Message-ID: abort: error: Connection timed out From tjreedy at udel.edu Sun Sep 8 21:16:56 2013 From: tjreedy at udel.edu (Terry Reedy) Date: Sun, 08 Sep 2013 15:16:56 -0400 Subject: [Python-checkins] peps: PEP 454: add clear_filters(), filter() and get_filters() functions In-Reply-To: <3cXtKm47bSz7LjM@mail.python.org> References: <3cXtKm47bSz7LjM@mail.python.org> Message-ID: <522CCD28.9000001@udel.edu> On 9/8/2013 9:15 AM, victor.stinner wrote: > http://hg.python.org/peps/rev/47234db47d7b > changeset: 5102:47234db47d7b > user: Victor Stinner > date: Sun Sep 08 15:15:26 2013 +0200 > summary: > PEP 454: add clear_filters(), filter() and get_filters() functions > > files: > pep-0454.txt | 24 ++++++++++++++++++++++++ > 1 files changed, 24 insertions(+), 0 deletions(-) > > > diff --git a/pep-0454.txt b/pep-0454.txt > --- a/pep-0454.txt > +++ b/pep-0454.txt > @@ -84,6 +84,10 @@ > Functions > --------- > > +``clear_filters()`` function: > + > + Reset the filter list. > + > ``clear_traces()`` function: > > Clear all traces and statistics of memory allocations. > @@ -97,6 +101,26 @@ > > Start tracing Python memory allocations. > > +``filter(include: bool, filename: str, lineno: int=None)`` function: > + > + Add a filter. If *include* is ``True``, only trace memory blocks > + allocated in a file with a name matching *filename*. If > + *include* is ``False``, don't trace memory blocks allocated in a > + file with a name matching *filename*. > + > + The match is done using *filename* as a prefix. For example, > + ``'/usr/bin/'`` only matchs files the ``/usr/bin`` directories. > + > + *lineno* is a line number. If *lineno* is ``None`` or lesser than 'lesser' .> 'less' > + ``1``, it matches any line number. and otherwise (None, > 1)? > + > +``get_filters()`` function: > + > + Get the filters as list of > + ``(include: bool, filename: str, lineno: int)`` tuples. > + > + If *lineno* is ``None``, a filter matchs any line number. This should be moved up to the definition of filter From python-checkins at python.org Mon Sep 9 01:46:45 2013 From: python-checkins at python.org (tim.peters) Date: Mon, 9 Sep 2013 01:46:45 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_18984=3A__Remove_=2E?= =?utf-8?q?=5Fstopped_Event_from_Thread_internals=2E?= Message-ID: <3cY8Kx0Wc6z7LjM@mail.python.org> http://hg.python.org/cpython/rev/aff959a3ba13 changeset: 85631:aff959a3ba13 parent: 85629:9dc5bdab157e user: Tim Peters date: Sun Sep 08 18:44:40 2013 -0500 summary: Issue 18984: Remove ._stopped Event from Thread internals. The fix for issue 18808 left us checking two things to be sure a Thread was done: an Event (._stopped) and a mutex (._tstate_lock). Clumsy & brittle. This patch removes the Event, leaving just a happy lock :-) The bulk of the patch removes two excruciating tests, which were verifying sanity of the internals of the ._stopped Event after a fork. Thanks to Antoine Pitrou for verifying that's the only real value these tests had. One consequence of moving from an Event to a mutex: waiters (threads calling Thread.join()) used to block each on their own unique mutex (internal to the ._stopped event), but now all contend on the same mutex (._tstate_lock). These approaches have different performance characteristics on different platforms. I don't think it matters in this context. files: Lib/test/test_threading.py | 138 +------------------------ Lib/threading.py | 59 +++++----- 2 files changed, 30 insertions(+), 167 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -647,144 +647,8 @@ """ self._run_and_join(script) - def assertScriptHasOutput(self, script, expected_output): - rc, out, err = assert_python_ok("-c", script) - data = out.decode().replace('\r', '') - self.assertEqual(data, expected_output) - - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_4_joining_across_fork_in_worker_thread(self): - # There used to be a possible deadlock when forking from a child - # thread. See http://bugs.python.org/issue6643. - - # The script takes the following steps: - # - The main thread in the parent process starts a new thread and then - # tries to join it. - # - The join operation acquires the Lock inside the thread's _block - # Condition. (See threading.py:Thread.join().) - # - We stub out the acquire method on the condition to force it to wait - # until the child thread forks. (See LOCK ACQUIRED HERE) - # - The child thread forks. (See LOCK HELD and WORKER THREAD FORKS - # HERE) - # - The main thread of the parent process enters Condition.wait(), - # which releases the lock on the child thread. - # - The child process returns. Without the necessary fix, when the - # main thread of the child process (which used to be the child thread - # in the parent process) attempts to exit, it will try to acquire the - # lock in the Thread._block Condition object and hang, because the - # lock was held across the fork. - - script = """if 1: - import os, time, threading - - finish_join = False - start_fork = False - - def worker(): - # Wait until this thread's lock is acquired before forking to - # create the deadlock. - global finish_join - while not start_fork: - time.sleep(0.01) - # LOCK HELD: Main thread holds lock across this call. - childpid = os.fork() - finish_join = True - if childpid != 0: - # Parent process just waits for child. - os.waitpid(childpid, 0) - # Child process should just return. - - w = threading.Thread(target=worker) - - # Stub out the private condition variable's lock acquire method. - # This acquires the lock and then waits until the child has forked - # before returning, which will release the lock soon after. If - # someone else tries to fix this test case by acquiring this lock - # before forking instead of resetting it, the test case will - # deadlock when it shouldn't. - condition = w._stopped._cond - orig_acquire = condition.acquire - call_count_lock = threading.Lock() - call_count = 0 - def my_acquire(): - global call_count - global start_fork - orig_acquire() # LOCK ACQUIRED HERE - start_fork = True - if call_count == 0: - while not finish_join: - time.sleep(0.01) # WORKER THREAD FORKS HERE - with call_count_lock: - call_count += 1 - condition.acquire = my_acquire - - w.start() - w.join() - print('end of main') - """ - self.assertScriptHasOutput(script, "end of main\n") - - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_5_clear_waiter_locks_to_avoid_crash(self): - # Check that a spawned thread that forks doesn't segfault on certain - # platforms, namely OS X. This used to happen if there was a waiter - # lock in the thread's condition variable's waiters list. Even though - # we know the lock will be held across the fork, it is not safe to - # release locks held across forks on all platforms, so releasing the - # waiter lock caused a segfault on OS X. Furthermore, since locks on - # OS X are (as of this writing) implemented with a mutex + condition - # variable instead of a semaphore, while we know that the Python-level - # lock will be acquired, we can't know if the internal mutex will be - # acquired at the time of the fork. - - script = """if True: - import os, time, threading - - start_fork = False - - def worker(): - # Wait until the main thread has attempted to join this thread - # before continuing. - while not start_fork: - time.sleep(0.01) - childpid = os.fork() - if childpid != 0: - # Parent process just waits for child. - (cpid, rc) = os.waitpid(childpid, 0) - assert cpid == childpid - assert rc == 0 - print('end of worker thread') - else: - # Child process should just return. - pass - - w = threading.Thread(target=worker) - - # Stub out the private condition variable's _release_save method. - # This releases the condition's lock and flips the global that - # causes the worker to fork. At this point, the problematic waiter - # lock has been acquired once by the waiter and has been put onto - # the waiters list. - condition = w._stopped._cond - orig_release_save = condition._release_save - def my_release_save(): - global start_fork - orig_release_save() - # Waiter lock held here, condition lock released. - start_fork = True - condition._release_save = my_release_save - - w.start() - w.join() - print('end of main thread') - """ - output = "end of worker thread\nend of main thread\n" - self.assertScriptHasOutput(script, output) - - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_6_daemon_threads(self): + def test_4_daemon_threads(self): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -549,7 +549,7 @@ self._ident = None self._tstate_lock = None self._started = Event() - self._stopped = Event() + self._is_stopped = False self._initialized = True # sys.stderr is not stored in the class like # sys.exc_info since it can be changed between instances @@ -561,7 +561,6 @@ # private! Called by _after_fork() to reset our internal locks as # they may be in an invalid state leading to a deadlock or crash. self._started._reset_internal_locks() - self._stopped._reset_internal_locks() if is_alive: self._set_tstate_lock() else: @@ -574,7 +573,7 @@ status = "initial" if self._started.is_set(): status = "started" - if self._stopped.is_set(): + if self._is_stopped: status = "stopped" if self._daemonic: status += " daemon" @@ -696,7 +695,6 @@ pass finally: with _active_limbo_lock: - self._stop() try: # We don't call self._delete() because it also # grabs _active_limbo_lock. @@ -705,7 +703,8 @@ pass def _stop(self): - self._stopped.set() + self._is_stopped = True + self._tstate_lock = None def _delete(self): "Remove current thread from the dict of currently running threads." @@ -749,29 +748,24 @@ raise RuntimeError("cannot join thread before it is started") if self is current_thread(): raise RuntimeError("cannot join current thread") - if not self.is_alive(): - return - self._stopped.wait(timeout) - if self._stopped.is_set(): - self._wait_for_tstate_lock(timeout is None) + if timeout is None: + self._wait_for_tstate_lock() + else: + self._wait_for_tstate_lock(timeout=timeout) - def _wait_for_tstate_lock(self, block): + def _wait_for_tstate_lock(self, block=True, timeout=-1): # Issue #18808: wait for the thread state to be gone. - # When self._stopped is set, the Python part of the thread is done, - # but the thread's tstate has not yet been destroyed. The C code - # releases self._tstate_lock when the C part of the thread is done - # (the code at the end of the thread's life to remove all knowledge - # of the thread from the C data structures). - # This method waits to acquire _tstate_lock if `block` is True, or - # sees whether it can be acquired immediately if `block` is False. - # If it does acquire the lock, the C code is done, and _tstate_lock - # is set to None. + # At the end of the thread's life, after all knowledge of the thread + # is removed from C data structures, C code releases our _tstate_lock. + # This method passes its arguments to _tstate_lock.aquire(). + # If the lock is acquired, the C code is done, and self._stop() is + # called. That sets ._is_stopped to True, and ._tstate_lock to None. lock = self._tstate_lock - if lock is None: - return # already determined that the C code is done - if lock.acquire(block): + if lock is None: # already determined that the C code is done + assert self._is_stopped + elif lock.acquire(block, timeout): lock.release() - self._tstate_lock = None + self._stop() @property def name(self): @@ -790,14 +784,10 @@ def is_alive(self): assert self._initialized, "Thread.__init__() not called" - if not self._started.is_set(): + if self._is_stopped or not self._started.is_set(): return False - if not self._stopped.is_set(): - return True - # The Python part of the thread is done, but the C part may still be - # waiting to run. self._wait_for_tstate_lock(False) - return self._tstate_lock is not None + return not self._is_stopped isAlive = is_alive @@ -861,6 +851,7 @@ def __init__(self): Thread.__init__(self, name="MainThread", daemon=False) + self._set_tstate_lock() self._started.set() self._set_ident() with _active_limbo_lock: @@ -925,6 +916,14 @@ _main_thread = _MainThread() def _shutdown(): + # Obscure: other threads may be waiting to join _main_thread. That's + # dubious, but some code does it. We can't wait for C code to release + # the main thread's tstate_lock - that won't happen until the interpreter + # is nearly dead. So we release it here. Note that just calling _stop() + # isn't enough: other threads may already be waiting on _tstate_lock. + assert _main_thread._tstate_lock is not None + assert _main_thread._tstate_lock.locked() + _main_thread._tstate_lock.release() _main_thread._stop() t = _pickSomeNonDaemonThread() while t: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 04:11:53 2013 From: python-checkins at python.org (guido.van.rossum) Date: Mon, 9 Sep 2013 04:11:53 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Remove_=40task=3B_add_gather?= =?utf-8?q?=28=29=3B_remote_run=5Funtil=5Fcomplete=28=29_timeout=3B_some_w?= =?utf-8?q?ording?= Message-ID: <3cYCYP4dtRz7LjV@mail.python.org> http://hg.python.org/peps/rev/aabbd1324891 changeset: 5104:aabbd1324891 user: Guido van Rossum date: Sun Sep 08 19:11:55 2013 -0700 summary: Remove @task; add gather(); remote run_until_complete() timeout; some wording cleanup. files: pep-3156.txt | 41 ++++++++++++++++++++++----------------- 1 files changed, 23 insertions(+), 18 deletions(-) diff --git a/pep-3156.txt b/pep-3156.txt --- a/pep-3156.txt +++ b/pep-3156.txt @@ -334,13 +334,10 @@ and in part because there shouldn't be many places where this is called anyway.) -- ``run_until_complete(future, timeout=None)``. Runs the event loop - until the Future is done. If a timeout is given, it waits at most - that long. If the Future is done, its result is returned, or its - exception is raised; if the timeout expires before the Future is - done, or if ``stop()`` is called, ``TimeoutError`` is raised (but - the Future is not cancelled). This cannot be called when the event - loop is already running. +- ``run_until_complete(future)``. Runs the event loop until the + Future is done. If the Future is done, its result is returned, or + its exception is raised. This cannot be called when the event loop + is already running. - ``stop()``. Stops the event loop as soon as it is convenient. It is fine to restart the loop with ``run_forever()`` or @@ -1301,14 +1298,14 @@ ``FIRST_COMPLETED``, ``FIRST_EXCEPTION``, ``ALL_COMPLETED`` are defined with the same values and the same meanings as in PEP 3148: - - ``ALL_COMPLETED`` (default): Wait until all Futures are done or - completed (or until the timeout occurs). + - ``ALL_COMPLETED`` (default): Wait until all Futures are done (or + until the timeout occurs). - - ``FIRST_COMPLETED``: Wait until at least one Future is done or - cancelled (or until the timeout occurs). + - ``FIRST_COMPLETED``: Wait until at least one Future is done (or + until the timeout occurs). - - ``FIRST_EXCEPTION``: Wait until at least one Future is done (not - cancelled) with an exception set. (The exclusion of cancelled + - ``FIRST_EXCEPTION``: Wait until at least one Future is done but + not cancelled with an exception set. (The exclusion of cancelled Futures from the condition is surprising, but PEP 3148 does it this way.) @@ -1335,6 +1332,16 @@ returning the result or raising the exception if it is completed within the timeout, raising ``TimeoutError`` otherwise. +- ``tulip.gather(f1, f2, ...)``. Returns a Future which waits until + all arguments (Futures or coroutines) are done and return a list of + their corresponding results. If one or more of the arguments is + cancelled or raises an exception, the returned Future is cancelled + or has its exception set (matching what happened to the first + argument), and the remaining arguments are left running in the + background. Cancelling the returned Future does not affect the + arguments. Note that coroutine arguments are converted to Futures + using ``tulip.async()``. + Sleeping -------- @@ -1363,11 +1370,9 @@ callback-based frameworks like Twisted. After converting a coroutine into a Task, callbacks can be added to the Task. -There are two ways to convert a coroutine into a task: explicitly, by -calling the coroutine function and then passing the resulting -coroutine object to the ``tulip.Task()`` constructor; or implicitly, -by decorating the coroutine with ``@tulip.task`` (instead of -``@tulip.coroutine``). +To convert a coroutine into a task, call the coroutine function and +pass the resulting coroutine object to the ``tulip.Task()`` +constructor. You may also use ``tulip.async()`` for this purpose. You may ask, why not automatically convert all coroutines to Tasks? The ``@tulip.coroutine`` decorator could do this. However, this would -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Mon Sep 9 06:21:25 2013 From: python-checkins at python.org (guido.van.rossum) Date: Mon, 9 Sep 2013 06:21:25 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_450_update_from_Steven_D?= =?utf-8?q?=27Aprano=2E?= Message-ID: <3cYGQs2S9Xz7Lkb@mail.python.org> http://hg.python.org/peps/rev/4997fe3f926d changeset: 5105:4997fe3f926d user: Guido van Rossum date: Sun Sep 08 21:21:27 2013 -0700 summary: PEP 450 update from Steven D'Aprano. files: pep-0450.txt | 104 ++++++++++++++++++++++++++++++++++++++- 1 files changed, 102 insertions(+), 2 deletions(-) diff --git a/pep-0450.txt b/pep-0450.txt --- a/pep-0450.txt +++ b/pep-0450.txt @@ -15,7 +15,7 @@ This PEP proposes the addition of a module for common statistics functions such as mean, median, variance and standard deviation to the Python - standard library. + standard library. See also http://bugs.python.org/issue18606 Rationale @@ -250,6 +250,102 @@ - But avoid going into tedious[20] mathematical detail. +API + + The initial version of the library will provide univariate (single + variable) statistics functions. The general API will be based on a + functional model ``function(data, ...) -> result``, where ``data`` + is a mandatory iterable of (usually) numeric data. + + The author expects that lists will be the most common data type used, + but any iterable type should be acceptable. Where necessary, functions + may convert to lists internally. Where possible, functions are + expected to conserve the type of the data values, for example, the mean + of a list of Decimals should be a Decimal rather than float. + + + Calculating mean, median and mode + + The ``mean``, ``median*`` and ``mode`` functions take a single + mandatory argument and return the appropriate statistic, e.g.: + + >>> mean([1, 2, 3]) + 2.0 + + Functions provided are: + + * mean(data) -> arithmetic mean of data. + + * median(data) -> median (middle value) of data, taking the + average of the two middle values when there are an even + number of values. + + * median_high(data) -> high median of data, taking the + larger of the two middle values when the number of items + is even. + + * median_low(data) -> low median of data, taking the smaller + of the two middle values when the number of items is even. + + * median_grouped(data, interval=1) -> 50th percentile of + grouped data, using interpolation. + + * mode(data) -> most common data point. + + ``mode`` is the sole exception to the rule that the data argument + must be numeric. It will also accept an iterable of nominal data, + such as strings. + + + Calculating variance and standard deviation + + In order to be similar to scientific calculators, the statistics + module will include separate functions for population and sample + variance and standard deviation. All four functions have similar + signatures, with a single mandatory argument, an iterable of + numeric data, e.g.: + + >>> variance([1, 2, 2, 2, 3]) + 0.5 + + All four functions also accept a second, optional, argument, the + mean of the data. This is modelled on a similar API provided by + the GNU Scientific Library[18]. There are three use-cases for + using this argument, in no particular order: + + 1) The value of the mean is known *a priori*. + + 2) You have already calculated the mean, and wish to avoid + calculating it again. + + 3) You wish to (ab)use the variance functions to calculate + the second moment about some given point other than the + mean. + + In each case, it is the caller's responsibility to ensure that + given argument is meaningful. + + Functions provided are: + + * variance(data, xbar=None) -> sample variance of data, + optionally using xbar as the sample mean. + + * stdev(data, xbar=None) -> sample standard deviation of + data, optionally using xbar as the sample mean. + + * pvariance(data, mu=None) -> population variance of data, + optionally using mu as the population mean. + + * pstdev(data, mu=None) -> population standard deviation of + data, optionally using mu as the population mean. + + Other functions + + There is one other public function: + + * sum(data, start=0) -> high-precision sum of numeric data. + + Specification As the proposed reference implementation is in pure Python, @@ -317,7 +413,7 @@ level somewhere between "use numpy" and "roll your own version". -Open and Deferred Issues +Future Work - At this stage, I am unsure of the best API for multivariate statistical functions such as linear regression, correlation coefficient, and @@ -329,6 +425,8 @@ * A single argument for (x, y) data: function([(x0, y0), (x1, y1), ...]) + This API is preferred by GvR[24]. + * Selecting arbitrary columns from a 2D array: function([[a0, x0, y0, z0], [a1, x1, y1, z1], ...], x=1, y=2) @@ -404,6 +502,8 @@ [23] http://mail.python.org/pipermail/python-ideas/2013-August/022630.html + [24] https://mail.python.org/pipermail/python-dev/2013-September/128429.html + Copyright -- Repository URL: http://hg.python.org/peps From solipsis at pitrou.net Mon Sep 9 06:24:21 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Mon, 09 Sep 2013 06:24:21 +0200 Subject: [Python-checkins] Daily reference leaks (aff959a3ba13): sum=0 Message-ID: results for aff959a3ba13 on branch "default" -------------------------------------------- Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogrs9GaC', '-x'] From python-checkins at python.org Mon Sep 9 08:30:14 2013 From: python-checkins at python.org (raymond.hettinger) Date: Mon, 9 Sep 2013 08:30:14 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_18752=3A__Make_chain?= =?utf-8?q?=2Efrom=5Fiterable=28=29_more_visible_in_the_documentation=2E?= Message-ID: <3cYKHV6DqXz7LkK@mail.python.org> http://hg.python.org/cpython/rev/fa1fa88b685b changeset: 85632:fa1fa88b685b user: Raymond Hettinger date: Mon Sep 09 01:29:40 2013 -0500 summary: Issue 18752: Make chain.from_iterable() more visible in the documentation. files: Doc/library/itertools.rst | 1 + Modules/itertoolsmodule.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -48,6 +48,7 @@ ==================== ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` +chain.from_iterable iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` :func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` :func:`filterfalse` pred, seq elements of seq where pred(elem) is False ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4,7 +4,7 @@ /* Itertools module written and maintained by Raymond D. Hettinger - Copyright (c) 2003 Python Software Foundation. + Copyright (c) 2003-2013 Python Software Foundation. All rights reserved. */ @@ -4456,6 +4456,7 @@ Iterators terminating on the shortest input sequence:\n\ accumulate(p[, func]) --> p0, p0+p1, p0+p1+p2\n\ chain(p, q, ...) --> p0, p1, ... plast, q0, q1, ... \n\ +chain.from_iterable([p, q, ...]) --> p0, p1, ... plast, q0, q1, ... \n\ compress(data, selectors) --> (d[0] if s[0]), (d[1] if s[1]), ...\n\ dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails\n\ groupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v)\n\ -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 08:48:50 2013 From: python-checkins at python.org (ethan.furman) Date: Mon, 9 Sep 2013 08:48:50 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Close_=2318980=3A_Enum_doc?= =?utf-8?q?_fixes=2E__Patch_from_Elazar_Gershuni=2E?= Message-ID: <3cYKhy3JKGz7Lkb@mail.python.org> http://hg.python.org/cpython/rev/0e05b07a6f84 changeset: 85633:0e05b07a6f84 user: Ethan Furman date: Sun Sep 08 23:48:34 2013 -0700 summary: Close #18980: Enum doc fixes. Patch from Elazar Gershuni. files: Doc/library/enum.rst | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -38,7 +38,8 @@ ... blue = 3 ... -..note: Nomenclature +.. note:: Nomenclature + - The class :class:`Color` is an *enumeration* (or *enum*) - The attributes :attr:`Color.red`, :attr:`Color.green`, etc., are *enumeration members* (or *enum members*). @@ -474,7 +475,7 @@ 4. %-style formatting: `%s` and `%r` call :class:`Enum`'s :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for IntEnum) treat the enum member as its mixed-in type. -5. :class:`str`.:meth:`__format__` (or :func:`format`) will use the mixed-in +5. :meth:`str.__format__` (or :func:`format`) will use the mixed-in type's :meth:`__format__`. If the :class:`Enum`'s :func:`str` or :func:`repr` is desired use the `!s` or `!r` :class:`str` format codes. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 08:55:43 2013 From: python-checkins at python.org (raymond.hettinger) Date: Mon, 9 Sep 2013 08:55:43 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgMTgzMDE6?= =?utf-8?q?__The_classmethod_decorator_didn=27t_fit_well_with_the?= Message-ID: <3cYKrv5lP4z7Lkb@mail.python.org> http://hg.python.org/cpython/rev/29fa1f418796 changeset: 85634:29fa1f418796 branch: 3.3 parent: 85628:c08e92529f62 user: Raymond Hettinger date: Mon Sep 09 01:54:27 2013 -0500 summary: Issue 18301: The classmethod decorator didn't fit well with the rough-equivalent example code. files: Doc/library/itertools.rst | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -156,9 +156,8 @@ .. classmethod:: chain.from_iterable(iterable) Alternate constructor for :func:`chain`. Gets chained inputs from a - single iterable argument that is evaluated lazily. Equivalent to:: + single iterable argument that is evaluated lazily. Roughly equivalent to:: - @classmethod def from_iterable(iterables): # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F for it in iterables: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 08:55:45 2013 From: python-checkins at python.org (raymond.hettinger) Date: Mon, 9 Sep 2013 08:55:45 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_merge?= Message-ID: <3cYKrx0MX0z7LlL@mail.python.org> http://hg.python.org/cpython/rev/8f8c614dc284 changeset: 85635:8f8c614dc284 parent: 85632:fa1fa88b685b parent: 85634:29fa1f418796 user: Raymond Hettinger date: Mon Sep 09 01:55:07 2013 -0500 summary: merge files: Doc/library/itertools.rst | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -157,9 +157,8 @@ .. classmethod:: chain.from_iterable(iterable) Alternate constructor for :func:`chain`. Gets chained inputs from a - single iterable argument that is evaluated lazily. Equivalent to:: + single iterable argument that is evaluated lazily. Roughly equivalent to:: - @classmethod def from_iterable(iterables): # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F for it in iterables: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 08:55:46 2013 From: python-checkins at python.org (raymond.hettinger) Date: Mon, 9 Sep 2013 08:55:46 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?= =?utf-8?q?=29=3A_merge?= Message-ID: <3cYKry22VRz7Ll6@mail.python.org> http://hg.python.org/cpython/rev/e235ab6dda5d changeset: 85636:e235ab6dda5d parent: 85635:8f8c614dc284 parent: 85633:0e05b07a6f84 user: Raymond Hettinger date: Mon Sep 09 01:55:31 2013 -0500 summary: merge files: Doc/library/enum.rst | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -38,7 +38,8 @@ ... blue = 3 ... -..note: Nomenclature +.. note:: Nomenclature + - The class :class:`Color` is an *enumeration* (or *enum*) - The attributes :attr:`Color.red`, :attr:`Color.green`, etc., are *enumeration members* (or *enum members*). @@ -474,7 +475,7 @@ 4. %-style formatting: `%s` and `%r` call :class:`Enum`'s :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for IntEnum) treat the enum member as its mixed-in type. -5. :class:`str`.:meth:`__format__` (or :func:`format`) will use the mixed-in +5. :meth:`str.__format__` (or :func:`format`) will use the mixed-in type's :meth:`__format__`. If the :class:`Enum`'s :func:`str` or :func:`repr` is desired use the `!s` or `!r` :class:`str` format codes. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 09:02:20 2013 From: python-checkins at python.org (raymond.hettinger) Date: Mon, 9 Sep 2013 09:02:20 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgMTg3NTI6?= =?utf-8?q?__Make_chain=2Efrom=5Fiterable=28=29_more_visible_in_the_docume?= =?utf-8?q?ntation=2E?= Message-ID: <3cYL0X3jtHz7Ljc@mail.python.org> http://hg.python.org/cpython/rev/23f77dc58979 changeset: 85637:23f77dc58979 branch: 3.3 parent: 85634:29fa1f418796 user: Raymond Hettinger date: Mon Sep 09 02:01:35 2013 -0500 summary: Issue 18752: Make chain.from_iterable() more visible in the documentation. files: Doc/library/itertools.rst | 1 + Modules/itertoolsmodule.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -48,6 +48,7 @@ ==================== ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` +chain.from_iterable iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` :func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` :func:`filterfalse` pred, seq elements of seq where pred(elem) is False ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4,7 +4,7 @@ /* Itertools module written and maintained by Raymond D. Hettinger - Copyright (c) 2003 Python Software Foundation. + Copyright (c) 2003-2013 Python Software Foundation. All rights reserved. */ @@ -4456,6 +4456,7 @@ Iterators terminating on the shortest input sequence:\n\ accumulate(p[, func]) --> p0, p0+p1, p0+p1+p2\n\ chain(p, q, ...) --> p0, p1, ... plast, q0, q1, ... \n\ +chain.from_iterable([p, q, ...]) --> p0, p1, ... plast, q0, q1, ... \n\ compress(data, selectors) --> (d[0] if s[0]), (d[1] if s[1]), ...\n\ dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails\n\ groupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v)\n\ -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 09:02:21 2013 From: python-checkins at python.org (raymond.hettinger) Date: Mon, 9 Sep 2013 09:02:21 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_merge?= Message-ID: <3cYL0Y5Vm9z7LlH@mail.python.org> http://hg.python.org/cpython/rev/6af5a355a931 changeset: 85638:6af5a355a931 parent: 85636:e235ab6dda5d parent: 85637:23f77dc58979 user: Raymond Hettinger date: Mon Sep 09 02:02:05 2013 -0500 summary: merge files: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 11:32:18 2013 From: python-checkins at python.org (mark.dickinson) Date: Mon, 9 Sep 2013 11:32:18 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Docstring_typo?= =?utf-8?q?_fix=3A_Arithmentic_-=3E_Arithmetic=2E?= Message-ID: <3cYPKZ1kL8z7Ljc@mail.python.org> http://hg.python.org/cpython/rev/740bd510a888 changeset: 85639:740bd510a888 branch: 2.7 parent: 85630:6ecdf5de6252 user: Mark Dickinson date: Mon Sep 09 10:32:08 2013 +0100 summary: Docstring typo fix: Arithmentic -> Arithmetic. files: Lib/unittest/__init__.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -11,7 +11,7 @@ import unittest - class IntegerArithmenticTestCase(unittest.TestCase): + class IntegerArithmeticTestCase(unittest.TestCase): def testAdd(self): ## test method names begin 'test*' self.assertEqual((1 + 2), 3) self.assertEqual(0 + 1, 1) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 11:35:19 2013 From: python-checkins at python.org (mark.dickinson) Date: Mon, 9 Sep 2013 11:35:19 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Docstring_typo?= =?utf-8?q?_fix=3A_Arithmentic_-=3E_Arithmetic=2E?= Message-ID: <3cYPP3621Bz7LjS@mail.python.org> http://hg.python.org/cpython/rev/ce29d00470d1 changeset: 85640:ce29d00470d1 branch: 3.3 parent: 85637:23f77dc58979 user: Mark Dickinson date: Mon Sep 09 10:34:24 2013 +0100 summary: Docstring typo fix: Arithmentic -> Arithmetic. files: Lib/unittest/__init__.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -11,7 +11,7 @@ import unittest - class IntegerArithmenticTestCase(unittest.TestCase): + class IntegerArithmeticTestCase(unittest.TestCase): def testAdd(self): ## test method names begin 'test*' self.assertEqual((1 + 2), 3) self.assertEqual(0 + 1, 1) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 11:35:21 2013 From: python-checkins at python.org (mark.dickinson) Date: Mon, 9 Sep 2013 11:35:21 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Merge_docstring_typo_fix_from_3=2E3?= Message-ID: <3cYPP50jcNz7LjS@mail.python.org> http://hg.python.org/cpython/rev/02cdd725a6eb changeset: 85641:02cdd725a6eb parent: 85638:6af5a355a931 parent: 85640:ce29d00470d1 user: Mark Dickinson date: Mon Sep 09 10:34:56 2013 +0100 summary: Merge docstring typo fix from 3.3 files: Lib/unittest/__init__.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -11,7 +11,7 @@ import unittest - class IntegerArithmenticTestCase(unittest.TestCase): + class IntegerArithmeticTestCase(unittest.TestCase): def testAdd(self): ## test method names begin 'test*' self.assertEqual((1 + 2), 3) self.assertEqual(0 + 1, 1) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 14:14:24 2013 From: python-checkins at python.org (larry.hastings) Date: Mon, 9 Sep 2013 14:14:24 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Update_pydoc_topic_index_f?= =?utf-8?q?or_Python_3=2E4=2E0a2=2E?= Message-ID: <3cYSwc2hGBz7LjS@mail.python.org> http://hg.python.org/cpython/rev/b94c73c0eaca changeset: 85642:b94c73c0eaca parent: 85584:98f82b124c7d user: Larry Hastings date: Sat Sep 07 23:41:12 2013 +1200 summary: Update pydoc topic index for Python 3.4.0a2. files: Lib/pydoc_data/topics.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Sat Aug 3 12:46:08 2013 +# Autogenerated by Sphinx on Sat Sep 7 23:40:23 2013 topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, ``assert expression``, is equivalent to\n\n if __debug__:\n if not expression: raise AssertionError\n\nThe extended form, ``assert expression1, expression2``, is equivalent\nto\n\n if __debug__:\n if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that ``__debug__`` and ``AssertionError``\nrefer to the built-in variables with those names. In the current\nimplementation, the built-in variable ``__debug__`` is ``True`` under\nnormal circumstances, ``False`` when optimization is requested\n(command line option -O). The current code generator emits no code\nfor an assert statement when optimization is requested at compile\ntime. Note that it is unnecessary to include the source code for the\nexpression that failed in the error message; it will be displayed as\npart of the stack trace.\n\nAssignments to ``__debug__`` are illegal. The value for the built-in\nvariable is determined when the interpreter starts.\n', 'assignment': '\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n | "*" target\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list, optionally enclosed in\nparentheses or square brackets, is recursively defined as follows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets.\n\n * If the target list contains one target prefixed with an asterisk,\n called a "starred" target: The object must be a sequence with at\n least as many items as there are targets in the target list, minus\n one. The first items of the sequence are assigned, from left to\n right, to the targets before the starred target. The final items\n of the sequence are assigned to the targets after the starred\n target. A list of the remaining items in the sequence is then\n assigned to the starred target (the list can be empty).\n\n * Else: The object must be a sequence with the same number of items\n as there are targets in the target list, and the items are\n assigned, from left to right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` or ``nonlocal``\n statement in the current code block: the name is bound to the\n object in the current local namespace.\n\n * Otherwise: the name is bound to the object in the global namespace\n or the outer namespace determined by ``nonlocal``, respectively.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield an integer. If it is negative, the sequence\'s\n length is added to it. The resulting value must be a nonnegative\n integer less than the sequence\'s length, and the sequence is asked\n to assign the assigned object to its item with that index. If the\n index is out of range, ``IndexError`` is raised (assignment to a\n subscripted sequence cannot add new items to a list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n For user-defined objects, the ``__setitem__()`` method is called\n with appropriate arguments.\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to integers. If either bound is\n negative, the sequence\'s length is added to it. The resulting\n bounds are clipped to lie between zero and the sequence\'s length,\n inclusive. Finally, the sequence object is asked to replace the\n slice with the items of the assigned sequence. The length of the\n slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print(x)\n\nSee also:\n\n **PEP 3132** - Extended Iterable Unpacking\n The specification for the ``*target`` feature.\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', 'atom-identifiers': '\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name, with leading underscores removed and a single underscore\ninserted, in front of the name. For example, the identifier\n``__spam`` occurring in a class named ``Ham`` will be transformed to\n``_Ham__spam``. This transformation is independent of the syntactical\ncontext in which the identifier is used. If the transformed name is\nextremely long (longer than 255 characters), implementation defined\ntruncation may happen. If the class name consists only of underscores,\nno transformation is done.\n', @@ -66,7 +66,7 @@ 'subscriptions': '\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object that supports subscription,\ne.g. a list or dictionary. User-defined objects can support\nsubscription by defining a ``__getitem__()`` method.\n\nFor built-in objects, there are two types of objects that support\nsubscription:\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey. (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to\nan integer or a slice (as discussed in the following section).\n\nThe formal syntax makes no special provision for negative indices in\nsequences; however, built-in sequences all provide a ``__getitem__()``\nmethod that interprets negative indices by adding the length of the\nsequence to the index (so that ``x[-1]`` selects the last item of\n``x``). The resulting value must be a nonnegative integer less than\nthe number of items in the sequence, and the subscription selects the\nitem whose index is that value (counting from zero). Since the support\nfor negative indices and slicing occurs in the object\'s\n``__getitem__()`` method, subclasses overriding this method will need\nto explicitly add that support.\n\nA string\'s items are characters. A character is not a separate data\ntype but a string of exactly one character.\n', 'truth': "\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an ``if`` or\n``while`` condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* ``None``\n\n* ``False``\n\n* zero of any numeric type, for example, ``0``, ``0.0``, ``0j``.\n\n* any empty sequence, for example, ``''``, ``()``, ``[]``.\n\n* any empty mapping, for example, ``{}``.\n\n* instances of user-defined classes, if the class defines a\n ``__bool__()`` or ``__len__()`` method, when that method returns the\n integer zero or ``bool`` value ``False``. [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn ``0`` or ``False`` for false and ``1`` or ``True`` for true,\nunless otherwise stated. (Important exception: the Boolean operations\n``or`` and ``and`` always return one of their operands.)\n", 'try': '\nThe ``try`` statement\n*********************\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression ["as" target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object or a tuple containing an item compatible with the\nexception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the ``as`` keyword in that except clause,\nif present, and the except clause\'s suite is executed. All except\nclauses must have an executable block. When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using ``as target``, it is cleared\nat the end of the except clause. This is as if\n\n except E as N:\n foo\n\nwas translated to\n\n except E as N:\n try:\n foo\n finally:\n del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause. Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the ``sys`` module and can be access via\n``sys.exc_info()``. ``sys.exc_info()`` returns a 3-tuple consisting of\nthe exception class, the exception instance and a traceback object\n(see section *The standard type hierarchy*) identifying the point in\nthe program where the exception occurred. ``sys.exc_info()`` values\nare restored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception it is re-raised at the end of\nthe ``finally`` clause. If the ``finally`` clause raises another\nexception, the saved exception is set as the context of the new\nexception. If the ``finally`` clause executes a ``return`` or\n``break`` statement, the saved exception is discarded:\n\n def f():\n try:\n 1/0\n finally:\n return 42\n\n >>> f()\n 42\n\nThe exception information is not available to the program during\nexecution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n', - 'types': '\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.), although such additions\nwill often be provided via the standard library instead.\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name ``None``.\n It is used to signify the absence of a value in many situations,\n e.g., it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``NotImplemented``. Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the literal ``...`` or the\n built-in name ``Ellipsis``. Its truth value is true.\n\n``numbers.Number``\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n ``numbers.Integral``\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are two types of integers:\n\n Integers (``int``)\n\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans (``bool``)\n These represent the truth values False and True. The two\n objects representing the values False and True are the only\n Boolean objects. The Boolean type is a subtype of the integer\n type, and Boolean values behave like the values 0 and 1,\n respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ``"False"`` or\n ``"True"`` are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers.\n\n ``numbers.Real`` (``float``)\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n ``numbers.Complex`` (``complex``)\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number ``z`` can be retrieved through the read-only\n attributes ``z.real`` and ``z.imag``.\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function ``len()`` returns the number of\n items of a sequence. When the length of a sequence is *n*, the\n index set contains the numbers 0, 1, ..., *n*-1. Item *i* of\n sequence *a* is selected by ``a[i]``.\n\n Sequences also support slicing: ``a[i:j]`` selects all items with\n index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n A string is a sequence of values that represent Unicode\n codepoints. All the codepoints in range ``U+0000 - U+10FFFF``\n can be represented in a string. Python doesn\'t have a\n ``chr`` type, and every character in the string is\n represented as a string object with length ``1``. The built-\n in function ``ord()`` converts a character to its codepoint\n (as an integer); ``chr()`` converts an integer in range ``0 -\n 10FFFF`` to the corresponding character. ``str.encode()`` can\n be used to convert a ``str`` to ``bytes`` using the given\n encoding, and ``bytes.decode()`` can be used to achieve the\n opposite.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Bytes\n A bytes object is an immutable array. The items are 8-bit\n bytes, represented by integers in the range 0 <= x < 256.\n Bytes literals (like ``b\'abc\'``) and the built-in function\n ``bytes()`` can be used to construct bytes objects. Also,\n bytes objects can be decoded to strings via the ``decode()``\n method.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and ``del`` (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in ``bytearray()`` constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module ``array`` provides an additional example of\n a mutable sequence type, as does the ``collections`` module.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function ``len()``\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n ``set()`` constructor and can be modified afterwards by several\n methods, such as ``add()``.\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in ``frozenset()`` constructor. As a frozenset is\n immutable and *hashable*, it can be used again as an element of\n another set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation ``a[k]`` selects the item indexed by\n ``k`` from the mapping ``a``; this can be used in expressions and\n as the target of assignments or ``del`` statements. The built-in\n function ``len()`` returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``) then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the ``{...}``\n notation (see section *Dictionary displays*).\n\n The extension modules ``dbm.ndbm`` and ``dbm.gnu`` provide\n additional examples of mapping types, as does the\n ``collections`` module.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +---------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +===========================+=================================+=============+\n | ``__doc__`` | The function\'s documentation | Writable |\n | | string, or ``None`` if | |\n | | unavailable | |\n +---------------------------+---------------------------------+-------------+\n | ``__name__`` | The function\'s name | Writable |\n +---------------------------+---------------------------------+-------------+\n | ``__qualname__`` | The function\'s *qualified name* | Writable |\n | | New in version 3.3. | |\n +---------------------------+---------------------------------+-------------+\n | ``__module__`` | The name of the module the | Writable |\n | | function was defined in, or | |\n | | ``None`` if unavailable. | |\n +---------------------------+---------------------------------+-------------+\n | ``__defaults__`` | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or ``None`` if no arguments | |\n | | have a default value | |\n +---------------------------+---------------------------------+-------------+\n | ``__code__`` | The code object representing | Writable |\n | | the compiled function body. | |\n +---------------------------+---------------------------------+-------------+\n | ``__globals__`` | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +---------------------------+---------------------------------+-------------+\n | ``__dict__`` | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +---------------------------+---------------------------------+-------------+\n | ``__closure__`` | ``None`` or a tuple of cells | Read-only |\n | | that contain bindings for the | |\n | | function\'s free variables. | |\n +---------------------------+---------------------------------+-------------+\n | ``__annotations__`` | A dict containing annotations | Writable |\n | | of parameters. The keys of the | |\n | | dict are the parameter names, | |\n | | or ``\'return\'`` for the return | |\n | | annotation, if provided. | |\n +---------------------------+---------------------------------+-------------+\n | ``__kwdefaults__`` | A dict containing defaults for | Writable |\n | | keyword-only parameters. | |\n +---------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n Instance methods\n An instance method object combines a class, a class instance and\n any callable object (normally a user-defined function).\n\n Special read-only attributes: ``__self__`` is the class instance\n object, ``__func__`` is the function object; ``__doc__`` is the\n method\'s documentation (same as ``__func__.__doc__``);\n ``__name__`` is the method name (same as ``__func__.__name__``);\n ``__module__`` is the name of the module the method was defined\n in, or ``None`` if unavailable.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object or a class\n method object.\n\n When an instance method object is created by retrieving a user-\n defined function object from a class via one of its instances,\n its ``__self__`` attribute is the instance, and the method\n object is said to be bound. The new method\'s ``__func__``\n attribute is the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the ``__func__``\n attribute of the new instance is not the original method object\n but its ``__func__`` attribute.\n\n When an instance method object is created by retrieving a class\n method object from a class or instance, its ``__self__``\n attribute is the class itself, and its ``__func__`` attribute is\n the function object underlying the class method.\n\n When an instance method object is called, the underlying\n function (``__func__``) is called, inserting the class instance\n (``__self__``) in front of the argument list. For instance,\n when ``C`` is a class which contains a definition for a function\n ``f()``, and ``x`` is an instance of ``C``, calling ``x.f(1)``\n is equivalent to calling ``C.f(x, 1)``.\n\n When an instance method object is derived from a class method\n object, the "class instance" stored in ``__self__`` will\n actually be the class itself, so that calling either ``x.f(1)``\n or ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n the underlying function.\n\n Note that the transformation from function object to instance\n method object happens each time the attribute is retrieved from\n the instance. In some cases, a fruitful optimization is to\n assign the attribute to a local variable and call that local\n variable. Also notice that this transformation only happens for\n user-defined functions; other callable objects (and all non-\n callable objects) are retrieved without transformation. It is\n also important to note that user-defined functions which are\n attributes of a class instance are not converted to bound\n methods; this *only* happens when the function is an attribute\n of the class.\n\n Generator functions\n A function or method which uses the ``yield`` statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s ``iterator__next__()`` method will cause the\n function to execute until it provides a value using the\n ``yield`` statement. When the function executes a ``return``\n statement or falls off the end, a ``StopIteration`` exception is\n raised and the iterator will have reached the end of the set of\n values to be returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are ``len()`` and ``math.sin()``\n (``math`` is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: ``__doc__`` is the function\'s documentation\n string, or ``None`` if unavailable; ``__name__`` is the\n function\'s name; ``__self__`` is set to ``None`` (but see the\n next item); ``__module__`` is the name of the module the\n function was defined in or ``None`` if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n ``alist.append()``, assuming *alist* is a list object. In this\n case, the special read-only attribute ``__self__`` is set to the\n object denoted by *alist*.\n\n Classes\n Classes are callable. These objects normally act as factories\n for new instances of themselves, but variations are possible for\n class types that override ``__new__()``. The arguments of the\n call are passed to ``__new__()`` and, in the typical case, to\n ``__init__()`` to initialize the new instance.\n\n Class Instances\n Instances of arbitrary classes can be made callable by defining\n a ``__call__()`` method in their class.\n\nModules\n Modules are a basic organizational unit of Python code, and are\n created by the *import system* as invoked either by the ``import``\n statement (see ``import``), or by calling functions such as\n ``importlib.import_module()`` and built-in ``__import__()``. A\n module object has a namespace implemented by a dictionary object\n (this is the dictionary referenced by the ``__globals__`` attribute\n of functions defined in the module). Attribute references are\n translated to lookups in this dictionary, e.g., ``m.x`` is\n equivalent to ``m.__dict__["x"]``. A module object does not contain\n the code object used to initialize the module (since it isn\'t\n needed once the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n Special read-only attribute: ``__dict__`` is the module\'s namespace\n as a dictionary object.\n\n **CPython implementation detail:** Because of the way CPython\n clears module dictionaries, the module dictionary will be cleared\n when the module falls out of scope even if the dictionary still has\n live references. To avoid this, copy the dictionary or keep the\n module around while using its dictionary directly.\n\n Predefined (writable) attributes: ``__name__`` is the module\'s\n name; ``__doc__`` is the module\'s documentation string, or ``None``\n if unavailable; ``__file__`` is the pathname of the file from which\n the module was loaded, if it was loaded from a file. The\n ``__file__`` attribute may be missing for certain types of modules,\n such as C modules that are statically linked into the interpreter;\n for extension modules loaded dynamically from a shared library, it\n is the pathname of the shared library file.\n\nCustom classes\n Custom class types are typically created by class definitions (see\n section *Class definitions*). A class has a namespace implemented\n by a dictionary object. Class attribute references are translated\n to lookups in this dictionary, e.g., ``C.x`` is translated to\n ``C.__dict__["x"]`` (although there are a number of hooks which\n allow for other means of locating attributes). When the attribute\n name is not found there, the attribute search continues in the base\n classes. This search of the base classes uses the C3 method\n resolution order which behaves correctly even in the presence of\n \'diamond\' inheritance structures where there are multiple\n inheritance paths leading back to a common ancestor. Additional\n details on the C3 MRO used by Python can be found in the\n documentation accompanying the 2.3 release at\n http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class ``C``, say) would yield\n a class method object, it is transformed into an instance method\n object whose ``__self__`` attributes is ``C``. When it would yield\n a static method object, it is transformed into the object wrapped\n by the static method object. See section *Implementing Descriptors*\n for another way in which attributes retrieved from a class may\n differ from those actually contained in its ``__dict__``.\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: ``__name__`` is the class name; ``__module__``\n is the module name in which the class was defined; ``__dict__`` is\n the dictionary containing the class\'s namespace; ``__bases__`` is a\n tuple (possibly empty or a singleton) containing the base classes,\n in the order of their occurrence in the base class list;\n ``__doc__`` is the class\'s documentation string, or None if\n undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object, it is transformed into an instance method object\n whose ``__self__`` attribute is the instance. Static method and\n class method objects are also transformed; see above under\n "Classes". See section *Implementing Descriptors* for another way\n in which attributes of a class retrieved via its instances may\n differ from the objects actually stored in the class\'s\n ``__dict__``. If no class attribute is found, and the object\'s\n class has a ``__getattr__()`` method, that is called to satisfy the\n lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n ``__setattr__()`` or ``__delattr__()`` method, this is called\n instead of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: ``__dict__`` is the attribute dictionary;\n ``__class__`` is the instance\'s class.\n\nI/O objects (also known as file objects)\n A *file object* represents an open file. Various shortcuts are\n available to create file objects: the ``open()`` built-in function,\n and also ``os.popen()``, ``os.fdopen()``, and the ``makefile()``\n method of socket objects (and perhaps by other functions or methods\n provided by extension modules).\n\n The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are\n initialized to file objects corresponding to the interpreter\'s\n standard input, output and error streams; they are all open in text\n mode and therefore follow the interface defined by the\n ``io.TextIOBase`` abstract class.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: ``co_name`` gives the function\n name; ``co_argcount`` is the number of positional arguments\n (including arguments with default values); ``co_nlocals`` is the\n number of local variables used by the function (including\n arguments); ``co_varnames`` is a tuple containing the names of\n the local variables (starting with the argument names);\n ``co_cellvars`` is a tuple containing the names of local\n variables that are referenced by nested functions;\n ``co_freevars`` is a tuple containing the names of free\n variables; ``co_code`` is a string representing the sequence of\n bytecode instructions; ``co_consts`` is a tuple containing the\n literals used by the bytecode; ``co_names`` is a tuple\n containing the names used by the bytecode; ``co_filename`` is\n the filename from which the code was compiled;\n ``co_firstlineno`` is the first line number of the function;\n ``co_lnotab`` is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); ``co_stacksize`` is the required stack size\n (including local variables); ``co_flags`` is an integer encoding\n a number of flags for the interpreter.\n\n The following flag bits are defined for ``co_flags``: bit\n ``0x04`` is set if the function uses the ``*arguments`` syntax\n to accept an arbitrary number of positional arguments; bit\n ``0x08`` is set if the function uses the ``**keywords`` syntax\n to accept arbitrary keyword arguments; bit ``0x20`` is set if\n the function is a generator.\n\n Future feature declarations (``from __future__ import\n division``) also use bits in ``co_flags`` to indicate whether a\n code object was compiled with a particular feature enabled: bit\n ``0x2000`` is set if the function was compiled with future\n division enabled; bits ``0x10`` and ``0x1000`` were used in\n earlier versions of Python.\n\n Other bits in ``co_flags`` are reserved for internal use.\n\n If a code object represents a function, the first item in\n ``co_consts`` is the documentation string of the function, or\n ``None`` if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: ``f_back`` is to the previous\n stack frame (towards the caller), or ``None`` if this is the\n bottom stack frame; ``f_code`` is the code object being executed\n in this frame; ``f_locals`` is the dictionary used to look up\n local variables; ``f_globals`` is used for global variables;\n ``f_builtins`` is used for built-in (intrinsic) names;\n ``f_lasti`` gives the precise instruction (this is an index into\n the bytecode string of the code object).\n\n Special writable attributes: ``f_trace``, if not ``None``, is a\n function called at the start of each source code line (this is\n used by the debugger); ``f_lineno`` is the current line number\n of the frame --- writing to this from within a trace function\n jumps to the given line (only for the bottom-most frame). A\n debugger can implement a Jump command (aka Set Next Statement)\n by writing to f_lineno.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as the third item of the\n tuple returned by ``sys.exc_info()``. When the program contains\n no suitable handler, the stack trace is written (nicely\n formatted) to the standard error stream; if the interpreter is\n interactive, it is also made available to the user as\n ``sys.last_traceback``.\n\n Special read-only attributes: ``tb_next`` is the next level in\n the stack trace (towards the frame where the exception\n occurred), or ``None`` if there is no next level; ``tb_frame``\n points to the execution frame of the current level;\n ``tb_lineno`` gives the line number where the exception\n occurred; ``tb_lasti`` indicates the precise instruction. The\n line number and last instruction in the traceback may differ\n from the line number of its frame object if the exception\n occurred in a ``try`` statement with no matching except clause\n or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices for ``__getitem__()``\n methods. They are also created by the built-in ``slice()``\n function.\n\n Special read-only attributes: ``start`` is the lower bound;\n ``stop`` is the upper bound; ``step`` is the step value; each is\n ``None`` if omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the slice that the slice object\n would describe if applied to a sequence of *length* items.\n It returns a tuple of three integers; respectively these are\n the *start* and *stop* indices and the *step* or stride\n length of the slice. Missing or out-of-bounds indices are\n handled in a manner consistent with regular slices.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n ``staticmethod()`` constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in ``classmethod()`` constructor.\n', + 'types': '\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.), although such additions\nwill often be provided via the standard library instead.\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name ``None``.\n It is used to signify the absence of a value in many situations,\n e.g., it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``NotImplemented``. Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the literal ``...`` or the\n built-in name ``Ellipsis``. Its truth value is true.\n\n``numbers.Number``\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n ``numbers.Integral``\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are two types of integers:\n\n Integers (``int``)\n\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans (``bool``)\n These represent the truth values False and True. The two\n objects representing the values False and True are the only\n Boolean objects. The Boolean type is a subtype of the integer\n type, and Boolean values behave like the values 0 and 1,\n respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ``"False"`` or\n ``"True"`` are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers.\n\n ``numbers.Real`` (``float``)\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n ``numbers.Complex`` (``complex``)\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number ``z`` can be retrieved through the read-only\n attributes ``z.real`` and ``z.imag``.\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function ``len()`` returns the number of\n items of a sequence. When the length of a sequence is *n*, the\n index set contains the numbers 0, 1, ..., *n*-1. Item *i* of\n sequence *a* is selected by ``a[i]``.\n\n Sequences also support slicing: ``a[i:j]`` selects all items with\n index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n A string is a sequence of values that represent Unicode\n codepoints. All the codepoints in range ``U+0000 - U+10FFFF``\n can be represented in a string. Python doesn\'t have a\n ``chr`` type, and every character in the string is\n represented as a string object with length ``1``. The built-\n in function ``ord()`` converts a character to its codepoint\n (as an integer); ``chr()`` converts an integer in range ``0 -\n 10FFFF`` to the corresponding character. ``str.encode()`` can\n be used to convert a ``str`` to ``bytes`` using the given\n encoding, and ``bytes.decode()`` can be used to achieve the\n opposite.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Bytes\n A bytes object is an immutable array. The items are 8-bit\n bytes, represented by integers in the range 0 <= x < 256.\n Bytes literals (like ``b\'abc\'``) and the built-in function\n ``bytes()`` can be used to construct bytes objects. Also,\n bytes objects can be decoded to strings via the ``decode()``\n method.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and ``del`` (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in ``bytearray()`` constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module ``array`` provides an additional example of\n a mutable sequence type, as does the ``collections`` module.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function ``len()``\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n ``set()`` constructor and can be modified afterwards by several\n methods, such as ``add()``.\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in ``frozenset()`` constructor. As a frozenset is\n immutable and *hashable*, it can be used again as an element of\n another set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation ``a[k]`` selects the item indexed by\n ``k`` from the mapping ``a``; this can be used in expressions and\n as the target of assignments or ``del`` statements. The built-in\n function ``len()`` returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``) then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the ``{...}``\n notation (see section *Dictionary displays*).\n\n The extension modules ``dbm.ndbm`` and ``dbm.gnu`` provide\n additional examples of mapping types, as does the\n ``collections`` module.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +---------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +===========================+=================================+=============+\n | ``__doc__`` | The function\'s documentation | Writable |\n | | string, or ``None`` if | |\n | | unavailable | |\n +---------------------------+---------------------------------+-------------+\n | ``__name__`` | The function\'s name | Writable |\n +---------------------------+---------------------------------+-------------+\n | ``__qualname__`` | The function\'s *qualified name* | Writable |\n | | New in version 3.3. | |\n +---------------------------+---------------------------------+-------------+\n | ``__module__`` | The name of the module the | Writable |\n | | function was defined in, or | |\n | | ``None`` if unavailable. | |\n +---------------------------+---------------------------------+-------------+\n | ``__defaults__`` | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or ``None`` if no arguments | |\n | | have a default value | |\n +---------------------------+---------------------------------+-------------+\n | ``__code__`` | The code object representing | Writable |\n | | the compiled function body. | |\n +---------------------------+---------------------------------+-------------+\n | ``__globals__`` | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +---------------------------+---------------------------------+-------------+\n | ``__dict__`` | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +---------------------------+---------------------------------+-------------+\n | ``__closure__`` | ``None`` or a tuple of cells | Read-only |\n | | that contain bindings for the | |\n | | function\'s free variables. | |\n +---------------------------+---------------------------------+-------------+\n | ``__annotations__`` | A dict containing annotations | Writable |\n | | of parameters. The keys of the | |\n | | dict are the parameter names, | |\n | | or ``\'return\'`` for the return | |\n | | annotation, if provided. | |\n +---------------------------+---------------------------------+-------------+\n | ``__kwdefaults__`` | A dict containing defaults for | Writable |\n | | keyword-only parameters. | |\n +---------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n Instance methods\n An instance method object combines a class, a class instance and\n any callable object (normally a user-defined function).\n\n Special read-only attributes: ``__self__`` is the class instance\n object, ``__func__`` is the function object; ``__doc__`` is the\n method\'s documentation (same as ``__func__.__doc__``);\n ``__name__`` is the method name (same as ``__func__.__name__``);\n ``__module__`` is the name of the module the method was defined\n in, or ``None`` if unavailable.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object or a class\n method object.\n\n When an instance method object is created by retrieving a user-\n defined function object from a class via one of its instances,\n its ``__self__`` attribute is the instance, and the method\n object is said to be bound. The new method\'s ``__func__``\n attribute is the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the ``__func__``\n attribute of the new instance is not the original method object\n but its ``__func__`` attribute.\n\n When an instance method object is created by retrieving a class\n method object from a class or instance, its ``__self__``\n attribute is the class itself, and its ``__func__`` attribute is\n the function object underlying the class method.\n\n When an instance method object is called, the underlying\n function (``__func__``) is called, inserting the class instance\n (``__self__``) in front of the argument list. For instance,\n when ``C`` is a class which contains a definition for a function\n ``f()``, and ``x`` is an instance of ``C``, calling ``x.f(1)``\n is equivalent to calling ``C.f(x, 1)``.\n\n When an instance method object is derived from a class method\n object, the "class instance" stored in ``__self__`` will\n actually be the class itself, so that calling either ``x.f(1)``\n or ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n the underlying function.\n\n Note that the transformation from function object to instance\n method object happens each time the attribute is retrieved from\n the instance. In some cases, a fruitful optimization is to\n assign the attribute to a local variable and call that local\n variable. Also notice that this transformation only happens for\n user-defined functions; other callable objects (and all non-\n callable objects) are retrieved without transformation. It is\n also important to note that user-defined functions which are\n attributes of a class instance are not converted to bound\n methods; this *only* happens when the function is an attribute\n of the class.\n\n Generator functions\n A function or method which uses the ``yield`` statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s ``iterator__next__()`` method will cause the\n function to execute until it provides a value using the\n ``yield`` statement. When the function executes a ``return``\n statement or falls off the end, a ``StopIteration`` exception is\n raised and the iterator will have reached the end of the set of\n values to be returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are ``len()`` and ``math.sin()``\n (``math`` is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: ``__doc__`` is the function\'s documentation\n string, or ``None`` if unavailable; ``__name__`` is the\n function\'s name; ``__self__`` is set to ``None`` (but see the\n next item); ``__module__`` is the name of the module the\n function was defined in or ``None`` if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n ``alist.append()``, assuming *alist* is a list object. In this\n case, the special read-only attribute ``__self__`` is set to the\n object denoted by *alist*.\n\n Classes\n Classes are callable. These objects normally act as factories\n for new instances of themselves, but variations are possible for\n class types that override ``__new__()``. The arguments of the\n call are passed to ``__new__()`` and, in the typical case, to\n ``__init__()`` to initialize the new instance.\n\n Class Instances\n Instances of arbitrary classes can be made callable by defining\n a ``__call__()`` method in their class.\n\nModules\n Modules are a basic organizational unit of Python code, and are\n created by the *import system* as invoked either by the ``import``\n statement (see ``import``), or by calling functions such as\n ``importlib.import_module()`` and built-in ``__import__()``. A\n module object has a namespace implemented by a dictionary object\n (this is the dictionary referenced by the ``__globals__`` attribute\n of functions defined in the module). Attribute references are\n translated to lookups in this dictionary, e.g., ``m.x`` is\n equivalent to ``m.__dict__["x"]``. A module object does not contain\n the code object used to initialize the module (since it isn\'t\n needed once the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n Special read-only attribute: ``__dict__`` is the module\'s namespace\n as a dictionary object.\n\n **CPython implementation detail:** Because of the way CPython\n clears module dictionaries, the module dictionary will be cleared\n when the module falls out of scope even if the dictionary still has\n live references. To avoid this, copy the dictionary or keep the\n module around while using its dictionary directly.\n\n Predefined (writable) attributes: ``__name__`` is the module\'s\n name; ``__doc__`` is the module\'s documentation string, or ``None``\n if unavailable; ``__file__`` is the pathname of the file from which\n the module was loaded, if it was loaded from a file. The\n ``__file__`` attribute may be missing for certain types of modules,\n such as C modules that are statically linked into the interpreter;\n for extension modules loaded dynamically from a shared library, it\n is the pathname of the shared library file.\n\nCustom classes\n Custom class types are typically created by class definitions (see\n section *Class definitions*). A class has a namespace implemented\n by a dictionary object. Class attribute references are translated\n to lookups in this dictionary, e.g., ``C.x`` is translated to\n ``C.__dict__["x"]`` (although there are a number of hooks which\n allow for other means of locating attributes). When the attribute\n name is not found there, the attribute search continues in the base\n classes. This search of the base classes uses the C3 method\n resolution order which behaves correctly even in the presence of\n \'diamond\' inheritance structures where there are multiple\n inheritance paths leading back to a common ancestor. Additional\n details on the C3 MRO used by Python can be found in the\n documentation accompanying the 2.3 release at\n http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class ``C``, say) would yield\n a class method object, it is transformed into an instance method\n object whose ``__self__`` attributes is ``C``. When it would yield\n a static method object, it is transformed into the object wrapped\n by the static method object. See section *Implementing Descriptors*\n for another way in which attributes retrieved from a class may\n differ from those actually contained in its ``__dict__``.\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: ``__name__`` is the class name; ``__module__``\n is the module name in which the class was defined; ``__dict__`` is\n the dictionary containing the class\'s namespace; ``__bases__`` is a\n tuple (possibly empty or a singleton) containing the base classes,\n in the order of their occurrence in the base class list;\n ``__doc__`` is the class\'s documentation string, or None if\n undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object, it is transformed into an instance method object\n whose ``__self__`` attribute is the instance. Static method and\n class method objects are also transformed; see above under\n "Classes". See section *Implementing Descriptors* for another way\n in which attributes of a class retrieved via its instances may\n differ from the objects actually stored in the class\'s\n ``__dict__``. If no class attribute is found, and the object\'s\n class has a ``__getattr__()`` method, that is called to satisfy the\n lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n ``__setattr__()`` or ``__delattr__()`` method, this is called\n instead of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: ``__dict__`` is the attribute dictionary;\n ``__class__`` is the instance\'s class.\n\nI/O objects (also known as file objects)\n A *file object* represents an open file. Various shortcuts are\n available to create file objects: the ``open()`` built-in function,\n and also ``os.popen()``, ``os.fdopen()``, and the ``makefile()``\n method of socket objects (and perhaps by other functions or methods\n provided by extension modules).\n\n The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are\n initialized to file objects corresponding to the interpreter\'s\n standard input, output and error streams; they are all open in text\n mode and therefore follow the interface defined by the\n ``io.TextIOBase`` abstract class.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: ``co_name`` gives the function\n name; ``co_argcount`` is the number of positional arguments\n (including arguments with default values); ``co_nlocals`` is the\n number of local variables used by the function (including\n arguments); ``co_varnames`` is a tuple containing the names of\n the local variables (starting with the argument names);\n ``co_cellvars`` is a tuple containing the names of local\n variables that are referenced by nested functions;\n ``co_freevars`` is a tuple containing the names of free\n variables; ``co_code`` is a string representing the sequence of\n bytecode instructions; ``co_consts`` is a tuple containing the\n literals used by the bytecode; ``co_names`` is a tuple\n containing the names used by the bytecode; ``co_filename`` is\n the filename from which the code was compiled;\n ``co_firstlineno`` is the first line number of the function;\n ``co_lnotab`` is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); ``co_stacksize`` is the required stack size\n (including local variables); ``co_flags`` is an integer encoding\n a number of flags for the interpreter.\n\n The following flag bits are defined for ``co_flags``: bit\n ``0x04`` is set if the function uses the ``*arguments`` syntax\n to accept an arbitrary number of positional arguments; bit\n ``0x08`` is set if the function uses the ``**keywords`` syntax\n to accept arbitrary keyword arguments; bit ``0x20`` is set if\n the function is a generator.\n\n Future feature declarations (``from __future__ import\n division``) also use bits in ``co_flags`` to indicate whether a\n code object was compiled with a particular feature enabled: bit\n ``0x2000`` is set if the function was compiled with future\n division enabled; bits ``0x10`` and ``0x1000`` were used in\n earlier versions of Python.\n\n Other bits in ``co_flags`` are reserved for internal use.\n\n If a code object represents a function, the first item in\n ``co_consts`` is the documentation string of the function, or\n ``None`` if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: ``f_back`` is to the previous\n stack frame (towards the caller), or ``None`` if this is the\n bottom stack frame; ``f_code`` is the code object being executed\n in this frame; ``f_locals`` is the dictionary used to look up\n local variables; ``f_globals`` is used for global variables;\n ``f_builtins`` is used for built-in (intrinsic) names;\n ``f_lasti`` gives the precise instruction (this is an index into\n the bytecode string of the code object).\n\n Special writable attributes: ``f_trace``, if not ``None``, is a\n function called at the start of each source code line (this is\n used by the debugger); ``f_lineno`` is the current line number\n of the frame --- writing to this from within a trace function\n jumps to the given line (only for the bottom-most frame). A\n debugger can implement a Jump command (aka Set Next Statement)\n by writing to f_lineno.\n\n Frame objects support one method:\n\n frame.clear()\n\n This method clears all references to local variables held by\n the frame. Also, if the frame belonged to a generator, the\n generator is finalized. This helps break reference cycles\n involving frame objects (for example when catching an\n exception and storing its traceback for later use).\n\n ``RuntimeError`` is raised if the frame is currently\n executing.\n\n New in version 3.4.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as the third item of the\n tuple returned by ``sys.exc_info()``. When the program contains\n no suitable handler, the stack trace is written (nicely\n formatted) to the standard error stream; if the interpreter is\n interactive, it is also made available to the user as\n ``sys.last_traceback``.\n\n Special read-only attributes: ``tb_next`` is the next level in\n the stack trace (towards the frame where the exception\n occurred), or ``None`` if there is no next level; ``tb_frame``\n points to the execution frame of the current level;\n ``tb_lineno`` gives the line number where the exception\n occurred; ``tb_lasti`` indicates the precise instruction. The\n line number and last instruction in the traceback may differ\n from the line number of its frame object if the exception\n occurred in a ``try`` statement with no matching except clause\n or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices for ``__getitem__()``\n methods. They are also created by the built-in ``slice()``\n function.\n\n Special read-only attributes: ``start`` is the lower bound;\n ``stop`` is the upper bound; ``step`` is the step value; each is\n ``None`` if omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the slice that the slice object\n would describe if applied to a sequence of *length* items.\n It returns a tuple of three integers; respectively these are\n the *start* and *stop* indices and the *step* or stride\n length of the slice. Missing or out-of-bounds indices are\n handled in a manner consistent with regular slices.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n ``staticmethod()`` constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in ``classmethod()`` constructor.\n', 'typesfunctions': '\nFunctions\n*********\n\nFunction objects are created by function definitions. The only\noperation on a function object is to call it: ``func(argument-list)``.\n\nThere are really two flavors of function objects: built-in functions\nand user-defined functions. Both support the same operation (to call\nthe function), but the implementation is different, hence the\ndifferent object types.\n\nSee *Function definitions* for more information.\n', 'typesmapping': '\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built-\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry. (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict(**kwarg)\nclass class dict(mapping, **kwarg)\nclass class dict(iterable, **kwarg)\n\n Return a new dictionary initialized from an optional positional\n argument and a possibly empty set of keyword arguments.\n\n If no positional argument is given, an empty dictionary is created.\n If a positional argument is given and it is a mapping object, a\n dictionary is created with the same key-value pairs as the mapping\n object. Otherwise, the positional argument must be an *iterator*\n object. Each item in the iterable must itself be an iterator with\n exactly two objects. The first object of each item becomes a key\n in the new dictionary, and the second object the corresponding\n value. If a key occurs more than once, the last value for that key\n becomes the corresponding value in the new dictionary.\n\n If keyword arguments are given, the keyword arguments and their\n values are added to the dictionary created from the positional\n argument. If a key being added is already present, the value from\n the keyword argument replaces the value from the positional\n argument.\n\n To illustrate, the following examples all return a dictionary equal\n to ``{"one": 1, "two": 2, "three": 3}``:\n\n >>> a = dict(one=1, two=2, three=3)\n >>> b = {\'one\': 1, \'two\': 2, \'three\': 3}\n >>> c = dict(zip([\'one\', \'two\', \'three\'], [1, 2, 3]))\n >>> d = dict([(\'two\', 2), (\'one\', 1), (\'three\', 3)])\n >>> e = dict({\'three\': 3, \'one\': 1, \'two\': 2})\n >>> a == b == c == d == e\n True\n\n Providing keyword arguments as in the first example only works for\n keys that are valid Python identifiers. Otherwise, any valid keys\n can be used.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a ``KeyError`` if\n *key* is not in the map.\n\n If a subclass of dict defines a method ``__missing__()``, if the\n key *key* is not present, the ``d[key]`` operation calls that\n method with the key *key* as argument. The ``d[key]`` operation\n then returns or raises whatever is returned or raised by the\n ``__missing__(key)`` call if the key is not present. No other\n operations or methods invoke ``__missing__()``. If\n ``__missing__()`` is not defined, ``KeyError`` is raised.\n ``__missing__()`` must be a method; it cannot be an instance\n variable:\n\n >>> class Counter(dict):\n ... def __missing__(self, key):\n ... return 0\n >>> c = Counter()\n >>> c[\'red\']\n 0\n >>> c[\'red\'] += 1\n >>> c[\'red\']\n 1\n\n See ``collections.Counter`` for a complete implementation\n including other methods helpful for accumulating and managing\n tallies.\n\n d[key] = value\n\n Set ``d[key]`` to *value*.\n\n del d[key]\n\n Remove ``d[key]`` from *d*. Raises a ``KeyError`` if *key* is\n not in the map.\n\n key in d\n\n Return ``True`` if *d* has a key *key*, else ``False``.\n\n key not in d\n\n Equivalent to ``not key in d``.\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for ``iter(d.keys())``.\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n classmethod fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n ``fromkeys()`` is a class method that returns a new dictionary.\n *value* defaults to ``None``.\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to ``None``,\n so that this method never raises a ``KeyError``.\n\n items()\n\n Return a new view of the dictionary\'s items (``(key, value)``\n pairs). See the *documentation of view objects*.\n\n keys()\n\n Return a new view of the dictionary\'s keys. See the\n *documentation of view objects*.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a ``KeyError`` is raised.\n\n popitem()\n\n Remove and return an arbitrary ``(key, value)`` pair from the\n dictionary.\n\n ``popitem()`` is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling ``popitem()`` raises a ``KeyError``.\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to ``None``.\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return ``None``.\n\n ``update()`` accepts either another dictionary object or an\n iterable of key/value pairs (as tuples or other iterables of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: ``d.update(red=1,\n blue=2)``.\n\n values()\n\n Return a new view of the dictionary\'s values. See the\n *documentation of view objects*.\n\nSee also:\n\n ``types.MappingProxyType`` can be used to create a read-only view\n of a ``dict``.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.keys()``, ``dict.values()`` and\n``dict.items()`` are *view objects*. They provide a dynamic view on\nthe dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of ``(key, value)``) in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of ``(value, key)`` pairs\n using ``zip()``: ``pairs = zip(d.values(), d.keys())``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.items()]``.\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n Return ``True`` if *x* is in the underlying dictionary\'s keys,\n values or items (in the latter case, *x* should be a ``(key,\n value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that ``(key, value)`` pairs are unique\nand hashable, then the items view is also set-like. (Values views are\nnot treated as set-like since the entries are generally not unique.)\nFor set-like views, all of the operations defined for the abstract\nbase class ``collections.abc.Set`` are available (for example, ``==``,\n``<``, or ``^``).\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.keys()\n >>> values = dishes.values()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n >>> keys ^ {\'sausage\', \'juice\'}\n {\'juice\', \'sausage\', \'bacon\', \'spam\'}\n', 'typesmethods': '\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as ``append()`` on\nlists) and class instance methods. Built-in methods are described\nwith the types that support them.\n\nIf you access a method (a function defined in a class namespace)\nthrough an instance, you get a special object: a *bound method* (also\ncalled *instance method*) object. When called, it will add the\n``self`` argument to the argument list. Bound methods have two\nspecial read-only attributes: ``m.__self__`` is the object on which\nthe method operates, and ``m.__func__`` is the function implementing\nthe method. Calling ``m(arg-1, arg-2, ..., arg-n)`` is completely\nequivalent to calling ``m.__func__(m.__self__, arg-1, arg-2, ...,\narg-n)``.\n\nLike function objects, bound method objects support getting arbitrary\nattributes. However, since method attributes are actually stored on\nthe underlying function object (``meth.__func__``), setting method\nattributes on bound methods is disallowed. Attempting to set an\nattribute on a method results in an ``AttributeError`` being raised.\nIn order to set a method attribute, you need to explicitly set it on\nthe underlying function object:\n\n >>> class C:\n ... def method(self):\n ... pass\n ...\n >>> c = C()\n >>> c.method.whoami = \'my name is method\' # can\'t set on the method\n Traceback (most recent call last):\n File "", line 1, in \n AttributeError: \'method\' object has no attribute \'whoami\'\n >>> c.method.__func__.whoami = \'my name is method\'\n >>> c.method.whoami\n \'my name is method\'\n\nSee *The standard type hierarchy* for more information.\n', -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 14:14:25 2013 From: python-checkins at python.org (larry.hastings) Date: Mon, 9 Sep 2013 14:14:25 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Version_number_bump_for_Py?= =?utf-8?q?thon_3=2E4=2E0a2=2E?= Message-ID: <3cYSwd4PtFz7Ljl@mail.python.org> http://hg.python.org/cpython/rev/9265a2168e2c changeset: 85643:9265a2168e2c tag: v3.4.0a2 user: Larry Hastings date: Sat Sep 07 23:42:07 2013 +1200 summary: Version number bump for Python 3.4.0a2. files: Include/patchlevel.h | 4 ++-- Lib/distutils/__init__.py | 2 +- Lib/idlelib/idlever.py | 2 +- Misc/RPM/python-3.4.spec | 2 +- README | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 4 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.4.0a1+" +#define PY_VERSION "3.4.0a2" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py --- a/Lib/distutils/__init__.py +++ b/Lib/distutils/__init__.py @@ -13,5 +13,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "3.4.0a1" +__version__ = "3.4.0a2" #--end constants-- diff --git a/Lib/idlelib/idlever.py b/Lib/idlelib/idlever.py --- a/Lib/idlelib/idlever.py +++ b/Lib/idlelib/idlever.py @@ -1,1 +1,1 @@ -IDLE_VERSION = "3.4.0a1" +IDLE_VERSION = "3.4.0a2" diff --git a/Misc/RPM/python-3.4.spec b/Misc/RPM/python-3.4.spec --- a/Misc/RPM/python-3.4.spec +++ b/Misc/RPM/python-3.4.spec @@ -39,7 +39,7 @@ %define name python #--start constants-- -%define version 3.4.0a1 +%define version 3.4.0a2 %define libvers 3.4 #--end constants-- %define release 1pydotorg diff --git a/README b/README --- a/README +++ b/README @@ -1,4 +1,4 @@ -This is Python version 3.4.0 alpha 1 +This is Python version 3.4.0 alpha 2 ==================================== Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 14:14:26 2013 From: python-checkins at python.org (larry.hastings) Date: Mon, 9 Sep 2013 14:14:26 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Added_tag_v3=2E4=2E0a2_for?= =?utf-8?q?_changeset_9265a2168e2c?= Message-ID: <3cYSwf66zKz7Lk9@mail.python.org> http://hg.python.org/cpython/rev/6bd266c1b28c changeset: 85644:6bd266c1b28c user: Larry Hastings date: Sat Sep 07 23:42:24 2013 +1200 summary: Added tag v3.4.0a2 for changeset 9265a2168e2c files: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -116,3 +116,4 @@ d9893d13c6289aa03d33559ec67f97dcbf5c9e3c v3.3.1 d047928ae3f6314a13b6137051315453d0ae89b6 v3.3.2 46535f65e7f3bcdcf176f36d34bc1fed719ffd2b v3.4.0a1 +9265a2168e2cb2a84785d8717792acc661e6b692 v3.4.0a2 -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 14:14:28 2013 From: python-checkins at python.org (larry.hastings) Date: Mon, 9 Sep 2013 14:14:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Post-3=2E4=2E0a2-release_f?= =?utf-8?q?ixups=2E?= Message-ID: <3cYSwh0rHdz7Lkm@mail.python.org> http://hg.python.org/cpython/rev/6b211a0c8042 changeset: 85645:6b211a0c8042 user: Larry Hastings date: Mon Sep 09 21:08:52 2013 +0900 summary: Post-3.4.0a2-release fixups. files: Include/patchlevel.h | 2 +- Misc/NEWS | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.4.0a2" +#define PY_VERSION "3.4.0a2+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,10 +2,22 @@ Python News +++++++++++ +What's New in Python 3.4.0 Alpha 3? +=================================== + +Projected Release date: 2013-09-29 + +Core and Builtins +----------------- + +Library +------- + + What's New in Python 3.4.0 Alpha 2? =================================== -Projected Release date: 2013-09-08 +Release date: 2013-09-09 Core and Builtins ----------------- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 14:14:30 2013 From: python-checkins at python.org (larry.hastings) Date: Mon, 9 Sep 2013 14:14:30 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?= =?utf-8?b?KTogTWVyZ2Uu?= Message-ID: <3cYSwk1Q6Pz7Ll2@mail.python.org> http://hg.python.org/cpython/rev/1cfe82916d98 changeset: 85646:1cfe82916d98 parent: 85645:6b211a0c8042 parent: 85641:02cdd725a6eb user: Larry Hastings date: Mon Sep 09 21:12:21 2013 +0900 summary: Merge. files: Doc/library/enum.rst | 5 +- Doc/library/itertools.rst | 4 +- Doc/library/os.rst | 2 - Doc/library/profile.rst | 8 +- Doc/library/test.rst | 5 +- Doc/library/unittest.rst | 5 +- Doc/using/cmdline.rst | 16 +- Include/pystate.h | 26 + Include/setobject.h | 1 - Lib/_dummy_thread.py | 4 + Lib/decimal.py | 2 +- Lib/multiprocessing/connection.py | 10 +- Lib/test/support/__init__.py | 31 +- Lib/test/test_faulthandler.py | 30 +- Lib/test/test_os.py | 35 +- Lib/test/test_regrtest.py | 3 + Lib/test/test_selectors.py | 3 +- Lib/test/test_site.py | 26 +- Lib/test/test_socket.py | 31 + Lib/test/test_tcl.py | 22 +- Lib/test/test_threading.py | 206 +++------ Lib/threading.py | 95 ++- Lib/unittest/__init__.py | 2 +- Misc/NEWS | 20 +- Modules/_multiprocessing/multiprocessing.c | 6 +- Modules/_threadmodule.c | 62 ++ Modules/faulthandler.c | 5 +- Modules/itertoolsmodule.c | 3 +- Objects/object.c | 1 - Objects/setobject.c | 217 ++++----- Python/pystate.c | 5 + 31 files changed, 528 insertions(+), 363 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -38,7 +38,8 @@ ... blue = 3 ... -..note: Nomenclature +.. note:: Nomenclature + - The class :class:`Color` is an *enumeration* (or *enum*) - The attributes :attr:`Color.red`, :attr:`Color.green`, etc., are *enumeration members* (or *enum members*). @@ -474,7 +475,7 @@ 4. %-style formatting: `%s` and `%r` call :class:`Enum`'s :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for IntEnum) treat the enum member as its mixed-in type. -5. :class:`str`.:meth:`__format__` (or :func:`format`) will use the mixed-in +5. :meth:`str.__format__` (or :func:`format`) will use the mixed-in type's :meth:`__format__`. If the :class:`Enum`'s :func:`str` or :func:`repr` is desired use the `!s` or `!r` :class:`str` format codes. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -48,6 +48,7 @@ ==================== ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` +chain.from_iterable iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` :func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` :func:`filterfalse` pred, seq elements of seq where pred(elem) is False ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` @@ -156,9 +157,8 @@ .. classmethod:: chain.from_iterable(iterable) Alternate constructor for :func:`chain`. Gets chained inputs from a - single iterable argument that is evaluated lazily. Equivalent to:: + single iterable argument that is evaluated lazily. Roughly equivalent to:: - @classmethod def from_iterable(iterables): # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F for it in iterables: diff --git a/Doc/library/os.rst b/Doc/library/os.rst --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -757,8 +757,6 @@ As of Python 3.3, this is equivalent to ``os.pathconf(fd, name)``. - Availability: Unix. - .. function:: fstat(fd) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -247,11 +247,13 @@ import cProfile, pstats, io pr = cProfile.Profile() pr.enable() - ... do something ... + # ... do something ... pr.disable() s = io.StringIO() - ps = pstats.Stats(pr, stream=s) - ps.print_results() + sortby = 'cumulative' + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats() + print(s.getvalue()) .. method:: enable() diff --git a/Doc/library/test.rst b/Doc/library/test.rst --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -263,12 +263,15 @@ Used when tests are executed by :mod:`test.regrtest`. -.. function:: findfile(filename) +.. function:: findfile(filename, subdir=None) Return the path to the file named *filename*. If no match is found *filename* is returned. This does not equal a failure since it could be the path to the file. + Setting *subdir* indicates a relative path to use to find the file + rather than looking directly in the path directories. + .. function:: run_unittest(\*classes) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1674,8 +1674,7 @@ A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.fail\*` or - :meth:`TestCase.assert\*` methods. + was explicitly signalled using the :meth:`TestCase.assert\*` methods. .. attribute:: skipped @@ -1772,7 +1771,7 @@ .. method:: addError(test, err) - Called when the test case *test* raises an unexpected exception *err* is a + Called when the test case *test* raises an unexpected exception. *err* is a tuple of the form returned by :func:`sys.exc_info`: ``(type, value, traceback)``. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -511,9 +511,9 @@ .. envvar:: PYTHONDONTWRITEBYTECODE - If this is set, Python won't try to write ``.pyc`` or ``.pyo`` files on the - import of source modules. This is equivalent to specifying the :option:`-B` - option. + If this is set to a non-empty string, Python won't try to write ``.pyc`` or + ``.pyo`` files on the import of source modules. This is equivalent to + specifying the :option:`-B` option. .. envvar:: PYTHONHASHSEED @@ -582,11 +582,11 @@ .. envvar:: PYTHONFAULTHANDLER - If this environment variable is set, :func:`faulthandler.enable` is called - at startup: install a handler for :const:`SIGSEGV`, :const:`SIGFPE`, - :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` signals to dump the - Python traceback. This is equivalent to :option:`-X` ``faulthandler`` - option. + If this environment variable is set to a non-empty string, + :func:`faulthandler.enable` is called at startup: install a handler for + :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and + :const:`SIGILL` signals to dump the Python traceback. This is equivalent to + :option:`-X` ``faulthandler`` option. .. versionadded:: 3.3 diff --git a/Include/pystate.h b/Include/pystate.h --- a/Include/pystate.h +++ b/Include/pystate.h @@ -118,6 +118,32 @@ int trash_delete_nesting; PyObject *trash_delete_later; + /* Called when a thread state is deleted normally, but not when it + * is destroyed after fork(). + * Pain: to prevent rare but fatal shutdown errors (issue 18808), + * Thread.join() must wait for the join'ed thread's tstate to be unlinked + * from the tstate chain. That happens at the end of a thread's life, + * in pystate.c. + * The obvious way doesn't quite work: create a lock which the tstate + * unlinking code releases, and have Thread.join() wait to acquire that + * lock. The problem is that we _are_ at the end of the thread's life: + * if the thread holds the last reference to the lock, decref'ing the + * lock will delete the lock, and that may trigger arbitrary Python code + * if there's a weakref, with a callback, to the lock. But by this time + * _PyThreadState_Current is already NULL, so only the simplest of C code + * can be allowed to run (in particular it must not be possible to + * release the GIL). + * So instead of holding the lock directly, the tstate holds a weakref to + * the lock: that's the value of on_delete_data below. Decref'ing a + * weakref is harmless. + * on_delete points to _threadmodule.c's static release_sentinel() function. + * After the tstate is unlinked, release_sentinel is called with the + * weakref-to-lock (on_delete_data) argument, and release_sentinel releases + * the indirectly held lock. + */ + void (*on_delete)(void *); + void *on_delete_data; + /* XXX signal handlers should also be here */ } PyThreadState; diff --git a/Include/setobject.h b/Include/setobject.h --- a/Include/setobject.h +++ b/Include/setobject.h @@ -105,7 +105,6 @@ PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); PyAPI_FUNC(int) PySet_ClearFreeList(void); -PyAPI_FUNC(void) _PySet_DebugMallocStats(FILE *out); #endif #ifdef __cplusplus diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py --- a/Lib/_dummy_thread.py +++ b/Lib/_dummy_thread.py @@ -81,6 +81,10 @@ raise error("setting thread stack size not supported") return 0 +def _set_sentinel(): + """Dummy implementation of _thread._set_sentinel().""" + return LockType() + class LockType(object): """Class implementing dummy implementation of _thread.LockType. diff --git a/Lib/decimal.py b/Lib/decimal.py --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -21,7 +21,7 @@ and IEEE standard 854-1987: - www.cs.berkeley.edu/~ejr/projects/754/private/drafts/854-1987/dir.html + http://en.wikipedia.org/wiki/IEEE_854-1987 Decimal floating point has finite precision with arbitrarily large bounds. diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -878,13 +878,21 @@ import selectors + # poll/select have the advantage of not requiring any extra file + # descriptor, contrarily to epoll/kqueue (also, they require a single + # syscall). + if hasattr(selectors, 'PollSelector'): + _WaitSelector = selectors.PollSelector + else: + _WaitSelector = selectors.SelectSelector + def wait(object_list, timeout=None): ''' Wait till an object in object_list is ready/readable. Returns list of those objects in object_list which are ready/readable. ''' - with selectors.DefaultSelector() as selector: + with _WaitSelector() as selector: for obj in object_list: selector.register(obj, selectors.EVENT_READ) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -860,24 +860,31 @@ finally: os.umask(oldmask) -# TEST_HOME refers to the top level directory of the "test" package +# TEST_HOME_DIR refers to the top level directory of the "test" package # that contains Python's regression test suite -TEST_HOME = os.path.dirname(os.path.abspath(__file__)) +TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR) -def findfile(file, here=TEST_HOME, subdir=None): +# TEST_DATA_DIR is used as a target download location for remote resources +TEST_DATA_DIR = os.path.join(TEST_HOME_DIR, "data") + +def findfile(filename, subdir=None): """Try to find a file on sys.path or in the test directory. If it is not found the argument passed to the function is returned (this does not - necessarily signal failure; could still be the legitimate path).""" - if os.path.isabs(file): - return file + necessarily signal failure; could still be the legitimate path). + + Setting *subdir* indicates a relative path to use to find the file + rather than looking directly in the path directories. + """ + if os.path.isabs(filename): + return filename if subdir is not None: - file = os.path.join(subdir, file) - path = sys.path - path = [os.path.dirname(here)] + path + filename = os.path.join(subdir, filename) + path = [TEST_HOME_DIR] + sys.path for dn in path: - fn = os.path.join(dn, file) + fn = os.path.join(dn, filename) if os.path.exists(fn): return fn - return file + return filename def create_empty_file(filename): """Create an empty file. If the file already exists, truncate it.""" @@ -914,7 +921,7 @@ filename = urllib.parse.urlparse(url)[2].split('/')[-1] # '/': it's URL! - fn = os.path.join(os.path.dirname(__file__), "data", filename) + fn = os.path.join(TEST_DATA_DIR, filename) def check_valid_file(fn): f = open(fn, *args, **kw) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -265,17 +265,33 @@ # By default, the module should be disabled code = "import faulthandler; print(faulthandler.is_enabled())" args = (sys.executable, '-E', '-c', code) - # use subprocess module directly because test.script_helper adds - # "-X faulthandler" to the command line - stdout = subprocess.check_output(args) - self.assertEqual(stdout.rstrip(), b"False") + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args) + self.assertEqual(output.rstrip(), b"False") def test_sys_xoptions(self): # Test python -X faulthandler code = "import faulthandler; print(faulthandler.is_enabled())" - rc, stdout, stderr = assert_python_ok("-X", "faulthandler", "-c", code) - stdout = (stdout + stderr).strip() - self.assertEqual(stdout, b"True") + args = (sys.executable, "-E", "-X", "faulthandler", "-c", code) + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args) + self.assertEqual(output.rstrip(), b"True") + + def test_env_var(self): + # empty env var + code = "import faulthandler; print(faulthandler.is_enabled())" + args = (sys.executable, "-c", code) + env = os.environ.copy() + env['PYTHONFAULTHANDLER'] = '' + # don't use assert_python_ok() because it always enable faulthandler + output = subprocess.check_output(args, env=env) + self.assertEqual(output.rstrip(), b"False") + + # non-empty env var + env = os.environ.copy() + env['PYTHONFAULTHANDLER'] = '1' + output = subprocess.check_output(args, env=env) + self.assertEqual(output.rstrip(), b"True") def check_dump_traceback(self, filename): """ diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -34,6 +34,10 @@ import resource except ImportError: resource = None +try: + import fcntl +except ImportError: + fcntl = None from test.script_helper import assert_python_ok @@ -2300,18 +2304,37 @@ class FDInheritanceTests(unittest.TestCase): - def test_get_inheritable(self): + def test_get_set_inheritable(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) - for inheritable in (False, True): - os.set_inheritable(fd, inheritable) - self.assertEqual(os.get_inheritable(fd), inheritable) + self.assertEqual(os.get_inheritable(fd), False) - def test_set_inheritable(self): + os.set_inheritable(fd, True) + self.assertEqual(os.get_inheritable(fd), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(os.get_inheritable(fd), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + os.set_inheritable(fd, True) - self.assertEqual(os.get_inheritable(fd), True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) def test_open(self): fd = os.open(__file__, os.O_RDONLY) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -3,6 +3,7 @@ """ import argparse +import faulthandler import getopt import os.path import unittest @@ -25,6 +26,8 @@ regrtest._parse_args([opt]) self.assertIn('Run Python regression tests.', out.getvalue()) + @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), + "faulthandler.dump_traceback_later() required") def test_timeout(self): ns = regrtest._parse_args(['--timeout', '4.2']) self.assertEqual(ns.timeout, 4.2) diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -301,6 +301,7 @@ class ScalableSelectorMixIn: + # see issue #18963 for why it's skipped on older OS X versions @support.requires_mac_ver(10, 5) @unittest.skipUnless(resource, "Test needs resource module") def test_above_fd_setsize(self): @@ -313,7 +314,7 @@ self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, (soft, hard)) NUM_FDS = hard - except OSError: + except (OSError, ValueError): NUM_FDS = soft # guard for already allocated FDs (stdin, stdout...) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -5,6 +5,7 @@ """ import unittest +import test.support from test.support import run_unittest, TESTFN, EnvironmentVarGuard from test.support import captured_stderr import builtins @@ -373,9 +374,10 @@ self.assertTrue(hasattr(builtins, "exit")) def test_setting_copyright(self): - # 'copyright' and 'credits' should be in builtins + # 'copyright', 'credits', and 'license' should be in builtins self.assertTrue(hasattr(builtins, "copyright")) self.assertTrue(hasattr(builtins, "credits")) + self.assertTrue(hasattr(builtins, "license")) def test_setting_help(self): # 'help' should be set in builtins @@ -402,5 +404,27 @@ self.fail("sitecustomize not imported automatically") +class LicenseURL(unittest.TestCase): + """Test accessibility of the license.""" + + @unittest.skipUnless(str(license).startswith('See http://'), + 'license is available as a file') + def test_license_page(self): + """urlopen should return the license page""" + pat = r'^See (http://www\.python\.org/download/releases/[^/]+/license/)$' + mo = re.search(pat, str(license)) + self.assertIsNotNone(mo, msg='can\'t find appropriate url in license') + if mo is not None: + url = mo.group(1) + with test.support.transient_internet(url): + import urllib.request, urllib.error + try: + with urllib.request.urlopen(url) as data: + code = data.getcode() + except urllib.error.HTTPError as e: + code = e.code + self.assertEqual(code, 200, msg=url) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -26,6 +26,10 @@ import multiprocessing except ImportError: multiprocessing = False +try: + import fcntl +except ImportError: + fcntl = None HOST = support.HOST MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return @@ -4804,6 +4808,33 @@ sock.set_inheritable(False) self.assertEqual(sock.get_inheritable(), False) + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(sock.get_inheritable(), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(sock.get_inheritable(), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + sock.set_inheritable(True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + + @unittest.skipUnless(hasattr(socket, "socketpair"), "need socket.socketpair()") def test_socketpair(self): diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -15,6 +15,14 @@ from tkinter import Tcl from _tkinter import TclError +tcl_version = _tkinter.TCL_VERSION.split('.') +try: + for i in range(len(tcl_version)): + tcl_version[i] = int(tcl_version[i]) +except ValueError: + pass +tcl_version = tuple(tcl_version) + class TkinterTest(unittest.TestCase): @@ -200,9 +208,12 @@ (('a', 3.4), ('a', 3.4)), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), - (call('dict', 'create', 1, '\u20ac', b'\xe2\x82\xac', (3.4,)), - (1, '\u20ac', '\u20ac', (3.4,))), ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 1, '\u20ac', b'\xe2\x82\xac', (3.4,)), + (1, '\u20ac', '\u20ac', (3.4,))), + ] for arg, res in testcases: self.assertEqual(splitlist(arg), res, msg=arg) self.assertRaises(TclError, splitlist, '{') @@ -234,9 +245,12 @@ (('a', (2, 3.4)), ('a', (2, 3.4))), ((), ()), (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), - (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), - (12, '\u20ac', '\u20ac', (3.4,))), ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), + (12, '\u20ac', '\u20ac', (3.4,))), + ] for arg, res in testcases: self.assertEqual(split(arg), res, msg=arg) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -109,7 +109,7 @@ if verbose: print('waiting for all tasks to complete') for t in threads: - t.join(NUMTASKS) + t.join() self.assertTrue(not t.is_alive()) self.assertNotEqual(t.ident, 0) self.assertFalse(t.ident is None) @@ -539,6 +539,40 @@ self.assertEqual(err, b"") self.assertEqual(data, "Thread-1\nTrue\nTrue\n") + def test_tstate_lock(self): + # Test an implementation detail of Thread objects. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + time.sleep(0.01) + # The tstate lock is None until the thread is started + t = threading.Thread(target=f) + self.assertIs(t._tstate_lock, None) + t.start() + started.acquire() + self.assertTrue(t.is_alive()) + # The tstate lock can't be acquired when the thread is running + # (or suspended). + tstate_lock = t._tstate_lock + self.assertFalse(tstate_lock.acquire(timeout=0), False) + finish.release() + # When the thread ends, the state_lock can be successfully + # acquired. + self.assertTrue(tstate_lock.acquire(timeout=5), False) + # But is_alive() is still True: we hold _tstate_lock now, which + # prevents is_alive() from knowing the thread's end-of-life C code + # is done. + self.assertTrue(t.is_alive()) + # Let is_alive() find out the C code is done. + tstate_lock.release() + self.assertFalse(t.is_alive()) + # And verify the thread disposed of _tstate_lock. + self.assertTrue(t._tstate_lock is None) + class ThreadJoinOnShutdown(BaseTestCase): @@ -613,144 +647,8 @@ """ self._run_and_join(script) - def assertScriptHasOutput(self, script, expected_output): - rc, out, err = assert_python_ok("-c", script) - data = out.decode().replace('\r', '') - self.assertEqual(data, expected_output) - - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_4_joining_across_fork_in_worker_thread(self): - # There used to be a possible deadlock when forking from a child - # thread. See http://bugs.python.org/issue6643. - - # The script takes the following steps: - # - The main thread in the parent process starts a new thread and then - # tries to join it. - # - The join operation acquires the Lock inside the thread's _block - # Condition. (See threading.py:Thread.join().) - # - We stub out the acquire method on the condition to force it to wait - # until the child thread forks. (See LOCK ACQUIRED HERE) - # - The child thread forks. (See LOCK HELD and WORKER THREAD FORKS - # HERE) - # - The main thread of the parent process enters Condition.wait(), - # which releases the lock on the child thread. - # - The child process returns. Without the necessary fix, when the - # main thread of the child process (which used to be the child thread - # in the parent process) attempts to exit, it will try to acquire the - # lock in the Thread._block Condition object and hang, because the - # lock was held across the fork. - - script = """if 1: - import os, time, threading - - finish_join = False - start_fork = False - - def worker(): - # Wait until this thread's lock is acquired before forking to - # create the deadlock. - global finish_join - while not start_fork: - time.sleep(0.01) - # LOCK HELD: Main thread holds lock across this call. - childpid = os.fork() - finish_join = True - if childpid != 0: - # Parent process just waits for child. - os.waitpid(childpid, 0) - # Child process should just return. - - w = threading.Thread(target=worker) - - # Stub out the private condition variable's lock acquire method. - # This acquires the lock and then waits until the child has forked - # before returning, which will release the lock soon after. If - # someone else tries to fix this test case by acquiring this lock - # before forking instead of resetting it, the test case will - # deadlock when it shouldn't. - condition = w._block - orig_acquire = condition.acquire - call_count_lock = threading.Lock() - call_count = 0 - def my_acquire(): - global call_count - global start_fork - orig_acquire() # LOCK ACQUIRED HERE - start_fork = True - if call_count == 0: - while not finish_join: - time.sleep(0.01) # WORKER THREAD FORKS HERE - with call_count_lock: - call_count += 1 - condition.acquire = my_acquire - - w.start() - w.join() - print('end of main') - """ - self.assertScriptHasOutput(script, "end of main\n") - - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_5_clear_waiter_locks_to_avoid_crash(self): - # Check that a spawned thread that forks doesn't segfault on certain - # platforms, namely OS X. This used to happen if there was a waiter - # lock in the thread's condition variable's waiters list. Even though - # we know the lock will be held across the fork, it is not safe to - # release locks held across forks on all platforms, so releasing the - # waiter lock caused a segfault on OS X. Furthermore, since locks on - # OS X are (as of this writing) implemented with a mutex + condition - # variable instead of a semaphore, while we know that the Python-level - # lock will be acquired, we can't know if the internal mutex will be - # acquired at the time of the fork. - - script = """if True: - import os, time, threading - - start_fork = False - - def worker(): - # Wait until the main thread has attempted to join this thread - # before continuing. - while not start_fork: - time.sleep(0.01) - childpid = os.fork() - if childpid != 0: - # Parent process just waits for child. - (cpid, rc) = os.waitpid(childpid, 0) - assert cpid == childpid - assert rc == 0 - print('end of worker thread') - else: - # Child process should just return. - pass - - w = threading.Thread(target=worker) - - # Stub out the private condition variable's _release_save method. - # This releases the condition's lock and flips the global that - # causes the worker to fork. At this point, the problematic waiter - # lock has been acquired once by the waiter and has been put onto - # the waiters list. - condition = w._block - orig_release_save = condition._release_save - def my_release_save(): - global start_fork - orig_release_save() - # Waiter lock held here, condition lock released. - start_fork = True - condition._release_save = my_release_save - - w.start() - w.join() - print('end of main thread') - """ - output = "end of worker thread\nend of main thread\n" - self.assertScriptHasOutput(script, output) - - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_6_daemon_threads(self): + def test_4_daemon_threads(self): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. @@ -867,6 +765,38 @@ # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") + def test_threads_join_2(self): + # Same as above, but a delay gets introduced after the thread's + # Python code returned but before the thread state is deleted. + # To achieve this, we register a thread-local object which sleeps + # a bit when deallocated. + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + class Sleeper: + def __del__(self): + time.sleep(0.05) + + tls = threading.local() + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + tls.x = Sleeper() + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = _testcapi.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + def test_daemon_threads_fatal_error(self): subinterp_code = r"""if 1: import os diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -33,6 +33,7 @@ # Rename some stuff so "from threading import *" is safe _start_new_thread = _thread.start_new_thread _allocate_lock = _thread.allocate_lock +_set_sentinel = _thread._set_sentinel get_ident = _thread.get_ident ThreadError = _thread.error try: @@ -516,8 +517,6 @@ _active_limbo_lock = _allocate_lock() _active = {} # maps thread id to Thread object _limbo = {} - -# For debug and leak testing _dangling = WeakSet() # Main class for threads @@ -548,28 +547,33 @@ else: self._daemonic = current_thread().daemon self._ident = None + self._tstate_lock = None self._started = Event() - self._stopped = False - self._block = Condition(Lock()) + self._is_stopped = False self._initialized = True # sys.stderr is not stored in the class like # sys.exc_info since it can be changed between instances self._stderr = _sys.stderr + # For debugging and _after_fork() _dangling.add(self) - def _reset_internal_locks(self): + def _reset_internal_locks(self, is_alive): # private! Called by _after_fork() to reset our internal locks as # they may be in an invalid state leading to a deadlock or crash. - if hasattr(self, '_block'): # DummyThread deletes _block - self._block.__init__() self._started._reset_internal_locks() + if is_alive: + self._set_tstate_lock() + else: + # The thread isn't alive after fork: it doesn't have a tstate + # anymore. + self._tstate_lock = None def __repr__(self): assert self._initialized, "Thread.__init__() was not called" status = "initial" if self._started.is_set(): status = "started" - if self._stopped: + if self._is_stopped: status = "stopped" if self._daemonic: status += " daemon" @@ -625,9 +629,18 @@ def _set_ident(self): self._ident = get_ident() + def _set_tstate_lock(self): + """ + Set a lock object which will be released by the interpreter when + the underlying thread state (see pystate.h) gets deleted. + """ + self._tstate_lock = _set_sentinel() + self._tstate_lock.acquire() + def _bootstrap_inner(self): try: self._set_ident() + self._set_tstate_lock() self._started.set() with _active_limbo_lock: _active[self._ident] = self @@ -682,7 +695,6 @@ pass finally: with _active_limbo_lock: - self._stop() try: # We don't call self._delete() because it also # grabs _active_limbo_lock. @@ -691,10 +703,8 @@ pass def _stop(self): - self._block.acquire() - self._stopped = True - self._block.notify_all() - self._block.release() + self._is_stopped = True + self._tstate_lock = None def _delete(self): "Remove current thread from the dict of currently running threads." @@ -738,21 +748,24 @@ raise RuntimeError("cannot join thread before it is started") if self is current_thread(): raise RuntimeError("cannot join current thread") + if timeout is None: + self._wait_for_tstate_lock() + else: + self._wait_for_tstate_lock(timeout=timeout) - self._block.acquire() - try: - if timeout is None: - while not self._stopped: - self._block.wait() - else: - deadline = _time() + timeout - while not self._stopped: - delay = deadline - _time() - if delay <= 0: - break - self._block.wait(delay) - finally: - self._block.release() + def _wait_for_tstate_lock(self, block=True, timeout=-1): + # Issue #18808: wait for the thread state to be gone. + # At the end of the thread's life, after all knowledge of the thread + # is removed from C data structures, C code releases our _tstate_lock. + # This method passes its arguments to _tstate_lock.aquire(). + # If the lock is acquired, the C code is done, and self._stop() is + # called. That sets ._is_stopped to True, and ._tstate_lock to None. + lock = self._tstate_lock + if lock is None: # already determined that the C code is done + assert self._is_stopped + elif lock.acquire(block, timeout): + lock.release() + self._stop() @property def name(self): @@ -771,7 +784,10 @@ def is_alive(self): assert self._initialized, "Thread.__init__() not called" - return self._started.is_set() and not self._stopped + if self._is_stopped or not self._started.is_set(): + return False + self._wait_for_tstate_lock(False) + return not self._is_stopped isAlive = is_alive @@ -835,6 +851,7 @@ def __init__(self): Thread.__init__(self, name="MainThread", daemon=False) + self._set_tstate_lock() self._started.set() self._set_ident() with _active_limbo_lock: @@ -854,11 +871,6 @@ def __init__(self): Thread.__init__(self, name=_newname("Dummy-%d"), daemon=True) - # Thread._block consumes an OS-level locking primitive, which - # can never be used by a _DummyThread. Since a _DummyThread - # instance is immortal, that's bad, so release this resource. - del self._block - self._started.set() self._set_ident() with _active_limbo_lock: @@ -904,6 +916,14 @@ _main_thread = _MainThread() def _shutdown(): + # Obscure: other threads may be waiting to join _main_thread. That's + # dubious, but some code does it. We can't wait for C code to release + # the main thread's tstate_lock - that won't happen until the interpreter + # is nearly dead. So we release it here. Note that just calling _stop() + # isn't enough: other threads may already be waiting on _tstate_lock. + assert _main_thread._tstate_lock is not None + assert _main_thread._tstate_lock.locked() + _main_thread._tstate_lock.release() _main_thread._stop() t = _pickSomeNonDaemonThread() while t: @@ -949,18 +969,23 @@ current = current_thread() _main_thread = current with _active_limbo_lock: - for thread in _enumerate(): + # Dangling thread instances must still have their locks reset, + # because someone may join() them. + threads = set(_enumerate()) + threads.update(_dangling) + for thread in threads: # Any lock/condition variable may be currently locked or in an # invalid state, so we reinitialize them. - thread._reset_internal_locks() if thread is current: # There is only one active thread. We reset the ident to # its new value since it can have changed. + thread._reset_internal_locks(True) ident = get_ident() thread._ident = ident new_active[ident] = thread else: # All the others are already stopped. + thread._reset_internal_locks(False) thread._stop() _limbo.clear() diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -11,7 +11,7 @@ import unittest - class IntegerArithmenticTestCase(unittest.TestCase): + class IntegerArithmeticTestCase(unittest.TestCase): def testAdd(self): ## test method names begin 'test*' self.assertEqual((1 + 2), 3) self.assertEqual(0 + 1, 1) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,9 +2,6 @@ Python News +++++++++++ -What's New in Python 3.4.0 Alpha 3? -=================================== - Projected Release date: 2013-09-29 Core and Builtins @@ -13,6 +10,17 @@ Library ------- +- The :envvar:`PYTHONFAULTHANDLER` environment variable now only enables the + faulthandler module if the variable is non-empty. Same behaviour than other + variables like :envvar:`PYTHONDONTWRITEBYTECODE`. + +Tests +----- + +- Issue #18952: Fix regression in support data downloads introduced when + test.support was converted to a package. Regression noticed by Zachary + Ware. + What's New in Python 3.4.0 Alpha 2? =================================== @@ -68,6 +76,10 @@ Library ------- +- Issue #18808: Thread.join() now waits for the underlying thread state to + be destroyed before returning. This prevents unpredictable aborts in + Py_EndInterpreter() when some non-daemon threads are still running. + - Issue #18458: Prevent crashes with newer versions of libedit. Its readline emulation has changed from 0-based indexing to 1-based like gnu readline. @@ -75,7 +87,7 @@ readline activation code in ``site.py``. - Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in - the _sre moduel. + the _sre module. - Issue #18830: inspect.getclasstree() no more produces duplicated entries even when input list contains duplicates. diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -99,13 +99,15 @@ { HANDLE handle; Py_buffer buf; - int ret; + int ret, length; if (!PyArg_ParseTuple(args, F_HANDLE "y*:send" , &handle, &buf)) return NULL; + length = (int)Py_MIN(buf.len, INT_MAX); + Py_BEGIN_ALLOW_THREADS - ret = send((SOCKET) handle, buf.buf, buf.len, 0); + ret = send((SOCKET) handle, buf.buf, length, 0); Py_END_ALLOW_THREADS PyBuffer_Release(&buf); diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1172,6 +1172,66 @@ This function is meant for internal and specialized purposes only.\n\ In most applications `threading.enumerate()` should be used instead."); +static void +release_sentinel(void *wr) +{ + /* Tricky: this function is called when the current thread state + is being deleted. Therefore, only simple C code can safely + execute here. */ + PyObject *obj = PyWeakref_GET_OBJECT(wr); + lockobject *lock; + if (obj != Py_None) { + assert(Py_TYPE(obj) == &Locktype); + lock = (lockobject *) obj; + if (lock->locked) { + PyThread_release_lock(lock->lock_lock); + lock->locked = 0; + } + } + /* Deallocating a weakref with a NULL callback only calls + PyObject_GC_Del(), which can't call any Python code. */ + Py_DECREF(wr); +} + +static PyObject * +thread__set_sentinel(PyObject *self) +{ + PyObject *wr; + PyThreadState *tstate = PyThreadState_Get(); + lockobject *lock; + + if (tstate->on_delete_data != NULL) { + /* We must support the re-creation of the lock from a + fork()ed child. */ + assert(tstate->on_delete == &release_sentinel); + wr = (PyObject *) tstate->on_delete_data; + tstate->on_delete = NULL; + tstate->on_delete_data = NULL; + Py_DECREF(wr); + } + lock = newlockobject(); + if (lock == NULL) + return NULL; + /* The lock is owned by whoever called _set_sentinel(), but the weakref + hangs to the thread state. */ + wr = PyWeakref_NewRef((PyObject *) lock, NULL); + if (wr == NULL) { + Py_DECREF(lock); + return NULL; + } + tstate->on_delete_data = (void *) wr; + tstate->on_delete = &release_sentinel; + return (PyObject *) lock; +} + +PyDoc_STRVAR(_set_sentinel_doc, +"_set_sentinel() -> lock\n\ +\n\ +Set a sentinel lock that will be released when the current thread\n\ +state is finalized (after it is untied from the interpreter).\n\ +\n\ +This is a private API for the threading module."); + static PyObject * thread_stack_size(PyObject *self, PyObject *args) { @@ -1247,6 +1307,8 @@ METH_NOARGS, _count_doc}, {"stack_size", (PyCFunction)thread_stack_size, METH_VARARGS, stack_size_doc}, + {"_set_sentinel", (PyCFunction)thread__set_sentinel, + METH_NOARGS, _set_sentinel_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1048,8 +1048,11 @@ { PyObject *xoptions, *key, *module, *res; _Py_IDENTIFIER(enable); + char *p; - if (!Py_GETENV("PYTHONFAULTHANDLER")) { + if (!((p = Py_GETENV("PYTHONFAULTHANDLER")) && *p != '\0')) { + /* PYTHONFAULTHANDLER environment variable is missing + or an empty string */ int has_key; xoptions = PySys_GetXOptions(); diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4,7 +4,7 @@ /* Itertools module written and maintained by Raymond D. Hettinger - Copyright (c) 2003 Python Software Foundation. + Copyright (c) 2003-2013 Python Software Foundation. All rights reserved. */ @@ -4456,6 +4456,7 @@ Iterators terminating on the shortest input sequence:\n\ accumulate(p[, func]) --> p0, p0+p1, p0+p1+p2\n\ chain(p, q, ...) --> p0, p1, ... plast, q0, q1, ... \n\ +chain.from_iterable([p, q, ...]) --> p0, p1, ... plast, q0, q1, ... \n\ compress(data, selectors) --> (d[0] if s[0]), (d[1] if s[1]), ...\n\ dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails\n\ groupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v)\n\ diff --git a/Objects/object.c b/Objects/object.c --- a/Objects/object.c +++ b/Objects/object.c @@ -1955,7 +1955,6 @@ _PyFrame_DebugMallocStats(out); _PyList_DebugMallocStats(out); _PyMethod_DebugMallocStats(out); - _PySet_DebugMallocStats(out); _PyTuple_DebugMallocStats(out); } diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1,22 +1,36 @@ /* set object implementation + Written and maintained by Raymond D. Hettinger Derived from Lib/sets.py and Objects/dictobject.c. Copyright (c) 2003-2013 Python Software Foundation. All rights reserved. + + The basic lookup function used by all operations. + This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. + + The initial probe index is computed as hash mod the table size. + Subsequent probe indices are computed as explained in Objects/dictobject.c. + + To improve cache locality, each probe inspects a series of consecutive + nearby entries before moving on to probes elsewhere in memory. This leaves + us with a hybrid of linear probing and open addressing. The linear probing + reduces the cost of hash collisions because consecutive memory accesses + tend to be much cheaper than scattered probes. After LINEAR_PROBES steps, + we then use open addressing with the upper bits from the hash value. This + helps break-up long chains of collisions. + + All arithmetic on hash should ignore overflow. + + Unlike the dictionary implementation, the lookkey functions can return + NULL if the rich comparison returns an error. */ #include "Python.h" #include "structmember.h" #include "stringlib/eq.h" -/* This must be >= 1 */ -#define PERTURB_SHIFT 5 - -/* This should be >= PySet_MINSIZE - 1 */ -#define LINEAR_PROBES 9 - /* Object used as dummy key to fill deleted entries */ static PyObject _dummy_struct; @@ -25,46 +39,15 @@ /* Exported for the gdb plugin's benefit. */ PyObject *_PySet_Dummy = dummy; -#define INIT_NONZERO_SET_SLOTS(so) do { \ - (so)->table = (so)->smalltable; \ - (so)->mask = PySet_MINSIZE - 1; \ - (so)->hash = -1; \ - } while(0) -#define EMPTY_TO_MINSIZE(so) do { \ - memset((so)->smalltable, 0, sizeof((so)->smalltable)); \ - (so)->used = (so)->fill = 0; \ - INIT_NONZERO_SET_SLOTS(so); \ - } while(0) +/* ======================================================================== */ +/* ======= Begin logic for probing the hash table ========================= */ -/* Reuse scheme to save calls to malloc, free, and memset */ -#ifndef PySet_MAXFREELIST -#define PySet_MAXFREELIST 80 -#endif -static PySetObject *free_list[PySet_MAXFREELIST]; -static int numfree = 0; +/* This should be >= PySet_MINSIZE - 1 */ +#define LINEAR_PROBES 9 - -/* -The basic lookup function used by all operations. -This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. - -The initial probe index is computed as hash mod the table size. -Subsequent probe indices are computed as explained in Objects/dictobject.c. - -To improve cache locality, each probe inspects a series of consecutive -nearby entries before moving on to probes elsewhere in memory. This leaves -us with a hybrid of linear probing and open addressing. The linear probing -reduces the cost of hash collisions because consecutive memory accesses -tend to be much cheaper than scattered probes. After LINEAR_PROBES steps, -we then use open addressing with the upper bits from the hash value. This -helps break-up long chains of collisions. - -All arithmetic on hash should ignore overflow. - -Unlike the dictionary implementation, the lookkey functions can return -NULL if the rich comparison returns an error. -*/ +/* This must be >= 1 */ +#define PERTURB_SHIFT 5 static setentry * set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) @@ -168,8 +151,8 @@ while (1) { if (entry->key == key || (entry->hash == hash - && entry->key != dummy - && unicode_eq(entry->key, key))) + && entry->key != dummy + && unicode_eq(entry->key, key))) return entry; if (entry->key == dummy && freeslot == NULL) freeslot = entry; @@ -200,38 +183,6 @@ } /* -Internal routine to insert a new key into the table. -Used by the public insert routine. -Eats a reference to key. -*/ -static int -set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) -{ - setentry *entry; - - assert(so->lookup != NULL); - entry = so->lookup(so, key, hash); - if (entry == NULL) - return -1; - if (entry->key == NULL) { - /* UNUSED */ - so->fill++; - entry->key = key; - entry->hash = hash; - so->used++; - } else if (entry->key == dummy) { - /* DUMMY */ - entry->key = key; - entry->hash = hash; - so->used++; - } else { - /* ACTIVE */ - Py_DECREF(key); - } - return 0; -} - -/* Internal routine used by set_table_resize() to insert an item which is known to be absent from the set. This routine also assumes that the set contains no deleted entries. Besides the performance benefit, @@ -268,6 +219,42 @@ so->used++; } +/* ======== End logic for probing the hash table ========================== */ +/* ======================================================================== */ + + +/* +Internal routine to insert a new key into the table. +Used by the public insert routine. +Eats a reference to key. +*/ +static int +set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash) +{ + setentry *entry; + + assert(so->lookup != NULL); + entry = so->lookup(so, key, hash); + if (entry == NULL) + return -1; + if (entry->key == NULL) { + /* UNUSED */ + so->fill++; + entry->key = key; + entry->hash = hash; + so->used++; + } else if (entry->key == dummy) { + /* DUMMY */ + entry->key = key; + entry->hash = hash; + so->used++; + } else { + /* ACTIVE */ + Py_DECREF(key); + } + return 0; +} + /* Restructure the table by allocating a new table and reinserting all keys again. When entries have been deleted, the new table may @@ -441,6 +428,17 @@ return DISCARD_FOUND; } +static void +set_empty_to_minsize(PySetObject *so) +{ + memset(so->smalltable, 0, sizeof(so->smalltable)); + so->fill = 0; + so->used = 0; + so->mask = PySet_MINSIZE - 1; + so->table = so->smalltable; + so->hash = -1; +} + static int set_clear_internal(PySetObject *so) { @@ -448,14 +446,13 @@ int table_is_malloced; Py_ssize_t fill; setentry small_copy[PySet_MINSIZE]; + #ifdef Py_DEBUG - Py_ssize_t i, n; - assert (PyAnySet_Check(so)); - - n = so->mask + 1; - i = 0; + Py_ssize_t i = 0; + Py_ssize_t n = so->mask + 1; #endif + assert (PyAnySet_Check(so)); table = so->table; assert(table != NULL); table_is_malloced = table != so->smalltable; @@ -468,7 +465,7 @@ */ fill = so->fill; if (table_is_malloced) - EMPTY_TO_MINSIZE(so); + set_empty_to_minsize(so); else if (fill > 0) { /* It's a small table with something that needs to be cleared. @@ -477,7 +474,7 @@ */ memcpy(small_copy, table, sizeof(small_copy)); table = small_copy; - EMPTY_TO_MINSIZE(so); + set_empty_to_minsize(so); } /* else it's a small table that's already empty */ @@ -560,10 +557,7 @@ } if (so->table != so->smalltable) PyMem_DEL(so->table); - if (numfree < PySet_MAXFREELIST && PyAnySet_CheckExact(so)) - free_list[numfree++] = so; - else - Py_TYPE(so)->tp_free(so); + Py_TYPE(so)->tp_free(so); Py_TRASHCAN_SAFE_END(so) } @@ -1018,24 +1012,16 @@ PySetObject *so = NULL; /* create PySetObject structure */ - if (numfree && - (type == &PySet_Type || type == &PyFrozenSet_Type)) { - so = free_list[--numfree]; - assert (so != NULL && PyAnySet_CheckExact(so)); - Py_TYPE(so) = type; - _Py_NewReference((PyObject *)so); - EMPTY_TO_MINSIZE(so); - PyObject_GC_Track(so); - } else { - so = (PySetObject *)type->tp_alloc(type, 0); - if (so == NULL) - return NULL; - /* tp_alloc has already zeroed the structure */ - assert(so->table == NULL && so->fill == 0 && so->used == 0); - INIT_NONZERO_SET_SLOTS(so); - } + so = (PySetObject *)type->tp_alloc(type, 0); + if (so == NULL) + return NULL; + so->fill = 0; + so->used = 0; + so->mask = PySet_MINSIZE - 1; + so->table = so->smalltable; so->lookup = set_lookkey_unicode; + so->hash = -1; so->weakreflist = NULL; if (iterable != NULL) { @@ -1098,34 +1084,15 @@ int PySet_ClearFreeList(void) { - int freelist_size = numfree; - PySetObject *so; - - while (numfree) { - numfree--; - so = free_list[numfree]; - PyObject_GC_Del(so); - } - return freelist_size; + return 0; } void PySet_Fini(void) { - PySet_ClearFreeList(); Py_CLEAR(emptyfrozenset); } -/* Print summary info about the state of the optimized allocator */ -void -_PySet_DebugMallocStats(FILE *out) -{ - _PyDebugAllocatorStats(out, - "free PySetObject", - numfree, sizeof(PySetObject)); -} - - static PyObject * set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -2398,7 +2365,7 @@ Py_ssize_t count; char *s; Py_ssize_t i; - PyObject *elem=NULL, *dup=NULL, *t, *f, *dup2, *x; + PyObject *elem=NULL, *dup=NULL, *t, *f, *dup2, *x=NULL; PyObject *ob = (PyObject *)so; Py_hash_t hash; PyObject *str; diff --git a/Python/pystate.c b/Python/pystate.c --- a/Python/pystate.c +++ b/Python/pystate.c @@ -208,6 +208,8 @@ tstate->trash_delete_nesting = 0; tstate->trash_delete_later = NULL; + tstate->on_delete = NULL; + tstate->on_delete_data = NULL; if (init) _PyThreadState_Init(tstate); @@ -390,6 +392,9 @@ if (tstate->next) tstate->next->prev = tstate->prev; HEAD_UNLOCK(); + if (tstate->on_delete != NULL) { + tstate->on_delete(tstate->on_delete_data); + } PyMem_RawFree(tstate); } -- Repository URL: http://hg.python.org/cpython From ncoghlan at gmail.com Mon Sep 9 16:20:54 2013 From: ncoghlan at gmail.com (Nick Coghlan) Date: Tue, 10 Sep 2013 00:20:54 +1000 Subject: [Python-checkins] cpython: Post-3.4.0a2-release fixups. In-Reply-To: <3cYSwh0rHdz7Lkm@mail.python.org> References: <3cYSwh0rHdz7Lkm@mail.python.org> Message-ID: On 9 Sep 2013 22:15, "larry.hastings" wrote: > > http://hg.python.org/cpython/rev/6b211a0c8042 > changeset: 85645:6b211a0c8042 > user: Larry Hastings > date: Mon Sep 09 21:08:52 2013 +0900 > summary: > Post-3.4.0a2-release fixups. > > files: > Include/patchlevel.h | 2 +- > Misc/NEWS | 14 +++++++++++++- > 2 files changed, 14 insertions(+), 2 deletions(-) > > > diff --git a/Include/patchlevel.h b/Include/patchlevel.h > --- a/Include/patchlevel.h > +++ b/Include/patchlevel.h > @@ -23,7 +23,7 @@ > #define PY_RELEASE_SERIAL 2 > > /* Version as a string */ > -#define PY_VERSION "3.4.0a2" > +#define PY_VERSION "3.4.0a2+" > /*--end constants--*/ > > /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. > diff --git a/Misc/NEWS b/Misc/NEWS > --- a/Misc/NEWS > +++ b/Misc/NEWS > @@ -2,10 +2,22 @@ > Python News > +++++++++++ > > +What's New in Python 3.4.0 Alpha 3? > +=================================== > + > +Projected Release date: 2013-09-29 > + > +Core and Builtins > +----------------- > + > +Library > +------- > + > + I had already pushed alpha 3 entries in NEWS, so something seems to have gone wrong here. Perhaps, if RMs are preparing the release out of tree, we could get the NEWS file headings on default updated immediately after the last included commit? Cheers, Nick. > What's New in Python 3.4.0 Alpha 2? > =================================== > > -Projected Release date: 2013-09-08 > +Release date: 2013-09-09 > > Core and Builtins > ----------------- > > -- > Repository URL: http://hg.python.org/cpython > > _______________________________________________ > Python-checkins mailing list > Python-checkins at python.org > https://mail.python.org/mailman/listinfo/python-checkins > -------------- next part -------------- An HTML attachment was scrubbed... URL: From benjamin at python.org Mon Sep 9 17:32:20 2013 From: benjamin at python.org (Benjamin Peterson) Date: Mon, 9 Sep 2013 11:32:20 -0400 Subject: [Python-checkins] [Python-Dev] cpython: Post-3.4.0a2-release fixups. In-Reply-To: References: <3cYSwh0rHdz7Lkm@mail.python.org> Message-ID: Well, it's important for the release manager to make sure what the script is doing is sane. :) 2013/9/9 Nick Coghlan : > > On 9 Sep 2013 22:15, "larry.hastings" wrote: >> >> http://hg.python.org/cpython/rev/6b211a0c8042 >> changeset: 85645:6b211a0c8042 >> user: Larry Hastings >> date: Mon Sep 09 21:08:52 2013 +0900 >> summary: >> Post-3.4.0a2-release fixups. >> >> files: >> Include/patchlevel.h | 2 +- >> Misc/NEWS | 14 +++++++++++++- >> 2 files changed, 14 insertions(+), 2 deletions(-) >> >> >> diff --git a/Include/patchlevel.h b/Include/patchlevel.h >> --- a/Include/patchlevel.h >> +++ b/Include/patchlevel.h >> @@ -23,7 +23,7 @@ >> #define PY_RELEASE_SERIAL 2 >> >> /* Version as a string */ >> -#define PY_VERSION "3.4.0a2" >> +#define PY_VERSION "3.4.0a2+" >> /*--end constants--*/ >> >> /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. >> diff --git a/Misc/NEWS b/Misc/NEWS >> --- a/Misc/NEWS >> +++ b/Misc/NEWS >> @@ -2,10 +2,22 @@ >> Python News >> +++++++++++ >> >> +What's New in Python 3.4.0 Alpha 3? >> +=================================== >> + >> +Projected Release date: 2013-09-29 >> + >> +Core and Builtins >> +----------------- >> + >> +Library >> +------- >> + >> + > > I had already pushed alpha 3 entries in NEWS, so something seems to have > gone wrong here. > > Perhaps, if RMs are preparing the release out of tree, we could get the NEWS > file headings on default updated immediately after the last included commit? > > Cheers, > Nick. > >> What's New in Python 3.4.0 Alpha 2? >> =================================== >> >> -Projected Release date: 2013-09-08 >> +Release date: 2013-09-09 >> >> Core and Builtins >> ----------------- >> >> -- >> Repository URL: http://hg.python.org/cpython >> >> _______________________________________________ >> Python-checkins mailing list >> Python-checkins at python.org >> https://mail.python.org/mailman/listinfo/python-checkins >> > > > _______________________________________________ > Python-Dev mailing list > Python-Dev at python.org > https://mail.python.org/mailman/listinfo/python-dev > Unsubscribe: > https://mail.python.org/mailman/options/python-dev/benjamin%40python.org > -- Regards, Benjamin From ncoghlan at gmail.com Mon Sep 9 18:12:27 2013 From: ncoghlan at gmail.com (Nick Coghlan) Date: Tue, 10 Sep 2013 02:12:27 +1000 Subject: [Python-checkins] [Python-Dev] cpython: Post-3.4.0a2-release fixups. In-Reply-To: References: <3cYSwh0rHdz7Lkm@mail.python.org> Message-ID: On 10 Sep 2013 01:32, "Benjamin Peterson" wrote: > > Well, it's important for the release manager to make sure what the > script is doing is sane. :) Sure, preparing out of tree is fine and sensible. But we should either freeze the tree or update the NEWS headers immediately, otherwise we're going to have updates going into the wrong section. Cheers, Nick. > > 2013/9/9 Nick Coghlan : > > > > On 9 Sep 2013 22:15, "larry.hastings" wrote: > >> > >> http://hg.python.org/cpython/rev/6b211a0c8042 > >> changeset: 85645:6b211a0c8042 > >> user: Larry Hastings > >> date: Mon Sep 09 21:08:52 2013 +0900 > >> summary: > >> Post-3.4.0a2-release fixups. > >> > >> files: > >> Include/patchlevel.h | 2 +- > >> Misc/NEWS | 14 +++++++++++++- > >> 2 files changed, 14 insertions(+), 2 deletions(-) > >> > >> > >> diff --git a/Include/patchlevel.h b/Include/patchlevel.h > >> --- a/Include/patchlevel.h > >> +++ b/Include/patchlevel.h > >> @@ -23,7 +23,7 @@ > >> #define PY_RELEASE_SERIAL 2 > >> > >> /* Version as a string */ > >> -#define PY_VERSION "3.4.0a2" > >> +#define PY_VERSION "3.4.0a2+" > >> /*--end constants--*/ > >> > >> /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. > >> diff --git a/Misc/NEWS b/Misc/NEWS > >> --- a/Misc/NEWS > >> +++ b/Misc/NEWS > >> @@ -2,10 +2,22 @@ > >> Python News > >> +++++++++++ > >> > >> +What's New in Python 3.4.0 Alpha 3? > >> +=================================== > >> + > >> +Projected Release date: 2013-09-29 > >> + > >> +Core and Builtins > >> +----------------- > >> + > >> +Library > >> +------- > >> + > >> + > > > > I had already pushed alpha 3 entries in NEWS, so something seems to have > > gone wrong here. > > > > Perhaps, if RMs are preparing the release out of tree, we could get the NEWS > > file headings on default updated immediately after the last included commit? > > > > Cheers, > > Nick. > > > >> What's New in Python 3.4.0 Alpha 2? > >> =================================== > >> > >> -Projected Release date: 2013-09-08 > >> +Release date: 2013-09-09 > >> > >> Core and Builtins > >> ----------------- > >> > >> -- > >> Repository URL: http://hg.python.org/cpython > >> > >> _______________________________________________ > >> Python-checkins mailing list > >> Python-checkins at python.org > >> https://mail.python.org/mailman/listinfo/python-checkins > >> > > > > > > _______________________________________________ > > Python-Dev mailing list > > Python-Dev at python.org > > https://mail.python.org/mailman/listinfo/python-dev > > Unsubscribe: > > https://mail.python.org/mailman/options/python-dev/benjamin%40python.org > > > > > > -- > Regards, > Benjamin -------------- next part -------------- An HTML attachment was scrubbed... URL: From python-checkins at python.org Mon Sep 9 19:57:35 2013 From: python-checkins at python.org (tim.peters) Date: Mon, 9 Sep 2013 19:57:35 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Minor_cleanup_of_the_new_s?= =?utf-8?q?cheme_for_detecting_thread_termination=2E?= Message-ID: <3cYcXb5cNJz7LjT@mail.python.org> http://hg.python.org/cpython/rev/1f5a7853680c changeset: 85647:1f5a7853680c user: Tim Peters date: Mon Sep 09 12:57:10 2013 -0500 summary: Minor cleanup of the new scheme for detecting thread termination. Documented some obscurities, and assert'ed ._stop()'s crucial precondition. files: Lib/threading.py | 31 ++++++++++++++++++++++++++----- 1 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -703,8 +703,28 @@ pass def _stop(self): - self._is_stopped = True - self._tstate_lock = None + # After calling .stop(), .is_alive() returns False and .join() returns + # immediately. ._tstate_lock must be released before calling ._stop(). + # + # Normal case: C code at the end of the thread's life + # (release_sentinel in _threadmodule.c) releases ._tstate_lock, and + # that's detected by our ._wait_for_tstate_lock(), called by .join() + # and .is_alive(). Any number of threads _may_ call ._stop() + # simultaneously (for example, if multiple threads are blocked in + # .join() calls), and they're not serialized. That's harmless - + # they'll just make redundant rebindings of ._is_stopped and + # ._tstate_lock. Obscure: we rebind ._tstate_lock last so that the + # "assert self._is_stopped" in ._wait_for_tstate_lock() always works + # (the assert is executed only if ._tstate_lock is None). + # + # Special case: _main_thread releases ._tstate_lock via this module's + # _shutdown() function. + tlock = self._tstate_lock + if tlock is not None: + # It's OK if multiple threads get in here (see above). + assert not tlock.locked() + self._is_stopped = True + self._tstate_lock = None def _delete(self): "Remove current thread from the dict of currently running threads." @@ -921,9 +941,10 @@ # the main thread's tstate_lock - that won't happen until the interpreter # is nearly dead. So we release it here. Note that just calling _stop() # isn't enough: other threads may already be waiting on _tstate_lock. - assert _main_thread._tstate_lock is not None - assert _main_thread._tstate_lock.locked() - _main_thread._tstate_lock.release() + tlock = _main_thread._tstate_lock + assert tlock is not None + assert tlock.locked() + tlock.release() _main_thread._stop() t = _pickSomeNonDaemonThread() while t: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 20:12:09 2013 From: python-checkins at python.org (guido.van.rossum) Date: Mon, 9 Sep 2013 20:12:09 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Make_start=5Fserving=28=29_a_?= =?utf-8?q?coroutine=2E?= Message-ID: <3cYcsP4M3Sz7LjT@mail.python.org> http://hg.python.org/peps/rev/64569b37c943 changeset: 5106:64569b37c943 user: Guido van Rossum date: Mon Sep 09 11:11:52 2013 -0700 summary: Make start_serving() a coroutine. files: pep-3156.txt | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/pep-3156.txt b/pep-3156.txt --- a/pep-3156.txt +++ b/pep-3156.txt @@ -523,7 +523,7 @@ port are looked up using ``getaddrinfo()``. - ``start_serving(protocol_factory, host, port, **kwds)``. Enters a - serving loop that accepts connections. This is a Task that + serving loop that accepts connections. This is a coroutine that completes once the serving loop is set up to serve. The return value is a list of one or more sockets in listening mode. (Multiple sockets may be returned if the specified address allows both IPv4 -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Mon Sep 9 20:49:17 2013 From: python-checkins at python.org (tim.peters) Date: Mon, 9 Sep 2013 20:49:17 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Backed_out_changeset_1f5a7?= =?utf-8?q?853680c?= Message-ID: <3cYdhF1GVtzMdM@mail.python.org> http://hg.python.org/cpython/rev/e61780cd2c78 changeset: 85648:e61780cd2c78 user: Tim Peters date: Mon Sep 09 13:47:16 2013 -0500 summary: Backed out changeset 1f5a7853680c Unixy buildbots were failing the thread + fork tests :-( files: Lib/threading.py | 31 +++++-------------------------- 1 files changed, 5 insertions(+), 26 deletions(-) diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -703,28 +703,8 @@ pass def _stop(self): - # After calling .stop(), .is_alive() returns False and .join() returns - # immediately. ._tstate_lock must be released before calling ._stop(). - # - # Normal case: C code at the end of the thread's life - # (release_sentinel in _threadmodule.c) releases ._tstate_lock, and - # that's detected by our ._wait_for_tstate_lock(), called by .join() - # and .is_alive(). Any number of threads _may_ call ._stop() - # simultaneously (for example, if multiple threads are blocked in - # .join() calls), and they're not serialized. That's harmless - - # they'll just make redundant rebindings of ._is_stopped and - # ._tstate_lock. Obscure: we rebind ._tstate_lock last so that the - # "assert self._is_stopped" in ._wait_for_tstate_lock() always works - # (the assert is executed only if ._tstate_lock is None). - # - # Special case: _main_thread releases ._tstate_lock via this module's - # _shutdown() function. - tlock = self._tstate_lock - if tlock is not None: - # It's OK if multiple threads get in here (see above). - assert not tlock.locked() - self._is_stopped = True - self._tstate_lock = None + self._is_stopped = True + self._tstate_lock = None def _delete(self): "Remove current thread from the dict of currently running threads." @@ -941,10 +921,9 @@ # the main thread's tstate_lock - that won't happen until the interpreter # is nearly dead. So we release it here. Note that just calling _stop() # isn't enough: other threads may already be waiting on _tstate_lock. - tlock = _main_thread._tstate_lock - assert tlock is not None - assert tlock.locked() - tlock.release() + assert _main_thread._tstate_lock is not None + assert _main_thread._tstate_lock.locked() + _main_thread._tstate_lock.release() _main_thread._stop() t = _pickSomeNonDaemonThread() while t: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 9 21:42:19 2013 From: python-checkins at python.org (tim.peters) Date: Mon, 9 Sep 2013 21:42:19 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Another_stab_at_the_thread?= =?utf-8?q?_cleanup_patch=2E?= Message-ID: <3cYfsR0BcFz7LjT@mail.python.org> http://hg.python.org/cpython/rev/eac63e7ceb03 changeset: 85649:eac63e7ceb03 user: Tim Peters date: Mon Sep 09 14:41:50 2013 -0500 summary: Another stab at the thread cleanup patch. Antoine Pitrou found a variation that worked for him on the thread+fork tests, and added an important self._is_stopped = True to the after-fork code. I confess I don't know why things passed before. But then mixing fork with threads is insane ;-) files: Lib/threading.py | 29 ++++++++++++++++++++++++++--- 1 files changed, 26 insertions(+), 3 deletions(-) diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -566,6 +566,7 @@ else: # The thread isn't alive after fork: it doesn't have a tstate # anymore. + self._is_stopped = True self._tstate_lock = None def __repr__(self): @@ -703,6 +704,25 @@ pass def _stop(self): + # After calling ._stop(), .is_alive() returns False and .join() returns + # immediately. ._tstate_lock must be released before calling ._stop(). + # + # Normal case: C code at the end of the thread's life + # (release_sentinel in _threadmodule.c) releases ._tstate_lock, and + # that's detected by our ._wait_for_tstate_lock(), called by .join() + # and .is_alive(). Any number of threads _may_ call ._stop() + # simultaneously (for example, if multiple threads are blocked in + # .join() calls), and they're not serialized. That's harmless - + # they'll just make redundant rebindings of ._is_stopped and + # ._tstate_lock. Obscure: we rebind ._tstate_lock last so that the + # "assert self._is_stopped" in ._wait_for_tstate_lock() always works + # (the assert is executed only if ._tstate_lock is None). + # + # Special case: _main_thread releases ._tstate_lock via this + # module's _shutdown() function. + lock = self._tstate_lock + if lock is not None: + assert not lock.locked() self._is_stopped = True self._tstate_lock = None @@ -921,9 +941,12 @@ # the main thread's tstate_lock - that won't happen until the interpreter # is nearly dead. So we release it here. Note that just calling _stop() # isn't enough: other threads may already be waiting on _tstate_lock. - assert _main_thread._tstate_lock is not None - assert _main_thread._tstate_lock.locked() - _main_thread._tstate_lock.release() + tlock = _main_thread._tstate_lock + # The main thread isn't finished yet, so its thread state lock can't have + # been released. + assert tlock is not None + assert tlock.locked() + tlock.release() _main_thread._stop() t = _pickSomeNonDaemonThread() while t: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 10 01:49:10 2013 From: python-checkins at python.org (tim.peters) Date: Tue, 10 Sep 2013 01:49:10 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Get_=22stopped=22_back_int?= =?utf-8?q?o_repr=28Thread=29_when_appropriate=2E?= Message-ID: <3cYmLG16gjz7LjS@mail.python.org> http://hg.python.org/cpython/rev/967af1815967 changeset: 85650:967af1815967 user: Tim Peters date: Mon Sep 09 18:48:24 2013 -0500 summary: Get "stopped" back into repr(Thread) when appropriate. Due to recent changes, a Thread doesn't know that it's over before someone calls .join() or .is_alive(). That meant repr(Thread) continued to include "started" (and not "stopped") before one of those methods was called, even if hours passed since the thread ended. Repaired that. files: Lib/test/test_threading.py | 25 +++++++++++++++++++++++++ Lib/threading.py | 1 + 2 files changed, 26 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -573,6 +573,31 @@ # And verify the thread disposed of _tstate_lock. self.assertTrue(t._tstate_lock is None) + def test_repr_stopped(self): + # Verify that "stopped" shows up in repr(Thread) appropriately. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + t = threading.Thread(target=f) + t.start() + started.acquire() + self.assertIn("started", repr(t)) + finish.release() + # "stopped" should appear in the repr in a reasonable amount of time. + # Implementation detail: as of this writing, that's trivially true + # if .join() is called, and almost trivially true if .is_alive() is + # called. The detail we're testing here is that "stopped" shows up + # "all on its own". + LOOKING_FOR = "stopped" + for i in range(500): + if LOOKING_FOR in repr(t): + break + time.sleep(0.01) + self.assertIn(LOOKING_FOR, repr(t)) # we waited at least 5 seconds class ThreadJoinOnShutdown(BaseTestCase): diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -574,6 +574,7 @@ status = "initial" if self._started.is_set(): status = "started" + self.is_alive() # easy way to get ._is_stopped set when appropriate if self._is_stopped: status = "stopped" if self._daemonic: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 10 04:58:29 2013 From: python-checkins at python.org (senthil.kumaran) Date: Tue, 10 Sep 2013 04:58:29 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Document_Fract?= =?utf-8?q?ion=27s_numerator_and_denominator_properties=2E?= Message-ID: <3cYrXj1JSJz7LjZ@mail.python.org> http://hg.python.org/cpython/rev/fe5c03fb0ff6 changeset: 85651:fe5c03fb0ff6 branch: 3.3 parent: 85640:ce29d00470d1 user: Senthil Kumaran date: Mon Sep 09 19:57:37 2013 -0700 summary: Document Fraction's numerator and denominator properties. Addresses issue #18800 files: Doc/library/fractions.rst | 11 ++++++++++- 1 files changed, 10 insertions(+), 1 deletions(-) diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -77,13 +77,22 @@ :class:`numbers.Rational`, and implements all of the methods and operations from that class. :class:`Fraction` instances are hashable, and should be treated as immutable. In addition, - :class:`Fraction` has the following methods: + :class:`Fraction` has the following properties and methods: .. versionchanged:: 3.2 The :class:`Fraction` constructor now accepts :class:`float` and :class:`decimal.Decimal` instances. + .. attribute:: numerator + + Numerator of the Fraction in lowest term. + + .. attribute:: denominator + + Denominator of the Fraction in lowest term. + + .. method:: from_float(flt) This class method constructs a :class:`Fraction` representing the exact -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 10 04:58:30 2013 From: python-checkins at python.org (senthil.kumaran) Date: Tue, 10 Sep 2013 04:58:30 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_merge_from_3=2E3?= Message-ID: <3cYrXk353Cz7LkD@mail.python.org> http://hg.python.org/cpython/rev/5fb700ca3fd5 changeset: 85652:5fb700ca3fd5 parent: 85650:967af1815967 parent: 85651:fe5c03fb0ff6 user: Senthil Kumaran date: Mon Sep 09 19:58:20 2013 -0700 summary: merge from 3.3 Document Fraction's numerator and denominator properties. Addresses issue #18800 files: Doc/library/fractions.rst | 11 ++++++++++- 1 files changed, 10 insertions(+), 1 deletions(-) diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -77,13 +77,22 @@ :class:`numbers.Rational`, and implements all of the methods and operations from that class. :class:`Fraction` instances are hashable, and should be treated as immutable. In addition, - :class:`Fraction` has the following methods: + :class:`Fraction` has the following properties and methods: .. versionchanged:: 3.2 The :class:`Fraction` constructor now accepts :class:`float` and :class:`decimal.Decimal` instances. + .. attribute:: numerator + + Numerator of the Fraction in lowest term. + + .. attribute:: denominator + + Denominator of the Fraction in lowest term. + + .. method:: from_float(flt) This class method constructs a :class:`Fraction` representing the exact -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Tue Sep 10 06:24:33 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Tue, 10 Sep 2013 06:24:33 +0200 Subject: [Python-checkins] Daily reference leaks (967af1815967): sum=0 Message-ID: results for 967af1815967 on branch "default" -------------------------------------------- Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogjT10f8', '-x'] From python-checkins at python.org Tue Sep 10 07:40:21 2013 From: python-checkins at python.org (senthil.kumaran) Date: Tue, 10 Sep 2013 07:40:21 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Clarify_mmap?= =?utf-8?q?=2Eclose_method__behavior=2E__Addresses_issue__=2318815?= Message-ID: <3cYw7T6lKwz7LjZ@mail.python.org> http://hg.python.org/cpython/rev/443d12b61e5b changeset: 85653:443d12b61e5b branch: 2.7 parent: 85639:740bd510a888 user: Senthil Kumaran date: Mon Sep 09 22:38:58 2013 -0700 summary: Clarify mmap.close method behavior. Addresses issue #18815 Patch contributed by Anoop Thomas Mathew. files: Doc/library/mmap.rst | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -152,8 +152,9 @@ .. method:: close() - Close the file. Subsequent calls to other methods of the object will - result in an exception being raised. + Closes the mmap. Subsequent calls to other methods of the object will + result in a ValueError exception being raised. This will not close + the open file. .. method:: find(string[, start[, end]]) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 10 07:40:23 2013 From: python-checkins at python.org (senthil.kumaran) Date: Tue, 10 Sep 2013 07:40:23 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Clarify_mmap?= =?utf-8?q?=2Eclose_method__behavior=2E__Addresses_issue__=2318815?= Message-ID: <3cYw7W1R27z7Ljc@mail.python.org> http://hg.python.org/cpython/rev/373907ca13e0 changeset: 85654:373907ca13e0 branch: 3.3 parent: 85651:fe5c03fb0ff6 user: Senthil Kumaran date: Mon Sep 09 22:39:28 2013 -0700 summary: Clarify mmap.close method behavior. Addresses issue #18815 Patch contributed by Anoop Thomas Mathew. files: Doc/library/mmap.rst | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -155,8 +155,9 @@ .. method:: close() - Close the file. Subsequent calls to other methods of the object will - result in an exception being raised. + Closes the mmap. Subsequent calls to other methods of the object will + result in a ValueError exception being raised. This will not close + the open file. .. attribute:: closed -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 10 07:40:24 2013 From: python-checkins at python.org (senthil.kumaran) Date: Tue, 10 Sep 2013 07:40:24 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_merge_from_3=2E3?= Message-ID: <3cYw7X38b5z7Ll6@mail.python.org> http://hg.python.org/cpython/rev/377bd6e0f61c changeset: 85655:377bd6e0f61c parent: 85652:5fb700ca3fd5 parent: 85654:373907ca13e0 user: Senthil Kumaran date: Mon Sep 09 22:40:13 2013 -0700 summary: merge from 3.3 Clarify mmap.close method behavior. Addresses issue #18815 Patch contributed by Anoop Thomas Mathew. files: Doc/library/mmap.rst | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -155,8 +155,9 @@ .. method:: close() - Close the file. Subsequent calls to other methods of the object will - result in an exception being raised. + Closes the mmap. Subsequent calls to other methods of the object will + result in a ValueError exception being raised. This will not close + the open file. .. attribute:: closed -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Wed Sep 11 06:26:18 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Wed, 11 Sep 2013 06:26:18 +0200 Subject: [Python-checkins] Daily reference leaks (377bd6e0f61c): sum=5 Message-ID: results for 377bd6e0f61c on branch "default" -------------------------------------------- test_support leaked [1, 0, 0] references, sum=1 test_support leaked [1, 2, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogFKp5RF', '-x'] From python-checkins at python.org Wed Sep 11 08:16:00 2013 From: python-checkins at python.org (raymond.hettinger) Date: Wed, 11 Sep 2013 08:16:00 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318962=3A__Optimiz?= =?utf-8?q?e_the_single_iterator_case_for__heapq=2Emerge=28=29?= Message-ID: <3cZXt868Mrz7Ljb@mail.python.org> http://hg.python.org/cpython/rev/0e70bf1f32a3 changeset: 85656:0e70bf1f32a3 user: Raymond Hettinger date: Wed Sep 11 01:15:40 2013 -0500 summary: Issue #18962: Optimize the single iterator case for heapq.merge() Suggested by Wouter Bolsterlee. files: Lib/heapq.py | 14 +++++++++----- Misc/ACKS | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Lib/heapq.py b/Lib/heapq.py --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -358,6 +358,7 @@ ''' _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration + _len = len h = [] h_append = h.append @@ -369,17 +370,20 @@ pass heapify(h) - while 1: + while _len(h) > 1: try: - while 1: - v, itnum, next = s = h[0] # raises IndexError when h is empty + while True: + v, itnum, next = s = h[0] yield v s[0] = next() # raises StopIteration when exhausted _heapreplace(h, s) # restore heap condition except _StopIteration: _heappop(h) # remove empty iterator - except IndexError: - return + if h: + # fast case when only a single iterator remains + v, itnum, next = h[0] + yield v + yield from next.__self__ # Extend the implementations of nsmallest and nlargest to use a key= argument _nsmallest = nsmallest diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -135,6 +135,7 @@ Matthew Boedicker Robin Boerdijk David Bolen +Wouter Bolsterlee Gawain Bolton Forest Bond Gregory Bond -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 11 22:00:17 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Wed, 11 Sep 2013 22:00:17 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4OTg4?= =?utf-8?q?=3A_The_=22Tab=22_key_now_works_when_a_word_is_already_autocomp?= =?utf-8?q?leted=2E?= Message-ID: <3cZv9F2Hjpz7LjY@mail.python.org> http://hg.python.org/cpython/rev/c882395e8ed8 changeset: 85657:c882395e8ed8 branch: 3.3 parent: 85654:373907ca13e0 user: Serhiy Storchaka date: Wed Sep 11 22:46:27 2013 +0300 summary: Issue #18988: The "Tab" key now works when a word is already autocompleted. files: Lib/idlelib/AutoComplete.py | 9 +++------ Lib/idlelib/AutoCompleteWindow.py | 3 ++- Misc/NEWS | 2 ++ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/idlelib/AutoComplete.py b/Lib/idlelib/AutoComplete.py --- a/Lib/idlelib/AutoComplete.py +++ b/Lib/idlelib/AutoComplete.py @@ -160,12 +160,9 @@ if not comp_lists[0]: return self.autocompletewindow = self._make_autocomplete_window() - self.autocompletewindow.show_window(comp_lists, - "insert-%dc" % len(comp_start), - complete, - mode, - userWantsWin) - return True + return not self.autocompletewindow.show_window( + comp_lists, "insert-%dc" % len(comp_start), + complete, mode, userWantsWin) def fetch_completions(self, what, mode): """Return a pair of lists of completions for something. The first list diff --git a/Lib/idlelib/AutoCompleteWindow.py b/Lib/idlelib/AutoCompleteWindow.py --- a/Lib/idlelib/AutoCompleteWindow.py +++ b/Lib/idlelib/AutoCompleteWindow.py @@ -157,13 +157,14 @@ self.start = self.widget.get(self.startindex, "insert") if complete: completed = self._complete_string(self.start) + start = self.start self._change_start(completed) i = self._binary_search(completed) if self.completions[i] == completed and \ (i == len(self.completions)-1 or self.completions[i+1][:len(completed)] != completed): # There is exactly one matching completion - return + return completed == start self.userwantswindow = userWantsWin self.lasttypedstart = self.start diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -293,6 +293,8 @@ IDLE ---- +- Issue #18988: The "Tab" key now works when a word is already autocompleted. + - Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster. - Issue #18429: Format / Format Paragraph, now works when comment blocks -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 11 22:00:18 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Wed, 11 Sep 2013 22:00:18 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2318988=3A_The_=22Tab=22_key_now_works_when_a_wor?= =?utf-8?q?d_is_already_autocompleted=2E?= Message-ID: <3cZv9G4Cp4z7LjY@mail.python.org> http://hg.python.org/cpython/rev/7d38784092d8 changeset: 85658:7d38784092d8 parent: 85656:0e70bf1f32a3 parent: 85657:c882395e8ed8 user: Serhiy Storchaka date: Wed Sep 11 22:48:52 2013 +0300 summary: Issue #18988: The "Tab" key now works when a word is already autocompleted. files: Lib/idlelib/AutoComplete.py | 9 +++------ Lib/idlelib/AutoCompleteWindow.py | 3 ++- Misc/NEWS | 5 +++++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Lib/idlelib/AutoComplete.py b/Lib/idlelib/AutoComplete.py --- a/Lib/idlelib/AutoComplete.py +++ b/Lib/idlelib/AutoComplete.py @@ -160,12 +160,9 @@ if not comp_lists[0]: return self.autocompletewindow = self._make_autocomplete_window() - self.autocompletewindow.show_window(comp_lists, - "insert-%dc" % len(comp_start), - complete, - mode, - userWantsWin) - return True + return not self.autocompletewindow.show_window( + comp_lists, "insert-%dc" % len(comp_start), + complete, mode, userWantsWin) def fetch_completions(self, what, mode): """Return a pair of lists of completions for something. The first list diff --git a/Lib/idlelib/AutoCompleteWindow.py b/Lib/idlelib/AutoCompleteWindow.py --- a/Lib/idlelib/AutoCompleteWindow.py +++ b/Lib/idlelib/AutoCompleteWindow.py @@ -157,13 +157,14 @@ self.start = self.widget.get(self.startindex, "insert") if complete: completed = self._complete_string(self.start) + start = self.start self._change_start(completed) i = self._binary_search(completed) if self.completions[i] == completed and \ (i == len(self.completions)-1 or self.completions[i+1][:len(completed)] != completed): # There is exactly one matching completion - return + return completed == start self.userwantswindow = userWantsWin self.lasttypedstart = self.start diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,11 @@ test.support was converted to a package. Regression noticed by Zachary Ware. +IDLE +---- + +- Issue #18988: The "Tab" key now works when a word is already autocompleted. + What's New in Python 3.4.0 Alpha 2? =================================== -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 11 22:00:21 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Wed, 11 Sep 2013 22:00:21 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4OTg4?= =?utf-8?q?=3A_The_=22Tab=22_key_now_works_when_a_word_is_already_autocomp?= =?utf-8?q?leted=2E?= Message-ID: <3cZv9K0K6Xz7LkX@mail.python.org> http://hg.python.org/cpython/rev/8d320204d5d6 changeset: 85659:8d320204d5d6 branch: 2.7 parent: 85653:443d12b61e5b user: Serhiy Storchaka date: Wed Sep 11 22:49:06 2013 +0300 summary: Issue #18988: The "Tab" key now works when a word is already autocompleted. files: Lib/idlelib/AutoComplete.py | 9 +++------ Lib/idlelib/AutoCompleteWindow.py | 3 ++- Misc/NEWS | 2 ++ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/idlelib/AutoComplete.py b/Lib/idlelib/AutoComplete.py --- a/Lib/idlelib/AutoComplete.py +++ b/Lib/idlelib/AutoComplete.py @@ -156,12 +156,9 @@ if not comp_lists[0]: return self.autocompletewindow = self._make_autocomplete_window() - self.autocompletewindow.show_window(comp_lists, - "insert-%dc" % len(comp_start), - complete, - mode, - userWantsWin) - return True + return not self.autocompletewindow.show_window( + comp_lists, "insert-%dc" % len(comp_start), + complete, mode, userWantsWin) def fetch_completions(self, what, mode): """Return a pair of lists of completions for something. The first list diff --git a/Lib/idlelib/AutoCompleteWindow.py b/Lib/idlelib/AutoCompleteWindow.py --- a/Lib/idlelib/AutoCompleteWindow.py +++ b/Lib/idlelib/AutoCompleteWindow.py @@ -157,13 +157,14 @@ self.start = self.widget.get(self.startindex, "insert") if complete: completed = self._complete_string(self.start) + start = self.start self._change_start(completed) i = self._binary_search(completed) if self.completions[i] == completed and \ (i == len(self.completions)-1 or self.completions[i+1][:len(completed)] != completed): # There is exactly one matching completion - return + return completed == start self.userwantswindow = userWantsWin self.lasttypedstart = self.start diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -191,6 +191,8 @@ IDLE ---- +- Issue #18988: The "Tab" key now works when a word is already autocompleted. + - Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster. - Issue #18429: Format / Format Paragraph, now works when comment blocks -- Repository URL: http://hg.python.org/cpython From root at python.org Thu Sep 12 00:05:22 2013 From: root at python.org (Cron Daemon) Date: Thu, 12 Sep 2013 00:05:22 +0200 Subject: [Python-checkins] Cron /home/docs/build-devguide Message-ID: abort: error: Connection timed out From solipsis at pitrou.net Thu Sep 12 06:25:12 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Thu, 12 Sep 2013 06:25:12 +0200 Subject: [Python-checkins] Daily reference leaks (7d38784092d8): sum=0 Message-ID: results for 7d38784092d8 on branch "default" -------------------------------------------- Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogQzl9Xq', '-x'] From python-checkins at python.org Thu Sep 12 07:57:22 2013 From: python-checkins at python.org (senthil.kumaran) Date: Thu, 12 Sep 2013 07:57:22 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Improve_the_do?= =?utf-8?q?cstring_of_random=2Eshuffle=2E_Inform_users_not_to_provide_int_?= =?utf-8?q?arg=2E?= Message-ID: <3cb8QB6mMSz7LjS@mail.python.org> http://hg.python.org/cpython/rev/82bdd5fc7a71 changeset: 85660:82bdd5fc7a71 branch: 2.7 parent: 85653:443d12b61e5b user: Senthil Kumaran date: Wed Sep 11 22:52:58 2013 -0700 summary: Improve the docstring of random.shuffle. Inform users not to provide int arg. Addresses issue #14927 files: Lib/random.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/Lib/random.py b/Lib/random.py --- a/Lib/random.py +++ b/Lib/random.py @@ -278,6 +278,8 @@ Optional arg random is a 0-argument function returning a random float in [0.0, 1.0); by default, the standard random.random. + + Do not supply the 'int' argument. """ if random is None: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 12 07:57:24 2013 From: python-checkins at python.org (senthil.kumaran) Date: Thu, 12 Sep 2013 07:57:24 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Improve_the_do?= =?utf-8?q?cstring_of_random=2Eshuffle=2E_Inform_users_not_to_provide_int_?= =?utf-8?q?arg=2E?= Message-ID: <3cb8QD1HKNz7LjR@mail.python.org> http://hg.python.org/cpython/rev/4782faf29480 changeset: 85661:4782faf29480 branch: 3.3 parent: 85654:373907ca13e0 user: Senthil Kumaran date: Wed Sep 11 22:54:31 2013 -0700 summary: Improve the docstring of random.shuffle. Inform users not to provide int arg. Addresses issue #14927 files: Lib/random.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/Lib/random.py b/Lib/random.py --- a/Lib/random.py +++ b/Lib/random.py @@ -256,6 +256,8 @@ Optional arg random is a 0-argument function returning a random float in [0.0, 1.0); by default, the standard random.random. + + Do not supply the 'int' argument. """ randbelow = self._randbelow -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 12 07:57:25 2013 From: python-checkins at python.org (senthil.kumaran) Date: Thu, 12 Sep 2013 07:57:25 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_merge_from_3=2E3?= Message-ID: <3cb8QF2vy7z7Ljh@mail.python.org> http://hg.python.org/cpython/rev/15096b93ae5a changeset: 85662:15096b93ae5a parent: 85655:377bd6e0f61c parent: 85661:4782faf29480 user: Senthil Kumaran date: Wed Sep 11 22:55:54 2013 -0700 summary: merge from 3.3 Improve the docstring of random.shuffle. Inform users not to provide int arg. Addresses issue #14927 files: Lib/random.py | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diff --git a/Lib/random.py b/Lib/random.py --- a/Lib/random.py +++ b/Lib/random.py @@ -257,9 +257,16 @@ def shuffle(self, x, random=None, int=int): """Shuffle list x in place, and return None. +<<<<<<< local Optional argument random is a 0-argument function returning a random float in [0.0, 1.0); if it is the default None, the standard random.random will be used. +======= + Optional arg random is a 0-argument function returning a random + float in [0.0, 1.0); by default, the standard random.random. + + Do not supply the 'int' argument. +>>>>>>> other """ randbelow = self._randbelow -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 12 07:57:26 2013 From: python-checkins at python.org (senthil.kumaran) Date: Thu, 12 Sep 2013 07:57:26 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?= =?utf-8?q?=29=3A_Automated_merge_with_ssh=3A//hg=2Epython=2Eorg/cpython?= Message-ID: <3cb8QG4XDCz7Ljh@mail.python.org> http://hg.python.org/cpython/rev/51c252f69a68 changeset: 85663:51c252f69a68 parent: 85658:7d38784092d8 parent: 85662:15096b93ae5a user: Senthil Kumaran date: Wed Sep 11 22:56:28 2013 -0700 summary: Automated merge with ssh://hg.python.org/cpython files: Lib/random.py | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diff --git a/Lib/random.py b/Lib/random.py --- a/Lib/random.py +++ b/Lib/random.py @@ -257,9 +257,16 @@ def shuffle(self, x, random=None, int=int): """Shuffle list x in place, and return None. +<<<<<<< local Optional argument random is a 0-argument function returning a random float in [0.0, 1.0); if it is the default None, the standard random.random will be used. +======= + Optional arg random is a 0-argument function returning a random + float in [0.0, 1.0); by default, the standard random.random. + + Do not supply the 'int' argument. +>>>>>>> other """ randbelow = self._randbelow -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 12 07:57:27 2013 From: python-checkins at python.org (senthil.kumaran) Date: Thu, 12 Sep 2013 07:57:27 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMi43IC0+IDIuNyk6?= =?utf-8?q?_Automated_merge_with_file=3A///Users/skumaran/python/cpython?= Message-ID: <3cb8QH6LKLz7Ljr@mail.python.org> http://hg.python.org/cpython/rev/07fbc1aec77c changeset: 85664:07fbc1aec77c branch: 2.7 parent: 85659:8d320204d5d6 parent: 85660:82bdd5fc7a71 user: Senthil Kumaran date: Wed Sep 11 22:56:34 2013 -0700 summary: Automated merge with file:///Users/skumaran/python/cpython files: Lib/random.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/Lib/random.py b/Lib/random.py --- a/Lib/random.py +++ b/Lib/random.py @@ -278,6 +278,8 @@ Optional arg random is a 0-argument function returning a random float in [0.0, 1.0); by default, the standard random.random. + + Do not supply the 'int' argument. """ if random is None: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 12 07:57:29 2013 From: python-checkins at python.org (senthil.kumaran) Date: Thu, 12 Sep 2013 07:57:29 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy4zIC0+IDMuMyk6?= =?utf-8?q?_Automated_merge_with_file=3A///Users/skumaran/python/cpython?= Message-ID: <3cb8QK0zbGz7Ljg@mail.python.org> http://hg.python.org/cpython/rev/246095bb2309 changeset: 85665:246095bb2309 branch: 3.3 parent: 85657:c882395e8ed8 parent: 85661:4782faf29480 user: Senthil Kumaran date: Wed Sep 11 22:56:30 2013 -0700 summary: Automated merge with file:///Users/skumaran/python/cpython files: Lib/random.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/Lib/random.py b/Lib/random.py --- a/Lib/random.py +++ b/Lib/random.py @@ -256,6 +256,8 @@ Optional arg random is a 0-argument function returning a random float in [0.0, 1.0); by default, the standard random.random. + + Do not supply the 'int' argument. """ randbelow = self._randbelow -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 12 16:07:04 2013 From: python-checkins at python.org (senthil.kumaran) Date: Thu, 12 Sep 2013 16:07:04 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Removing_the_merge_conflic?= =?utf-8?q?t_markers=2E?= Message-ID: <3cbMHD48k8z7LjN@mail.python.org> http://hg.python.org/cpython/rev/1398dfb59fd9 changeset: 85666:1398dfb59fd9 parent: 85663:51c252f69a68 user: Senthil Kumaran date: Thu Sep 12 07:06:49 2013 -0700 summary: Removing the merge conflict markers. - my previous removal and hg resolve mark had still left them and hooks did not catch it too! files: Lib/random.py | 5 ----- 1 files changed, 0 insertions(+), 5 deletions(-) diff --git a/Lib/random.py b/Lib/random.py --- a/Lib/random.py +++ b/Lib/random.py @@ -257,16 +257,11 @@ def shuffle(self, x, random=None, int=int): """Shuffle list x in place, and return None. -<<<<<<< local Optional argument random is a 0-argument function returning a random float in [0.0, 1.0); if it is the default None, the standard random.random will be used. -======= - Optional arg random is a 0-argument function returning a random - float in [0.0, 1.0); by default, the standard random.random. Do not supply the 'int' argument. ->>>>>>> other """ randbelow = self._randbelow -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Fri Sep 13 06:30:28 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Fri, 13 Sep 2013 06:30:28 +0200 Subject: [Python-checkins] Daily reference leaks (1398dfb59fd9): sum=2 Message-ID: results for 1398dfb59fd9 on branch "default" -------------------------------------------- test_support leaked [0, -1, 1] references, sum=0 test_support leaked [0, -1, 3] memory blocks, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflog8ByhKb', '-x'] From python-checkins at python.org Fri Sep 13 07:09:16 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Fri, 13 Sep 2013 07:09:16 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4Nzg0?= =?utf-8?q?=3A_The_uuid_module_no_more_attempts_to_load_libc_via_ctypes=2E?= =?utf-8?q?CDLL=2C?= Message-ID: <3cblJD02GVz7Ljd@mail.python.org> http://hg.python.org/cpython/rev/7aaba721ebc0 changeset: 85667:7aaba721ebc0 branch: 3.3 parent: 85665:246095bb2309 user: Serhiy Storchaka date: Fri Sep 13 07:46:40 2013 +0300 summary: Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, if all necessary functions are already found in libuuid. Patch by Evgeny Sologubov. files: Lib/uuid.py | 2 ++ Misc/ACKS | 1 + Misc/NEWS | 4 ++++ 3 files changed, 7 insertions(+), 0 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -429,6 +429,8 @@ _uuid_generate_random = lib.uuid_generate_random if hasattr(lib, 'uuid_generate_time'): _uuid_generate_time = lib.uuid_generate_time + if _uuid_generate_random is not None: + break # found everything we were looking for # The uuid_generate_* functions are broken on MacOS X 10.5, as noted # in issue #8621 the function generates the same sequence of values diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1160,6 +1160,7 @@ Eric Snow Dirk Soede Paul Sokolovsky +Evgeny Sologubov Cody Somerville Edoardo Spadolini Clay Spence diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -68,6 +68,10 @@ Library ------- +- Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, + if all necessary functions are already found in libuuid. + Patch by Evgeny Sologubov. + - Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in the _sre module. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 07:09:17 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Fri, 13 Sep 2013 07:09:17 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2318784=3A_The_uuid_module_no_more_attempts_to_lo?= =?utf-8?q?ad_libc_via_ctypes=2ECDLL=2C?= Message-ID: <3cblJF1v9Tz7Ljx@mail.python.org> http://hg.python.org/cpython/rev/66ec8431032d changeset: 85668:66ec8431032d parent: 85666:1398dfb59fd9 parent: 85667:7aaba721ebc0 user: Serhiy Storchaka date: Fri Sep 13 07:49:36 2013 +0300 summary: Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, if all necessary functions are already found in libuuid. Patch by Evgeny Sologubov. files: Lib/uuid.py | 2 ++ Misc/ACKS | 1 + Misc/NEWS | 4 ++++ 3 files changed, 7 insertions(+), 0 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -429,6 +429,8 @@ _uuid_generate_random = lib.uuid_generate_random if hasattr(lib, 'uuid_generate_time'): _uuid_generate_time = lib.uuid_generate_time + if _uuid_generate_random is not None: + break # found everything we were looking for # The uuid_generate_* functions are broken on MacOS X 10.5, as noted # in issue #8621 the function generates the same sequence of values diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1199,6 +1199,7 @@ Eric Snow Dirk Soede Paul Sokolovsky +Evgeny Sologubov Cody Somerville Edoardo Spadolini Clay Spence diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Library ------- +- Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, + if all necessary functions are already found in libuuid. + Patch by Evgeny Sologubov. + - The :envvar:`PYTHONFAULTHANDLER` environment variable now only enables the faulthandler module if the variable is non-empty. Same behaviour than other variables like :envvar:`PYTHONDONTWRITEBYTECODE`. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 07:09:18 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Fri, 13 Sep 2013 07:09:18 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4Nzg0?= =?utf-8?q?=3A_The_uuid_module_no_more_attempts_to_load_libc_via_ctypes=2E?= =?utf-8?q?CDLL=2C?= Message-ID: <3cblJG3qjNz7LkG@mail.python.org> http://hg.python.org/cpython/rev/6d8a5cbb94c9 changeset: 85669:6d8a5cbb94c9 branch: 2.7 parent: 85664:07fbc1aec77c user: Serhiy Storchaka date: Fri Sep 13 07:52:00 2013 +0300 summary: Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, if all necessary functions are already found in libuuid. Patch by Evgeny Sologubov. files: Lib/uuid.py | 2 ++ Misc/ACKS | 1 + Misc/NEWS | 4 ++++ 3 files changed, 7 insertions(+), 0 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -408,6 +408,8 @@ _uuid_generate_random = lib.uuid_generate_random if hasattr(lib, 'uuid_generate_time'): _uuid_generate_time = lib.uuid_generate_time + if _uuid_generate_random is not None: + break # found everything we were looking for # The uuid_generate_* functions are broken on MacOS X 10.5, as noted # in issue #8621 the function generates the same sequence of values diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -970,6 +970,7 @@ Rafal Smotrzyk Dirk Soede Paul Sokolovsky +Evgeny Sologubov Cody Somerville Clay Spence Stefan Sperling diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,10 @@ Library ------- +- Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, + if all necessary functions are already found in libuuid. + Patch by Evgeny Sologubov. + - Issue #14971: unittest test discovery no longer gets confused when a function has a different __name__ than its name in the TestCase class dictionary. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 09:22:53 2013 From: python-checkins at python.org (senthil.kumaran) Date: Fri, 13 Sep 2013 09:22:53 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Fix_SimpleHTTP?= =?utf-8?q?Server=27s_request_handling_case_on_trailing_=27/=27=2E?= Message-ID: <3cbpGP5GSczT1G@mail.python.org> http://hg.python.org/cpython/rev/a58b620e4dc9 changeset: 85670:a58b620e4dc9 branch: 2.7 user: Senthil Kumaran date: Fri Sep 13 00:18:55 2013 -0700 summary: Fix SimpleHTTPServer's request handling case on trailing '/'. Patch contributed by Vajrasky Kok. Addresses Issue #17324 files: Lib/SimpleHTTPServer.py | 4 ++++ Lib/test/test_httpservers.py | 3 +++ Misc/NEWS | 3 +++ 3 files changed, 10 insertions(+), 0 deletions(-) diff --git a/Lib/SimpleHTTPServer.py b/Lib/SimpleHTTPServer.py --- a/Lib/SimpleHTTPServer.py +++ b/Lib/SimpleHTTPServer.py @@ -149,6 +149,8 @@ # abandon query parameters path = path.split('?',1)[0] path = path.split('#',1)[0] + # Don't forget explicit trailing slash when normalizing. Issue17324 + trailing_slash = True if path.rstrip().endswith('/') else False path = posixpath.normpath(urllib.unquote(path)) words = path.split('/') words = filter(None, words) @@ -158,6 +160,8 @@ head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) + if trailing_slash: + path += '/' return path def copyfile(self, source, outputfile): diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -313,6 +313,9 @@ #constructs the path relative to the root directory of the HTTPServer response = self.request(self.tempdir_name + '/test') self.check_status_and_reason(response, 200, data=self.data) + # check for trailing "/" which should return 404. See Issue17324 + response = self.request(self.tempdir_name + '/test/') + self.check_status_and_reason(response, 404) response = self.request(self.tempdir_name + '/') self.check_status_and_reason(response, 200) response = self.request(self.tempdir_name) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Library ------- +- Issue #17324: Fix http.server's request handling case on trailing '/'. Patch + contributed by Vajrasky Kok. + - Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, if all necessary functions are already found in libuuid. Patch by Evgeny Sologubov. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 09:22:55 2013 From: python-checkins at python.org (senthil.kumaran) Date: Fri, 13 Sep 2013 09:22:55 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogRml4IGh0dHAuc2Vy?= =?utf-8?q?ver=27s_request_handling_case_on_trailing_=27/=27=2E?= Message-ID: <3cbpGR06MQz7Ljh@mail.python.org> http://hg.python.org/cpython/rev/1fcccbbe15e2 changeset: 85671:1fcccbbe15e2 branch: 3.3 parent: 85667:7aaba721ebc0 user: Senthil Kumaran date: Fri Sep 13 00:21:18 2013 -0700 summary: Fix http.server's request handling case on trailing '/'. Patch contributed by Vajrasky Kok. Addresses Issue #17324 files: Lib/http/server.py | 4 ++++ Lib/test/test_httpservers.py | 3 +++ Misc/NEWS | 4 ++++ 3 files changed, 11 insertions(+), 0 deletions(-) diff --git a/Lib/http/server.py b/Lib/http/server.py --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -780,6 +780,8 @@ # abandon query parameters path = path.split('?',1)[0] path = path.split('#',1)[0] + # Don't forget explicit trailing slash when normalizing. Issue17324 + trailing_slash = True if path.rstrip().endswith('/') else False path = posixpath.normpath(urllib.parse.unquote(path)) words = path.split('/') words = filter(None, words) @@ -789,6 +791,8 @@ head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) + if trailing_slash: + path += '/' return path def copyfile(self, source, outputfile): diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -249,6 +249,9 @@ #constructs the path relative to the root directory of the HTTPServer response = self.request(self.tempdir_name + '/test') self.check_status_and_reason(response, 200, data=self.data) + # check for trailing "/" which should return 404. See Issue17324 + response = self.request(self.tempdir_name + '/test/') + self.check_status_and_reason(response, 404) response = self.request(self.tempdir_name + '/') self.check_status_and_reason(response, 200) response = self.request(self.tempdir_name) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -68,6 +68,10 @@ Library ------- + +- Issue #17324: Fix http.server's request handling case on trailing '/'. Patch + contributed by Vajrasky Kok. + - Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, if all necessary functions are already found in libuuid. Patch by Evgeny Sologubov. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 09:22:56 2013 From: python-checkins at python.org (senthil.kumaran) Date: Fri, 13 Sep 2013 09:22:56 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Fix_http=2Eserver=27s_request_handling_case_on_trailing_?= =?utf-8?b?Jy8nLg==?= Message-ID: <3cbpGS1zZSz7Lk4@mail.python.org> http://hg.python.org/cpython/rev/b85c9d2a5227 changeset: 85672:b85c9d2a5227 parent: 85668:66ec8431032d parent: 85671:1fcccbbe15e2 user: Senthil Kumaran date: Fri Sep 13 00:22:45 2013 -0700 summary: Fix http.server's request handling case on trailing '/'. Patch contributed by Vajrasky Kok. Addresses Issue #17324 files: Lib/http/server.py | 4 ++++ Lib/test/test_httpservers.py | 3 +++ Misc/NEWS | 4 ++++ 3 files changed, 11 insertions(+), 0 deletions(-) diff --git a/Lib/http/server.py b/Lib/http/server.py --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -788,6 +788,8 @@ # abandon query parameters path = path.split('?',1)[0] path = path.split('#',1)[0] + # Don't forget explicit trailing slash when normalizing. Issue17324 + trailing_slash = True if path.rstrip().endswith('/') else False path = posixpath.normpath(urllib.parse.unquote(path)) words = path.split('/') words = filter(None, words) @@ -797,6 +799,8 @@ head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) + if trailing_slash: + path += '/' return path def copyfile(self, source, outputfile): diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -270,6 +270,9 @@ #constructs the path relative to the root directory of the HTTPServer response = self.request(self.tempdir_name + '/test') self.check_status_and_reason(response, 200, data=self.data) + # check for trailing "/" which should return 404. See Issue17324 + response = self.request(self.tempdir_name + '/test/') + self.check_status_and_reason(response, 404) response = self.request(self.tempdir_name + '/') self.check_status_and_reason(response, 200) response = self.request(self.tempdir_name) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Library ------- + +- Issue #17324: Fix http.server's request handling case on trailing '/'. Patch + contributed by Vajrasky Kok. + - Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, if all necessary functions are already found in libuuid. Patch by Evgeny Sologubov. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 10:46:52 2013 From: python-checkins at python.org (serhiy.storchaka) Date: Fri, 13 Sep 2013 10:46:52 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318818=3A_The_=22e?= =?utf-8?q?ncodingname=22_part_of_PYTHONIOENCODING_is_now_optional=2E?= Message-ID: <3cbr7J6psQz7LjQ@mail.python.org> http://hg.python.org/cpython/rev/c7fdb0637d0b changeset: 85673:c7fdb0637d0b user: Serhiy Storchaka date: Fri Sep 13 11:46:24 2013 +0300 summary: Issue #18818: The "encodingname" part of PYTHONIOENCODING is now optional. files: Doc/using/cmdline.rst | 9 +++++-- Lib/test/test_sys.py | 36 +++++++++++++++++++++++++++++++ Misc/NEWS | 2 + Python/pythonrun.c | 20 ++++++++++------ 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -538,13 +538,16 @@ .. envvar:: PYTHONIOENCODING If this is set before running the interpreter, it overrides the encoding used - for stdin/stdout/stderr, in the syntax ``encodingname:errorhandler``. The - ``:errorhandler`` part is optional and has the same meaning as in - :func:`str.encode`. + for stdin/stdout/stderr, in the syntax ``encodingname:errorhandler``. Both + the ``encodingname`` and the ``:errorhandler`` parts are optional and have + the same meaning as in :func:`str.encode`. For stderr, the ``:errorhandler`` part is ignored; the handler will always be ``'backslashreplace'``. + .. versionchanged:: 3.4 + The ``encodingname`` part is now optional. + .. envvar:: PYTHONNOUSERSITE diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -544,6 +544,42 @@ out = p.communicate()[0].strip() self.assertEqual(out, b'?') + env["PYTHONIOENCODING"] = "ascii" + p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env) + out, err = p.communicate() + self.assertEqual(out, b'') + self.assertIn(b'UnicodeEncodeError:', err) + self.assertIn(rb"'\xa2'", err) + + env["PYTHONIOENCODING"] = "ascii:" + p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env) + out, err = p.communicate() + self.assertEqual(out, b'') + self.assertIn(b'UnicodeEncodeError:', err) + self.assertIn(rb"'\xa2'", err) + + env["PYTHONIOENCODING"] = ":surrogateescape" + p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xdcbd))'], + stdout=subprocess.PIPE, env=env) + out = p.communicate()[0].strip() + self.assertEqual(out, b'\xbd') + + @unittest.skipUnless(test.support.FS_NONASCII, + 'requires OS support of non-ASCII encodings') + def test_ioencoding_nonascii(self): + env = dict(os.environ) + + env["PYTHONIOENCODING"] = "" + p = subprocess.Popen([sys.executable, "-c", + 'print(%a)' % test.support.FS_NONASCII], + stdout=subprocess.PIPE, env=env) + out = p.communicate()[0].strip() + self.assertEqual(out, os.fsencode(test.support.FS_NONASCII)) + @unittest.skipIf(sys.base_prefix != sys.prefix, 'Test is not venv-compatible') def test_executable(self): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -7,6 +7,8 @@ Core and Builtins ----------------- +- Issue #18818: The "encodingname" part of PYTHONIOENCODING is now optional. + Library ------- diff --git a/Python/pythonrun.c b/Python/pythonrun.c --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1056,7 +1056,7 @@ PyObject *std = NULL; int status = 0, fd; PyObject * encoding_attr; - char *encoding = NULL, *errors; + char *pythonioencoding = NULL, *encoding, *errors; /* Hack to avoid a nasty recursion issue when Python is invoked in verbose mode: pre-import the Latin-1 and UTF-8 codecs */ @@ -1088,19 +1088,23 @@ } Py_DECREF(wrapper); - encoding = Py_GETENV("PYTHONIOENCODING"); - errors = NULL; - if (encoding) { - encoding = _PyMem_Strdup(encoding); - if (encoding == NULL) { + pythonioencoding = Py_GETENV("PYTHONIOENCODING"); + encoding = errors = NULL; + if (pythonioencoding) { + pythonioencoding = _PyMem_Strdup(pythonioencoding); + if (pythonioencoding == NULL) { PyErr_NoMemory(); goto error; } - errors = strchr(encoding, ':'); + errors = strchr(pythonioencoding, ':'); if (errors) { *errors = '\0'; errors++; + if (!*errors) + errors = NULL; } + if (*pythonioencoding) + encoding = pythonioencoding; } /* Set sys.stdin */ @@ -1180,7 +1184,7 @@ status = -1; } - PyMem_Free(encoding); + PyMem_Free(pythonioencoding); Py_XDECREF(bimod); Py_XDECREF(iomod); return status; -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 14:30:21 2013 From: python-checkins at python.org (eli.bendersky) Date: Fri, 13 Sep 2013 14:30:21 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4OTQ1?= =?utf-8?q?=3A_Add_tests_for_tempfile_name_collision_handling=2E?= Message-ID: <3cbx5910yVz7LjR@mail.python.org> http://hg.python.org/cpython/rev/63f25483c8f6 changeset: 85674:63f25483c8f6 branch: 3.3 parent: 85671:1fcccbbe15e2 user: Eli Bendersky date: Fri Sep 13 05:28:20 2013 -0700 summary: Issue #18945: Add tests for tempfile name collision handling. Patch by Vlad Shcherbina files: Lib/test/test_tempfile.py | 85 ++++++++++++++++++++------ 1 files changed, 64 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -7,6 +7,7 @@ import sys import re import warnings +import contextlib import unittest from test import support @@ -255,6 +256,22 @@ self.assertTrue(a is b) + at contextlib.contextmanager +def _inside_empty_temp_dir(): + dir = tempfile.mkdtemp() + try: + with support.swap_attr(tempfile, 'tempdir', dir): + yield + finally: + support.rmtree(dir) + + +def _mock_candidate_names(*names): + return support.swap_attr(tempfile, + '_get_candidate_names', + lambda: iter(names)) + + class TestMkstempInner(BaseTestCase): """Test the internal function _mkstemp_inner.""" @@ -372,31 +389,36 @@ os.lseek(f.fd, 0, os.SEEK_SET) self.assertEqual(os.read(f.fd, 20), b"blat") + def default_mkstemp_inner(self): + return tempfile._mkstemp_inner(tempfile.gettempdir(), + tempfile.template, + '', + tempfile._bin_openflags) + + def test_collision_with_existing_file(self): + # _mkstemp_inner tries another name when a file with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + (fd1, name1) = self.default_mkstemp_inner() + os.close(fd1) + self.assertTrue(name1.endswith('aaa')) + + (fd2, name2) = self.default_mkstemp_inner() + os.close(fd2) + self.assertTrue(name2.endswith('bbb')) + def test_collision_with_existing_directory(self): # _mkstemp_inner tries another name when a directory with # the chosen name already exists - container_dir = tempfile.mkdtemp() - try: - def mock_get_candidate_names(): - return iter(['aaa', 'aaa', 'bbb']) - with support.swap_attr(tempfile, - '_get_candidate_names', - mock_get_candidate_names): - dir = tempfile.mkdtemp(dir=container_dir) - self.assertTrue(dir.endswith('aaa')) + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + dir = tempfile.mkdtemp() + self.assertTrue(dir.endswith('aaa')) - flags = tempfile._bin_openflags - (fd, name) = tempfile._mkstemp_inner(container_dir, - tempfile.template, - '', - flags) - try: - self.assertTrue(name.endswith('bbb')) - finally: - os.close(fd) - os.unlink(name) - finally: - support.rmtree(container_dir) + (fd, name) = self.default_mkstemp_inner() + os.close(fd) + self.assertTrue(name.endswith('bbb')) class TestGetTempPrefix(BaseTestCase): @@ -553,6 +575,27 @@ finally: os.rmdir(dir) + def test_collision_with_existing_file(self): + # mkdtemp tries another name when a file with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + file = tempfile.NamedTemporaryFile(delete=False) + file.close() + self.assertTrue(file.name.endswith('aaa')) + dir = tempfile.mkdtemp() + self.assertTrue(dir.endswith('bbb')) + + def test_collision_with_existing_directory(self): + # mkdtemp tries another name when a directory with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + dir1 = tempfile.mkdtemp() + self.assertTrue(dir1.endswith('aaa')) + dir2 = tempfile.mkdtemp() + self.assertTrue(dir2.endswith('bbb')) + class TestMktemp(BaseTestCase): """Test mktemp().""" -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 14:30:22 2013 From: python-checkins at python.org (eli.bendersky) Date: Fri, 13 Sep 2013 14:30:22 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2318945=3A_Add_tests_for_tempfile_name_collision_?= =?utf-8?q?handling=2E?= Message-ID: <3cbx5B3n2Sz7Ljx@mail.python.org> http://hg.python.org/cpython/rev/c902ceaf7825 changeset: 85675:c902ceaf7825 parent: 85673:c7fdb0637d0b parent: 85674:63f25483c8f6 user: Eli Bendersky date: Fri Sep 13 05:30:00 2013 -0700 summary: Issue #18945: Add tests for tempfile name collision handling. Patch by Vlad Shcherbina files: Lib/test/test_tempfile.py | 85 ++++++++++++++++++++------ 1 files changed, 64 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -7,6 +7,7 @@ import sys import re import warnings +import contextlib import unittest from test import support @@ -255,6 +256,22 @@ self.assertTrue(a is b) + at contextlib.contextmanager +def _inside_empty_temp_dir(): + dir = tempfile.mkdtemp() + try: + with support.swap_attr(tempfile, 'tempdir', dir): + yield + finally: + support.rmtree(dir) + + +def _mock_candidate_names(*names): + return support.swap_attr(tempfile, + '_get_candidate_names', + lambda: iter(names)) + + class TestMkstempInner(BaseTestCase): """Test the internal function _mkstemp_inner.""" @@ -373,31 +390,36 @@ os.lseek(f.fd, 0, os.SEEK_SET) self.assertEqual(os.read(f.fd, 20), b"blat") + def default_mkstemp_inner(self): + return tempfile._mkstemp_inner(tempfile.gettempdir(), + tempfile.template, + '', + tempfile._bin_openflags) + + def test_collision_with_existing_file(self): + # _mkstemp_inner tries another name when a file with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + (fd1, name1) = self.default_mkstemp_inner() + os.close(fd1) + self.assertTrue(name1.endswith('aaa')) + + (fd2, name2) = self.default_mkstemp_inner() + os.close(fd2) + self.assertTrue(name2.endswith('bbb')) + def test_collision_with_existing_directory(self): # _mkstemp_inner tries another name when a directory with # the chosen name already exists - container_dir = tempfile.mkdtemp() - try: - def mock_get_candidate_names(): - return iter(['aaa', 'aaa', 'bbb']) - with support.swap_attr(tempfile, - '_get_candidate_names', - mock_get_candidate_names): - dir = tempfile.mkdtemp(dir=container_dir) - self.assertTrue(dir.endswith('aaa')) + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + dir = tempfile.mkdtemp() + self.assertTrue(dir.endswith('aaa')) - flags = tempfile._bin_openflags - (fd, name) = tempfile._mkstemp_inner(container_dir, - tempfile.template, - '', - flags) - try: - self.assertTrue(name.endswith('bbb')) - finally: - os.close(fd) - os.unlink(name) - finally: - support.rmtree(container_dir) + (fd, name) = self.default_mkstemp_inner() + os.close(fd) + self.assertTrue(name.endswith('bbb')) class TestGetTempPrefix(BaseTestCase): @@ -554,6 +576,27 @@ finally: os.rmdir(dir) + def test_collision_with_existing_file(self): + # mkdtemp tries another name when a file with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + file = tempfile.NamedTemporaryFile(delete=False) + file.close() + self.assertTrue(file.name.endswith('aaa')) + dir = tempfile.mkdtemp() + self.assertTrue(dir.endswith('bbb')) + + def test_collision_with_existing_directory(self): + # mkdtemp tries another name when a directory with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + dir1 = tempfile.mkdtemp() + self.assertTrue(dir1.endswith('aaa')) + dir2 = tempfile.mkdtemp() + self.assertTrue(dir2.endswith('bbb')) + class TestMktemp(BaseTestCase): """Test mktemp().""" -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 15:28:13 2013 From: python-checkins at python.org (eli.bendersky) Date: Fri, 13 Sep 2013 15:28:13 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4OTk3?= =?utf-8?q?=3A_fix_ElementTree_crash_with_using_pickle_and_=5F=5Fgetstate?= =?utf-8?b?X18u?= Message-ID: <3cbyMx3CCmz7LjN@mail.python.org> http://hg.python.org/cpython/rev/39823ebfc731 changeset: 85676:39823ebfc731 branch: 3.3 parent: 85674:63f25483c8f6 user: Eli Bendersky date: Fri Sep 13 06:24:25 2013 -0700 summary: Issue #18997: fix ElementTree crash with using pickle and __getstate__. Based on report and initial patch from Germ?n M. Bravo files: Lib/test/test_xml_etree.py | 13 ++++++ Modules/_elementtree.c | 52 ++++++++++++------------- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1462,6 +1462,7 @@ ET.register_namespace('test10777', 'http://myuri/') ET.register_namespace('test10777', 'http://myuri/') + # -------------------------------------------------------------------- @@ -1542,6 +1543,18 @@ self.assertEqual(len(e2), 2) self.assertEqualElements(e, e2) + def test_pickle_issue18997(self): + for dumper, loader in product(self.modules, repeat=2): + XMLTEXT = """ + 4 + """ + e1 = dumper.fromstring(XMLTEXT) + if hasattr(e1, '__getstate__'): + self.assertEqual(e1.__getstate__()['tag'], 'group') + e2 = self.pickleRoundTrip(e1, 'xml.etree.ElementTree', dumper, loader) + self.assertEqual(e2.tag, 'group') + self.assertEqual(e2[0].tag, 'dogs') + class ElementTreeTypeTest(unittest.TestCase): def test_istype(self): diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -100,6 +100,18 @@ #define JOIN_SET(p, flag) ((void*) ((Py_uintptr_t) (JOIN_OBJ(p)) | (flag))) #define JOIN_OBJ(p) ((PyObject*) ((Py_uintptr_t) (p) & ~(Py_uintptr_t)1)) +/* Py_CLEAR for a PyObject* that uses a join flag. Pass the pointer by + * reference since this function sets it to NULL. +*/ +void _clear_joined_ptr(PyObject **p) +{ + if (*p) { + PyObject *tmp = JOIN_OBJ(*p); + *p = NULL; + Py_DECREF(tmp); + } +} + /* Types defined by this extension */ static PyTypeObject Element_Type; static PyTypeObject ElementIter_Type; @@ -606,22 +618,8 @@ element_gc_clear(ElementObject *self) { Py_CLEAR(self->tag); - - /* The following is like Py_CLEAR for self->text and self->tail, but - * written explicitily because the real pointers hide behind access - * macros. - */ - if (self->text) { - PyObject *tmp = JOIN_OBJ(self->text); - self->text = NULL; - Py_DECREF(tmp); - } - - if (self->tail) { - PyObject *tmp = JOIN_OBJ(self->tail); - self->tail = NULL; - Py_DECREF(tmp); - } + _clear_joined_ptr(&self->text); + _clear_joined_ptr(&self->tail); /* After dropping all references from extra, it's no longer valid anyway, * so fully deallocate it. @@ -859,15 +857,15 @@ PICKLED_TAG, self->tag, PICKLED_CHILDREN, children, PICKLED_ATTRIB, - PICKLED_TEXT, self->text, - PICKLED_TAIL, self->tail); + PICKLED_TEXT, JOIN_OBJ(self->text), + PICKLED_TAIL, JOIN_OBJ(self->tail)); else instancedict = Py_BuildValue("{sOsOsOsOsO}", PICKLED_TAG, self->tag, PICKLED_CHILDREN, children, PICKLED_ATTRIB, self->extra->attrib, - PICKLED_TEXT, self->text, - PICKLED_TAIL, self->tail); + PICKLED_TEXT, JOIN_OBJ(self->text), + PICKLED_TAIL, JOIN_OBJ(self->tail)); if (instancedict) { Py_DECREF(children); return instancedict; @@ -900,13 +898,13 @@ self->tag = tag; Py_INCREF(self->tag); - Py_CLEAR(self->text); - self->text = text ? text : Py_None; - Py_INCREF(self->text); - - Py_CLEAR(self->tail); - self->tail = tail ? tail : Py_None; - Py_INCREF(self->tail); + _clear_joined_ptr(&self->text); + self->text = text ? JOIN_SET(text, PyList_CheckExact(text)) : Py_None; + Py_INCREF(JOIN_OBJ(self->text)); + + _clear_joined_ptr(&self->tail); + self->tail = tail ? JOIN_SET(tail, PyList_CheckExact(tail)) : Py_None; + Py_INCREF(JOIN_OBJ(self->tail)); /* Handle ATTRIB and CHILDREN. */ if (!children && !attrib) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 15:28:14 2013 From: python-checkins at python.org (eli.bendersky) Date: Fri, 13 Sep 2013 15:28:14 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogQWRkIEdlcm3DoW4g?= =?utf-8?q?M=2E_Bravo_to_Misc/ACKS?= Message-ID: <3cbyMy4r5Yz7Ljd@mail.python.org> http://hg.python.org/cpython/rev/db01fc8c1a7b changeset: 85677:db01fc8c1a7b branch: 3.3 user: Eli Bendersky date: Fri Sep 13 06:24:59 2013 -0700 summary: Add Germ?n M. Bravo to Misc/ACKS files: Misc/ACKS | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -150,6 +150,7 @@ Georg Brandl Christopher Brannon Terrence Brannon +Germ?n M. Bravo Erik Bray Brian Brazil Dave Brennan -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 15:28:16 2013 From: python-checkins at python.org (eli.bendersky) Date: Fri, 13 Sep 2013 15:28:16 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Merge_for_Issue_=2318997=3A_Issue_=2318997=3A_fix_Elemen?= =?utf-8?q?tTree_crash_with_using_pickle?= Message-ID: <3cbyN00kgfz7LkG@mail.python.org> http://hg.python.org/cpython/rev/bda5a87df1c8 changeset: 85678:bda5a87df1c8 parent: 85675:c902ceaf7825 parent: 85677:db01fc8c1a7b user: Eli Bendersky date: Fri Sep 13 06:27:52 2013 -0700 summary: Merge for Issue #18997: Issue #18997: fix ElementTree crash with using pickle and __getstate__. files: Lib/test/test_xml_etree.py | 12 ++++++ Misc/ACKS | 1 + Modules/_elementtree.c | 52 ++++++++++++------------- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1697,6 +1697,18 @@ self.assertEqual(len(e2), 2) self.assertEqualElements(e, e2) + def test_pickle_issue18997(self): + for dumper, loader in product(self.modules, repeat=2): + XMLTEXT = """ + 4 + """ + e1 = dumper.fromstring(XMLTEXT) + if hasattr(e1, '__getstate__'): + self.assertEqual(e1.__getstate__()['tag'], 'group') + e2 = self.pickleRoundTrip(e1, 'xml.etree.ElementTree', dumper, loader) + self.assertEqual(e2.tag, 'group') + self.assertEqual(e2[0].tag, 'dogs') + class ElementTreeTypeTest(unittest.TestCase): def test_istype(self): diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -154,6 +154,7 @@ Georg Brandl Christopher Brannon Terrence Brannon +Germ?n M. Bravo Sven Brauch Erik Bray Brian Brazil diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -59,6 +59,18 @@ #define JOIN_SET(p, flag) ((void*) ((Py_uintptr_t) (JOIN_OBJ(p)) | (flag))) #define JOIN_OBJ(p) ((PyObject*) ((Py_uintptr_t) (p) & ~(Py_uintptr_t)1)) +/* Py_CLEAR for a PyObject* that uses a join flag. Pass the pointer by + * reference since this function sets it to NULL. +*/ +void _clear_joined_ptr(PyObject **p) +{ + if (*p) { + PyObject *tmp = JOIN_OBJ(*p); + *p = NULL; + Py_DECREF(tmp); + } +} + /* Types defined by this extension */ static PyTypeObject Element_Type; static PyTypeObject ElementIter_Type; @@ -613,22 +625,8 @@ element_gc_clear(ElementObject *self) { Py_CLEAR(self->tag); - - /* The following is like Py_CLEAR for self->text and self->tail, but - * written explicitily because the real pointers hide behind access - * macros. - */ - if (self->text) { - PyObject *tmp = JOIN_OBJ(self->text); - self->text = NULL; - Py_DECREF(tmp); - } - - if (self->tail) { - PyObject *tmp = JOIN_OBJ(self->tail); - self->tail = NULL; - Py_DECREF(tmp); - } + _clear_joined_ptr(&self->text); + _clear_joined_ptr(&self->tail); /* After dropping all references from extra, it's no longer valid anyway, * so fully deallocate it. @@ -866,15 +864,15 @@ PICKLED_TAG, self->tag, PICKLED_CHILDREN, children, PICKLED_ATTRIB, - PICKLED_TEXT, self->text, - PICKLED_TAIL, self->tail); + PICKLED_TEXT, JOIN_OBJ(self->text), + PICKLED_TAIL, JOIN_OBJ(self->tail)); else instancedict = Py_BuildValue("{sOsOsOsOsO}", PICKLED_TAG, self->tag, PICKLED_CHILDREN, children, PICKLED_ATTRIB, self->extra->attrib, - PICKLED_TEXT, self->text, - PICKLED_TAIL, self->tail); + PICKLED_TEXT, JOIN_OBJ(self->text), + PICKLED_TAIL, JOIN_OBJ(self->tail)); if (instancedict) { Py_DECREF(children); return instancedict; @@ -907,13 +905,13 @@ self->tag = tag; Py_INCREF(self->tag); - Py_CLEAR(self->text); - self->text = text ? text : Py_None; - Py_INCREF(self->text); - - Py_CLEAR(self->tail); - self->tail = tail ? tail : Py_None; - Py_INCREF(self->tail); + _clear_joined_ptr(&self->text); + self->text = text ? JOIN_SET(text, PyList_CheckExact(text)) : Py_None; + Py_INCREF(JOIN_OBJ(self->text)); + + _clear_joined_ptr(&self->tail); + self->tail = tail ? JOIN_SET(tail, PyList_CheckExact(tail)) : Py_None; + Py_INCREF(JOIN_OBJ(self->tail)); /* Handle ATTRIB and CHILDREN. */ if (!children && !attrib) -- Repository URL: http://hg.python.org/cpython From eric at trueblade.com Fri Sep 13 16:49:05 2013 From: eric at trueblade.com (Eric V. Smith) Date: Fri, 13 Sep 2013 10:49:05 -0400 Subject: [Python-checkins] cpython (merge 3.3 -> default): Fix http.server's request handling case on trailing '/'. In-Reply-To: <3cbpGS1zZSz7Lk4@mail.python.org> References: <3cbpGS1zZSz7Lk4@mail.python.org> Message-ID: <523325E1.70608@trueblade.com> On 9/13/2013 3:22 AM, senthil.kumaran wrote: > http://hg.python.org/cpython/rev/b85c9d2a5227 > changeset: 85672:b85c9d2a5227 > parent: 85668:66ec8431032d > parent: 85671:1fcccbbe15e2 > user: Senthil Kumaran > date: Fri Sep 13 00:22:45 2013 -0700 > summary: > Fix http.server's request handling case on trailing '/'. > > Patch contributed by Vajrasky Kok. Addresses Issue #17324 > + trailing_slash = True if path.rstrip().endswith('/') else False Wouldn't this be better just as: trailing_slash = path.rstrip().endswith('/') -- Eric. From python-checkins at python.org Fri Sep 13 19:54:21 2013 From: python-checkins at python.org (charles-francois.natali) Date: Fri, 13 Sep 2013 19:54:21 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2316201=3A_socket?= =?utf-8?q?=3A_Use_inet=5Fpton=28=29/inet=5Faddr=28=29_instead_of_ad-hoc_p?= =?utf-8?q?arsing_for?= Message-ID: <3cc4H16Mm8z7LjN@mail.python.org> http://hg.python.org/cpython/rev/540a9c69c2ea changeset: 85679:540a9c69c2ea user: Charles-Fran?ois Natali date: Fri Sep 13 19:53:08 2013 +0200 summary: Issue #16201: socket: Use inet_pton()/inet_addr() instead of ad-hoc parsing for numeric IP addresses. files: Lib/test/test_socket.py | 14 +++++ Modules/socketmodule.c | 73 +++++++++++++++++++++------- 2 files changed, 68 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -757,6 +757,20 @@ if not fqhn in all_host_names: self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) + def test_host_resolution(self): + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + self.assertRaises(OSError, socket.gethostbyname, addr) + self.assertRaises(OSError, socket.gethostbyaddr, addr) + + for addr in [support.HOST, '10.0.0.1', '255.255.255.255']: + self.assertEqual(socket.gethostbyname(addr), addr) + + # we don't test support.HOSTv6 because there's a chance it doesn't have + # a matching name entry (e.g. 'ip6-localhost') + for host in [support.HOST]: + self.assertIn(host, socket.gethostbyaddr(host)[2]) + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") def test_sethostname(self): diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -425,6 +425,10 @@ #define INVALID_SOCKET (-1) #endif +#ifndef INADDR_NONE +#define INADDR_NONE (-1) +#endif + /* XXX There's a problem here: *static* functions are not supposed to have a Py prefix (or use CapitalizedWords). Later... */ @@ -787,8 +791,6 @@ { struct addrinfo hints, *res; int error; - int d1, d2, d3, d4; - char ch; memset((void *) addr_ret, '\0', sizeof(*addr_ret)); if (name[0] == '\0') { @@ -837,7 +839,10 @@ freeaddrinfo(res); return siz; } - if (name[0] == '<' && strcmp(name, "") == 0) { + /* special-case broadcast - inet_addr() below can return INADDR_NONE for + * this */ + if (strcmp(name, "255.255.255.255") == 0 || + strcmp(name, "") == 0) { struct sockaddr_in *sin; if (af != AF_INET && af != AF_UNSPEC) { PyErr_SetString(PyExc_OSError, @@ -853,20 +858,53 @@ sin->sin_addr.s_addr = INADDR_BROADCAST; return sizeof(sin->sin_addr); } - if (sscanf(name, "%d.%d.%d.%d%c", &d1, &d2, &d3, &d4, &ch) == 4 && - 0 <= d1 && d1 <= 255 && 0 <= d2 && d2 <= 255 && - 0 <= d3 && d3 <= 255 && 0 <= d4 && d4 <= 255) { - struct sockaddr_in *sin; - sin = (struct sockaddr_in *)addr_ret; - sin->sin_addr.s_addr = htonl( - ((long) d1 << 24) | ((long) d2 << 16) | - ((long) d3 << 8) | ((long) d4 << 0)); - sin->sin_family = AF_INET; + + /* avoid a name resolution in case of numeric address */ +#ifdef HAVE_INET_PTON + /* check for an IPv4 address */ + if (af == AF_UNSPEC || af == AF_INET) { + struct sockaddr_in *sin = (struct sockaddr_in *)addr_ret; + memset(sin, 0, sizeof(*sin)); + if (inet_pton(AF_INET, name, &sin->sin_addr) > 0) { + sin->sin_family = AF_INET; #ifdef HAVE_SOCKADDR_SA_LEN - sin->sin_len = sizeof(*sin); -#endif - return 4; - } + sin->sin_len = sizeof(*sin); +#endif + return 4; + } + } +#ifdef ENABLE_IPV6 + /* check for an IPv6 address - if the address contains a scope ID, we + * fallback to getaddrinfo(), which can handle translation from interface + * name to interface index */ + if ((af == AF_UNSPEC || af == AF_INET6) && !strchr(name, '%')) { + struct sockaddr_in6 *sin = (struct sockaddr_in6 *)addr_ret; + memset(sin, 0, sizeof(*sin)); + if (inet_pton(AF_INET6, name, &sin->sin6_addr) > 0) { + sin->sin6_family = AF_INET6; +#ifdef HAVE_SOCKADDR_SA_LEN + sin->sin6_len = sizeof(*sin); +#endif + return 16; + } + } +#endif /* ENABLE_IPV6 */ +#else /* HAVE_INET_PTON */ + /* check for an IPv4 address */ + if (af == AF_INET || af == AF_UNSPEC) { + struct sockaddr_in *sin = (struct sockaddr_in *)addr_ret; + memset(sin, 0, sizeof(*sin)); + if ((sin->sin_addr.s_addr = inet_addr(name)) != INADDR_NONE) { + sin->sin_family = AF_INET; +#ifdef HAVE_SOCKADDR_SA_LEN + sin->sin_len = sizeof(*sin); +#endif + return 4; + } + } +#endif /* HAVE_INET_PTON */ + + /* perform a name resolution */ memset(&hints, 0, sizeof(hints)); hints.ai_family = af; Py_BEGIN_ALLOW_THREADS @@ -4896,9 +4934,6 @@ static PyObject* socket_inet_aton(PyObject *self, PyObject *args) { -#ifndef INADDR_NONE -#define INADDR_NONE (-1) -#endif #ifdef HAVE_INET_ATON struct in_addr buf; #endif -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 20:38:44 2013 From: python-checkins at python.org (antoine.pitrou) Date: Fri, 13 Sep 2013 20:38:44 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Adding_a_key-transforming_dic?= =?utf-8?q?tionary_to_collections?= Message-ID: <3cc5GD2jqsz7LjN@mail.python.org> http://hg.python.org/peps/rev/5e9b9068ff20 changeset: 5107:5e9b9068ff20 user: Antoine Pitrou date: Fri Sep 13 20:38:36 2013 +0200 summary: Adding a key-transforming dictionary to collections files: pep-0455.txt | 195 +++++++++++++++++++++++++++++++++++++++ 1 files changed, 195 insertions(+), 0 deletions(-) diff --git a/pep-0455.txt b/pep-0455.txt new file mode 100644 --- /dev/null +++ b/pep-0455.txt @@ -0,0 +1,195 @@ +PEP: 455 +Title: Adding a key-transforming dictionary to collections +Version: $Revision$ +Last-Modified: $Date$ +Author: Antoine Pitrou +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 13-Sep-2013 +Python-Version: 3.4 +Post-History: + + +Abstract +======== + +This PEP proposes a new data structure for the ``collections`` module, +called "TransformDict" in this PEP. This structure is a mutable mapping +which transforms the key using a given function when doing a lookup, but +retains the original key when reading. + + +Rationale +========= + +Numerous specialized versions of this pattern exist. The most common +is a case-insensitive case-preserving dict, i.e. a dict-like container +which matches keys in a case-insensitive fashion but retains the original +casing. It is a very common need in network programming, as many +protocols feature some arrays of "key / value" properties in their +messages, where the keys are textual strings whose casing isn't relevant. + +Another common request is an identity dict, where keys are matched +according to their respective id()s instead of normal matching. + +Both are instances of a more general pattern, where a given transformation +function is applied to keys when looking them up: that function being +``str.lower`` in the former example and the built-in ``id`` function in +the latter. + +(it can be said that the pattern *projects* keys from the user-visible +set onto the internal lookup set, hence this PEP's title) + + +Semantics +========= + +TransformDict is a ``MutableMapping`` implementation: it faithfully +implements the well-known API of mutable mappings, as ``dict`` itself +and other dict-like classes in the standard library. Therefore, this PEP +won't rehash the semantics of most TransformDict methods. + +The transformation function needn't be bijective, it can be strictly +surjective as in the case-insensitive example:: + + >>> d = TransformDict(str.lower) + >>> d['SomeKey'] = 5 + >>> d['somekey'] + 5 + >>> d['SOMEKEY'] + 5 + +TransformDict retains the first key used when creating an entry:: + + >>> d = TransformDict(str.lower) + >>> d['SomeKey'] = 1 + >>> d['somekey'] = 2 + >>> list(d.items()) + [('SomeKey', 2)] + +The original keys needn't be hashable, as long as the transformation +function returns a hashable one:: + + >>> d = TransformDict(id) + >>> l = [None] + >>> d[l] = 5 + >>> l in d + True + +Constructor +----------- + +As shown in the example aboves, creating a TransformDict requires passing +the key transformation function as the first argument (much like creating +a ``defaultdict`` requires passing the factory function as first argument). + +The constructor also takes other optional arguments which can be used +to initialize the TransformDict with certain key-value pairs. Those +optional arguments are the same as in the ``dict`` and ``defaultdict`` +constructors:: + + >>> d = TransformDict(str.lower, [('Foo': 1)], Bar=2) + >>> sorted(d.items()) + [('Bar', 2), ('Foo', 1)] + + +Alternative proposals and questions +=================================== + +Retaining the last original key +------------------------------- + +Most python-dev respondents found retaining the first user-supplied key +more intuitive than retaining the last. Also, it matches the dict +object's own behaviour when using different but equal keys:: + + >>> d = {} + >>> d[1] = 'hello' + >>> d[1.0] = 'world' + >>> d + {1: 'world'} + +Furthermore, explicitly retaining the last key in a first-key-retaining +scheme is still possible using the following approach:: + + d.pop(key, None) + d[key] = value + +while the converse (retaining the first key in a last-key-retaining +scheme) doesn't look possible without rewriting part of the container's +code. + +Using an encoder / decoder pair +------------------------------- + +Using a function pair isn't necessary, since the original key is retained +by the container. Moreover, an encoder / decoder pair would require the +transformation to be bijective, which prevents important use cases +like case-insensitive matching. + +Providing a transformation function for values +---------------------------------------------- + +Dictionary values are not used for lookup, their semantics are totally +irrelevant to the container's operation. Therefore, there is no point in +having both an "original" and a "transformed" value: the transformed +value wouldn't be used for anything. + +Providing a specialized container, not generic +---------------------------------------------- + +It was asked why we would provide the generic TransformDict construct +rather than a specialized case-insensitive dict variant. The answer +is that it's nearly as cheap (code-wise and performance-wise) to provide +the generic construct, and it can fill more use cases. + + +Implementation +============== + +A patch for the collections module is tracked on the bug tracker at +http://bugs.python.org/issue18986. + + +Existing work +============= + +Case-insensitive dicts are a popular request: + +* http://twistedmatrix.com/documents/current/api/twisted.python.util.InsensitiveDict.html +* https://mail.python.org/pipermail/python-list/2013-May/647243.html +* https://mail.python.org/pipermail/python-list/2005-April/296208.html +* https://mail.python.org/pipermail/python-list/2004-June/241748.html +* http://bugs.python.org/msg197376 +* http://stackoverflow.com/a/2082169 +* http://stackoverflow.com/a/3296782 +* http://code.activestate.com/recipes/66315-case-insensitive-dictionary/ +* https://gist.github.com/babakness/3901174 +* http://www.wikier.org/blog/key-insensitive-dictionary-in-python +* http://en.sharejs.com/python/14534 +* http://www.voidspace.org.uk/python/archive.shtml#caseless + +Identity dicts have been requested too: + +* https://mail.python.org/pipermail/python-ideas/2010-May/007235.html +* http://www.gossamer-threads.com/lists/python/python/209527 + +Python's own pickle module uses identity lookups for object +memoization: http://hg.python.org/cpython/file/0e70bf1f32a3/Lib/pickle.py#l234 + + +Copyright +========= + +This document has been placed in the public domain. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Fri Sep 13 21:20:41 2013 From: python-checkins at python.org (ezio.melotti) Date: Fri, 13 Sep 2013 21:20:41 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE4OTUxOiB1c2Ug?= =?utf-8?q?consistent_names_in_unittest_docs=2E?= Message-ID: <3cc6Bd53kzzSZJ@mail.python.org> http://hg.python.org/cpython/rev/03e94f9884ce changeset: 85680:03e94f9884ce branch: 3.3 parent: 85677:db01fc8c1a7b user: Ezio Melotti date: Fri Sep 13 22:17:40 2013 +0300 summary: #18951: use consistent names in unittest docs. files: Doc/library/unittest.rst | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -815,14 +815,14 @@ | :meth:`assertRaises(exc, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | | | ` | | | +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertRaisesRegex(exc, re, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 3.1 | - | ` | and the message matches *re* | | + | :meth:`assertRaisesRegex(exc, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 3.1 | + | ` | and the message matches regex *r* | | +---------------------------------------------------------+--------------------------------------+------------+ | :meth:`assertWarns(warn, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | | ` | | | +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertWarnsRegex(warn, re, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | - | ` | and the message matches *re* | | + | :meth:`assertWarnsRegex(warn, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | + | ` | and the message matches regex *r* | | +---------------------------------------------------------+--------------------------------------+------------+ .. method:: assertRaises(exception, callable, *args, **kwds) @@ -978,10 +978,10 @@ | :meth:`assertLessEqual(a, b) | ``a <= b`` | 3.1 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertRegex(s, re) | ``regex.search(s)`` | 3.1 | + | :meth:`assertRegex(s, r) | ``r.search(s)`` | 3.1 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotRegex(s, re) | ``not regex.search(s)`` | 3.2 | + | :meth:`assertNotRegex(s, r) | ``not r.search(s)`` | 3.2 | | ` | | | +---------------------------------------+--------------------------------+--------------+ | :meth:`assertCountEqual(a, b) | *a* and *b* have the same | 3.2 | -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 21:20:42 2013 From: python-checkins at python.org (ezio.melotti) Date: Fri, 13 Sep 2013 21:20:42 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?b?KTogIzE4OTUxOiBtZXJnZSB3aXRoIDMuMy4=?= Message-ID: <3cc6Bf6zsYz7LjY@mail.python.org> http://hg.python.org/cpython/rev/eb332e3dc303 changeset: 85681:eb332e3dc303 parent: 85679:540a9c69c2ea parent: 85680:03e94f9884ce user: Ezio Melotti date: Fri Sep 13 22:18:02 2013 +0300 summary: #18951: merge with 3.3. files: Doc/library/unittest.rst | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -892,14 +892,14 @@ | :meth:`assertRaises(exc, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | | | ` | | | +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertRaisesRegex(exc, re, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 3.1 | - | ` | and the message matches *re* | | + | :meth:`assertRaisesRegex(exc, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 3.1 | + | ` | and the message matches regex *r* | | +---------------------------------------------------------+--------------------------------------+------------+ | :meth:`assertWarns(warn, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | | ` | | | +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertWarnsRegex(warn, re, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | - | ` | and the message matches *re* | | + | :meth:`assertWarnsRegex(warn, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | + | ` | and the message matches regex *r* | | +---------------------------------------------------------+--------------------------------------+------------+ .. method:: assertRaises(exception, callable, *args, **kwds) @@ -1055,10 +1055,10 @@ | :meth:`assertLessEqual(a, b) | ``a <= b`` | 3.1 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertRegex(s, re) | ``regex.search(s)`` | 3.1 | + | :meth:`assertRegex(s, r) | ``r.search(s)`` | 3.1 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotRegex(s, re) | ``not regex.search(s)`` | 3.2 | + | :meth:`assertNotRegex(s, r) | ``not r.search(s)`` | 3.2 | | ` | | | +---------------------------------------+--------------------------------+--------------+ | :meth:`assertCountEqual(a, b) | *a* and *b* have the same | 3.2 | -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 21:20:44 2013 From: python-checkins at python.org (ezio.melotti) Date: Fri, 13 Sep 2013 21:20:44 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogIzE4OTUxOiB1c2Ug?= =?utf-8?q?consistent_names_in_unittest_docs=2E?= Message-ID: <3cc6Bh224cz7Ljh@mail.python.org> http://hg.python.org/cpython/rev/6d8ff8c57d38 changeset: 85682:6d8ff8c57d38 branch: 2.7 parent: 85670:a58b620e4dc9 user: Ezio Melotti date: Fri Sep 13 22:17:40 2013 +0300 summary: #18951: use consistent names in unittest docs. files: Doc/library/unittest.rst | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -916,8 +916,8 @@ | :meth:`assertRaises(exc, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | | | ` | | | +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertRaisesRegexp(exc, re, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 2.7 | - | ` | and the message matches *re* | | + | :meth:`assertRaisesRegexp(exc, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 2.7 | + | ` | and the message matches regex *r* | | +---------------------------------------------------------+--------------------------------------+------------+ .. method:: assertRaises(exception, callable, *args, **kwds) @@ -993,10 +993,10 @@ | :meth:`assertLessEqual(a, b) | ``a <= b`` | 2.7 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertRegexpMatches(s, re) | ``regex.search(s)`` | 2.7 | + | :meth:`assertRegexpMatches(s, r) | ``r.search(s)`` | 2.7 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotRegexpMatches(s, re) | ``not regex.search(s)`` | 2.7 | + | :meth:`assertNotRegexpMatches(s, r) | ``not r.search(s)`` | 2.7 | | ` | | | +---------------------------------------+--------------------------------+--------------+ | :meth:`assertItemsEqual(a, b) | sorted(a) == sorted(b) and | 2.7 | -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 21:40:43 2013 From: python-checkins at python.org (antoine.pitrou) Date: Fri, 13 Sep 2013 21:40:43 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Address_Serhiy=27s_comments?= Message-ID: <3cc6dl22jPzSjT@mail.python.org> http://hg.python.org/peps/rev/4c4adcc0e3b2 changeset: 5108:4c4adcc0e3b2 user: Antoine Pitrou date: Fri Sep 13 21:40:34 2013 +0200 summary: Address Serhiy's comments files: pep-0455.txt | 19 ++++++++++--------- 1 files changed, 10 insertions(+), 9 deletions(-) diff --git a/pep-0455.txt b/pep-0455.txt --- a/pep-0455.txt +++ b/pep-0455.txt @@ -35,11 +35,11 @@ Both are instances of a more general pattern, where a given transformation function is applied to keys when looking them up: that function being -``str.lower`` in the former example and the built-in ``id`` function in -the latter. +``str.lower`` or ``str.casefold`` in the former example and the built-in +``id`` function in the latter. -(it can be said that the pattern *projects* keys from the user-visible -set onto the internal lookup set, hence this PEP's title) +(it could be said that the pattern *projects* keys from the user-visible +set onto the internal lookup set) Semantics @@ -53,7 +53,7 @@ The transformation function needn't be bijective, it can be strictly surjective as in the case-insensitive example:: - >>> d = TransformDict(str.lower) + >>> d = TransformDict(str.casefold) >>> d['SomeKey'] = 5 >>> d['somekey'] 5 @@ -62,7 +62,7 @@ TransformDict retains the first key used when creating an entry:: - >>> d = TransformDict(str.lower) + >>> d = TransformDict(str.casefold) >>> d['SomeKey'] = 1 >>> d['somekey'] = 2 >>> list(d.items()) @@ -89,7 +89,7 @@ optional arguments are the same as in the ``dict`` and ``defaultdict`` constructors:: - >>> d = TransformDict(str.lower, [('Foo': 1)], Bar=2) + >>> d = TransformDict(str.casefold, [('Foo', 1)], Bar=2) >>> sorted(d.items()) [('Bar', 2), ('Foo', 1)] @@ -175,8 +175,9 @@ * https://mail.python.org/pipermail/python-ideas/2010-May/007235.html * http://www.gossamer-threads.com/lists/python/python/209527 -Python's own pickle module uses identity lookups for object -memoization: http://hg.python.org/cpython/file/0e70bf1f32a3/Lib/pickle.py#l234 +Several modules in the standard library use identity lookups for object +memoization, for example ``pickle``, ``json``, ``copy``, ``cProfile``, +``doctest`` and ``_threading_local``. Copyright -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Fri Sep 13 22:01:28 2013 From: python-checkins at python.org (ezio.melotti) Date: Fri, 13 Sep 2013 22:01:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Fix_markup_in_?= =?utf-8?q?email=2Eparser_docs=2E?= Message-ID: <3cc75h0kKxz7LkR@mail.python.org> http://hg.python.org/cpython/rev/eaae175f1ff7 changeset: 85683:eaae175f1ff7 branch: 2.7 user: Ezio Melotti date: Fri Sep 13 22:55:08 2013 +0300 summary: Fix markup in email.parser docs. files: Doc/library/email.parser.rst | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -168,7 +168,7 @@ Return a message object structure from a string. This is exactly equivalent to ``Parser().parsestr(s)``. Optional *_class* and *strict* are interpreted as - with the :class:``~email.parser.Parser` class constructor. + with the :class:`~email.parser.Parser` class constructor. .. versionchanged:: 2.2.2 The *strict* flag was added. @@ -178,7 +178,7 @@ Return a message object structure tree from an open file object. This is exactly equivalent to ``Parser().parse(fp)``. Optional *_class* and *strict* - are interpreted as with the :class:``~email.parser.Parser` class constructor. + are interpreted as with the :class:`~email.parser.Parser` class constructor. .. versionchanged:: 2.2.2 The *strict* flag was added. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 22:43:38 2013 From: python-checkins at python.org (antoine.pitrou) Date: Fri, 13 Sep 2013 22:43:38 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Address_more_comments?= Message-ID: <3cc82L1fdvz7LjT@mail.python.org> http://hg.python.org/peps/rev/3663de7135a2 changeset: 5109:3663de7135a2 user: Antoine Pitrou date: Fri Sep 13 22:43:34 2013 +0200 summary: Address more comments files: pep-0455.txt | 23 ++++++++++++++++++++++- 1 files changed, 22 insertions(+), 1 deletions(-) diff --git a/pep-0455.txt b/pep-0455.txt --- a/pep-0455.txt +++ b/pep-0455.txt @@ -28,7 +28,9 @@ which matches keys in a case-insensitive fashion but retains the original casing. It is a very common need in network programming, as many protocols feature some arrays of "key / value" properties in their -messages, where the keys are textual strings whose casing isn't relevant. +messages, where the keys are textual strings whose case is specified to +be ignored on receipt but by either specification or custom is to be +preserved or non-trivially canonicalized when retransmitted. Another common request is an identity dict, where keys are matched according to their respective id()s instead of normal matching. @@ -144,6 +146,25 @@ is that it's nearly as cheap (code-wise and performance-wise) to provide the generic construct, and it can fill more use cases. +Other constructor patterns +-------------------------- + +Two other constructor patterns were proposed by Serhiy Storchaka: + +* A type factory scheme:: + + d = TransformDict(str.casefold)(Foo=1) + +* A subclassing scheme:: + + class CaseInsensitiveDict(TransformDict): + __transform__ = str.casefold + + d = CaseInsensitiveDict(Foo=1) + +While both approaches can be defended, they don't follow established +practices in the standard library, and therefore were rejected. + Implementation ============== -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Fri Sep 13 22:52:34 2013 From: python-checkins at python.org (brett.cannon) Date: Fri, 13 Sep 2013 22:52:34 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318955=3A_clarify_?= =?utf-8?q?what_is_removed_by_importlib=2Eutil=2Emodule=5Ffor=5Floader=2E?= Message-ID: <3cc8Df2JhPz7LkS@mail.python.org> http://hg.python.org/cpython/rev/ecc18ec4a12e changeset: 85684:ecc18ec4a12e parent: 85681:eb332e3dc303 user: Brett Cannon date: Fri Sep 13 16:52:19 2013 -0400 summary: Issue #18955: clarify what is removed by importlib.util.module_for_loader. files: Doc/library/importlib.rst | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -975,9 +975,9 @@ reloading. If an exception is raised by the decorated method and a module was added to - :data:`sys.modules` it will be removed to prevent a partially initialized - module from being in left in :data:`sys.modules`. If the module was already - in :data:`sys.modules` then it is left alone. + :data:`sys.modules`, then the module will be removed to prevent a partially + initialized module from being in left in :data:`sys.modules`. If the module + was already in :data:`sys.modules` then it is left alone. .. versionchanged:: 3.3 :attr:`__loader__` and :attr:`__package__` are automatically set -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 22:54:52 2013 From: python-checkins at python.org (ezio.melotti) Date: Fri, 13 Sep 2013 22:54:52 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogIzE4OTgxOiBmaXgg?= =?utf-8?q?a_typo_in_a_comment_=28noticed_by_Anoop_Thomas_Mathew=29=2E?= Message-ID: <3cc8HJ6ZW0zMGY@mail.python.org> http://hg.python.org/cpython/rev/b90ba60c5029 changeset: 85685:b90ba60c5029 branch: 2.7 parent: 85683:eaae175f1ff7 user: Ezio Melotti date: Fri Sep 13 23:52:12 2013 +0300 summary: #18981: fix a typo in a comment (noticed by Anoop Thomas Mathew). files: Lib/ctypes/test/test_integers.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/ctypes/test/test_integers.py b/Lib/ctypes/test/test_integers.py --- a/Lib/ctypes/test/test_integers.py +++ b/Lib/ctypes/test/test_integers.py @@ -1,4 +1,4 @@ -# superseeded by test_numbers.py +# superseded by test_numbers.py import unittest if __name__ == '__main__': -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 22:54:54 2013 From: python-checkins at python.org (ezio.melotti) Date: Fri, 13 Sep 2013 22:54:54 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE4OTgxOiBmaXgg?= =?utf-8?q?a_typo_in_a_comment_=28noticed_by_Anoop_Thomas_Mathew=29=2E?= Message-ID: <3cc8HL1ns9z7LjM@mail.python.org> http://hg.python.org/cpython/rev/9fcb9deb70f5 changeset: 85686:9fcb9deb70f5 branch: 3.3 parent: 85680:03e94f9884ce user: Ezio Melotti date: Fri Sep 13 23:52:12 2013 +0300 summary: #18981: fix a typo in a comment (noticed by Anoop Thomas Mathew). files: Lib/ctypes/test/test_integers.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/ctypes/test/test_integers.py b/Lib/ctypes/test/test_integers.py --- a/Lib/ctypes/test/test_integers.py +++ b/Lib/ctypes/test/test_integers.py @@ -1,4 +1,4 @@ -# superseeded by test_numbers.py +# superseded by test_numbers.py import unittest if __name__ == '__main__': -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 22:54:55 2013 From: python-checkins at python.org (ezio.melotti) Date: Fri, 13 Sep 2013 22:54:55 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?b?KTogIzE4OTgxOiBtZXJnZSB3aXRoIDMuMy4=?= Message-ID: <3cc8HM3cZBz7LjT@mail.python.org> http://hg.python.org/cpython/rev/dae3e64e1743 changeset: 85687:dae3e64e1743 parent: 85684:ecc18ec4a12e parent: 85686:9fcb9deb70f5 user: Ezio Melotti date: Fri Sep 13 23:54:41 2013 +0300 summary: #18981: merge with 3.3. files: Lib/ctypes/test/test_integers.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/ctypes/test/test_integers.py b/Lib/ctypes/test/test_integers.py --- a/Lib/ctypes/test/test_integers.py +++ b/Lib/ctypes/test/test_integers.py @@ -1,4 +1,4 @@ -# superseeded by test_numbers.py +# superseded by test_numbers.py import unittest if __name__ == '__main__': -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 23:48:38 2013 From: python-checkins at python.org (antoine.pitrou) Date: Fri, 13 Sep 2013 23:48:38 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Propose_a_getitem=28=29_metho?= =?utf-8?q?d?= Message-ID: <3cc9TL0n6gz7LjT@mail.python.org> http://hg.python.org/peps/rev/4a6b87d0ee91 changeset: 5110:4a6b87d0ee91 user: Antoine Pitrou date: Fri Sep 13 23:48:33 2013 +0200 summary: Propose a getitem() method files: pep-0455.txt | 17 +++++++++++++++++ 1 files changed, 17 insertions(+), 0 deletions(-) diff --git a/pep-0455.txt b/pep-0455.txt --- a/pep-0455.txt +++ b/pep-0455.txt @@ -95,6 +95,23 @@ >>> sorted(d.items()) [('Bar', 2), ('Foo', 1)] +Getting the original key +------------------------ + +TransformDict also features a lookup method returning the stored key +together with the corresponding value:: + + >>> d = TransformDict(str.casefold, {'Foo': 1}) + >>> d.getitem('FOO') + ('Foo', 1) + >>> d.getitem('bar') + Traceback (most recent call last): + File "", line 1, in + KeyError: 'bar' + +The method name ``getitem()`` mirrors the standard ``popitem()`` method +on mutable mappings. + Alternative proposals and questions =================================== -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Fri Sep 13 23:54:08 2013 From: python-checkins at python.org (antoine.pitrou) Date: Fri, 13 Sep 2013 23:54:08 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2319013=3A_add_unit?= =?utf-8?q?test=2Emain=28=29_epilogs_to_unittest=27s_own_test_modules?= Message-ID: <3cc9bh3P1hz7LjY@mail.python.org> http://hg.python.org/cpython/rev/1d08ce952592 changeset: 85688:1d08ce952592 user: Antoine Pitrou date: Fri Sep 13 23:52:46 2013 +0200 summary: Issue #19013: add unittest.main() epilogs to unittest's own test modules files: Lib/unittest/test/test_assertions.py | 4 ++++ Lib/unittest/test/test_break.py | 4 ++++ Lib/unittest/test/test_case.py | 4 ++++ Lib/unittest/test/test_functiontestcase.py | 4 ++++ Lib/unittest/test/test_loader.py | 4 ++++ Lib/unittest/test/test_runner.py | 4 ++++ Lib/unittest/test/test_setups.py | 1 + Lib/unittest/test/test_skipping.py | 4 ++++ 8 files changed, 29 insertions(+), 0 deletions(-) diff --git a/Lib/unittest/test/test_assertions.py b/Lib/unittest/test/test_assertions.py --- a/Lib/unittest/test/test_assertions.py +++ b/Lib/unittest/test/test_assertions.py @@ -361,3 +361,7 @@ ['^"regex" does not match "foo"$', '^oops$', '^"regex" does not match "foo"$', '^"regex" does not match "foo" : oops$']) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/unittest/test/test_break.py b/Lib/unittest/test/test_break.py --- a/Lib/unittest/test/test_break.py +++ b/Lib/unittest/test/test_break.py @@ -282,3 +282,7 @@ "if threads have been used") class TestBreakSignalDefault(TestBreak): int_handler = signal.SIG_DFL + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py --- a/Lib/unittest/test/test_case.py +++ b/Lib/unittest/test/test_case.py @@ -1405,3 +1405,7 @@ with support.disable_gc(): del case self.assertFalse(wr()) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/unittest/test/test_functiontestcase.py b/Lib/unittest/test/test_functiontestcase.py --- a/Lib/unittest/test/test_functiontestcase.py +++ b/Lib/unittest/test/test_functiontestcase.py @@ -142,3 +142,7 @@ test = unittest.FunctionTestCase(lambda: None, description=desc) self.assertEqual(test.shortDescription(), "this tests foo") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py --- a/Lib/unittest/test/test_loader.py +++ b/Lib/unittest/test/test_loader.py @@ -1306,3 +1306,7 @@ def test_suiteClass__default_value(self): loader = unittest.TestLoader() self.assertTrue(loader.suiteClass is unittest.TestSuite) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -339,3 +339,7 @@ f = io.StringIO() runner = unittest.TextTestRunner(f) self.assertTrue(runner.stream.stream is f) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/unittest/test/test_setups.py b/Lib/unittest/test/test_setups.py --- a/Lib/unittest/test/test_setups.py +++ b/Lib/unittest/test/test_setups.py @@ -501,5 +501,6 @@ with self.assertRaisesRegex(Exception, msg): suite.debug() + if __name__ == '__main__': unittest.main() diff --git a/Lib/unittest/test/test_skipping.py b/Lib/unittest/test/test_skipping.py --- a/Lib/unittest/test/test_skipping.py +++ b/Lib/unittest/test/test_skipping.py @@ -221,3 +221,7 @@ suite = unittest.TestSuite([test]) suite.run(result) self.assertEqual(result.skipped, [(test, "testing")]) + + +if __name__ == "__main__": + unittest.main() -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 13 23:54:09 2013 From: python-checkins at python.org (antoine.pitrou) Date: Fri, 13 Sep 2013 23:54:09 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2319013=3A_add_unit?= =?utf-8?q?test=2Emain=28=29_epilogs_to_unittest=2Emock=27s_own_test_modul?= =?utf-8?q?es?= Message-ID: <3cc9bj55mgz7Lk0@mail.python.org> http://hg.python.org/cpython/rev/f81a53580c45 changeset: 85689:f81a53580c45 user: Antoine Pitrou date: Fri Sep 13 23:54:01 2013 +0200 summary: Issue #19013: add unittest.main() epilogs to unittest.mock's own test modules files: Lib/unittest/test/testmock/testcallable.py | 4 ++++ Lib/unittest/test/testmock/testmock.py | 1 - Lib/unittest/test/testmock/testpatch.py | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/unittest/test/testmock/testcallable.py b/Lib/unittest/test/testmock/testcallable.py --- a/Lib/unittest/test/testmock/testcallable.py +++ b/Lib/unittest/test/testmock/testcallable.py @@ -145,3 +145,7 @@ mock.wibble.assert_called_once_with() self.assertRaises(TypeError, mock.wibble, 'some', 'args') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py --- a/Lib/unittest/test/testmock/testmock.py +++ b/Lib/unittest/test/testmock/testmock.py @@ -1399,6 +1399,5 @@ mock.foo - if __name__ == '__main__': unittest.main() diff --git a/Lib/unittest/test/testmock/testpatch.py b/Lib/unittest/test/testmock/testpatch.py --- a/Lib/unittest/test/testmock/testpatch.py +++ b/Lib/unittest/test/testmock/testpatch.py @@ -1780,6 +1780,5 @@ self.assertIs(os.path, path) - if __name__ == '__main__': unittest.main() -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 00:38:03 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sat, 14 Sep 2013 00:38:03 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2319013=3A_add_a_?= =?utf-8?q?=5F=5Fmain=5F=5F_to_unittest=2Etest_to_ease_CLI_invocation?= Message-ID: <3ccBZM3QvTz7Ljn@mail.python.org> http://hg.python.org/cpython/rev/bbcdcdb06c66 changeset: 85690:bbcdcdb06c66 user: Antoine Pitrou date: Sat Sep 14 00:37:18 2013 +0200 summary: Issue #19013: add a __main__ to unittest.test to ease CLI invocation files: Lib/unittest/test/__main__.py | 18 ++++++++++++++++++ 1 files changed, 18 insertions(+), 0 deletions(-) diff --git a/Lib/unittest/test/__main__.py b/Lib/unittest/test/__main__.py new file mode 100644 --- /dev/null +++ b/Lib/unittest/test/__main__.py @@ -0,0 +1,18 @@ +import os +import unittest + + +def load_tests(loader, standard_tests, pattern): + # top level directory cached on loader instance + this_dir = os.path.dirname(__file__) + pattern = pattern or "test_*.py" + # We are inside unittest.test, so the top-level is two notches up + top_level_dir = os.path.dirname(os.path.dirname(this_dir)) + package_tests = loader.discover(start_dir=this_dir, pattern=pattern, + top_level_dir=top_level_dir) + standard_tests.addTests(package_tests) + return standard_tests + + +if __name__ == '__main__': + unittest.main() -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 00:39:22 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sat, 14 Sep 2013 00:39:22 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2319013=3A_add_a_?= =?utf-8?q?=5F=5Fmain=5F=5F_to_unittest=2Etest=2Etestmock_to_ease_CLI_invo?= =?utf-8?q?cation?= Message-ID: <3ccBbt0LSbz7Ljn@mail.python.org> http://hg.python.org/cpython/rev/c2b926c0d0c2 changeset: 85691:c2b926c0d0c2 user: Antoine Pitrou date: Sat Sep 14 00:39:15 2013 +0200 summary: Issue #19013: add a __main__ to unittest.test.testmock to ease CLI invocation files: Lib/unittest/test/testmock/__main__.py | 18 ++++++++++++++ 1 files changed, 18 insertions(+), 0 deletions(-) diff --git a/Lib/unittest/test/testmock/__main__.py b/Lib/unittest/test/testmock/__main__.py new file mode 100644 --- /dev/null +++ b/Lib/unittest/test/testmock/__main__.py @@ -0,0 +1,18 @@ +import os +import unittest + + +def load_tests(loader, standard_tests, pattern): + # top level directory cached on loader instance + this_dir = os.path.dirname(__file__) + pattern = pattern or "test*.py" + # We are inside unittest.test, so the top-level is two notches up + top_level_dir = os.path.dirname(os.path.dirname(os.path.dirname(this_dir))) + package_tests = loader.discover(start_dir=this_dir, pattern=pattern, + top_level_dir=top_level_dir) + standard_tests.addTests(package_tests) + return standard_tests + + +if __name__ == '__main__': + unittest.main() -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 00:40:54 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sat, 14 Sep 2013 00:40:54 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Adjust_comment?= Message-ID: <3ccBdf4gPJz7Ljn@mail.python.org> http://hg.python.org/cpython/rev/3894e17e59c4 changeset: 85692:3894e17e59c4 user: Antoine Pitrou date: Sat Sep 14 00:40:46 2013 +0200 summary: Adjust comment files: Lib/unittest/test/testmock/__main__.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/unittest/test/testmock/__main__.py b/Lib/unittest/test/testmock/__main__.py --- a/Lib/unittest/test/testmock/__main__.py +++ b/Lib/unittest/test/testmock/__main__.py @@ -6,7 +6,7 @@ # top level directory cached on loader instance this_dir = os.path.dirname(__file__) pattern = pattern or "test*.py" - # We are inside unittest.test, so the top-level is two notches up + # We are inside unittest.test.testmock, so the top-level is three notches up top_level_dir = os.path.dirname(os.path.dirname(os.path.dirname(this_dir))) package_tests = loader.discover(start_dir=this_dir, pattern=pattern, top_level_dir=top_level_dir) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 03:38:00 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sat, 14 Sep 2013 03:38:00 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Clarify_surjectivity=2C_for_d?= =?utf-8?q?iversity=27s_sake_=28not_the_mailing-list=2C_though=29?= Message-ID: <3ccGZ01wPNz7Ljn@mail.python.org> http://hg.python.org/peps/rev/9f8656f36392 changeset: 5111:9f8656f36392 user: Antoine Pitrou date: Sat Sep 14 03:37:52 2013 +0200 summary: Clarify surjectivity, for diversity's sake (not the mailing-list, though) files: pep-0455.txt | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/pep-0455.txt b/pep-0455.txt --- a/pep-0455.txt +++ b/pep-0455.txt @@ -53,7 +53,8 @@ won't rehash the semantics of most TransformDict methods. The transformation function needn't be bijective, it can be strictly -surjective as in the case-insensitive example:: +surjective as in the case-insensitive example (in other words, different +keys can lookup the same value):: >>> d = TransformDict(str.casefold) >>> d['SomeKey'] = 5 -- Repository URL: http://hg.python.org/peps From solipsis at pitrou.net Sat Sep 14 06:29:13 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sat, 14 Sep 2013 06:29:13 +0200 Subject: [Python-checkins] Daily reference leaks (3894e17e59c4): sum=0 Message-ID: results for 3894e17e59c4 on branch "default" -------------------------------------------- Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogoRFlQR', '-x'] From python-checkins at python.org Sat Sep 14 07:08:16 2013 From: python-checkins at python.org (georg.brandl) Date: Sat, 14 Sep 2013 07:08:16 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_small_PEP_101_updates?= Message-ID: <3ccMDc73wHz7LjY@mail.python.org> http://hg.python.org/peps/rev/29ea1675720c changeset: 5112:29ea1675720c parent: 4889:26aff864a750 user: Georg Brandl date: Wed May 15 19:41:19 2013 +0200 summary: small PEP 101 updates files: pep-0101.txt | 28 ++++++++++++++++------------ 1 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pep-0101.txt b/pep-0101.txt --- a/pep-0101.txt +++ b/pep-0101.txt @@ -380,13 +380,13 @@ directory, You'll move everything in there when the final release comes out. - ___ Move the release .tgz, tar.bz2, and .msi files into place, as well - as the .asc GPG signature files. + ___ Move the release .tgz, tar.bz2, .tar.xz, .dmg and .msi files into + place, as well as the .asc GPG signature files. Make sure they are world readable. They should also be group writable, and group-owned by webmaster. - ___ md5sum the files and make sure they got uploaded intact. + ___ Use `gpg --verify` to make sure they got uploaded intact. ___ If this is a final release: Move the doc zips and tarballs to /data/ftp.python.org/pub/python/doc/X.Y.Z creating the directory @@ -400,6 +400,9 @@ If it is a release of a security-fix-only version, tell the DE to build a version with the "version switcher" and put it there. + ___ For branches out of maintenance: adapt the symlink in + /docs.python.org/release/X.Y to point to the recent version. + ___ Let the DE check if the docs are built and work all right. ___ If this is a major release: Tell the DE to adapt redirects for @@ -430,9 +433,8 @@ permissions. It's insane for you not to have it. I'm not going to go into the details of building the site or pushing it - live. Plenty of people on pydotorg can help you, and there's a good README - once you get the branch. All the directories below are named relative to - the data subdirectory unless otherwise noted. + live. All the directories below are named relative to the data subdirectory + unless otherwise noted. This page will probably come in handy: @@ -466,8 +468,8 @@ ___ update the download page, editing `download/content.ht` - ___ edit the previous release's last release content.ht page to point to - the new release. + ___ for X.Y.Z, edit all the previous X.Y releases' content.ht page to + point to the new release. ___ update `doc/content.ht` to indicate the new current documentation version, and remove the current version from any 'in development' @@ -475,7 +477,7 @@ ___ Add the new version to `doc/versions/content.ht`. - ___ Edit download/releases/content.ht to update the version numbers for + ___ Edit `download/releases/content.ht` to update the version numbers for this release. There are a bunch of places you need to touch: ___ The subdirectory name as the first element in the Nav rows. @@ -484,14 +486,16 @@ ___ Update the version specific pages. - ___ cd to download/releases/X.Y.Z + ___ cd to `download/releases/X.Y.Z` ___ Edit the version numbers in content.ht ___ Comment out the link to the CHM file if this is not a final, remove the comment if it is. ___ Update the md5 checksums - Note, you don't have to copy the actual .tgz or tar.bz2 tarballs into - this directory because they only live on dinsdale in the ftp directory. + ___ Update the license in `download/releases/X.Y.Z/license` + + Note, you don't have to copy any release files into this directory; + they only live on dinsdale in the ftp directory. ___ When everything looks good, `svn commit` in the data directory. This will trigger the live site to update itself, and at that point the -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sat Sep 14 07:08:18 2013 From: python-checkins at python.org (georg.brandl) Date: Sat, 14 Sep 2013 07:08:18 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps_=28merge_default_-=3E_default=29?= =?utf-8?q?=3A_merge?= Message-ID: <3ccMDf2p9Nz7Ljr@mail.python.org> http://hg.python.org/peps/rev/cbdbff7cf367 changeset: 5113:cbdbff7cf367 parent: 5111:9f8656f36392 parent: 5112:29ea1675720c user: Georg Brandl date: Sat Sep 14 07:08:32 2013 +0200 summary: merge files: pep-0101.txt | 28 ++++++++++++++++------------ 1 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pep-0101.txt b/pep-0101.txt --- a/pep-0101.txt +++ b/pep-0101.txt @@ -406,13 +406,13 @@ directory, You'll move everything in there when the final release comes out. - ___ Move the release .tgz, tar.bz2, and .msi files into place, as well - as the .asc GPG signature files. + ___ Move the release .tgz, tar.bz2, .tar.xz, .dmg and .msi files into + place, as well as the .asc GPG signature files. Make sure they are world readable. They should also be group writable, and group-owned by webmaster. - ___ md5sum the files and make sure they got uploaded intact. + ___ Use `gpg --verify` to make sure they got uploaded intact. ___ If this is a final release: Move the doc zips and tarballs to /data/ftp.python.org/pub/python/doc/X.Y.Z creating the directory @@ -426,6 +426,9 @@ If it is a release of a security-fix-only version, tell the DE to build a version with the "version switcher" and put it there. + ___ For branches out of maintenance: adapt the symlink in + /docs.python.org/release/X.Y to point to the recent version. + ___ Let the DE check if the docs are built and work all right. ___ If this is a major release: Tell the DE to adapt redirects for @@ -456,9 +459,8 @@ permissions. It's insane for you not to have it. I'm not going to go into the details of building the site or pushing it - live. Plenty of people on pydotorg can help you, and there's a good README - once you get the branch. All the directories below are named relative to - the data subdirectory unless otherwise noted. + live. All the directories below are named relative to the data subdirectory + unless otherwise noted. This page will probably come in handy: @@ -492,8 +494,8 @@ ___ update the download page, editing `download/content.ht` - ___ edit the previous release's last release content.ht page to point to - the new release. + ___ for X.Y.Z, edit all the previous X.Y releases' content.ht page to + point to the new release. ___ update `doc/content.ht` to indicate the new current documentation version, and remove the current version from any 'in development' @@ -501,7 +503,7 @@ ___ Add the new version to `doc/versions/content.ht`. - ___ Edit download/releases/content.ht to update the version numbers for + ___ Edit `download/releases/content.ht` to update the version numbers for this release. There are a bunch of places you need to touch: ___ The subdirectory name as the first element in the Nav rows. @@ -510,14 +512,16 @@ ___ Update the version specific pages. - ___ cd to download/releases/X.Y.Z + ___ cd to `download/releases/X.Y.Z` ___ Edit the version numbers in content.ht ___ Comment out the link to the CHM file if this is not a final, remove the comment if it is. ___ Update the md5 checksums - Note, you don't have to copy the actual .tgz or tar.bz2 tarballs into - this directory because they only live on dinsdale in the ftp directory. + ___ Update the license in `download/releases/X.Y.Z/license` + + Note, you don't have to copy any release files into this directory; + they only live on dinsdale in the ftp directory. ___ When everything looks good, `svn commit` in the data directory. This will trigger the live site to update itself, and at that point the -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sat Sep 14 09:12:15 2013 From: python-checkins at python.org (georg.brandl) Date: Sat, 14 Sep 2013 09:12:15 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E2=29=3A_Fix_tkinter_re?= =?utf-8?q?gression_introduced_by_the_security_fix_in_=2316248=2E?= Message-ID: <3ccPzg5rysz7LjR@mail.python.org> http://hg.python.org/cpython/rev/c18c18774e24 changeset: 85693:c18c18774e24 branch: 3.2 parent: 83826:b9b521efeba3 user: Georg Brandl date: Sat Sep 14 09:08:09 2013 +0200 summary: Fix tkinter regression introduced by the security fix in #16248. files: Lib/tkinter/__init__.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1722,7 +1722,7 @@ # ensure that self.tk is always _something_. self.tk = None if baseName is None: - import sys, os + import os baseName = os.path.basename(sys.argv[0]) baseName, ext = os.path.splitext(baseName) if ext not in ('.py', '.pyc', '.pyo'): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 09:12:17 2013 From: python-checkins at python.org (georg.brandl) Date: Sat, 14 Sep 2013 09:12:17 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E2=29=3A_Add_NEWS_entry?= =?utf-8?q?_for_c18c18774e24=2E?= Message-ID: <3ccPzj0WWtz7LkD@mail.python.org> http://hg.python.org/cpython/rev/6aca1f584fcb changeset: 85694:6aca1f584fcb branch: 3.2 user: Georg Brandl date: Sat Sep 14 09:09:18 2013 +0200 summary: Add NEWS entry for c18c18774e24. files: Misc/NEWS | 11 +++++++++++ 1 files changed, 11 insertions(+), 0 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,17 @@ Python News +++++++++++ +What's New in Python 3.2.6? +=========================== + +*Release date: TBD* + +Library +------- + +- Fix tkinter regression introduced by the security fix in issue #16248. + + What's New in Python 3.2.5? =========================== -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 09:12:18 2013 From: python-checkins at python.org (georg.brandl) Date: Sat, 14 Sep 2013 09:12:18 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E2=29=3A_Add_a_NEWS_ent?= =?utf-8?q?ry_for_b9b521efeba3=2E?= Message-ID: <3ccPzk2Mz0z7LkT@mail.python.org> http://hg.python.org/cpython/rev/58babe506bfa changeset: 85695:58babe506bfa branch: 3.2 user: Georg Brandl date: Sat Sep 14 09:10:21 2013 +0200 summary: Add a NEWS entry for b9b521efeba3. files: Misc/NEWS | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,9 @@ - Fix tkinter regression introduced by the security fix in issue #16248. +- Issue #17980: Fix possible abuse of ssl.match_hostname() for denial of + service using certificates with many wildcards (CVE-2013-2099). + What's New in Python 3.2.5? =========================== -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 09:12:19 2013 From: python-checkins at python.org (georg.brandl) Date: Sat, 14 Sep 2013 09:12:19 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy4yIC0+IDMuMyk6?= =?utf-8?q?_merge_with_3=2E2?= Message-ID: <3ccPzl49L4z7Lkn@mail.python.org> http://hg.python.org/cpython/rev/1aef1b265f73 changeset: 85696:1aef1b265f73 branch: 3.3 parent: 85686:9fcb9deb70f5 parent: 85695:58babe506bfa user: Georg Brandl date: Sat Sep 14 09:11:09 2013 +0200 summary: merge with 3.2 files: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 09:12:20 2013 From: python-checkins at python.org (georg.brandl) Date: Sat, 14 Sep 2013 09:12:20 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_merge_with_3=2E3?= Message-ID: <3ccPzm62Chz7Lk5@mail.python.org> http://hg.python.org/cpython/rev/c9644c1f9c94 changeset: 85697:c9644c1f9c94 parent: 85692:3894e17e59c4 parent: 85696:1aef1b265f73 user: Georg Brandl date: Sat Sep 14 09:11:21 2013 +0200 summary: merge with 3.3 files: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 19:32:13 2013 From: python-checkins at python.org (r.david.murray) Date: Sat, 14 Sep 2013 19:32:13 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE4MjA2OiBGaXgg?= =?utf-8?q?test_for_existence_of_license_URL=2E?= Message-ID: <3ccgl15xmlz7Lkp@mail.python.org> http://hg.python.org/cpython/rev/7c18b799841e changeset: 85698:7c18b799841e branch: 3.3 parent: 85696:1aef1b265f73 user: R David Murray date: Sat Sep 14 13:28:37 2013 -0400 summary: #18206: Fix test for existence of license URL. It now always checks, instead of only when the LICENSE file doesn't exist. It is also protected by the 'network' resource, and uses a HEAD request since we are only doing an existence check. files: Lib/test/test_site.py | 40 ++++++++++++------------------ 1 files changed, 16 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -13,6 +13,8 @@ import sys import re import encodings +import urllib.request +import urllib.error import subprocess import sysconfig from copy import copy @@ -407,30 +409,20 @@ else: self.fail("sitecustomize not imported automatically") + @test.support.requires_resource('network') + def test_license_exists_at_url(self): + # This test is a bit fragile since it depends on the format of the + # string displayed by license in the absence of a LICENSE file. + url = license._Printer__data.split()[1] + req = urllib.request.Request(url, method='HEAD') + try: + with test.support.transient_internet(url): + with urllib.request.urlopen(req) as data: + code = data.getcode() + except urllib.error.HTTPError as e: + code = e.code + self.assertEqual(code, 200, msg="Can't find " + url) -class LicenseURL(unittest.TestCase): - """Test accessibility of the license.""" - - @unittest.skipUnless(str(license).startswith('See http://'), - 'license is available as a file') - def test_license_page(self): - """urlopen should return the license page""" - pat = r'^See (http://www\.python\.org/download/releases/[^/]+/license/)$' - mo = re.search(pat, str(license)) - self.assertIsNotNone(mo, msg='can\'t find appropriate url in license') - if mo is not None: - url = mo.group(1) - with test.support.transient_internet(url): - import urllib.request, urllib.error - try: - with urllib.request.urlopen(url) as data: - code = data.getcode() - except urllib.error.HTTPError as e: - code = e.code - self.assertEqual(code, 200, msg=url) - -def test_main(): - run_unittest(HelperFunctionsTests, ImportSideEffectTests, LicenseURL) if __name__ == "__main__": - test_main() + unittest.main() -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 19:32:15 2013 From: python-checkins at python.org (r.david.murray) Date: Sat, 14 Sep 2013 19:32:15 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_Merge_=2318206=3A_Fix_test_for_existence_of_license_URL?= =?utf-8?q?=2E?= Message-ID: <3ccgl30mTtz7LkB@mail.python.org> http://hg.python.org/cpython/rev/321c3c9cd1b5 changeset: 85699:321c3c9cd1b5 parent: 85697:c9644c1f9c94 parent: 85698:7c18b799841e user: R David Murray date: Sat Sep 14 13:31:14 2013 -0400 summary: Merge #18206: Fix test for existence of license URL. This test will fail because a previous attempt to fix a merge error in site.py was incorrect, but the test wasn't running so it wasn't caught. The next commit will fix the site.py bug. files: Lib/test/test_site.py | 34 ++++++++++++------------------ 1 files changed, 14 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -13,6 +13,8 @@ import sys import re import encodings +import urllib.request +import urllib.error import subprocess import sysconfig from copy import copy @@ -403,27 +405,19 @@ else: self.fail("sitecustomize not imported automatically") - -class LicenseURL(unittest.TestCase): - """Test accessibility of the license.""" - - @unittest.skipUnless(str(license).startswith('See http://'), - 'license is available as a file') - def test_license_page(self): - """urlopen should return the license page""" - pat = r'^See (http://www\.python\.org/download/releases/[^/]+/license/)$' - mo = re.search(pat, str(license)) - self.assertIsNotNone(mo, msg='can\'t find appropriate url in license') - if mo is not None: - url = mo.group(1) + @test.support.requires_resource('network') + def test_license_exists_at_url(self): + # This test is a bit fragile since it depends on the format of the + # string displayed by license in the absence of a LICENSE file. + url = license._Printer__data.split()[1] + req = urllib.request.Request(url, method='HEAD') + try: with test.support.transient_internet(url): - import urllib.request, urllib.error - try: - with urllib.request.urlopen(url) as data: - code = data.getcode() - except urllib.error.HTTPError as e: - code = e.code - self.assertEqual(code, 200, msg=url) + with urllib.request.urlopen(req) as data: + code = data.getcode() + except urllib.error.HTTPError as e: + code = e.code + self.assertEqual(code, 200, msg="Can't find " + url) if __name__ == "__main__": -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 19:32:16 2013 From: python-checkins at python.org (r.david.murray) Date: Sat, 14 Sep 2013 19:32:16 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_=2318206=3A_Re-fix_license?= =?utf-8?q?_URL=2E?= Message-ID: <3ccgl42Xj4z7LkY@mail.python.org> http://hg.python.org/cpython/rev/9354325d2ee4 changeset: 85700:9354325d2ee4 user: R David Murray date: Sat Sep 14 13:31:44 2013 -0400 summary: #18206: Re-fix license URL. files: Lib/site.py | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/Lib/site.py b/Lib/site.py --- a/Lib/site.py +++ b/Lib/site.py @@ -362,7 +362,8 @@ for supporting Python development. See www.python.org for more information.""") here = os.path.dirname(os.__file__) builtins.license = _sitebuiltins._Printer( - "license", "See http://www.python.org/%.3s/license.html" % sys.version, + "license", + "See http://www.python.org/download/releases/%.5s/license" % sys.version, ["LICENSE.txt", "LICENSE"], [os.path.join(here, os.pardir), here, os.curdir]) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 19:45:54 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sat, 14 Sep 2013 19:45:54 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318937=3A_Add_an_a?= =?utf-8?q?ssertLogs=28=29_context_manager_to_unittest=2ETestCase_to?= Message-ID: <3cch2p3mLCz7LjT@mail.python.org> http://hg.python.org/cpython/rev/4f5815747f58 changeset: 85701:4f5815747f58 user: Antoine Pitrou date: Sat Sep 14 19:45:47 2013 +0200 summary: Issue #18937: Add an assertLogs() context manager to unittest.TestCase to ensure that a block of code emits a message using the logging module. files: Doc/library/unittest.rst | 41 +++++++ Lib/unittest/case.py | 109 +++++++++++++++++++- Lib/unittest/test/test_case.py | 96 ++++++++++++++++++ Misc/NEWS | 2 + 4 files changed, 242 insertions(+), 6 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1031,6 +1031,47 @@ .. versionchanged:: 3.3 Added the *msg* keyword argument when used as a context manager. + .. method:: assertLogs(logger=None, level=None) + + A context manager to test that at least one message is logged on + the *logger* or one of its children, with at least the given + *level*. + + If given, *logger* should be a :class:`logging.Logger` object or a + :class:`str` giving the name of a logger. The default is the root + logger, which will catch all messages. + + If given, *level* should be either a numeric logging level or + its string equivalent (for example either ``"ERROR"`` or + :attr:`logging.ERROR`). The default is :attr:`logging.INFO`. + + The test passes if at least one message emitted inside the ``with`` + block matches the *logger* and *level* conditions, otherwise it fails. + + The object returned by the context manager is a recording helper + which keeps tracks of the matching log messages. It has two + attributes: + + .. attribute:: records + + A list of :class:`logging.LogRecord` objects of the matching + log messages. + + .. attribute:: output + + A list of :class:`str` objects with the formatted output of + matching messages. + + Example:: + + with self.assertLogs('foo', level='INFO') as cm: + logging.getLogger('foo').info('first message') + logging.getLogger('foo.bar').error('second message') + self.assertEqual(cm.output, ['INFO:foo:first message', + 'ERROR:foo.bar:second message']) + + .. versionadded:: 3.4 + There are also other methods used to perform more specific checks, such as: diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -3,6 +3,7 @@ import sys import functools import difflib +import logging import pprint import re import warnings @@ -115,10 +116,21 @@ return test_item -class _AssertRaisesBaseContext(object): +class _BaseTestCaseContext: + + def __init__(self, test_case): + self.test_case = test_case + + def _raiseFailure(self, standardMsg): + msg = self.test_case._formatMessage(self.msg, standardMsg) + raise self.test_case.failureException(msg) + + +class _AssertRaisesBaseContext(_BaseTestCaseContext): def __init__(self, expected, test_case, callable_obj=None, expected_regex=None): + _BaseTestCaseContext.__init__(self, test_case) self.expected = expected self.test_case = test_case if callable_obj is not None: @@ -133,10 +145,6 @@ self.expected_regex = expected_regex self.msg = None - def _raiseFailure(self, standardMsg): - msg = self.test_case._formatMessage(self.msg, standardMsg) - raise self.test_case.failureException(msg) - def handle(self, name, callable_obj, args, kwargs): """ If callable_obj is None, assertRaises/Warns is being used as a @@ -150,7 +158,6 @@ callable_obj(*args, **kwargs) - class _AssertRaisesContext(_AssertRaisesBaseContext): """A context manager used to implement TestCase.assertRaises* methods.""" @@ -232,6 +239,74 @@ self._raiseFailure("{} not triggered".format(exc_name)) + +_LoggingWatcher = collections.namedtuple("_LoggingWatcher", + ["records", "output"]) + + +class _CapturingHandler(logging.Handler): + """ + A logging handler capturing all (raw and formatted) logging output. + """ + + def __init__(self): + logging.Handler.__init__(self) + self.watcher = _LoggingWatcher([], []) + + def flush(self): + pass + + def emit(self, record): + self.watcher.records.append(record) + msg = self.format(record) + self.watcher.output.append(msg) + + + +class _AssertLogsContext(_BaseTestCaseContext): + """A context manager used to implement TestCase.assertLogs().""" + + LOGGING_FORMAT = "%(levelname)s:%(name)s:%(message)s" + + def __init__(self, test_case, logger_name, level): + _BaseTestCaseContext.__init__(self, test_case) + self.logger_name = logger_name + if level: + self.level = logging._nameToLevel.get(level, level) + else: + self.level = logging.INFO + self.msg = None + + def __enter__(self): + if isinstance(self.logger_name, logging.Logger): + logger = self.logger = self.logger_name + else: + logger = self.logger = logging.getLogger(self.logger_name) + formatter = logging.Formatter(self.LOGGING_FORMAT) + handler = _CapturingHandler() + handler.setFormatter(formatter) + self.watcher = handler.watcher + self.old_handlers = logger.handlers[:] + self.old_level = logger.level + self.old_propagate = logger.propagate + logger.handlers = [handler] + logger.setLevel(self.level) + logger.propagate = False + return handler.watcher + + def __exit__(self, exc_type, exc_value, tb): + self.logger.handlers = self.old_handlers + self.logger.propagate = self.old_propagate + self.logger.setLevel(self.old_level) + if exc_type is not None: + # let unexpected exceptions pass through + return False + if len(self.watcher.records) == 0: + self._raiseFailure( + "no logs of level {} or higher triggered on {}" + .format(logging.getLevelName(self.level), self.logger.name)) + + class TestCase(object): """A class whose instances are single test cases. @@ -644,6 +719,28 @@ context = _AssertWarnsContext(expected_warning, self, callable_obj) return context.handle('assertWarns', callable_obj, args, kwargs) + def assertLogs(self, logger=None, level=None): + """Fail unless a log message of level *level* or higher is emitted + on *logger_name* or its children. If omitted, *level* defaults to + INFO and *logger* defaults to the root logger. + + This method must be used as a context manager, and will yield + a recording object with two attributes: `output` and `records`. + At the end of the context manager, the `output` attribute will + be a list of the matching formatted log messages and the + `records` attribute will be a list of the corresponding LogRecord + objects. + + Example:: + + with self.assertLogs('foo', level='INFO') as cm: + logging.getLogger('foo').info('first message') + logging.getLogger('foo.bar').error('second message') + self.assertEqual(cm.output, ['INFO:foo:first message', + 'ERROR:foo.bar:second message']) + """ + return _AssertLogsContext(self, logger, level) + def _getAssertEqualityFunc(self, first, second): """Get a detailed comparison function for the types of the two args. diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py --- a/Lib/unittest/test/test_case.py +++ b/Lib/unittest/test/test_case.py @@ -1,8 +1,10 @@ +import contextlib import difflib import pprint import pickle import re import sys +import logging import warnings import weakref import inspect @@ -16,6 +18,12 @@ TestEquality, TestHashing, LoggingResult, LegacyLoggingResult, ResultWithNoStartTestRunStopTestRun ) +from test.support import captured_stderr + + +log_foo = logging.getLogger('foo') +log_foobar = logging.getLogger('foo.bar') +log_quux = logging.getLogger('quux') class Test(object): @@ -1251,6 +1259,94 @@ with self.assertWarnsRegex(RuntimeWarning, "o+"): _runtime_warn("barz") + @contextlib.contextmanager + def assertNoStderr(self): + with captured_stderr() as buf: + yield + self.assertEqual(buf.getvalue(), "") + + def assertLogRecords(self, records, matches): + self.assertEqual(len(records), len(matches)) + for rec, match in zip(records, matches): + self.assertIsInstance(rec, logging.LogRecord) + for k, v in match.items(): + self.assertEqual(getattr(rec, k), v) + + def testAssertLogsDefaults(self): + # defaults: root logger, level INFO + with self.assertNoStderr(): + with self.assertLogs() as cm: + log_foo.info("1") + log_foobar.debug("2") + self.assertEqual(cm.output, ["INFO:foo:1"]) + self.assertLogRecords(cm.records, [{'name': 'foo'}]) + + def testAssertLogsTwoMatchingMessages(self): + # Same, but with two matching log messages + with self.assertNoStderr(): + with self.assertLogs() as cm: + log_foo.info("1") + log_foobar.debug("2") + log_quux.warning("3") + self.assertEqual(cm.output, ["INFO:foo:1", "WARNING:quux:3"]) + self.assertLogRecords(cm.records, + [{'name': 'foo'}, {'name': 'quux'}]) + + def checkAssertLogsPerLevel(self, level): + # Check level filtering + with self.assertNoStderr(): + with self.assertLogs(level=level) as cm: + log_foo.warning("1") + log_foobar.error("2") + log_quux.critical("3") + self.assertEqual(cm.output, ["ERROR:foo.bar:2", "CRITICAL:quux:3"]) + self.assertLogRecords(cm.records, + [{'name': 'foo.bar'}, {'name': 'quux'}]) + + def testAssertLogsPerLevel(self): + self.checkAssertLogsPerLevel(logging.ERROR) + self.checkAssertLogsPerLevel('ERROR') + + def checkAssertLogsPerLogger(self, logger): + # Check per-logger fitering + with self.assertNoStderr(): + with self.assertLogs(level='DEBUG') as outer_cm: + with self.assertLogs(logger, level='DEBUG') as cm: + log_foo.info("1") + log_foobar.debug("2") + log_quux.warning("3") + self.assertEqual(cm.output, ["INFO:foo:1", "DEBUG:foo.bar:2"]) + self.assertLogRecords(cm.records, + [{'name': 'foo'}, {'name': 'foo.bar'}]) + # The outer catchall caught the quux log + self.assertEqual(outer_cm.output, ["WARNING:quux:3"]) + + def testAssertLogsPerLogger(self): + self.checkAssertLogsPerLogger(logging.getLogger('foo')) + self.checkAssertLogsPerLogger('foo') + + def testAssertLogsFailureNoLogs(self): + # Failure due to no logs + with self.assertNoStderr(): + with self.assertRaises(self.failureException): + with self.assertLogs(): + pass + + def testAssertLogsFailureLevelTooHigh(self): + # Failure due to level too high + with self.assertNoStderr(): + with self.assertRaises(self.failureException): + with self.assertLogs(level='WARNING'): + log_foo.info("1") + + def testAssertLogsFailureMismatchingLogger(self): + # Failure due to mismatching logger (and the logged message is + # passed through) + with self.assertLogs('quux', level='ERROR'): + with self.assertRaises(self.failureException): + with self.assertLogs('foo'): + log_quux.error("1") + def testDeprecatedMethodNames(self): """ Test that the deprecated methods raise a DeprecationWarning. See #9424. diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,8 @@ Library ------- +- Issue #18937: Add an assertLogs() context manager to unittest.TestCase + to ensure that a block of code emits a message using the logging module. - Issue #17324: Fix http.server's request handling case on trailing '/'. Patch contributed by Vajrasky Kok. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 20:01:40 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sat, 14 Sep 2013 20:01:40 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Add_a_=60=60transform=5Ffunc?= =?utf-8?q?=60=60_property?= Message-ID: <3cchP05wWXzRmv@mail.python.org> http://hg.python.org/peps/rev/d0f2e932e13a changeset: 5114:d0f2e932e13a user: Antoine Pitrou date: Sat Sep 14 20:01:00 2013 +0200 summary: Add a ``transform_func`` property files: pep-0455.txt | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/pep-0455.txt b/pep-0455.txt --- a/pep-0455.txt +++ b/pep-0455.txt @@ -113,6 +113,12 @@ The method name ``getitem()`` mirrors the standard ``popitem()`` method on mutable mappings. +Getting the transformation function +----------------------------------- + +TransformDict has a simple read-only property ``transform_func`` which +gives back the transformation function. + Alternative proposals and questions =================================== -- Repository URL: http://hg.python.org/peps From tjreedy at udel.edu Sat Sep 14 20:14:10 2013 From: tjreedy at udel.edu (Terry Reedy) Date: Sat, 14 Sep 2013 14:14:10 -0400 Subject: [Python-checkins] cpython: Issue #18937: Add an assertLogs() context manager to unittest.TestCase to In-Reply-To: <3cch2p3mLCz7LjT@mail.python.org> References: <3cch2p3mLCz7LjT@mail.python.org> Message-ID: <5234A772.4000800@udel.edu> On 9/14/2013 1:45 PM, antoine.pitrou wrote: > http://hg.python.org/cpython/rev/4f5815747f58 > changeset: 85701:4f5815747f58 > user: Antoine Pitrou > date: Sat Sep 14 19:45:47 2013 +0200 > summary: > Issue #18937: Add an assertLogs() context manager to unittest.TestCase to ensure that a block of code emits a message using the logging module. > > files: > Doc/library/unittest.rst | 41 +++++++ > Lib/unittest/case.py | 109 +++++++++++++++++++- > Lib/unittest/test/test_case.py | 96 ++++++++++++++++++ > Misc/NEWS | 2 + > 4 files changed, 242 insertions(+), 6 deletions(-) > > > diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst > --- a/Doc/library/unittest.rst > +++ b/Doc/library/unittest.rst > @@ -1031,6 +1031,47 @@ > .. versionchanged:: 3.3 > Added the *msg* keyword argument when used as a context manager. assertWarnsRegex > + .. method:: assertLogs(logger=None, level=None) > + > + A context manager to test that at least one message is logged on > + the *logger* or one of its children, with at least the given > + *level*. > + > + If given, *logger* should be a :class:`logging.Logger` object or a > + :class:`str` giving the name of a logger. The default is the root > + logger, which will catch all messages. > + > + If given, *level* should be either a numeric logging level or > + its string equivalent (for example either ``"ERROR"`` or > + :attr:`logging.ERROR`). The default is :attr:`logging.INFO`. > + > + The test passes if at least one message emitted inside the ``with`` > + block matches the *logger* and *level* conditions, otherwise it fails. > + > + The object returned by the context manager is a recording helper > + which keeps tracks of the matching log messages. It has two > + attributes: > + > + .. attribute:: records > + > + A list of :class:`logging.LogRecord` objects of the matching > + log messages. > + > + .. attribute:: output > + > + A list of :class:`str` objects with the formatted output of > + matching messages. > + > + Example:: > + > + with self.assertLogs('foo', level='INFO') as cm: > + logging.getLogger('foo').info('first message') > + logging.getLogger('foo.bar').error('second message') > + self.assertEqual(cm.output, ['INFO:foo:first message', > + 'ERROR:foo.bar:second message']) > + > + .. versionadded:: 3.4 > + > > There are also other methods used to perform more specific checks, such as: The new method should be added to the table of raises/warns methods that starts this group. The table intro sentence "It is also possible to check that exceptions and warnings are raised using the following methods:" should be modified to something like "It is also possible to check the production of exception, warning, and log messages using the following methods:" Terry From python-checkins at python.org Sat Sep 14 21:16:45 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sat, 14 Sep 2013 21:16:45 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Avoid_test=5Flogging_failu?= =?utf-8?q?re_when_run_after_test=5Funittest=2C_by_renaming_a?= Message-ID: <3cck3d1xNsz7LlW@mail.python.org> http://hg.python.org/cpython/rev/1dc925ee441a changeset: 85702:1dc925ee441a user: Antoine Pitrou date: Sat Sep 14 21:16:39 2013 +0200 summary: Avoid test_logging failure when run after test_unittest, by renaming a conflicting logger files: Lib/test/test_logging.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1345,7 +1345,7 @@ def test_logger_disabling(self): self.apply_config(self.disable_test) - logger = logging.getLogger('foo') + logger = logging.getLogger('some_pristine_logger') self.assertFalse(logger.disabled) self.apply_config(self.disable_test) self.assertTrue(logger.disabled) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 14 22:11:59 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sat, 14 Sep 2013 22:11:59 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Add_one_more_point_about_the_?= =?utf-8?q?benefits_of_a_generic_construct?= Message-ID: <3cclHM2McTzSg7@mail.python.org> http://hg.python.org/peps/rev/893e09bea72f changeset: 5115:893e09bea72f user: Antoine Pitrou date: Sat Sep 14 22:11:52 2013 +0200 summary: Add one more point about the benefits of a generic construct files: pep-0455.txt | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/pep-0455.txt b/pep-0455.txt --- a/pep-0455.txt +++ b/pep-0455.txt @@ -170,6 +170,10 @@ is that it's nearly as cheap (code-wise and performance-wise) to provide the generic construct, and it can fill more use cases. +Even case-insensitive dicts can actually elicit different transformation +functions: ``str.lower``, ``str.casefold`` or in some cases ``bytes.lower`` +when working with text encoded in a ASCII-compatible encoding. + Other constructor patterns -------------------------- -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sun Sep 15 02:01:44 2013 From: python-checkins at python.org (antoine.pitrou) Date: Sun, 15 Sep 2013 02:01:44 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Address_Terry=27s_comments?= Message-ID: <3ccrNS6JN2z7LjR@mail.python.org> http://hg.python.org/cpython/rev/7bf4406c972e changeset: 85703:7bf4406c972e user: Antoine Pitrou date: Sun Sep 15 02:01:39 2013 +0200 summary: Address Terry's comments files: Doc/library/unittest.rst | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -883,8 +883,8 @@ - It is also possible to check that exceptions and warnings are raised using - the following methods: + It is also possible to check the production of exceptions, warnings and + log messages using the following methods: +---------------------------------------------------------+--------------------------------------+------------+ | Method | Checks that | New in | @@ -901,6 +901,9 @@ | :meth:`assertWarnsRegex(warn, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | | ` | and the message matches regex *r* | | +---------------------------------------------------------+--------------------------------------+------------+ + | :meth:`assertLogs(logger, level)` | The ``with`` block logs on *logger* | 3.4 | + | ` | with minimum *level* | | + +---------------------------------------------------------+--------------------------------------+------------+ .. method:: assertRaises(exception, callable, *args, **kwds) assertRaises(exception, msg=None) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 03:07:28 2013 From: python-checkins at python.org (ezio.melotti) Date: Sun, 15 Sep 2013 03:07:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE4ODU2OiBpbXBy?= =?utf-8?q?ove_test_coverage_of_the_calendar_module=2E__Patch_by_Madison_M?= =?utf-8?b?YXku?= Message-ID: <3ccsrJ37bmz7LjQ@mail.python.org> http://hg.python.org/cpython/rev/442e23bbec48 changeset: 85704:442e23bbec48 branch: 3.3 parent: 85686:9fcb9deb70f5 user: Ezio Melotti date: Sat Sep 14 04:55:53 2013 +0300 summary: #18856: improve test coverage of the calendar module. Patch by Madison May. files: Lib/test/test_calendar.py | 21 +++++++++++++++++++++ 1 files changed, 21 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -399,6 +399,27 @@ 'January' ) + def test_prweek(self): + with support.captured_stdout() as out: + week = [(1,0), (2,1), (3,2), (4,3), (5,4), (6,5), (7,6)] + calendar.TextCalendar().prweek(week, 1) + self.assertEqual(out.getvalue().strip(), "1 2 3 4 5 6 7") + + def test_prmonth(self): + with support.captured_stdout() as out: + calendar.TextCalendar().prmonth(2004, 1) + output = out.getvalue().strip() + self.assertEqual(output, result_2004_01_text.strip()) + + def test_pryear(self): + with support.captured_stdout() as out: + calendar.TextCalendar().pryear(2004) + self.assertEqual(out.getvalue().strip(), result_2004_text.strip()) + + def test_format(self): + with support.captured_stdout() as out: + calendar.format(["1", "2", "3"], colwidth=3, spacing=1) + self.assertEqual(out.getvalue().strip(), "1 2 3") class CalendarTestCase(unittest.TestCase): def test_isleap(self): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 03:07:29 2013 From: python-checkins at python.org (ezio.melotti) Date: Sun, 15 Sep 2013 03:07:29 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy4zIC0+IDMuMyk6?= =?utf-8?q?_Merge_heads=2E?= Message-ID: <3ccsrK4wDVz7LjQ@mail.python.org> http://hg.python.org/cpython/rev/63b43935964f changeset: 85705:63b43935964f branch: 3.3 parent: 85698:7c18b799841e parent: 85704:442e23bbec48 user: Ezio Melotti date: Sun Sep 15 03:00:42 2013 +0300 summary: Merge heads. files: Lib/test/test_calendar.py | 21 +++++++++++++++++++++ 1 files changed, 21 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -399,6 +399,27 @@ 'January' ) + def test_prweek(self): + with support.captured_stdout() as out: + week = [(1,0), (2,1), (3,2), (4,3), (5,4), (6,5), (7,6)] + calendar.TextCalendar().prweek(week, 1) + self.assertEqual(out.getvalue().strip(), "1 2 3 4 5 6 7") + + def test_prmonth(self): + with support.captured_stdout() as out: + calendar.TextCalendar().prmonth(2004, 1) + output = out.getvalue().strip() + self.assertEqual(output, result_2004_01_text.strip()) + + def test_pryear(self): + with support.captured_stdout() as out: + calendar.TextCalendar().pryear(2004) + self.assertEqual(out.getvalue().strip(), result_2004_text.strip()) + + def test_format(self): + with support.captured_stdout() as out: + calendar.format(["1", "2", "3"], colwidth=3, spacing=1) + self.assertEqual(out.getvalue().strip(), "1 2 3") class CalendarTestCase(unittest.TestCase): def test_isleap(self): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 03:07:30 2013 From: python-checkins at python.org (ezio.melotti) Date: Sun, 15 Sep 2013 03:07:30 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?b?KTogIzE4ODU2OiBtZXJnZSB3aXRoIDMuMy4=?= Message-ID: <3ccsrL6dNdz7LjY@mail.python.org> http://hg.python.org/cpython/rev/81da2ed0c563 changeset: 85706:81da2ed0c563 parent: 85703:7bf4406c972e parent: 85705:63b43935964f user: Ezio Melotti date: Sun Sep 15 04:05:04 2013 +0300 summary: #18856: merge with 3.3. files: Lib/test/test_calendar.py | 21 +++++++++++++++++++++ 1 files changed, 21 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -399,6 +399,27 @@ 'January' ) + def test_prweek(self): + with support.captured_stdout() as out: + week = [(1,0), (2,1), (3,2), (4,3), (5,4), (6,5), (7,6)] + calendar.TextCalendar().prweek(week, 1) + self.assertEqual(out.getvalue().strip(), "1 2 3 4 5 6 7") + + def test_prmonth(self): + with support.captured_stdout() as out: + calendar.TextCalendar().prmonth(2004, 1) + output = out.getvalue().strip() + self.assertEqual(output, result_2004_01_text.strip()) + + def test_pryear(self): + with support.captured_stdout() as out: + calendar.TextCalendar().pryear(2004) + self.assertEqual(out.getvalue().strip(), result_2004_text.strip()) + + def test_format(self): + with support.captured_stdout() as out: + calendar.format(["1", "2", "3"], colwidth=3, spacing=1) + self.assertEqual(out.getvalue().strip(), "1 2 3") class CalendarTestCase(unittest.TestCase): def test_isleap(self): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 03:12:04 2013 From: python-checkins at python.org (ethan.furman) Date: Sun, 15 Sep 2013 03:12:04 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbjogQWRkIF9fcmV2ZXJzZWRfXyB0?= =?utf-8?q?o_Enum=2E__Minor_code_reorg_=28moved_=5F=5Fmembers=5F=5F_to_be_?= =?utf-8?q?in_alpha?= Message-ID: <3ccsxc44MBzNHS@mail.python.org> http://hg.python.org/cpython/rev/24aa6f98fe25 changeset: 85707:24aa6f98fe25 user: Ethan Furman date: Sat Sep 14 18:11:24 2013 -0700 summary: Add __reversed__ to Enum. Minor code reorg (moved __members__ to be in alpha order). files: Lib/enum.py | 23 +++++++++++++---------- Lib/test/test_enum.py | 7 +++++++ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Lib/enum.py b/Lib/enum.py --- a/Lib/enum.py +++ b/Lib/enum.py @@ -225,16 +225,6 @@ def __dir__(self): return ['__class__', '__doc__', '__members__'] + self._member_names_ - @property - def __members__(cls): - """Returns a mapping of member name->value. - - This mapping lists all enum members, including aliases. Note that this - is a read-only view of the internal mapping. - - """ - return MappingProxyType(cls._member_map_) - def __getattr__(cls, name): """Return the enum member matching `name` @@ -260,9 +250,22 @@ def __len__(cls): return len(cls._member_names_) + @property + def __members__(cls): + """Returns a mapping of member name->value. + + This mapping lists all enum members, including aliases. Note that this + is a read-only view of the internal mapping. + + """ + return MappingProxyType(cls._member_map_) + def __repr__(cls): return "" % cls.__name__ + def __reversed__(cls): + return (cls._member_map_[name] for name in reversed(cls._member_names_)) + def __setattr__(cls, name, value): """Block attempts to reassign Enum members. diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -477,6 +477,13 @@ [Season.SUMMER, Season.WINTER, Season.AUTUMN, Season.SPRING], ) + def test_reversed_iteration_order(self): + self.assertEqual( + list(reversed(self.Season)), + [self.Season.WINTER, self.Season.AUTUMN, self.Season.SUMMER, + self.Season.SPRING] + ) + def test_programatic_function_string(self): SummerMonth = Enum('SummerMonth', 'june july august') lst = list(SummerMonth) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 03:53:46 2013 From: python-checkins at python.org (ethan.furman) Date: Sun, 15 Sep 2013 03:53:46 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Close_=2318929=3A_inspect?= =?utf-8?q?=2Eclassify=5Fclass=5Fattrs_will_now_search_the_metaclasses?= Message-ID: <3cctsk1w7Dz7Ljb@mail.python.org> http://hg.python.org/cpython/rev/b0517bb271ad changeset: 85708:b0517bb271ad user: Ethan Furman date: Sat Sep 14 18:53:26 2013 -0700 summary: Close #18929: inspect.classify_class_attrs will now search the metaclasses (last) to find where an attr was defined. files: Doc/library/inspect.rst | 5 +++-- Lib/inspect.py | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -173,8 +173,9 @@ .. note:: - :func:`getmembers` does not return metaclass attributes when the argument - is a class (this behavior is inherited from the :func:`dir` function). + :func:`getmembers` will only return metaclass attributes when the + argument is a class and those attributes have been listed in a custom + :meth:`__dir__`. .. function:: getmoduleinfo(path) diff --git a/Lib/inspect.py b/Lib/inspect.py --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -308,9 +308,14 @@ data attributes: C.data is just a data object, but C.__dict__['data'] may be a data descriptor with additional info, like a __doc__ string. + + If one of the items in dir(cls) is stored in the metaclass it will now + be discovered and not have None be listed as the class in which it was + defined. """ mro = getmro(cls) + metamro = getmro(type(cls)) # for attributes stored in the metaclass names = dir(cls) result = [] for name in names: @@ -321,7 +326,7 @@ # getattr(). This is the case with some descriptors (bug #1785). # Thus, we only use getattr() as a last resort. homecls = None - for base in (cls,) + mro: + for base in (cls,) + mro + metamro: if name in base.__dict__: obj = base.__dict__[name] homecls = base -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 05:53:11 2013 From: python-checkins at python.org (raymond.hettinger) Date: Sun, 15 Sep 2013 05:53:11 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE5MDE4?= =?utf-8?q?=3A_The_heapq=2Emerge=28=29_function_no_longer_suppresses_Index?= =?utf-8?q?Error?= Message-ID: <3ccxWW6pX6z7Lk5@mail.python.org> http://hg.python.org/cpython/rev/0ff5bb61c6a1 changeset: 85709:0ff5bb61c6a1 branch: 3.3 parent: 85705:63b43935964f user: Raymond Hettinger date: Sat Sep 14 20:51:57 2013 -0700 summary: Issue #19018: The heapq.merge() function no longer suppresses IndexError files: Lib/heapq.py | 14 +++++++++----- Lib/test/test_heapq.py | 9 +++++++++ Misc/ACKS | 2 ++ Misc/NEWS | 3 +++ 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Lib/heapq.py b/Lib/heapq.py --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -358,6 +358,7 @@ ''' _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration + _len = len h = [] h_append = h.append @@ -369,17 +370,20 @@ pass heapify(h) - while 1: + while _len(h) > 1: try: - while 1: - v, itnum, next = s = h[0] # raises IndexError when h is empty + while True: + v, itnum, next = s = h[0] yield v s[0] = next() # raises StopIteration when exhausted _heapreplace(h, s) # restore heap condition except _StopIteration: _heappop(h) # remove empty iterator - except IndexError: - return + if h: + # fast case when only a single iterator remains + v, itnum, next = h[0] + yield v + yield from next.__self__ # Extend the implementations of nsmallest and nlargest to use a key= argument _nsmallest = nsmallest diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -158,6 +158,15 @@ self.assertEqual(sorted(chain(*inputs)), list(self.module.merge(*inputs))) self.assertEqual(list(self.module.merge()), []) + def test_merge_does_not_suppress_index_error(self): + # Issue 19018: Heapq.merge suppresses IndexError from user generator + def iterable(): + s = list(range(10)) + for i in range(20): + yield s[i] # IndexError when i > 10 + with self.assertRaises(IndexError): + list(self.module.merge(iterable(), iterable())) + def test_merge_stability(self): class Int(int): pass diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -133,6 +133,7 @@ Matthew Boedicker Robin Boerdijk David Bolen +Wouter Bolsterlee Gawain Bolton Forest Bond Gregory Bond @@ -382,6 +383,7 @@ Frederik Fix Matt Fleming Hern?n Mart?nez Foffani +Artem Fokin Arnaud Fontaine Michael Foord Amaury Forgeot d'Arc diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -72,6 +72,9 @@ - Issue #17324: Fix http.server's request handling case on trailing '/'. Patch contributed by Vajrasky Kok. +- Issue #19018: The heapq.merge() function no longer suppresses IndexError + in the underlying iterables. + - Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, if all necessary functions are already found in libuuid. Patch by Evgeny Sologubov. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 05:53:13 2013 From: python-checkins at python.org (raymond.hettinger) Date: Sun, 15 Sep 2013 05:53:13 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?= =?utf-8?q?=29=3A_merge?= Message-ID: <3ccxWY1S8Pz7Lkc@mail.python.org> http://hg.python.org/cpython/rev/21994951ef5b changeset: 85710:21994951ef5b parent: 85708:b0517bb271ad parent: 85709:0ff5bb61c6a1 user: Raymond Hettinger date: Sat Sep 14 20:52:54 2013 -0700 summary: merge files: Lib/test/test_heapq.py | 9 +++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 3 files changed, 13 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -158,6 +158,15 @@ self.assertEqual(sorted(chain(*inputs)), list(self.module.merge(*inputs))) self.assertEqual(list(self.module.merge()), []) + def test_merge_does_not_suppress_index_error(self): + # Issue 19018: Heapq.merge suppresses IndexError from user generator + def iterable(): + s = list(range(10)) + for i in range(20): + yield s[i] # IndexError when i > 10 + with self.assertRaises(IndexError): + list(self.module.merge(iterable(), iterable())) + def test_merge_stability(self): class Int(int): pass diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -394,6 +394,7 @@ Frederik Fix Matt Fleming Hern?n Mart?nez Foffani +Artem Fokin Arnaud Fontaine Michael Foord Amaury Forgeot d'Arc diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -18,6 +18,9 @@ - Issue #17324: Fix http.server's request handling case on trailing '/'. Patch contributed by Vajrasky Kok. +- Issue #19018: The heapq.merge() function no longer suppresses IndexError + in the underlying iterables. + - Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, if all necessary functions are already found in libuuid. Patch by Evgeny Sologubov. -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Sun Sep 15 06:25:17 2013 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sun, 15 Sep 2013 06:25:17 +0200 Subject: [Python-checkins] Daily reference leaks (24aa6f98fe25): sum=4 Message-ID: results for 24aa6f98fe25 on branch "default" -------------------------------------------- test_support leaked [-1, 1, 0] references, sum=0 test_support leaked [-1, 3, 2] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogqXtKcA', '-x'] From python-checkins at python.org Sun Sep 15 07:17:51 2013 From: python-checkins at python.org (raymond.hettinger) Date: Sun, 15 Sep 2013 07:17:51 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE5MDE4?= =?utf-8?q?=3A_The_heapq=2Emerge=28=29_function_no_longer_suppresses_Index?= =?utf-8?q?Error?= Message-ID: <3cczPC0wCgz7LjT@mail.python.org> http://hg.python.org/cpython/rev/56a3c0bc4634 changeset: 85711:56a3c0bc4634 branch: 2.7 parent: 85685:b90ba60c5029 user: Raymond Hettinger date: Sat Sep 14 22:17:39 2013 -0700 summary: Issue #19018: The heapq.merge() function no longer suppresses IndexError files: Lib/heapq.py | 15 ++++++++++----- Lib/test/test_heapq.py | 9 +++++++++ Misc/ACKS | 2 ++ Misc/NEWS | 3 +++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/Lib/heapq.py b/Lib/heapq.py --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -366,6 +366,7 @@ ''' _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration + _len = len h = [] h_append = h.append @@ -377,17 +378,21 @@ pass heapify(h) - while 1: + while _len(h) > 1: try: - while 1: - v, itnum, next = s = h[0] # raises IndexError when h is empty + while True: + v, itnum, next = s = h[0] yield v s[0] = next() # raises StopIteration when exhausted _heapreplace(h, s) # restore heap condition except _StopIteration: _heappop(h) # remove empty iterator - except IndexError: - return + if h: + # fast case when only a single iterator remains + v, itnum, next = h[0] + yield v + for v in next.__self__: + yield v # Extend the implementations of nsmallest and nlargest to use a key= argument _nsmallest = nsmallest diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -158,6 +158,15 @@ self.assertEqual(sorted(chain(*inputs)), list(self.module.merge(*inputs))) self.assertEqual(list(self.module.merge()), []) + def test_merge_does_not_suppress_index_error(self): + # Issue 19018: Heapq.merge suppresses IndexError from user generator + def iterable(): + s = list(range(10)) + for i in range(20): + yield s[i] # IndexError when i > 10 + with self.assertRaises(IndexError): + list(self.module.merge(iterable(), iterable())) + def test_merge_stability(self): class Int(int): pass diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -110,6 +110,7 @@ Matthew Boedicker Robin Boerdijk David Bolen +Wouter Bolsterlee Gawain Bolton Gregory Bond Jurjen Bos @@ -313,6 +314,7 @@ Frederik Fix Matt Fleming Hern?n Mart?nez Foffani +Artem Fokin Arnaud Fontaine Michael Foord Amaury Forgeot d'Arc diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -35,6 +35,9 @@ - Issue #17324: Fix http.server's request handling case on trailing '/'. Patch contributed by Vajrasky Kok. +- Issue #19018: The heapq.merge() function no longer suppresses IndexError + in the underlying iterables. + - Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, if all necessary functions are already found in libuuid. Patch by Evgeny Sologubov. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 08:20:30 2013 From: python-checkins at python.org (nick.coghlan) Date: Sun, 15 Sep 2013 08:20:30 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Assorted_PEP_453_=28pip_boots?= =?utf-8?q?trapping=29_updates?= Message-ID: <3cd0nV1MPDz7LjT@mail.python.org> http://hg.python.org/peps/rev/0522bee830a6 changeset: 5116:0522bee830a6 user: Nick Coghlan date: Sun Sep 15 16:20:11 2013 +1000 summary: Assorted PEP 453 (pip bootstrapping) updates - revert to being explicitly 3.4 only (with rationale) - open question re uninstallation - open question re script execution on Windows - explicit cover handling of the setuptools dependency - spell out the proposed CLI options for getpip - note that venv will also support --no-download - explicitly note that pip may gain new features in CPython maintenance releases and continues using its own release cycle - explicitly note that bundling pip doesn't preclude the use of alternate installers, but instead better *enables* them by making them easier to bootstrap - note we should update to new pip version when they're released, so the release process just checks they're up to date. files: pep-0453.txt | 201 +++++++++++++++++++++++++++++++------- 1 files changed, 160 insertions(+), 41 deletions(-) diff --git a/pep-0453.txt b/pep-0453.txt --- a/pep-0453.txt +++ b/pep-0453.txt @@ -21,29 +21,31 @@ redistributors of Python to also provide pip by default (in a way reasonable for their distributions). -This PEP does *not* propose the inclusion of pip itself in the standard -library. - Proposal ======== This PEP proposes the inclusion of a ``getpip`` bootstrapping module in -Python 3.4, as well as in the upcoming maintenance releases of Python 2.7 -and Python 3.3. +Python 3.4. + +This PEP does *not* propose making pip (or any dependencies) part of the +standard library. Instead, pip will be a bundled application provided +along with CPython for the convenience of Python users, but subject to +its own development life cycle and able to be upgraded independently of +the core interpreter and standard library. Rationale ========= -Installing a third party package into a freshly installed Python requires first -installing the package manager. This requires users ahead of time to know what -the package manager is, where to get them from, and how to install them. The -effect of this is that these external projects are required to either blindly -assume the user already has the package manager installed, needs to duplicate -the instructions and tell their users how to install the package manager, or -completely forgo the use of dependencies to ease installation concerns for -their users. +Installing a third party package into a freshly installed Python requires +first installing the package manager. This requires users ahead of time to +know what the package manager is, where to get them from, and how to install +them. The effect of this is that these external projects are required to +either blindly assume the user already has the package manager installed, +needs to duplicate the instructions and tell their users how to install +the package manager, or completely forgo the use of dependencies to ease +installation concerns for their users. All of the available options have their own drawbacks. @@ -68,9 +70,11 @@ experience involved still isn't good (especially on Windows, where downloading and running a Python script with the default OS configuration is significantly more painful than downloading and running a binary executable -or installer). +or installer). The situation becomes even more complicated when multiple +Python versions are involved (for example, parallel installations of Python +2 and Python 3). -The projects that have decided to forgo dependencies all together are forced +The projects that have decided to forgo dependencies altogether are forced to either duplicate the efforts of other projects by inventing their own solutions to problems or are required to simply include the other projects in their own source trees. Both of these options present their own problems @@ -92,6 +96,14 @@ have a particular tool out of the box instead of needing to use the difficulty in installing a package as justification for inclusion. +Providing a standard installation system also helps with bootstrapping +alternate build and installer systems, such as ``setuptools``, +``zc.buildout`` and the ``hashdist``/``conda`` combination that is aimed +specifically at the scientific community. So long as +``pip install `` works, then a standard Python-specific installer +provides a reasonably secure, cross platform mechanism to get access to +these utilities. + Explicit Bootstrapping ====================== @@ -113,30 +125,79 @@ right, and evolves along with the rest of packaging. Instead of attempting to maintain a "mini pip" for the sole purpose of -installing pip the ``getpip`` module will, as an implementation detail, include -a private copy of pip which will be used to discover and install pip from PyPI. -It is important to stress that this private copy of pip is *only* an -implementation detail and it should *not* be relied on or assumed to exist. +installing pip the ``getpip`` module will, as an implementation detail, +include a private copy of pip and its dependencies which will be used to +discover and install pip from PyPI. It is important to stress that this +private copy of pip is *only* an implementation detail and it should *not* +be relied on or assumed to exist. -Not all users will have network access to PyPI whenever they run the bootstrap. -In order to ensure that these users will still be able to bootstrap pip the -bootstrap will fallback to simply installing the included copy of pip. +Not all users will have network access to PyPI whenever they run the +bootstrap. In order to ensure that these users will still be able to +bootstrap pip the bootstrap will fallback to simply installing the included +copy of pip. The pip ``--no-download`` command line option will be supported +to force installation of the bundled version, without even attempting to +contact PyPI. -This presents a balance between giving users the latest version of pip, saving -them from needing to immediately upgrade pip after bootstrapping it, and -allowing the bootstrap to work offline in situations where users might already -have packages downloaded that they wish to install. +This presents a balance between giving users the latest version of pip, +saving them from needing to immediately upgrade pip after bootstrapping it, +and allowing the bootstrap to work offline in situations where users might +already have packages downloaded that they wish to install. -Updating the Bundled pip +Proposed CLI +------------ + +The proposed CLI is based on a subset of the existing ``pip install`` +options:: + + Usage: + python -m getpip [options] + + Download Options: + --no-download Install the bundled version, don't attempt to download + -i, --index-url Base URL of Python Package Index (default https://pypi.python.org/simple/). + --proxy Specify a proxy in the form [user:passwd@]proxy.server:port. + --timeout Set the socket timeout (default 15 seconds). + --cert Path to alternate CA bundle. + + Installation Options: + -U, --upgrade Upgrade pip and dependencies, even if already installed + --user Install using the user scheme. + --root Install everything relative to this alternate root directory. + +Additional options (such as verbosity and logging options) may also +be supported. + + +Automatic installation of setuptools +------------------------------------ + +``pip`` currently depends on ``setuptools`` to handle metadata generation +during the build process, along with some other features. While work is +ongoing to reduce or eliminate this dependency, it is not clear if that +work will be complete for pip 1.5 (which is the version likely to be +bundled with Python 3.4.0). + +This PEP proposes that, if pip still requires it, ``setuptools`` will be +bundled along with pip itself, and thus installed when running +``python -m getpip``. + +However, this behaviour will be officially declared an implementation +detail. Other projects which explicitly require setuptools should still +provide an appropriate dependency declaration, rather than assuming +``setuptools`` will always be installed alongside ``pip``. + + +Updating the bundled pip ------------------------ In order to keep up with evolutions in packaging as well as providing users who are using the offline installation method with as recent version as -possible the ``getpip`` module should be updates to the latest versions of -everything it bootstraps. During the preparation for any release of Python, a -script, provided as part of this PEP, should be run to update the bundled -packages to the latest versions. +possible the ``getpip`` module should be updated to the latest versions of +everything it bootstraps. After each new pip release, and again during the +preparation for any release of Python, a script, provided as part of this +PEP, should be run to ensure the bundled packages have been updated to the +latest versions. This means that maintenance releases of the CPython installers will include an updated version of the ``getpip`` bootstrap module. @@ -153,16 +214,54 @@ The Windows and OSX installers distributed by Python.org will automatically attempt to run ``python -m getpip`` by default however the ``make install`` -and ``make altinstall`` commands of the source distribution will not. +and ``make altinstall`` commands of the source distribution will not. Note +that ``getpip`` itself will still be installed normally (as it is a regular +part of the standard library), only Keeping the pip bootstrapping as a separate step for make based installations should minimize the changes CPython redistributors need to make to their build processes. Avoiding the layer of indirection through -make for the getpip invocation also ensures those installing from a custom +``make`` for the getpip invocation also ensures those installing from a custom source build can easily force an offline installation of pip, install it from a private index server, or skip installing pip entirely. +Open Question: Uninstallation +============================= + +No changes are currently proposed to the uninstallation process. The +bootstrapped pip will be installed the same way as any other pip +installed packages, and will be handled in the same way as any other +post-install additions to the Python environment. + +At least on Windows, that means the bootstrapped files will be +left behind after uninstallation, since those files won't be associated +with the Python MSI installer. + +.. note:: + + Perhaps the installer needs to be updated to clobber everything in + site-packages and the Scripts directory, but I would prefer not to make + this PEP conditional on that change. + + +Open Question: Script Execution on Windows +========================================== + +While the Windows installer was updated in Python 3.3 to make ``python`` +available on the PATH, no such change was made to include the scripts +directory. This PEP proposes that this be changed to also add the scripts +directory. + +Without this change, the most reliable way to invoke pip on Windows (without +tinkering with paths) is actually be ``py -m pip`` (or ``py -3 -m pip`` +if both Python 2 and 3 are installed) rather than simply calling ``pip``. + +Adding the scripts directory to the system PATH would mean that ``pip`` +works reliably in the "only one Python installation" case, with +``py -m pip`` needed only for the parallel installation case. + + Python Virtual Environments =========================== @@ -178,9 +277,11 @@ will allow people the same convenience inside of the virtual environment as this PEP provides outside of it as well as bringing the ``venv`` module closer to feature parity with the external ``virtualenv`` package making it a more -suitable replacement. In the case that a user does not wish to have pip +suitable replacement. To handles cases where a user does not wish to have pip bootstrapped into their virtual environment a ``--without-pip`` option will be -added. +added. The ``--no-download`` option will also be supported, to force the +use of the bundled ``pip`` rather than retrieving the latest version from +PyPI. Recommendations for Downstream Distributors @@ -195,9 +296,9 @@ * Ensure that whenever Python is installed pip is also installed. - * This may take the form of separate with dependencies on each either so that - installing the python package installs the pip package and installing the - pip package installs the Python package. + * This may take the form of separate packages with dependencies on each + other so that installing the Python package installs the pip package + and installing the pip package installs the Python package. * Do not remove the bundled copy of pip. @@ -211,8 +312,8 @@ * Migrate build systems to utilize `pip`_ and `Wheel`_ instead of directly using ``setup.py``. - * This will ensure that downstream packages can utilize the new formats which - will not have a ``setup.py`` easier. + * This will ensure that downstream packages can more easily utilize the + new metadata formats which may not have a ``setup.py``. * Ensure that all features of this PEP continue to work with any modifications made. @@ -247,6 +348,10 @@ backwards compatibility policy of Python for its standard library. The externally developed software that this PEP bundles does not. +Most importantly, this means that the bundled version of pip may gain new +features in CPython maintenance releases, and pip continues to operate on +its own 6 month release cycle rather than CPython's 18-24 month cycle. + Security Releases ----------------- @@ -260,10 +365,24 @@ ============================ +Bundling the installer in Python 2.7 and 3.3 Maintenance Releases +----------------------------------------------------------------- + +Unlike earlier Python versions, Python 3.4 provides access to the system +certificate store on Windows systems. This allows ``getpip`` to create a +verified connection to PyPI without needing to include a custom certificate +bundle with CPython. + +Rather than trying to come up with a secure bootstrapping alternative for +earlier Python versions, the existing manual bootstrapping mechanism (which +relies on SSL verification in other tools like curl, wget and web browsers) +will continue to be used. + + Implicit Bootstrap ------------------ -`PEP439`_, the predecessor for this PEP, proposes it's own solution. Its +`PEP439`_, the predecessor for this PEP, proposes its own solution. Its solution involves shipping a fake ``pip`` command that when executed would implicitly bootstrap and install pip if it does not already exist. This has been rejected because it is too "magical". It hides from the end user when -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sun Sep 15 08:30:56 2013 From: python-checkins at python.org (nick.coghlan) Date: Sun, 15 Sep 2013 08:30:56 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Update_the_PEP_list_for_3=2E4?= Message-ID: <3cd11X1D9Hz7LjY@mail.python.org> http://hg.python.org/peps/rev/4a133ecd8fb2 changeset: 5117:4a133ecd8fb2 user: Nick Coghlan date: Sun Sep 15 16:30:43 2013 +1000 summary: Update the PEP list for 3.4 files: pep-0429.txt | 10 ++++++++-- 1 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pep-0429.txt b/pep-0429.txt --- a/pep-0429.txt +++ b/pep-0429.txt @@ -71,6 +71,7 @@ * PEP 442, improved semantics for object finalization * PEP 443, adding single-dispatch generic functions to the standard library * PEP 445, a new C API for implementing custom memory allocators +* PEP 446, make newly created file descriptors not inherited by default Other final large-scale changes: @@ -80,17 +81,22 @@ * PEP 431, improved support for time zone databases * PEP 436, a build-time preprocessor for builtin argument parsing -* PEP 446, explicit controls on file descriptor inheritance +* PEP 441, improved Python zip application support * PEP 447, support for __locallookup__ metaclass method * PEP 448, additional unpacking generalizations +* PEP 450, basic statistics module for the standard library +* PEP 451, making custom import hooks easier to implement correctly +* PEP 453, pip bootstrapping/bundling with CPython +* PEP 454, PEP 445 based malloc tracing support +* PEP 455, key transforming dictionary * PEP 3154, Pickle protocol revision 4 +* PEP 3156, improved asynchronous IO support Other proposed large-scale changes: * Introspection information for builtins * Addition of the "regex" module * Email version 6 -* A standard event-loop interface Deferred to post-3.4: -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sun Sep 15 10:37:33 2013 From: python-checkins at python.org (georg.brandl) Date: Sun, 15 Sep 2013 10:37:33 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Fixup_reST_syntax_errors_a?= =?utf-8?q?nd_streamline_docs_of_PEP_446=2E?= Message-ID: <3cd3qd5D05zQPw@mail.python.org> http://hg.python.org/cpython/rev/ed590800fde7 changeset: 85712:ed590800fde7 parent: 85710:21994951ef5b user: Georg Brandl date: Sun Sep 15 10:37:57 2013 +0200 summary: Fixup reST syntax errors and streamline docs of PEP 446. files: Doc/library/os.rst | 25 ++++++++++++------------- Doc/whatsnew/3.4.rst | 8 ++++---- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1206,41 +1206,40 @@ Inheritance of File Descriptors ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A file descriptor has a inheritable flag which indicates if the file descriptor -can be inherited or not in child processes. Since Python 3.4, file descriptors +.. versionadded:: 3.4 + +A file descriptor has an "inheritable" flag which indicates if the file descriptor +can be inherited by child processes. Since Python 3.4, file descriptors created by Python are non-inheritable by default. On UNIX, non-inheritable file descriptors are closed in child processes at the execution of a new program, other file descriptors are inherited. On Windows, non-inheritable handles and file descriptors are closed in child -processes, except standard streams (file descriptors 0, 1 and 2: stdin, stdout -and stderr) which are always inherited. Using :func:`os.spawn*` functions, +processes, except for standard streams (file descriptors 0, 1 and 2: stdin, stdout +and stderr), which are always inherited. Using :func:`os.spawn*` functions, all inheritable handles and all inheritable file descriptors are inherited. Using the :mod:`subprocess` module, all file descriptors except standard -streams are closed, inheritable handles are only inherited if the *close_fds* -parameter is ``False``. - -.. versionadded:: 3.4 +streams are closed, and inheritable handles are only inherited if the +*close_fds* parameter is ``False``. .. function:: get_inheritable(fd) - Get the `inheritable flag `_ of the specified file - descriptor. Return a :class:`bool`. + Get the "inheritable" flag of the specified file descriptor (a boolean). .. function:: set_inheritable(fd, inheritable) - Set the `inheritable flag `_ of the specified file descriptor. + Set the "inheritable" flag of the specified file descriptor. .. function:: get_handle_inheritable(handle) - Get the `inheritable flag `_ of the specified handle. Return a :class:`bool`. + Get the "inheritable" flag of the specified handle (a boolean). Availability: Windows. .. function:: set_handle_inheritable(handle, inheritable) - Set the `inheritable flag `_ of the specified handle. + Set the "inheritable" flag of the specified handle. Availability: Windows. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -126,8 +126,8 @@ PEP 446: Make newly created file descriptors non-inheritable ============================================================ -The :pep:`446` makes newly created file descriptors `non-inheritable -`_. New functions and methods: +The :pep:`446` makes newly created file descriptors :ref:`non-inheritable +`. New functions and methods: * :func:`os.get_inheritable`, :func:`os.set_inheritable` * :func:`os.get_handle_inheritable`, :func:`os.set_handle_inheritable` @@ -318,8 +318,8 @@ socket ------ -Socket objects have new methods to get or set their `inheritable flag -`_: +Socket objects have new methods to get or set their :ref:`inheritable flag +`: * :meth:`socket.socket.get_inheritable`, :meth:`socket.socket.set_inheritable` -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 15:49:55 2013 From: python-checkins at python.org (nick.coghlan) Date: Sun, 15 Sep 2013 15:49:55 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Add_3=2E3_and_2=2E7_back_to_p?= =?utf-8?q?ip_boostrap_PEP?= Message-ID: <3cdBm341kBzRqY@mail.python.org> http://hg.python.org/peps/rev/ea9babf171fb changeset: 5118:ea9babf171fb user: Nick Coghlan date: Sun Sep 15 23:49:45 2013 +1000 summary: Add 3.3 and 2.7 back to pip boostrap PEP Donald reminded me of both why we originally proposed that, and how the current implementation supports older Python versions files: pep-0453.txt | 57 ++++++++++++++++++++++++++++----------- 1 files changed, 41 insertions(+), 16 deletions(-) diff --git a/pep-0453.txt b/pep-0453.txt --- a/pep-0453.txt +++ b/pep-0453.txt @@ -26,7 +26,8 @@ ======== This PEP proposes the inclusion of a ``getpip`` bootstrapping module in -Python 3.4. +Python 3.4, as well as in the next maintenance releases of Python 3.3 and +2.7. This PEP does *not* propose making pip (or any dependencies) part of the standard library. Instead, pip will be a bundled application provided @@ -203,6 +204,28 @@ an updated version of the ``getpip`` bootstrap module. +Feature Addition in Maintenance Releases +======================================== + +Adding a new module to the standard library in Python 2.7 and 3.3 +maintenance releases breaks the usual policy of "no new features in +maintenance releases". + +It is being proposed in this case as the bootstrapping problem greatly +affects the experience of new users, especially on Python 2 where many +Python 3 standard library improvements are available as backports on PyPI, +but are not included in the Python 2 standard library. + +By updating Python 2.7, 3.3 and 3.4 to easily bootstrap the PyPI ecosystem, +this should aid the vast majority of Python users, rather than only those +with the freedom to adopt Python 3.4 as soon as it is released. + +This is also a matter of starting as we mean to continue: similar to IDLE +(see PEP 434), ``getpip`` will be permanently exempted from the "no new +features in maintenance releases" restriction, as it will include (and +rely on) upgraded versions of ``pip`` even in maintenance releases. + + Pre-installation ================ @@ -216,7 +239,8 @@ attempt to run ``python -m getpip`` by default however the ``make install`` and ``make altinstall`` commands of the source distribution will not. Note that ``getpip`` itself will still be installed normally (as it is a regular -part of the standard library), only +part of the standard library), only the installation of pip and its +dependencies will be skipped. Keeping the pip bootstrapping as a separate step for make based installations should minimize the changes CPython redistributors need to @@ -284,6 +308,18 @@ PyPI. +Bundling CA Certificates with CPython +===================================== + +The reference ``getpip`` implementation includes the ``pip`` CA +bundle along with the rest of pip. This means CPython effectively includes +a CA bundle that is used solely for ``getpip``. + +This is considered desirable, as it ensures that ``pip`` will behave the +same across all supported versions of Python, even those prior to Python +3.4 that cannot access the system certificate store on Windows. + + Recommendations for Downstream Distributors =========================================== @@ -308,6 +344,9 @@ "debundling" policy. * This does mean that if ``pip`` needs to be updated due to a security issue, so does the bundled version in the ``getpip`` bootstrap module + * However, altering the bundled version of pip to remove the embedded + CA certificate bundle and rely the system CA bundle instead is a + reasonable change. * Migrate build systems to utilize `pip`_ and `Wheel`_ instead of directly using ``setup.py``. @@ -365,20 +404,6 @@ ============================ -Bundling the installer in Python 2.7 and 3.3 Maintenance Releases ------------------------------------------------------------------ - -Unlike earlier Python versions, Python 3.4 provides access to the system -certificate store on Windows systems. This allows ``getpip`` to create a -verified connection to PyPI without needing to include a custom certificate -bundle with CPython. - -Rather than trying to come up with a secure bootstrapping alternative for -earlier Python versions, the existing manual bootstrapping mechanism (which -relies on SSL verification in other tools like curl, wget and web browsers) -will continue to be used. - - Implicit Bootstrap ------------------ -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sun Sep 15 16:37:01 2013 From: python-checkins at python.org (nick.coghlan) Date: Sun, 15 Sep 2013 16:37:01 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Defer_PEP_422_=28too_many_oth?= =?utf-8?q?er_things_going_on=29?= Message-ID: <3cdCpP0RPxz7Lkx@mail.python.org> http://hg.python.org/peps/rev/5193ab61bae9 changeset: 5119:5193ab61bae9 user: Nick Coghlan date: Mon Sep 16 00:36:52 2013 +1000 summary: Defer PEP 422 (too many other things going on) files: pep-0422.txt | 11 +++++++++-- 1 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pep-0422.txt b/pep-0422.txt --- a/pep-0422.txt +++ b/pep-0422.txt @@ -4,11 +4,11 @@ Last-Modified: $Date$ Author: Nick Coghlan , Daniel Urban -Status: Draft +Status: Deferred Type: Standards Track Content-Type: text/x-rst Created: 5-Jun-2012 -Python-Version: 3.4 +Python-Version: 3.5 Post-History: 5-Jun-2012, 10-Feb-2013 @@ -27,6 +27,13 @@ implementing a custom metaclass, and thus should provide a gentler introduction to the full power Python's metaclass machinery. +PEP Deferral +============ + +Deferred until 3.5 at the earliest. The last review raised a few interesting +points that I (Nick) need to consider further before proposing it for +inclusion, and that's not going to happen in the 3.4 timeframe. + Background ========== -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sun Sep 15 17:35:57 2013 From: python-checkins at python.org (donald.stufft) Date: Sun, 15 Sep 2013 17:35:57 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Update_post_history?= Message-ID: <3cdF6P0tn1z7LjQ@mail.python.org> http://hg.python.org/peps/rev/e9f14f91d334 changeset: 5120:e9f14f91d334 user: Donald Stufft date: Sun Sep 15 11:35:45 2013 -0400 summary: Update post history files: pep-0453.txt | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/pep-0453.txt b/pep-0453.txt --- a/pep-0453.txt +++ b/pep-0453.txt @@ -8,7 +8,7 @@ Type: Process Content-Type: text/x-rst Created: 10-Aug-2013 -Post-History: 30-Aug-2013 +Post-History: 30-Aug-2013, 15-Sep-2013 Abstract -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sun Sep 15 18:02:28 2013 From: python-checkins at python.org (brett.cannon) Date: Sun, 15 Sep 2013 18:02:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Grammar_touch-ups?= Message-ID: <3cdFj03DQvzPBn@mail.python.org> http://hg.python.org/peps/rev/5412ea80c0a8 changeset: 5121:5412ea80c0a8 user: Brett Cannon date: Sun Sep 15 12:02:22 2013 -0400 summary: Grammar touch-ups files: pep-0453.txt | 26 +++++++++++++------------- 1 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pep-0453.txt b/pep-0453.txt --- a/pep-0453.txt +++ b/pep-0453.txt @@ -17,7 +17,7 @@ This PEP proposes the inclusion of a method for explicitly bootstrapping `pip`_ as the default package manager for Python. It also proposes that the distributions of Python available via Python.org will automatically run this -explicit bootstrapping method and a recommendation to third party +explicit bootstrapping method and a recommendation to third-party redistributors of Python to also provide pip by default (in a way reasonable for their distributions). @@ -39,7 +39,7 @@ Rationale ========= -Installing a third party package into a freshly installed Python requires +Installing a third-party package into a freshly installed Python requires first installing the package manager. This requires users ahead of time to know what the package manager is, where to get them from, and how to install them. The effect of this is that these external projects are required to @@ -126,7 +126,7 @@ right, and evolves along with the rest of packaging. Instead of attempting to maintain a "mini pip" for the sole purpose of -installing pip the ``getpip`` module will, as an implementation detail, +installing pip, the ``getpip`` module will, as an implementation detail, include a private copy of pip and its dependencies which will be used to discover and install pip from PyPI. It is important to stress that this private copy of pip is *only* an implementation detail and it should *not* @@ -135,7 +135,7 @@ Not all users will have network access to PyPI whenever they run the bootstrap. In order to ensure that these users will still be able to bootstrap pip the bootstrap will fallback to simply installing the included -copy of pip. The pip ``--no-download`` command line option will be supported +copy of pip. The pip ``--no-download`` command-line option will be supported to force installation of the bundled version, without even attempting to contact PyPI. @@ -194,7 +194,7 @@ In order to keep up with evolutions in packaging as well as providing users who are using the offline installation method with as recent version as -possible the ``getpip`` module should be updated to the latest versions of +possible, the ``getpip`` module should be updated to the latest versions of everything it bootstraps. After each new pip release, and again during the preparation for any release of Python, a script, provided as part of this PEP, should be run to ensure the bundled packages have been updated to the @@ -229,20 +229,20 @@ Pre-installation ================ -During the installation of Python from Python.org ``python -m getpip`` should +During the installation of Python from Python.org, ``python -m getpip`` should be executed, leaving people using the Windows or OSX installers with a working copy of pip once the installation has completed. The exact method of this is left up to the maintainers of the installers, however if the bootstrapping is optional it should be opt-out rather than opt-in. The Windows and OSX installers distributed by Python.org will automatically -attempt to run ``python -m getpip`` by default however the ``make install`` +attempt to run ``python -m getpip`` by default, however the ``make install`` and ``make altinstall`` commands of the source distribution will not. Note that ``getpip`` itself will still be installed normally (as it is a regular part of the standard library), only the installation of pip and its dependencies will be skipped. -Keeping the pip bootstrapping as a separate step for make based +Keeping the pip bootstrapping as a separate step for ``make``-based installations should minimize the changes CPython redistributors need to make to their build processes. Avoiding the layer of indirection through ``make`` for the getpip invocation also ensures those installing from a custom @@ -278,7 +278,7 @@ directory. Without this change, the most reliable way to invoke pip on Windows (without -tinkering with paths) is actually be ``py -m pip`` (or ``py -3 -m pip`` +tinkering with paths) will actually be ``py -m pip`` (or ``py -3 -m pip`` if both Python 2 and 3 are installed) rather than simply calling ``pip``. Adding the scripts directory to the system PATH would mean that ``pip`` @@ -300,8 +300,8 @@ pip bootstrap by default inside of the new environment while creating it. This will allow people the same convenience inside of the virtual environment as this PEP provides outside of it as well as bringing the ``venv`` module closer -to feature parity with the external ``virtualenv`` package making it a more -suitable replacement. To handles cases where a user does not wish to have pip +to feature parity with the external ``virtualenv`` package, making it a more +suitable replacement. To handle cases where a user does not wish to have pip bootstrapped into their virtual environment a ``--without-pip`` option will be added. The ``--no-download`` option will also be supported, to force the use of the bundled ``pip`` rather than retrieving the latest version from @@ -325,8 +325,8 @@ A common source of Python installations are through downstream distributors such as the various Linux Distributions [#ubuntu]_ [#debian]_ [#fedora]_, OSX -package managers [#homebrew]_, or python specific tools [#conda]_. In order to -provide a consistent, user friendly experience to all users of Python +package managers [#homebrew]_, or python-specific tools [#conda]_. In order to +provide a consistent, user-friendly experience to all users of Python regardless of how they attained Python this PEP recommends and asks that downstream distributors: -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Sun Sep 15 18:37:33 2013 From: python-checkins at python.org (senthil.kumaran) Date: Sun, 15 Sep 2013 18:37:33 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Expose_--bind_argument_for?= =?utf-8?q?_http=2Eserver=2C_enable_http=2Eserver_to_bind_to_a_user?= Message-ID: <3cdGTT670Vz7Lkx@mail.python.org> http://hg.python.org/cpython/rev/a248655c8261 changeset: 85713:a248655c8261 user: Senthil Kumaran date: Sun Sep 15 09:37:27 2013 -0700 summary: Expose --bind argument for http.server, enable http.server to bind to a user specified network interface. Patch contributed by Malte Swart. Addresses issue #17764. HG :Enter commit message. Lines beginning with 'HG:' are removed. files: Doc/library/http.server.rst | 9 +++++++++ Lib/http/server.py | 14 +++++++++----- Misc/NEWS | 3 +++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -368,6 +368,15 @@ python -m http.server 8000 +By default, server binds itself to all interfaces. To restrict it to bind to a +particular interface only, ``--bind ADDRESS`` argument can be used. For e.g, to +restrict the server to bind only to localhost. :: + + python -m http.server 8000 --bind 127.0.0.1 + +.. versionadded:: 3.4 + ``--bind`` argument was introduced. + .. class:: CGIHTTPRequestHandler(request, client_address, server) diff --git a/Lib/http/server.py b/Lib/http/server.py --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -1186,15 +1186,15 @@ self.log_message("CGI script exited OK") -def test(HandlerClass = BaseHTTPRequestHandler, - ServerClass = HTTPServer, protocol="HTTP/1.0", port=8000): +def test(HandlerClass=BaseHTTPRequestHandler, + ServerClass=HTTPServer, protocol="HTTP/1.0", port=8000, bind=""): """Test the HTTP request handler class. This runs an HTTP server on port 8000 (or the first command line argument). """ - server_address = ('', port) + server_address = (bind, port) HandlerClass.protocol_version = protocol httpd = ServerClass(server_address, HandlerClass) @@ -1212,12 +1212,16 @@ parser = argparse.ArgumentParser() parser.add_argument('--cgi', action='store_true', help='Run as CGI Server') + parser.add_argument('--bind', '-b', default='', metavar='ADDRESS', + help='Specify alternate bind address ' + '[default: all interfaces]') parser.add_argument('port', action='store', default=8000, type=int, nargs='?', help='Specify alternate port [default: 8000]') args = parser.parse_args() if args.cgi: - test(HandlerClass=CGIHTTPRequestHandler, port=args.port) + handler_class = CGIHTTPRequestHandler else: - test(HandlerClass=SimpleHTTPRequestHandler, port=args.port) + handler_class = SimpleHTTPRequestHandler + test(HandlerClass=handler_class, port=args.port, bind=args.bind) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,9 @@ Library ------- +- Issue #17764: Enable http.server to bind to a user specified network + interface. Patch contributed by Malte Swart. + - Issue #18937: Add an assertLogs() context manager to unittest.TestCase to ensure that a block of code emits a message using the logging module. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 19:15:13 2013 From: python-checkins at python.org (andrew.kuchling) Date: Sun, 15 Sep 2013 19:15:13 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi42KTogIzE2MDQyOiBDVkUt?= =?utf-8?q?2013-1752=3A_Limit_amount_of_data_read_by_limiting_the_call_to?= Message-ID: <3cdHJx0pknz7LlY@mail.python.org> http://hg.python.org/cpython/rev/8a6def3add5b changeset: 85714:8a6def3add5b branch: 2.6 parent: 85537:07ee48ce4513 user: Andrew Kuchling date: Sun Sep 15 13:11:47 2013 -0400 summary: #16042: CVE-2013-1752: Limit amount of data read by limiting the call to readline(). The SSLFakeFile.readline() method needs to support limiting readline() as well. It's not a full emulation of readline()'s signature, but this class is only used by smtplib's code, so it doesn't have to be. Modified version of original patch by Christian Heimes. files: Lib/smtplib.py | 13 +++++++++-- Lib/test/test_smtplib.py | 29 +++++++++++++++++++++++++++- Misc/NEWS | 5 +++- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/Lib/smtplib.py b/Lib/smtplib.py --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -57,6 +57,7 @@ SMTP_PORT = 25 SMTP_SSL_PORT = 465 CRLF="\r\n" +_MAXLINE = 8192 # more than 8 times larger than RFC 821, 4.5.3 OLDSTYLE_AUTH = re.compile(r"auth=(.*)", re.I) @@ -170,10 +171,14 @@ def __init__(self, sslobj): self.sslobj = sslobj - def readline(self): + def readline(self, size=-1): + if size < 0: + size = None str = "" chr = None while chr != "\n": + if size is not None and len(str) >= size: + break chr = self.sslobj.read(1) if not chr: break str += chr @@ -334,11 +339,13 @@ if self.file is None: self.file = self.sock.makefile('rb') while 1: - line = self.file.readline() + line = self.file.readline(_MAXLINE + 1) if line == '': self.close() raise SMTPServerDisconnected("Connection unexpectedly closed") - if self.debuglevel > 0: print>>stderr, 'reply:', repr(line) + if self.debuglevel > 0: print >>stderr, 'reply:', repr(line) + if len(line) > _MAXLINE: + raise SMTPResponseException(500, "Line too long.") resp.append(line[4:].strip()) code=line[:3] # Check that the error code is syntactically correct. diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -273,6 +273,32 @@ HOST, self.port, 'localhost', 3) +class TooLongLineTests(TestCase): + respdata = '250 OK' + ('.' * smtplib._MAXLINE * 2) + '\n' + + def setUp(self): + self.old_stdout = sys.stdout + self.output = StringIO.StringIO() + sys.stdout = self.output + + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(15) + self.port = test_support.bind_port(self.sock) + servargs = (self.evt, self.respdata, self.sock) + threading.Thread(target=server, args=servargs).start() + self.evt.wait() + self.evt.clear() + + def tearDown(self): + self.evt.wait() + sys.stdout = self.old_stdout + + def testLineTooLong(self): + self.assertRaises(smtplib.SMTPResponseException, smtplib.SMTP, + HOST, self.port, 'localhost', 3) + + sim_users = {'Mr.A at somewhere.com':'John A', 'Ms.B at somewhere.com':'Sally B', 'Mrs.C at somewhereesle.com':'Ruth C', @@ -482,7 +508,8 @@ def test_main(verbose=None): test_support.run_unittest(GeneralTests, DebuggingServerTests, NonConnectingTests, - BadHELOServerTests, SMTPSimTests) + BadHELOServerTests, SMTPSimTests, + TooLongLineTests) if __name__ == '__main__': test_main() diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -16,13 +16,16 @@ - Issue #16248: Disable code execution from the user's home directory by tkinter when the -E flag is passed to Python. Patch by Zachary Ware. +- Issue #16042: CVE-2013-1752: smtplib: Limit amount of data read by + limiting the call to readline(). Original patch by Christian Heimes. + Extension Modules ----------------- - Issue #18709: Fix CVE-2013-4238. The SSL module now handles NULL bytes inside subjectAltName correctly. Formerly the module has used OpenSSL's GENERAL_NAME_print() function to get the string represention of ASN.1 - strings for `rfc822Name` (email), `dNSName` (DNS) and + strings for `rfc822Name` (email), `dNSName` (DNS) and `uniformResourceIdentifier` (URI). -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 19:24:51 2013 From: python-checkins at python.org (andrew.kuchling) Date: Sun, 15 Sep 2013 19:24:51 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMi42IC0+IDIuNyk6?= =?utf-8?q?_Null_merge_with_2=2E6?= Message-ID: <3cdHX30PM1z7LkK@mail.python.org> http://hg.python.org/cpython/rev/a9f147749b68 changeset: 85715:a9f147749b68 branch: 2.7 parent: 85711:56a3c0bc4634 parent: 85714:8a6def3add5b user: Andrew Kuchling date: Sun Sep 15 13:24:05 2013 -0400 summary: Null merge with 2.6 files: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 21:08:40 2013 From: python-checkins at python.org (eli.bendersky) Date: Sun, 15 Sep 2013 21:08:40 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogQ2xvc2UgIzE4OTQ1?= =?utf-8?q?=3A_Add_tests_for_tempfile_name_collision_handling=2E?= Message-ID: <3cdKqr6Kn0z7LkK@mail.python.org> http://hg.python.org/cpython/rev/fa54069eb8c4 changeset: 85716:fa54069eb8c4 branch: 2.7 user: Eli Bendersky date: Sun Sep 15 12:08:14 2013 -0700 summary: Close #18945: Add tests for tempfile name collision handling. Patch by Vlad Shcherbina files: Lib/test/test_tempfile.py | 85 ++++++++++++++++++++------ 1 files changed, 64 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -8,6 +8,7 @@ import sys import re import warnings +import contextlib import unittest from test import test_support as support @@ -270,6 +271,22 @@ test_classes.append(test__get_candidate_names) + at contextlib.contextmanager +def _inside_empty_temp_dir(): + dir = tempfile.mkdtemp() + try: + with support.swap_attr(tempfile, 'tempdir', dir): + yield + finally: + support.rmtree(dir) + + +def _mock_candidate_names(*names): + return support.swap_attr(tempfile, + '_get_candidate_names', + lambda: iter(names)) + + class test__mkstemp_inner(TC): """Test the internal function _mkstemp_inner.""" @@ -386,31 +403,36 @@ self.do_create(bin=0).write("blat\n") # XXX should test that the file really is a text file + def default_mkstemp_inner(self): + return tempfile._mkstemp_inner(tempfile.gettempdir(), + tempfile.template, + '', + tempfile._bin_openflags) + + def test_collision_with_existing_file(self): + # _mkstemp_inner tries another name when a file with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + (fd1, name1) = self.default_mkstemp_inner() + os.close(fd1) + self.assertTrue(name1.endswith('aaa')) + + (fd2, name2) = self.default_mkstemp_inner() + os.close(fd2) + self.assertTrue(name2.endswith('bbb')) + def test_collision_with_existing_directory(self): # _mkstemp_inner tries another name when a directory with # the chosen name already exists - container_dir = tempfile.mkdtemp() - try: - def mock_get_candidate_names(): - return iter(['aaa', 'aaa', 'bbb']) - with support.swap_attr(tempfile, - '_get_candidate_names', - mock_get_candidate_names): - dir = tempfile.mkdtemp(dir=container_dir) - self.assertTrue(dir.endswith('aaa')) + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + dir = tempfile.mkdtemp() + self.assertTrue(dir.endswith('aaa')) - flags = tempfile._bin_openflags - (fd, name) = tempfile._mkstemp_inner(container_dir, - tempfile.template, - '', - flags) - try: - self.assertTrue(name.endswith('bbb')) - finally: - os.close(fd) - os.unlink(name) - finally: - support.rmtree(container_dir) + (fd, name) = self.default_mkstemp_inner() + os.close(fd) + self.assertTrue(name.endswith('bbb')) test_classes.append(test__mkstemp_inner) @@ -587,6 +609,27 @@ finally: os.rmdir(dir) + def test_collision_with_existing_file(self): + # mkdtemp tries another name when a file with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + file = tempfile.NamedTemporaryFile(delete=False) + file.close() + self.assertTrue(file.name.endswith('aaa')) + dir = tempfile.mkdtemp() + self.assertTrue(dir.endswith('bbb')) + + def test_collision_with_existing_directory(self): + # mkdtemp tries another name when a directory with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + dir1 = tempfile.mkdtemp() + self.assertTrue(dir1.endswith('aaa')) + dir2 = tempfile.mkdtemp() + self.assertTrue(dir2.endswith('bbb')) + test_classes.append(test_mkdtemp) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sun Sep 15 21:34:58 2013 From: python-checkins at python.org (ethan.furman) Date: Sun, 15 Sep 2013 21:34:58 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Close_=2318989=3A_enum_mem?= =?utf-8?q?bers_will_no_longer_overwrite_other_attributes=2C_nor_be?= Message-ID: <3cdLQB2zX0z7LkK@mail.python.org> http://hg.python.org/cpython/rev/54ddd1124df8 changeset: 85717:54ddd1124df8 parent: 85713:a248655c8261 user: Ethan Furman date: Sun Sep 15 12:34:36 2013 -0700 summary: Close #18989: enum members will no longer overwrite other attributes, nor be overwritten by them. files: Doc/library/enum.rst | 6 +++++ Lib/enum.py | 36 +++++++++++++++++------------ Lib/test/test_enum.py | 37 +++++++++++++++++++++--------- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -154,6 +154,12 @@ >>> Shape(2) +.. note:: + + Attempting to create a member with the same name as an already + defined attribute (another member, a method, etc.) or attempting to create + an attribute with the same name as a member is not allowed. + Ensuring unique enumeration values ---------------------------------- diff --git a/Lib/enum.py b/Lib/enum.py --- a/Lib/enum.py +++ b/Lib/enum.py @@ -29,6 +29,14 @@ raise AttributeError("can't delete attribute") +def _is_descriptor(obj): + """Returns True if obj is a descriptor, False otherwise.""" + return ( + hasattr(obj, '__get__') or + hasattr(obj, '__set__') or + hasattr(obj, '__delete__')) + + def _is_dunder(name): """Returns True if a __dunder__ name, False otherwise.""" return (name[:2] == name[-2:] == '__' and @@ -50,8 +58,9 @@ cls.__reduce__ = _break_on_call_reduce cls.__module__ = '' + class _EnumDict(dict): - """Keeps track of definition order of the enum items. + """Track enum member order and ensure member names are not reused. EnumMeta will use the names found in self._member_names as the enumeration member names. @@ -62,11 +71,7 @@ self._member_names = [] def __setitem__(self, key, value): - """Changes anything not dundered or that doesn't have __get__. - - If a descriptor is added with the same name as an enum member, the name - is removed from _member_names (this may leave a hole in the numerical - sequence of values). + """Changes anything not dundered or not a descriptor. If an enum member name is used twice, an error is raised; duplicate values are not checked for. @@ -76,19 +81,20 @@ """ if _is_sunder(key):