From python-checkins at python.org  Mon Jul  1 00:38:19 2013
From: python-checkins at python.org (terry.reedy)
Date: Mon,  1 Jul 2013 00:38:19 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4MTg5?=
 =?utf-8?q?=3A_add_test=5Fdelegator_for_Idle_Delegator_class=2E?=
Message-ID: <3bk67H5QwNz7LnM@mail.python.org>

http://hg.python.org/cpython/rev/231c122b44b6
changeset:   84399:231c122b44b6
branch:      2.7
parent:      84392:a568a5426a16
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sun Jun 30 18:36:53 2013 -0400
summary:
  Issue #18189: add test_delegator for Idle Delegator class.
Also change private dict used as a set to a set.

files:
  Lib/idlelib/Delegator.py                |   6 +-
  Lib/idlelib/idle_test/test_delegator.py |  37 +++++++++++++
  2 files changed, 40 insertions(+), 3 deletions(-)


diff --git a/Lib/idlelib/Delegator.py b/Lib/idlelib/Delegator.py
--- a/Lib/idlelib/Delegator.py
+++ b/Lib/idlelib/Delegator.py
@@ -4,16 +4,16 @@
 
     def __init__(self, delegate=None):
         self.delegate = delegate
-        self.__cache = {}
+        self.__cache = set()
 
     def __getattr__(self, name):
         attr = getattr(self.delegate, name) # May raise AttributeError
         setattr(self, name, attr)
-        self.__cache[name] = attr
+        self.__cache.add(name)
         return attr
 
     def resetcache(self):
-        for key in self.__cache.keys():
+        for key in self.__cache:
             try:
                 delattr(self, key)
             except AttributeError:
diff --git a/Lib/idlelib/idle_test/test_delegator.py b/Lib/idlelib/idle_test/test_delegator.py
new file mode 100644
--- /dev/null
+++ b/Lib/idlelib/idle_test/test_delegator.py
@@ -0,0 +1,37 @@
+import unittest
+from idlelib.Delegator import Delegator
+
+class DelegatorTest(unittest.TestCase):
+
+    def test_mydel(self):
+        # test a simple use scenario
+
+        # initialize
+        mydel = Delegator(int)
+        self.assertIs(mydel.delegate, int)
+        self.assertEqual(mydel._Delegator__cache, set())
+
+        # add an attribute:
+        self.assertRaises(AttributeError, mydel.__getattr__, 'xyz')
+        bl = mydel.bit_length
+        self.assertIs(bl, int.bit_length)
+        self.assertIs(mydel.__dict__['bit_length'], int.bit_length)
+        self.assertEqual(mydel._Delegator__cache, {'bit_length'})
+
+        # add a second attribute
+        mydel.numerator
+        self.assertEqual(mydel._Delegator__cache, {'bit_length', 'numerator'})
+
+        # delete the second (which, however, leaves it in the name cache)
+        del mydel.numerator
+        self.assertNotIn('numerator', mydel.__dict__)
+        self.assertIn('numerator', mydel._Delegator__cache)
+
+        # reset by calling .setdelegate, which calls .resetcache
+        mydel.setdelegate(float)
+        self.assertIs(mydel.delegate, float)
+        self.assertNotIn('bit_length', mydel.__dict__)
+        self.assertEqual(mydel._Delegator__cache, set())
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2, exit=2)

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 00:38:21 2013
From: python-checkins at python.org (terry.reedy)
Date: Mon,  1 Jul 2013 00:38:21 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4MTg5?=
 =?utf-8?q?=3A_add_test=5Fdelegator_for_Idle_Delegator_class=2E?=
Message-ID: <3bk67K0FNdz7Lnc@mail.python.org>

http://hg.python.org/cpython/rev/c7605471e8ae
changeset:   84400:c7605471e8ae
branch:      3.3
parent:      84397:c17fa2cbad43
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sun Jun 30 18:37:05 2013 -0400
summary:
  Issue #18189: add test_delegator for Idle Delegator class.
Also change private dict used as a set to a set.

files:
  Lib/idlelib/Delegator.py                |   4 +-
  Lib/idlelib/idle_test/test_delegator.py |  37 +++++++++++++
  2 files changed, 39 insertions(+), 2 deletions(-)


diff --git a/Lib/idlelib/Delegator.py b/Lib/idlelib/Delegator.py
--- a/Lib/idlelib/Delegator.py
+++ b/Lib/idlelib/Delegator.py
@@ -4,12 +4,12 @@
 
     def __init__(self, delegate=None):
         self.delegate = delegate
-        self.__cache = {}
+        self.__cache = set()
 
     def __getattr__(self, name):
         attr = getattr(self.delegate, name) # May raise AttributeError
         setattr(self, name, attr)
-        self.__cache[name] = attr
+        self.__cache.add(name)
         return attr
 
     def resetcache(self):
diff --git a/Lib/idlelib/idle_test/test_delegator.py b/Lib/idlelib/idle_test/test_delegator.py
new file mode 100644
--- /dev/null
+++ b/Lib/idlelib/idle_test/test_delegator.py
@@ -0,0 +1,37 @@
+import unittest
+from idlelib.Delegator import Delegator
+
+class DelegatorTest(unittest.TestCase):
+
+    def test_mydel(self):
+        # test a simple use scenario
+
+        # initialize
+        mydel = Delegator(int)
+        self.assertIs(mydel.delegate, int)
+        self.assertEqual(mydel._Delegator__cache, set())
+
+        # add an attribute:
+        self.assertRaises(AttributeError, mydel.__getattr__, 'xyz')
+        bl = mydel.bit_length
+        self.assertIs(bl, int.bit_length)
+        self.assertIs(mydel.__dict__['bit_length'], int.bit_length)
+        self.assertEqual(mydel._Delegator__cache, {'bit_length'})
+
+        # add a second attribute
+        mydel.numerator
+        self.assertEqual(mydel._Delegator__cache, {'bit_length', 'numerator'})
+
+        # delete the second (which, however, leaves it in the name cache)
+        del mydel.numerator
+        self.assertNotIn('numerator', mydel.__dict__)
+        self.assertIn('numerator', mydel._Delegator__cache)
+
+        # reset by calling .setdelegate, which calls .resetcache
+        mydel.setdelegate(float)
+        self.assertIs(mydel.delegate, float)
+        self.assertNotIn('bit_length', mydel.__dict__)
+        self.assertEqual(mydel._Delegator__cache, set())
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2, exit=2)

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 00:38:22 2013
From: python-checkins at python.org (terry.reedy)
Date: Mon,  1 Jul 2013 00:38:22 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
	=?utf-8?q?=29=3A_Merge_with_3=2E3?=
Message-ID: <3bk67L25v2z7LnM@mail.python.org>

http://hg.python.org/cpython/rev/dbdb6f7f9a1a
changeset:   84401:dbdb6f7f9a1a
parent:      84398:ae69436eb7c2
parent:      84400:c7605471e8ae
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sun Jun 30 18:37:51 2013 -0400
summary:
  Merge with 3.3

files:
  Lib/idlelib/Delegator.py                |   4 +-
  Lib/idlelib/idle_test/test_delegator.py |  37 +++++++++++++
  2 files changed, 39 insertions(+), 2 deletions(-)


diff --git a/Lib/idlelib/Delegator.py b/Lib/idlelib/Delegator.py
--- a/Lib/idlelib/Delegator.py
+++ b/Lib/idlelib/Delegator.py
@@ -4,12 +4,12 @@
 
     def __init__(self, delegate=None):
         self.delegate = delegate
-        self.__cache = {}
+        self.__cache = set()
 
     def __getattr__(self, name):
         attr = getattr(self.delegate, name) # May raise AttributeError
         setattr(self, name, attr)
-        self.__cache[name] = attr
+        self.__cache.add(name)
         return attr
 
     def resetcache(self):
diff --git a/Lib/idlelib/idle_test/test_delegator.py b/Lib/idlelib/idle_test/test_delegator.py
new file mode 100644
--- /dev/null
+++ b/Lib/idlelib/idle_test/test_delegator.py
@@ -0,0 +1,37 @@
+import unittest
+from idlelib.Delegator import Delegator
+
+class DelegatorTest(unittest.TestCase):
+
+    def test_mydel(self):
+        # test a simple use scenario
+
+        # initialize
+        mydel = Delegator(int)
+        self.assertIs(mydel.delegate, int)
+        self.assertEqual(mydel._Delegator__cache, set())
+
+        # add an attribute:
+        self.assertRaises(AttributeError, mydel.__getattr__, 'xyz')
+        bl = mydel.bit_length
+        self.assertIs(bl, int.bit_length)
+        self.assertIs(mydel.__dict__['bit_length'], int.bit_length)
+        self.assertEqual(mydel._Delegator__cache, {'bit_length'})
+
+        # add a second attribute
+        mydel.numerator
+        self.assertEqual(mydel._Delegator__cache, {'bit_length', 'numerator'})
+
+        # delete the second (which, however, leaves it in the name cache)
+        del mydel.numerator
+        self.assertNotIn('numerator', mydel.__dict__)
+        self.assertIn('numerator', mydel._Delegator__cache)
+
+        # reset by calling .setdelegate, which calls .resetcache
+        mydel.setdelegate(float)
+        self.assertIs(mydel.delegate, float)
+        self.assertNotIn('bit_length', mydel.__dict__)
+        self.assertEqual(mydel._Delegator__cache, set())
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2, exit=2)

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 01:08:18 2013
From: python-checkins at python.org (terry.reedy)
Date: Mon,  1 Jul 2013 01:08:18 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzg1MTU6?=
 =?utf-8?q?_Set_=5F=5Ffile=5F=5F_when_run_file_in_IDLE=2E_Backport_2c276d0?=
 =?utf-8?q?553ff_by?=
Message-ID: <3bk6nt4kTkz7LmZ@mail.python.org>

http://hg.python.org/cpython/rev/a958b7f16a7d
changeset:   84402:a958b7f16a7d
branch:      2.7
parent:      84399:231c122b44b6
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sun Jun 30 19:07:49 2013 -0400
summary:
  Issue #8515: Set __file__ when run file in IDLE. Backport 2c276d0553ff by
Andrew Svetlov, based on initial patch by Bruce Frederiksen.

files:
  Lib/idlelib/ScriptBinding.py |  12 ++++++------
  Misc/ACKS                    |   1 +
  Misc/NEWS                    |   3 +++
  3 files changed, 10 insertions(+), 6 deletions(-)


diff --git a/Lib/idlelib/ScriptBinding.py b/Lib/idlelib/ScriptBinding.py
--- a/Lib/idlelib/ScriptBinding.py
+++ b/Lib/idlelib/ScriptBinding.py
@@ -152,16 +152,16 @@
         dirname = os.path.dirname(filename)
         # XXX Too often this discards arguments the user just set...
         interp.runcommand("""if 1:
-            _filename = %r
+            __file__ = {filename!r}
             import sys as _sys
             from os.path import basename as _basename
             if (not _sys.argv or
-                _basename(_sys.argv[0]) != _basename(_filename)):
-                _sys.argv = [_filename]
+                _basename(_sys.argv[0]) != _basename(__file__)):
+                _sys.argv = [__file__]
             import os as _os
-            _os.chdir(%r)
-            del _filename, _sys, _basename, _os
-            \n""" % (filename, dirname))
+            _os.chdir({dirname!r})
+            del _sys, _basename, _os
+            \n""".format(filename=filename, dirname=dirname))
         interp.prepend_syspath(filename)
         # XXX KBK 03Jul04 When run w/o subprocess, runtime warnings still
         #         go to __stderr__.  With subprocess, they go to the shell.
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -317,6 +317,7 @@
 John Fouhy
 Stefan Franke
 Martin Franklin
+Bruce Frederiksen
 Robin Friedrich
 Bradley Froehle
 Ivan Frohne
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -73,6 +73,9 @@
 IDLE
 ----
 
+- Issue #8515: Set __file__ when run file in IDLE.
+  Initial patch by Bruce Frederiksen.
+
 - Issue #5492: Avoid traceback when exiting IDLE caused by a race condition.
 
 - Issue #17511: Keep IDLE find dialog open after clicking "Find Next".

-- 
Repository URL: http://hg.python.org/cpython

From solipsis at pitrou.net  Mon Jul  1 05:45:20 2013
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Mon, 01 Jul 2013 05:45:20 +0200
Subject: [Python-checkins] Daily reference leaks (dbdb6f7f9a1a): sum=0
Message-ID: <E1UtV3I-0007Sz-KH@ap.vmr.nerim.net>

results for dbdb6f7f9a1a on branch "default"
--------------------------------------------



Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/refloguuf90q', '-x']

From python-checkins at python.org  Mon Jul  1 06:52:44 2013
From: python-checkins at python.org (terry.reedy)
Date: Mon,  1 Jul 2013 06:52:44 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzcxMzY6?=
 =?utf-8?q?_In_the_Idle_File_menu=2C_=22New_Window=22_is_renamed_=22New_Fi?=
 =?utf-8?q?le=22=2E?=
Message-ID: <3bkGRJ1p8Rz7Lkw@mail.python.org>

http://hg.python.org/cpython/rev/26ef5d5d5c3e
changeset:   84403:26ef5d5d5c3e
branch:      2.7
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Mon Jul 01 00:42:44 2013 -0400
summary:
  Issue #7136: In the Idle File menu, "New Window" is renamed "New File".
Patch by Tal Einat, Roget Serwy, and Todd Rovito.

files:
  Doc/library/idle.rst    |  4 ++--
  Lib/idlelib/Bindings.py |  2 +-
  Lib/idlelib/help.txt    |  2 +-
  Misc/NEWS               |  3 +++
  4 files changed, 7 insertions(+), 4 deletions(-)


diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst
--- a/Doc/library/idle.rst
+++ b/Doc/library/idle.rst
@@ -33,8 +33,8 @@
 File menu
 ^^^^^^^^^
 
-New window
-   create a new editing window
+New file
+   create a new file editing window
 
 Open...
    open an existing file
diff --git a/Lib/idlelib/Bindings.py b/Lib/idlelib/Bindings.py
--- a/Lib/idlelib/Bindings.py
+++ b/Lib/idlelib/Bindings.py
@@ -15,7 +15,7 @@
 menudefs = [
  # underscore prefixes character to underscore
  ('file', [
-   ('_New Window', '<<open-new-window>>'),
+   ('_New File', '<<open-new-window>>'),
    ('_Open...', '<<open-window-from-file>>'),
    ('Open _Module...', '<<open-module>>'),
    ('Class _Browser', '<<open-class-browser>>'),
diff --git a/Lib/idlelib/help.txt b/Lib/idlelib/help.txt
--- a/Lib/idlelib/help.txt
+++ b/Lib/idlelib/help.txt
@@ -5,7 +5,7 @@
 
 File Menu:
 
-	New Window       -- Create a new editing window
+	New File         -- Create a new editing window
 	Open...          -- Open an existing file
 	Recent Files...  -- Open a list of recent files
 	Open Module...   -- Open an existing module (searches sys.path)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -73,6 +73,9 @@
 IDLE
 ----
 
+- Issue #7136: In the Idle File menu, "New Window" is renamed "New File".
+  Patch by Tal Einat, Roget Serwy, and Todd Rovito.
+  
 - Issue #8515: Set __file__ when run file in IDLE.
   Initial patch by Bruce Frederiksen.
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 06:52:45 2013
From: python-checkins at python.org (terry.reedy)
Date: Mon,  1 Jul 2013 06:52:45 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzcxMzY6?=
 =?utf-8?q?_In_the_Idle_File_menu=2C_=22New_Window=22_is_renamed_=22New_Fi?=
 =?utf-8?q?le=22=2E?=
Message-ID: <3bkGRK3tzmz7Llg@mail.python.org>

http://hg.python.org/cpython/rev/c39ddff53694
changeset:   84404:c39ddff53694
branch:      3.3
parent:      84400:c7605471e8ae
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Mon Jul 01 00:42:52 2013 -0400
summary:
  Issue #7136: In the Idle File menu, "New Window" is renamed "New File".
Patch by Tal Einat, Roget Serwy, and Todd Rovito.

files:
  Doc/library/idle.rst    |  4 ++--
  Lib/idlelib/Bindings.py |  2 +-
  Lib/idlelib/help.txt    |  2 +-
  Misc/NEWS               |  3 +++
  4 files changed, 7 insertions(+), 4 deletions(-)


diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst
--- a/Doc/library/idle.rst
+++ b/Doc/library/idle.rst
@@ -33,8 +33,8 @@
 File menu
 ^^^^^^^^^
 
-New window
-   create a new editing window
+New file
+   create a new file editing window
 
 Open...
    open an existing file
diff --git a/Lib/idlelib/Bindings.py b/Lib/idlelib/Bindings.py
--- a/Lib/idlelib/Bindings.py
+++ b/Lib/idlelib/Bindings.py
@@ -15,7 +15,7 @@
 menudefs = [
  # underscore prefixes character to underscore
  ('file', [
-   ('_New Window', '<<open-new-window>>'),
+   ('_New File', '<<open-new-window>>'),
    ('_Open...', '<<open-window-from-file>>'),
    ('Open _Module...', '<<open-module>>'),
    ('Class _Browser', '<<open-class-browser>>'),
diff --git a/Lib/idlelib/help.txt b/Lib/idlelib/help.txt
--- a/Lib/idlelib/help.txt
+++ b/Lib/idlelib/help.txt
@@ -5,7 +5,7 @@
 
 File Menu:
 
-	New Window       -- Create a new editing window
+	New File         -- Create a new file editing window
 	Open...          -- Open an existing file
 	Recent Files...  -- Open a list of recent files
 	Open Module...   -- Open an existing module (searches sys.path)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -116,6 +116,9 @@
 IDLE
 ----
 
+- Issue #7136: In the Idle File menu, "New Window" is renamed "New File".
+  Patch by Tal Einat, Roget Serwy, and Todd Rovito.
+
 - Remove dead imports of imp.
 
 - Issue #18196: Avoid displaying spurious SystemExit tracebacks.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 06:52:46 2013
From: python-checkins at python.org (terry.reedy)
Date: Mon,  1 Jul 2013 06:52:46 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_=28merge=29_Issue_=237136=3A_In_the_Idle_File_menu=2C_?=
 =?utf-8?q?=22New_Window=22_is_renamed_=22New_File=22=2E?=
Message-ID: <3bkGRL5rkCz7Lpd@mail.python.org>

http://hg.python.org/cpython/rev/5bc3d8d22a93
changeset:   84405:5bc3d8d22a93
parent:      84401:dbdb6f7f9a1a
parent:      84404:c39ddff53694
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Mon Jul 01 00:52:18 2013 -0400
summary:
  (merge) Issue #7136: In the Idle File menu, "New Window" is renamed "New File".
Patch by Tal Einat, Roget Serwy, and Todd Rovito.

files:
  Doc/library/idle.rst    |  4 ++--
  Lib/idlelib/Bindings.py |  2 +-
  Lib/idlelib/help.txt    |  2 +-
  Misc/NEWS               |  3 +++
  4 files changed, 7 insertions(+), 4 deletions(-)


diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst
--- a/Doc/library/idle.rst
+++ b/Doc/library/idle.rst
@@ -39,8 +39,8 @@
 File menu (Shell and Editor)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-New window
-   Create a new editing window
+New file
+   Create a new file editing window
 
 Open...
    Open an existing file
diff --git a/Lib/idlelib/Bindings.py b/Lib/idlelib/Bindings.py
--- a/Lib/idlelib/Bindings.py
+++ b/Lib/idlelib/Bindings.py
@@ -15,7 +15,7 @@
 menudefs = [
  # underscore prefixes character to underscore
  ('file', [
-   ('_New Window', '<<open-new-window>>'),
+   ('_New File', '<<open-new-window>>'),
    ('_Open...', '<<open-window-from-file>>'),
    ('Open _Module...', '<<open-module>>'),
    ('Class _Browser', '<<open-class-browser>>'),
diff --git a/Lib/idlelib/help.txt b/Lib/idlelib/help.txt
--- a/Lib/idlelib/help.txt
+++ b/Lib/idlelib/help.txt
@@ -21,7 +21,7 @@
 
 File Menu (Shell and Editor):
 
-        New Window       -- Create a new editing window
+        New File         -- Create a new file editing window
         Open...          -- Open an existing file
         Open Module...   -- Open an existing module (searches sys.path)
         Recent Files...  -- Open a list of recent files
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -543,6 +543,9 @@
 IDLE
 ----
 
+- Issue #7136: In the Idle File menu, "New Window" is renamed "New File".
+  Patch by Tal Einat, Roget Serwy, and Todd Rovito.
+
 - Remove dead imports of imp.
 
 - Issue #18196: Avoid displaying spurious SystemExit tracebacks.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 13:08:57 2013
From: python-checkins at python.org (christian.heimes)
Date: Mon,  1 Jul 2013 13:08:57 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_18240=3A_The_HMAC_mo?=
 =?utf-8?q?dule_is_no_longer_restricted_to_bytes_and_accepts?=
Message-ID: <3bkQnP0KNnz7LkG@mail.python.org>

http://hg.python.org/cpython/rev/636947fe131e
changeset:   84406:636947fe131e
user:        Christian Heimes <christian at cheimes.de>
date:        Mon Jul 01 13:08:42 2013 +0200
summary:
  Issue 18240: The HMAC module is no longer restricted to bytes and accepts
any bytes-like object, e.g. memoryview. Original patch by Jonas Borgstr?m.

files:
  Doc/library/hmac.rst  |  18 ++++++++++++------
  Lib/hmac.py           |   8 +++-----
  Lib/test/test_hmac.py |  14 ++++++++++++++
  Misc/ACKS             |   1 +
  Misc/NEWS             |   3 +++
  5 files changed, 33 insertions(+), 11 deletions(-)


diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst
--- a/Doc/library/hmac.rst
+++ b/Doc/library/hmac.rst
@@ -16,20 +16,26 @@
 
 .. function:: new(key, msg=None, digestmod=None)
 
-   Return a new hmac object.  *key* is a bytes object giving the secret key.  If
-   *msg* is present, the method call ``update(msg)`` is made.  *digestmod* is
-   the digest constructor or module for the HMAC object to use. It defaults to
-   the :func:`hashlib.md5` constructor.
+   Return a new hmac object.  *key* is a bytes or bytearray object giving the
+   secret key.  If *msg* is present, the method call ``update(msg)`` is made.
+   *digestmod* is the digest constructor or module for the HMAC object to use.
+   It defaults to the :func:`hashlib.md5` constructor.
 
+   .. versionchanged:: 3.4
+      Parameter *key* can be a bytes or bytearray object. Parameter *msg* can
+      be of any type supported by :mod:`hashlib`.
 
 An HMAC object has the following methods:
 
 .. method:: HMAC.update(msg)
 
-   Update the hmac object with the bytes object *msg*.  Repeated calls are
-   equivalent to a single call with the concatenation of all the arguments:
+   Update the hmac object with *msg*.  Repeated calls are equivalent to a
+   single call with the concatenation of all the arguments:
    ``m.update(a); m.update(b)`` is equivalent to ``m.update(a + b)``.
 
+   .. versionchanged:: 3.4
+      Parameter *msg* can be of any type supported by :mod:`hashlib`.
+
 
 .. method:: HMAC.digest()
 
diff --git a/Lib/hmac.py b/Lib/hmac.py
--- a/Lib/hmac.py
+++ b/Lib/hmac.py
@@ -31,11 +31,11 @@
                    A hashlib constructor returning a new hash object.
                    Defaults to hashlib.md5.
 
-        Note: key and msg must be bytes objects.
+        Note: key and msg must be a bytes or bytearray objects.
         """
 
-        if not isinstance(key, bytes):
-            raise TypeError("key: expected bytes, but got %r" % type(key).__name__)
+        if not isinstance(key, (bytes, bytearray)):
+            raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__)
 
         if digestmod is None:
             import hashlib
@@ -75,8 +75,6 @@
     def update(self, msg):
         """Update this hashing object with the string msg.
         """
-        if not isinstance(msg, bytes):
-            raise TypeError("expected bytes, but got %r" % type(msg).__name__)
         self.inner.update(msg)
 
     def copy(self):
diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py
--- a/Lib/test/test_hmac.py
+++ b/Lib/test/test_hmac.py
@@ -253,6 +253,20 @@
         except:
             self.fail("Constructor call with text argument raised exception.")
 
+    def test_with_bytearray(self):
+        try:
+            h = hmac.HMAC(bytearray(b"key"), bytearray(b"hash this!"))
+            self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+        except:
+            self.fail("Constructor call with bytearray arguments raised exception.")
+
+    def test_with_memoryview_msg(self):
+        try:
+            h = hmac.HMAC(b"key", memoryview(b"hash this!"))
+            self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+        except:
+            self.fail("Constructor call with memoryview msg raised exception.")
+
     def test_withmodule(self):
         # Constructor call with text and digest module.
         try:
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -138,6 +138,7 @@
 Forest Bond
 Gregory Bond
 Matias Bordese
+Jonas Borgstr?m
 Jurjen Bos
 Peter Bosch
 Dan Boswell
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -135,6 +135,9 @@
 Library
 -------
 
+- Issue 18240: The HMAC module is no longer restricted to bytes and accepts
+  any bytes-like object, e.g. memoryview. Original patch by Jonas Borgstr?m.
+
 - Issue #18224: Removed pydoc script from created venv, as it causes problems
   on Windows and adds no value over and above python -m pydoc ...
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 14:46:25 2013
From: python-checkins at python.org (lukasz.langa)
Date: Mon,  1 Jul 2013 14:46:25 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_Use_C3-based_linearization_fo?=
 =?utf-8?q?r_ABC_support_to_improve_predictability?=
Message-ID: <3bkSxs70tYz7LkL@mail.python.org>

http://hg.python.org/peps/rev/000a8986ef73
changeset:   4968:000a8986ef73
user:        ?ukasz Langa <lukasz at langa.pl>
date:        Mon Jul 01 14:46:15 2013 +0200
summary:
  Use C3-based linearization for ABC support to improve predictability

files:
  pep-0443.txt |  73 ++++++++++++++++++++++-----------------
  1 files changed, 41 insertions(+), 32 deletions(-)


diff --git a/pep-0443.txt b/pep-0443.txt
--- a/pep-0443.txt
+++ b/pep-0443.txt
@@ -193,48 +193,37 @@
 importantly, it introduces support for Abstract Base Classes (ABC).
 
 When a generic function implementation is registered for an ABC, the
-dispatch algorithm switches to a mode of MRO calculation for the
-provided argument which includes the relevant ABCs. The algorithm is as
-follows::
+dispatch algorithm switches to an extended form of C3 linearization,
+which includes the relevant ABCs in the MRO of the provided argument.
+The algorithm inserts ABCs where their functionality is introduced, i.e.
+``issubclass(cls, abc)`` returns ``True`` for the class itself but
+returns ``False`` for all its direct base classes. Implicit ABCs for
+a given class (either registered or inferred from the presence of
+a special method like ``__len__()``) are inserted directly after the
+last ABC explicitly listed in the MRO of said class.
 
-  def _compose_mro(cls, haystack):
-      """Calculates the MRO for a given class `cls`, including relevant
-      abstract base classes from `haystack`."""
-      bases = set(cls.__mro__)
-      mro = list(cls.__mro__)
-      for regcls in haystack:
-          if regcls in bases or not issubclass(cls, regcls):
-              continue   # either present in the __mro__ or unrelated
-          for index, base in enumerate(mro):
-              if not issubclass(base, regcls):
-                  break
-          if base in bases and not issubclass(regcls, base):
-              # Conflict resolution: put classes present in __mro__
-              # and their subclasses first.
-              index += 1
-          mro.insert(index, regcls)
-      return mro
-
-In its most basic form, it returns the MRO for the given type::
+In its most basic form, this linearization returns the MRO for the given
+type::
 
   >>> _compose_mro(dict, [])
   [<class 'dict'>, <class 'object'>]
 
-When the haystack consists of ABCs that the specified type is a subclass
-of, they are inserted in a predictable order::
+When the second argument contains ABCs that the specified type is
+a subclass of, they are inserted in a predictable order::
 
   >>> _compose_mro(dict, [Sized, MutableMapping, str,
   ...                     Sequence, Iterable])
   [<class 'dict'>, <class 'collections.abc.MutableMapping'>,
-   <class 'collections.abc.Iterable'>, <class 'collections.abc.Sized'>,
+   <class 'collections.abc.Mapping'>, <class 'collections.abc.Sized'>,
+   <class 'collections.abc.Iterable'>, <class 'collections.abc.Container'>,
    <class 'object'>]
 
 While this mode of operation is significantly slower, all dispatch
 decisions are cached. The cache is invalidated on registering new
 implementations on the generic function or when user code calls
-``register()`` on an ABC to register a new virtual subclass. In the
-latter case, it is possible to create a situation with ambiguous
-dispatch, for instance::
+``register()`` on an ABC to implicitly subclass it. In the latter case,
+it is possible to create a situation with ambiguous dispatch, for
+instance::
 
   >>> from collections import Iterable, Container
   >>> class P:
@@ -261,20 +250,38 @@
   RuntimeError: Ambiguous dispatch: <class 'collections.abc.Container'>
   or <class 'collections.abc.Iterable'>
 
-Note that this exception would not be raised if ``Iterable`` and
-``Container`` had been provided as base classes during class definition.
-In this case dispatch happens in the MRO order::
+Note that this exception would not be raised if one or more ABCs had
+been provided explicitly as base classes during class definition. In
+this case dispatch happens in the MRO order::
 
   >>> class Ten(Iterable, Container):
   ...     def __iter__(self):
   ...         for i in range(10):
   ...             yield i
   ...     def __contains__(self, value):
-  ...       return value in range(10)
+  ...         return value in range(10)
   ...
   >>> g(Ten())
   'iterable'
 
+A similar conflict arises when subclassing an ABC is inferred from the
+presence of a special method like ``__len__()`` or ``__contains__()``::
+
+  >>> class Q:
+  ...   def __contains__(self, value):
+  ...     return False
+  ...
+  >>> issubclass(Q, Container)
+  True
+  >>> Iterable.register(Q)
+  >>> g(Q())
+  Traceback (most recent call last):
+  ...
+  RuntimeError: Ambiguous dispatch: <class 'collections.abc.Container'>
+  or <class 'collections.abc.Iterable'>
+
+An early version of the PEP contained a custom approach that was simpler
+but created a number of edge cases with surprising results [#why-c3]_.
 
 Usage Patterns
 ==============
@@ -378,6 +385,8 @@
    a particular annotation style".
    (http://www.python.org/dev/peps/pep-0008)
 
+.. [#why-c3] http://bugs.python.org/issue18244
+
 .. [#pep-3124] http://www.python.org/dev/peps/pep-3124/
 
 .. [#peak-rules] http://peak.telecommunity.com/DevCenter/PEAK_2dRules

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Mon Jul  1 15:19:00 2013
From: python-checkins at python.org (christian.heimes)
Date: Mon,  1 Jul 2013 15:19:00 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4MzM5?=
 =?utf-8?q?=3A_Negative_ints_keys_in_unpickler=2Ememo_dict_no_longer_cause?=
 =?utf-8?q?_a?=
Message-ID: <3bkTgS1264zLrK@mail.python.org>

http://hg.python.org/cpython/rev/17b7af660f82
changeset:   84407:17b7af660f82
branch:      3.3
parent:      84404:c39ddff53694
user:        Christian Heimes <christian at cheimes.de>
date:        Mon Jul 01 15:17:45 2013 +0200
summary:
  Issue #18339: Negative ints keys in unpickler.memo dict no longer cause a
segfault inside the _pickle C extension.

files:
  Lib/test/test_pickle.py |  7 +++++++
  Misc/NEWS               |  3 +++
  Modules/_pickle.c       |  5 +++++
  3 files changed, 15 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -115,6 +115,13 @@
         pickler_class = _pickle.Pickler
         unpickler_class = _pickle.Unpickler
 
+        def test_issue18339(self):
+            unpickler = self.unpickler_class(io.BytesIO())
+            self.assertRaises(TypeError, setattr, unpickler, "memo", object)
+            # used to cause a segfault
+            self.assertRaises(ValueError, setattr, unpickler, "memo", {-1: None})
+            unpickler.memo = {1: None}
+
     class CDispatchTableTests(AbstractDispatchTableTests):
         pickler_class = pickle.Pickler
         def get_dispatch_table(self):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -38,6 +38,9 @@
 Library
 -------
 
+- Issue #18339: Negative ints keys in unpickler.memo dict no longer cause a
+  segfault inside the _pickle C extension.
+
 - Issue #18224: Removed pydoc script from created venv, as it causes problems
   on Windows and adds no value over and above python -m pydoc ...
 
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -5931,6 +5931,11 @@
             idx = PyLong_AsSsize_t(key);
             if (idx == -1 && PyErr_Occurred())
                 goto error;
+            if (idx < 0) {
+                PyErr_SetString(PyExc_ValueError,
+                                "memos key must be positive integers.");
+                goto error;
+            }
             if (_Unpickler_MemoPut(self, idx, value) < 0)
                 goto error;
         }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 15:19:01 2013
From: python-checkins at python.org (christian.heimes)
Date: Mon,  1 Jul 2013 15:19:01 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2318339=3A_Negative_ints_keys_in_unpickler=2Ememo?=
 =?utf-8?q?_dict_no_longer_cause_a?=
Message-ID: <3bkTgT4JDJz7Lkr@mail.python.org>

http://hg.python.org/cpython/rev/f3372692ca20
changeset:   84408:f3372692ca20
parent:      84406:636947fe131e
parent:      84407:17b7af660f82
user:        Christian Heimes <christian at cheimes.de>
date:        Mon Jul 01 15:18:49 2013 +0200
summary:
  Issue #18339: Negative ints keys in unpickler.memo dict no longer cause a
segfault inside the _pickle C extension.

files:
  Lib/test/test_pickle.py |  7 +++++++
  Misc/NEWS               |  3 +++
  Modules/_pickle.c       |  5 +++++
  3 files changed, 15 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -115,6 +115,13 @@
         pickler_class = _pickle.Pickler
         unpickler_class = _pickle.Unpickler
 
+        def test_issue18339(self):
+            unpickler = self.unpickler_class(io.BytesIO())
+            self.assertRaises(TypeError, setattr, unpickler, "memo", object)
+            # used to cause a segfault
+            self.assertRaises(ValueError, setattr, unpickler, "memo", {-1: None})
+            unpickler.memo = {1: None}
+
     class CDispatchTableTests(AbstractDispatchTableTests):
         pickler_class = pickle.Pickler
         def get_dispatch_table(self):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -135,6 +135,9 @@
 Library
 -------
 
+- Issue #18339: Negative ints keys in unpickler.memo dict no longer cause a
+  segfault inside the _pickle C extension.
+
 - Issue 18240: The HMAC module is no longer restricted to bytes and accepts
   any bytes-like object, e.g. memoryview. Original patch by Jonas Borgstr?m.
 
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -5952,6 +5952,11 @@
             idx = PyLong_AsSsize_t(key);
             if (idx == -1 && PyErr_Occurred())
                 goto error;
+            if (idx < 0) {
+                PyErr_SetString(PyExc_ValueError,
+                                "memos key must be positive integers.");
+                goto error;
+            }
             if (_Unpickler_MemoPut(self, idx, value) < 0)
                 goto error;
         }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 15:24:07 2013
From: python-checkins at python.org (christian.heimes)
Date: Mon,  1 Jul 2013 15:24:07 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Singular_form_?=
 =?utf-8?q?just_like_the_other_error_message=2E?=
Message-ID: <3bkTnM3ccBz7Lk3@mail.python.org>

http://hg.python.org/cpython/rev/61b6cd7b9819
changeset:   84409:61b6cd7b9819
branch:      3.3
parent:      84407:17b7af660f82
user:        Christian Heimes <christian at cheimes.de>
date:        Mon Jul 01 15:23:39 2013 +0200
summary:
  Singular form just like the other error message.

files:
  Modules/_pickle.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Modules/_pickle.c b/Modules/_pickle.c
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -5933,7 +5933,7 @@
                 goto error;
             if (idx < 0) {
                 PyErr_SetString(PyExc_ValueError,
-                                "memos key must be positive integers.");
+                                "memo key must be positive integers.");
                 goto error;
             }
             if (_Unpickler_MemoPut(self, idx, value) < 0)

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 15:24:08 2013
From: python-checkins at python.org (christian.heimes)
Date: Mon,  1 Jul 2013 15:24:08 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Singular_form_just_like_the_other_error_message=2E?=
Message-ID: <3bkTnN6vzPz7LkK@mail.python.org>

http://hg.python.org/cpython/rev/f17647ce8f8a
changeset:   84410:f17647ce8f8a
parent:      84408:f3372692ca20
parent:      84409:61b6cd7b9819
user:        Christian Heimes <christian at cheimes.de>
date:        Mon Jul 01 15:23:48 2013 +0200
summary:
  Singular form just like the other error message.

files:
  Modules/_pickle.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Modules/_pickle.c b/Modules/_pickle.c
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -5954,7 +5954,7 @@
                 goto error;
             if (idx < 0) {
                 PyErr_SetString(PyExc_ValueError,
-                                "memos key must be positive integers.");
+                                "memo key must be positive integers.");
                 goto error;
             }
             if (_Unpickler_MemoPut(self, idx, value) < 0)

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 16:03:39 2013
From: python-checkins at python.org (lukasz.langa)
Date: Mon,  1 Jul 2013 16:03:39 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318244=3A_Adopt_C3?=
 =?utf-8?q?-based_linearization_in_functools=2Esingledispatch_for?=
Message-ID: <3bkVfz4jBQz7Ljg@mail.python.org>

http://hg.python.org/cpython/rev/a2672dc7c805
changeset:   84411:a2672dc7c805
parent:      84406:636947fe131e
user:        ?ukasz Langa <lukasz at langa.pl>
date:        Mon Jul 01 16:00:38 2013 +0200
summary:
  Issue #18244: Adopt C3-based linearization in functools.singledispatch for improved ABC support

files:
  Lib/functools.py           |  178 ++++++++++++++++++------
  Lib/test/test_functools.py |  174 +++++++++++++++++++++--
  Misc/ACKS                  |    1 +
  3 files changed, 289 insertions(+), 64 deletions(-)


diff --git a/Lib/functools.py b/Lib/functools.py
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -365,46 +365,138 @@
 ### singledispatch() - single-dispatch generic function decorator
 ################################################################################
 
-def _compose_mro(cls, haystack):
-    """Calculates the MRO for a given class `cls`, including relevant abstract
-    base classes from `haystack`.
+def _c3_merge(sequences):
+    """Merges MROs in *sequences* to a single MRO using the C3 algorithm.
+
+    Adapted from http://www.python.org/download/releases/2.3/mro/.
+
+    """
+    result = []
+    while True:
+        sequences = [s for s in sequences if s]   # purge empty sequences
+        if not sequences:
+            return result
+        for s1 in sequences:   # find merge candidates among seq heads
+            candidate = s1[0]
+            for s2 in sequences:
+                if candidate in s2[1:]:
+                    candidate = None
+                    break      # reject the current head, it appears later
+            else:
+                break
+        if not candidate:
+            raise RuntimeError("Inconsistent hierarchy")
+        result.append(candidate)
+        # remove the chosen candidate
+        for seq in sequences:
+            if seq[0] == candidate:
+                del seq[0]
+
+def _c3_mro(cls, abcs=None):
+    """Computes the method resolution order using extended C3 linearization.
+
+    If no *abcs* are given, the algorithm works exactly like the built-in C3
+    linearization used for method resolution.
+
+    If given, *abcs* is a list of abstract base classes that should be inserted
+    into the resulting MRO. Unrelated ABCs are ignored and don't end up in the
+    result. The algorithm inserts ABCs where their functionality is introduced,
+    i.e. issubclass(cls, abc) returns True for the class itself but returns
+    False for all its direct base classes. Implicit ABCs for a given class
+    (either registered or inferred from the presence of a special method like
+    __len__) are inserted directly after the last ABC explicitly listed in the
+    MRO of said class. If two implicit ABCs end up next to each other in the
+    resulting MRO, their ordering depends on the order of types in *abcs*.
+
+    """
+    for i, base in enumerate(reversed(cls.__bases__)):
+        if hasattr(base, '__abstractmethods__'):
+            boundary = len(cls.__bases__) - i
+            break   # Bases up to the last explicit ABC are considered first.
+    else:
+        boundary = 0
+    abcs = list(abcs) if abcs else []
+    explicit_bases = list(cls.__bases__[:boundary])
+    abstract_bases = []
+    other_bases = list(cls.__bases__[boundary:])
+    for base in abcs:
+        if issubclass(cls, base) and not any(
+                issubclass(b, base) for b in cls.__bases__
+            ):
+            # If *cls* is the class that introduces behaviour described by
+            # an ABC *base*, insert said ABC to its MRO.
+            abstract_bases.append(base)
+    for base in abstract_bases:
+        abcs.remove(base)
+    explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases]
+    abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases]
+    other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases]
+    return _c3_merge(
+        [[cls]] +
+        explicit_c3_mros + abstract_c3_mros + other_c3_mros +
+        [explicit_bases] + [abstract_bases] + [other_bases]
+    )
+
+def _compose_mro(cls, types):
+    """Calculates the method resolution order for a given class *cls*.
+
+    Includes relevant abstract base classes (with their respective bases) from
+    the *types* iterable. Uses a modified C3 linearization algorithm.
 
     """
     bases = set(cls.__mro__)
-    mro = list(cls.__mro__)
-    for needle in haystack:
-        if (needle in bases or not hasattr(needle, '__mro__')
-                            or not issubclass(cls, needle)):
-            continue   # either present in the __mro__ already or unrelated
-        for index, base in enumerate(mro):
-            if not issubclass(base, needle):
-                break
-        if base in bases and not issubclass(needle, base):
-            # Conflict resolution: put classes present in __mro__ and their
-            # subclasses first. See test_mro_conflicts() in test_functools.py
-            # for examples.
-            index += 1
-        mro.insert(index, needle)
-    return mro
+    # Remove entries which are already present in the __mro__ or unrelated.
+    def is_related(typ):
+        return (typ not in bases and hasattr(typ, '__mro__')
+                                 and issubclass(cls, typ))
+    types = [n for n in types if is_related(n)]
+    # Remove entries which are strict bases of other entries (they will end up
+    # in the MRO anyway.
+    def is_strict_base(typ):
+        for other in types:
+            if typ != other and typ in other.__mro__:
+                return True
+        return False
+    types = [n for n in types if not is_strict_base(n)]
+    # Subclasses of the ABCs in *types* which are also implemented by
+    # *cls* can be used to stabilize ABC ordering.
+    type_set = set(types)
+    mro = []
+    for typ in types:
+        found = []
+        for sub in typ.__subclasses__():
+            if sub not in bases and issubclass(cls, sub):
+                found.append([s for s in sub.__mro__ if s in type_set])
+        if not found:
+            mro.append(typ)
+            continue
+        # Favor subclasses with the biggest number of useful bases
+        found.sort(key=len, reverse=True)
+        for sub in found:
+            for subcls in sub:
+                if subcls not in mro:
+                    mro.append(subcls)
+    return _c3_mro(cls, abcs=mro)
 
 def _find_impl(cls, registry):
-    """Returns the best matching implementation for the given class `cls` in
-    `registry`. Where there is no registered implementation for a specific
-    type, its method resolution order is used to find a more generic
-    implementation.
+    """Returns the best matching implementation from *registry* for type *cls*.
 
-    Note: if `registry` does not contain an implementation for the base
-    `object` type, this function may return None.
+    Where there is no registered implementation for a specific type, its method
+    resolution order is used to find a more generic implementation.
+
+    Note: if *registry* does not contain an implementation for the base
+    *object* type, this function may return None.
 
     """
     mro = _compose_mro(cls, registry.keys())
     match = None
     for t in mro:
         if match is not None:
-            # If `match` is an ABC but there is another unrelated, equally
-            # matching ABC. Refuse the temptation to guess.
-            if (t in registry and not issubclass(match, t)
-                              and match not in cls.__mro__):
+            # If *match* is an implicit ABC but there is another unrelated,
+            # equally matching implicit ABC, refuse the temptation to guess.
+            if (t in registry and t not in cls.__mro__
+                              and match not in cls.__mro__
+                              and not issubclass(match, t)):
                 raise RuntimeError("Ambiguous dispatch: {} or {}".format(
                     match, t))
             break
@@ -418,19 +510,19 @@
     Transforms a function into a generic function, which can have different
     behaviours depending upon the type of its first argument. The decorated
     function acts as the default implementation, and additional
-    implementations can be registered using the 'register()' attribute of
-    the generic function.
+    implementations can be registered using the register() attribute of the
+    generic function.
 
     """
     registry = {}
     dispatch_cache = WeakKeyDictionary()
     cache_token = None
 
-    def dispatch(typ):
-        """generic_func.dispatch(type) -> <function implementation>
+    def dispatch(cls):
+        """generic_func.dispatch(cls) -> <function implementation>
 
         Runs the dispatch algorithm to return the best available implementation
-        for the given `type` registered on `generic_func`.
+        for the given *cls* registered on *generic_func*.
 
         """
         nonlocal cache_token
@@ -440,26 +532,26 @@
                 dispatch_cache.clear()
                 cache_token = current_token
         try:
-            impl = dispatch_cache[typ]
+            impl = dispatch_cache[cls]
         except KeyError:
             try:
-                impl = registry[typ]
+                impl = registry[cls]
             except KeyError:
-                impl = _find_impl(typ, registry)
-            dispatch_cache[typ] = impl
+                impl = _find_impl(cls, registry)
+            dispatch_cache[cls] = impl
         return impl
 
-    def register(typ, func=None):
-        """generic_func.register(type, func) -> func
+    def register(cls, func=None):
+        """generic_func.register(cls, func) -> func
 
-        Registers a new implementation for the given `type` on a `generic_func`.
+        Registers a new implementation for the given *cls* on a *generic_func*.
 
         """
         nonlocal cache_token
         if func is None:
-            return lambda f: register(typ, f)
-        registry[typ] = func
-        if cache_token is None and hasattr(typ, '__abstractmethods__'):
+            return lambda f: register(cls, f)
+        registry[cls] = func
+        if cache_token is None and hasattr(cls, '__abstractmethods__'):
             cache_token = get_cache_token()
         dispatch_cache.clear()
         return func
diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py
--- a/Lib/test/test_functools.py
+++ b/Lib/test/test_functools.py
@@ -929,22 +929,55 @@
         self.assertEqual(g(rnd), ("Number got rounded",))
 
     def test_compose_mro(self):
+        # None of the examples in this test depend on haystack ordering.
         c = collections
         mro = functools._compose_mro
         bases = [c.Sequence, c.MutableMapping, c.Mapping, c.Set]
         for haystack in permutations(bases):
             m = mro(dict, haystack)
-            self.assertEqual(m, [dict, c.MutableMapping, c.Mapping, object])
+            self.assertEqual(m, [dict, c.MutableMapping, c.Mapping, c.Sized,
+                                 c.Iterable, c.Container, object])
         bases = [c.Container, c.Mapping, c.MutableMapping, c.OrderedDict]
         for haystack in permutations(bases):
             m = mro(c.ChainMap, haystack)
             self.assertEqual(m, [c.ChainMap, c.MutableMapping, c.Mapping,
                                  c.Sized, c.Iterable, c.Container, object])
-        # Note: The MRO order below depends on haystack ordering.
-        m = mro(c.defaultdict, [c.Sized, c.Container, str])
-        self.assertEqual(m, [c.defaultdict, dict, c.Container, c.Sized, object])
-        m = mro(c.defaultdict, [c.Container, c.Sized, str])
-        self.assertEqual(m, [c.defaultdict, dict, c.Sized, c.Container, object])
+
+        # If there's a generic function with implementations registered for
+        # both Sized and Container, passing a defaultdict to it results in an
+        # ambiguous dispatch which will cause a RuntimeError (see
+        # test_mro_conflicts).
+        bases = [c.Container, c.Sized, str]
+        for haystack in permutations(bases):
+            m = mro(c.defaultdict, [c.Sized, c.Container, str])
+            self.assertEqual(m, [c.defaultdict, dict, c.Sized, c.Container,
+                                 object])
+
+        # MutableSequence below is registered directly on D. In other words, it
+        # preceeds MutableMapping which means single dispatch will always
+        # choose MutableSequence here.
+        class D(c.defaultdict):
+            pass
+        c.MutableSequence.register(D)
+        bases = [c.MutableSequence, c.MutableMapping]
+        for haystack in permutations(bases):
+            m = mro(D, bases)
+            self.assertEqual(m, [D, c.MutableSequence, c.Sequence,
+                                 c.defaultdict, dict, c.MutableMapping,
+                                 c.Mapping, c.Sized, c.Iterable, c.Container,
+                                 object])
+
+        # Container and Callable are registered on different base classes and
+        # a generic function supporting both should always pick the Callable
+        # implementation if a C instance is passed.
+        class C(c.defaultdict):
+            def __call__(self):
+                pass
+        bases = [c.Sized, c.Callable, c.Container, c.Mapping]
+        for haystack in permutations(bases):
+            m = mro(C, haystack)
+            self.assertEqual(m, [C, c.Callable, c.defaultdict, dict, c.Mapping,
+                                 c.Sized, c.Iterable, c.Container, object])
 
     def test_register_abc(self):
         c = collections
@@ -1040,17 +1073,37 @@
         self.assertEqual(g(f), "frozen-set")
         self.assertEqual(g(t), "tuple")
 
+    def test_c3_abc(self):
+        c = collections
+        mro = functools._c3_mro
+        class A(object):
+            pass
+        class B(A):
+            def __len__(self):
+                return 0   # implies Sized
+        @c.Container.register
+        class C(object):
+            pass
+        class D(object):
+            pass   # unrelated
+        class X(D, C, B):
+            def __call__(self):
+                pass   # implies Callable
+        expected = [X, c.Callable, D, C, c.Container, B, c.Sized, A, object]
+        for abcs in permutations([c.Sized, c.Callable, c.Container]):
+            self.assertEqual(mro(X, abcs=abcs), expected)
+        # unrelated ABCs don't appear in the resulting MRO
+        many_abcs = [c.Mapping, c.Sized, c.Callable, c.Container, c.Iterable]
+        self.assertEqual(mro(X, abcs=many_abcs), expected)
+
     def test_mro_conflicts(self):
         c = collections
-
         @functools.singledispatch
         def g(arg):
             return "base"
-
         class O(c.Sized):
             def __len__(self):
                 return 0
-
         o = O()
         self.assertEqual(g(o), "base")
         g.register(c.Iterable, lambda arg: "iterable")
@@ -1062,35 +1115,114 @@
         self.assertEqual(g(o), "sized")   # because it's explicitly in __mro__
         c.Container.register(O)
         self.assertEqual(g(o), "sized")   # see above: Sized is in __mro__
-
+        c.Set.register(O)
+        self.assertEqual(g(o), "set")     # because c.Set is a subclass of
+                                          # c.Sized and c.Container
         class P:
             pass
-
         p = P()
         self.assertEqual(g(p), "base")
         c.Iterable.register(P)
         self.assertEqual(g(p), "iterable")
         c.Container.register(P)
-        with self.assertRaises(RuntimeError) as re:
+        with self.assertRaises(RuntimeError) as re_one:
             g(p)
-            self.assertEqual(
-                str(re),
-                ("Ambiguous dispatch: <class 'collections.abc.Container'> "
-                    "or <class 'collections.abc.Iterable'>"),
-            )
-
+        self.assertIn(
+            str(re_one.exception),
+            (("Ambiguous dispatch: <class 'collections.abc.Container'> "
+              "or <class 'collections.abc.Iterable'>"),
+             ("Ambiguous dispatch: <class 'collections.abc.Iterable'> "
+              "or <class 'collections.abc.Container'>")),
+        )
         class Q(c.Sized):
             def __len__(self):
                 return 0
-
         q = Q()
         self.assertEqual(g(q), "sized")
         c.Iterable.register(Q)
         self.assertEqual(g(q), "sized")   # because it's explicitly in __mro__
         c.Set.register(Q)
         self.assertEqual(g(q), "set")     # because c.Set is a subclass of
-                                          # c.Sized which is explicitly in
-                                          # __mro__
+                                          # c.Sized and c.Iterable
+        @functools.singledispatch
+        def h(arg):
+            return "base"
+        @h.register(c.Sized)
+        def _(arg):
+            return "sized"
+        @h.register(c.Container)
+        def _(arg):
+            return "container"
+        # Even though Sized and Container are explicit bases of MutableMapping,
+        # this ABC is implicitly registered on defaultdict which makes all of
+        # MutableMapping's bases implicit as well from defaultdict's
+        # perspective.
+        with self.assertRaises(RuntimeError) as re_two:
+            h(c.defaultdict(lambda: 0))
+        self.assertIn(
+            str(re_two.exception),
+            (("Ambiguous dispatch: <class 'collections.abc.Container'> "
+              "or <class 'collections.abc.Sized'>"),
+             ("Ambiguous dispatch: <class 'collections.abc.Sized'> "
+              "or <class 'collections.abc.Container'>")),
+        )
+        class R(c.defaultdict):
+            pass
+        c.MutableSequence.register(R)
+        @functools.singledispatch
+        def i(arg):
+            return "base"
+        @i.register(c.MutableMapping)
+        def _(arg):
+            return "mapping"
+        @i.register(c.MutableSequence)
+        def _(arg):
+            return "sequence"
+        r = R()
+        self.assertEqual(i(r), "sequence")
+        class S:
+            pass
+        class T(S, c.Sized):
+            def __len__(self):
+                return 0
+        t = T()
+        self.assertEqual(h(t), "sized")
+        c.Container.register(T)
+        self.assertEqual(h(t), "sized")   # because it's explicitly in the MRO
+        class U:
+            def __len__(self):
+                return 0
+        u = U()
+        self.assertEqual(h(u), "sized")   # implicit Sized subclass inferred
+                                          # from the existence of __len__()
+        c.Container.register(U)
+        # There is no preference for registered versus inferred ABCs.
+        with self.assertRaises(RuntimeError) as re_three:
+            h(u)
+        self.assertIn(
+            str(re_three.exception),
+            (("Ambiguous dispatch: <class 'collections.abc.Container'> "
+              "or <class 'collections.abc.Sized'>"),
+             ("Ambiguous dispatch: <class 'collections.abc.Sized'> "
+              "or <class 'collections.abc.Container'>")),
+        )
+        class V(c.Sized, S):
+            def __len__(self):
+                return 0
+        @functools.singledispatch
+        def j(arg):
+            return "base"
+        @j.register(S)
+        def _(arg):
+            return "s"
+        @j.register(c.Container)
+        def _(arg):
+            return "container"
+        v = V()
+        self.assertEqual(j(v), "s")
+        c.Container.register(V)
+        self.assertEqual(j(v), "container")   # because it ends up right after
+                                              # Sized in the MRO
 
     def test_cache_invalidation(self):
         from collections import UserDict
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -195,6 +195,7 @@
 Mike Carlton
 Pierre Carrier
 Terry Carroll
+Edward Catmur
 Lorenzo M. Catucci
 Donn Cave
 Charles Cazabon

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 16:03:41 2013
From: python-checkins at python.org (lukasz.langa)
Date: Mon,  1 Jul 2013 16:03:41 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?=
 =?utf-8?q?=29=3A_Merge_with_current_default?=
Message-ID: <3bkVg101cNz7LkS@mail.python.org>

http://hg.python.org/cpython/rev/879ee6762f3d
changeset:   84412:879ee6762f3d
parent:      84411:a2672dc7c805
parent:      84410:f17647ce8f8a
user:        ?ukasz Langa <lukasz at langa.pl>
date:        Mon Jul 01 16:03:17 2013 +0200
summary:
  Merge with current default

files:
  Lib/test/test_pickle.py |  7 +++++++
  Misc/NEWS               |  3 +++
  Modules/_pickle.c       |  5 +++++
  3 files changed, 15 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -115,6 +115,13 @@
         pickler_class = _pickle.Pickler
         unpickler_class = _pickle.Unpickler
 
+        def test_issue18339(self):
+            unpickler = self.unpickler_class(io.BytesIO())
+            self.assertRaises(TypeError, setattr, unpickler, "memo", object)
+            # used to cause a segfault
+            self.assertRaises(ValueError, setattr, unpickler, "memo", {-1: None})
+            unpickler.memo = {1: None}
+
     class CDispatchTableTests(AbstractDispatchTableTests):
         pickler_class = pickle.Pickler
         def get_dispatch_table(self):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -135,6 +135,9 @@
 Library
 -------
 
+- Issue #18339: Negative ints keys in unpickler.memo dict no longer cause a
+  segfault inside the _pickle C extension.
+
 - Issue 18240: The HMAC module is no longer restricted to bytes and accepts
   any bytes-like object, e.g. memoryview. Original patch by Jonas Borgstr?m.
 
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -5952,6 +5952,11 @@
             idx = PyLong_AsSsize_t(key);
             if (idx == -1 && PyErr_Occurred())
                 goto error;
+            if (idx < 0) {
+                PyErr_SetString(PyExc_ValueError,
+                                "memo key must be positive integers.");
+                goto error;
+            }
             if (_Unpickler_MemoPut(self, idx, value) < 0)
                 goto error;
         }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 20:12:10 2013
From: python-checkins at python.org (richard.oudkerk)
Date: Mon,  1 Jul 2013 20:12:10 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE3MDk3?=
 =?utf-8?q?=3A_Make_multiprocessing_ignore_EINTR=2E?=
Message-ID: <3bkc9k3GQlz7Ljx@mail.python.org>

http://hg.python.org/cpython/rev/bc34fe4a0d58
changeset:   84413:bc34fe4a0d58
branch:      2.7
parent:      84403:26ef5d5d5c3e
user:        Richard Oudkerk <shibturn at gmail.com>
date:        Mon Jul 01 18:45:28 2013 +0100
summary:
  Issue #17097: Make multiprocessing ignore EINTR.

files:
  Lib/multiprocessing/connection.py            |   9 +-
  Lib/test/test_multiprocessing.py             |  70 +++++++++-
  Misc/NEWS                                    |   2 +
  Modules/_multiprocessing/socket_connection.c |  57 ++++++-
  4 files changed, 127 insertions(+), 11 deletions(-)


diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -270,7 +270,14 @@
             self._unlink = None
 
     def accept(self):
-        s, self._last_accepted = self._socket.accept()
+        while True:
+            try:
+                s, self._last_accepted = self._socket.accept()
+            except socket.error as e:
+                if e.args[0] != errno.EINTR:
+                    raise
+            else:
+                break
         s.setblocking(True)
         fd = duplicate(s.fileno())
         conn = _multiprocessing.Connection(fd)
diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py
--- a/Lib/test/test_multiprocessing.py
+++ b/Lib/test/test_multiprocessing.py
@@ -2461,12 +2461,80 @@
         self.assertLessEqual(new_size, old_size)
 
 #
+# Issue #17097: EINTR should be ignored by recv(), send(), accept() etc
+#
+
+class TestIgnoreEINTR(unittest.TestCase):
+
+    @classmethod
+    def _test_ignore(cls, conn):
+        def handler(signum, frame):
+            pass
+        signal.signal(signal.SIGUSR1, handler)
+        conn.send('ready')
+        x = conn.recv()
+        conn.send(x)
+        conn.send_bytes(b'x'*(1024*1024))   # sending 1 MB should block
+
+    @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1')
+    def test_ignore(self):
+        conn, child_conn = multiprocessing.Pipe()
+        try:
+            p = multiprocessing.Process(target=self._test_ignore,
+                                        args=(child_conn,))
+            p.daemon = True
+            p.start()
+            child_conn.close()
+            self.assertEqual(conn.recv(), 'ready')
+            time.sleep(0.1)
+            os.kill(p.pid, signal.SIGUSR1)
+            time.sleep(0.1)
+            conn.send(1234)
+            self.assertEqual(conn.recv(), 1234)
+            time.sleep(0.1)
+            os.kill(p.pid, signal.SIGUSR1)
+            self.assertEqual(conn.recv_bytes(), b'x'*(1024*1024))
+            time.sleep(0.1)
+            p.join()
+        finally:
+            conn.close()
+
+    @classmethod
+    def _test_ignore_listener(cls, conn):
+        def handler(signum, frame):
+            pass
+        signal.signal(signal.SIGUSR1, handler)
+        l = multiprocessing.connection.Listener()
+        conn.send(l.address)
+        a = l.accept()
+        a.send('welcome')
+
+    @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1')
+    def test_ignore_listener(self):
+        conn, child_conn = multiprocessing.Pipe()
+        try:
+            p = multiprocessing.Process(target=self._test_ignore_listener,
+                                        args=(child_conn,))
+            p.daemon = True
+            p.start()
+            child_conn.close()
+            address = conn.recv()
+            time.sleep(0.1)
+            os.kill(p.pid, signal.SIGUSR1)
+            time.sleep(0.1)
+            client = multiprocessing.connection.Client(address)
+            self.assertEqual(client.recv(), 'welcome')
+            p.join()
+        finally:
+            conn.close()
+
+#
 #
 #
 
 testcases_other = [OtherTest, TestInvalidHandle, TestInitializers,
                    TestStdinBadfiledescriptor, TestTimeouts, TestNoForkBomb,
-                   TestFlags, TestForkAwareThreadLock]
+                   TestFlags, TestForkAwareThreadLock, TestIgnoreEINTR]
 
 #
 #
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -24,6 +24,8 @@
 Library
 -------
 
+- Issue #17097: Make multiprocessing ignore EINTR.
+
 - Issue #18155: The csv module now correctly handles csv files that use
   a delimiter character that has a special meaning in regexes, instead of
   throwing an exception.
diff --git a/Modules/_multiprocessing/socket_connection.c b/Modules/_multiprocessing/socket_connection.c
--- a/Modules/_multiprocessing/socket_connection.c
+++ b/Modules/_multiprocessing/socket_connection.c
@@ -23,6 +23,21 @@
 #endif
 
 /*
+ * Wrapper for PyErr_CheckSignals() which can be called without the GIL
+ */
+
+static int
+check_signals(void)
+{
+    PyGILState_STATE state;
+    int res;
+    state = PyGILState_Ensure();
+    res = PyErr_CheckSignals();
+    PyGILState_Release(state);
+    return res;
+}
+
+/*
  * Send string to file descriptor
  */
 
@@ -34,8 +49,14 @@
 
     while (length > 0) {
         res = WRITE(h, p, length);
-        if (res < 0)
+        if (res < 0) {
+            if (errno == EINTR) {
+                if (check_signals() < 0)
+                    return MP_EXCEPTION_HAS_BEEN_SET;
+                continue;
+            }
             return MP_SOCKET_ERROR;
+        }
         length -= res;
         p += res;
     }
@@ -56,12 +77,16 @@
 
     while (remaining > 0) {
         temp = READ(h, p, remaining);
-        if (temp <= 0) {
-            if (temp == 0)
-                return remaining == length ?
-                    MP_END_OF_FILE : MP_EARLY_END_OF_FILE;
-            else
-                return temp;
+        if (temp < 0) {
+            if (errno == EINTR) {
+                if (check_signals() < 0)
+                    return MP_EXCEPTION_HAS_BEEN_SET;
+                continue;
+            }
+            return temp;
+        }
+        else if (temp == 0) {
+            return remaining == length ? MP_END_OF_FILE : MP_EARLY_END_OF_FILE;
         }
         remaining -= temp;
         p += temp;
@@ -171,9 +196,16 @@
     p.revents = 0;
 
     if (timeout < 0) {
-        res = poll(&p, 1, -1);
+        do {
+            res = poll(&p, 1, -1);
+        } while (res < 0 && errno == EINTR);
     } else {
         res = poll(&p, 1, (int)(timeout * 1000 + 0.5));
+        if (res < 0 && errno == EINTR) {
+            /* We were interrupted by a signal.  Just indicate a
+               timeout even though we are early. */
+            return FALSE;
+        }
     }
 
     if (res < 0) {
@@ -209,12 +241,19 @@
     FD_SET((SOCKET)conn->handle, &rfds);
 
     if (timeout < 0.0) {
-        res = select((int)conn->handle+1, &rfds, NULL, NULL, NULL);
+        do {
+            res = select((int)conn->handle+1, &rfds, NULL, NULL, NULL);
+        } while (res < 0 && errno == EINTR);
     } else {
         struct timeval tv;
         tv.tv_sec = (long)timeout;
         tv.tv_usec = (long)((timeout - tv.tv_sec) * 1e6 + 0.5);
         res = select((int)conn->handle+1, &rfds, NULL, NULL, &tv);
+        if (res < 0 && errno == EINTR) {
+            /* We were interrupted by a signal.  Just indicate a
+               timeout even though we are early. */
+            return FALSE;
+        }
     }
 
     if (res < 0) {

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 20:12:11 2013
From: python-checkins at python.org (richard.oudkerk)
Date: Mon,  1 Jul 2013 20:12:11 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE3MDk3?=
 =?utf-8?q?=3A_Make_multiprocessing_ignore_EINTR=2E?=
Message-ID: <3bkc9l6lKJz7LkX@mail.python.org>

http://hg.python.org/cpython/rev/3ec5267f51ff
changeset:   84414:3ec5267f51ff
branch:      3.3
parent:      84409:61b6cd7b9819
user:        Richard Oudkerk <shibturn at gmail.com>
date:        Mon Jul 01 18:59:26 2013 +0100
summary:
  Issue #17097: Make multiprocessing ignore EINTR.

files:
  Lib/multiprocessing/connection.py |  18 ++++-
  Lib/test/test_multiprocessing.py  |  70 ++++++++++++++++++-
  Misc/NEWS                         |   2 +
  3 files changed, 86 insertions(+), 4 deletions(-)


diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -366,7 +366,10 @@
     def _send(self, buf, write=_write):
         remaining = len(buf)
         while True:
-            n = write(self._handle, buf)
+            try:
+                n = write(self._handle, buf)
+            except InterruptedError:
+                continue
             remaining -= n
             if remaining == 0:
                 break
@@ -377,7 +380,10 @@
         handle = self._handle
         remaining = size
         while remaining > 0:
-            chunk = read(handle, remaining)
+            try:
+                chunk = read(handle, remaining)
+            except InterruptedError:
+                continue
             n = len(chunk)
             if n == 0:
                 if remaining == size:
@@ -581,7 +587,13 @@
             self._unlink = None
 
     def accept(self):
-        s, self._last_accepted = self._socket.accept()
+        while True:
+            try:
+                s, self._last_accepted = self._socket.accept()
+            except InterruptedError:
+                pass
+            else:
+                break
         s.setblocking(True)
         return Connection(s.detach())
 
diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py
--- a/Lib/test/test_multiprocessing.py
+++ b/Lib/test/test_multiprocessing.py
@@ -3461,13 +3461,81 @@
         self.assertLessEqual(new_size, old_size)
 
 #
+# Issue #17097: EINTR should be ignored by recv(), send(), accept() etc
+#
+
+class TestIgnoreEINTR(unittest.TestCase):
+
+    @classmethod
+    def _test_ignore(cls, conn):
+        def handler(signum, frame):
+            pass
+        signal.signal(signal.SIGUSR1, handler)
+        conn.send('ready')
+        x = conn.recv()
+        conn.send(x)
+        conn.send_bytes(b'x'*(1024*1024))   # sending 1 MB should block
+
+    @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1')
+    def test_ignore(self):
+        conn, child_conn = multiprocessing.Pipe()
+        try:
+            p = multiprocessing.Process(target=self._test_ignore,
+                                        args=(child_conn,))
+            p.daemon = True
+            p.start()
+            child_conn.close()
+            self.assertEqual(conn.recv(), 'ready')
+            time.sleep(0.1)
+            os.kill(p.pid, signal.SIGUSR1)
+            time.sleep(0.1)
+            conn.send(1234)
+            self.assertEqual(conn.recv(), 1234)
+            time.sleep(0.1)
+            os.kill(p.pid, signal.SIGUSR1)
+            self.assertEqual(conn.recv_bytes(), b'x'*(1024*1024))
+            time.sleep(0.1)
+            p.join()
+        finally:
+            conn.close()
+
+    @classmethod
+    def _test_ignore_listener(cls, conn):
+        def handler(signum, frame):
+            pass
+        signal.signal(signal.SIGUSR1, handler)
+        l = multiprocessing.connection.Listener()
+        conn.send(l.address)
+        a = l.accept()
+        a.send('welcome')
+
+    @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1')
+    def test_ignore_listener(self):
+        conn, child_conn = multiprocessing.Pipe()
+        try:
+            p = multiprocessing.Process(target=self._test_ignore_listener,
+                                        args=(child_conn,))
+            p.daemon = True
+            p.start()
+            child_conn.close()
+            address = conn.recv()
+            time.sleep(0.1)
+            os.kill(p.pid, signal.SIGUSR1)
+            time.sleep(0.1)
+            client = multiprocessing.connection.Client(address)
+            self.assertEqual(client.recv(), 'welcome')
+            p.join()
+        finally:
+            conn.close()
+
+#
 #
 #
 
 testcases_other = [OtherTest, TestInvalidHandle, TestInitializers,
                    TestStdinBadfiledescriptor, TestWait, TestInvalidFamily,
                    TestFlags, TestTimeouts, TestNoForkBomb,
-                   TestForkAwareThreadLock]
+                   TestForkAwareThreadLock, TestIgnoreEINTR]
 
 #
 #
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -38,6 +38,8 @@
 Library
 -------
 
+- Issue #17097: Make multiprocessing ignore EINTR.
+
 - Issue #18339: Negative ints keys in unpickler.memo dict no longer cause a
   segfault inside the _pickle C extension.
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 20:12:13 2013
From: python-checkins at python.org (richard.oudkerk)
Date: Mon,  1 Jul 2013 20:12:13 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?b?KTogSXNzdWUgIzE3MDk3OiBNZXJnZS4=?=
Message-ID: <3bkc9n41pWz7LlC@mail.python.org>

http://hg.python.org/cpython/rev/035577781505
changeset:   84415:035577781505
parent:      84412:879ee6762f3d
parent:      84414:3ec5267f51ff
user:        Richard Oudkerk <shibturn at gmail.com>
date:        Mon Jul 01 19:10:39 2013 +0100
summary:
  Issue #17097: Merge.

files:
  Lib/multiprocessing/connection.py |  18 ++++-
  Lib/test/test_multiprocessing.py  |  70 ++++++++++++++++++-
  Misc/NEWS                         |   2 +
  3 files changed, 86 insertions(+), 4 deletions(-)


diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -363,7 +363,10 @@
     def _send(self, buf, write=_write):
         remaining = len(buf)
         while True:
-            n = write(self._handle, buf)
+            try:
+                n = write(self._handle, buf)
+            except InterruptedError:
+                continue
             remaining -= n
             if remaining == 0:
                 break
@@ -374,7 +377,10 @@
         handle = self._handle
         remaining = size
         while remaining > 0:
-            chunk = read(handle, remaining)
+            try:
+                chunk = read(handle, remaining)
+            except InterruptedError:
+                continue
             n = len(chunk)
             if n == 0:
                 if remaining == size:
@@ -578,7 +584,13 @@
             self._unlink = None
 
     def accept(self):
-        s, self._last_accepted = self._socket.accept()
+        while True:
+            try:
+                s, self._last_accepted = self._socket.accept()
+            except InterruptedError:
+                pass
+            else:
+                break
         s.setblocking(True)
         return Connection(s.detach())
 
diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py
--- a/Lib/test/test_multiprocessing.py
+++ b/Lib/test/test_multiprocessing.py
@@ -3514,13 +3514,81 @@
         self.assertLessEqual(new_size, old_size)
 
 #
+# Issue #17097: EINTR should be ignored by recv(), send(), accept() etc
+#
+
+class TestIgnoreEINTR(unittest.TestCase):
+
+    @classmethod
+    def _test_ignore(cls, conn):
+        def handler(signum, frame):
+            pass
+        signal.signal(signal.SIGUSR1, handler)
+        conn.send('ready')
+        x = conn.recv()
+        conn.send(x)
+        conn.send_bytes(b'x'*(1024*1024))   # sending 1 MB should block
+
+    @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1')
+    def test_ignore(self):
+        conn, child_conn = multiprocessing.Pipe()
+        try:
+            p = multiprocessing.Process(target=self._test_ignore,
+                                        args=(child_conn,))
+            p.daemon = True
+            p.start()
+            child_conn.close()
+            self.assertEqual(conn.recv(), 'ready')
+            time.sleep(0.1)
+            os.kill(p.pid, signal.SIGUSR1)
+            time.sleep(0.1)
+            conn.send(1234)
+            self.assertEqual(conn.recv(), 1234)
+            time.sleep(0.1)
+            os.kill(p.pid, signal.SIGUSR1)
+            self.assertEqual(conn.recv_bytes(), b'x'*(1024*1024))
+            time.sleep(0.1)
+            p.join()
+        finally:
+            conn.close()
+
+    @classmethod
+    def _test_ignore_listener(cls, conn):
+        def handler(signum, frame):
+            pass
+        signal.signal(signal.SIGUSR1, handler)
+        l = multiprocessing.connection.Listener()
+        conn.send(l.address)
+        a = l.accept()
+        a.send('welcome')
+
+    @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1')
+    def test_ignore_listener(self):
+        conn, child_conn = multiprocessing.Pipe()
+        try:
+            p = multiprocessing.Process(target=self._test_ignore_listener,
+                                        args=(child_conn,))
+            p.daemon = True
+            p.start()
+            child_conn.close()
+            address = conn.recv()
+            time.sleep(0.1)
+            os.kill(p.pid, signal.SIGUSR1)
+            time.sleep(0.1)
+            client = multiprocessing.connection.Client(address)
+            self.assertEqual(client.recv(), 'welcome')
+            p.join()
+        finally:
+            conn.close()
+
+#
 #
 #
 
 testcases_other = [OtherTest, TestInvalidHandle, TestInitializers,
                    TestStdinBadfiledescriptor, TestWait, TestInvalidFamily,
                    TestFlags, TestTimeouts, TestNoForkBomb,
-                   TestForkAwareThreadLock]
+                   TestForkAwareThreadLock, TestIgnoreEINTR]
 
 #
 #
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -135,6 +135,8 @@
 Library
 -------
 
+- Issue #17097: Make multiprocessing ignore EINTR.
+
 - Issue #18339: Negative ints keys in unpickler.memo dict no longer cause a
   segfault inside the _pickle C extension.
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 22:29:26 2013
From: python-checkins at python.org (victor.stinner)
Date: Mon,  1 Jul 2013 22:29:26 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_445=3A_cleanup?=
Message-ID: <3bkgD63LG2z7Ljg@mail.python.org>

http://hg.python.org/peps/rev/4de91c027ae2
changeset:   4969:4de91c027ae2
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Jul 01 22:29:08 2013 +0200
summary:
  PEP 445: cleanup

Avoid "should", "may" and "might". Rephrase some sentences

files:
  pep-0445.txt |  218 ++++++++++++++++++++------------------
  1 files changed, 114 insertions(+), 104 deletions(-)


diff --git a/pep-0445.txt b/pep-0445.txt
--- a/pep-0445.txt
+++ b/pep-0445.txt
@@ -12,7 +12,8 @@
 Abstract
 ========
 
-Add new APIs to customize Python memory allocators.
+Add new Application Programming Interfaces (API) to customize Python
+memory allocators.
 
 
 Rationale
@@ -20,21 +21,22 @@
 
 Use cases:
 
-* Application embedding Python may want to isolate Python memory from
-  the memory of the application, or may want to use a different memory
+* Applications embedding Python which want to isolate Python memory from
+  the memory of the application, or want to use a different memory
   allocator optimized for its Python usage
 * Python running on embedded devices with low memory and slow CPU.
-  A custom memory allocator may be required to use efficiently the
-  memory and/or to be able to use all the memory of the device.
-* Debug tool to:
+  A custom memory allocator can be used for efficiency and/or to get
+  access all the memory of the device.
+* Debug tools for memory allocators:
 
-  - track the memory usage (memory leaks)
-  - get the Python filename and line number where an object was
-    allocated
-  - detect buffer underflow, buffer overflow and detect misuse of Python
-    allocator APIs (builtin Python debug hooks)
-  - force allocation to fail to test handling of ``MemoryError``
-    exception
+  - track the memory usage (find memory leaks)
+  - get the location of a memory allocation: Python filename and line
+    number, and the size of a memory block
+  - detect buffer underflow, buffer overflow and misuse of Python
+    allocator APIs (see `Redesign Debug Checks on Memory Block
+    Allocators as Hooks`_)
+  - force memory allocations to fail to test handling of the
+    ``MemoryError`` exception
 
 
 Proposal
@@ -56,8 +58,7 @@
 * Add a new ``PyMemAllocator`` structure::
 
     typedef struct {
-        /* user context passed as the first argument
-           to the 3 functions */
+        /* user context passed as the first argument to the 3 functions */
         void *ctx;
 
         /* allocate a memory block */
@@ -82,7 +83,7 @@
   - ``PYMEM_DOMAIN_OBJ``: ``PyObject_Malloc()``, ``PyObject_Realloc()``
     and ``PyObject_Free()``
 
-* Add new functions to get and set memory allocators:
+* Add new functions to get and set memory block allocators:
 
   - ``void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)``
   - ``void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)``
@@ -94,8 +95,7 @@
 * Add a new ``PyObjectArenaAllocator`` structure::
 
     typedef struct {
-        /* user context passed as the first argument
-           to the 2 functions */
+        /* user context passed as the first argument to the 2 functions */
         void *ctx;
 
         /* allocate an arena */
@@ -111,18 +111,17 @@
   - ``void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)``
   - ``void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)``
 
-* Add a new function to setup the debug checks on memory allocators when
-  a memory allocator is replaced:
+* Add a new function to reinstall the debug checks on memory allocators when
+  a memory allocator is replaced with ``PyMem_SetAllocator()``:
 
   - ``void PyMem_SetupDebugHooks(void)``
-  - Install the debug hook on all memory block allocators. The function
-    can be called more than once, hooks are not reinstalled if they
-    were already installed.
-  - The function does nothing is Python is not compiled in debug mode
+  - Install the debug hooks on all memory block allocators. The function can be
+    called more than once, hooks are only installed once.
+  - The function does nothing is Python is not compiled in debug mode.
 
-* Memory allocators always returns *NULL* if size is greater than
-  ``PY_SSIZE_T_MAX``. The check is done before calling the
-  inner function.
+* Memory block allocators always return *NULL* if *size* is greater than
+  ``PY_SSIZE_T_MAX``. The check is done before calling the inner
+  function.
 
 The *pymalloc* allocator is optimized for objects smaller than 512 bytes
 with a short lifetime. It uses memory mappings with a fixed size of 256
@@ -140,8 +139,8 @@
   and ``free()``
 
 
-Redesign Debug Checks on  Memory Allocators as Hooks
-----------------------------------------------------
+Redesign Debug Checks on Memory Block Allocators as Hooks
+---------------------------------------------------------
 
 Since Python 2.3, Python implements different checks on memory
 allocators in debug mode:
@@ -157,7 +156,8 @@
 ``PyMem_Realloc()``, ``PyMem_Free()``, ``PyObject_Malloc()``,
 ``PyObject_Realloc()`` and ``PyObject_Free()`` using macros. The new
 allocator allocates a larger buffer and write a pattern to detect buffer
-underflow and overflow. It uses the original ``PyObject_Malloc()``
+underflow, buffer overflow and use after free (fill the buffer with the
+pattern ``0xDB``). It uses the original ``PyObject_Malloc()``
 function to allocate memory. So ``PyMem_Malloc()`` and
 ``PyMem_Realloc()`` call indirectly ``PyObject_Malloc()`` and
 ``PyObject_Realloc()``.
@@ -178,13 +178,14 @@
 * ``PyObject_Free()`` => ``_PyMem_DebugFree()``
   => ``_PyObject_Free()``
 
-As a result, ``PyMem_Malloc()`` and ``PyMem_Realloc()`` now always call
-``malloc()`` and ``realloc()``, instead of calling ``PyObject_Malloc()``
-and ``PyObject_Realloc()`` in debug mode.
+As a result, ``PyMem_Malloc()`` and ``PyMem_Realloc()`` now call
+``malloc()`` and ``realloc()`` in release mode and in debug mode,
+instead of calling ``PyObject_Malloc()`` and ``PyObject_Realloc()`` in
+debug mode.
 
 When at least one memory allocator is replaced with
 ``PyMem_SetAllocator()``, the ``PyMem_SetupDebugHooks()`` function must
-be called to install the debug hooks on top on the new allocator.
+be called to reinstall the debug hooks on top on the new allocator.
 
 
 Don't call malloc() directly anymore
@@ -195,7 +196,7 @@
 ``PyObject_Realloc()`` falls back on ``PyMem_Realloc()`` instead of
 ``realloc()``
 
-Replace direct calls to ``malloc()`` with ``PyMem_Malloc()``, or
+Direct calls to ``malloc()`` are replaced with ``PyMem_Malloc()``, or
 ``PyMem_RawMalloc()`` if the GIL is not held.
 
 Configure external libraries like zlib or OpenSSL to allocate memory
@@ -205,22 +206,22 @@
 
 For the "track memory usage" use case, it is important to track memory
 allocated in external libraries to have accurate reports, because these
-allocations may be large.
+allocations can be large (can raise a ``MemoryError`` exception).
 
 If an hook is used to the track memory usage, the memory allocated by
-``malloc()`` will not be tracked. Remaining ``malloc()`` in external
-libraries like OpenSSL or bz2 may allocate large memory blocks and so
-would be missed in memory usage reports.
+direct calls to ``malloc()`` will not be tracked. Remaining ``malloc()``
+in external libraries like OpenSSL or bz2 can allocate large memory
+blocks and so would be missed in memory usage reports.
 
 
 Examples
 ========
 
-Use case 1: Replace Memory Allocator, keep pymalloc
+Use case 1: Replace Memory Allocators, keep pymalloc
 ----------------------------------------------------
 
 Dummy example wasting 2 bytes per memory block,
-and 10 bytes per memory mapping::
+and 10 bytes per *pymalloc* arena::
 
     #include <stdlib.h>
 
@@ -267,6 +268,7 @@
 
         PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc);
         PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc);
+        /* leave PYMEM_DOMAIN_OBJ unchanged, use pymalloc */
 
         arena.ctx = &arena_padding;
         arena.alloc = my_alloc_arena;
@@ -277,7 +279,7 @@
     }
 
 
-Use case 2: Replace Memory Allocator, override pymalloc
+Use case 2: Replace Memory Allocators, override pymalloc
 --------------------------------------------------------
 
 If your allocator is optimized for allocations of objects smaller than
@@ -322,11 +324,14 @@
         PyMem_SetupDebugHooks();
     }
 
+The *pymalloc* arena does not need to be replaced, because it is no more
+used by the new allocator.
 
-Use case 3: Setup Allocator Hooks
----------------------------------
 
-Example to setup hooks on all memory allocators::
+Use case 3: Setup Hooks On Memory Block Allocators
+--------------------------------------------------
+
+Example to setup hooks on all memory block allocators::
 
     struct {
         PyMemAllocator raw;
@@ -390,22 +395,23 @@
     }
 
 .. note::
-   ``PyMem_SetupDebugHooks()`` does not need to be called because the
-   allocator is not replaced: Python debug hooks are installed
-   automatically at startup.
+   ``PyMem_SetupDebugHooks()`` does not need to be called because
+   memory allocator are not replaced: the debug checks on memory
+   block allocators are installed automatically at startup.
 
 
 Performances
 ============
 
+The implementation of this PEP (issue #3329) has no visible overhead on
+the Python benchmark suite.
+
 Results of the `Python benchmarks suite
 <http://hg.python.org/benchmarks>`_ (-b 2n3): some tests are 1.04x
-faster, some tests are 1.04 slower, significant is between 115 and -191.
+faster, some tests are 1.04 slower. Results of pybench microbenchmark:
+"+0.1%" slower globally (diff between -4.9% and +5.6%).
 
-Results of pybench benchmark: "+0.1%" slower globally (diff between
--4.9% and +5.6%).
-
-The full reports are attached to the issue #3329.
+The full output of benchmarks is attached to the issue #3329.
 
 
 Rejected Alternatives
@@ -428,8 +434,9 @@
 * ``void PyMem_SetAllocator(PyMemAllocator *allocator)``
 * ``void PyObject_SetAllocator(PyMemAllocator *allocator)``
 
-With more specific functions, it becomes more difficult to write generic
-code, like reusing the same code for different allocator domains.
+This alternative was rejected because it is not possible to write
+generic code with more specific functions: code must be duplicated for
+each memory allocator domain.
 
 
 Make PyMem_Malloc() reuse PyMem_RawMalloc() by default
@@ -439,25 +446,25 @@
 calling ``PyMem_SetAllocator(PYMEM_DOMAIN_RAW, alloc)`` would also also
 patch ``PyMem_Malloc()`` indirectly.
 
-This option was rejected because ``PyMem_SetAllocator()`` would have a
-different behaviour depending on the domain. Always having the same
-behaviour is less error-prone.
+This alternative was rejected because ``PyMem_SetAllocator()`` would
+have a different behaviour depending on the domain. Always having the
+same behaviour is less error-prone.
 
 
 Add a new PYDEBUGMALLOC environment variable
 --------------------------------------------
 
-To be able to use the Python builtin debug hooks even when a custom
-memory allocator replaces the default Python allocator, an environment
-variable ``PYDEBUGMALLOC`` can be added to setup these debug function
-hooks, instead of adding the new function ``PyMem_SetupDebugHooks()``.
-If the environment variable is present, ``PyMem_SetRawAllocator()``,
-``PyMem_SetAllocator()`` and ``PyObject_SetAllocator()`` will reinstall
-automatically the hook on top of the new allocator.
+Add a new ``PYDEBUGMALLOC`` environment variable to enable debug checks
+on memory block allocators. The environment variable replaces the new
+function ``PyMem_SetupDebugHooks()`` which is not needed anymore.
+Another advantage is to allow to enable debug checks even in release
+mode: debug checks are always compiled, but only enabled when the
+environment variable is present and non-empty.
 
-A new environment variable would make the Python initialization even
-more complex. The `PEP 432 <http://www.python.org/dev/peps/pep-0432/>`_
-tries to simply the CPython startup sequence.
+This alternative was rejected because a new environment variable would
+make the Python initialization even more complex. The `PEP 432
+<http://www.python.org/dev/peps/pep-0432/>`_ tries to simply the CPython
+startup sequence.
 
 
 Use macros to get customizable allocators
@@ -503,7 +510,7 @@
     void* _PyMem_MallocTrace(const char *filename, int lineno,
                              size_t size);
 
-    /* need also a function for the Python stable ABI */
+    /* the function is still needed for the Python stable ABI */
     void* PyMem_Malloc(size_t size);
 
     #define PyMem_Malloc(size) \
@@ -527,19 +534,19 @@
 calls indirectly ``PyObject_Malloc()`` which requires the GIL to be
 held.  That's why ``PyMem_Malloc()`` must be called with the GIL held.
 
-This PEP proposes changes ``PyMem_Malloc()``: it now always call
-``malloc()``.  The "GIL must be held" restriction can be removed from
+This PEP changes ``PyMem_Malloc()``: it now always call ``malloc()``.
+The "GIL must be held" restriction could be removed from
 ``PyMem_Malloc()``.
 
 This alternative was rejected because allowing to call
-``PyMem_Malloc()`` without holding the GIL might break applications
+``PyMem_Malloc()`` without holding the GIL can break applications
 which setup their own allocators or allocator hooks.  Holding the GIL is
 convinient to develop a custom allocator: no need to care of other
 threads. It is also convinient for a debug allocator hook: Python
 internal objects can be safetly inspected.
 
 Calling ``PyGILState_Ensure()`` in
-a memory allocator may have unexpected behaviour, especially at Python
+a memory allocator has unexpected behaviour, especially at Python
 startup and at creation of a new Python thread state.
 
 
@@ -552,13 +559,14 @@
 The ``PyMem_Malloc()`` is used without the GIL held in some Python
 functions.  For example, the ``main()`` and ``Py_Main()`` functions of
 Python call ``PyMem_Malloc()`` whereas the GIL do not exist yet. In this
-case, ``PyMem_Malloc()`` should be replaced with ``malloc()`` (or
+case, ``PyMem_Malloc()`` would be replaced with ``malloc()`` (or
 ``PyMem_RawMalloc()``).
 
-If an hook is used to the track memory usage, the memory allocated by
-direct calls to ``malloc()`` will not be tracked. External libraries
-like OpenSSL or bz2 should not call ``malloc()`` directly, so large
-allocated will be included in memory usage reports.
+This alternative was rejected because ``PyMem_RawMalloc()`` is required
+for accurate reports of the memory usage. When a debug hook is used to
+track the memory usage, the memory allocated by direct calls to
+``malloc()`` cannot be tracked. ``PyMem_RawMalloc()`` can be hooked and
+so all the memory allocated by Python can be tracked.
 
 
 Use existing debug tools to analyze the memory
@@ -571,11 +579,11 @@
 <http://www.nongnu.org/failmalloc/>`_, etc.
 
 The problem is to retrieve the Python object related to a memory pointer
-to read its type and/or content. Another issue is to retrieve the
+to read its type and/or its content. Another issue is to retrieve the
 location of the memory allocation: the C backtrace is usually useless
-(same reasoning than macros using ``__FILE__`` and ``__LINE__``), the
-Python filename and line number (or even the Python traceback) is more
-useful.
+(same reasoning than macros using ``__FILE__`` and ``__LINE__``, see
+`Pass the C filename and line number`_), the Python filename and line
+number (or even the Python traceback) is more useful.
 
 This alternative was rejected because classic tools are unable to
 introspect Python internals to collect such information. Being able to
@@ -586,8 +594,8 @@
 Add a msize() function
 ----------------------
 
-Add another field to ``PyMemAllocator`` and ``PyObjectArenaAllocator``
-structures::
+Add another function to ``PyMemAllocator`` and
+``PyObjectArenaAllocator`` structures::
 
     size_t msize(void *ptr);
 
@@ -607,8 +615,8 @@
 platforms implement it. For example, Linux with the GNU libc does not
 provide a function to get the size of a memory block. ``msize()`` is not
 currently used in the Python source code. The function is only used to
-track the memory usage, but makes the API more complex. A debug hook can
-implemente the function internally, there is no need to add it to
+track the memory usage, and makes the API more complex. A debug hook can
+implement the function internally, there is no need to add it to
 ``PyMemAllocator`` and ``PyObjectArenaAllocator`` structures.
 
 
@@ -653,17 +661,19 @@
 * lzma: `LZMA SDK - How to Use
   <http://www.asawicki.info/news_1368_lzma_sdk_-_how_to_use.html>`_,
   pass an opaque pointer
-* lipmpdec doesn't have this extra *ctx* parameter
+* lipmpdec: no opaque pointer (classic malloc API)
 
 Other libraries:
 
 * glib: `g_mem_set_vtable()
   <http://developer.gnome.org/glib/unstable/glib-Memory-Allocation.html#g-mem-set-vtable>`_
-* libxml2: `xmlGcMemSetup() <http://xmlsoft.org/html/libxml-xmlmemory.html>`_,
+* libxml2:
+  `xmlGcMemSetup() <http://xmlsoft.org/html/libxml-xmlmemory.html>`_,
   global
 * Oracle's OCI: `Oracle Call Interface Programmer's Guide,
   Release 2 (9.2)
-  <http://docs.oracle.com/cd/B10501_01/appdev.920/a96584/oci15re4.htm>`_
+  <http://docs.oracle.com/cd/B10501_01/appdev.920/a96584/oci15re4.htm>`_,
+  pass an opaque pointer
 
 See also the `GNU libc: Memory Allocation Hooks
 <http://www.gnu.org/software/libc/manual/html_node/Hooks-for-Malloc.html>`_.
@@ -676,30 +686,30 @@
 Its implementation depends on the platform and of the C library. The GNU
 C library uses a modified ptmalloc2, based on "Doug Lea's Malloc"
 (dlmalloc). FreeBSD uses `jemalloc
-<http://www.canonware.com/jemalloc/>`_. Google provides tcmalloc which
+<http://www.canonware.com/jemalloc/>`_. Google provides *tcmalloc* which
 is part of `gperftools <http://code.google.com/p/gperftools/>`_.
 
 ``malloc()`` uses two kinds of memory: heap and memory mappings. Memory
 mappings are usually used for large allocations (ex: larger than 256
 KB), whereas the heap is used for small allocations.
 
-On UNIX, the heap is handled by ``brk()`` and ``sbrk()`` system calls on
-Linux, and it is contiguous.  On Windows, the heap is handled by
-``HeapAlloc()`` and may be discontiguous. Memory mappings are handled by
-``mmap()`` on UNIX and ``VirtualAlloc()`` on Windows, they may be
+On UNIX, the heap is handled by ``brk()`` and ``sbrk()`` system calls,
+and it is contiguous.  On Windows, the heap is handled by
+``HeapAlloc()`` and can be discontiguous. Memory mappings are handled by
+``mmap()`` on UNIX and ``VirtualAlloc()`` on Windows, they can be
 discontiguous.
 
 Releasing a memory mapping gives back immediatly the memory to the
-system. On UNIX, heap memory is only given back to the system if it is
-at the end of the heap. Otherwise, the memory will only be given back to
-the system when all the memory located after the released memory are
-also released.
+system. On UNIX, the heap memory is only given back to the system if the
+released block is located at the end of the heap. Otherwise, the memory
+will only be given back to the system when all the memory located after
+the released memory is also released.
 
-To allocate memory in the heap, the allocator tries to reuse free space.
-If there is no contiguous space big enough, the heap must be increased,
-even if we have more free space than required size.  This issue is
+To allocate memory on the heap, an allocator tries to reuse free space.
+If there is no contiguous space big enough, the heap must be enlarged,
+even if there is more free space than required size.  This issue is
 called the "memory fragmentation": the memory usage seen by the system
-may be much higher than real usage. On Windows, ``HeapAlloc()`` creates
+is higher than real usage. On Windows, ``HeapAlloc()`` creates
 a new memory mapping with ``VirtualAlloc()`` if there is not enough free
 contiguous memory.
 
@@ -730,8 +740,8 @@
   <http://bugs.python.org/issue3329>`_
 * `Issue #13483: Use VirtualAlloc to allocate memory arenas
   <http://bugs.python.org/issue13483>`_
-* `Issue #16742: PyOS_Readline drops GIL and calls PyOS_StdioReadline, which
-  isn't thread safe <http://bugs.python.org/issue16742>`_
+* `Issue #16742: PyOS_Readline drops GIL and calls PyOS_StdioReadline,
+  which isn't thread safe <http://bugs.python.org/issue16742>`_
 * `Issue #18203: Replace calls to malloc() with PyMem_Malloc() or
   PyMem_RawMalloc() <http://bugs.python.org/issue18203>`_
 * `Issue #18227: Use Python memory allocators in external libraries like

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Mon Jul  1 22:45:04 2013
From: python-checkins at python.org (victor.stinner)
Date: Mon,  1 Jul 2013 22:45:04 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_445=3A_rephrase_abstract?=
Message-ID: <3bkgZ86Lxyz7Ll3@mail.python.org>

http://hg.python.org/peps/rev/f3032259c28e
changeset:   4970:f3032259c28e
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Jul 01 22:44:48 2013 +0200
summary:
  PEP 445: rephrase abstract

Remove also a wrong sentence: using the same prototypes than external libraries
is not a must-have. In fact, it was neither a guideline for this PEP, but just
an inspiration.

files:
  pep-0445.txt |  7 ++-----
  1 files changed, 2 insertions(+), 5 deletions(-)


diff --git a/pep-0445.txt b/pep-0445.txt
--- a/pep-0445.txt
+++ b/pep-0445.txt
@@ -12,8 +12,8 @@
 Abstract
 ========
 
-Add new Application Programming Interfaces (API) to customize Python
-memory allocators.
+This PEP proposes new Application Programming Interfaces (API) to customize
+Python memory allocators.
 
 
 Rationale
@@ -642,9 +642,6 @@
 External libraries
 ==================
 
-Python should try to reuse the same prototypes for allocator functions
-than other libraries.
-
 Libraries used by Python:
 
 * OpenSSL: `CRYPTO_set_mem_functions()

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Mon Jul  1 23:00:33 2013
From: python-checkins at python.org (christian.heimes)
Date: Mon,  1 Jul 2013 23:00:33 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4MzM5?=
 =?utf-8?q?=3A_use_with_self=2EassertRaises=28=29_to_make_test_case_more_r?=
 =?utf-8?q?eadable?=
Message-ID: <3bkgw171JqzPPY@mail.python.org>

http://hg.python.org/cpython/rev/fa0a03afe359
changeset:   84416:fa0a03afe359
branch:      3.3
parent:      84414:3ec5267f51ff
user:        Christian Heimes <christian at cheimes.de>
date:        Mon Jul 01 23:00:13 2013 +0200
summary:
  Issue #18339: use with self.assertRaises() to make test case more readable

files:
  Lib/test/test_pickle.py |  6 ++++--
  1 files changed, 4 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -117,9 +117,11 @@
 
         def test_issue18339(self):
             unpickler = self.unpickler_class(io.BytesIO())
-            self.assertRaises(TypeError, setattr, unpickler, "memo", object)
+            with self.assertRaises(TypeError):
+                unpickler.memo = object
             # used to cause a segfault
-            self.assertRaises(ValueError, setattr, unpickler, "memo", {-1: None})
+            with self.assertRaises(ValueError):
+                unpickler.memo = {-1: None}
             unpickler.memo = {1: None}
 
     class CDispatchTableTests(AbstractDispatchTableTests):

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 23:00:35 2013
From: python-checkins at python.org (christian.heimes)
Date: Mon,  1 Jul 2013 23:00:35 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2318339=3A_use_with_self=2EassertRaises=28=29_to_?=
 =?utf-8?q?make_test_case_more_readable?=
Message-ID: <3bkgw31j8nz7Lkq@mail.python.org>

http://hg.python.org/cpython/rev/de44cd866bb0
changeset:   84417:de44cd866bb0
parent:      84415:035577781505
parent:      84416:fa0a03afe359
user:        Christian Heimes <christian at cheimes.de>
date:        Mon Jul 01 23:00:22 2013 +0200
summary:
  Issue #18339: use with self.assertRaises() to make test case more readable

files:
  Lib/test/test_pickle.py |  6 ++++--
  1 files changed, 4 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -117,9 +117,11 @@
 
         def test_issue18339(self):
             unpickler = self.unpickler_class(io.BytesIO())
-            self.assertRaises(TypeError, setattr, unpickler, "memo", object)
+            with self.assertRaises(TypeError):
+                unpickler.memo = object
             # used to cause a segfault
-            self.assertRaises(ValueError, setattr, unpickler, "memo", {-1: None})
+            with self.assertRaises(ValueError):
+                unpickler.memo = {-1: None}
             unpickler.memo = {1: None}
 
     class CDispatchTableTests(AbstractDispatchTableTests):

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 23:15:59 2013
From: python-checkins at python.org (victor.stinner)
Date: Mon,  1 Jul 2013 23:15:59 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_445=3A_explain_why_Extern?=
 =?utf-8?q?al_Libraries_and_Memory_Allocators_sections_are?=
Message-ID: <3bkhFq2lJ8z7Ll8@mail.python.org>

http://hg.python.org/peps/rev/358aa763611a
changeset:   4971:358aa763611a
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Jul 01 23:15:45 2013 +0200
summary:
  PEP 445: explain why External Libraries and Memory Allocators sections are present

files:
  pep-0445.txt |  16 +++++++++++++---
  1 files changed, 13 insertions(+), 3 deletions(-)


diff --git a/pep-0445.txt b/pep-0445.txt
--- a/pep-0445.txt
+++ b/pep-0445.txt
@@ -639,9 +639,11 @@
 In C++, the context can be used to pass *this*.
 
 
-External libraries
+External Libraries
 ==================
 
+Examples of API used to customize memory allocators.
+
 Libraries used by Python:
 
 * OpenSSL: `CRYPTO_set_mem_functions()
@@ -672,11 +674,15 @@
   <http://docs.oracle.com/cd/B10501_01/appdev.920/a96584/oci15re4.htm>`_,
   pass an opaque pointer
 
+The new *ctx* parameter of this PEP was inspired by the API of zlib and
+Oracle's OCI libraries.
+
 See also the `GNU libc: Memory Allocation Hooks
-<http://www.gnu.org/software/libc/manual/html_node/Hooks-for-Malloc.html>`_.
+<http://www.gnu.org/software/libc/manual/html_node/Hooks-for-Malloc.html>`_
+which uses a different approach to hook memory allocators.
 
 
-Memory allocators
+Memory Allocators
 =================
 
 The C standard library provides the well known ``malloc()`` function.
@@ -727,6 +733,10 @@
   <https://developer.gnome.org/glib/unstable/glib-Memory-Slices.html>`_:
   efficient way to allocate groups of equal-sized chunks of memory
 
+This PEP permits to choose exactly which memory allocator is used for your
+application depending on its usage of the memory (number of allocation, size of
+allocations, lifetime of objects, etc.).
+
 
 Links
 =====

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Mon Jul  1 23:43:20 2013
From: python-checkins at python.org (christian.heimes)
Date: Mon,  1 Jul 2013 23:43:20 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4MzI4?=
 =?utf-8?q?=3A_Reorder_ops_in_PyThreadState=5FDelete*=28=29_functions=2E_N?=
 =?utf-8?q?ow_the?=
Message-ID: <3bkhsN24yzz7LlT@mail.python.org>

http://hg.python.org/cpython/rev/ff30bf84b378
changeset:   84418:ff30bf84b378
branch:      3.3
parent:      84416:fa0a03afe359
user:        Christian Heimes <christian at cheimes.de>
date:        Mon Jul 01 23:42:28 2013 +0200
summary:
  Issue #18328: Reorder ops in PyThreadState_Delete*() functions. Now the
tstate is first removed from TLS and then deallocated.
CID 1019639 (#1 of 1): Use after free (USE_AFTER_FREE)
 use_after_free: Using freed pointer tstate.

files:
  Misc/NEWS        |  3 +++
  Python/pystate.c |  4 ++--
  2 files changed, 5 insertions(+), 2 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,9 @@
 Core and Builtins
 -----------------
 
+- Issue #18328: Reorder ops in PyThreadState_Delete*() functions. Now the
+  tstate is first removed from TLS and then deallocated.
+
 - Issue #18184: PyUnicode_FromFormat() and PyUnicode_FromFormatV() now raise
   OverflowError when an argument of %c format is out of range.
 
diff --git a/Python/pystate.c b/Python/pystate.c
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -388,11 +388,11 @@
 {
     if (tstate == _Py_atomic_load_relaxed(&_PyThreadState_Current))
         Py_FatalError("PyThreadState_Delete: tstate is still current");
-    tstate_delete_common(tstate);
 #ifdef WITH_THREAD
     if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate)
         PyThread_delete_key_value(autoTLSkey);
 #endif /* WITH_THREAD */
+    tstate_delete_common(tstate);
 }
 
 
@@ -406,9 +406,9 @@
         Py_FatalError(
             "PyThreadState_DeleteCurrent: no current tstate");
     _Py_atomic_store_relaxed(&_PyThreadState_Current, NULL);
-    tstate_delete_common(tstate);
     if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate)
         PyThread_delete_key_value(autoTLSkey);
+    tstate_delete_common(tstate);
     PyEval_ReleaseLock();
 }
 #endif /* WITH_THREAD */

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  1 23:43:21 2013
From: python-checkins at python.org (christian.heimes)
Date: Mon,  1 Jul 2013 23:43:21 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2318328=3A_Reorder_ops_in_PyThreadState=5FDelete*?=
 =?utf-8?q?=28=29_functions=2E_Now_the?=
Message-ID: <3bkhsP48V2z7LlQ@mail.python.org>

http://hg.python.org/cpython/rev/ebe064e542ef
changeset:   84419:ebe064e542ef
parent:      84417:de44cd866bb0
parent:      84418:ff30bf84b378
user:        Christian Heimes <christian at cheimes.de>
date:        Mon Jul 01 23:43:09 2013 +0200
summary:
  Issue #18328: Reorder ops in PyThreadState_Delete*() functions. Now the
tstate is first removed from TLS and then deallocated.
CID 1019639 (#1 of 1): Use after free (USE_AFTER_FREE)
 use_after_free: Using freed pointer tstate.

files:
  Misc/NEWS        |  3 +++
  Python/pystate.c |  4 ++--
  2 files changed, 5 insertions(+), 2 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #18328: Reorder ops in PyThreadState_Delete*() functions. Now the
+  tstate is first removed from TLS and then deallocated.
+
 - Issue #13483: Use VirtualAlloc in obmalloc on Windows.
 
 - Issue #18184: PyUnicode_FromFormat() and PyUnicode_FromFormatV() now raise
diff --git a/Python/pystate.c b/Python/pystate.c
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -374,11 +374,11 @@
 {
     if (tstate == _Py_atomic_load_relaxed(&_PyThreadState_Current))
         Py_FatalError("PyThreadState_Delete: tstate is still current");
-    tstate_delete_common(tstate);
 #ifdef WITH_THREAD
     if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate)
         PyThread_delete_key_value(autoTLSkey);
 #endif /* WITH_THREAD */
+    tstate_delete_common(tstate);
 }
 
 
@@ -392,9 +392,9 @@
         Py_FatalError(
             "PyThreadState_DeleteCurrent: no current tstate");
     _Py_atomic_store_relaxed(&_PyThreadState_Current, NULL);
-    tstate_delete_common(tstate);
     if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate)
         PyThread_delete_key_value(autoTLSkey);
+    tstate_delete_common(tstate);
     PyEval_ReleaseLock();
 }
 #endif /* WITH_THREAD */

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 00:17:42 2013
From: python-checkins at python.org (victor.stinner)
Date: Tue,  2 Jul 2013 00:17:42 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4MzQz?=
 =?utf-8?q?=3A_faulthandler=2Eregister=28=29_now_keeps_the_previous_signal?=
 =?utf-8?q?_handler?=
Message-ID: <3bkjd214Fqz7LlC@mail.python.org>

http://hg.python.org/cpython/rev/229dde749ed6
changeset:   84420:229dde749ed6
branch:      3.3
parent:      84418:ff30bf84b378
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Tue Jul 02 00:14:56 2013 +0200
summary:
  Issue #18343: faulthandler.register() now keeps the previous signal handler
when the function is called twice, so faulthandler.unregister() restores
correctly the original signal handler.

files:
  Misc/NEWS              |  4 ++++
  Modules/faulthandler.c |  3 ++-
  2 files changed, 6 insertions(+), 1 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -41,6 +41,10 @@
 Library
 -------
 
+- Issue #18343: faulthandler.register() now keeps the previous signal handler
+  when the function is called twice, so faulthandler.unregister() restores
+  correctly the original signal handler.
+
 - Issue #17097: Make multiprocessing ignore EINTR.
 
 - Issue #18339: Negative ints keys in unpickler.memo dict no longer cause a
diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c
--- a/Modules/faulthandler.c
+++ b/Modules/faulthandler.c
@@ -742,6 +742,8 @@
             PyErr_SetFromErrno(PyExc_OSError);
             return NULL;
         }
+
+        user->previous = previous;
     }
 
     Py_XDECREF(user->file);
@@ -750,7 +752,6 @@
     user->fd = fd;
     user->all_threads = all_threads;
     user->chain = chain;
-    user->previous = previous;
     user->interp = tstate->interp;
     user->enabled = 1;
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 00:17:43 2013
From: python-checkins at python.org (victor.stinner)
Date: Tue,  2 Jul 2013 00:17:43 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_=28Merge_3=2E3=29_Issue_=2318343=3A_faulthandler=2Eregis?=
 =?utf-8?q?ter=28=29_now_keeps_the_previous_signal?=
Message-ID: <3bkjd3369bz7LlT@mail.python.org>

http://hg.python.org/cpython/rev/74b7ff20e0e4
changeset:   84421:74b7ff20e0e4
parent:      84419:ebe064e542ef
parent:      84420:229dde749ed6
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Tue Jul 02 00:17:14 2013 +0200
summary:
  (Merge 3.3) Issue #18343: faulthandler.register() now keeps the previous signal
handler when the function is called twice, so faulthandler.unregister()
restores correctly the original signal handler.

files:
  Misc/NEWS              |  4 ++++
  Modules/faulthandler.c |  3 ++-
  2 files changed, 6 insertions(+), 1 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -138,6 +138,10 @@
 Library
 -------
 
+- Issue #18343: faulthandler.register() now keeps the previous signal handler
+  when the function is called twice, so faulthandler.unregister() restores
+  correctly the original signal handler.
+
 - Issue #17097: Make multiprocessing ignore EINTR.
 
 - Issue #18339: Negative ints keys in unpickler.memo dict no longer cause a
diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c
--- a/Modules/faulthandler.c
+++ b/Modules/faulthandler.c
@@ -741,6 +741,8 @@
             PyErr_SetFromErrno(PyExc_OSError);
             return NULL;
         }
+
+        user->previous = previous;
     }
 
     Py_XDECREF(user->file);
@@ -749,7 +751,6 @@
     user->fd = fd;
     user->all_threads = all_threads;
     user->chain = chain;
-    user->previous = previous;
     user->interp = tstate->interp;
     user->enabled = 1;
 

-- 
Repository URL: http://hg.python.org/cpython

From solipsis at pitrou.net  Tue Jul  2 05:46:19 2013
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Tue, 02 Jul 2013 05:46:19 +0200
Subject: [Python-checkins] Daily reference leaks (74b7ff20e0e4): sum=0
Message-ID: <E1UtrXn-0002d3-SS@ap.vmr.nerim.net>

results for 74b7ff20e0e4 on branch "default"
--------------------------------------------



Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogheinsp', '-x']

From python-checkins at python.org  Tue Jul  2 13:42:33 2013
From: python-checkins at python.org (richard.oudkerk)
Date: Tue,  2 Jul 2013 13:42:33 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE3Mjcz?=
 =?utf-8?q?=3A_Clarify_that_pool_methods_can_only_be_used_by_parent_proces?=
 =?utf-8?q?s=2E?=
Message-ID: <3bl3Tj23zyz7Ljk@mail.python.org>

http://hg.python.org/cpython/rev/389788ba6bcb
changeset:   84422:389788ba6bcb
branch:      2.7
parent:      84413:bc34fe4a0d58
user:        Richard Oudkerk <shibturn at gmail.com>
date:        Tue Jul 02 12:31:50 2013 +0100
summary:
  Issue #17273: Clarify that pool methods can only be used by parent process.

files:
  Doc/library/multiprocessing.rst |  6 ++++++
  1 files changed, 6 insertions(+), 0 deletions(-)


diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -287,6 +287,9 @@
        print result.get(timeout=1)           # prints "100" unless your computer is *very* slow
        print pool.map(f, range(10))          # prints "[0, 1, 4,..., 81]"
 
+Note that the methods of a pool should only ever be used by the
+process which created it.
+
 
 Reference
 ---------
@@ -1599,6 +1602,9 @@
    *initializer* is not ``None`` then each worker process will call
    ``initializer(*initargs)`` when it starts.
 
+   Note that the methods of the pool object should only be called by
+   the process which created the pool.
+
    .. versionadded:: 2.7
       *maxtasksperchild* is the number of tasks a worker process can complete
       before it will exit and be replaced with a fresh worker process, to enable

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 13:42:34 2013
From: python-checkins at python.org (richard.oudkerk)
Date: Tue,  2 Jul 2013 13:42:34 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE3Mjcz?=
 =?utf-8?q?=3A_Clarify_that_pool_methods_can_only_be_used_by_parent_proces?=
 =?utf-8?q?s=2E?=
Message-ID: <3bl3Tk48p6z7Ljk@mail.python.org>

http://hg.python.org/cpython/rev/57fe80fda9be
changeset:   84423:57fe80fda9be
branch:      3.3
parent:      84420:229dde749ed6
user:        Richard Oudkerk <shibturn at gmail.com>
date:        Tue Jul 02 12:32:00 2013 +0100
summary:
  Issue #17273: Clarify that pool methods can only be used by parent process.

files:
  Doc/library/multiprocessing.rst |  6 ++++++
  1 files changed, 6 insertions(+), 0 deletions(-)


diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -284,6 +284,9 @@
            print(result.get(timeout=1))       # prints "100" unless your computer is *very* slow
            print(pool.map(f, range(10)))      # prints "[0, 1, 4,..., 81]"
 
+Note that the methods of a pool should only ever be used by the
+process which created it.
+
 
 Reference
 ---------
@@ -1665,6 +1668,9 @@
    *initializer* is not ``None`` then each worker process will call
    ``initializer(*initargs)`` when it starts.
 
+   Note that the methods of the pool object should only be called by
+   the process which created the pool.
+
    .. versionadded:: 3.2
       *maxtasksperchild* is the number of tasks a worker process can complete
       before it will exit and be replaced with a fresh worker process, to enable

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 13:42:35 2013
From: python-checkins at python.org (richard.oudkerk)
Date: Tue,  2 Jul 2013 13:42:35 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2317273=3A_Clarify_that_pool_methods_can_only_be_?=
 =?utf-8?q?used_by_parent_process=2E?=
Message-ID: <3bl3Tl69hcz7Lkw@mail.python.org>

http://hg.python.org/cpython/rev/7ccf3d36ad13
changeset:   84424:7ccf3d36ad13
parent:      84421:74b7ff20e0e4
parent:      84423:57fe80fda9be
user:        Richard Oudkerk <shibturn at gmail.com>
date:        Tue Jul 02 12:41:00 2013 +0100
summary:
  Issue #17273: Clarify that pool methods can only be used by parent process.

files:
  Doc/library/multiprocessing.rst |  6 ++++++
  1 files changed, 6 insertions(+), 0 deletions(-)


diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -284,6 +284,9 @@
            print(result.get(timeout=1))       # prints "100" unless your computer is *very* slow
            print(pool.map(f, range(10)))      # prints "[0, 1, 4,..., 81]"
 
+Note that the methods of a pool should only ever be used by the
+process which created it.
+
 
 Reference
 ---------
@@ -1668,6 +1671,9 @@
    *initializer* is not ``None`` then each worker process will call
    ``initializer(*initargs)`` when it starts.
 
+   Note that the methods of the pool object should only be called by
+   the process which created the pool.
+
    .. versionadded:: 3.2
       *maxtasksperchild* is the number of tasks a worker process can complete
       before it will exit and be replaced with a fresh worker process, to enable

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 14:02:32 2013
From: python-checkins at python.org (richard.oudkerk)
Date: Tue,  2 Jul 2013 14:02:32 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE0MjA2?=
 =?utf-8?q?=3A_Clarify_docs_for_Queue=2Ejoin=5Fcancel=5Fthread=28=29=2E?=
Message-ID: <3bl3wm34bxz7LlR@mail.python.org>

http://hg.python.org/cpython/rev/f35401dba89f
changeset:   84425:f35401dba89f
branch:      2.7
parent:      84422:389788ba6bcb
user:        Richard Oudkerk <shibturn at gmail.com>
date:        Tue Jul 02 12:58:21 2013 +0100
summary:
  Issue #14206: Clarify docs for Queue.join_cancel_thread().

files:
  Doc/library/multiprocessing.rst |  7 +++++++
  1 files changed, 7 insertions(+), 0 deletions(-)


diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -630,6 +630,13 @@
       the background thread from being joined automatically when the process
       exits -- see :meth:`join_thread`.
 
+      A better name for this method might be
+      ``allow_exit_without_flush()``.  It is likely to cause enqueued
+      data to lost, and you almost certainly will not need to use it.
+      It is really only there if you need the current process to exit
+      immediately without waiting to flush enqueued data to the
+      underlying pipe, and you don't care about lost data.
+
 
 .. class:: multiprocessing.queues.SimpleQueue()
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 14:02:33 2013
From: python-checkins at python.org (richard.oudkerk)
Date: Tue,  2 Jul 2013 14:02:33 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE0MjA2?=
 =?utf-8?q?=3A_Clarify_docs_for_Queue=2Ejoin=5Fcancel=5Fthread=28=29=2E?=
Message-ID: <3bl3wn4yvVz7LlT@mail.python.org>

http://hg.python.org/cpython/rev/9746f217a270
changeset:   84426:9746f217a270
branch:      3.3
parent:      84423:57fe80fda9be
user:        Richard Oudkerk <shibturn at gmail.com>
date:        Tue Jul 02 12:59:55 2013 +0100
summary:
  Issue #14206: Clarify docs for Queue.join_cancel_thread().

files:
  Doc/library/multiprocessing.rst |  7 +++++++
  1 files changed, 7 insertions(+), 0 deletions(-)


diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -659,6 +659,13 @@
       the background thread from being joined automatically when the process
       exits -- see :meth:`join_thread`.
 
+      A better name for this method might be
+      ``allow_exit_without_flush()``.  It is likely to cause enqueued
+      data to lost, and you almost certainly will not need to use it.
+      It is really only there if you need the current process to exit
+      immediately without waiting to flush enqueued data to the
+      underlying pipe, and you don't care about lost data.
+
 
 .. class:: SimpleQueue()
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 14:02:34 2013
From: python-checkins at python.org (richard.oudkerk)
Date: Tue,  2 Jul 2013 14:02:34 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2314206=3A_Clarify_docs_for_Queue=2Ejoin=5Fcancel?=
 =?utf-8?b?X3RocmVhZCgpLg==?=
Message-ID: <3bl3wp6wS5z7LlT@mail.python.org>

http://hg.python.org/cpython/rev/d0b48efac9de
changeset:   84427:d0b48efac9de
parent:      84424:7ccf3d36ad13
parent:      84426:9746f217a270
user:        Richard Oudkerk <shibturn at gmail.com>
date:        Tue Jul 02 13:01:31 2013 +0100
summary:
  Issue #14206: Clarify docs for Queue.join_cancel_thread().

files:
  Doc/library/multiprocessing.rst |  7 +++++++
  1 files changed, 7 insertions(+), 0 deletions(-)


diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -659,6 +659,13 @@
       the background thread from being joined automatically when the process
       exits -- see :meth:`join_thread`.
 
+      A better name for this method might be
+      ``allow_exit_without_flush()``.  It is likely to cause enqueued
+      data to lost, and you almost certainly will not need to use it.
+      It is really only there if you need the current process to exit
+      immediately without waiting to flush enqueued data to the
+      underlying pipe, and you don't care about lost data.
+
 
 .. class:: SimpleQueue()
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 14:41:05 2013
From: python-checkins at python.org (richard.oudkerk)
Date: Tue,  2 Jul 2013 14:41:05 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE3MjYx?=
 =?utf-8?q?=3A_Ensure_multiprocessing=27s_proxies_use_proper_address=2E?=
Message-ID: <3bl4nF2VCszS2M@mail.python.org>

http://hg.python.org/cpython/rev/20b59fec1123
changeset:   84428:20b59fec1123
branch:      2.7
parent:      84425:f35401dba89f
user:        Richard Oudkerk <shibturn at gmail.com>
date:        Tue Jul 02 13:31:43 2013 +0100
summary:
  Issue #17261: Ensure multiprocessing's proxies use proper address.

files:
  Lib/multiprocessing/managers.py |  1 +
  Misc/NEWS                       |  2 ++
  2 files changed, 3 insertions(+), 0 deletions(-)


diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py
--- a/Lib/multiprocessing/managers.py
+++ b/Lib/multiprocessing/managers.py
@@ -763,6 +763,7 @@
         elif kind == '#PROXY':
             exposed, token = result
             proxytype = self._manager._registry[token.typeid][-1]
+            token.address = self._token.address
             proxy = proxytype(
                 token, self._serializer, manager=self._manager,
                 authkey=self._authkey, exposed=exposed
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -24,6 +24,8 @@
 Library
 -------
 
+- Issue #17261: Ensure multiprocessing's proxies use proper address.
+
 - Issue #17097: Make multiprocessing ignore EINTR.
 
 - Issue #18155: The csv module now correctly handles csv files that use

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 14:41:06 2013
From: python-checkins at python.org (richard.oudkerk)
Date: Tue,  2 Jul 2013 14:41:06 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE3MjYx?=
 =?utf-8?q?=3A_Ensure_multiprocessing=27s_proxies_use_proper_address=2E?=
Message-ID: <3bl4nG4WX5z7Ll3@mail.python.org>

http://hg.python.org/cpython/rev/be4b9e677187
changeset:   84429:be4b9e677187
branch:      3.3
parent:      84426:9746f217a270
user:        Richard Oudkerk <shibturn at gmail.com>
date:        Tue Jul 02 13:37:43 2013 +0100
summary:
  Issue #17261: Ensure multiprocessing's proxies use proper address.

files:
  Lib/multiprocessing/managers.py |  1 +
  Misc/NEWS                       |  2 ++
  2 files changed, 3 insertions(+), 0 deletions(-)


diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py
--- a/Lib/multiprocessing/managers.py
+++ b/Lib/multiprocessing/managers.py
@@ -731,6 +731,7 @@
         elif kind == '#PROXY':
             exposed, token = result
             proxytype = self._manager._registry[token.typeid][-1]
+            token.address = self._token.address
             proxy = proxytype(
                 token, self._serializer, manager=self._manager,
                 authkey=self._authkey, exposed=exposed
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -41,6 +41,8 @@
 Library
 -------
 
+- Issue #17261: Ensure multiprocessing's proxies use proper address.
+
 - Issue #18343: faulthandler.register() now keeps the previous signal handler
   when the function is called twice, so faulthandler.unregister() restores
   correctly the original signal handler.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 14:41:07 2013
From: python-checkins at python.org (richard.oudkerk)
Date: Tue,  2 Jul 2013 14:41:07 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2317261=3A_Ensure_multiprocessing=27s_proxies_use?=
 =?utf-8?q?_proper_address=2E?=
Message-ID: <3bl4nH6ggFz7LlQ@mail.python.org>

http://hg.python.org/cpython/rev/7387b884f833
changeset:   84430:7387b884f833
parent:      84427:d0b48efac9de
parent:      84429:be4b9e677187
user:        Richard Oudkerk <shibturn at gmail.com>
date:        Tue Jul 02 13:38:58 2013 +0100
summary:
  Issue #17261: Ensure multiprocessing's proxies use proper address.

files:
  Lib/multiprocessing/managers.py |  1 +
  Misc/NEWS                       |  2 ++
  2 files changed, 3 insertions(+), 0 deletions(-)


diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py
--- a/Lib/multiprocessing/managers.py
+++ b/Lib/multiprocessing/managers.py
@@ -731,6 +731,7 @@
         elif kind == '#PROXY':
             exposed, token = result
             proxytype = self._manager._registry[token.typeid][-1]
+            token.address = self._token.address
             proxy = proxytype(
                 token, self._serializer, manager=self._manager,
                 authkey=self._authkey, exposed=exposed
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -138,6 +138,8 @@
 Library
 -------
 
+- Issue #17261: Ensure multiprocessing's proxies use proper address.
+
 - Issue #18343: faulthandler.register() now keeps the previous signal handler
   when the function is called twice, so faulthandler.unregister() restores
   correctly the original signal handler.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 14:46:02 2013
From: python-checkins at python.org (nick.coghlan)
Date: Tue,  2 Jul 2013 14:46:02 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?devinabox=3A_Tweak_a_few_things?=
Message-ID: <3bl4ty1HgZzRk0@mail.python.org>

http://hg.python.org/devinabox/rev/53a1f44a6219
changeset:   48:53a1f44a6219
user:        Nick Coghlan <ncoghlan at gmail.com>
date:        Tue Jul 02 22:45:52 2013 +1000
summary:
  Tweak a few things

files:
  .hgignore        |   2 ++
  README           |  22 +++++++++++++++-------
  full_coverage.py |  13 ++++++++++---
  3 files changed, 27 insertions(+), 10 deletions(-)


diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -5,6 +5,8 @@
 cpython
 devguide
 peps
+# Unversioned downloads
+Visual C++ Express/vc_web.exe
 # Generated
 coverage_report
 original_coverage_report
diff --git a/README b/README
--- a/README
+++ b/README
@@ -26,6 +26,10 @@
 get updates, sparing the probably taxed internet connection at the sprint from
 doing complete repository cloning.
 
+If recreating from an old checkout ``hg purge --all`` in the individual
+clones is a handy way to ensure old build artifacts have been removed.
+You will need to enable the purge extension in ``~/.hgrc``.
+
 
 Mercurial
 ---------
@@ -104,15 +108,16 @@
 
 01. Build the CPython repository
 02. Clone the repository from https://bitbucket.org/ned/coveragepy
-03. Download Distribute from https://pypi.python.org/pypi/distribute
-04. Unpack and build Distribute with Python 3 in the coveragepy repository
-05. Symlink the ``distribute-N.N.N/build/lib/setuptools`` directory into
-   ``coveragepy``
-06. Symlink ``distribute-N.N.N/build/lib/pkg_resources.py`` into ``coveragepy``
+03. Clone setuptools from https://bitbucket.org/pypa/setuptools/
+04. Run ``python3 setup.py build`` in the ``setuptools`` directory
+05. Run ``ln -s ../setuptools/build/lib/setuptools`` in the ``coveragepy``
+    directory
+06. Run ``ln -s ../setuptools/build/lib/pkg_resources.py`` in the
+    ``coveragepy`` directory
 07. Run ``./cpython/python full_coverage.py build``
 08. Run ``./cpython/python full_coverage.py run``
 09. Run ``./cpython/python full_coverage.py html original_coverage_report``
-10. ``make distclean`` the CPython repository
+10. Run ``hg purge --all`` in the CPython repository
 
 
 All these steps will generate a complete coverage report for the standard
@@ -150,9 +155,12 @@
 just simplifies this by having a single command subsume both the configure and
 build steps. It also uses reasonable defaults (e.g. all cores on the CPU).
 
+(You may need to cd into the CPython directory and run ``make`` to get the
+extension modules to build)
+
 
 ``index.html``
 --------------
 
 An HTML file with links to the various pieces of documentation you built
-previously and the helper scripts.
\ No newline at end of file
+previously and the helper scripts.
diff --git a/full_coverage.py b/full_coverage.py
--- a/full_coverage.py
+++ b/full_coverage.py
@@ -29,6 +29,13 @@
     yield
     os.chdir(original_directory)
 
+def make_setup_cmd(*args):
+    # Silence a spurious warning from setuptools
+    # See https://bitbucket.org/pypa/setuptools/issue/29/
+    cmd = [sys.executable, '-W', 'ignore::UserWarning:distutils.dist',
+           'setup.py']
+    cmd.extend(args)
+    return cmd
 
 def build(args):
     """Build coverage.py's C-based tracer.
@@ -43,11 +50,11 @@
                 os.unlink(tracer_path)
             except FileNotFoundError:
                 pass
-        subprocess.check_call([sys.executable, 'setup.py', 'clean'])
+        subprocess.check_call(make_setup_cmd('clean'))
         env = os.environ.copy()
         env['CPPFLAGS'] = '-I {} -I {}'.format(CPYTHON,
                                                os.path.join(CPYTHON, 'Include'))
-        command = [sys.executable, 'setup.py', 'build_ext', '--inplace']
+        command = make_setup_cmd('build_ext', '--inplace')
         process = subprocess.Popen(command, env=env)
         process.wait()
 
@@ -124,4 +131,4 @@
         help_args = ['-h']
         if args.command:
             help_args.insert(0, args.command)
-        parser.parse_args(help_args)
\ No newline at end of file
+        parser.parse_args(help_args)

-- 
Repository URL: http://hg.python.org/devinabox

From python-checkins at python.org  Tue Jul  2 15:08:49 2013
From: python-checkins at python.org (eric.smith)
Date: Tue,  2 Jul 2013 15:08:49 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogQ2xvc2VzICMxODMx?=
 =?utf-8?q?2=3A_=27make_distclean=27_no_longer_deletes_files_in_dot-direct?=
 =?utf-8?q?ories=2E?=
Message-ID: <3bl5PF6xfVzS29@mail.python.org>

http://hg.python.org/cpython/rev/5896f887a93a
changeset:   84431:5896f887a93a
branch:      2.7
parent:      84428:20b59fec1123
user:        Eric V. Smith <eric at trueblade.com>
date:        Tue Jul 02 09:02:03 2013 -0400
summary:
  Closes #18312: 'make distclean' no longer deletes files in dot-directories.

files:
  Makefile.pre.in |  10 +++++-----
  1 files changed, 5 insertions(+), 5 deletions(-)


diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1295,11 +1295,11 @@
 		Modules/ld_so_aix Modules/python.exp Misc/python.pc
 	-rm -f python*-gdb.py
 	-rm -f pybuilddir.txt
-	find $(srcdir) '(' -name '*.fdc' -o -name '*~' \
-			   -o -name '[@,#]*' -o -name '*.old' \
-			   -o -name '*.orig' -o -name '*.rej' \
-			   -o -name '*.bak' ')' \
-			   -exec rm -f {} ';'
+	find $(srcdir)/[a-zA-Z]* '(' -name '*.fdc' -o -name '*~' \
+				     -o -name '[@,#]*' -o -name '*.old' \
+				     -o -name '*.orig' -o -name '*.rej' \
+				     -o -name '*.bak' ')' \
+				     -exec rm -f {} ';'
 
 # Check for smelly exported symbols (not starting with Py/_Py)
 smelly: all

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 15:08:51 2013
From: python-checkins at python.org (eric.smith)
Date: Tue,  2 Jul 2013 15:08:51 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogQ2xvc2VzICMxODMx?=
 =?utf-8?q?2=3A_=27make_distclean=27_no_longer_deletes_files_in_dot-direct?=
 =?utf-8?q?ories=2E?=
Message-ID: <3bl5PH1hJvzS2M@mail.python.org>

http://hg.python.org/cpython/rev/910ec3471d55
changeset:   84432:910ec3471d55
branch:      3.3
parent:      84429:be4b9e677187
user:        Eric V. Smith <eric at trueblade.com>
date:        Tue Jul 02 09:06:54 2013 -0400
summary:
  Closes #18312: 'make distclean' no longer deletes files in dot-directories.

files:
  Makefile.pre.in |  10 +++++-----
  1 files changed, 5 insertions(+), 5 deletions(-)


diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1413,11 +1413,11 @@
 		Modules/Setup Modules/Setup.local Modules/Setup.config \
 		Modules/ld_so_aix Modules/python.exp Misc/python.pc
 	-rm -f python*-gdb.py
-	find $(srcdir) '(' -name '*.fdc' -o -name '*~' \
-			   -o -name '[@,#]*' -o -name '*.old' \
-			   -o -name '*.orig' -o -name '*.rej' \
-			   -o -name '*.bak' ')' \
-			   -exec rm -f {} ';'
+	find $(srcdir)/[a-zA-Z]* '(' -name '*.fdc' -o -name '*~' \
+				     -o -name '[@,#]*' -o -name '*.old' \
+				     -o -name '*.orig' -o -name '*.rej' \
+				     -o -name '*.bak' ')' \
+				     -exec rm -f {} ';'
 
 # Check for smelly exported symbols (not starting with Py/_Py)
 smelly: all

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  2 15:08:52 2013
From: python-checkins at python.org (eric.smith)
Date: Tue,  2 Jul 2013 15:08:52 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?b?KTogIzE4MzEyOiBtZXJnZSBmcm9tIDMuMy4=?=
Message-ID: <3bl5PJ3RpnzS8n@mail.python.org>

http://hg.python.org/cpython/rev/8e838598eed1
changeset:   84433:8e838598eed1
parent:      84430:7387b884f833
parent:      84432:910ec3471d55
user:        Eric V. Smith <eric at trueblade.com>
date:        Tue Jul 02 09:07:53 2013 -0400
summary:
  #18312: merge from 3.3.

files:
  Makefile.pre.in |  10 +++++-----
  1 files changed, 5 insertions(+), 5 deletions(-)


diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1416,11 +1416,11 @@
 		Modules/Setup Modules/Setup.local Modules/Setup.config \
 		Modules/ld_so_aix Modules/python.exp Misc/python.pc
 	-rm -f python*-gdb.py
-	find $(srcdir) '(' -name '*.fdc' -o -name '*~' \
-			   -o -name '[@,#]*' -o -name '*.old' \
-			   -o -name '*.orig' -o -name '*.rej' \
-			   -o -name '*.bak' ')' \
-			   -exec rm -f {} ';'
+	find $(srcdir)/[a-zA-Z]* '(' -name '*.fdc' -o -name '*~' \
+				     -o -name '[@,#]*' -o -name '*.old' \
+				     -o -name '*.orig' -o -name '*.rej' \
+				     -o -name '*.bak' ')' \
+				     -exec rm -f {} ';'
 
 # Check for smelly exported symbols (not starting with Py/_Py)
 smelly: all

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Wed Jul  3 00:45:10 2013
From: python-checkins at python.org (victor.stinner)
Date: Wed,  3 Jul 2013 00:45:10 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_445=3A_fix_typo?=
Message-ID: <3blLBG3m3fzNyn@mail.python.org>

http://hg.python.org/peps/rev/bd8e5c86fb27
changeset:   4972:bd8e5c86fb27
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Wed Jul 03 00:44:58 2013 +0200
summary:
  PEP 445: fix typo

files:
  pep-0445.txt |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/pep-0445.txt b/pep-0445.txt
--- a/pep-0445.txt
+++ b/pep-0445.txt
@@ -134,7 +134,7 @@
   bytes
 * ``PYMEM_DOMAIN_OBJ``: *pymalloc* allocator which falls back on
   ``PyMem_Malloc()`` for allocations larger than 512 bytes
-* *pymalloc* arena allocator: ``VirualAlloc()`` and ``VirtualFree()`` on
+* *pymalloc* arena allocator: ``VirtualAlloc()`` and ``VirtualFree()`` on
   Windows, ``mmap()`` and ``munmap()`` when available, or ``malloc()``
   and ``free()``
 

-- 
Repository URL: http://hg.python.org/peps

From solipsis at pitrou.net  Wed Jul  3 05:43:42 2013
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Wed, 03 Jul 2013 05:43:42 +0200
Subject: [Python-checkins] Daily reference leaks (8e838598eed1): sum=0
Message-ID: <E1UuDyo-0006Um-Qu@ap.vmr.nerim.net>

results for 8e838598eed1 on branch "default"
--------------------------------------------



Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogn4j0Cs', '-x']

From python-checkins at python.org  Wed Jul  3 19:26:44 2013
From: python-checkins at python.org (barry.warsaw)
Date: Wed,  3 Jul 2013 19:26:44 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_Relax_PEP_8=27s_perceived_=28?=
 =?utf-8?q?but_incorrect=29_prohibition_against_backslashed=2E?=
Message-ID: <3blq4N4v2PzRHr@mail.python.org>

http://hg.python.org/peps/rev/98f42236d1af
changeset:   4973:98f42236d1af
user:        Barry Warsaw <barry at python.org>
date:        Wed Jul 03 13:26:36 2013 -0400
summary:
  Relax PEP 8's perceived (but incorrect) prohibition against backslashed.

files:
  pep-0008.txt |  18 +++++++++++++++---
  1 files changed, 15 insertions(+), 3 deletions(-)


diff --git a/pep-0008.txt b/pep-0008.txt
--- a/pep-0008.txt
+++ b/pep-0008.txt
@@ -158,9 +158,21 @@
 line continuation inside parentheses, brackets and braces.  Long lines
 can be broken over multiple lines by wrapping expressions in
 parentheses. These should be used in preference to using a backslash
-for line continuation.  Make sure to indent the continued line
-appropriately.  The preferred place to break around a binary operator
-is *after* the operator, not before it.  Some examples::
+for line continuation.
+
+Backslashes may still be appropriate at times.  For example, long,
+multiple ``with``-statements cannot use implicit continuation, so
+backslashes are acceptable::
+
+    with open('/path/to/some/file/you/want/to/read') as file_1, \
+            open('/path/to/some/file/being/written', 'w') as file_2:
+        file_2.write(file_1.read())
+
+Another such case is with ``assert`` statements.
+
+Make sure to indent the continued line appropriately.  The preferred
+place to break around a binary operator is *after* the operator, not
+before it.  Some examples::
 
     class Rectangle(Blob):
 

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Wed Jul  3 22:37:09 2013
From: python-checkins at python.org (victor.stinner)
Date: Wed,  3 Jul 2013 22:37:09 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogdGVzdF9mYXVsdGhh?=
 =?utf-8?q?ndler=3A_skip_test=5Fread=5Fnull=28=29_on_AIX?=
Message-ID: <3blvJ55SNNz7Lmg@mail.python.org>

http://hg.python.org/cpython/rev/217535af4567
changeset:   84434:217535af4567
branch:      3.3
parent:      84432:910ec3471d55
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Wed Jul 03 22:29:42 2013 +0200
summary:
  test_faulthandler: skip test_read_null() on AIX

AIX maps the first page of memory at address zero as valid, read-only.  Reading
NULL is not a fault on AIX. This is utilized by IBM compiler optimizations.
One speculatively can indirect through a pointer which may be null without
first testing if null and defer the test before using the value.

files:
  Lib/test/test_faulthandler.py |  2 ++
  1 files changed, 2 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py
--- a/Lib/test/test_faulthandler.py
+++ b/Lib/test/test_faulthandler.py
@@ -107,6 +107,8 @@
         self.assertRegex(output, regex)
         self.assertNotEqual(exitcode, 0)
 
+    @unittest.skipIf(sys.platform.startswith('aix'),
+                     "the first page of memory is a mapped read-only on AIX")
     def test_read_null(self):
         self.check_fatal_error("""
 import faulthandler

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Wed Jul  3 22:37:11 2013
From: python-checkins at python.org (victor.stinner)
Date: Wed,  3 Jul 2013 22:37:11 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_=28Merge_3=2E3=29_test=5Ffaulthandler=3A_skip_test=5Frea?=
 =?utf-8?b?ZF9udWxsKCkgb24gQUlY?=
Message-ID: <3blvJ708HSz7LjX@mail.python.org>

http://hg.python.org/cpython/rev/eea9680599a1
changeset:   84435:eea9680599a1
parent:      84433:8e838598eed1
parent:      84434:217535af4567
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Wed Jul 03 22:35:39 2013 +0200
summary:
  (Merge 3.3) test_faulthandler: skip test_read_null() on AIX

AIX maps the first page of memory at address zero as valid, read-only.  Reading
NULL is not a fault on AIX. This is utilized by IBM compiler optimizations.
One speculatively can indirect through a pointer which may be null without
first testing if null and defer the test before using the value.

files:
  Lib/test/test_faulthandler.py |  2 ++
  1 files changed, 2 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py
--- a/Lib/test/test_faulthandler.py
+++ b/Lib/test/test_faulthandler.py
@@ -107,6 +107,8 @@
         self.assertRegex(output, regex)
         self.assertNotEqual(exitcode, 0)
 
+    @unittest.skipIf(sys.platform.startswith('aix'),
+                     "the first page of memory is a mapped read-only on AIX")
     def test_read_null(self):
         self.check_fatal_error("""
 import faulthandler

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Wed Jul  3 23:08:23 2013
From: python-checkins at python.org (victor.stinner)
Date: Wed,  3 Jul 2013 23:08:23 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogdGVzdF90aW1lLnRl?=
 =?utf-8?q?st=5Fmonotonic=28=29=3A_use_a_longer_sleep_to_try_to_make_the_t?=
 =?utf-8?q?est_more?=
Message-ID: <3blw070G3pzSlk@mail.python.org>

http://hg.python.org/cpython/rev/13e154067de3
changeset:   84436:13e154067de3
branch:      3.3
parent:      84434:217535af4567
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Wed Jul 03 23:07:37 2013 +0200
summary:
  test_time.test_monotonic(): use a longer sleep to try to make the test more reliable

files:
  Lib/test/test_time.py |  4 ++--
  1 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py
--- a/Lib/test/test_time.py
+++ b/Lib/test/test_time.py
@@ -368,11 +368,11 @@
                          'need time.monotonic')
     def test_monotonic(self):
         t1 = time.monotonic()
-        time.sleep(0.1)
+        time.sleep(0.5)
         t2 = time.monotonic()
         dt = t2 - t1
         self.assertGreater(t2, t1)
-        self.assertAlmostEqual(dt, 0.1, delta=0.2)
+        self.assertAlmostEqual(dt, 0.5, delta=0.2)
 
         info = time.get_clock_info('monotonic')
         self.assertTrue(info.monotonic)

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Wed Jul  3 23:08:24 2013
From: python-checkins at python.org (victor.stinner)
Date: Wed,  3 Jul 2013 23:08:24 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?b?KTogKE1lcmdlIDMuMykgdGVzdF90aW1lLnRlc3RfbW9ub3RvbmljKCk6IHVz?=
 =?utf-8?q?e_a_longer_sleep_to_try_to_make_the?=
Message-ID: <3blw082677z7Llw@mail.python.org>

http://hg.python.org/cpython/rev/c9545c2386c4
changeset:   84437:c9545c2386c4
parent:      84435:eea9680599a1
parent:      84436:13e154067de3
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Wed Jul 03 23:07:59 2013 +0200
summary:
  (Merge 3.3) test_time.test_monotonic(): use a longer sleep to try to make the test more reliable

files:
  Lib/test/test_time.py |  4 ++--
  1 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py
--- a/Lib/test/test_time.py
+++ b/Lib/test/test_time.py
@@ -368,11 +368,11 @@
                          'need time.monotonic')
     def test_monotonic(self):
         t1 = time.monotonic()
-        time.sleep(0.1)
+        time.sleep(0.5)
         t2 = time.monotonic()
         dt = t2 - t1
         self.assertGreater(t2, t1)
-        self.assertAlmostEqual(dt, 0.1, delta=0.2)
+        self.assertAlmostEqual(dt, 0.5, delta=0.2)
 
         info = time.get_clock_info('monotonic')
         self.assertTrue(info.monotonic)

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul  4 01:52:29 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu,  4 Jul 2013 01:52:29 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_Add_PEP_446=3A_based_on_PEP_4?=
 =?utf-8?q?33=2C_but_simpler_and_more_conservative?=
Message-ID: <3blzdT2mPHz7Ln5@mail.python.org>

http://hg.python.org/peps/rev/7c1f78c181bb
changeset:   4974:7c1f78c181bb
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 04 01:32:07 2013 +0200
summary:
  Add PEP 446: based on PEP 433, but simpler and more conservative

files:
  pep-0446.txt |  172 +++++++++++++++++++++++++++++++++++++++
  1 files changed, 172 insertions(+), 0 deletions(-)


diff --git a/pep-0446.txt b/pep-0446.txt
new file mode 100644
--- /dev/null
+++ b/pep-0446.txt
@@ -0,0 +1,172 @@
+PEP: 446
+Title: Add new cloexec and blocking parameters to functions creating file descriptors
+Version: $Revision$
+Last-Modified: $Date$
+Author: Victor Stinner <victor.stinner at gmail.com>
+Status: Draft
+Type: Standards Track
+Content-Type: text/x-rst
+Created: 3-July-2013
+Python-Version: 3.4
+
+
+Abstract
+========
+
+This PEP proposes new portable parameters and functions to configure the
+inherance of file descriptors and the non-blocking flag of sockets.
+
+
+Rationale
+=========
+
+Inherance of file descriptors
+-----------------------------
+
+The inherance of a file descriptor in a child process at the execution
+of a new program can be configured on each file descriptor using a
+*close-on-exec* flag. By default, the close-on-exec flag is not set. On
+Windows, file descriptors are not inherited if the ``bInheritHandles``
+parameter of the ``CreateProcess()`` function is ``FALSE``, even if the
+close-on-exec flag is not set. On UNIX, file descriptors are inherited
+by default.
+
+Inherance of file descriptors causes issues. For example, closing a file
+descriptor in the parent process does not release the resource (file,
+socket, ...), because the file descriptor is still open in the child
+process.
+
+Leaking file descriptors is a major security vulnerability. An
+untrusted child process can read sensitive data like passwords and
+take control of the parent process though leaked file descriptors. It
+is for example a known vulnerability to escape from a chroot.
+
+
+Non-blocking sockets
+--------------------
+
+To handle multiple network clients in a single thread, a multiplexing
+function like ``select()`` can be used. For best performances, sockets
+must be configured as non-blocking.
+
+By default, newly created sockets are blocking. Setting the non-blocking
+mode requires extra system calls.
+
+
+Setting flags at the creation of the file descriptor
+----------------------------------------------------
+
+Windows and recent versions of other operating systems like Linux
+support setting close-on-exec and blocking flags directly at the
+creation of file descriptors and sockets.
+
+Setting these flags at the creation is atomic and avoids extra system
+calls.
+
+
+Proposal
+========
+
+New cloexec And blocking Parameters
+-----------------------------------
+
+Add a new optional *cloexec* on functions creating file descriptors:
+
+* ``io.FileIO``
+* ``io.open()``
+* ``open()``
+* ``os.dup()``
+* ``os.dup2()``
+* ``os.fdopen()``
+* ``os.open()``
+* ``os.openpty()``
+* ``os.pipe()``
+* ``select.devpoll()``
+* ``select.epoll()``
+* ``select.kqueue()``
+
+Add new optional *cloexec* and *blocking* parameters to functions
+creating sockets:
+
+* ``asyncore.dispatcher.create_socket()``
+* ``socket.socket()``
+* ``socket.socket.accept()``
+* ``socket.socket.dup()``
+* ``socket.socket.fromfd``
+* ``socket.socketpair()``
+
+The default value of *cloexec* is ``False``, and the default value of
+*blocking* is ``True``.
+
+The atomicity is not guaranteed. If the platform does not support
+setting close-on-exec and blocking flags at the creation of the file
+descriptor, the flags are set using extra system calls.
+
+New Functions
+-------------
+
+Add new functions the get and set the close-on-exec flag of a file
+descriptor:
+
+* ``os.get_cloexec(fd:int) -> bool``
+* ``os.set_cloexec(fd:int, cloexec: bool)``
+
+
+Other Changes
+-------------
+
+The ``subprocess.Popen`` class must clear the close-on-exec flag of file
+descriptors of the ``pass_fds`` parameter.
+
+The close-on-exec flag must also be set on private file descriptors and
+sockets in the Python standard library. For example, on UNIX,
+os.urandom() opens ``/dev/urandom`` to read some random bytes, the file
+descriptor is closed at function exit.  The file descriptor is not
+expected to be inherited on execution of a new program in a child
+process.
+
+
+Alternatives
+============
+
+The PEP 433 is a previous attempt proposing various other alternatives,
+but no consensus could be reached.
+
+This PEP is much simpler, more conservative (no backward compatibility
+issue) and has a well defined behaviour (the default value of the new
+*cloexec* parameter is not configurable).
+
+
+Links
+=====
+
+Python issues:
+
+* `#10115: Support accept4() for atomic setting of flags at socket
+  creation <http://bugs.python.org/issue10115>`_
+* `#12105: open() does not able to set flags, such as O_CLOEXEC
+  <http://bugs.python.org/issue12105>`_
+* `#12107: TCP listening sockets created without FD_CLOEXEC flag
+  <http://bugs.python.org/issue12107>`_
+* `#16850: Add "e" mode to open(): close-and-exec
+  (O_CLOEXEC) / O_NOINHERIT <http://bugs.python.org/issue16850>`_
+* `#16860: Use O_CLOEXEC in the tempfile module
+  <http://bugs.python.org/issue16860>`_
+* `#16946: subprocess: _close_open_fd_range_safe() does not set
+  close-on-exec flag on Linux < 2.6.23 if O_CLOEXEC is defined
+  <http://bugs.python.org/issue16946>`_
+* `#17070: PEP 433: Use the new cloexec to improve security and avoid
+  bugs <http://bugs.python.org/issue17070>`_
+
+Other links:
+
+* `Secure File Descriptor Handling
+  <http://udrepper.livejournal.com/20407.html>`_ (Ulrich Drepper,
+  2008)
+
+
+Copyright
+=========
+
+This document has been placed into the public domain.
+

-- 
Repository URL: http://hg.python.org/peps

From solipsis at pitrou.net  Thu Jul  4 05:43:36 2013
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Thu, 04 Jul 2013 05:43:36 +0200
Subject: [Python-checkins] Daily reference leaks (c9545c2386c4): sum=2
Message-ID: <E1UuaSH-0002g7-00@ap.vmr.nerim.net>

results for c9545c2386c4 on branch "default"
--------------------------------------------

test_concurrent_futures leaked [-2, 3, 1] memory blocks, sum=2


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflog4RbBJl', '-x']

From python-checkins at python.org  Thu Jul  4 12:58:38 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu,  4 Jul 2013 12:58:38 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_446=3A_add_an_=22Overlapp?=
	=?utf-8?q?ed_I/O=22_alternative?=
Message-ID: <3bmGQ63tKqz7LjM@mail.python.org>

http://hg.python.org/peps/rev/20d12c860d89
changeset:   4975:20d12c860d89
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 04 12:58:03 2013 +0200
summary:
  PEP 446: add an "Overlapped I/O" alternative

files:
  pep-0446.txt |  108 +++++++++++++++++++++++++++-----------
  1 files changed, 76 insertions(+), 32 deletions(-)


diff --git a/pep-0446.txt b/pep-0446.txt
--- a/pep-0446.txt
+++ b/pep-0446.txt
@@ -23,23 +23,31 @@
 Inherance of file descriptors
 -----------------------------
 
-The inherance of a file descriptor in a child process at the execution
-of a new program can be configured on each file descriptor using a
-*close-on-exec* flag. By default, the close-on-exec flag is not set. On
-Windows, file descriptors are not inherited if the ``bInheritHandles``
-parameter of the ``CreateProcess()`` function is ``FALSE``, even if the
-close-on-exec flag is not set. On UNIX, file descriptors are inherited
-by default.
+The inherance of file descriptors in child processes can be configured
+on each file descriptor using a *close-on-exec* flag. By default, the
+close-on-exec flag is not set.
+
+On Windows, file descriptors are not inherited if the
+``bInheritHandles`` parameter of the ``CreateProcess()`` function is
+``FALSE``, even if the close-on-exec flag is not set.
+
+On UNIX, file descriptors with the close-and-exec flag set are closed at
+the execution of a new program (ex: when calling ``execv()``). The flag
+has no effect on ``fork()``, all file descriptors are inherited by the
+child process.
+
+Issues of the inherance of file descriptors
+-------------------------------------------
 
 Inherance of file descriptors causes issues. For example, closing a file
 descriptor in the parent process does not release the resource (file,
 socket, ...), because the file descriptor is still open in the child
 process.
 
-Leaking file descriptors is a major security vulnerability. An
-untrusted child process can read sensitive data like passwords and
-take control of the parent process though leaked file descriptors. It
-is for example a known vulnerability to escape from a chroot.
+Leaking file descriptors is also a major security vulnerability. An
+untrusted child process can read sensitive data like passwords and take
+control of the parent process though leaked file descriptors. It is for
+example a known vulnerability to escape from a chroot.
 
 
 Non-blocking sockets
@@ -47,21 +55,24 @@
 
 To handle multiple network clients in a single thread, a multiplexing
 function like ``select()`` can be used. For best performances, sockets
-must be configured as non-blocking.
+must be configured as non-blocking. Operations like ``send()`` and
+``recv()`` return an ``EAGAIN`` or ``EWOULDBLOCK`` error if the
+operation would block.
 
 By default, newly created sockets are blocking. Setting the non-blocking
-mode requires extra system calls.
+mode requires additional system calls.
 
 
 Setting flags at the creation of the file descriptor
 ----------------------------------------------------
 
 Windows and recent versions of other operating systems like Linux
-support setting close-on-exec and blocking flags directly at the
-creation of file descriptors and sockets.
+support setting the close-on-exec flag directly at the creation of file
+descriptors, and close-on-exec and blocking flags at the creation of
+sockets.
 
-Setting these flags at the creation is atomic and avoids extra system
-calls.
+Setting these flags at the creation is atomic and avoids additional
+system calls.
 
 
 Proposal
@@ -95,12 +106,12 @@
 * ``socket.socket.fromfd``
 * ``socket.socketpair()``
 
-The default value of *cloexec* is ``False``, and the default value of
+The default value of *cloexec* is ``False`` and the default value of
 *blocking* is ``True``.
 
 The atomicity is not guaranteed. If the platform does not support
 setting close-on-exec and blocking flags at the creation of the file
-descriptor, the flags are set using extra system calls.
+descriptor or socket, the flags are set using additional system calls.
 
 New Functions
 -------------
@@ -120,21 +131,54 @@
 
 The close-on-exec flag must also be set on private file descriptors and
 sockets in the Python standard library. For example, on UNIX,
-os.urandom() opens ``/dev/urandom`` to read some random bytes, the file
-descriptor is closed at function exit.  The file descriptor is not
-expected to be inherited on execution of a new program in a child
-process.
+os.urandom() opens ``/dev/urandom`` to read some random bytes and the
+file descriptor is closed at function exit. The file descriptor is not
+expected to be inherited by child processes.
 
 
-Alternatives
-============
+Rejected Alternatives
+=====================
 
-The PEP 433 is a previous attempt proposing various other alternatives,
-but no consensus could be reached.
+PEP 433
+-------
 
-This PEP is much simpler, more conservative (no backward compatibility
-issue) and has a well defined behaviour (the default value of the new
-*cloexec* parameter is not configurable).
+The PEP 433 entitled "Easier suppression of file descriptor inheritance"
+is a previous attempt proposing various other alternatives, but no
+consensus could be reached.
+
+This PEP has a well defined behaviour (the default value of the new
+*cloexec* parameter is not configurable), is more conservative (no
+backward compatibility issue), and is much simpler.
+
+
+Add blocking parameter for file descriptors and Windows overlapped I/O
+----------------------------------------------------------------------
+
+Windows supports non-blocking operations on files using an extension of
+the Windows API called "Overlapped I/O". Using this extension requires
+to modify the Python standard library and applications to pass a
+``OVERLAPPED`` structure and an event loop to wait for the completion of
+operations.
+
+This PEP only tries to expose portable flags on file descriptors and
+sockets. Supporting overlapped I/O requires an abstraction providing a
+high-level and portable API for asynchronous operations on files and
+sockets. Overlapped I/O are out of the scope of this PEP.
+
+UNIX supports non-blocking files, moreover recent versions of operating
+systems support setting the non-blocking flag at the creation of a file
+descriptor. It would be possible to add a new optional *blocking*
+parameter to Python functions creating file descriptors. On Windows,
+creating a file descriptor with ``blocking=False``  would raise a
+``NotImplementedError``. This behaviour is not acceptable for the ``os``
+module which is designed as a thin wrapper on the C functions of the
+operating system. If a platform does not support a function, the
+function should not be available on the platform. For example,
+the ``os.fork()`` function is not available on Windows.
+
+For all these reasons, this alternative was rejected. The PEP 3156
+proposes an abstraction for asynchronous I/O supporting non-blocking
+files on Windows.
 
 
 Links
@@ -155,8 +199,8 @@
 * `#16946: subprocess: _close_open_fd_range_safe() does not set
   close-on-exec flag on Linux < 2.6.23 if O_CLOEXEC is defined
   <http://bugs.python.org/issue16946>`_
-* `#17070: PEP 433: Use the new cloexec to improve security and avoid
-  bugs <http://bugs.python.org/issue17070>`_
+* `#17070: Use the new cloexec to improve security and avoid bugs
+  <http://bugs.python.org/issue17070>`_
 
 Other links:
 

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Thu Jul  4 13:01:44 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu,  4 Jul 2013 13:01:44 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_446=3A_better_title?=
Message-ID: <3bmGTh6fRDz7LjM@mail.python.org>

http://hg.python.org/peps/rev/105a96abaebe
changeset:   4976:105a96abaebe
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 04 13:01:38 2013 +0200
summary:
  PEP 446: better title

files:
  pep-0446.txt |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/pep-0446.txt b/pep-0446.txt
--- a/pep-0446.txt
+++ b/pep-0446.txt
@@ -1,5 +1,5 @@
 PEP: 446
-Title: Add new cloexec and blocking parameters to functions creating file descriptors
+Title: Add new parameters to configure the inherance of files and for non-blocking sockets
 Version: $Revision$
 Last-Modified: $Date$
 Author: Victor Stinner <victor.stinner at gmail.com>

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Thu Jul  4 21:05:43 2013
From: python-checkins at python.org (antoine.pitrou)
Date: Thu,  4 Jul 2013 21:05:43 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzExMTg1?=
 =?utf-8?q?=3A_Fix_test=5Fwait4_under_AIX=2E__Patch_by_S=C3=A9bastien_Sabl?=
 =?utf-8?b?w6ku?=
Message-ID: <3bmTD74NZrz7Lkm@mail.python.org>

http://hg.python.org/cpython/rev/b3ea1b5a1617
changeset:   84438:b3ea1b5a1617
branch:      3.3
parent:      84436:13e154067de3
user:        Antoine Pitrou <solipsis at pitrou.net>
date:        Thu Jul 04 21:03:10 2013 +0200
summary:
  Issue #11185: Fix test_wait4 under AIX.  Patch by S?bastien Sabl?.

files:
  Lib/test/test_wait4.py |  8 +++++++-
  Misc/NEWS              |  2 ++
  2 files changed, 9 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_wait4.py b/Lib/test/test_wait4.py
--- a/Lib/test/test_wait4.py
+++ b/Lib/test/test_wait4.py
@@ -3,6 +3,7 @@
 
 import os
 import time
+import sys
 from test.fork_wait import ForkWait
 from test.support import run_unittest, reap_children, get_attribute
 
@@ -13,10 +14,15 @@
 
 class Wait4Test(ForkWait):
     def wait_impl(self, cpid):
+        option = os.WNOHANG
+        if sys.platform.startswith('aix'):
+            # Issue #11185: wait4 is broken on AIX and will always return 0
+            # with WNOHANG.
+            option = 0
         for i in range(10):
             # wait4() shouldn't hang, but some of the buildbots seem to hang
             # in the forking tests.  This is an attempt to fix the problem.
-            spid, status, rusage = os.wait4(cpid, os.WNOHANG)
+            spid, status, rusage = os.wait4(cpid, option)
             if spid == cpid:
                 break
             time.sleep(1.0)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -155,6 +155,8 @@
 Tests
 -----
 
+- Issue #11185: Fix test_wait4 under AIX.  Patch by S?bastien Sabl?.
+
 - Issue #17691: test_univnewlines now works with unittest test discovery.
   Patch by Zachary Ware.
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul  4 21:05:44 2013
From: python-checkins at python.org (antoine.pitrou)
Date: Thu,  4 Jul 2013 21:05:44 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2311185=3A_Fix_test=5Fwait4_under_AIX=2E__Patch_b?=
 =?utf-8?b?eSBTw6liYXN0aWVuIFNhYmzDqS4=?=
Message-ID: <3bmTD86bsJz7Lpx@mail.python.org>

http://hg.python.org/cpython/rev/8055521e372f
changeset:   84439:8055521e372f
parent:      84437:c9545c2386c4
parent:      84438:b3ea1b5a1617
user:        Antoine Pitrou <solipsis at pitrou.net>
date:        Thu Jul 04 21:05:30 2013 +0200
summary:
  Issue #11185: Fix test_wait4 under AIX.  Patch by S?bastien Sabl?.

files:
  Lib/test/test_wait4.py |  8 +++++++-
  Misc/NEWS              |  3 ++-
  2 files changed, 9 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_wait4.py b/Lib/test/test_wait4.py
--- a/Lib/test/test_wait4.py
+++ b/Lib/test/test_wait4.py
@@ -3,6 +3,7 @@
 
 import os
 import time
+import sys
 from test.fork_wait import ForkWait
 from test.support import run_unittest, reap_children, get_attribute
 
@@ -13,10 +14,15 @@
 
 class Wait4Test(ForkWait):
     def wait_impl(self, cpid):
+        option = os.WNOHANG
+        if sys.platform.startswith('aix'):
+            # Issue #11185: wait4 is broken on AIX and will always return 0
+            # with WNOHANG.
+            option = 0
         for i in range(10):
             # wait4() shouldn't hang, but some of the buildbots seem to hang
             # in the forking tests.  This is an attempt to fix the problem.
-            spid, status, rusage = os.wait4(cpid, os.WNOHANG)
+            spid, status, rusage = os.wait4(cpid, option)
             if spid == cpid:
                 break
             time.sleep(1.0)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -465,10 +465,11 @@
 
 - Implement PEP 435 "Adding an Enum type to the Python standard library".
 
-
 Tests
 -----
 
+- Issue #11185: Fix test_wait4 under AIX.  Patch by S?bastien Sabl?.
+
 - Issue #18207: Fix test_ssl for some versions of OpenSSL that ignore seconds
   in ASN1_TIME fields.
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul  4 21:06:19 2013
From: python-checkins at python.org (antoine.pitrou)
Date: Thu,  4 Jul 2013 21:06:19 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Consolidate_tests_section?=
	=?utf-8?q?=2E?=
Message-ID: <3bmTDq2yZpz7Lmd@mail.python.org>

http://hg.python.org/cpython/rev/efdb4320219b
changeset:   84440:efdb4320219b
user:        Antoine Pitrou <solipsis at pitrou.net>
date:        Thu Jul 04 21:06:12 2013 +0200
summary:
  Consolidate tests section.

files:
  Misc/NEWS |  7 ++-----
  1 files changed, 2 insertions(+), 5 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -519,6 +519,8 @@
 - Issue #17692: test_sqlite now works with unittest test discovery.
   Patch by Zachary Ware.
 
+- Issue #11995: test_pydoc doesn't import all sys.path modules anymore.
+
 Documentation
 -------------
 
@@ -534,11 +536,6 @@
 - Issue #6696: add documentation for the Profile objects, and improve
   profile/cProfile docs.  Patch by Tom Pinckney.
 
-Tests
------
-
-- Issue #11995: test_pydoc doesn't import all sys.path modules anymore.
-
 C-API
 -----
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul  4 21:11:20 2013
From: python-checkins at python.org (antoine.pitrou)
Date: Thu,  4 Jul 2013 21:11:20 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzExMTg1?=
 =?utf-8?q?=3A_Fix_test=5Fwait4_under_AIX=2E__Patch_by_S=C3=A9bastien_Sabl?=
 =?utf-8?b?w6ku?=
Message-ID: <3bmTLc542jz7Lpk@mail.python.org>

http://hg.python.org/cpython/rev/e3fd5fc5dc47
changeset:   84441:e3fd5fc5dc47
branch:      2.7
parent:      84431:5896f887a93a
user:        Antoine Pitrou <solipsis at pitrou.net>
date:        Thu Jul 04 21:03:10 2013 +0200
summary:
  Issue #11185: Fix test_wait4 under AIX.  Patch by S?bastien Sabl?.

files:
  Lib/test/test_wait4.py |  8 +++++++-
  Misc/NEWS              |  2 ++
  2 files changed, 9 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_wait4.py b/Lib/test/test_wait4.py
--- a/Lib/test/test_wait4.py
+++ b/Lib/test/test_wait4.py
@@ -3,6 +3,7 @@
 
 import os
 import time
+import sys
 from test.fork_wait import ForkWait
 from test.test_support import run_unittest, reap_children, get_attribute
 
@@ -13,10 +14,15 @@
 
 class Wait4Test(ForkWait):
     def wait_impl(self, cpid):
+        option = os.WNOHANG
+        if sys.platform.startswith('aix'):
+            # Issue #11185: wait4 is broken on AIX and will always return 0
+            # with WNOHANG.
+            option = 0
         for i in range(10):
             # wait4() shouldn't hang, but some of the buildbots seem to hang
             # in the forking tests.  This is an attempt to fix the problem.
-            spid, status, rusage = os.wait4(cpid, os.WNOHANG)
+            spid, status, rusage = os.wait4(cpid, option)
             if spid == cpid:
                 break
             time.sleep(1.0)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -99,6 +99,8 @@
 Tests
 -----
 
+- Issue #11185: Fix test_wait4 under AIX.  Patch by S?bastien Sabl?.
+
 - Issue #18094: test_uuid no more reports skipped tests as passed.
 
 - Issue #11995: test_pydoc doesn't import all sys.path modules anymore.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul  4 22:10:56 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu,  4 Jul 2013 22:10:56 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_446=3A_typo=3A_inherance_?=
	=?utf-8?q?=3D=3E_inheritance?=
Message-ID: <3bmVgN4M6kz7LqQ@mail.python.org>

http://hg.python.org/peps/rev/39553edc98e2
changeset:   4977:39553edc98e2
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 04 22:10:39 2013 +0200
summary:
  PEP 446: typo: inherance => inheritance

files:
  pep-0446.txt |  22 +++++++++++-----------
  1 files changed, 11 insertions(+), 11 deletions(-)


diff --git a/pep-0446.txt b/pep-0446.txt
--- a/pep-0446.txt
+++ b/pep-0446.txt
@@ -1,5 +1,5 @@
 PEP: 446
-Title: Add new parameters to configure the inherance of files and for non-blocking sockets
+Title: Add new parameters to configure the inheritance of files and for non-blocking sockets
 Version: $Revision$
 Last-Modified: $Date$
 Author: Victor Stinner <victor.stinner at gmail.com>
@@ -14,16 +14,16 @@
 ========
 
 This PEP proposes new portable parameters and functions to configure the
-inherance of file descriptors and the non-blocking flag of sockets.
+inheritance of file descriptors and the non-blocking flag of sockets.
 
 
 Rationale
 =========
 
-Inherance of file descriptors
------------------------------
+Inheritance of file descriptors
+-------------------------------
 
-The inherance of file descriptors in child processes can be configured
+The inheritance of file descriptors in child processes can be configured
 on each file descriptor using a *close-on-exec* flag. By default, the
 close-on-exec flag is not set.
 
@@ -36,13 +36,13 @@
 has no effect on ``fork()``, all file descriptors are inherited by the
 child process.
 
-Issues of the inherance of file descriptors
--------------------------------------------
+Issues of the inheritance of file descriptors
+---------------------------------------------
 
-Inherance of file descriptors causes issues. For example, closing a file
-descriptor in the parent process does not release the resource (file,
-socket, ...), because the file descriptor is still open in the child
-process.
+Inheritance of file descriptors causes issues. For example, closing a
+file descriptor in the parent process does not release the resource
+(file, socket, ...), because the file descriptor is still open in the
+child process.
 
 Leaking file descriptors is also a major security vulnerability. An
 untrusted child process can read sensitive data like passwords and take

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Fri Jul  5 00:16:32 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri,  5 Jul 2013 00:16:32 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318200=3A_Back_out?=
 =?utf-8?q?_usage_of_ModuleNotFoundError_=288d28d44f3a9a=29?=
Message-ID: <3bmYSJ293zz7LlN@mail.python.org>

http://hg.python.org/cpython/rev/7769c4d72806
changeset:   84442:7769c4d72806
parent:      84440:efdb4320219b
user:        Brett Cannon <brett at python.org>
date:        Thu Jul 04 17:43:24 2013 -0400
summary:
  Issue #18200: Back out usage of ModuleNotFoundError (8d28d44f3a9a)

files:
  Lib/_dummy_thread.py              |   2 +-
  Lib/_osx_support.py               |   2 +-
  Lib/_pyio.py                      |   4 ++--
  Lib/_strptime.py                  |   2 +-
  Lib/bisect.py                     |   2 +-
  Lib/bz2.py                        |   2 +-
  Lib/cmd.py                        |   4 ++--
  Lib/code.py                       |   2 +-
  Lib/collections/__init__.py       |   2 +-
  Lib/copy.py                       |   2 +-
  Lib/datetime.py                   |   2 +-
  Lib/decimal.py                    |   8 ++++----
  Lib/distutils/archive_util.py     |   2 +-
  Lib/distutils/ccompiler.py        |   7 ++++---
  Lib/distutils/dist.py             |   9 +++++----
  Lib/distutils/msvccompiler.py     |   4 ++--
  Lib/distutils/util.py             |   2 +-
  Lib/encodings/__init__.py         |   7 ++++---
  Lib/ftplib.py                     |   2 +-
  Lib/functools.py                  |   6 +++---
  Lib/getopt.py                     |   2 +-
  Lib/getpass.py                    |   2 +-
  Lib/hashlib.py                    |   4 ++--
  Lib/heapq.py                      |   2 +-
  Lib/http/client.py                |   2 +-
  Lib/http/cookiejar.py             |   2 +-
  Lib/http/server.py                |   2 +-
  Lib/imaplib.py                    |   2 +-
  Lib/imp.py                        |   2 +-
  Lib/importlib/__init__.py         |   2 +-
  Lib/importlib/abc.py              |   2 +-
  Lib/inspect.py                    |   2 +-
  Lib/json/decoder.py               |   2 +-
  Lib/json/encoder.py               |   4 ++--
  Lib/json/scanner.py               |   2 +-
  Lib/lib2to3/refactor.py           |   2 +-
  Lib/locale.py                     |   2 +-
  Lib/logging/__init__.py           |   2 +-
  Lib/logging/config.py             |   2 +-
  Lib/logging/handlers.py           |   4 ++--
  Lib/macpath.py                    |   2 +-
  Lib/mailbox.py                    |   2 +-
  Lib/mimetypes.py                  |   2 +-
  Lib/multiprocessing/connection.py |   2 +-
  Lib/multiprocessing/forking.py    |   2 +-
  Lib/nntplib.py                    |   2 +-
  Lib/ntpath.py                     |   4 ++--
  Lib/operator.py                   |   2 +-
  Lib/optparse.py                   |   2 +-
  Lib/os.py                         |  14 +++++++-------
  Lib/pdb.py                        |   2 +-
  Lib/pickle.py                     |   4 ++--
  Lib/platform.py                   |  14 +++++++-------
  Lib/poplib.py                     |   2 +-
  Lib/pstats.py                     |   2 +-
  Lib/pty.py                        |   2 +-
  Lib/pydoc.py                      |   4 ++--
  Lib/queue.py                      |   4 ++--
  Lib/quopri.py                     |   2 +-
  Lib/reprlib.py                    |   2 +-
  Lib/rlcompleter.py                |   2 +-
  Lib/sched.py                      |   4 ++--
  Lib/shutil.py                     |  10 +++++-----
  Lib/site.py                       |   6 +++---
  Lib/smtpd.py                      |   2 +-
  Lib/smtplib.py                    |   2 +-
  Lib/socket.py                     |   2 +-
  Lib/socketserver.py               |   2 +-
  Lib/sqlite3/test/dbapi.py         |   2 +-
  Lib/sqlite3/test/types.py         |   2 +-
  Lib/sre_compile.py                |   2 +-
  Lib/ssl.py                        |   4 ++--
  Lib/subprocess.py                 |   2 +-
  Lib/tarfile.py                    |  12 ++++++------
  Lib/tempfile.py                   |   4 ++--
  Lib/threading.py                  |   6 +++---
  Lib/trace.py                      |   4 ++--
  Lib/urllib/request.py             |   6 +++---
  Lib/venv/__init__.py              |   2 +-
  Lib/warnings.py                   |   6 +++---
  Lib/xml/etree/ElementTree.py      |  10 +++++-----
  Lib/xml/sax/expatreader.py        |  17 ++++++++++++++---
  Lib/xmlrpc/client.py              |   2 +-
  83 files changed, 158 insertions(+), 144 deletions(-)


diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py
--- a/Lib/_dummy_thread.py
+++ b/Lib/_dummy_thread.py
@@ -7,7 +7,7 @@
 
     try:
         import _thread
-    except ModuleNotFoundError:
+    except ImportError:
         import _dummy_thread as _thread
 
 """
diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py
--- a/Lib/_osx_support.py
+++ b/Lib/_osx_support.py
@@ -62,7 +62,7 @@
     try:
         import tempfile
         fp = tempfile.NamedTemporaryFile()
-    except ModuleNotFoundError:
+    except ImportError:
         fp = open("/tmp/_osx_support.%s"%(
             os.getpid(),), "w+b")
 
diff --git a/Lib/_pyio.py b/Lib/_pyio.py
--- a/Lib/_pyio.py
+++ b/Lib/_pyio.py
@@ -9,7 +9,7 @@
 # Import _thread instead of threading to reduce startup cost
 try:
     from _thread import allocate_lock as Lock
-except ModuleNotFoundError:
+except ImportError:
     from _dummy_thread import allocate_lock as Lock
 
 import io
@@ -1486,7 +1486,7 @@
             if encoding is None:
                 try:
                     import locale
-                except ModuleNotFoundError:
+                except ImportError:
                     # Importing locale may fail if Python is being built
                     encoding = "ascii"
                 else:
diff --git a/Lib/_strptime.py b/Lib/_strptime.py
--- a/Lib/_strptime.py
+++ b/Lib/_strptime.py
@@ -21,7 +21,7 @@
                       timezone as datetime_timezone)
 try:
     from _thread import allocate_lock as _thread_allocate_lock
-except ModuleNotFoundError:
+except ImportError:
     from _dummy_thread import allocate_lock as _thread_allocate_lock
 
 __all__ = []
diff --git a/Lib/bisect.py b/Lib/bisect.py
--- a/Lib/bisect.py
+++ b/Lib/bisect.py
@@ -88,5 +88,5 @@
 # Overwrite above definitions with a fast C implementation
 try:
     from _bisect import *
-except ModuleNotFoundError:
+except ImportError:
     pass
diff --git a/Lib/bz2.py b/Lib/bz2.py
--- a/Lib/bz2.py
+++ b/Lib/bz2.py
@@ -14,7 +14,7 @@
 
 try:
     from threading import RLock
-except ModuleNotFoundError:
+except ImportError:
     from dummy_threading import RLock
 
 from _bz2 import BZ2Compressor, BZ2Decompressor
diff --git a/Lib/cmd.py b/Lib/cmd.py
--- a/Lib/cmd.py
+++ b/Lib/cmd.py
@@ -109,7 +109,7 @@
                 self.old_completer = readline.get_completer()
                 readline.set_completer(self.complete)
                 readline.parse_and_bind(self.completekey+": complete")
-            except ModuleNotFoundError:
+            except ImportError:
                 pass
         try:
             if intro is not None:
@@ -143,7 +143,7 @@
                 try:
                     import readline
                     readline.set_completer(self.old_completer)
-                except ModuleNotFoundError:
+                except ImportError:
                     pass
 
 
diff --git a/Lib/code.py b/Lib/code.py
--- a/Lib/code.py
+++ b/Lib/code.py
@@ -293,7 +293,7 @@
     else:
         try:
             import readline
-        except ModuleNotFoundError:
+        except ImportError:
             pass
     console.interact(banner)
 
diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py
--- a/Lib/collections/__init__.py
+++ b/Lib/collections/__init__.py
@@ -395,7 +395,7 @@
 
 try:                                    # Load C helper function if available
     from _collections import _count_elements
-except ModuleNotFoundError:
+except ImportError:
     pass
 
 class Counter(dict):
diff --git a/Lib/copy.py b/Lib/copy.py
--- a/Lib/copy.py
+++ b/Lib/copy.py
@@ -59,7 +59,7 @@
 
 try:
     from org.python.core import PyStringMap
-except ModuleNotFoundError:
+except ImportError:
     PyStringMap = None
 
 __all__ = ["Error", "copy", "deepcopy"]
diff --git a/Lib/datetime.py b/Lib/datetime.py
--- a/Lib/datetime.py
+++ b/Lib/datetime.py
@@ -2116,7 +2116,7 @@
 
 try:
     from _datetime import *
-except ModuleNotFoundError:
+except ImportError:
     pass
 else:
     # Clean up unused names
diff --git a/Lib/decimal.py b/Lib/decimal.py
--- a/Lib/decimal.py
+++ b/Lib/decimal.py
@@ -149,7 +149,7 @@
 try:
     from collections import namedtuple as _namedtuple
     DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent')
-except ModuleNotFoundError:
+except ImportError:
     DecimalTuple = lambda *args: args
 
 # Rounding
@@ -430,7 +430,7 @@
 
 try:
     import threading
-except ModuleNotFoundError:
+except ImportError:
     # Python was compiled without threads; create a mock object instead
     class MockThreading(object):
         def local(self, sys=sys):
@@ -6147,7 +6147,7 @@
 # don't care too much if locale isn't present.
 try:
     import locale as _locale
-except ModuleNotFoundError:
+except ImportError:
     pass
 
 def _parse_format_specifier(format_spec, _localeconv=None):
@@ -6391,7 +6391,7 @@
 
 try:
     import _decimal
-except ModuleNotFoundError:
+except ImportError:
     pass
 else:
     s1 = set(dir())
diff --git a/Lib/distutils/archive_util.py b/Lib/distutils/archive_util.py
--- a/Lib/distutils/archive_util.py
+++ b/Lib/distutils/archive_util.py
@@ -9,7 +9,7 @@
 
 try:
     import zipfile
-except ModuleNotFoundError:
+except ImportError:
     zipfile = None
 
 
diff --git a/Lib/distutils/ccompiler.py b/Lib/distutils/ccompiler.py
--- a/Lib/distutils/ccompiler.py
+++ b/Lib/distutils/ccompiler.py
@@ -3,7 +3,7 @@
 Contains CCompiler, an abstract base class that defines the interface
 for the Distutils compiler abstraction model."""
 
-import importlib, sys, os, re
+import sys, os, re
 from distutils.errors import *
 from distutils.spawn import spawn
 from distutils.file_util import move_file
@@ -1013,9 +1013,10 @@
 
     try:
         module_name = "distutils." + module_name
-        module = importlib.import_module(module_name)
+        __import__ (module_name)
+        module = sys.modules[module_name]
         klass = vars(module)[class_name]
-    except ModuleNotFoundError:
+    except ImportError:
         raise DistutilsModuleError(
               "can't compile C/C++ code: unable to load module '%s'" % \
               module_name)
diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py
--- a/Lib/distutils/dist.py
+++ b/Lib/distutils/dist.py
@@ -4,11 +4,11 @@
 being built/installed/distributed.
 """
 
-import importlib, sys, os, re
+import sys, os, re
 
 try:
     import warnings
-except ModuleNotFoundError:
+except ImportError:
     warnings = None
 
 from distutils.errors import *
@@ -788,8 +788,9 @@
             klass_name = command
 
             try:
-                module = importlib.import_module(module_name)
-            except ModuleNotFoundError:
+                __import__ (module_name)
+                module = sys.modules[module_name]
+            except ImportError:
                 continue
 
             try:
diff --git a/Lib/distutils/msvccompiler.py b/Lib/distutils/msvccompiler.py
--- a/Lib/distutils/msvccompiler.py
+++ b/Lib/distutils/msvccompiler.py
@@ -28,7 +28,7 @@
     RegEnumValue = winreg.EnumValue
     RegError = winreg.error
 
-except ModuleNotFoundError:
+except ImportError:
     try:
         import win32api
         import win32con
@@ -39,7 +39,7 @@
         RegEnumKey = win32api.RegEnumKey
         RegEnumValue = win32api.RegEnumValue
         RegError = win32api.error
-    except ModuleNotFoundError:
+    except ImportError:
         log.info("Warning: Can't read registry to find the "
                  "necessary compiler setting\n"
                  "Make sure that Python modules winreg, "
diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py
--- a/Lib/distutils/util.py
+++ b/Lib/distutils/util.py
@@ -388,7 +388,7 @@
         try:
             from tempfile import mkstemp
             (script_fd, script_name) = mkstemp(".py")
-        except ModuleNotFoundError:
+        except ImportError:
             from tempfile import mktemp
             (script_fd, script_name) = None, mktemp(".py")
         log.info("writing byte-compilation script '%s'", script_name)
diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py
--- a/Lib/encodings/__init__.py
+++ b/Lib/encodings/__init__.py
@@ -29,11 +29,11 @@
 """#"
 
 import codecs
-import importlib
 from . import aliases
 
 _cache = {}
 _unknown = '--unknown--'
+_import_tail = ['*']
 _aliases = aliases.aliases
 
 class CodecRegistryError(LookupError, SystemError):
@@ -94,8 +94,9 @@
         try:
             # Import is absolute to prevent the possibly malicious import of a
             # module with side-effects that is not in the 'encodings' package.
-            mod = importlib.import_module('encodings.' + modname)
-        except ModuleNotFoundError:
+            mod = __import__('encodings.' + modname, fromlist=_import_tail,
+                             level=0)
+        except ImportError:
             pass
         else:
             break
diff --git a/Lib/ftplib.py b/Lib/ftplib.py
--- a/Lib/ftplib.py
+++ b/Lib/ftplib.py
@@ -667,7 +667,7 @@
 
 try:
     import ssl
-except ModuleNotFoundError:
+except ImportError:
     _SSLSocket = None
 else:
     _SSLSocket = ssl.SSLSocket
diff --git a/Lib/functools.py b/Lib/functools.py
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -15,7 +15,7 @@
 
 try:
     from _functools import reduce
-except ModuleNotFoundError:
+except ImportError:
     pass
 from abc import get_cache_token
 from collections import namedtuple
@@ -143,7 +143,7 @@
 
 try:
     from _functools import cmp_to_key
-except ModuleNotFoundError:
+except ImportError:
     pass
 
 
@@ -166,7 +166,7 @@
 
 try:
     from _functools import partial
-except ModuleNotFoundError:
+except ImportError:
     pass
 
 
diff --git a/Lib/getopt.py b/Lib/getopt.py
--- a/Lib/getopt.py
+++ b/Lib/getopt.py
@@ -36,7 +36,7 @@
 import os
 try:
     from gettext import gettext as _
-except ModuleNotFoundError:
+except ImportError:
     # Bootstrapping Python: gettext's dependencies not built yet
     def _(s): return s
 
diff --git a/Lib/getpass.py b/Lib/getpass.py
--- a/Lib/getpass.py
+++ b/Lib/getpass.py
@@ -164,7 +164,7 @@
 except (ImportError, AttributeError):
     try:
         import msvcrt
-    except ModuleNotFoundError:
+    except ImportError:
         getpass = fallback_getpass
     else:
         getpass = win_getpass
diff --git a/Lib/hashlib.py b/Lib/hashlib.py
--- a/Lib/hashlib.py
+++ b/Lib/hashlib.py
@@ -98,7 +98,7 @@
                 return _sha3.sha3_384
             elif bs == '512':
                 return _sha3.sha3_512
-    except ModuleNotFoundError:
+    except ImportError:
         pass  # no extension module, this hash is unsupported.
 
     raise ValueError('unsupported hash type ' + name)
@@ -143,7 +143,7 @@
     __get_hash = __get_openssl_constructor
     algorithms_available = algorithms_available.union(
             _hashlib.openssl_md_meth_names)
-except ModuleNotFoundError:
+except ImportError:
     new = __py_new
     __get_hash = __get_builtin_constructor
 
diff --git a/Lib/heapq.py b/Lib/heapq.py
--- a/Lib/heapq.py
+++ b/Lib/heapq.py
@@ -343,7 +343,7 @@
 # If available, use C implementation
 try:
     from _heapq import *
-except ModuleNotFoundError:
+except ImportError:
     pass
 
 def merge(*iterables):
diff --git a/Lib/http/client.py b/Lib/http/client.py
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -1156,7 +1156,7 @@
 
 try:
     import ssl
-except ModuleNotFoundError:
+except ImportError:
     pass
 else:
     class HTTPSConnection(HTTPConnection):
diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py
--- a/Lib/http/cookiejar.py
+++ b/Lib/http/cookiejar.py
@@ -35,7 +35,7 @@
 import urllib.parse, urllib.request
 try:
     import threading as _threading
-except ModuleNotFoundError:
+except ImportError:
     import dummy_threading as _threading
 import http.client  # only for the default HTTP port
 from calendar import timegm
diff --git a/Lib/http/server.py b/Lib/http/server.py
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -904,7 +904,7 @@
         return nobody
     try:
         import pwd
-    except ModuleNotFoundError:
+    except ImportError:
         return -1
     try:
         nobody = pwd.getpwnam('nobody')[2]
diff --git a/Lib/imaplib.py b/Lib/imaplib.py
--- a/Lib/imaplib.py
+++ b/Lib/imaplib.py
@@ -29,7 +29,7 @@
 try:
     import ssl
     HAVE_SSL = True
-except ModuleNotFoundError:
+except ImportError:
     HAVE_SSL = False
 
 __all__ = ["IMAP4", "IMAP4_stream", "Internaldate2tuple",
diff --git a/Lib/imp.py b/Lib/imp.py
--- a/Lib/imp.py
+++ b/Lib/imp.py
@@ -12,7 +12,7 @@
                   _fix_co_filename)
 try:
     from _imp import load_dynamic
-except ModuleNotFoundError:
+except ImportError:
     # Platform doesn't support dynamic loading.
     load_dynamic = None
 
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py
--- a/Lib/importlib/__init__.py
+++ b/Lib/importlib/__init__.py
@@ -15,7 +15,7 @@
 
 try:
     import _frozen_importlib as _bootstrap
-except ModuleNotFoundError:
+except ImportError:
     from . import _bootstrap
     _bootstrap._setup(sys, _imp)
 else:
diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py
--- a/Lib/importlib/abc.py
+++ b/Lib/importlib/abc.py
@@ -3,7 +3,7 @@
 from . import machinery
 try:
     import _frozen_importlib
-except ModuleNotFoundError as exc:
+except ImportError as exc:
     if exc.name != '_frozen_importlib':
         raise
     _frozen_importlib = None
diff --git a/Lib/inspect.py b/Lib/inspect.py
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -50,7 +50,7 @@
 # back to hardcording so the dependency is optional
 try:
     from dis import COMPILER_FLAG_NAMES as _flag_names
-except ModuleNotFoundError:
+except ImportError:
     CO_OPTIMIZED, CO_NEWLOCALS = 0x1, 0x2
     CO_VARARGS, CO_VARKEYWORDS = 0x4, 0x8
     CO_NESTED, CO_GENERATOR, CO_NOFREE = 0x10, 0x20, 0x40
diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py
--- a/Lib/json/decoder.py
+++ b/Lib/json/decoder.py
@@ -5,7 +5,7 @@
 from json import scanner
 try:
     from _json import scanstring as c_scanstring
-except ModuleNotFoundError:
+except ImportError:
     c_scanstring = None
 
 __all__ = ['JSONDecoder']
diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py
--- a/Lib/json/encoder.py
+++ b/Lib/json/encoder.py
@@ -4,11 +4,11 @@
 
 try:
     from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ModuleNotFoundError:
+except ImportError:
     c_encode_basestring_ascii = None
 try:
     from _json import make_encoder as c_make_encoder
-except ModuleNotFoundError:
+except ImportError:
     c_make_encoder = None
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
diff --git a/Lib/json/scanner.py b/Lib/json/scanner.py
--- a/Lib/json/scanner.py
+++ b/Lib/json/scanner.py
@@ -3,7 +3,7 @@
 import re
 try:
     from _json import make_scanner as c_make_scanner
-except ModuleNotFoundError:
+except ImportError:
     c_make_scanner = None
 
 __all__ = ['make_scanner']
diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py
--- a/Lib/lib2to3/refactor.py
+++ b/Lib/lib2to3/refactor.py
@@ -706,7 +706,7 @@
                 items, write, doctests_only)
         try:
             import multiprocessing
-        except ModuleNotFoundError:
+        except ImportError:
             raise MultiprocessingUnsupported
         if self.queue is not None:
             raise RuntimeError("already doing multiple processes")
diff --git a/Lib/locale.py b/Lib/locale.py
--- a/Lib/locale.py
+++ b/Lib/locale.py
@@ -47,7 +47,7 @@
 
     from _locale import *
 
-except ModuleNotFoundError:
+except ImportError:
 
     # Locale emulation
 
diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py
--- a/Lib/logging/__init__.py
+++ b/Lib/logging/__init__.py
@@ -37,7 +37,7 @@
 
 try:
     import threading
-except ModuleNotFoundError: #pragma: no cover
+except ImportError: #pragma: no cover
     threading = None
 
 __author__  = "Vinay Sajip <vinay_sajip at red-dove.com>"
diff --git a/Lib/logging/config.py b/Lib/logging/config.py
--- a/Lib/logging/config.py
+++ b/Lib/logging/config.py
@@ -30,7 +30,7 @@
 try:
     import _thread as thread
     import threading
-except ModuleNotFoundError: #pragma: no cover
+except ImportError: #pragma: no cover
     thread = None
 
 from socketserver import ThreadingTCPServer, StreamRequestHandler
diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py
--- a/Lib/logging/handlers.py
+++ b/Lib/logging/handlers.py
@@ -29,7 +29,7 @@
 import queue
 try:
     import threading
-except ModuleNotFoundError: #pragma: no cover
+except ImportError: #pragma: no cover
     threading = None
 
 #
@@ -995,7 +995,7 @@
                 logging.ERROR   : win32evtlog.EVENTLOG_ERROR_TYPE,
                 logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE,
          }
-        except ModuleNotFoundError:
+        except ImportError:
             print("The Python Win32 extensions for NT (service, event "\
                         "logging) appear not to be available.")
             self._welu = None
diff --git a/Lib/macpath.py b/Lib/macpath.py
--- a/Lib/macpath.py
+++ b/Lib/macpath.py
@@ -187,7 +187,7 @@
     path = abspath(path)
     try:
         import Carbon.File
-    except ModuleNotFoundError:
+    except ImportError:
         return path
     if not path:
         return path
diff --git a/Lib/mailbox.py b/Lib/mailbox.py
--- a/Lib/mailbox.py
+++ b/Lib/mailbox.py
@@ -23,7 +23,7 @@
 import contextlib
 try:
     import fcntl
-except ModuleNotFoundError:
+except ImportError:
     fcntl = None
 
 __all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py
--- a/Lib/mimetypes.py
+++ b/Lib/mimetypes.py
@@ -29,7 +29,7 @@
 import urllib.parse
 try:
     import winreg as _winreg
-except ModuleNotFoundError:
+except ImportError:
     _winreg = None
 
 __all__ = [
diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -27,7 +27,7 @@
 try:
     import _winapi
     from _winapi import WAIT_OBJECT_0, WAIT_TIMEOUT, INFINITE
-except ModuleNotFoundError:
+except ImportError:
     if sys.platform == 'win32':
         raise
     _winapi = None
diff --git a/Lib/multiprocessing/forking.py b/Lib/multiprocessing/forking.py
--- a/Lib/multiprocessing/forking.py
+++ b/Lib/multiprocessing/forking.py
@@ -73,7 +73,7 @@
 
 try:
     from functools import partial
-except ModuleNotFoundError:
+except ImportError:
     pass
 else:
     def _reduce_partial(p):
diff --git a/Lib/nntplib.py b/Lib/nntplib.py
--- a/Lib/nntplib.py
+++ b/Lib/nntplib.py
@@ -71,7 +71,7 @@
 
 try:
     import ssl
-except ModuleNotFoundError:
+except ImportError:
     _have_ssl = False
 else:
     _have_ssl = True
diff --git a/Lib/ntpath.py b/Lib/ntpath.py
--- a/Lib/ntpath.py
+++ b/Lib/ntpath.py
@@ -566,7 +566,7 @@
 try:
     from nt import _getfullpathname
 
-except ModuleNotFoundError: # not running on Windows - mock up something sensible
+except ImportError: # not running on Windows - mock up something sensible
     def abspath(path):
         """Return the absolute version of a path."""
         if not isabs(path):
@@ -659,6 +659,6 @@
     # This is overkill on Windows - just pass the path to GetFileAttributes
     # and check the attribute from there.
     from nt import _isdir as isdir
-except ModuleNotFoundError:
+except ImportError:
     # Use genericpath.isdir as imported above.
     pass
diff --git a/Lib/operator.py b/Lib/operator.py
--- a/Lib/operator.py
+++ b/Lib/operator.py
@@ -360,7 +360,7 @@
 
 try:
     from _operator import *
-except ModuleNotFoundError:
+except ImportError:
     pass
 else:
     from _operator import __doc__
diff --git a/Lib/optparse.py b/Lib/optparse.py
--- a/Lib/optparse.py
+++ b/Lib/optparse.py
@@ -87,7 +87,7 @@
 
 try:
     from gettext import gettext, ngettext
-except ModuleNotFoundError:
+except ImportError:
     def gettext(message):
         return message
 
diff --git a/Lib/os.py b/Lib/os.py
--- a/Lib/os.py
+++ b/Lib/os.py
@@ -52,13 +52,13 @@
     try:
         from posix import _exit
         __all__.append('_exit')
-    except ModuleNotFoundError:
+    except ImportError:
         pass
     import posixpath as path
 
     try:
         from posix import _have_functions
-    except ModuleNotFoundError:
+    except ImportError:
         pass
 
 elif 'nt' in _names:
@@ -68,7 +68,7 @@
     try:
         from nt import _exit
         __all__.append('_exit')
-    except ModuleNotFoundError:
+    except ImportError:
         pass
     import ntpath as path
 
@@ -78,7 +78,7 @@
 
     try:
         from nt import _have_functions
-    except ModuleNotFoundError:
+    except ImportError:
         pass
 
 elif 'ce' in _names:
@@ -88,7 +88,7 @@
     try:
         from ce import _exit
         __all__.append('_exit')
-    except ModuleNotFoundError:
+    except ImportError:
         pass
     # We can use the standard Windows path.
     import ntpath as path
@@ -99,11 +99,11 @@
 
     try:
         from ce import _have_functions
-    except ModuleNotFoundError:
+    except ImportError:
         pass
 
 else:
-    raise ModuleNotFoundError('no os specific module found')
+    raise ImportError('no os specific module found')
 
 sys.modules['os.path'] = path
 from os.path import (curdir, pardir, sep, pathsep, defpath, extsep, altsep,
diff --git a/Lib/pdb.py b/Lib/pdb.py
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -158,7 +158,7 @@
             import readline
             # remove some common file name delimiters
             readline.set_completer_delims(' \t\n`@#$%^&*()=+[{]}\\|;:\'",<>?')
-        except ModuleNotFoundError:
+        except ImportError:
             pass
         self.allow_kbdint = False
         self.nosigint = nosigint
diff --git a/Lib/pickle.py b/Lib/pickle.py
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -90,7 +90,7 @@
 # Jython has PyStringMap; it's a dict subclass with string keys
 try:
     from org.python.core import PyStringMap
-except ModuleNotFoundError:
+except ImportError:
     PyStringMap = None
 
 # Pickle opcodes.  See pickletools.py for extensive docs.  The listing
@@ -1296,7 +1296,7 @@
 # Use the faster _pickle if possible
 try:
     from _pickle import *
-except ModuleNotFoundError:
+except ImportError:
     Pickler, Unpickler = _Pickler, _Unpickler
 
 # Doctest
diff --git a/Lib/platform.py b/Lib/platform.py
--- a/Lib/platform.py
+++ b/Lib/platform.py
@@ -460,7 +460,7 @@
     try:
         # Use win32api if available
         from win32api import RegQueryValueEx
-    except ModuleNotFoundError:
+    except ImportError:
         # On Python 2.0 and later, emulate using winreg
         import winreg
         RegQueryValueEx = winreg.QueryValueEx
@@ -503,7 +503,7 @@
              RegCloseKey, GetVersionEx
         from win32con import HKEY_LOCAL_MACHINE, VER_PLATFORM_WIN32_NT, \
              VER_PLATFORM_WIN32_WINDOWS, VER_NT_WORKSTATION
-    except ModuleNotFoundError:
+    except ImportError:
         # Emulate the win32api module using Python APIs
         try:
             sys.getwindowsversion
@@ -661,7 +661,7 @@
     # Check whether the version info module is available
     try:
         import _gestalt
-    except ModuleNotFoundError:
+    except ImportError:
         return None
     # Get the infos
     sysv, sysa = _mac_ver_lookup(('sysv','sysa'))
@@ -697,7 +697,7 @@
 
     try:
         import plistlib
-    except ModuleNotFoundError:
+    except ImportError:
         return None
 
     pl = plistlib.readPlist(fn)
@@ -762,7 +762,7 @@
     # Import the needed APIs
     try:
         import java.lang
-    except ModuleNotFoundError:
+    except ImportError:
         return release,vendor,vminfo,osinfo
 
     vendor = _java_getprop('java.vendor', vendor)
@@ -874,7 +874,7 @@
     """
     try:
         import socket
-    except ModuleNotFoundError:
+    except ImportError:
         # No sockets...
         return default
     try:
@@ -1138,7 +1138,7 @@
         # Get processor information
         try:
             import vms_lib
-        except ModuleNotFoundError:
+        except ImportError:
             pass
         else:
             csid, cpu_number = vms_lib.getsyi('SYI$_CPU',0)
diff --git a/Lib/poplib.py b/Lib/poplib.py
--- a/Lib/poplib.py
+++ b/Lib/poplib.py
@@ -20,7 +20,7 @@
 try:
     import ssl
     HAVE_SSL = True
-except ModuleNotFoundError:
+except ImportError:
     HAVE_SSL = False
 
 __all__ = ["POP3","error_proto"]
diff --git a/Lib/pstats.py b/Lib/pstats.py
--- a/Lib/pstats.py
+++ b/Lib/pstats.py
@@ -528,7 +528,7 @@
     import cmd
     try:
         import readline
-    except ModuleNotFoundError:
+    except ImportError:
         pass
 
     class ProfileBrowser(cmd.Cmd):
diff --git a/Lib/pty.py b/Lib/pty.py
--- a/Lib/pty.py
+++ b/Lib/pty.py
@@ -67,7 +67,7 @@
     result = os.open(tty_name, os.O_RDWR)
     try:
         from fcntl import ioctl, I_PUSH
-    except ModuleNotFoundError:
+    except ImportError:
         return result
     try:
         ioctl(result, I_PUSH, "ptem")
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -1892,7 +1892,7 @@
     def showtopic(self, topic, more_xrefs=''):
         try:
             import pydoc_data.topics
-        except ModuleNotFoundError:
+        except ImportError:
             self.output.write('''
 Sorry, topic and keyword documentation is not available because the
 module "pydoc_data.topics" could not be found.
@@ -1932,7 +1932,7 @@
         """
         try:
             import pydoc_data.topics
-        except ModuleNotFoundError:
+        except ImportError:
             return('''
 Sorry, topic and keyword documentation is not available because the
 module "pydoc_data.topics" could not be found.
diff --git a/Lib/queue.py b/Lib/queue.py
--- a/Lib/queue.py
+++ b/Lib/queue.py
@@ -2,13 +2,13 @@
 
 try:
     import threading
-except ModuleNotFoundError:
+except ImportError:
     import dummy_threading as threading
 from collections import deque
 from heapq import heappush, heappop
 try:
     from time import monotonic as time
-except ModuleNotFoundError:
+except ImportError:
     from time import time
 
 __all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue']
diff --git a/Lib/quopri.py b/Lib/quopri.py
--- a/Lib/quopri.py
+++ b/Lib/quopri.py
@@ -13,7 +13,7 @@
 
 try:
     from binascii import a2b_qp, b2a_qp
-except ModuleNotFoundError:
+except ImportError:
     a2b_qp = None
     b2a_qp = None
 
diff --git a/Lib/reprlib.py b/Lib/reprlib.py
--- a/Lib/reprlib.py
+++ b/Lib/reprlib.py
@@ -6,7 +6,7 @@
 from itertools import islice
 try:
     from _thread import get_ident
-except ModuleNotFoundError:
+except ImportError:
     from _dummy_thread import get_ident
 
 def recursive_repr(fillvalue='...'):
diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py
--- a/Lib/rlcompleter.py
+++ b/Lib/rlcompleter.py
@@ -154,7 +154,7 @@
 
 try:
     import readline
-except ModuleNotFoundError:
+except ImportError:
     pass
 else:
     readline.set_completer(Completer().complete)
diff --git a/Lib/sched.py b/Lib/sched.py
--- a/Lib/sched.py
+++ b/Lib/sched.py
@@ -33,11 +33,11 @@
 from collections import namedtuple
 try:
     import threading
-except ModuleNotFoundError:
+except ImportError:
     import dummy_threading as threading
 try:
     from time import monotonic as _time
-except ModuleNotFoundError:
+except ImportError:
     from time import time as _time
 
 __all__ = ["scheduler"]
diff --git a/Lib/shutil.py b/Lib/shutil.py
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -17,17 +17,17 @@
     import bz2
     del bz2
     _BZ2_SUPPORTED = True
-except ModuleNotFoundError:
+except ImportError:
     _BZ2_SUPPORTED = False
 
 try:
     from pwd import getpwnam
-except ModuleNotFoundError:
+except ImportError:
     getpwnam = None
 
 try:
     from grp import getgrnam
-except ModuleNotFoundError:
+except ImportError:
     getgrnam = None
 
 __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
@@ -668,7 +668,7 @@
     # command.
     try:
         import zipfile
-    except ModuleNotFoundError:
+    except ImportError:
         zipfile = None
 
     if zipfile is None:
@@ -858,7 +858,7 @@
     """
     try:
         import zipfile
-    except ModuleNotFoundError:
+    except ImportError:
         raise ReadError('zlib not supported, cannot unpack this archive.')
 
     if not zipfile.is_zipfile(filename):
diff --git a/Lib/site.py b/Lib/site.py
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -469,7 +469,7 @@
         try:
             import readline
             import rlcompleter
-        except ModuleNotFoundError:
+        except ImportError:
             return
 
         # Reading the initialization (config) file may not be enough to set a
@@ -570,7 +570,7 @@
     """Run custom site specific code, if available."""
     try:
         import sitecustomize
-    except ModuleNotFoundError:
+    except ImportError:
         pass
     except Exception as err:
         if os.environ.get("PYTHONVERBOSE"):
@@ -586,7 +586,7 @@
     """Run custom user specific code, if available."""
     try:
         import usercustomize
-    except ModuleNotFoundError:
+    except ImportError:
         pass
     except Exception as err:
         if os.environ.get("PYTHONVERBOSE"):
diff --git a/Lib/smtpd.py b/Lib/smtpd.py
--- a/Lib/smtpd.py
+++ b/Lib/smtpd.py
@@ -846,7 +846,7 @@
     if options.setuid:
         try:
             import pwd
-        except ModuleNotFoundError:
+        except ImportError:
             print('Cannot import module "pwd"; try running with -n option.', file=sys.stderr)
             sys.exit(1)
         nobody = pwd.getpwnam('nobody')[2]
diff --git a/Lib/smtplib.py b/Lib/smtplib.py
--- a/Lib/smtplib.py
+++ b/Lib/smtplib.py
@@ -171,7 +171,7 @@
 
 try:
     import ssl
-except ModuleNotFoundError:
+except ImportError:
     _have_ssl = False
 else:
     _have_ssl = True
diff --git a/Lib/socket.py b/Lib/socket.py
--- a/Lib/socket.py
+++ b/Lib/socket.py
@@ -51,7 +51,7 @@
 
 try:
     import errno
-except ModuleNotFoundError:
+except ImportError:
     errno = None
 EBADF = getattr(errno, 'EBADF', 9)
 EAGAIN = getattr(errno, 'EAGAIN', 11)
diff --git a/Lib/socketserver.py b/Lib/socketserver.py
--- a/Lib/socketserver.py
+++ b/Lib/socketserver.py
@@ -136,7 +136,7 @@
 import errno
 try:
     import threading
-except ModuleNotFoundError:
+except ImportError:
     import dummy_threading as threading
 
 __all__ = ["TCPServer","UDPServer","ForkingUDPServer","ForkingTCPServer",
diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py
--- a/Lib/sqlite3/test/dbapi.py
+++ b/Lib/sqlite3/test/dbapi.py
@@ -25,7 +25,7 @@
 import sqlite3 as sqlite
 try:
     import threading
-except ModuleNotFoundError:
+except ImportError:
     threading = None
 
 from test.support import TESTFN, unlink
diff --git a/Lib/sqlite3/test/types.py b/Lib/sqlite3/test/types.py
--- a/Lib/sqlite3/test/types.py
+++ b/Lib/sqlite3/test/types.py
@@ -26,7 +26,7 @@
 import sqlite3 as sqlite
 try:
     import zlib
-except ModuleNotFoundError:
+except ImportError:
     zlib = None
 
 
diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py
--- a/Lib/sre_compile.py
+++ b/Lib/sre_compile.py
@@ -295,7 +295,7 @@
 def _optimize_unicode(charset, fixup):
     try:
         import array
-    except ModuleNotFoundError:
+    except ImportError:
         return charset
     charmap = [0]*65536
     negate = 0
diff --git a/Lib/ssl.py b/Lib/ssl.py
--- a/Lib/ssl.py
+++ b/Lib/ssl.py
@@ -127,14 +127,14 @@
 try:
     from _ssl import PROTOCOL_SSLv2
     _SSLv2_IF_EXISTS = PROTOCOL_SSLv2
-except ModuleNotFoundError:
+except ImportError:
     _SSLv2_IF_EXISTS = None
 else:
     _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
 
 try:
     from _ssl import PROTOCOL_TLSv1_1, PROTOCOL_TLSv1_2
-except ModuleNotFoundError:
+except ImportError:
     pass
 else:
     _PROTOCOL_NAMES[PROTOCOL_TLSv1_1] = "TLSv1.1"
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -353,7 +353,7 @@
 import errno
 try:
     from time import monotonic as _time
-except ModuleNotFoundError:
+except ImportError:
     from time import time as _time
 
 # Exception classes used by this module.
diff --git a/Lib/tarfile.py b/Lib/tarfile.py
--- a/Lib/tarfile.py
+++ b/Lib/tarfile.py
@@ -50,7 +50,7 @@
 
 try:
     import grp, pwd
-except ModuleNotFoundError:
+except ImportError:
     grp = pwd = None
 
 # os.symlink on Windows prior to 6.0 raises NotImplementedError
@@ -381,7 +381,7 @@
             if comptype == "gz":
                 try:
                     import zlib
-                except ModuleNotFoundError:
+                except ImportError:
                     raise CompressionError("zlib module is not available")
                 self.zlib = zlib
                 self.crc = zlib.crc32(b"")
@@ -394,7 +394,7 @@
             elif comptype == "bz2":
                 try:
                     import bz2
-                except ModuleNotFoundError:
+                except ImportError:
                     raise CompressionError("bz2 module is not available")
                 if mode == "r":
                     self.dbuf = b""
@@ -406,7 +406,7 @@
             elif comptype == "xz":
                 try:
                     import lzma
-                except ModuleNotFoundError:
+                except ImportError:
                     raise CompressionError("lzma module is not available")
                 if mode == "r":
                     self.dbuf = b""
@@ -1654,7 +1654,7 @@
 
         try:
             import bz2
-        except ModuleNotFoundError:
+        except ImportError:
             raise CompressionError("bz2 module is not available")
 
         fileobj = bz2.BZ2File(fileobj or name, mode,
@@ -1678,7 +1678,7 @@
 
         try:
             import lzma
-        except ModuleNotFoundError:
+        except ImportError:
             raise CompressionError("lzma module is not available")
 
         fileobj = lzma.LZMAFile(fileobj or name, mode, preset=preset)
diff --git a/Lib/tempfile.py b/Lib/tempfile.py
--- a/Lib/tempfile.py
+++ b/Lib/tempfile.py
@@ -36,7 +36,7 @@
 
 try:
     import fcntl as _fcntl
-except ModuleNotFoundError:
+except ImportError:
     def _set_cloexec(fd):
         pass
 else:
@@ -53,7 +53,7 @@
 
 try:
     import _thread
-except ModuleNotFoundError:
+except ImportError:
     import _dummy_thread as _thread
 _allocate_lock = _thread.allocate_lock
 
diff --git a/Lib/threading.py b/Lib/threading.py
--- a/Lib/threading.py
+++ b/Lib/threading.py
@@ -6,14 +6,14 @@
 from time import sleep as _sleep
 try:
     from time import monotonic as _time
-except ModuleNotFoundError:
+except ImportError:
     from time import time as _time
 from traceback import format_exc as _format_exc
 from _weakrefset import WeakSet
 from itertools import islice as _islice
 try:
     from _collections import deque as _deque
-except ModuleNotFoundError:
+except ImportError:
     from collections import deque as _deque
 
 # Note regarding PEP 8 compliant names
@@ -922,7 +922,7 @@
 
 try:
     from _thread import _local as local
-except ModuleNotFoundError:
+except ImportError:
     from _threading_local import local
 
 
diff --git a/Lib/trace.py b/Lib/trace.py
--- a/Lib/trace.py
+++ b/Lib/trace.py
@@ -61,12 +61,12 @@
 from warnings import warn as _warn
 try:
     from time import monotonic as _time
-except ModuleNotFoundError:
+except ImportError:
     from time import time as _time
 
 try:
     import threading
-except ModuleNotFoundError:
+except ImportError:
     _settrace = sys.settrace
 
     def _unsettrace():
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
--- a/Lib/urllib/request.py
+++ b/Lib/urllib/request.py
@@ -110,7 +110,7 @@
 # check for SSL
 try:
     import ssl
-except ModuleNotFoundError:
+except ImportError:
     _have_ssl = False
 else:
     _have_ssl = True
@@ -2512,7 +2512,7 @@
         proxies = {}
         try:
             import winreg
-        except ModuleNotFoundError:
+        except ImportError:
             # Std module, so should be around - but you never know!
             return proxies
         try:
@@ -2560,7 +2560,7 @@
     def proxy_bypass_registry(host):
         try:
             import winreg
-        except ModuleNotFoundError:
+        except ImportError:
             # Std modules, so should be around - but you never know!
             return 0
         try:
diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py
--- a/Lib/venv/__init__.py
+++ b/Lib/venv/__init__.py
@@ -35,7 +35,7 @@
 import sysconfig
 try:
     import threading
-except ModuleNotFoundError:
+except ImportError:
     threading = None
 
 logger = logging.getLogger(__name__)
diff --git a/Lib/warnings.py b/Lib/warnings.py
--- a/Lib/warnings.py
+++ b/Lib/warnings.py
@@ -144,8 +144,8 @@
         module = category[:i]
         klass = category[i+1:]
         try:
-            m = __import__(module, fromlist[klass])
-        except ModuleNotFoundError:
+            m = __import__(module, None, None, [klass])
+        except ImportError:
             raise _OptionError("invalid module name: %r" % (module,))
         try:
             cat = getattr(m, klass)
@@ -362,7 +362,7 @@
     defaultaction = _defaultaction
     onceregistry = _onceregistry
     _warnings_defaults = True
-except ModuleNotFoundError:
+except ImportError:
     filters = []
     defaultaction = "default"
     onceregistry = {}
diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py
--- a/Lib/xml/etree/ElementTree.py
+++ b/Lib/xml/etree/ElementTree.py
@@ -1439,13 +1439,13 @@
     def __init__(self, html=0, target=None, encoding=None):
         try:
             from xml.parsers import expat
-        except ModuleNotFoundError:
+        except ImportError:
             try:
                 import pyexpat as expat
-            except ModuleNotFoundError:
-                raise ModuleNotFoundError(
-                    "No module named expat; use SimpleXMLTreeBuilder instead",
-                    name='expat')
+            except ImportError:
+                raise ImportError(
+                    "No module named expat; use SimpleXMLTreeBuilder instead"
+                    )
         parser = expat.ParserCreate(encoding, "}")
         if target is None:
             target = TreeBuilder()
diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py
--- a/Lib/xml/sax/expatreader.py
+++ b/Lib/xml/sax/expatreader.py
@@ -20,7 +20,7 @@
 
 try:
     from xml.parsers import expat
-except ModuleNotFoundError:
+except ImportError:
     raise SAXReaderNotAvailable("expat not supported", None)
 else:
     if not hasattr(expat, "ParserCreate"):
@@ -30,7 +30,18 @@
 AttributesImpl = xmlreader.AttributesImpl
 AttributesNSImpl = xmlreader.AttributesNSImpl
 
-import weakref
+# If we're using a sufficiently recent version of Python, we can use
+# weak references to avoid cycles between the parser and content
+# handler, otherwise we'll just have to pretend.
+try:
+    import _weakref
+except ImportError:
+    def _mkproxy(o):
+        return o
+else:
+    import weakref
+    _mkproxy = weakref.proxy
+    del weakref, _weakref
 
 # --- ExpatLocator
 
@@ -41,7 +52,7 @@
     a circular reference between the parser and the content handler.
     """
     def __init__(self, parser):
-        self._ref = weakref.proxy(parser)
+        self._ref = _mkproxy(parser)
 
     def getColumnNumber(self):
         parser = self._ref
diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py
--- a/Lib/xmlrpc/client.py
+++ b/Lib/xmlrpc/client.py
@@ -139,7 +139,7 @@
 from io import BytesIO
 try:
     import gzip
-except ModuleNotFoundError:
+except ImportError:
     gzip = None #python can be built without zlib/gzip support
 
 # --------------------------------------------------------------------

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul  5 00:16:33 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri,  5 Jul 2013 00:16:33 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2315767=3A_Back_out?=
 =?utf-8?q?_8d28d44f3a9a_related_to_ModuleNotFoundError=2E?=
Message-ID: <3bmYSK47S3z7Lqw@mail.python.org>

http://hg.python.org/cpython/rev/0e4e062751fa
changeset:   84443:0e4e062751fa
user:        Brett Cannon <brett at python.org>
date:        Thu Jul 04 17:44:08 2013 -0400
summary:
  Issue #15767: Back out 8d28d44f3a9a related to ModuleNotFoundError.

files:
  Lib/test/test_importlib/import_/test_api.py |  7 -------
  1 files changed, 0 insertions(+), 7 deletions(-)


diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py
--- a/Lib/test/test_importlib/import_/test_api.py
+++ b/Lib/test/test_importlib/import_/test_api.py
@@ -26,13 +26,6 @@
         with self.assertRaises(ModuleNotFoundError):
             util.import_('some module that does not exist')
 
-    def test_raises_ModuleNotFoundError_for_None(self):
-        # None in sys.modules should raise ModuleNotFoundError.
-        with importlib_test_util.uncache('not_here'):
-            sys.modules['not_here'] = None
-            with self.assertRaises(ModuleNotFoundError):
-                util.import_('not_here')
-
     def test_name_requires_rparition(self):
         # Raise TypeError if a non-string is passed in for the module name.
         with self.assertRaises(TypeError):

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul  5 00:16:35 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri,  5 Jul 2013 00:16:35 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2315767=3A_Revert_3?=
 =?utf-8?q?a50025f1900_for_ModuleNotFoundError?=
Message-ID: <3bmYSM0HhKz7Lqj@mail.python.org>

http://hg.python.org/cpython/rev/e3ec8b176a80
changeset:   84444:e3ec8b176a80
user:        Brett Cannon <brett at python.org>
date:        Thu Jul 04 17:48:16 2013 -0400
summary:
  Issue #15767: Revert 3a50025f1900 for ModuleNotFoundError

files:
  Doc/c-api/exceptions.rst    |     7 -
  Doc/library/exceptions.rst  |     3 +-
  Doc/reference/import.rst    |    12 +-
  Doc/whatsnew/3.4.rst        |     3 +-
  Include/pyerrors.h          |     3 -
  Lib/importlib/_bootstrap.py |     2 +-
  Misc/NEWS                   |     4 -
  Python/errors.c             |    25 +-
  Python/import.c             |     3 +-
  Python/importlib.h          |  1589 +++++++++++-----------
  10 files changed, 808 insertions(+), 843 deletions(-)


diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
--- a/Doc/c-api/exceptions.rst
+++ b/Doc/c-api/exceptions.rst
@@ -292,13 +292,6 @@
 
    .. versionadded:: 3.3
 
-.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *msg, PyObject *name, PyObject *path)
-
-   Much like :c:func:`PyErr_SetImportError` but this function allows for
-   specifying a subclass of :exc:`ImportError` to raise.
-
-   .. versionadded:: 3.4
-
 
 .. c:function:: void PyErr_SyntaxLocationEx(char *filename, int lineno, int col_offset)
 
diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst
--- a/Doc/library/exceptions.rst
+++ b/Doc/library/exceptions.rst
@@ -185,8 +185,7 @@
    A subclass of :exc:`ImportError` which is raised by :keyword:`import` when a
    module could not be located. This includes ``from ... import`` statements as
    the specific attribute being requested cannot be known a priori to be a module
-   or some other type of object. It is also raised when ``None`` is found in
-   :data:`sys.modules`.
+   or some other type of object.
 
    .. versionadded:: 3.4
 
diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst
--- a/Doc/reference/import.rst
+++ b/Doc/reference/import.rst
@@ -37,7 +37,7 @@
 
 When a module is first imported, Python searches for the module and if found,
 it creates a module object [#fnmo]_, initializing it.  If the named module
-cannot be found, an :exc:`ModuleNotFoundError` is raised.  Python implements various
+cannot be found, an :exc:`ImportError` is raised.  Python implements various
 strategies to search for the named module when the import machinery is
 invoked.  These strategies can be modified and extended by using various hooks
 described in the sections below.
@@ -168,7 +168,7 @@
 This name will be used in various phases of the import search, and it may be
 the dotted path to a submodule, e.g. ``foo.bar.baz``.  In this case, Python
 first tries to import ``foo``, then ``foo.bar``, and finally ``foo.bar.baz``.
-If any of the intermediate imports fail, an :exc:`ModuleNotFoundError` is raised.
+If any of the intermediate imports fail, an :exc:`ImportError` is raised.
 
 
 The module cache
@@ -187,7 +187,7 @@
 During import, the module name is looked up in :data:`sys.modules` and if
 present, the associated value is the module satisfying the import, and the
 process completes.  However, if the value is ``None``, then an
-:exc:`ModuleNotFoundError` is raised.  If the module name is missing, Python will
+:exc:`ImportError` is raised.  If the module name is missing, Python will
 continue searching for the module.
 
 :data:`sys.modules` is writable.  Deleting a key may not destroy the
@@ -195,7 +195,7 @@
 but it will invalidate the cache entry for the named module, causing
 Python to search anew for the named module upon its next
 import. The key can also be assigned to ``None``, forcing the next import
-of the module to result in an :exc:`ModuleNotFoundError`.
+of the module to result in an :exc:`ImportError`.
 
 Beware though, as if you keep a reference to the module object,
 invalidate its cache entry in :data:`sys.modules`, and then re-import the
@@ -284,7 +284,7 @@
 If the meta path finder knows how to handle the named module, it returns a
 loader object.  If it cannot handle the named module, it returns ``None``.  If
 :data:`sys.meta_path` processing reaches the end of its list without returning
-a loader, then an :exc:`ModuleNotFoundError` is raised.  Any other exceptions raised
+a loader, then an :exc:`ImportError` is raised.  Any other exceptions raised
 are simply propagated up, aborting the import process.
 
 The :meth:`find_module()` method of meta path finders is called with two
@@ -647,7 +647,7 @@
 
 To selectively prevent import of some modules from a hook early on the
 meta path (rather than disabling the standard import system entirely),
-it is sufficient to raise :exc:`ModuleNotFoundError` directly from
+it is sufficient to raise :exc:`ImportError` directly from
 :meth:`find_module` instead of returning ``None``. The latter indicates
 that the meta path search should continue. while raising an exception
 terminates it immediately.
diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst
--- a/Doc/whatsnew/3.4.rst
+++ b/Doc/whatsnew/3.4.rst
@@ -312,7 +312,8 @@
   using ``hasattr(module, '__path__')``.
 
 * :c:func:`PyErr_SetImportError` now sets :exc:`TypeError` when its **msg**
-  argument is not set. Previously only ``NULL`` was returned.
+  argument is not set. Previously only ``NULL`` was returned with no exception
+  set.
 
 * :func:`py_compile.compile` now raises :exc:`FileExistsError` if the file path
   it would write to is a symlink or a non-regular file. This is to act as a
diff --git a/Include/pyerrors.h b/Include/pyerrors.h
--- a/Include/pyerrors.h
+++ b/Include/pyerrors.h
@@ -268,9 +268,6 @@
 
 PyAPI_FUNC(PyObject *) PyErr_SetExcWithArgsKwargs(PyObject *, PyObject *,
     PyObject *);
-
-PyAPI_FUNC(PyObject *) PyErr_SetImportErrorSubclass(PyObject *, PyObject *,
-    PyObject *, PyObject *);
 PyAPI_FUNC(PyObject *) PyErr_SetImportError(PyObject *, PyObject *,
     PyObject *);
 
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -1617,7 +1617,7 @@
         _imp.release_lock()
         message = ("import of {} halted; "
                     "None in sys.modules".format(name))
-        raise ModuleNotFoundError(message, name=name)
+        raise ImportError(message, name=name)
     _lock_unlock_module(name)
     return module
 
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -539,10 +539,6 @@
 C-API
 -----
 
-- Issue #15767: Added PyErr_SetImportErrorSubclass().
-
-- PyErr_SetImportError() now sets TypeError when its msg argument is set.
-
 - Issue #9369: The types of `char*` arguments of PyObject_CallFunction() and
   PyObject_CallMethod() now changed to `const char*`.  Based on patches by
   J?rg M?ller and Lars Buitinck.
diff --git a/Python/errors.c b/Python/errors.c
--- a/Python/errors.c
+++ b/Python/errors.c
@@ -619,25 +619,12 @@
 #endif /* MS_WINDOWS */
 
 PyObject *
-PyErr_SetImportErrorSubclass(PyObject *exception, PyObject *msg,
-    PyObject *name, PyObject *path)
+PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path)
 {
-    int issubclass;
     PyObject *args, *kwargs, *error;
 
-    issubclass = PyObject_IsSubclass(exception, PyExc_ImportError);
-    if (issubclass < 0) {
+    if (msg == NULL)
         return NULL;
-    }
-    else if (!issubclass) {
-        PyErr_SetString(PyExc_TypeError, "expected a subclass of ImportError");
-        return NULL;
-    }
-
-    if (msg == NULL) {
-        PyErr_SetString(PyExc_TypeError, "expected a message argument");
-        return NULL;
-    }
 
     args = PyTuple_New(1);
     if (args == NULL)
@@ -662,7 +649,7 @@
     PyDict_SetItemString(kwargs, "name", name);
     PyDict_SetItemString(kwargs, "path", path);
 
-    error = PyObject_Call(exception, args, kwargs);
+    error = PyObject_Call(PyExc_ImportError, args, kwargs);
     if (error != NULL) {
         PyErr_SetObject((PyObject *)Py_TYPE(error), error);
         Py_DECREF(error);
@@ -674,12 +661,6 @@
     return NULL;
 }
 
-PyObject *
-PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path)
-{
-    return PyErr_SetImportErrorSubclass(PyExc_ImportError, msg, name, path);
-}
-
 void
 _PyErr_BadInternalCall(const char *filename, int lineno)
 {
diff --git a/Python/import.c b/Python/import.c
--- a/Python/import.c
+++ b/Python/import.c
@@ -1428,8 +1428,7 @@
         PyObject *msg = PyUnicode_FromFormat("import of %R halted; "
                                              "None in sys.modules", abs_name);
         if (msg != NULL) {
-            PyErr_SetImportErrorSubclass(PyExc_ModuleNotFoundError, msg,
-                    abs_name, NULL);
+            PyErr_SetImportError(msg, abs_name, NULL);
             Py_DECREF(msg);
         }
         mod = NULL;
diff --git a/Python/importlib.h b/Python/importlib.h
--- a/Python/importlib.h
+++ b/Python/importlib.h
[stripped]

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul  5 00:16:36 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri,  5 Jul 2013 00:16:36 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2315767=3A_back_out?=
 =?utf-8?q?_8a0ed9f63c6e=2C_finishing_the_removal_of?=
Message-ID: <3bmYSN3sZhz7Lnt@mail.python.org>

http://hg.python.org/cpython/rev/ee9662d77ebb
changeset:   84445:ee9662d77ebb
user:        Brett Cannon <brett at python.org>
date:        Thu Jul 04 17:51:50 2013 -0400
summary:
  Issue #15767: back out 8a0ed9f63c6e, finishing the removal of
ModuleNotFoundError.

files:
  Doc/c-api/exceptions.rst                         |    2 -
  Doc/library/exceptions.rst                       |   13 +-
  Doc/whatsnew/3.4.rst                             |    3 -
  Include/pyerrors.h                               |    1 -
  Lib/importlib/_bootstrap.py                      |   15 +-
  Lib/pydoc.py                                     |    2 +-
  Lib/test/exception_hierarchy.txt                 |    1 -
  Lib/test/test_exceptions.py                      |    3 +
  Lib/test/test_import.py                          |   25 +-
  Lib/test/test_importlib/import_/test_api.py      |    4 -
  Lib/test/test_importlib/import_/test_fromlist.py |    8 +-
  Lib/test/test_pydoc.py                           |    2 +-
  Lib/test/test_site.py                            |    2 +-
  Misc/NEWS                                        |    3 -
  Objects/exceptions.c                             |    9 -
  Python/ceval.c                                   |    2 +-
  Python/importlib.h                               |  744 +++++----
  17 files changed, 412 insertions(+), 427 deletions(-)


diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
--- a/Doc/c-api/exceptions.rst
+++ b/Doc/c-api/exceptions.rst
@@ -686,8 +686,6 @@
 +-----------------------------------------+---------------------------------+----------+
 | :c:data:`PyExc_ImportError`             | :exc:`ImportError`              |          |
 +-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ModuleNotFoundError`     | :exc:`ModuleNotFoundError`      |          |
-+-----------------------------------------+---------------------------------+----------+
 | :c:data:`PyExc_IndexError`              | :exc:`IndexError`               |          |
 +-----------------------------------------+---------------------------------+----------+
 | :c:data:`PyExc_InterruptedError`        | :exc:`InterruptedError`         |          |
diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst
--- a/Doc/library/exceptions.rst
+++ b/Doc/library/exceptions.rst
@@ -169,8 +169,8 @@
 
 .. exception:: ImportError
 
-   Raised when the :keyword:`import` statement has troubles trying to load a
-   module.
+   Raised when an :keyword:`import` statement fails to find the module definition
+   or when a ``from ... import`` fails to find a name that is to be imported.
 
    The :attr:`name` and :attr:`path` attributes can be set using keyword-only
    arguments to the constructor. When set they represent the name of the module
@@ -180,15 +180,6 @@
    .. versionchanged:: 3.3
       Added the :attr:`name` and :attr:`path` attributes.
 
-.. exception:: ModuleNotFoundError
-
-   A subclass of :exc:`ImportError` which is raised by :keyword:`import` when a
-   module could not be located. This includes ``from ... import`` statements as
-   the specific attribute being requested cannot be known a priori to be a module
-   or some other type of object.
-
-   .. versionadded:: 3.4
-
 
 .. exception:: IndexError
 
diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst
--- a/Doc/whatsnew/3.4.rst
+++ b/Doc/whatsnew/3.4.rst
@@ -137,9 +137,6 @@
 
 * Unicode database updated to UCD version 6.2.
 
-* Import now raises the new exception :exc:`ModuleNotFoundError` (subclass of
-  :exc:`ImportError`) when it cannot find something.
-
 * :func:`min` and :func:`max` now accept a *default* argument that can be used
   to specify the value they return if the iterable they are evaluating has no
   elements.  Contributed by Julian Berman in :issue:`18111`.
diff --git a/Include/pyerrors.h b/Include/pyerrors.h
--- a/Include/pyerrors.h
+++ b/Include/pyerrors.h
@@ -152,7 +152,6 @@
 PyAPI_DATA(PyObject *) PyExc_FloatingPointError;
 PyAPI_DATA(PyObject *) PyExc_OSError;
 PyAPI_DATA(PyObject *) PyExc_ImportError;
-PyAPI_DATA(PyObject *) PyExc_ModuleNotFoundError;
 PyAPI_DATA(PyObject *) PyExc_IndexError;
 PyAPI_DATA(PyObject *) PyExc_KeyError;
 PyAPI_DATA(PyObject *) PyExc_KeyboardInterrupt;
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -1556,7 +1556,11 @@
             raise ImportError(msg, name=name)
     loader = _find_module(name, path)
     if loader is None:
-        raise ModuleNotFoundError(_ERR_MSG.format(name), name=name)
+        exc = ImportError(_ERR_MSG.format(name), name=name)
+        # TODO(brett): switch to a proper ModuleNotFound exception in Python
+        # 3.4.
+        exc._not_found = True
+        raise exc
     elif name not in sys.modules:
         # The parent import may have already imported this module.
         loader.load_module(name)
@@ -1642,12 +1646,15 @@
                 from_name = '{}.{}'.format(module.__name__, x)
                 try:
                     _call_with_frames_removed(import_, from_name)
-                except ModuleNotFoundError as exc:
+                except ImportError as exc:
                     # Backwards-compatibility dictates we ignore failed
                     # imports triggered by fromlist for modules that don't
                     # exist.
-                    if exc.name == from_name:
-                        continue
+                    # TODO(brett): In Python 3.4, have import raise
+                    #   ModuleNotFound and catch that.
+                    if getattr(exc, '_not_found', False):
+                        if exc.name == from_name:
+                            continue
                     raise
     return module
 
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -316,7 +316,7 @@
         elif exc is SyntaxError:
             # A SyntaxError occurred before we could execute the module.
             raise ErrorDuringImport(value.filename, info)
-        elif issubclass(exc, ImportError) and value.name == path:
+        elif exc is ImportError and value.name == path:
             # No such module in the path.
             return None
         else:
diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt
--- a/Lib/test/exception_hierarchy.txt
+++ b/Lib/test/exception_hierarchy.txt
@@ -13,7 +13,6 @@
       +-- BufferError
       +-- EOFError
       +-- ImportError
-           +-- ModuleNotFoundError
       +-- LookupError
       |    +-- IndexError
       |    +-- KeyError
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -953,5 +953,8 @@
             self.assertEqual(str(arg), str(exc))
 
 
+def test_main():
+    run_unittest(ExceptionTests, ImportErrorTests)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
--- a/Lib/test/test_import.py
+++ b/Lib/test/test_import.py
@@ -68,15 +68,7 @@
     def tearDown(self):
         unload(TESTFN)
 
-    def test_import_raises_ModuleNotFoundError(self):
-        with self.assertRaises(ModuleNotFoundError):
-            import something_that_should_not_exist_anywhere
-
-    def test_from_import_raises_ModuleNotFoundError(self):
-        with self.assertRaises(ModuleNotFoundError):
-            from something_that_should_not_exist_anywhere import blah
-        with self.assertRaises(ModuleNotFoundError):
-            from importlib import something_that_should_not_exist_anywhere
+    setUp = tearDown
 
     def test_case_sensitivity(self):
         # Brief digression to test that import is case-sensitive:  if we got
@@ -495,7 +487,7 @@
             header = f.read(12)
             code = marshal.load(f)
         constants = list(code.co_consts)
-        foreign_code = importlib.import_module.__code__
+        foreign_code = test_main.__code__
         pos = constants.index(1)
         constants[pos] = foreign_code
         code = type(code)(code.co_argcount, code.co_kwonlyargcount,
@@ -1021,5 +1013,16 @@
             importlib.SourceLoader.load_module = old_load_module
 
 
+def test_main(verbose=None):
+    run_unittest(ImportTests, PycacheTests, FilePermissionTests,
+                 PycRewritingTests, PathsTests, RelativeImportTests,
+                 OverridingImportBuiltinTests,
+                 ImportlibBootstrapTests,
+                 TestSymbolicallyLinkedPackage,
+                 ImportTracebackTests)
+
+
 if __name__ == '__main__':
-    unittest.main()
+    # Test needs to be a package, so we can do relative imports.
+    from test.test_import import test_main
+    test_main()
diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py
--- a/Lib/test/test_importlib/import_/test_api.py
+++ b/Lib/test/test_importlib/import_/test_api.py
@@ -22,10 +22,6 @@
     """Test API-specific details for __import__ (e.g. raising the right
     exception when passing in an int for the module name)."""
 
-    def test_raises_ModuleNotFoundError(self):
-        with self.assertRaises(ModuleNotFoundError):
-            util.import_('some module that does not exist')
-
     def test_name_requires_rparition(self):
         # Raise TypeError if a non-string is passed in for the module name.
         with self.assertRaises(TypeError):
diff --git a/Lib/test/test_importlib/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py
--- a/Lib/test/test_importlib/import_/test_fromlist.py
+++ b/Lib/test/test_importlib/import_/test_fromlist.py
@@ -68,16 +68,16 @@
                 self.assertTrue(hasattr(module, 'module'))
                 self.assertEqual(module.module.__name__, 'pkg.module')
 
-    def test_module_from_package_triggers_ModuleNotFoundError(self):
-        # If a submodule causes an ModuleNotFoundError because it tries to import
-        # a module which doesn't exist, that should let the ModuleNotFoundError
+    def test_module_from_package_triggers_ImportError(self):
+        # If a submodule causes an ImportError because it tries to import
+        # a module which doesn't exist, that should let the ImportError
         # propagate.
         def module_code():
             import i_do_not_exist
         with util.mock_modules('pkg.__init__', 'pkg.mod',
                                module_code={'pkg.mod': module_code}) as importer:
             with util.import_state(meta_path=[importer]):
-                with self.assertRaises(ModuleNotFoundError) as exc:
+                with self.assertRaises(ImportError) as exc:
                     import_util.import_('pkg', fromlist=['mod'])
                 self.assertEqual('i_do_not_exist', exc.exception.name)
 
diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py
--- a/Lib/test/test_pydoc.py
+++ b/Lib/test/test_pydoc.py
@@ -206,7 +206,7 @@
 missing_pattern = "no Python documentation found for '%s'"
 
 # output pattern for module with bad imports
-badimport_pattern = "problem in %s - ModuleNotFoundError: No module named %r"
+badimport_pattern = "problem in %s - ImportError: No module named %r"
 
 def run_pydoc(module_name, *args, **env):
     """
diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py
--- a/Lib/test/test_site.py
+++ b/Lib/test/test_site.py
@@ -131,7 +131,7 @@
             re.escape(os.path.join(pth_dir, pth_fn)))
         # XXX: ditto previous XXX comment.
         self.assertRegex(err_out.getvalue(), 'Traceback')
-        self.assertRegex(err_out.getvalue(), 'ModuleNotFoundError')
+        self.assertRegex(err_out.getvalue(), 'ImportError')
 
     @unittest.skipIf(sys.platform == "win32", "Windows does not raise an "
                       "error for file paths containing null characters")
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -25,9 +25,6 @@
 - Issue #18137: Detect integer overflow on precision in float.__format__()
   and complex.__format__().
 
-- Issue #15767: Introduce ModuleNotFoundError which is raised when a module
-  could not be found.
-
 - Issue #18183: Fix various unicode operations on strings with large unicode
   codepoints.
 
diff --git a/Objects/exceptions.c b/Objects/exceptions.c
--- a/Objects/exceptions.c
+++ b/Objects/exceptions.c
@@ -710,13 +710,6 @@
                         "module.");
 
 /*
- *    ModuleNotFoundError extends ImportError
- */
-
-MiddlingExtendsException(PyExc_ImportError, ModuleNotFoundError, ImportError,
-                         "Module not found.");
-
-/*
  *    OSError extends Exception
  */
 
@@ -2402,7 +2395,6 @@
     PRE_INIT(SystemExit)
     PRE_INIT(KeyboardInterrupt)
     PRE_INIT(ImportError)
-    PRE_INIT(ModuleNotFoundError)
     PRE_INIT(OSError)
     PRE_INIT(EOFError)
     PRE_INIT(RuntimeError)
@@ -2473,7 +2465,6 @@
     POST_INIT(SystemExit)
     POST_INIT(KeyboardInterrupt)
     POST_INIT(ImportError)
-    POST_INIT(ModuleNotFoundError)
     POST_INIT(OSError)
     INIT_ALIAS(EnvironmentError, OSError)
     INIT_ALIAS(IOError, OSError)
diff --git a/Python/ceval.c b/Python/ceval.c
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -4588,7 +4588,7 @@
 
     x = PyObject_GetAttr(v, name);
     if (x == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) {
-        PyErr_Format(PyExc_ModuleNotFoundError, "cannot import name %S", name);
+        PyErr_Format(PyExc_ImportError, "cannot import name %S", name);
     }
     return x;
 }
diff --git a/Python/importlib.h b/Python/importlib.h
--- a/Python/importlib.h
+++ b/Python/importlib.h
[stripped]

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul  5 00:16:37 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri,  5 Jul 2013 00:16:37 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Move_test=5Fimport_over_to?=
	=?utf-8?b?IHVuaXR0ZXN0Lm1haW4oKS4=?=
Message-ID: <3bmYSP5jJVz7LqS@mail.python.org>

http://hg.python.org/cpython/rev/200b1a099ed8
changeset:   84446:200b1a099ed8
user:        Brett Cannon <brett at python.org>
date:        Thu Jul 04 18:03:57 2013 -0400
summary:
  Move test_import over to unittest.main().

files:
  Lib/test/test_import.py |  16 ++--------------
  1 files changed, 2 insertions(+), 14 deletions(-)


diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
--- a/Lib/test/test_import.py
+++ b/Lib/test/test_import.py
@@ -68,8 +68,6 @@
     def tearDown(self):
         unload(TESTFN)
 
-    setUp = tearDown
-
     def test_case_sensitivity(self):
         # Brief digression to test that import is case-sensitive:  if we got
         # this far, we know for sure that "random" exists.
@@ -487,7 +485,7 @@
             header = f.read(12)
             code = marshal.load(f)
         constants = list(code.co_consts)
-        foreign_code = test_main.__code__
+        foreign_code = importlib.import_module.__code__
         pos = constants.index(1)
         constants[pos] = foreign_code
         code = type(code)(code.co_argcount, code.co_kwonlyargcount,
@@ -1013,16 +1011,6 @@
             importlib.SourceLoader.load_module = old_load_module
 
 
-def test_main(verbose=None):
-    run_unittest(ImportTests, PycacheTests, FilePermissionTests,
-                 PycRewritingTests, PathsTests, RelativeImportTests,
-                 OverridingImportBuiltinTests,
-                 ImportlibBootstrapTests,
-                 TestSymbolicallyLinkedPackage,
-                 ImportTracebackTests)
-
-
 if __name__ == '__main__':
     # Test needs to be a package, so we can do relative imports.
-    from test.test_import import test_main
-    test_main()
+    unittest.main()

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul  5 00:16:39 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri,  5 Jul 2013 00:16:39 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Remove_dead_code_in_test?=
	=?utf-8?q?=5Fexceptions=2E?=
Message-ID: <3bmYSR0R26z7Lqs@mail.python.org>

http://hg.python.org/cpython/rev/6a8ba235542a
changeset:   84447:6a8ba235542a
user:        Brett Cannon <brett at python.org>
date:        Thu Jul 04 18:04:20 2013 -0400
summary:
  Remove dead code in test_exceptions.

files:
  Lib/test/test_exceptions.py |  3 ---
  1 files changed, 0 insertions(+), 3 deletions(-)


diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -953,8 +953,5 @@
             self.assertEqual(str(arg), str(exc))
 
 
-def test_main():
-    run_unittest(ExceptionTests, ImportErrorTests)
-
 if __name__ == '__main__':
     unittest.main()

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul  5 00:16:40 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri,  5 Jul 2013 00:16:40 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2315767=3A_Excise_t?=
 =?utf-8?q?he_remaining_instances_of_ModuleNotFoundError?=
Message-ID: <3bmYSS3ddfz7LqV@mail.python.org>

http://hg.python.org/cpython/rev/de947db308ba
changeset:   84448:de947db308ba
user:        Brett Cannon <brett at python.org>
date:        Thu Jul 04 18:16:15 2013 -0400
summary:
  Issue #15767: Excise the remaining instances of ModuleNotFoundError

files:
  Lib/stat.py              |   2 +-
  Lib/test/regrtest.py     |  16 ++++++++--------
  Lib/test/support.py      |  16 ++++++++--------
  Lib/test/test___all__.py |   2 +-
  Lib/test/test_tarfile.py |   6 +++---
  Lib/test/test_xmlrpc.py  |   4 ++--
  Lib/xmlrpc/server.py     |   2 +-
  Lib/zipfile.py           |   6 +++---
  8 files changed, 27 insertions(+), 27 deletions(-)


diff --git a/Lib/stat.py b/Lib/stat.py
--- a/Lib/stat.py
+++ b/Lib/stat.py
@@ -151,5 +151,5 @@
 # If available, use C implementation
 try:
     from _stat import *
-except ModuleNotFoundError:
+except ImportError:
     pass
diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -146,11 +146,11 @@
 
 try:
     import threading
-except ModuleNotFoundError:
+except ImportError:
     threading = None
 try:
     import multiprocessing.process
-except ModuleNotFoundError:
+except ImportError:
     multiprocessing = None
 
 
@@ -180,7 +180,7 @@
 if sys.platform == 'darwin':
     try:
         import resource
-    except ModuleNotFoundError:
+    except ImportError:
         pass
     else:
         soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
@@ -567,7 +567,7 @@
     if findleaks:
         try:
             import gc
-        except ModuleNotFoundError:
+        except ImportError:
             print('No GC available, disabling findleaks.')
             findleaks = False
         else:
@@ -688,7 +688,7 @@
     if use_mp:
         try:
             from threading import Thread
-        except ModuleNotFoundError:
+        except ImportError:
             print("Multiprocess option requires thread support")
             sys.exit(2)
         from queue import Queue
@@ -1399,7 +1399,7 @@
     pic = sys.path_importer_cache.copy()
     try:
         import zipimport
-    except ModuleNotFoundError:
+    except ImportError:
         zdc = None # Run unmodified on platforms without zipimport support
     else:
         zdc = zipimport._zip_directory_cache.copy()
@@ -1476,7 +1476,7 @@
     sys.path_importer_cache.update(pic)
     try:
         import zipimport
-    except ModuleNotFoundError:
+    except ImportError:
         pass # Run unmodified on platforms without zipimport support
     else:
         zipimport._zip_directory_cache.clear()
@@ -1513,7 +1513,7 @@
     doctest.master = None
     try:
         import ctypes
-    except ModuleNotFoundError:
+    except ImportError:
         # Don't worry about resetting the cache if ctypes is not supported
         pass
     else:
diff --git a/Lib/test/support.py b/Lib/test/support.py
--- a/Lib/test/support.py
+++ b/Lib/test/support.py
@@ -29,32 +29,32 @@
 
 try:
     import _thread, threading
-except ModuleNotFoundError:
+except ImportError:
     _thread = None
     threading = None
 try:
     import multiprocessing.process
-except ModuleNotFoundError:
+except ImportError:
     multiprocessing = None
 
 try:
     import zlib
-except ModuleNotFoundError:
+except ImportError:
     zlib = None
 
 try:
     import gzip
-except ModuleNotFoundError:
+except ImportError:
     gzip = None
 
 try:
     import bz2
-except ModuleNotFoundError:
+except ImportError:
     bz2 = None
 
 try:
     import lzma
-except ModuleNotFoundError:
+except ImportError:
     lzma = None
 
 __all__ = [
@@ -121,7 +121,7 @@
     with _ignore_deprecated_imports(deprecated):
         try:
             return importlib.import_module(name)
-        except ModuleNotFoundError as msg:
+        except ImportError as msg:
             if sys.platform.startswith(tuple(required_on)):
                 raise
             raise unittest.SkipTest(str(msg))
@@ -193,7 +193,7 @@
                 if not _save_and_block_module(blocked_name, orig_modules):
                     names_to_remove.append(blocked_name)
             fresh_module = importlib.import_module(name)
-        except ModuleNotFoundError:
+        except ImportError:
             fresh_module = None
         finally:
             for orig_name, module in orig_modules.items():
diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py
--- a/Lib/test/test___all__.py
+++ b/Lib/test/test___all__.py
@@ -75,7 +75,7 @@
         try:
             import rlcompleter
             import locale
-        except ModuleNotFoundError:
+        except ImportError:
             pass
         else:
             locale.setlocale(locale.LC_CTYPE, 'C')
diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py
--- a/Lib/test/test_tarfile.py
+++ b/Lib/test/test_tarfile.py
@@ -12,15 +12,15 @@
 # Check for our compression modules.
 try:
     import gzip
-except ModuleNotFoundError:
+except ImportError:
     gzip = None
 try:
     import bz2
-except ModuleNotFoundError:
+except ImportError:
     bz2 = None
 try:
     import lzma
-except ModuleNotFoundError:
+except ImportError:
     lzma = None
 
 def md5sum(data):
diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py
--- a/Lib/test/test_xmlrpc.py
+++ b/Lib/test/test_xmlrpc.py
@@ -15,11 +15,11 @@
 
 try:
     import gzip
-except ModuleNotFoundError:
+except ImportError:
     gzip = None
 try:
     import threading
-except ModuleNotFoundError:
+except ImportError:
     threading = None
 
 alist = [{'astring': 'foo at bar.baz.spam',
diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py
--- a/Lib/xmlrpc/server.py
+++ b/Lib/xmlrpc/server.py
@@ -116,7 +116,7 @@
 import traceback
 try:
     import fcntl
-except ModuleNotFoundError:
+except ImportError:
     fcntl = None
 
 def resolve_dotted_attribute(obj, attr, allow_dotted_names=True):
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
--- a/Lib/zipfile.py
+++ b/Lib/zipfile.py
@@ -18,18 +18,18 @@
 try:
     import zlib # We may need its compression method
     crc32 = zlib.crc32
-except ModuleNotFoundError:
+except ImportError:
     zlib = None
     crc32 = binascii.crc32
 
 try:
     import bz2 # We may need its compression method
-except ModuleNotFoundError:
+except ImportError:
     bz2 = None
 
 try:
     import lzma # We may need its compression method
-except ModuleNotFoundError:
+except ImportError:
     lzma = None
 
 __all__ = ["BadZipFile", "BadZipfile", "error",

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul  5 01:41:51 2013
From: python-checkins at python.org (christian.heimes)
Date: Fri,  5 Jul 2013 01:41:51 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4MzQ3?=
 =?utf-8?q?=3A_ElementTree=27s_html_serializer_now_preserves_the_case_of_c?=
 =?utf-8?q?losing?=
Message-ID: <3bmbLl2xcsz7Lk2@mail.python.org>

http://hg.python.org/cpython/rev/df79735b21c1
changeset:   84449:df79735b21c1
branch:      3.3
parent:      84438:b3ea1b5a1617
user:        Christian Heimes <christian at cheimes.de>
date:        Fri Jul 05 01:39:49 2013 +0200
summary:
  Issue #18347: ElementTree's html serializer now preserves the case of closing tags.

files:
  Lib/test/test_xml_etree.py   |  7 +++++++
  Lib/xml/etree/ElementTree.py |  6 +++---
  Misc/NEWS                    |  3 +++
  3 files changed, 13 insertions(+), 3 deletions(-)


diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
--- a/Lib/test/test_xml_etree.py
+++ b/Lib/test/test_xml_etree.py
@@ -751,6 +751,13 @@
                 '<html><link><script>1 < 2</script></html>\n')
         self.assertEqual(serialize(e, method="text"), '1 < 2\n')
 
+    def test_issue18347(self):
+        e = ET.XML('<html><CamelCase>text</CamelCase></html>')
+        self.assertEqual(serialize(e),
+                '<html><CamelCase>text</CamelCase></html>')
+        self.assertEqual(serialize(e, method="html"),
+                '<html><CamelCase>text</CamelCase></html>')
+
     def test_entity(self):
         # Test entity handling.
 
diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py
--- a/Lib/xml/etree/ElementTree.py
+++ b/Lib/xml/etree/ElementTree.py
@@ -1039,15 +1039,15 @@
                     # FIXME: handle boolean attributes
                     write(" %s=\"%s\"" % (qnames[k], v))
             write(">")
-            tag = tag.lower()
+            ltag = tag.lower()
             if text:
-                if tag == "script" or tag == "style":
+                if ltag == "script" or ltag == "style":
                     write(text)
                 else:
                     write(_escape_cdata(text))
             for e in elem:
                 _serialize_html(write, e, qnames, None)
-            if tag not in HTML_EMPTY:
+            if ltag not in HTML_EMPTY:
                 write("</" + tag + ">")
     if elem.tail:
         write(_escape_cdata(elem.tail))
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -41,6 +41,9 @@
 Library
 -------
 
+- Issue #18347: ElementTree's html serializer now preserves the case of
+  closing tags.
+
 - Issue #17261: Ensure multiprocessing's proxies use proper address.
 
 - Issue #18343: faulthandler.register() now keeps the previous signal handler

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul  5 01:41:52 2013
From: python-checkins at python.org (christian.heimes)
Date: Fri,  5 Jul 2013 01:41:52 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2318347=3A_ElementTree=27s_html_serializer_now_pr?=
 =?utf-8?q?eserves_the_case_of_closing?=
Message-ID: <3bmbLm4xvZz7Ll0@mail.python.org>

http://hg.python.org/cpython/rev/d5536c06a082
changeset:   84450:d5536c06a082
parent:      84448:de947db308ba
parent:      84449:df79735b21c1
user:        Christian Heimes <christian at cheimes.de>
date:        Fri Jul 05 01:40:52 2013 +0200
summary:
  Issue #18347: ElementTree's html serializer now preserves the case of closing tags.

files:
  Lib/test/test_xml_etree.py   |  7 +++++++
  Lib/xml/etree/ElementTree.py |  6 +++---
  Misc/NEWS                    |  3 +++
  3 files changed, 13 insertions(+), 3 deletions(-)


diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
--- a/Lib/test/test_xml_etree.py
+++ b/Lib/test/test_xml_etree.py
@@ -742,6 +742,13 @@
                 '<html><link><script>1 < 2</script></html>\n')
         self.assertEqual(serialize(e, method="text"), '1 < 2\n')
 
+    def test_issue18347(self):
+        e = ET.XML('<html><CamelCase>text</CamelCase></html>')
+        self.assertEqual(serialize(e),
+                '<html><CamelCase>text</CamelCase></html>')
+        self.assertEqual(serialize(e, method="html"),
+                '<html><CamelCase>text</CamelCase></html>')
+
     def test_entity(self):
         # Test entity handling.
 
diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py
--- a/Lib/xml/etree/ElementTree.py
+++ b/Lib/xml/etree/ElementTree.py
@@ -992,15 +992,15 @@
                     # FIXME: handle boolean attributes
                     write(" %s=\"%s\"" % (qnames[k], v))
             write(">")
-            tag = tag.lower()
+            ltag = tag.lower()
             if text:
-                if tag == "script" or tag == "style":
+                if ltag == "script" or ltag == "style":
                     write(text)
                 else:
                     write(_escape_cdata(text))
             for e in elem:
                 _serialize_html(write, e, qnames, None)
-            if tag not in HTML_EMPTY:
+            if ltag not in HTML_EMPTY:
                 write("</" + tag + ">")
     if elem.tail:
         write(_escape_cdata(elem.tail))
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -135,6 +135,9 @@
 Library
 -------
 
+- Issue #18347: ElementTree's html serializer now preserves the case of
+  closing tags.
+
 - Issue #17261: Ensure multiprocessing's proxies use proper address.
 
 - Issue #18343: faulthandler.register() now keeps the previous signal handler

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul  5 01:41:53 2013
From: python-checkins at python.org (christian.heimes)
Date: Fri,  5 Jul 2013 01:41:53 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4MzQ3?=
 =?utf-8?q?=3A_ElementTree=27s_html_serializer_now_preserves_the_case_of_c?=
 =?utf-8?q?losing?=
Message-ID: <3bmbLn6ztLz7Lmp@mail.python.org>

http://hg.python.org/cpython/rev/328781ae35d2
changeset:   84451:328781ae35d2
branch:      2.7
parent:      84441:e3fd5fc5dc47
user:        Christian Heimes <christian at cheimes.de>
date:        Fri Jul 05 01:41:30 2013 +0200
summary:
  Issue #18347: ElementTree's html serializer now preserves the case of closing tags.

files:
  Lib/test/test_xml_etree.py   |  10 ++++++++++
  Lib/xml/etree/ElementTree.py |   6 +++---
  Misc/NEWS                    |   3 +++
  3 files changed, 16 insertions(+), 3 deletions(-)


diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
--- a/Lib/test/test_xml_etree.py
+++ b/Lib/test/test_xml_etree.py
@@ -1769,6 +1769,16 @@
 
     """
 
+def bug_18347():
+    """
+
+    >>> e = ET.XML('<html><CamelCase>text</CamelCase></html>')
+    >>> serialize(e)
+    '<html><CamelCase>text</CamelCase></html>'
+    >>> serialize(e, method="html")
+    '<html><CamelCase>text</CamelCase></html>'
+    """
+
 # --------------------------------------------------------------------
 # reported on bugs.python.org
 
diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py
--- a/Lib/xml/etree/ElementTree.py
+++ b/Lib/xml/etree/ElementTree.py
@@ -988,15 +988,15 @@
                     # FIXME: handle boolean attributes
                     write(" %s=\"%s\"" % (qnames[k], v))
             write(">")
-            tag = tag.lower()
+            ltag = tag.lower()
             if text:
-                if tag == "script" or tag == "style":
+                if ltag == "script" or ltag == "style":
                     write(_encode(text, encoding))
                 else:
                     write(_escape_cdata(text, encoding))
             for e in elem:
                 _serialize_html(write, e, encoding, qnames, None)
-            if tag not in HTML_EMPTY:
+            if ltag not in HTML_EMPTY:
                 write("</" + tag + ">")
     if elem.tail:
         write(_escape_cdata(elem.tail, encoding))
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -24,6 +24,9 @@
 Library
 -------
 
+- Issue #18347: ElementTree's html serializer now preserves the case of
+  closing tags.
+
 - Issue #17261: Ensure multiprocessing's proxies use proper address.
 
 - Issue #17097: Make multiprocessing ignore EINTR.

-- 
Repository URL: http://hg.python.org/cpython

From solipsis at pitrou.net  Fri Jul  5 05:43:59 2013
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Fri, 05 Jul 2013 05:43:59 +0200
Subject: [Python-checkins] Daily reference leaks (d5536c06a082): sum=0
Message-ID: <E1UuwwB-0007jT-IY@ap.vmr.nerim.net>

results for d5536c06a082 on branch "default"
--------------------------------------------



Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogZ7wEhS', '-x']

From solipsis at pitrou.net  Sat Jul  6 05:43:24 2013
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Sat, 06 Jul 2013 05:43:24 +0200
Subject: [Python-checkins] Daily reference leaks (d5536c06a082): sum=2
Message-ID: <E1UvJPA-00036m-2m@ap.vmr.nerim.net>

results for d5536c06a082 on branch "default"
--------------------------------------------

test_unittest leaked [-1, 2, 1] memory blocks, sum=2


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogWCNA5b', '-x']

From python-checkins at python.org  Sat Jul  6 06:06:03 2013
From: python-checkins at python.org (raymond.hettinger)
Date: Sat,  6 Jul 2013 06:06:03 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Speed-up_deque_indexing_by?=
 =?utf-8?q?_changing_the_deque_block_length_to_a_power_of_two=2E?=
Message-ID: <3bnK9706nkz7Lk2@mail.python.org>

http://hg.python.org/cpython/rev/6d278f426417
changeset:   84452:6d278f426417
parent:      84450:d5536c06a082
user:        Raymond Hettinger <python at rcn.com>
date:        Fri Jul 05 18:05:29 2013 -1000
summary:
  Speed-up deque indexing by changing the deque block length to a power of two.

The division and modulo calculation in deque_item() can be compiled
to fast bitwise operations when the BLOCKLEN is a power of two.

Timing before:

 ~/cpython $ py -m timeit -r7 -s 'from collections import deque' -s 'd=deque(range(10))' 'd[5]'
10000000 loops, best of 7: 0.0627 usec per loop

Timing after:

~/cpython $ py -m timeit -r7 -s 'from collections import deque' -s 'd=deque(range(10))' 'd[5]'
10000000 loops, best of 7: 0.0581 usec per loop

files:
  Lib/test/test_deque.py       |  2 +-
  Modules/_collectionsmodule.c |  2 +-
  2 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py
--- a/Lib/test/test_deque.py
+++ b/Lib/test/test_deque.py
@@ -536,7 +536,7 @@
 
     @support.cpython_only
     def test_sizeof(self):
-        BLOCKLEN = 62
+        BLOCKLEN = 64
         basesize = support.calcobjsize('2P4nlP')
         blocksize = struct.calcsize('2P%dP' % BLOCKLEN)
         self.assertEqual(object.__sizeof__(deque()), basesize)
diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -14,7 +14,7 @@
  * division/modulo computations during indexing.
  */
 
-#define BLOCKLEN 62
+#define BLOCKLEN 64
 #define CENTER ((BLOCKLEN - 1) / 2)
 
 /* A `dequeobject` is composed of a doubly-linked list of `block` nodes.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 10:25:19 2013
From: python-checkins at python.org (ronald.oussoren)
Date: Sat,  6 Jul 2013 10:25:19 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE3ODYw?=
 =?utf-8?q?=3A_explicitly_mention_that_std*_streams_are_opened_in_binary_m?=
 =?utf-8?q?ode_by?=
Message-ID: <3bnQwH1z6Vz7LjW@mail.python.org>

http://hg.python.org/cpython/rev/a2c2ffa1a41c
changeset:   84453:a2c2ffa1a41c
branch:      3.3
parent:      84449:df79735b21c1
user:        Ronald Oussoren <ronaldoussoren at mac.com>
date:        Sat Jul 06 10:23:59 2013 +0200
summary:
  Issue #17860: explicitly mention that std* streams are opened in binary mode by default.

The documentation does mention that the streams are opened in text mode
when univeral_newlines is true, but not that that they are opened in
binary mode when that argument is false and that seems to confuse at
least some users.

files:
  Doc/library/subprocess.rst |  6 ++++--
  1 files changed, 4 insertions(+), 2 deletions(-)


diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -293,7 +293,8 @@
    If *universal_newlines* is ``True``, the file objects *stdin*, *stdout* and
    *stderr* will be opened as text streams in :term:`universal newlines` mode
    using the encoding returned by :func:`locale.getpreferredencoding(False)
-   <locale.getpreferredencoding>`.  For *stdin*, line ending characters
+   <locale.getpreferredencoding>`, otherwise these streams will be opened
+   as binary streams.  For *stdin*, line ending characters
    ``'\n'`` in the input will be converted to the default line separator
    :data:`os.linesep`.  For *stdout* and *stderr*, all line endings in the
    output will be converted to ``'\n'``.  For more information see the
@@ -537,7 +538,8 @@
 
    If *universal_newlines* is ``True``, the file objects *stdin*, *stdout*
    and *stderr* are opened as text streams in universal newlines mode, as
-   described above in :ref:`frequently-used-arguments`.
+   described above in :ref:`frequently-used-arguments`, otherwise they are
+   opened as binary streams.
 
    If given, *startupinfo* will be a :class:`STARTUPINFO` object, which is
    passed to the underlying ``CreateProcess`` function.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 10:25:20 2013
From: python-checkins at python.org (ronald.oussoren)
Date: Sat,  6 Jul 2013 10:25:20 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_=283=2E3-=3Edefault=29_Issue_=2317860=3A_explicitly_ment?=
 =?utf-8?q?ion_that_std*_streams_are_opened_in?=
Message-ID: <3bnQwJ3szqz7LjW@mail.python.org>

http://hg.python.org/cpython/rev/ae8b054155c1
changeset:   84454:ae8b054155c1
parent:      84452:6d278f426417
parent:      84453:a2c2ffa1a41c
user:        Ronald Oussoren <ronaldoussoren at mac.com>
date:        Sat Jul 06 10:25:04 2013 +0200
summary:
  (3.3->default) Issue #17860: explicitly mention that std* streams are opened in binary mode by default.

The documentation does mention that the streams are opened in text mode
when univeral_newlines is true, but not that that they are opened in
binary mode when that argument is false and that seems to confuse at
least some users.

files:
  Doc/library/subprocess.rst |  6 ++++--
  1 files changed, 4 insertions(+), 2 deletions(-)


diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -303,7 +303,8 @@
    If *universal_newlines* is ``True``, the file objects *stdin*, *stdout* and
    *stderr* will be opened as text streams in :term:`universal newlines` mode
    using the encoding returned by :func:`locale.getpreferredencoding(False)
-   <locale.getpreferredencoding>`.  For *stdin*, line ending characters
+   <locale.getpreferredencoding>`, otherwise these streams will be opened
+   as binary streams.  For *stdin*, line ending characters
    ``'\n'`` in the input will be converted to the default line separator
    :data:`os.linesep`.  For *stdout* and *stderr*, all line endings in the
    output will be converted to ``'\n'``.  For more information see the
@@ -547,7 +548,8 @@
 
    If *universal_newlines* is ``True``, the file objects *stdin*, *stdout*
    and *stderr* are opened as text streams in universal newlines mode, as
-   described above in :ref:`frequently-used-arguments`.
+   described above in :ref:`frequently-used-arguments`, otherwise they are
+   opened as binary streams.
 
    If given, *startupinfo* will be a :class:`STARTUPINFO` object, which is
    passed to the underlying ``CreateProcess`` function.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 12:28:13 2013
From: python-checkins at python.org (florent.xicluna)
Date: Sat,  6 Jul 2013 12:28:13 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4Mzc1?=
 =?utf-8?q?=3A_Assume_--randomize_when_--randseed_is_used_for_running_the?=
Message-ID: <3bnTf535D1z7Lk0@mail.python.org>

http://hg.python.org/cpython/rev/5bd3f9ed357e
changeset:   84455:5bd3f9ed357e
branch:      3.3
parent:      84453:a2c2ffa1a41c
user:        Florent Xicluna <florent.xicluna at gmail.com>
date:        Sat Jul 06 12:25:52 2013 +0200
summary:
  Issue #18375: Assume --randomize when --randseed is used for running the testsuite.

files:
  Lib/test/regrtest.py |  1 +
  Misc/NEWS            |  3 +++
  2 files changed, 4 insertions(+), 0 deletions(-)


diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -346,6 +346,7 @@
         elif o in ('-r', '--randomize'):
             randomize = True
         elif o == '--randseed':
+            randomize = True
             random_seed = int(a)
         elif o in ('-f', '--fromfile'):
             fromfile = a
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -158,6 +158,9 @@
 Tests
 -----
 
+- Issue #18375: Assume --randomize when --randseed is used for running the
+  testsuite.
+
 - Issue #11185: Fix test_wait4 under AIX.  Patch by S?bastien Sabl?.
 
 - Issue #17691: test_univnewlines now works with unittest test discovery.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 12:28:14 2013
From: python-checkins at python.org (florent.xicluna)
Date: Sat,  6 Jul 2013 12:28:14 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2318375=3A_merge_with_3=2E3?=
Message-ID: <3bnTf654BWz7LmK@mail.python.org>

http://hg.python.org/cpython/rev/7be081fa8db1
changeset:   84456:7be081fa8db1
parent:      84454:ae8b054155c1
parent:      84455:5bd3f9ed357e
user:        Florent Xicluna <florent.xicluna at gmail.com>
date:        Sat Jul 06 12:27:50 2013 +0200
summary:
  Issue #18375: merge with 3.3

files:
  Lib/test/regrtest.py |  1 +
  Misc/NEWS            |  3 +++
  2 files changed, 4 insertions(+), 0 deletions(-)


diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -428,6 +428,7 @@
         elif o in ('-r', '--randomize'):
             randomize = True
         elif o == '--randseed':
+            randomize = True
             random_seed = int(a)
         elif o in ('-f', '--fromfile'):
             fromfile = a
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -468,6 +468,9 @@
 Tests
 -----
 
+- Issue #18375: Assume --randomize when --randseed is used for running the
+  testsuite.
+
 - Issue #11185: Fix test_wait4 under AIX.  Patch by S?bastien Sabl?.
 
 - Issue #18207: Fix test_ssl for some versions of OpenSSL that ignore seconds

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 13:20:27 2013
From: python-checkins at python.org (ronald.oussoren)
Date: Sat,  6 Jul 2013 13:20:27 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzEyOTkw?=
 =?utf-8?q?=3A_The_=22Python_Launcher=22_on_OSX_could_not_launch_python_sc?=
 =?utf-8?q?ripts_that?=
Message-ID: <3bnVpM5lnTz7LmX@mail.python.org>

http://hg.python.org/cpython/rev/7ec9255d4189
changeset:   84457:7ec9255d4189
branch:      2.7
parent:      84451:328781ae35d2
user:        Ronald Oussoren <ronaldoussoren at mac.com>
date:        Sat Jul 06 13:19:58 2013 +0200
summary:
  Issue #12990: The "Python Launcher" on OSX could not launch python scripts that have paths that include wide characters.

files:
  Mac/PythonLauncher/MyDocument.m |  2 +-
  Misc/NEWS                       |  6 ++++++
  2 files changed, 7 insertions(+), 1 deletions(-)


diff --git a/Mac/PythonLauncher/MyDocument.m b/Mac/PythonLauncher/MyDocument.m
--- a/Mac/PythonLauncher/MyDocument.m
+++ b/Mac/PythonLauncher/MyDocument.m
@@ -76,7 +76,7 @@
     const char *cmdline;
     int sts;
     
-     cmdline = [[settings commandLineForScript: script] cString];
+     cmdline = [[settings commandLineForScript: script] UTF8String];
    if ([settings with_terminal]) {
         sts = doscript(cmdline);
     } else {
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -67,6 +67,12 @@
 - Issue #18113: Fixed a refcount leak in the curses.panel module's
   set_userptr() method.  Reported by Atsuo Ishimoto.
 
+Tools/Demos
+-----------
+
+- Issue #12990: The "Python Launcher" on OSX could not launch python scripts
+  that have paths that include wide characters.
+
 Build
 -----
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 13:26:12 2013
From: python-checkins at python.org (ronald.oussoren)
Date: Sat,  6 Jul 2013 13:26:12 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzEyOTkw?=
 =?utf-8?q?=3A_The_=22Python_Launcher=22_on_OSX_could_not_launch_python_sc?=
 =?utf-8?q?ripts_that?=
Message-ID: <3bnVx041H8z7LkW@mail.python.org>

http://hg.python.org/cpython/rev/27eb350d5056
changeset:   84458:27eb350d5056
branch:      3.3
parent:      84455:5bd3f9ed357e
user:        Ronald Oussoren <ronaldoussoren at mac.com>
date:        Sat Jul 06 13:20:57 2013 +0200
summary:
  Issue #12990: The "Python Launcher" on OSX could not launch python scripts that have paths that include wide characters.

files:
  Mac/PythonLauncher/MyDocument.m |  2 +-
  Misc/NEWS                       |  3 +++
  2 files changed, 4 insertions(+), 1 deletions(-)


diff --git a/Mac/PythonLauncher/MyDocument.m b/Mac/PythonLauncher/MyDocument.m
--- a/Mac/PythonLauncher/MyDocument.m
+++ b/Mac/PythonLauncher/MyDocument.m
@@ -76,7 +76,7 @@
     const char *cmdline;
     int sts;
     
-     cmdline = [[settings commandLineForScript: script] cString];
+     cmdline = [[settings commandLineForScript: script] UTF8String];
    if ([settings with_terminal]) {
         sts = doscript(cmdline);
     } else {
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -188,6 +188,9 @@
 
 - Issue #15239: Make mkstringprep.py work again on Python 3.
 
+- Issue #12990: The "Python Launcher" on OSX could not launch python scripts
+  that have paths that include wide characters.
+
 Build
 -----
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 13:26:13 2013
From: python-checkins at python.org (ronald.oussoren)
Date: Sat,  6 Jul 2013 13:26:13 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?b?KTogKDMuMy0+ZGVmYXVsdCkgSXNzdWUgIzEyOTkwOiBUaGUgIlB5dGhvbiBM?=
 =?utf-8?q?auncher=22_on_OSX_could_not_launch?=
Message-ID: <3bnVx16LTDz7LmM@mail.python.org>

http://hg.python.org/cpython/rev/b6ebc726d5fe
changeset:   84459:b6ebc726d5fe
parent:      84456:7be081fa8db1
parent:      84458:27eb350d5056
user:        Ronald Oussoren <ronaldoussoren at mac.com>
date:        Sat Jul 06 13:25:44 2013 +0200
summary:
  (3.3->default) Issue #12990: The "Python Launcher" on OSX could not launch python scripts that have paths that include wide characters.

files:
  Mac/PythonLauncher/MyDocument.m |  2 +-
  Misc/NEWS                       |  6 ++++++
  2 files changed, 7 insertions(+), 1 deletions(-)


diff --git a/Mac/PythonLauncher/MyDocument.m b/Mac/PythonLauncher/MyDocument.m
--- a/Mac/PythonLauncher/MyDocument.m
+++ b/Mac/PythonLauncher/MyDocument.m
@@ -76,7 +76,7 @@
     const char *cmdline;
     int sts;
     
-     cmdline = [[settings commandLineForScript: script] cString];
+     cmdline = [[settings commandLineForScript: script] UTF8String];
    if ([settings with_terminal]) {
         sts = doscript(cmdline);
     } else {
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -599,6 +599,12 @@
 - Issue #15172: Document NASM 2.10+ as requirement for building OpenSSL 1.0.1
   on Windows.
 
+Tools/Demos
+-----------
+
+- Issue #12990: The "Python Launcher" on OSX could not launch python scripts
+  that have paths that include wide characters.
+
 What's New in Python 3.3.1 release candidate 1?
 ===============================================
 

-- 
Repository URL: http://hg.python.org/cpython

From rdmurray at bitdance.com  Sat Jul  6 13:59:07 2013
From: rdmurray at bitdance.com (R. David Murray)
Date: Sat, 06 Jul 2013 07:59:07 -0400
Subject: [Python-checkins]
	=?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE3ODYw?=
	=?utf-8?q?=3A_explicitly_mention_that_std*_streams_are_opened_in_b?=
	=?utf-8?q?inary_mode_by?=
In-Reply-To: <3bnQwH1z6Vz7LjW@mail.python.org>
References: <3bnQwH1z6Vz7LjW@mail.python.org>
Message-ID: <20130706115908.2072B250185@webabinitio.net>

On Sat, 06 Jul 2013 10:25:19 +0200, ronald.oussoren <python-checkins at python.org> wrote:
> http://hg.python.org/cpython/rev/a2c2ffa1a41c
> changeset:   84453:a2c2ffa1a41c
> branch:      3.3
> parent:      84449:df79735b21c1
> user:        Ronald Oussoren <ronaldoussoren at mac.com>
> date:        Sat Jul 06 10:23:59 2013 +0200
> summary:
>   Issue #17860: explicitly mention that std* streams are opened in binary mode by default.
> 
> The documentation does mention that the streams are opened in text mode
> when univeral_newlines is true, but not that that they are opened in
> binary mode when that argument is false and that seems to confuse at
> least some users.
> 
> files:
>   Doc/library/subprocess.rst |  6 ++++--
>   1 files changed, 4 insertions(+), 2 deletions(-)
> 
> 
> diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
> --- a/Doc/library/subprocess.rst
> +++ b/Doc/library/subprocess.rst
> @@ -293,7 +293,8 @@
>     If *universal_newlines* is ``True``, the file objects *stdin*, *stdout* and
>     *stderr* will be opened as text streams in :term:`universal newlines` mode
>     using the encoding returned by :func:`locale.getpreferredencoding(False)
> -   <locale.getpreferredencoding>`.  For *stdin*, line ending characters
> +   <locale.getpreferredencoding>`, otherwise these streams will be opened
> +   as binary streams.  For *stdin*, line ending characters
>     ``'\n'`` in the input will be converted to the default line separator
>     :data:`os.linesep`.  For *stdout* and *stderr*, all line endings in the
>     output will be converted to ``'\n'``.  For more information see the

IMO, either the default should be mentioned first, or the default
should be mentioned in a parenthetical.  Otherwise it sounds like
newline translation is being done in both modes.  Logically that makes
no sense, so the above construction will likely lead to, at a minimum,
an interruption in the flow for the reader, and at worse even more
confusion than not mentioning it at all.

--David

From ronaldoussoren at mac.com  Sat Jul  6 14:09:38 2013
From: ronaldoussoren at mac.com (Ronald Oussoren)
Date: Sat, 6 Jul 2013 14:09:38 +0200
Subject: [Python-checkins] [Python-Dev]	 cpython (3.3): Issue #17860:
	explicitly mention that std* streams are opened in binary mode by
In-Reply-To: <20130706115908.2072B250185@webabinitio.net>
References: <3bnQwH1z6Vz7LjW@mail.python.org>
	<20130706115908.2072B250185@webabinitio.net>
Message-ID: <A6E4F77E-3D82-437B-B37F-DEA6E0719A55@mac.com>


On 6 Jul, 2013, at 13:59, R. David Murray <rdmurray at bitdance.com> wrote:

> On Sat, 06 Jul 2013 10:25:19 +0200, ronald.oussoren <python-checkins at python.org> wrote:
>> http://hg.python.org/cpython/rev/a2c2ffa1a41c
>> changeset:   84453:a2c2ffa1a41c
>> branch:      3.3
>> parent:      84449:df79735b21c1
>> user:        Ronald Oussoren <ronaldoussoren at mac.com>
>> date:        Sat Jul 06 10:23:59 2013 +0200
>> summary:
>>  Issue #17860: explicitly mention that std* streams are opened in binary mode by default.
>> 
>> The documentation does mention that the streams are opened in text mode
>> when univeral_newlines is true, but not that that they are opened in
>> binary mode when that argument is false and that seems to confuse at
>> least some users.
>> 
>> files:
>>  Doc/library/subprocess.rst |  6 ++++--
>>  1 files changed, 4 insertions(+), 2 deletions(-)
>> 
>> 
>> diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
>> --- a/Doc/library/subprocess.rst
>> +++ b/Doc/library/subprocess.rst
>> @@ -293,7 +293,8 @@
>>    If *universal_newlines* is ``True``, the file objects *stdin*, *stdout* and
>>    *stderr* will be opened as text streams in :term:`universal newlines` mode
>>    using the encoding returned by :func:`locale.getpreferredencoding(False)
>> -   <locale.getpreferredencoding>`.  For *stdin*, line ending characters
>> +   <locale.getpreferredencoding>`, otherwise these streams will be opened
>> +   as binary streams.  For *stdin*, line ending characters
>>    ``'\n'`` in the input will be converted to the default line separator
>>    :data:`os.linesep`.  For *stdout* and *stderr*, all line endings in the
>>    output will be converted to ``'\n'``.  For more information see the
> 
> IMO, either the default should be mentioned first, or the default
> should be mentioned in a parenthetical.  Otherwise it sounds like
> newline translation is being done in both modes.  Logically that makes
> no sense, so the above construction will likely lead to, at a minimum,
> an interruption in the flow for the reader, and at worse even more
> confusion than not mentioning it at all.

You've got a point there. Converting the next text (", otherwise ...") to a parententical
seems to be the cleanest fix, creating a separate sentence for the ``False`` case introduces
duplication unless I restructure the text.

Ronald

> 
> --David
> _______________________________________________
> Python-Dev mailing list
> Python-Dev at python.org
> http://mail.python.org/mailman/listinfo/python-dev
> Unsubscribe: http://mail.python.org/mailman/options/python-dev/ronaldoussoren%40mac.com


From python-checkins at python.org  Sat Jul  6 14:31:47 2013
From: python-checkins at python.org (victor.stinner)
Date: Sat,  6 Jul 2013 14:31:47 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_446=3A_enhance_the_Ration?=
 =?utf-8?q?ale_to_take_into_account_Cameron_Simpson=27s_remarks?=
Message-ID: <3bnXNg74PRz7Ln1@mail.python.org>

http://hg.python.org/peps/rev/425f831fddf7
changeset:   4978:425f831fddf7
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sat Jul 06 14:27:23 2013 +0200
summary:
  PEP 446: enhance the Rationale to take into account Cameron Simpson's remarks

 * mention the name of the close-on-exec flag: HANDLE_FLAG_INHERIT, O_CLOEXEC
 * mention the name of the blocking flag: O_NONBLOCK
 * explain that file attributes are duplicated at fork

files:
  pep-0446.txt |  28 +++++++++++++++++++++-------
  1 files changed, 21 insertions(+), 7 deletions(-)


diff --git a/pep-0446.txt b/pep-0446.txt
--- a/pep-0446.txt
+++ b/pep-0446.txt
@@ -27,14 +27,25 @@
 on each file descriptor using a *close-on-exec* flag. By default, the
 close-on-exec flag is not set.
 
-On Windows, file descriptors are not inherited if the
-``bInheritHandles`` parameter of the ``CreateProcess()`` function is
-``FALSE``, even if the close-on-exec flag is not set.
+On Windows, the close-on-exec flag is ``HANDLE_FLAG_INHERIT``. File
+descriptors are not inherited if the ``bInheritHandles`` parameter of
+the ``CreateProcess()`` function is ``FALSE``, even if the
+``HANDLE_FLAG_INHERIT`` flag is set. If ``bInheritHandles`` is ``TRUE``,
+only file descriptors with ``HANDLE_FLAG_INHERIT`` flag set are
+inherited, others are not.
 
-On UNIX, file descriptors with the close-and-exec flag set are closed at
-the execution of a new program (ex: when calling ``execv()``). The flag
-has no effect on ``fork()``, all file descriptors are inherited by the
-child process.
+On UNIX, the close-on-exec flag is ``O_CLOEXEC``. File descriptors with
+the ``O_CLOEXEC`` flag set are closed at the execution of a new program
+(ex: when calling ``execv()``).
+
+The ``O_CLOEXEC`` flag has no effect on ``fork()``, all file descriptors
+are inherited by the child process. Futhermore, most properties file
+descriptors are shared between the parent and the child processes,
+except file attributes which are duplicated (``O_CLOEXEC`` is the only
+file attribute).  Setting ``O_CLOEXEC`` flag of a file descriptor in the
+child process does not change the ``O_CLOEXEC`` flag of the file
+descriptor in the parent process.
+
 
 Issues of the inheritance of file descriptors
 ---------------------------------------------
@@ -62,6 +73,9 @@
 By default, newly created sockets are blocking. Setting the non-blocking
 mode requires additional system calls.
 
+On UNIX, the blocking flag is ``O_NONBLOCK``: a pipe and a socket are
+non-blocking if the ``O_NONBLOCK`` flag is set.
+
 
 Setting flags at the creation of the file descriptor
 ----------------------------------------------------

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Sat Jul  6 15:08:51 2013
From: python-checkins at python.org (florent.xicluna)
Date: Sat,  6 Jul 2013 15:08:51 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogdGVzdF9mdHBsaWI6?=
 =?utf-8?q?_silence_a_BytesWarning_when_checking_TypeError?=
Message-ID: <3bnYCR6CJlzRy8@mail.python.org>

http://hg.python.org/cpython/rev/851254748c6b
changeset:   84460:851254748c6b
branch:      3.3
parent:      84458:27eb350d5056
user:        Florent Xicluna <florent.xicluna at gmail.com>
date:        Sat Jul 06 15:08:21 2013 +0200
summary:
  test_ftplib: silence a BytesWarning when checking TypeError

files:
  Lib/test/test_ftplib.py |  3 ++-
  1 files changed, 2 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py
--- a/Lib/test/test_ftplib.py
+++ b/Lib/test/test_ftplib.py
@@ -590,7 +590,8 @@
 
         f = io.StringIO(RETR_DATA.replace('\r\n', '\n'))
         # storlines() expects a binary file, not a text file
-        self.assertRaises(TypeError, self.client.storlines, 'stor foo', f)
+        with support.check_warnings(('', BytesWarning), quiet=True):
+            self.assertRaises(TypeError, self.client.storlines, 'stor foo', f)
 
     def test_nlst(self):
         self.client.nlst()

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 15:08:53 2013
From: python-checkins at python.org (florent.xicluna)
Date: Sat,  6 Jul 2013 15:08:53 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_test=5Fftplib=3A_silence_a_BytesWarning_when_checking_Ty?=
 =?utf-8?q?peError?=
Message-ID: <3bnYCT1PFMzRy8@mail.python.org>

http://hg.python.org/cpython/rev/dbd301063e59
changeset:   84461:dbd301063e59
parent:      84459:b6ebc726d5fe
parent:      84460:851254748c6b
user:        Florent Xicluna <florent.xicluna at gmail.com>
date:        Sat Jul 06 15:08:29 2013 +0200
summary:
  test_ftplib: silence a BytesWarning when checking TypeError

files:
  Lib/test/test_ftplib.py |  3 ++-
  1 files changed, 2 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py
--- a/Lib/test/test_ftplib.py
+++ b/Lib/test/test_ftplib.py
@@ -591,7 +591,8 @@
 
         f = io.StringIO(RETR_DATA.replace('\r\n', '\n'))
         # storlines() expects a binary file, not a text file
-        self.assertRaises(TypeError, self.client.storlines, 'stor foo', f)
+        with support.check_warnings(('', BytesWarning), quiet=True):
+            self.assertRaises(TypeError, self.client.storlines, 'stor foo', f)
 
     def test_nlst(self):
         self.client.nlst()

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 17:17:57 2013
From: python-checkins at python.org (ezio.melotti)
Date: Sat,  6 Jul 2013 17:17:57 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE4MzgwOiBwYXNz?=
 =?utf-8?q?_regex_flags_to_the_right_argument=2E__Patch_by_Valentina?=
Message-ID: <3bnc4P5ZhPzSCt@mail.python.org>

http://hg.python.org/cpython/rev/c8fd1351c840
changeset:   84462:c8fd1351c840
branch:      3.3
parent:      84460:851254748c6b
user:        Ezio Melotti <ezio.melotti at gmail.com>
date:        Sat Jul 06 17:16:04 2013 +0200
summary:
  #18380: pass regex flags to the right argument.  Patch by Valentina Mukhamedzhanova.

files:
  Lib/email/quoprimime.py           |  2 +-
  Lib/test/test_email/test_email.py |  4 ++++
  Misc/ACKS                         |  1 +
  3 files changed, 6 insertions(+), 1 deletions(-)


diff --git a/Lib/email/quoprimime.py b/Lib/email/quoprimime.py
--- a/Lib/email/quoprimime.py
+++ b/Lib/email/quoprimime.py
@@ -319,4 +319,4 @@
     the high level email.header class for that functionality.
     """
     s = s.replace('_', ' ')
-    return re.sub(r'=[a-fA-F0-9]{2}', _unquote_match, s, re.ASCII)
+    return re.sub(r'=[a-fA-F0-9]{2}', _unquote_match, s, flags=re.ASCII)
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -4048,6 +4048,10 @@
     def test_header_decode_non_ascii(self):
         self._test_header_decode('hello=C7there', 'hello\xc7there')
 
+    def test_header_decode_re_bug_18380(self):
+        # Issue 18380: Call re.sub with a positional argument for flags in the wrong position
+        self.assertEqual(quoprimime.header_decode('=30' * 257), '0' * 257)
+
     def _test_decode(self, encoded, expected_decoded, eol=None):
         if eol is None:
             decoded = quoprimime.decode(encoded)
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -849,6 +849,7 @@
 Pablo Mouzo
 Mher Movsisyan
 Ruslan Mstoi
+Valentina Mukhamedzhanova
 Michael Mulich
 Sape Mullender
 Sjoerd Mullender

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 17:17:59 2013
From: python-checkins at python.org (ezio.melotti)
Date: Sat,  6 Jul 2013 17:17:59 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?b?KTogIzE4MzgwOiBtZXJnZSB3aXRoIDMuMy4=?=
Message-ID: <3bnc4R0pNNz7LjZ@mail.python.org>

http://hg.python.org/cpython/rev/6e23ce14c3c6
changeset:   84463:6e23ce14c3c6
parent:      84461:dbd301063e59
parent:      84462:c8fd1351c840
user:        Ezio Melotti <ezio.melotti at gmail.com>
date:        Sat Jul 06 17:17:45 2013 +0200
summary:
  #18380: merge with 3.3.

files:
  Lib/email/quoprimime.py           |  2 +-
  Lib/test/test_email/test_email.py |  4 ++++
  Misc/ACKS                         |  1 +
  3 files changed, 6 insertions(+), 1 deletions(-)


diff --git a/Lib/email/quoprimime.py b/Lib/email/quoprimime.py
--- a/Lib/email/quoprimime.py
+++ b/Lib/email/quoprimime.py
@@ -319,4 +319,4 @@
     the high level email.header class for that functionality.
     """
     s = s.replace('_', ' ')
-    return re.sub(r'=[a-fA-F0-9]{2}', _unquote_match, s, re.ASCII)
+    return re.sub(r'=[a-fA-F0-9]{2}', _unquote_match, s, flags=re.ASCII)
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -4048,6 +4048,10 @@
     def test_header_decode_non_ascii(self):
         self._test_header_decode('hello=C7there', 'hello\xc7there')
 
+    def test_header_decode_re_bug_18380(self):
+        # Issue 18380: Call re.sub with a positional argument for flags in the wrong position
+        self.assertEqual(quoprimime.header_decode('=30' * 257), '0' * 257)
+
     def _test_decode(self, encoded, expected_decoded, eol=None):
         if eol is None:
             decoded = quoprimime.decode(encoded)
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -871,6 +871,7 @@
 Pablo Mouzo
 Mher Movsisyan
 Ruslan Mstoi
+Valentina Mukhamedzhanova
 Michael Mulich
 Sape Mullender
 Sjoerd Mullender

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 20:48:29 2013
From: python-checkins at python.org (brett.cannon)
Date: Sat,  6 Jul 2013 20:48:29 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318364=3A_Stop_usi?=
 =?utf-8?q?ng_the_ImportError=2E=5Fnot=5Ffound_hack=2E?=
Message-ID: <3bnhlK320Wz7Lk0@mail.python.org>

http://hg.python.org/cpython/rev/0d590683c8d1
changeset:   84464:0d590683c8d1
user:        Brett Cannon <brett at python.org>
date:        Sat Jul 06 14:48:18 2013 -0400
summary:
  Issue #18364: Stop using the ImportError._not_found hack.

The private attribute was leaking out of importlib and led to at least
one person noticing it. Switch to another hack which won't leak
outside of importlib and is nearly as robust.

files:
  Lib/importlib/_bootstrap.py |    13 +-
  Python/importlib.h          |  7136 +++++++++++-----------
  2 files changed, 3572 insertions(+), 3577 deletions(-)


diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -1536,7 +1536,8 @@
         raise ValueError("Empty module name")
 
 
-_ERR_MSG = 'No module named {!r}'
+_ERR_MSG_PREFIX = 'No module named '
+_ERR_MSG = _ERR_MSG_PREFIX + '{!r}'
 
 def _find_and_load_unlocked(name, import_):
     path = None
@@ -1556,11 +1557,7 @@
             raise ImportError(msg, name=name)
     loader = _find_module(name, path)
     if loader is None:
-        exc = ImportError(_ERR_MSG.format(name), name=name)
-        # TODO(brett): switch to a proper ModuleNotFound exception in Python
-        # 3.4.
-        exc._not_found = True
-        raise exc
+        raise ImportError(_ERR_MSG.format(name), name=name)
     elif name not in sys.modules:
         # The parent import may have already imported this module.
         loader.load_module(name)
@@ -1650,9 +1647,7 @@
                     # Backwards-compatibility dictates we ignore failed
                     # imports triggered by fromlist for modules that don't
                     # exist.
-                    # TODO(brett): In Python 3.4, have import raise
-                    #   ModuleNotFound and catch that.
-                    if getattr(exc, '_not_found', False):
+                    if str(exc).startswith(_ERR_MSG_PREFIX):
                         if exc.name == from_name:
                             continue
                     raise
diff --git a/Python/importlib.h b/Python/importlib.h
--- a/Python/importlib.h
+++ b/Python/importlib.h
[stripped]

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 21:07:28 2013
From: python-checkins at python.org (raymond.hettinger)
Date: Sat,  6 Jul 2013 21:07:28 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Remove_unnecessary_branche?=
 =?utf-8?q?s_from_count=28=29_and_reverse=28=29=2E?=
Message-ID: <3bnj9D3bTcz7Ljy@mail.python.org>

http://hg.python.org/cpython/rev/49a9c734304d
changeset:   84465:49a9c734304d
user:        Raymond Hettinger <python at rcn.com>
date:        Sat Jul 06 09:07:06 2013 -1000
summary:
  Remove unnecessary branches from count() and reverse().

files:
  Modules/_collectionsmodule.c |  9 +++------
  1 files changed, 3 insertions(+), 6 deletions(-)


diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -551,6 +551,8 @@
     for (i=0 ; i<n ; i++) {
         /* Validate that pointers haven't met in the middle */
         assert(leftblock != rightblock || leftindex < rightindex);
+        assert(leftblock != NULL);
+        assert(rightblock != NULL);
 
         /* Swap */
         tmp = leftblock->data[leftindex];
@@ -560,8 +562,6 @@
         /* Advance left block/index pair */
         leftindex++;
         if (leftindex == BLOCKLEN) {
-            if (leftblock->rightlink == NULL)
-                break;
             leftblock = leftblock->rightlink;
             leftindex = 0;
         }
@@ -569,8 +569,6 @@
         /* Step backwards with the right block/index pair */
         rightindex--;
         if (rightindex == -1) {
-            if (rightblock->leftlink == NULL)
-                break;
             rightblock = rightblock->leftlink;
             rightindex = BLOCKLEN - 1;
         }
@@ -594,6 +592,7 @@
     int cmp;
 
     for (i=0 ; i<n ; i++) {
+        assert(leftblock != NULL);
         item = leftblock->data[leftindex];
         cmp = PyObject_RichCompareBool(item, v, Py_EQ);
         if (cmp > 0)
@@ -610,8 +609,6 @@
         /* Advance left block/index pair */
         leftindex++;
         if (leftindex == BLOCKLEN) {
-            if (leftblock->rightlink == NULL)  /* can occur when i==n-1 */
-                break;
             leftblock = leftblock->rightlink;
             leftindex = 0;
         }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sat Jul  6 22:39:00 2013
From: python-checkins at python.org (daniel.holth)
Date: Sat,  6 Jul 2013 22:39:00 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_pep-0426=3A_add_generator_fie?=
	=?utf-8?q?ld?=
Message-ID: <3bnlBr42QhzSfh@mail.python.org>

http://hg.python.org/peps/rev/cc4da4be3838
changeset:   4979:cc4da4be3838
user:        Daniel Holth <dholth at fastmail.fm>
date:        Sat Jul 06 16:38:22 2013 -0400
summary:
  pep-0426: add generator field

files:
  pep-0426.txt                |  12 ++++++++++++
  pep-0426/pymeta-schema.json |   5 +++++
  2 files changed, 17 insertions(+), 0 deletions(-)


diff --git a/pep-0426.txt b/pep-0426.txt
--- a/pep-0426.txt
+++ b/pep-0426.txt
@@ -415,6 +415,7 @@
 fields:
 
 * ``metadata_version``
+* ``generator``
 * ``name``
 * ``version``
 * ``source_label``
@@ -502,6 +503,17 @@
     "metadata_version": "2.0"
 
 
+Generator
+---------
+
+Name (and optional version) of the program that generated the file,
+if any.  A manually produced file would omit this field.
+
+Example::
+
+    "generator": "setuptools (0.8)"
+
+
 Name
 ----
 
diff --git a/pep-0426/pymeta-schema.json b/pep-0426/pymeta-schema.json
--- a/pep-0426/pymeta-schema.json
+++ b/pep-0426/pymeta-schema.json
@@ -9,6 +9,11 @@
       "type": "string",
       "pattern": "^(\\d+(\\.\\d+)*)$"
     },
+    "generator": {
+      "description": "Name and version of the program that produced this file.",
+      "type": "string",
+      "pattern": "^[0-9A-Za-z]([0-9A-Za-z_.-]*[0-9A-Za-z])( \\((\\d+(\\.\\d+)*)((a|b|c|rc)(\\d+))?(\\.(post)(\\d+))?(\\.(dev)(\\d+))\\))?$"
+    },
     "name": {
       "description": "The name of the distribution.",
       "type": "string",

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Sat Jul  6 22:49:42 2013
From: python-checkins at python.org (antoine.pitrou)
Date: Sat,  6 Jul 2013 22:49:42 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_Some_tweaks?=
Message-ID: <3bnlRB34LNzNF4@mail.python.org>

http://hg.python.org/peps/rev/eddd0a587bd9
changeset:   4980:eddd0a587bd9
user:        Antoine Pitrou <solipsis at pitrou.net>
date:        Sat Jul 06 22:48:32 2013 +0200
summary:
  Some tweaks

files:
  pep-0445.txt |  128 +++++++++++++++++++-------------------
  1 files changed, 63 insertions(+), 65 deletions(-)


diff --git a/pep-0445.txt b/pep-0445.txt
--- a/pep-0445.txt
+++ b/pep-0445.txt
@@ -13,7 +13,9 @@
 ========
 
 This PEP proposes new Application Programming Interfaces (API) to customize
-Python memory allocators.
+Python memory allocators.  The only implementation required to conform to
+this PEP is CPython, but other implementations may choose to be compatible,
+or to re-use a similar scheme.
 
 
 Rationale
@@ -123,11 +125,12 @@
   ``PY_SSIZE_T_MAX``. The check is done before calling the inner
   function.
 
-The *pymalloc* allocator is optimized for objects smaller than 512 bytes
-with a short lifetime. It uses memory mappings with a fixed size of 256
-KB called "arenas".
+.. note::
+    The *pymalloc* allocator is optimized for objects smaller than 512 bytes
+    with a short lifetime. It uses memory mappings with a fixed size of 256
+    KB called "arenas".
 
-Default allocators:
+Here is how the allocators are set up by default:
 
 * ``PYMEM_DOMAIN_RAW``, ``PYMEM_DOMAIN_MEM``: ``malloc()``,
   ``realloc()`` and ``free()``; call ``malloc(1)`` when requesting zero
@@ -155,11 +158,11 @@
 In Python 3.3, the checks are installed by replacing ``PyMem_Malloc()``,
 ``PyMem_Realloc()``, ``PyMem_Free()``, ``PyObject_Malloc()``,
 ``PyObject_Realloc()`` and ``PyObject_Free()`` using macros. The new
-allocator allocates a larger buffer and write a pattern to detect buffer
-underflow, buffer overflow and use after free (fill the buffer with the
-pattern ``0xDB``). It uses the original ``PyObject_Malloc()``
+allocator allocates a larger buffer and writes a pattern to detect buffer
+underflow, buffer overflow and use after free (by filling the buffer with
+the byte ``0xDB``). It uses the original ``PyObject_Malloc()``
 function to allocate memory. So ``PyMem_Malloc()`` and
-``PyMem_Realloc()`` call indirectly ``PyObject_Malloc()`` and
+``PyMem_Realloc()`` indirectly call``PyObject_Malloc()`` and
 ``PyObject_Realloc()``.
 
 This PEP redesigns the debug checks as hooks on the existing allocators
@@ -179,7 +182,7 @@
   => ``_PyObject_Free()``
 
 As a result, ``PyMem_Malloc()`` and ``PyMem_Realloc()`` now call
-``malloc()`` and ``realloc()`` in release mode and in debug mode,
+``malloc()`` and ``realloc()`` in both release mode and debug mode,
 instead of calling ``PyObject_Malloc()`` and ``PyObject_Realloc()`` in
 debug mode.
 
@@ -199,19 +202,15 @@
 Direct calls to ``malloc()`` are replaced with ``PyMem_Malloc()``, or
 ``PyMem_RawMalloc()`` if the GIL is not held.
 
-Configure external libraries like zlib or OpenSSL to allocate memory
+External libraries like zlib or OpenSSL can be configured to allocate memory
 using ``PyMem_Malloc()`` or ``PyMem_RawMalloc()``. If the allocator of a
-library can only be replaced globally, the allocator is not replaced if
-Python is embedded in an application.
+library can only be replaced globally (rather than on an object-by-object
+basis), it shouldn't be replaced when Python is embedded in an application.
 
 For the "track memory usage" use case, it is important to track memory
 allocated in external libraries to have accurate reports, because these
-allocations can be large (can raise a ``MemoryError`` exception).
-
-If an hook is used to the track memory usage, the memory allocated by
-direct calls to ``malloc()`` will not be tracked. Remaining ``malloc()``
-in external libraries like OpenSSL or bz2 can allocate large memory
-blocks and so would be missed in memory usage reports.
+allocations can be large (e.g. they can raise a ``MemoryError`` exception)
+and would otherwise be missed in memory usage reports.
 
 
 Examples
@@ -282,9 +281,9 @@
 Use case 2: Replace Memory Allocators, override pymalloc
 --------------------------------------------------------
 
-If your allocator is optimized for allocations of objects smaller than
-512 bytes with a short lifetime, pymalloc can be overriden (replace
-``PyObject_Malloc()``).
+If you have a dedicated allocator optimized for allocations of objects
+smaller than 512 bytes with a short lifetime, pymalloc can be overriden
+(replace ``PyObject_Malloc()``).
 
 Dummy example wasting 2 bytes per memory block::
 
@@ -420,12 +419,8 @@
 More specific functions to get/set memory allocators
 ----------------------------------------------------
 
-Replace the 2 functions:
-
-* ``void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)``
-* ``void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)``
-
-with:
+It was originally proposed a larger set of C API functions, with one pair
+of functions for each allocator domain:
 
 * ``void PyMem_GetRawAllocator(PyMemAllocator *allocator)``
 * ``void PyMem_GetAllocator(PyMemAllocator *allocator)``
@@ -442,8 +437,8 @@
 Make PyMem_Malloc() reuse PyMem_RawMalloc() by default
 ------------------------------------------------------
 
-If ``PyMem_Malloc()`` would call ``PyMem_RawMalloc()`` by default,
-calling ``PyMem_SetAllocator(PYMEM_DOMAIN_RAW, alloc)`` would also also
+If ``PyMem_Malloc()`` called ``PyMem_RawMalloc()`` by default,
+calling ``PyMem_SetAllocator(PYMEM_DOMAIN_RAW, alloc)`` would also
 patch ``PyMem_Malloc()`` indirectly.
 
 This alternative was rejected because ``PyMem_SetAllocator()`` would
@@ -454,17 +449,17 @@
 Add a new PYDEBUGMALLOC environment variable
 --------------------------------------------
 
-Add a new ``PYDEBUGMALLOC`` environment variable to enable debug checks
-on memory block allocators. The environment variable replaces the new
-function ``PyMem_SetupDebugHooks()`` which is not needed anymore.
-Another advantage is to allow to enable debug checks even in release
-mode: debug checks are always compiled, but only enabled when the
-environment variable is present and non-empty.
+It was proposed to add a new ``PYDEBUGMALLOC`` environment variable to
+enable debug checks on memory block allocators. It would have had the same
+effect as calling the ``PyMem_SetupDebugHooks()``, without the need
+to write any C code.  Another advantage is to allow to enable debug checks
+even in release mode: debug checks would always be compiled in, but only
+enabled when the environment variable is present and non-empty.
 
 This alternative was rejected because a new environment variable would
-make the Python initialization even more complex. The `PEP 432
-<http://www.python.org/dev/peps/pep-0432/>`_ tries to simply the CPython
-startup sequence.
+make Python initialization even more complex. `PEP 432
+<http://www.python.org/dev/peps/pep-0432/>`_ tries to simplify the
+CPython startup sequence.
 
 
 Use macros to get customizable allocators
@@ -474,7 +469,7 @@
 allocators would be an optional feature enabled by a configuration
 option or by macros.
 
-This alternative was rejected because the usage of macros implies having
+This alternative was rejected because the use of macros implies having
 to recompile extensions modules to use the new allocator and allocator
 hooks. Not having to recompile Python nor extension modules makes debug
 hooks easier to use in practice.
@@ -518,12 +513,12 @@
 
 The GC allocator functions would also have to be patched. For example,
 ``_PyObject_GC_Malloc()`` is used in many C functions and so objects of
-differenet types would have the same allocation location.
+different types would have the same allocation location.
 
 This alternative was rejected because passing a filename and a line
 number to each allocator makes the API more complex: pass 3 new
 arguments (ctx, filename, lineno) to each allocator function, instead of
-just a context argument (ctx). Having to modify also GC allocator
+just a context argument (ctx). Having to also modify GC allocator
 functions adds too much complexity for a little gain.
 
 
@@ -531,23 +526,25 @@
 -----------------------
 
 In Python 3.3, when Python is compiled in debug mode, ``PyMem_Malloc()``
-calls indirectly ``PyObject_Malloc()`` which requires the GIL to be
-held.  That's why ``PyMem_Malloc()`` must be called with the GIL held.
+indirectly calls ``PyObject_Malloc()`` which requires the GIL to be
+held (it isn't thread-safe).  That's why ``PyMem_Malloc()`` must be called
+with the GIL held.
 
-This PEP changes ``PyMem_Malloc()``: it now always call ``malloc()``.
-The "GIL must be held" restriction could be removed from
-``PyMem_Malloc()``.
+This PEP changes ``PyMem_Malloc()``: it now always calls ``malloc()``
+rather than ``PyObject_Malloc()``.  The "GIL must be held" restriction
+could therefore be removed from ``PyMem_Malloc()``.
 
 This alternative was rejected because allowing to call
 ``PyMem_Malloc()`` without holding the GIL can break applications
 which setup their own allocators or allocator hooks.  Holding the GIL is
-convinient to develop a custom allocator: no need to care of other
-threads. It is also convinient for a debug allocator hook: Python
-internal objects can be safetly inspected.
+convenient to develop a custom allocator: no need to care about other
+threads.  It is also convenient for a debug allocator hook: Python
+objects can be safely inspected, and the C API may be used for reporting.
 
-Calling ``PyGILState_Ensure()`` in
-a memory allocator has unexpected behaviour, especially at Python
-startup and at creation of a new Python thread state.
+Moreover, calling ``PyGILState_Ensure()`` in a memory allocator has
+unexpected behaviour, especially at Python startup and when creating of a
+new Python thread state.  It is better to free custom allocators of
+the responsibility of acquiring the GIL.
 
 
 Don't add PyMem_RawMalloc()
@@ -566,13 +563,14 @@
 for accurate reports of the memory usage. When a debug hook is used to
 track the memory usage, the memory allocated by direct calls to
 ``malloc()`` cannot be tracked. ``PyMem_RawMalloc()`` can be hooked and
-so all the memory allocated by Python can be tracked.
+so all the memory allocated by Python can be tracked, including
+memory allocated without holding the GIL.
 
 
-Use existing debug tools to analyze the memory
+Use existing debug tools to analyze memory use
 ----------------------------------------------
 
-There are many existing debug tools to analyze the memory. Some
+There are many existing debug tools to analyze memory use. Some
 examples: `Valgrind <http://valgrind.org/>`_, `Purify
 <http://ibm.com/software/awdtools/purify/>`_, `Clang AddressSanitizer
 <http://code.google.com/p/address-sanitizer/>`_, `failmalloc
@@ -580,14 +578,14 @@
 
 The problem is to retrieve the Python object related to a memory pointer
 to read its type and/or its content. Another issue is to retrieve the
-location of the memory allocation: the C backtrace is usually useless
+source of the memory allocation: the C backtrace is usually useless
 (same reasoning than macros using ``__FILE__`` and ``__LINE__``, see
 `Pass the C filename and line number`_), the Python filename and line
 number (or even the Python traceback) is more useful.
 
 This alternative was rejected because classic tools are unable to
 introspect Python internals to collect such information. Being able to
-setup a hook on allocators called with the GIL held allow to collect a
+setup a hook on allocators called with the GIL held allows to collect a
 lot of useful data from Python internals.
 
 
@@ -606,7 +604,7 @@
 On Windows, this function can be implemented using ``_msize()`` and
 ``VirtualQuery()``.
 
-The function can be used to implement an hook tracking the memory usage.
+The function can be used to implement a hook tracking the memory usage.
 The ``free()`` method of an allocator only gets the address of a memory
 block, whereas the size of the memory block is required to update the
 memory usage.
@@ -614,9 +612,9 @@
 The additional ``msize()`` function was rejected because only few
 platforms implement it. For example, Linux with the GNU libc does not
 provide a function to get the size of a memory block. ``msize()`` is not
-currently used in the Python source code. The function is only used to
-track the memory usage, and makes the API more complex. A debug hook can
-implement the function internally, there is no need to add it to
+currently used in the Python source code. The function would only be
+used to track memory use, and make the API more complex. A debug hook
+can implement the function internally, there is no need to add it to
 ``PyMemAllocator`` and ``PyObjectArenaAllocator`` structures.
 
 
@@ -733,9 +731,9 @@
   <https://developer.gnome.org/glib/unstable/glib-Memory-Slices.html>`_:
   efficient way to allocate groups of equal-sized chunks of memory
 
-This PEP permits to choose exactly which memory allocator is used for your
-application depending on its usage of the memory (number of allocation, size of
-allocations, lifetime of objects, etc.).
+This PEP allows to choose exactly which memory allocator is used for your
+application depending on its usage of the memory (number of allocations,
+size of allocations, lifetime of objects, etc.).
 
 
 Links

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Sat Jul  6 22:58:08 2013
From: python-checkins at python.org (antoine.pitrou)
Date: Sat,  6 Jul 2013 22:58:08 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_Mark_accepted?=
Message-ID: <3bnlcw22XvzSL9@mail.python.org>

http://hg.python.org/peps/rev/7558862588d6
changeset:   4981:7558862588d6
user:        Antoine Pitrou <solipsis at pitrou.net>
date:        Sat Jul 06 22:56:39 2013 +0200
summary:
  Mark accepted

files:
  pep-0445.txt |  3 ++-
  1 files changed, 2 insertions(+), 1 deletions(-)


diff --git a/pep-0445.txt b/pep-0445.txt
--- a/pep-0445.txt
+++ b/pep-0445.txt
@@ -3,11 +3,12 @@
 Version: $Revision$
 Last-Modified: $Date$
 Author: Victor Stinner <victor.stinner at gmail.com>
-Status: Draft
+Status: Accepted
 Type: Standards Track
 Content-Type: text/x-rst
 Created: 15-june-2013
 Python-Version: 3.4
+Resolution: http://mail.python.org/pipermail/python-dev/2013-July/127222.html
 
 Abstract
 ========

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Sat Jul  6 22:58:09 2013
From: python-checkins at python.org (antoine.pitrou)
Date: Sat,  6 Jul 2013 22:58:09 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_=2E=2E=2E_and_mark_myself_as_?=
	=?utf-8?q?delegate?=
Message-ID: <3bnlcx3x8XzSHQ@mail.python.org>

http://hg.python.org/peps/rev/677b773f44f3
changeset:   4982:677b773f44f3
user:        Antoine Pitrou <solipsis at pitrou.net>
date:        Sat Jul 06 22:57:08 2013 +0200
summary:
  ... and mark myself as delegate

files:
  pep-0445.txt |  1 +
  1 files changed, 1 insertions(+), 0 deletions(-)


diff --git a/pep-0445.txt b/pep-0445.txt
--- a/pep-0445.txt
+++ b/pep-0445.txt
@@ -3,6 +3,7 @@
 Version: $Revision$
 Last-Modified: $Date$
 Author: Victor Stinner <victor.stinner at gmail.com>
+BDFL-Delegate: Antoine Pitrou <solipsis at pitrou.net>
 Status: Accepted
 Type: Standards Track
 Content-Type: text/x-rst

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Sun Jul  7 00:02:47 2013
From: python-checkins at python.org (raymond.hettinger)
Date: Sun,  7 Jul 2013 00:02:47 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Refactor_deque=5Ftraverse?=
	=?utf-8?b?KCku?=
Message-ID: <3bnn3W6RSQzSbH@mail.python.org>

http://hg.python.org/cpython/rev/3555cc0ca35b
changeset:   84466:3555cc0ca35b
user:        Raymond Hettinger <python at rcn.com>
date:        Sat Jul 06 11:58:09 2013 -1000
summary:
  Refactor deque_traverse().

Hoist conditional expression out of the loop.
Use rightblock as the guard instead of checking for NULL.

files:
  Modules/_collectionsmodule.c |  12 ++++++------
  1 files changed, 6 insertions(+), 6 deletions(-)


diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -805,17 +805,17 @@
     Py_ssize_t index;
     Py_ssize_t indexlo = deque->leftindex;
 
-    for (b = deque->leftblock; b != NULL; b = b->rightlink) {
-        const Py_ssize_t indexhi = b == deque->rightblock ?
-                                 deque->rightindex :
-                     BLOCKLEN - 1;
-
-        for (index = indexlo; index <= indexhi; ++index) {
+    for (b = deque->leftblock; b != deque->rightblock; b = b->rightlink) {
+        for (index = indexlo; index < BLOCKLEN ; index++) {
             item = b->data[index];
             Py_VISIT(item);
         }
         indexlo = 0;
     }
+    for (index = indexlo; index <= deque->rightindex; index++) {
+        item = b->data[index];
+        Py_VISIT(item);
+    }
     return 0;
 }
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 00:05:13 2013
From: python-checkins at python.org (brett.cannon)
Date: Sun,  7 Jul 2013 00:05:13 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4MzUx?=
 =?utf-8?q?=3A_Fix_various_issues_with?=
Message-ID: <3bnn6K499KzSyb@mail.python.org>

http://hg.python.org/cpython/rev/b8028f74bac4
changeset:   84467:b8028f74bac4
branch:      3.3
parent:      84462:c8fd1351c840
user:        Brett Cannon <brett at python.org>
date:        Sat Jul 06 17:56:43 2013 -0400
summary:
  Issue #18351: Fix various issues with
importlib._bootstrap._get_sourcefile().

Thanks to its only use by the C API, it was never properly tested
until now.

Thanks to Neal Norwitz for discovering the bug and Madison May for the patch.

files:
  Lib/importlib/_bootstrap.py |    10 +-
  Lib/test/test_import.py     |    38 +-
  Misc/ACKS                   |     1 +
  Misc/NEWS                   |     6 +
  Python/importlib.h          |  6826 +++++++++++-----------
  5 files changed, 3460 insertions(+), 3421 deletions(-)


diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -471,16 +471,14 @@
     """
     if len(bytecode_path) == 0:
         return None
-    rest, _, extension = bytecode_path.rparition('.')
-    if not rest or extension.lower()[-3:-1] != '.py':
+    rest, _, extension = bytecode_path.rpartition('.')
+    if not rest or extension.lower()[-3:-1] != 'py':
         return bytecode_path
-
     try:
         source_path = source_from_cache(bytecode_path)
     except (NotImplementedError, ValueError):
-        source_path = bytcode_path[-1:]
-
-    return source_path if _path_isfile(source_stats) else bytecode_path
+        source_path = bytecode_path[:-1]
+    return source_path if _path_isfile(source_path) else bytecode_path
 
 
 def _verbose_message(message, *args, verbosity=1):
diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
--- a/Lib/test/test_import.py
+++ b/Lib/test/test_import.py
@@ -1,5 +1,6 @@
 # We import importlib *ASAP* in order to test #15386
 import importlib
+from importlib._bootstrap import _get_sourcefile
 import builtins
 import imp
 from test.test_importlib.import_ import util as importlib_util
@@ -11,6 +12,7 @@
 import stat
 import sys
 import unittest
+import unittest.mock as mock
 import textwrap
 import errno
 import shutil
@@ -864,6 +866,40 @@
         self.assertIs(imp.new_module, mod.new_module)
 
 
+ at cpython_only
+class GetSourcefileTests(unittest.TestCase):
+
+    """Test importlib._bootstrap._get_sourcefile() as used by the C API.
+
+    Because of the peculiarities of the need of this function, the tests are
+    knowingly whitebox tests.
+
+    """
+
+    def test_get_sourcefile(self):
+        # Given a valid bytecode path, return the path to the corresponding
+        # source file if it exists.
+        with mock.patch('importlib._bootstrap._path_isfile') as _path_isfile:
+            _path_isfile.return_value = True;
+            path = TESTFN + '.pyc'
+            expect = TESTFN + '.py'
+            self.assertEqual(_get_sourcefile(path), expect)
+
+    def test_get_sourcefile_no_source(self):
+        # Given a valid bytecode path without a corresponding source path,
+        # return the original bytecode path.
+        with mock.patch('importlib._bootstrap._path_isfile') as _path_isfile:
+            _path_isfile.return_value = False;
+            path = TESTFN + '.pyc'
+            self.assertEqual(_get_sourcefile(path), path)
+
+    def test_get_sourcefile_bad_ext(self):
+        # Given a path with an invalid bytecode extension, return the
+        # bytecode path passed as the argument.
+        path = TESTFN + '.bad_ext'
+        self.assertEqual(_get_sourcefile(path), path)
+
+
 class ImportTracebackTests(unittest.TestCase):
 
     def setUp(self):
@@ -1028,7 +1064,7 @@
     run_unittest(ImportTests, PycacheTests, FilePermissionTests,
                  PycRewritingTests, PathsTests, RelativeImportTests,
                  OverridingImportBuiltinTests,
-                 ImportlibBootstrapTests,
+                 ImportlibBootstrapTests, GetSourcefileTests,
                  TestSymbolicallyLinkedPackage,
                  ImportTracebackTests)
 
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -788,6 +788,7 @@
 Graham Matthews
 Dieter Maurer
 Daniel May
+Madison May
 Arnaud Mazin
 Rebecca McCreary
 Kirk McDonald
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -130,6 +130,12 @@
 - Issue #18113: Fixed a refcount leak in the curses.panel module's
   set_userptr() method.  Reported by Atsuo Ishimoto.
 
+C API
+-----
+
+- Issue #18351: Fix various issues with a helper function in importlib used
+  by PyImport_ExecCodeModuleWithPathnames() (and thus by extension PyImport_ExecCodeModule() and PyImport_ExecCodeModuleEx()).
+
 IDLE
 ----
 
diff --git a/Python/importlib.h b/Python/importlib.h
--- a/Python/importlib.h
+++ b/Python/importlib.h
[stripped]

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 00:05:15 2013
From: python-checkins at python.org (brett.cannon)
Date: Sun,  7 Jul 2013 00:05:15 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_merge_for_issue_=2318351=2E?=
Message-ID: <3bnn6M1B2fz7LjZ@mail.python.org>

http://hg.python.org/cpython/rev/e80634ad5a0e
changeset:   84468:e80634ad5a0e
parent:      84465:49a9c734304d
parent:      84467:b8028f74bac4
user:        Brett Cannon <brett at python.org>
date:        Sat Jul 06 18:04:41 2013 -0400
summary:
  merge for issue #18351.

files:
  Lib/importlib/_bootstrap.py |    10 +-
  Lib/test/test_import.py     |    36 +
  Misc/ACKS                   |     1 +
  Misc/NEWS                   |     3 +
  Python/importlib.h          |  1600 +++++++++++-----------
  5 files changed, 843 insertions(+), 807 deletions(-)


diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -443,16 +443,14 @@
     """
     if len(bytecode_path) == 0:
         return None
-    rest, _, extension = bytecode_path.rparition('.')
-    if not rest or extension.lower()[-3:-1] != '.py':
+    rest, _, extension = bytecode_path.rpartition('.')
+    if not rest or extension.lower()[-3:-1] != 'py':
         return bytecode_path
-
     try:
         source_path = source_from_cache(bytecode_path)
     except (NotImplementedError, ValueError):
-        source_path = bytcode_path[-1:]
-
-    return source_path if _path_isfile(source_stats) else bytecode_path
+        source_path = bytecode_path[:-1]
+    return source_path if _path_isfile(source_path) else bytecode_path
 
 
 def _calc_mode(path):
diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
--- a/Lib/test/test_import.py
+++ b/Lib/test/test_import.py
@@ -1,6 +1,7 @@
 # We import importlib *ASAP* in order to test #15386
 import importlib
 import importlib.util
+from importlib._bootstrap import _get_sourcefile
 import builtins
 from test.test_importlib.import_ import util as importlib_util
 import marshal
@@ -11,6 +12,7 @@
 import stat
 import sys
 import unittest
+import unittest.mock as mock
 import textwrap
 import errno
 import shutil
@@ -851,6 +853,40 @@
         self.assertIs(machinery.FileFinder, mod.FileFinder)
 
 
+ at cpython_only
+class GetSourcefileTests(unittest.TestCase):
+
+    """Test importlib._bootstrap._get_sourcefile() as used by the C API.
+
+    Because of the peculiarities of the need of this function, the tests are
+    knowingly whitebox tests.
+
+    """
+
+    def test_get_sourcefile(self):
+        # Given a valid bytecode path, return the path to the corresponding
+        # source file if it exists.
+        with mock.patch('importlib._bootstrap._path_isfile') as _path_isfile:
+            _path_isfile.return_value = True;
+            path = TESTFN + '.pyc'
+            expect = TESTFN + '.py'
+            self.assertEqual(_get_sourcefile(path), expect)
+
+    def test_get_sourcefile_no_source(self):
+        # Given a valid bytecode path without a corresponding source path,
+        # return the original bytecode path.
+        with mock.patch('importlib._bootstrap._path_isfile') as _path_isfile:
+            _path_isfile.return_value = False;
+            path = TESTFN + '.pyc'
+            self.assertEqual(_get_sourcefile(path), path)
+
+    def test_get_sourcefile_bad_ext(self):
+        # Given a path with an invalid bytecode extension, return the
+        # bytecode path passed as the argument.
+        path = TESTFN + '.bad_ext'
+        self.assertEqual(_get_sourcefile(path), path)
+
+
 class ImportTracebackTests(unittest.TestCase):
 
     def setUp(self):
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -808,6 +808,7 @@
 Graham Matthews
 Dieter Maurer
 Daniel May
+Madison May
 Lucas Maystre
 Arnaud Mazin
 Rebecca McCreary
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -542,6 +542,9 @@
 C-API
 -----
 
+- Issue #18351: Fix various issues in a function in importlib provided to help
+  PyImport_ExecCodeModuleWithPathnames() (and thus by extension PyImport_ExecCodeModule() and PyImport_ExecCodeModuleEx()).
+
 - Issue #9369: The types of `char*` arguments of PyObject_CallFunction() and
   PyObject_CallMethod() now changed to `const char*`.  Based on patches by
   J?rg M?ller and Lars Buitinck.
diff --git a/Python/importlib.h b/Python/importlib.h
--- a/Python/importlib.h
+++ b/Python/importlib.h
[stripped]

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 00:05:16 2013
From: python-checkins at python.org (brett.cannon)
Date: Sun,  7 Jul 2013 00:05:16 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?=
	=?utf-8?q?=29=3A_merge?=
Message-ID: <3bnn6N3Yxfz7Ljb@mail.python.org>

http://hg.python.org/cpython/rev/e8a1b4bcabcb
changeset:   84469:e8a1b4bcabcb
parent:      84468:e80634ad5a0e
parent:      84466:3555cc0ca35b
user:        Brett Cannon <brett at python.org>
date:        Sat Jul 06 18:05:02 2013 -0400
summary:
  merge

files:
  Modules/_collectionsmodule.c |  12 ++++++------
  1 files changed, 6 insertions(+), 6 deletions(-)


diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -805,17 +805,17 @@
     Py_ssize_t index;
     Py_ssize_t indexlo = deque->leftindex;
 
-    for (b = deque->leftblock; b != NULL; b = b->rightlink) {
-        const Py_ssize_t indexhi = b == deque->rightblock ?
-                                 deque->rightindex :
-                     BLOCKLEN - 1;
-
-        for (index = indexlo; index <= indexhi; ++index) {
+    for (b = deque->leftblock; b != deque->rightblock; b = b->rightlink) {
+        for (index = indexlo; index < BLOCKLEN ; index++) {
             item = b->data[index];
             Py_VISIT(item);
         }
         indexlo = 0;
     }
+    for (index = indexlo; index <= deque->rightindex; index++) {
+        item = b->data[index];
+        Py_VISIT(item);
+    }
     return 0;
 }
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 01:01:44 2013
From: python-checkins at python.org (raymond.hettinger)
Date: Sun,  7 Jul 2013 01:01:44 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Apply_the_PyObject=5FVAR?=
	=?utf-8?q?=5FHEAD_and_Py=5FSIZE_macros?=
Message-ID: <3bnpMX3lQjzSmK@mail.python.org>

http://hg.python.org/cpython/rev/9166aa413d6f
changeset:   84470:9166aa413d6f
user:        Raymond Hettinger <python at rcn.com>
date:        Sat Jul 06 13:01:13 2013 -1000
summary:
  Apply the PyObject_VAR_HEAD and Py_SIZE macros
to be consistent with practices in other modules.

files:
  Modules/_collectionsmodule.c |  79 +++++++++++------------
  1 files changed, 39 insertions(+), 40 deletions(-)


diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -93,12 +93,11 @@
 }
 
 typedef struct {
-    PyObject_HEAD
+    PyObject_VAR_HEAD
     block *leftblock;
     block *rightblock;
     Py_ssize_t leftindex;       /* in range(BLOCKLEN) */
     Py_ssize_t rightindex;      /* in range(BLOCKLEN) */
-    Py_ssize_t len;
     Py_ssize_t maxlen;
     long state;         /* incremented whenever the indices move */
     PyObject *weakreflist; /* List of weak references */
@@ -114,9 +113,9 @@
  */
 
 #define TRIM(d, popfunction)                                    \
-    if (d->maxlen != -1 && d->len > d->maxlen) {                \
+    if (d->maxlen != -1 && Py_SIZE(d) > d->maxlen) {       \
         PyObject *rv = popfunction(d, NULL);                \
-        assert(rv != NULL  &&  d->len <= d->maxlen);        \
+        assert(rv != NULL  &&  Py_SIZE(d) <= d->maxlen);    \
         Py_DECREF(rv);                                      \
     }
 
@@ -144,7 +143,7 @@
     deque->rightblock = b;
     deque->leftindex = CENTER + 1;
     deque->rightindex = CENTER;
-    deque->len = 0;
+    Py_SIZE(deque) = 0;
     deque->state = 0;
     deque->weakreflist = NULL;
     deque->maxlen = -1;
@@ -158,17 +157,17 @@
     PyObject *item;
     block *prevblock;
 
-    if (deque->len == 0) {
+    if (Py_SIZE(deque) == 0) {
         PyErr_SetString(PyExc_IndexError, "pop from an empty deque");
         return NULL;
     }
     item = deque->rightblock->data[deque->rightindex];
     deque->rightindex--;
-    deque->len--;
+    Py_SIZE(deque)--;
     deque->state++;
 
     if (deque->rightindex == -1) {
-        if (deque->len == 0) {
+        if (Py_SIZE(deque) == 0) {
             assert(deque->leftblock == deque->rightblock);
             assert(deque->leftindex == deque->rightindex+1);
             /* re-center instead of freeing a block */
@@ -194,18 +193,18 @@
     PyObject *item;
     block *prevblock;
 
-    if (deque->len == 0) {
+    if (Py_SIZE(deque) == 0) {
         PyErr_SetString(PyExc_IndexError, "pop from an empty deque");
         return NULL;
     }
     assert(deque->leftblock != NULL);
     item = deque->leftblock->data[deque->leftindex];
     deque->leftindex++;
-    deque->len--;
+    Py_SIZE(deque)--;
     deque->state++;
 
     if (deque->leftindex == BLOCKLEN) {
-        if (deque->len == 0) {
+        if (Py_SIZE(deque) == 0) {
             assert(deque->leftblock == deque->rightblock);
             assert(deque->leftindex == deque->rightindex+1);
             /* re-center instead of freeing a block */
@@ -231,7 +230,7 @@
 {
     deque->state++;
     if (deque->rightindex == BLOCKLEN-1) {
-        block *b = newblock(deque->rightblock, NULL, deque->len);
+        block *b = newblock(deque->rightblock, NULL, Py_SIZE(deque));
         if (b == NULL)
             return NULL;
         assert(deque->rightblock->rightlink == NULL);
@@ -240,7 +239,7 @@
         deque->rightindex = -1;
     }
     Py_INCREF(item);
-    deque->len++;
+    Py_SIZE(deque)++;
     deque->rightindex++;
     deque->rightblock->data[deque->rightindex] = item;
     TRIM(deque, deque_popleft);
@@ -254,7 +253,7 @@
 {
     deque->state++;
     if (deque->leftindex == 0) {
-        block *b = newblock(NULL, deque->leftblock, deque->len);
+        block *b = newblock(NULL, deque->leftblock, Py_SIZE(deque));
         if (b == NULL)
             return NULL;
         assert(deque->leftblock->leftlink == NULL);
@@ -263,7 +262,7 @@
         deque->leftindex = BLOCKLEN;
     }
     Py_INCREF(item);
-    deque->len++;
+    Py_SIZE(deque)++;
     deque->leftindex--;
     deque->leftblock->data[deque->leftindex] = item;
     TRIM(deque, deque_pop);
@@ -316,7 +315,7 @@
         deque->state++;
         if (deque->rightindex == BLOCKLEN-1) {
             block *b = newblock(deque->rightblock, NULL,
-                                deque->len);
+                                Py_SIZE(deque));
             if (b == NULL) {
                 Py_DECREF(item);
                 Py_DECREF(it);
@@ -327,7 +326,7 @@
             deque->rightblock = b;
             deque->rightindex = -1;
         }
-        deque->len++;
+        Py_SIZE(deque)++;
         deque->rightindex++;
         deque->rightblock->data[deque->rightindex] = item;
         TRIM(deque, deque_popleft);
@@ -368,7 +367,7 @@
         deque->state++;
         if (deque->leftindex == 0) {
             block *b = newblock(NULL, deque->leftblock,
-                                deque->len);
+                                Py_SIZE(deque));
             if (b == NULL) {
                 Py_DECREF(item);
                 Py_DECREF(it);
@@ -379,7 +378,7 @@
             deque->leftblock = b;
             deque->leftindex = BLOCKLEN;
         }
-        deque->len++;
+        Py_SIZE(deque)++;
         deque->leftindex--;
         deque->leftblock->data[deque->leftindex] = item;
         TRIM(deque, deque_pop);
@@ -413,7 +412,7 @@
     block *rightblock = deque->rightblock;
     Py_ssize_t leftindex = deque->leftindex;
     Py_ssize_t rightindex = deque->rightindex;
-    Py_ssize_t len=deque->len, halflen=len>>1;
+    Py_ssize_t len=Py_SIZE(deque), halflen=len>>1;
     int rv = 0;
 
     if (len <= 1)
@@ -544,7 +543,7 @@
     block *rightblock = deque->rightblock;
     Py_ssize_t leftindex = deque->leftindex;
     Py_ssize_t rightindex = deque->rightindex;
-    Py_ssize_t n = (deque->len)/2;
+    Py_ssize_t n = (Py_SIZE(deque))/2;
     Py_ssize_t i;
     PyObject *tmp;
 
@@ -584,7 +583,7 @@
 {
     block *leftblock = deque->leftblock;
     Py_ssize_t leftindex = deque->leftindex;
-    Py_ssize_t n = deque->len;
+    Py_ssize_t n = Py_SIZE(deque);
     Py_ssize_t i;
     Py_ssize_t count = 0;
     PyObject *item;
@@ -622,19 +621,19 @@
 static Py_ssize_t
 deque_len(dequeobject *deque)
 {
-    return deque->len;
+    return Py_SIZE(deque);
 }
 
 static PyObject *
 deque_remove(dequeobject *deque, PyObject *value)
 {
-    Py_ssize_t i, n=deque->len;
+    Py_ssize_t i, n=Py_SIZE(deque);
 
     for (i=0 ; i<n ; i++) {
         PyObject *item = deque->leftblock->data[deque->leftindex];
         int cmp = PyObject_RichCompareBool(item, value, Py_EQ);
 
-        if (deque->len != n) {
+        if (Py_SIZE(deque) != n) {
             PyErr_SetString(PyExc_IndexError,
                 "deque mutated during remove().");
             return NULL;
@@ -665,14 +664,14 @@
 {
     PyObject *item;
 
-    while (deque->len) {
+    while (Py_SIZE(deque)) {
         item = deque_pop(deque, NULL);
         assert (item != NULL);
         Py_DECREF(item);
     }
     assert(deque->leftblock == deque->rightblock &&
            deque->leftindex - 1 == deque->rightindex &&
-           deque->len == 0);
+           Py_SIZE(deque) == 0);
 }
 
 static PyObject *
@@ -682,7 +681,7 @@
     PyObject *item;
     Py_ssize_t n, index=i;
 
-    if (i < 0 || i >= deque->len) {
+    if (i < 0 || i >= Py_SIZE(deque)) {
         PyErr_SetString(PyExc_IndexError,
                         "deque index out of range");
         return NULL;
@@ -691,19 +690,19 @@
     if (i == 0) {
         i = deque->leftindex;
         b = deque->leftblock;
-    } else if (i == deque->len - 1) {
+    } else if (i == Py_SIZE(deque) - 1) {
         i = deque->rightindex;
         b = deque->rightblock;
     } else {
         i += deque->leftindex;
         n = i / BLOCKLEN;
         i %= BLOCKLEN;
-        if (index < (deque->len >> 1)) {
+        if (index < (Py_SIZE(deque) >> 1)) {
             b = deque->leftblock;
             while (n--)
                 b = b->rightlink;
         } else {
-            n = (deque->leftindex + deque->len - 1) / BLOCKLEN - n;
+            n = (deque->leftindex + Py_SIZE(deque) - 1) / BLOCKLEN - n;
             b = deque->rightblock;
             while (n--)
                 b = b->leftlink;
@@ -726,7 +725,7 @@
 {
     PyObject *item;
 
-    assert (i >= 0 && i < deque->len);
+    assert (i >= 0 && i < Py_SIZE(deque));
     if (_deque_rotate(deque, -i) == -1)
         return -1;
 
@@ -742,7 +741,7 @@
 {
     PyObject *old_value;
     block *b;
-    Py_ssize_t n, len=deque->len, halflen=(len+1)>>1, index=i;
+    Py_ssize_t n, len=Py_SIZE(deque), halflen=(len+1)>>1, index=i;
 
     if (i < 0 || i >= len) {
         PyErr_SetString(PyExc_IndexError,
@@ -905,8 +904,8 @@
     }
 
     /* Shortcuts */
-    vs = ((dequeobject *)v)->len;
-    ws = ((dequeobject *)w)->len;
+    vs = Py_SIZE((dequeobject *)v);
+    ws = Py_SIZE((dequeobject *)w);
     if (op == Py_EQ) {
         if (v == w)
             Py_RETURN_TRUE;
@@ -1007,8 +1006,8 @@
     Py_ssize_t blocks;
 
     res = sizeof(dequeobject);
-    blocks = (deque->leftindex + deque->len + BLOCKLEN - 1) / BLOCKLEN;
-    assert(deque->leftindex + deque->len - 1 ==
+    blocks = (deque->leftindex + Py_SIZE(deque) + BLOCKLEN - 1) / BLOCKLEN;
+    assert(deque->leftindex + Py_SIZE(deque) - 1 ==
            (blocks - 1) * BLOCKLEN + deque->rightindex);
     res += blocks * sizeof(block);
     return PyLong_FromSsize_t(res);
@@ -1161,7 +1160,7 @@
     Py_INCREF(deque);
     it->deque = deque;
     it->state = deque->state;
-    it->counter = deque->len;
+    it->counter = Py_SIZE(deque);
     PyObject_GC_Track(it);
     return (PyObject *)it;
 }
@@ -1248,7 +1247,7 @@
 static PyObject *
 dequeiter_reduce(dequeiterobject *it)
 {
-    return Py_BuildValue("O(On)", Py_TYPE(it), it->deque, it->deque->len - it->counter);
+    return Py_BuildValue("O(On)", Py_TYPE(it), it->deque, Py_SIZE(it->deque) - it->counter);
 }
 
 static PyMethodDef dequeiter_methods[] = {
@@ -1317,7 +1316,7 @@
     Py_INCREF(deque);
     it->deque = deque;
     it->state = deque->state;
-    it->counter = deque->len;
+    it->counter = Py_SIZE(deque);
     PyObject_GC_Track(it);
     return (PyObject *)it;
 }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 02:25:37 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 02:25:37 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=233329=3A_Implement?=
	=?utf-8?q?_the_PEP_445?=
Message-ID: <3bnrDK06HkzRZn@mail.python.org>

http://hg.python.org/cpython/rev/ca78c974e938
changeset:   84471:ca78c974e938
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 02:05:46 2013 +0200
summary:
  Issue #3329: Implement the PEP 445

Add new enum:

* PyMemAllocatorDomain

Add new structures:

* PyMemAllocator
* PyObjectArenaAllocator

Add new functions:

* PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree()
* PyMem_GetAllocator(), PyMem_SetAllocator()
* PyObject_GetArenaAllocator(), PyObject_SetArenaAllocator()
* PyMem_SetupDebugHooks()

Changes:

* PyMem_Malloc()/PyObject_Realloc() now always call malloc()/realloc(), instead
  of calling PyObject_Malloc()/PyObject_Realloc() in debug mode.
* PyObject_Malloc()/PyObject_Realloc() now falls back to
  PyMem_Malloc()/PyMem_Realloc() for allocations larger than 512 bytes.
* Redesign debug checks on memory block allocators as hooks, instead of using C
  macros

files:
  Doc/c-api/memory.rst      |  171 ++++++++-
  Doc/whatsnew/3.4.rst      |   18 +-
  Include/objimpl.h         |   60 +-
  Include/pymem.h           |   94 +++-
  Misc/NEWS                 |    3 +
  Modules/_testcapimodule.c |  163 ++++++++
  Objects/object.c          |   20 -
  Objects/obmalloc.c        |  504 ++++++++++++++++++-------
  8 files changed, 796 insertions(+), 237 deletions(-)


diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst
--- a/Doc/c-api/memory.rst
+++ b/Doc/c-api/memory.rst
@@ -84,6 +84,48 @@
 for the I/O buffer escapes completely the Python memory manager.
 
 
+Raw Memory Interface
+====================
+
+The following function sets are wrappers to the system allocator. These
+functions are thread-safe, the :term:`GIL <global interpreter lock>` does not
+need to be held.
+
+The default raw memory block allocator uses the following functions:
+:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
+requesting zero bytes.
+
+.. versionadded:: 3.4
+
+.. c:function:: void* PyMem_RawMalloc(size_t n)
+
+   Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the
+   allocated memory, or *NULL* if the request fails. Requesting zero bytes
+   returns a distinct non-*NULL* pointer if possible, as if
+   ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have
+   been initialized in any way.
+
+
+.. c:function:: void* PyMem_RawRealloc(void *p, size_t n)
+
+   Resizes the memory block pointed to by *p* to *n* bytes. The contents will
+   be unchanged to the minimum of the old and the new sizes. If *p* is *NULL*,
+   the call is equivalent to ``PyMem_RawMalloc(n)``; else if *n* is equal to
+   zero, the memory block is resized but is not freed, and the returned pointer
+   is non-*NULL*. Unless *p* is *NULL*, it must have been returned by a
+   previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`. If
+   the request fails, :c:func:`PyMem_RawRealloc` returns *NULL* and *p* remains
+   a valid pointer to the previous memory area.
+
+
+.. c:function:: void PyMem_RawFree(void *p)
+
+   Frees the memory block pointed to by *p*, which must have been returned by a
+   previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`.
+   Otherwise, or if ``PyMem_Free(p)`` has been called before, undefined
+   behavior occurs. If *p* is *NULL*, no operation is performed.
+
+
 .. _memoryinterface:
 
 Memory Interface
@@ -91,8 +133,16 @@
 
 The following function sets, modeled after the ANSI C standard, but specifying
 behavior when requesting zero bytes, are available for allocating and releasing
-memory from the Python heap:
+memory from the Python heap.
 
+The default memory block allocator uses the following functions:
+:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
+requesting zero bytes.
+
+.. warning::
+
+   The :term:`GIL <global interpreter lock>` must be held when using these
+   functions.
 
 .. c:function:: void* PyMem_Malloc(size_t n)
 
@@ -155,6 +205,125 @@
 :c:func:`PyMem_NEW`, :c:func:`PyMem_RESIZE`, :c:func:`PyMem_DEL`.
 
 
+Customize Memory Allocators
+===========================
+
+.. versionadded:: 3.4
+
+.. c:type:: PyMemAllocator
+
+   Structure used to describe a memory block allocator. The structure has
+   four fields:
+
+   +----------------------------------------------------------+---------------------------------------+
+   | Field                                                    | Meaning                               |
+   +==========================================================+=======================================+
+   | ``void *ctx``                                            | user context passed as first argument |
+   +----------------------------------------------------------+---------------------------------------+
+   | ``void* malloc(void *ctx, size_t size)``                 | allocate a memory block               |
+   +----------------------------------------------------------+---------------------------------------+
+   | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block     |
+   +----------------------------------------------------------+---------------------------------------+
+   | ``void free(void *ctx, void *ptr)``                      | free a memory block                   |
+   +----------------------------------------------------------+---------------------------------------+
+
+.. c:type:: PyMemAllocatorDomain
+
+   Enum used to identify an allocator domain. Domains:
+
+   * :c:data:`PYMEM_DOMAIN_RAW`: functions :c:func:`PyMem_RawMalloc`,
+     :c:func:`PyMem_RawRealloc` and :c:func:`PyMem_RawFree`
+   * :c:data:`PYMEM_DOMAIN_MEM`: functions :c:func:`PyMem_Malloc`,
+     :c:func:`PyMem_Realloc` and :c:func:`PyMem_Free`
+   * :c:data:`PYMEM_DOMAIN_OBJ`: functions :c:func:`PyObject_Malloc`,
+     :c:func:`PyObject_Realloc` and :c:func:`PyObject_Free`
+
+
+.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
+
+   Get the memory block allocator of the specified domain.
+
+
+.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
+
+   Set the memory block allocator of the specified domain.
+
+   The new allocator must return a distinct non-NULL pointer when requesting
+   zero bytes.
+
+   For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be
+   thread-safe: the :term:`GIL <global interpreter lock>` is not held when the
+   allocator is called.
+
+   If the new allocator is not a hook (does not call the previous allocator),
+   the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the
+   debug hooks on top on the new allocator.
+
+
+.. c:function:: void PyMem_SetupDebugHooks(void)
+
+   Setup hooks to detect bugs in the following Python memory allocator
+   functions:
+
+   - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc`,
+     :c:func:`PyMem_RawFree`
+   - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc`, :c:func:`PyMem_Free`
+   - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc`,
+     :c:func:`PyObject_Free`
+
+   Newly allocated memory is filled with the byte ``0xCB``, freed memory is
+   filled with the byte ``0xDB``. Additionnal checks:
+
+   - detect API violations, ex: :c:func:`PyObject_Free` called on a buffer
+     allocated by :c:func:`PyMem_Malloc`
+   - detect write before the start of the buffer (buffer underflow)
+   - detect write after the end of the buffer (buffer overflow)
+
+   The function does nothing if Python is not compiled is debug mode.
+
+
+Customize PyObject Arena Allocator
+==================================
+
+Python has a *pymalloc* allocator for allocations smaller than 512 bytes. This
+allocator is optimized for small objects with a short lifetime. It uses memory
+mappings called "arenas" with a fixed size of 256 KB. It falls back to
+:c:func:`PyMem_Malloc` and :c:func:`PyMem_Realloc` for allocations larger than
+512 bytes.  *pymalloc* is the default allocator used by
+:c:func:`PyObject_Malloc`.
+
+The default arena allocator uses the following functions:
+
+* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows,
+* :c:func:`mmap` and :c:func:`munmap` if available,
+* :c:func:`malloc` and :c:func:`free` otherwise.
+
+.. versionadded:: 3.4
+
+.. c:type:: PyObjectArenaAllocator
+
+   Structure used to describe an arena allocator. The structure has
+   three fields:
+
+   +--------------------------------------------------+---------------------------------------+
+   | Field                                            | Meaning                               |
+   +==================================================+=======================================+
+   | ``void *ctx``                                    | user context passed as first argument |
+   +--------------------------------------------------+---------------------------------------+
+   | ``void* alloc(void *ctx, size_t size)``          | allocate an arena of size bytes       |
+   +--------------------------------------------------+---------------------------------------+
+   | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena                         |
+   +--------------------------------------------------+---------------------------------------+
+
+.. c:function:: PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)
+
+   Get the arena allocator.
+
+.. c:function:: PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)
+
+   Set the arena allocator.
+
+
 .. _memoryexamples:
 
 Examples
diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst
--- a/Doc/whatsnew/3.4.rst
+++ b/Doc/whatsnew/3.4.rst
@@ -112,21 +112,11 @@
 Please read on for a comprehensive list of user-facing changes.
 
 
-.. PEP-sized items next.
+PEP 445: Add new APIs to customize Python memory allocators
+===========================================================
 
-.. _pep-4XX:
-
-.. PEP 4XX: Example PEP
-.. ====================
-
-
-.. (Implemented by Foo Bar.)
-
-.. .. seealso::
-
-   :pep:`4XX` - Example PEP
-      PEP written by Example Author
-
+The :pep:`445` adds new Application Programming Interfaces (API) to customize
+Python memory allocators.
 
 
 
diff --git a/Include/objimpl.h b/Include/objimpl.h
--- a/Include/objimpl.h
+++ b/Include/objimpl.h
@@ -94,9 +94,9 @@
    the object gets initialized via PyObject_{Init, InitVar} after obtaining
    the raw memory.
 */
-PyAPI_FUNC(void *) PyObject_Malloc(size_t);
-PyAPI_FUNC(void *) PyObject_Realloc(void *, size_t);
-PyAPI_FUNC(void) PyObject_Free(void *);
+PyAPI_FUNC(void *) PyObject_Malloc(size_t size);
+PyAPI_FUNC(void *) PyObject_Realloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyObject_Free(void *ptr);
 
 /* This function returns the number of allocated memory blocks, regardless of size */
 PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
@@ -106,41 +106,15 @@
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out);
 #endif /* #ifndef Py_LIMITED_API */
-#ifdef PYMALLOC_DEBUG   /* WITH_PYMALLOC && PYMALLOC_DEBUG */
-PyAPI_FUNC(void *) _PyObject_DebugMalloc(size_t nbytes);
-PyAPI_FUNC(void *) _PyObject_DebugRealloc(void *p, size_t nbytes);
-PyAPI_FUNC(void) _PyObject_DebugFree(void *p);
-PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p);
-PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p);
-PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes);
-PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes);
-PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p);
-PyAPI_FUNC(void) _PyObject_DebugCheckAddressApi(char api, const void *p);
-PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes);
-PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes);
-PyAPI_FUNC(void) _PyMem_DebugFree(void *p);
-#define PyObject_MALLOC         _PyObject_DebugMalloc
-#define PyObject_Malloc         _PyObject_DebugMalloc
-#define PyObject_REALLOC        _PyObject_DebugRealloc
-#define PyObject_Realloc        _PyObject_DebugRealloc
-#define PyObject_FREE           _PyObject_DebugFree
-#define PyObject_Free           _PyObject_DebugFree
+#endif
 
-#else   /* WITH_PYMALLOC && ! PYMALLOC_DEBUG */
+/* Macros */
 #define PyObject_MALLOC         PyObject_Malloc
 #define PyObject_REALLOC        PyObject_Realloc
 #define PyObject_FREE           PyObject_Free
-#endif
+#define PyObject_Del            PyObject_Free
+#define PyObject_DEL            PyObject_Free
 
-#else   /* ! WITH_PYMALLOC */
-#define PyObject_MALLOC         PyMem_MALLOC
-#define PyObject_REALLOC        PyMem_REALLOC
-#define PyObject_FREE           PyMem_FREE
-
-#endif  /* WITH_PYMALLOC */
-
-#define PyObject_Del            PyObject_Free
-#define PyObject_DEL            PyObject_FREE
 
 /*
  * Generic object allocator interface
@@ -224,6 +198,26 @@
    constructor you would start directly with PyObject_Init/InitVar
 */
 
+#ifndef Py_LIMITED_API
+typedef struct {
+    /* user context passed as the first argument to the 2 functions */
+    void *ctx;
+
+    /* allocate an arena of size bytes */
+    void* (*alloc) (void *ctx, size_t size);
+
+    /* free an arena */
+    void (*free) (void *ctx, void *ptr, size_t size);
+} PyObjectArenaAllocator;
+
+/* Get the arena allocator. */
+PyAPI_FUNC(void) PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator);
+
+/* Set the arena allocator. */
+PyAPI_FUNC(void) PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator);
+#endif
+
+
 /*
  * Garbage Collection Support
  * ==========================
diff --git a/Include/pymem.h b/Include/pymem.h
--- a/Include/pymem.h
+++ b/Include/pymem.h
@@ -11,6 +11,11 @@
 extern "C" {
 #endif
 
+PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
+PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
+
+
 /* BEWARE:
 
    Each interface exports both functions and macros.  Extension modules should
@@ -49,21 +54,11 @@
    performed on failure (no exception is set, no warning is printed, etc).
 */
 
-PyAPI_FUNC(void *) PyMem_Malloc(size_t);
-PyAPI_FUNC(void *) PyMem_Realloc(void *, size_t);
-PyAPI_FUNC(void) PyMem_Free(void *);
-
-/* Starting from Python 1.6, the wrappers Py_{Malloc,Realloc,Free} are
-   no longer supported. They used to call PyErr_NoMemory() on failure. */
+PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
+PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_Free(void *ptr);
 
 /* Macros. */
-#ifdef PYMALLOC_DEBUG
-/* Redirect all memory operations to Python's debugging allocator. */
-#define PyMem_MALLOC		_PyMem_DebugMalloc
-#define PyMem_REALLOC		_PyMem_DebugRealloc
-#define PyMem_FREE		_PyMem_DebugFree
-
-#else	/* ! PYMALLOC_DEBUG */
 
 /* PyMem_MALLOC(0) means malloc(1). Some systems would return NULL
    for malloc(0), which would be treated as an error. Some platforms
@@ -71,13 +66,9 @@
    pymalloc. To solve these problems, allocate an extra byte. */
 /* Returns NULL to indicate error if a negative size or size larger than
    Py_ssize_t can represent is supplied.  Helps prevents security holes. */
-#define PyMem_MALLOC(n)		((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \
-				: malloc((n) ? (n) : 1))
-#define PyMem_REALLOC(p, n)	((size_t)(n) > (size_t)PY_SSIZE_T_MAX  ? NULL \
-				: realloc((p), (n) ? (n) : 1))
-#define PyMem_FREE		free
-
-#endif	/* PYMALLOC_DEBUG */
+#define PyMem_MALLOC(n)         PyMem_Malloc(n)
+#define PyMem_REALLOC(p, n)     PyMem_Realloc(p, n)
+#define PyMem_FREE(p)           PyMem_Free(p)
 
 /*
  * Type-oriented memory interface
@@ -115,6 +106,69 @@
 #define PyMem_Del		PyMem_Free
 #define PyMem_DEL		PyMem_FREE
 
+#ifndef Py_LIMITED_API
+typedef enum {
+    /* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */
+    PYMEM_DOMAIN_RAW,
+
+    /* PyMem_Malloc(), PyMem_Realloc() and PyMem_Free() */
+    PYMEM_DOMAIN_MEM,
+
+    /* PyObject_Malloc(), PyObject_Realloc() and PyObject_Free() */
+    PYMEM_DOMAIN_OBJ
+} PyMemAllocatorDomain;
+
+typedef struct {
+    /* user context passed as the first argument to the 3 functions */
+    void *ctx;
+
+    /* allocate a memory block */
+    void* (*malloc) (void *ctx, size_t size);
+
+    /* allocate or resize a memory block */
+    void* (*realloc) (void *ctx, void *ptr, size_t new_size);
+
+    /* release a memory block */
+    void (*free) (void *ctx, void *ptr);
+} PyMemAllocator;
+
+/* Get the memory block allocator of the specified domain. */
+PyAPI_FUNC(void) PyMem_GetAllocator(PyMemAllocatorDomain domain,
+                                    PyMemAllocator *allocator);
+
+/* Set the memory block allocator of the specified domain.
+
+   The new allocator must return a distinct non-NULL pointer when requesting
+   zero bytes.
+
+   For the PYMEM_DOMAIN_RAW domain, the allocator must be thread-safe: the GIL
+   is not held when the allocator is called.
+
+   If the new allocator is not a hook (don't call the previous allocator), the
+   PyMem_SetupDebugHooks() function must be called to reinstall the debug hooks
+   on top on the new allocator. */
+PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain,
+                                    PyMemAllocator *allocator);
+
+/* Setup hooks to detect bugs in the following Python memory allocator
+   functions:
+
+   - PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree()
+   - PyMem_Malloc(), PyMem_Realloc(), PyMem_Free()
+   - PyObject_Malloc(), PyObject_Realloc() and PyObject_Free()
+
+   Newly allocated memory is filled with the byte 0xCB, freed memory is filled
+   with the byte 0xDB. Additionnal checks:
+
+   - detect API violations, ex: PyObject_Free() called on a buffer allocated
+     by PyMem_Malloc()
+   - detect write before the start of the buffer (buffer underflow)
+   - detect write after the end of the buffer (buffer overflow)
+
+   The function does nothing if Python is not compiled is debug mode. */
+PyAPI_FUNC(void) PyMem_SetupDebugHooks(void);
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #3329: Implement the PEP 445: Add new APIs to customize Python memory
+  allocators.
+
 - Issue #18328: Reorder ops in PyThreadState_Delete*() functions. Now the
   tstate is first removed from TLS and then deallocated.
 
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -2511,6 +2511,161 @@
     Py_RETURN_NONE;
 }
 
+static PyObject *
+test_pymem_alloc0(PyObject *self)
+{
+    void *ptr;
+
+    ptr = PyMem_Malloc(0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_Malloc(0) returns NULL");
+        return NULL;
+    }
+    PyMem_Free(ptr);
+
+    ptr = PyObject_Malloc(0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyObject_Malloc(0) returns NULL");
+        return NULL;
+    }
+    PyObject_Free(ptr);
+
+    Py_RETURN_NONE;
+}
+
+typedef struct {
+    PyMemAllocator alloc;
+
+    size_t malloc_size;
+    void *realloc_ptr;
+    size_t realloc_new_size;
+    void *free_ptr;
+} alloc_hook_t;
+
+static void* hook_malloc (void* ctx, size_t size)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->malloc_size = size;
+    return hook->alloc.malloc(hook->alloc.ctx, size);
+}
+
+static void* hook_realloc (void* ctx, void* ptr, size_t new_size)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->realloc_ptr = ptr;
+    hook->realloc_new_size = new_size;
+    return hook->alloc.realloc(hook->alloc.ctx, ptr, new_size);
+}
+
+static void hook_free (void *ctx, void *ptr)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->free_ptr = ptr;
+    hook->alloc.free(hook->alloc.ctx, ptr);
+}
+
+static PyObject *
+test_setallocators(PyMemAllocatorDomain domain)
+{
+    PyObject *res = NULL;
+    const char *error_msg;
+    alloc_hook_t hook;
+    PyMemAllocator alloc;
+    size_t size, size2;
+    void *ptr, *ptr2;
+
+    hook.malloc_size = 0;
+    hook.realloc_ptr = NULL;
+    hook.realloc_new_size = 0;
+    hook.free_ptr = NULL;
+
+    alloc.ctx = &hook;
+    alloc.malloc = &hook_malloc;
+    alloc.realloc = &hook_realloc;
+    alloc.free = &hook_free;
+    PyMem_GetAllocator(domain, &hook.alloc);
+    PyMem_SetAllocator(domain, &alloc);
+
+    size = 42;
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: ptr = PyMem_RawMalloc(size); break;
+    case PYMEM_DOMAIN_MEM: ptr = PyMem_Malloc(size); break;
+    case PYMEM_DOMAIN_OBJ: ptr = PyObject_Malloc(size); break;
+    default: ptr = NULL; break;
+    }
+
+    if (ptr == NULL) {
+        error_msg = "malloc failed";
+        goto fail;
+    }
+
+    if (hook.malloc_size != size) {
+        error_msg = "malloc invalid size";
+        goto fail;
+    }
+
+    size2 = 200;
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: ptr2 = PyMem_RawRealloc(ptr, size2); break;
+    case PYMEM_DOMAIN_MEM: ptr2 = PyMem_Realloc(ptr, size2); break;
+    case PYMEM_DOMAIN_OBJ: ptr2 = PyObject_Realloc(ptr, size2); break;
+    }
+
+    if (ptr2 == NULL) {
+        error_msg = "realloc failed";
+        goto fail;
+    }
+
+    if (hook.realloc_ptr != ptr
+        || hook.realloc_new_size != size2) {
+        error_msg = "realloc invalid parameters";
+        goto fail;
+    }
+
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: PyMem_RawFree(ptr2); break;
+    case PYMEM_DOMAIN_MEM: PyMem_Free(ptr2); break;
+    case PYMEM_DOMAIN_OBJ: PyObject_Free(ptr2); break;
+    }
+
+    if (hook.free_ptr != ptr2) {
+        error_msg = "free invalid pointer";
+        goto fail;
+    }
+
+    Py_INCREF(Py_None);
+    res = Py_None;
+    goto finally;
+
+fail:
+    PyErr_SetString(PyExc_RuntimeError, error_msg);
+
+finally:
+    PyMem_SetAllocator(domain, &hook.alloc);
+    return res;
+}
+
+static PyObject *
+test_pymem_setrawallocators(PyObject *self)
+{
+    return test_setallocators(PYMEM_DOMAIN_RAW);
+}
+
+static PyObject *
+test_pymem_setallocators(PyObject *self)
+{
+    return test_setallocators(PYMEM_DOMAIN_MEM);
+}
+
+static PyObject *
+test_pyobject_setallocators(PyObject *self)
+{
+    return test_setallocators(PYMEM_DOMAIN_OBJ);
+}
+
 static PyMethodDef TestMethods[] = {
     {"raise_exception",         raise_exception,                 METH_VARARGS},
     {"raise_memoryerror",   (PyCFunction)raise_memoryerror,  METH_NOARGS},
@@ -2611,6 +2766,14 @@
     {"pytime_object_to_time_t", test_pytime_object_to_time_t,  METH_VARARGS},
     {"pytime_object_to_timeval", test_pytime_object_to_timeval,  METH_VARARGS},
     {"pytime_object_to_timespec", test_pytime_object_to_timespec,  METH_VARARGS},
+    {"test_pymem",
+     (PyCFunction)test_pymem_alloc0, METH_NOARGS},
+    {"test_pymem_alloc0",
+     (PyCFunction)test_pymem_setrawallocators, METH_NOARGS},
+    {"test_pymem_setallocators",
+     (PyCFunction)test_pymem_setallocators, METH_NOARGS},
+    {"test_pyobject_setallocators",
+     (PyCFunction)test_pyobject_setallocators, METH_NOARGS},
     {NULL, NULL} /* sentinel */
 };
 
diff --git a/Objects/object.c b/Objects/object.c
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -1859,26 +1859,6 @@
 Py_ssize_t (*_Py_abstract_hack)(PyObject *) = PyObject_Size;
 
 
-/* Python's malloc wrappers (see pymem.h) */
-
-void *
-PyMem_Malloc(size_t nbytes)
-{
-    return PyMem_MALLOC(nbytes);
-}
-
-void *
-PyMem_Realloc(void *p, size_t nbytes)
-{
-    return PyMem_REALLOC(p, nbytes);
-}
-
-void
-PyMem_Free(void *p)
-{
-    PyMem_FREE(p);
-}
-
 void
 _PyObject_DebugTypeStats(FILE *out)
 {
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -1,18 +1,326 @@
 #include "Python.h"
 
+/* Python's malloc wrappers (see pymem.h) */
+
+#ifdef PYMALLOC_DEBUG   /* WITH_PYMALLOC && PYMALLOC_DEBUG */
+/* Forward declaration */
+static void* _PyMem_DebugMalloc(void *ctx, size_t size);
+static void _PyMem_DebugFree(void *ctx, void *p);
+static void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size);
+
+static void _PyObject_DebugDumpAddress(const void *p);
+static void _PyMem_DebugCheckAddress(char api_id, const void *p);
+#endif
+
 #ifdef WITH_PYMALLOC
 
-#ifdef HAVE_MMAP
- #include <sys/mman.h>
- #ifdef MAP_ANONYMOUS
-  #define ARENAS_USE_MMAP
- #endif
+#ifdef MS_WINDOWS
+#  include <windows.h>
+#elif defined(HAVE_MMAP)
+#  include <sys/mman.h>
+#  ifdef MAP_ANONYMOUS
+#    define ARENAS_USE_MMAP
+#  endif
 #endif
 
+/* Forward declaration */
+static void* _PyObject_Malloc(void *ctx, size_t size);
+static void _PyObject_Free(void *ctx, void *p);
+static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
+#endif
+
+
+static void *
+_PyMem_RawMalloc(void *ctx, size_t size)
+{
+    /* PyMem_Malloc(0) means malloc(1). Some systems would return NULL
+       for malloc(0), which would be treated as an error. Some platforms would
+       return a pointer with no memory behind it, which would break pymalloc.
+       To solve these problems, allocate an extra byte. */
+    if (size == 0)
+        size = 1;
+    return malloc(size);
+}
+
+static void *
+_PyMem_RawRealloc(void *ctx, void *ptr, size_t size)
+{
+    if (size == 0)
+        size = 1;
+    return realloc(ptr, size);
+}
+
+static void
+_PyMem_RawFree(void *ctx, void *ptr)
+{
+    free(ptr);
+}
+
+
 #ifdef MS_WINDOWS
-#include <windows.h>
+static void *
+_PyObject_ArenaVirtualAlloc(void *ctx, size_t size)
+{
+    return VirtualAlloc(NULL, size,
+                        MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
+}
+
+static void
+_PyObject_ArenaVirtualFree(void *ctx, void *ptr, size_t size)
+{
+    VirtualFree(ptr, size, MEM_RELEASE);
+}
+
+#elif defined(ARENAS_USE_MMAP)
+static void *
+_PyObject_ArenaMmap(void *ctx, size_t size)
+{
+    void *ptr;
+    ptr = mmap(NULL, size, PROT_READ|PROT_WRITE,
+               MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+    if (ptr == MAP_FAILED)
+        return NULL;
+    assert(ptr != NULL);
+    return ptr;
+}
+
+static void
+_PyObject_ArenaMunmap(void *ctx, void *ptr, size_t size)
+{
+    munmap(ptr, size);
+}
+
+#else
+static void *
+_PyObject_ArenaMalloc(void *ctx, size_t size)
+{
+    return malloc(size);
+}
+
+static void
+_PyObject_ArenaFree(void *ctx, void *ptr, size_t size)
+{
+    free(ptr);
+}
 #endif
 
+
+#define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawRealloc, _PyMem_RawFree
+#ifdef WITH_PYMALLOC
+#define PYOBJECT_FUNCS _PyObject_Malloc, _PyObject_Realloc, _PyObject_Free
+#else
+#define PYOBJECT_FUNCS PYRAW_FUNCS
+#endif
+
+#ifdef PYMALLOC_DEBUG
+typedef struct {
+    /* We tag each block with an API ID in order to tag API violations */
+    char api_id;
+    PyMemAllocator alloc;
+} debug_alloc_api_t;
+static struct {
+    debug_alloc_api_t raw;
+    debug_alloc_api_t mem;
+    debug_alloc_api_t obj;
+} _PyMem_Debug = {
+    {'r', {NULL, PYRAW_FUNCS}},
+    {'m', {NULL, PYRAW_FUNCS}},
+    {'o', {NULL, PYOBJECT_FUNCS}}
+    };
+
+#define PYDEBUG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugRealloc, _PyMem_DebugFree
+#endif
+
+static PyMemAllocator _PyMem_Raw = {
+#ifdef PYMALLOC_DEBUG
+    &_PyMem_Debug.raw, PYDEBUG_FUNCS
+#else
+    NULL, PYRAW_FUNCS
+#endif
+    };
+
+static PyMemAllocator _PyMem = {
+#ifdef PYMALLOC_DEBUG
+    &_PyMem_Debug.mem, PYDEBUG_FUNCS
+#else
+    NULL, PYRAW_FUNCS
+#endif
+    };
+
+static PyMemAllocator _PyObject = {
+#ifdef PYMALLOC_DEBUG
+    &_PyMem_Debug.obj, PYDEBUG_FUNCS
+#else
+    NULL, PYOBJECT_FUNCS
+#endif
+    };
+
+#undef PYRAW_FUNCS
+#undef PYOBJECT_FUNCS
+#undef PYDEBUG_FUNCS
+
+static PyObjectArenaAllocator _PyObject_Arena = {NULL,
+#ifdef MS_WINDOWS
+    _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree
+#elif defined(ARENAS_USE_MMAP)
+    _PyObject_ArenaMmap, _PyObject_ArenaMunmap
+#else
+    _PyObject_ArenaMalloc, _PyObject_ArenaFree
+#endif
+    };
+
+void
+PyMem_SetupDebugHooks(void)
+{
+#ifdef PYMALLOC_DEBUG
+    PyMemAllocator alloc;
+
+    alloc.malloc = _PyMem_DebugMalloc;
+    alloc.realloc = _PyMem_DebugRealloc;
+    alloc.free = _PyMem_DebugFree;
+
+    if (_PyMem_Raw.malloc != _PyMem_DebugMalloc) {
+        alloc.ctx = &_PyMem_Debug.raw;
+        PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &_PyMem_Debug.raw.alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc);
+    }
+
+    if (_PyMem.malloc != _PyMem_DebugMalloc) {
+        alloc.ctx = &_PyMem_Debug.mem;
+        PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc);
+    }
+
+    if (_PyObject.malloc != _PyMem_DebugMalloc) {
+        alloc.ctx = &_PyMem_Debug.obj;
+        PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc);
+    }
+#endif
+}
+
+void
+PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
+{
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: *allocator = _PyMem_Raw; break;
+    case PYMEM_DOMAIN_MEM: *allocator = _PyMem; break;
+    case PYMEM_DOMAIN_OBJ: *allocator = _PyObject; break;
+    default:
+        /* unknown domain */
+        allocator->ctx = NULL;
+        allocator->malloc = NULL;
+        allocator->realloc = NULL;
+        allocator->free = NULL;
+    }
+}
+
+void
+PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
+{
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: _PyMem_Raw = *allocator; break;
+    case PYMEM_DOMAIN_MEM: _PyMem = *allocator; break;
+    case PYMEM_DOMAIN_OBJ: _PyObject = *allocator; break;
+    /* ignore unknown domain */
+    }
+
+}
+
+void
+PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)
+{
+    *allocator = _PyObject_Arena;
+}
+
+void
+PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)
+{
+    _PyObject_Arena = *allocator;
+}
+
+void *
+PyMem_RawMalloc(size_t size)
+{
+    /*
+     * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.
+     * Most python internals blindly use a signed Py_ssize_t to track
+     * things without checking for overflows or negatives.
+     * As size_t is unsigned, checking for size < 0 is not required.
+     */
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+
+    return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);
+}
+
+void*
+PyMem_RawRealloc(void *ptr, size_t new_size)
+{
+    /* see PyMem_RawMalloc() */
+    if (new_size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size);
+}
+
+void PyMem_RawFree(void *ptr)
+{
+    _PyMem_Raw.free(_PyMem_Raw.ctx, ptr);
+}
+
+void *
+PyMem_Malloc(size_t size)
+{
+    /* see PyMem_RawMalloc() */
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    return _PyMem.malloc(_PyMem.ctx, size);
+}
+
+void *
+PyMem_Realloc(void *ptr, size_t new_size)
+{
+    /* see PyMem_RawMalloc() */
+    if (new_size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    return _PyMem.realloc(_PyMem.ctx, ptr, new_size);
+}
+
+void
+PyMem_Free(void *ptr)
+{
+    _PyMem.free(_PyMem.ctx, ptr);
+}
+
+void *
+PyObject_Malloc(size_t size)
+{
+    /* see PyMem_RawMalloc() */
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    return _PyObject.malloc(_PyObject.ctx, size);
+}
+
+void *
+PyObject_Realloc(void *ptr, size_t new_size)
+{
+    /* see PyMem_RawMalloc() */
+    if (new_size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    return _PyObject.realloc(_PyObject.ctx, ptr, new_size);
+}
+
+void
+PyObject_Free(void *ptr)
+{
+    _PyObject.free(_PyObject.ctx, ptr);
+}
+
+
+#ifdef WITH_PYMALLOC
+
 #ifdef WITH_VALGRIND
 #include <valgrind/valgrind.h>
 
@@ -549,7 +857,6 @@
     struct arena_object* arenaobj;
     uint excess;        /* number of bytes above pool alignment */
     void *address;
-    int err;
 
 #ifdef PYMALLOC_DEBUG
     if (Py_GETENV("PYTHONMALLOCSTATS"))
@@ -571,7 +878,7 @@
             return NULL;                /* overflow */
 #endif
         nbytes = numarenas * sizeof(*arenas);
-        arenaobj = (struct arena_object *)realloc(arenas, nbytes);
+        arenaobj = (struct arena_object *)PyMem_Realloc(arenas, nbytes);
         if (arenaobj == NULL)
             return NULL;
         arenas = arenaobj;
@@ -602,19 +909,8 @@
     arenaobj = unused_arena_objects;
     unused_arena_objects = arenaobj->nextarena;
     assert(arenaobj->address == 0);
-#ifdef MS_WINDOWS
-    address = (void*)VirtualAlloc(NULL, ARENA_SIZE,
-                                 MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
-    err = (address == NULL);
-#elif defined(ARENAS_USE_MMAP)
-    address = mmap(NULL, ARENA_SIZE, PROT_READ|PROT_WRITE,
-                                   MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
-    err = (address == MAP_FAILED);
-#else
-    address = malloc(ARENA_SIZE);
-    err = (address == 0);
-#endif
-    if (err) {
+    address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE);
+    if (address == NULL) {
         /* The allocation failed: return NULL after putting the
          * arenaobj back.
          */
@@ -777,9 +1073,8 @@
  * Unless the optimizer reorders everything, being too smart...
  */
 
-#undef PyObject_Malloc
-void *
-PyObject_Malloc(size_t nbytes)
+static void *
+_PyObject_Malloc(void *ctx, size_t nbytes)
 {
     block *bp;
     poolp pool;
@@ -796,17 +1091,6 @@
 #endif
 
     /*
-     * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.
-     * Most python internals blindly use a signed Py_ssize_t to track
-     * things without checking for overflows or negatives.
-     * As size_t is unsigned, checking for nbytes < 0 is not required.
-     */
-    if (nbytes > PY_SSIZE_T_MAX) {
-        _Py_AllocatedBlocks--;
-        return NULL;
-    }
-
-    /*
      * This implicitly redirects malloc(0).
      */
     if ((nbytes - 1) < SMALL_REQUEST_THRESHOLD) {
@@ -978,10 +1262,8 @@
      * last chance to serve the request) or when the max memory limit
      * has been reached.
      */
-    if (nbytes == 0)
-        nbytes = 1;
     {
-        void *result = malloc(nbytes);
+        void *result = PyMem_Malloc(nbytes);
         if (!result)
             _Py_AllocatedBlocks--;
         return result;
@@ -990,9 +1272,8 @@
 
 /* free */
 
-#undef PyObject_Free
-void
-PyObject_Free(void *p)
+static void
+_PyObject_Free(void *ctx, void *p)
 {
     poolp pool;
     block *lastfree;
@@ -1101,13 +1382,8 @@
                 unused_arena_objects = ao;
 
                 /* Free the entire arena. */
-#ifdef MS_WINDOWS
-                VirtualFree((void *)ao->address, 0, MEM_RELEASE);
-#elif defined(ARENAS_USE_MMAP)
-                munmap((void *)ao->address, ARENA_SIZE);
-#else
-                free((void *)ao->address);
-#endif
+                _PyObject_Arena.free(_PyObject_Arena.ctx,
+                                     (void *)ao->address, ARENA_SIZE);
                 ao->address = 0;                        /* mark unassociated */
                 --narenas_currently_allocated;
 
@@ -1216,7 +1492,7 @@
 redirect:
 #endif
     /* We didn't allocate this address. */
-    free(p);
+    PyMem_Free(p);
 }
 
 /* realloc.  If p is NULL, this acts like malloc(nbytes).  Else if nbytes==0,
@@ -1224,9 +1500,8 @@
  * return a non-NULL result.
  */
 
-#undef PyObject_Realloc
-void *
-PyObject_Realloc(void *p, size_t nbytes)
+static void *
+_PyObject_Realloc(void *ctx, void *p, size_t nbytes)
 {
     void *bp;
     poolp pool;
@@ -1236,16 +1511,7 @@
 #endif
 
     if (p == NULL)
-        return PyObject_Malloc(nbytes);
-
-    /*
-     * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.
-     * Most python internals blindly use a signed Py_ssize_t to track
-     * things without checking for overflows or negatives.
-     * As size_t is unsigned, checking for nbytes < 0 is not required.
-     */
-    if (nbytes > PY_SSIZE_T_MAX)
-        return NULL;
+        return _PyObject_Malloc(ctx, nbytes);
 
 #ifdef WITH_VALGRIND
     /* Treat running_on_valgrind == -1 the same as 0 */
@@ -1273,10 +1539,10 @@
             }
             size = nbytes;
         }
-        bp = PyObject_Malloc(nbytes);
+        bp = _PyObject_Malloc(ctx, nbytes);
         if (bp != NULL) {
             memcpy(bp, p, size);
-            PyObject_Free(p);
+            _PyObject_Free(ctx, p);
         }
         return bp;
     }
@@ -1294,14 +1560,14 @@
      * at p.  Instead we punt:  let C continue to manage this block.
      */
     if (nbytes)
-        return realloc(p, nbytes);
+        return PyMem_Realloc(p, nbytes);
     /* C doesn't define the result of realloc(p, 0) (it may or may not
      * return NULL then), but Python's docs promise that nbytes==0 never
      * returns NULL.  We don't pass 0 to realloc(), to avoid that endcase
      * to begin with.  Even then, we can't be sure that realloc() won't
      * return NULL.
      */
-    bp = realloc(p, 1);
+    bp = PyMem_Realloc(p, 1);
     return bp ? bp : p;
 }
 
@@ -1311,24 +1577,6 @@
 /* pymalloc not enabled:  Redirect the entry points to malloc.  These will
  * only be used by extensions that are compiled with pymalloc enabled. */
 
-void *
-PyObject_Malloc(size_t n)
-{
-    return PyMem_MALLOC(n);
-}
-
-void *
-PyObject_Realloc(void *p, size_t n)
-{
-    return PyMem_REALLOC(p, n);
-}
-
-void
-PyObject_Free(void *p)
-{
-    PyMem_FREE(p);
-}
-
 Py_ssize_t
 _Py_GetAllocatedBlocks(void)
 {
@@ -1354,10 +1602,6 @@
 #define DEADBYTE       0xDB    /* dead (newly freed) memory */
 #define FORBIDDENBYTE  0xFB    /* untouchable bytes at each end of a block */
 
-/* We tag each block with an API ID in order to tag API violations */
-#define _PYMALLOC_MEM_ID 'm'   /* the PyMem_Malloc() API */
-#define _PYMALLOC_OBJ_ID 'o'   /* The PyObject_Malloc() API */
-
 static size_t serialno = 0;     /* incremented on each debug {m,re}alloc */
 
 /* serialno is always incremented via calling this routine.  The point is
@@ -1440,58 +1684,18 @@
 p[2*S+n: 2*S+n+S]
     Copies of FORBIDDENBYTE.  Used to catch over- writes and reads.
 p[2*S+n+S: 2*S+n+2*S]
-    A serial number, incremented by 1 on each call to _PyObject_DebugMalloc
-    and _PyObject_DebugRealloc.
+    A serial number, incremented by 1 on each call to _PyMem_DebugMalloc
+    and _PyMem_DebugRealloc.
     This is a big-endian size_t.
     If "bad memory" is detected later, the serial number gives an
     excellent way to set a breakpoint on the next run, to capture the
     instant at which this block was passed out.
 */
 
-/* debug replacements for the PyMem_* memory API */
-void *
-_PyMem_DebugMalloc(size_t nbytes)
+static void *
+_PyMem_DebugMalloc(void *ctx, size_t nbytes)
 {
-    return _PyObject_DebugMallocApi(_PYMALLOC_MEM_ID, nbytes);
-}
-void *
-_PyMem_DebugRealloc(void *p, size_t nbytes)
-{
-    return _PyObject_DebugReallocApi(_PYMALLOC_MEM_ID, p, nbytes);
-}
-void
-_PyMem_DebugFree(void *p)
-{
-    _PyObject_DebugFreeApi(_PYMALLOC_MEM_ID, p);
-}
-
-/* debug replacements for the PyObject_* memory API */
-void *
-_PyObject_DebugMalloc(size_t nbytes)
-{
-    return _PyObject_DebugMallocApi(_PYMALLOC_OBJ_ID, nbytes);
-}
-void *
-_PyObject_DebugRealloc(void *p, size_t nbytes)
-{
-    return _PyObject_DebugReallocApi(_PYMALLOC_OBJ_ID, p, nbytes);
-}
-void
-_PyObject_DebugFree(void *p)
-{
-    _PyObject_DebugFreeApi(_PYMALLOC_OBJ_ID, p);
-}
-void
-_PyObject_DebugCheckAddress(const void *p)
-{
-    _PyObject_DebugCheckAddressApi(_PYMALLOC_OBJ_ID, p);
-}
-
-
-/* generic debug memory api, with an "id" to identify the API in use */
-void *
-_PyObject_DebugMallocApi(char id, size_t nbytes)
-{
+    debug_alloc_api_t *api = (debug_alloc_api_t *)ctx;
     uchar *p;           /* base address of malloc'ed block */
     uchar *tail;        /* p + 2*SST + nbytes == pointer to tail pad bytes */
     size_t total;       /* nbytes + 4*SST */
@@ -1502,14 +1706,14 @@
         /* overflow:  can't represent total as a size_t */
         return NULL;
 
-    p = (uchar *)PyObject_Malloc(total);
+    p = (uchar *)api->alloc.malloc(api->alloc.ctx, total);
     if (p == NULL)
         return NULL;
 
     /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
     write_size_t(p, nbytes);
-    p[SST] = (uchar)id;
-    memset(p + SST + 1 , FORBIDDENBYTE, SST-1);
+    p[SST] = (uchar)api->api_id;
+    memset(p + SST + 1, FORBIDDENBYTE, SST-1);
 
     if (nbytes > 0)
         memset(p + 2*SST, CLEANBYTE, nbytes);
@@ -1527,25 +1731,27 @@
    Then fills the original bytes with DEADBYTE.
    Then calls the underlying free.
 */
-void
-_PyObject_DebugFreeApi(char api, void *p)
+static void
+_PyMem_DebugFree(void *ctx, void *p)
 {
+    debug_alloc_api_t *api = (debug_alloc_api_t *)ctx;
     uchar *q = (uchar *)p - 2*SST;  /* address returned from malloc */
     size_t nbytes;
 
     if (p == NULL)
         return;
-    _PyObject_DebugCheckAddressApi(api, p);
+    _PyMem_DebugCheckAddress(api->api_id, p);
     nbytes = read_size_t(q);
     nbytes += 4*SST;
     if (nbytes > 0)
         memset(q, DEADBYTE, nbytes);
-    PyObject_Free(q);
+    api->alloc.free(api->alloc.ctx, q);
 }
 
-void *
-_PyObject_DebugReallocApi(char api, void *p, size_t nbytes)
+static void *
+_PyMem_DebugRealloc(void *ctx, void *p, size_t nbytes)
 {
+    debug_alloc_api_t *api = (debug_alloc_api_t *)ctx;
     uchar *q = (uchar *)p;
     uchar *tail;
     size_t total;       /* nbytes + 4*SST */
@@ -1553,9 +1759,9 @@
     int i;
 
     if (p == NULL)
-        return _PyObject_DebugMallocApi(api, nbytes);
+        return _PyMem_DebugMalloc(ctx, nbytes);
 
-    _PyObject_DebugCheckAddressApi(api, p);
+    _PyMem_DebugCheckAddress(api->api_id, p);
     bumpserialno();
     original_nbytes = read_size_t(q - 2*SST);
     total = nbytes + 4*SST;
@@ -1572,12 +1778,12 @@
      * case we didn't get the chance to mark the old memory with DEADBYTE,
      * but we live with that.
      */
-    q = (uchar *)PyObject_Realloc(q - 2*SST, total);
+    q = (uchar *)api->alloc.realloc(api->alloc.ctx, q - 2*SST, total);
     if (q == NULL)
         return NULL;
 
     write_size_t(q, nbytes);
-    assert(q[SST] == (uchar)api);
+    assert(q[SST] == (uchar)api->api_id);
     for (i = 1; i < SST; ++i)
         assert(q[SST + i] == FORBIDDENBYTE);
     q += 2*SST;
@@ -1599,8 +1805,8 @@
  * and call Py_FatalError to kill the program.
  * The API id, is also checked.
  */
- void
-_PyObject_DebugCheckAddressApi(char api, const void *p)
+static void
+_PyMem_DebugCheckAddress(char api, const void *p)
 {
     const uchar *q = (const uchar *)p;
     char msgbuf[64];
@@ -1652,7 +1858,7 @@
 }
 
 /* Display info to stderr about the memory block at p. */
-void
+static void
 _PyObject_DebugDumpAddress(const void *p)
 {
     const uchar *q = (const uchar *)p;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 02:50:07 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 02:50:07 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2317206=3A_On_Windo?=
 =?utf-8?q?ws=2C_increase_the_stack_size_from_2_MB_to_4=2E2_MB_to_fix?=
Message-ID: <3bnrmb2gtdz7LjW@mail.python.org>

http://hg.python.org/cpython/rev/fcf079dece0d
changeset:   84472:fcf079dece0d
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 02:49:07 2013 +0200
summary:
  Issue #17206: On Windows, increase the stack size from 2 MB to 4.2 MB to fix
a stack overflow in the marshal module (fix a crash in test_marshal).
Patch written by Jeremy Kloth.

files:
  Misc/NEWS              |  4 ++++
  PCbuild/python.vcxproj |  2 +-
  2 files changed, 5 insertions(+), 1 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,10 @@
 Core and Builtins
 -----------------
 
+- Issue #17206: On Windows, increase the stack size from 2 MB to 4.2 MB to fix
+  a stack overflow in the marshal module (fix a crash in test_marshal).
+  Patch written by Jeremy Kloth.
+
 - Issue #3329: Implement the PEP 445: Add new APIs to customize Python memory
   allocators.
 
diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj
--- a/PCbuild/python.vcxproj
+++ b/PCbuild/python.vcxproj
@@ -243,7 +243,7 @@
     <Link>
       <OutputFile>$(OutDir)python_d.exe</OutputFile>
       <SubSystem>Console</SubSystem>
-      <StackReserveSize>2100000</StackReserveSize>
+      <StackReserveSize>4194304</StackReserveSize>
       <BaseAddress>0x1d000000</BaseAddress>
     </Link>
   </ItemDefinitionGroup>

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 03:06:31 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 03:06:31 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=233329=3A_Fix_=5FPy?=
 =?utf-8?q?Object=5FArenaVirtualFree=28=29?=
Message-ID: <3bns7W2Fvbz7LkR@mail.python.org>

http://hg.python.org/cpython/rev/51ed51d10e60
changeset:   84473:51ed51d10e60
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 03:06:16 2013 +0200
summary:
  Issue #3329: Fix _PyObject_ArenaVirtualFree()

According to VirtualFree() documentation, the size must be zero if the "free
type" is MEM_RELEASE.

files:
  Objects/obmalloc.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -68,7 +68,7 @@
 static void
 _PyObject_ArenaVirtualFree(void *ctx, void *ptr, size_t size)
 {
-    VirtualFree(ptr, size, MEM_RELEASE);
+    VirtualFree(ptr, 0, MEM_RELEASE);
 }
 
 #elif defined(ARENAS_USE_MMAP)

-- 
Repository URL: http://hg.python.org/cpython

From solipsis at pitrou.net  Sun Jul  7 05:46:07 2013
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Sun, 07 Jul 2013 05:46:07 +0200
Subject: [Python-checkins] Daily reference leaks (51ed51d10e60): sum=5
Message-ID: <E1UvfvL-00078h-Ez@ap.vmr.nerim.net>

results for 51ed51d10e60 on branch "default"
--------------------------------------------

test_support leaked [1, 0, 0] references, sum=1
test_support leaked [1, 2, 1] memory blocks, sum=4


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflog6zGxiG', '-x']

From python-checkins at python.org  Sun Jul  7 05:50:11 2013
From: python-checkins at python.org (raymond.hettinger)
Date: Sun,  7 Jul 2013 05:50:11 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Improve_variable_names_in_?=
	=?utf-8?q?deque=5Fcount=28=29?=
Message-ID: <3bnwmM364Vz7LjX@mail.python.org>

http://hg.python.org/cpython/rev/e7025b1e69ce
changeset:   84474:e7025b1e69ce
parent:      84470:9166aa413d6f
user:        Raymond Hettinger <python at rcn.com>
date:        Sat Jul 06 17:49:06 2013 -1000
summary:
  Improve variable names in deque_count()

files:
  Modules/_collectionsmodule.c |  16 ++++++++--------
  1 files changed, 8 insertions(+), 8 deletions(-)


diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -581,8 +581,8 @@
 static PyObject *
 deque_count(dequeobject *deque, PyObject *v)
 {
-    block *leftblock = deque->leftblock;
-    Py_ssize_t leftindex = deque->leftindex;
+    block *b = deque->leftblock;
+    Py_ssize_t index = deque->leftindex;
     Py_ssize_t n = Py_SIZE(deque);
     Py_ssize_t i;
     Py_ssize_t count = 0;
@@ -591,8 +591,8 @@
     int cmp;
 
     for (i=0 ; i<n ; i++) {
-        assert(leftblock != NULL);
-        item = leftblock->data[leftindex];
+        assert(b != NULL);
+        item = b->data[index];
         cmp = PyObject_RichCompareBool(item, v, Py_EQ);
         if (cmp > 0)
             count++;
@@ -606,10 +606,10 @@
         }
 
         /* Advance left block/index pair */
-        leftindex++;
-        if (leftindex == BLOCKLEN) {
-            leftblock = leftblock->rightlink;
-            leftindex = 0;
+        index++;
+        if (index == BLOCKLEN) {
+            b = b->rightlink;
+            index = 0;
         }
     }
     return PyLong_FromSsize_t(count);

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 05:50:13 2013
From: python-checkins at python.org (raymond.hettinger)
Date: Sun,  7 Jul 2013 05:50:13 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?=
	=?utf-8?q?=29=3A_merge?=
Message-ID: <3bnwmP23bCz7LkC@mail.python.org>

http://hg.python.org/cpython/rev/cc0c8c1c9cca
changeset:   84475:cc0c8c1c9cca
parent:      84474:e7025b1e69ce
parent:      84473:51ed51d10e60
user:        Raymond Hettinger <python at rcn.com>
date:        Sat Jul 06 17:50:01 2013 -1000
summary:
  merge

files:
  Doc/c-api/memory.rst      |  171 ++++++++-
  Doc/whatsnew/3.4.rst      |   18 +-
  Include/objimpl.h         |   60 +-
  Include/pymem.h           |   94 +++-
  Misc/NEWS                 |    7 +
  Modules/_testcapimodule.c |  163 ++++++++
  Objects/object.c          |   20 -
  Objects/obmalloc.c        |  504 ++++++++++++++++++-------
  PCbuild/python.vcxproj    |    2 +-
  9 files changed, 801 insertions(+), 238 deletions(-)


diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst
--- a/Doc/c-api/memory.rst
+++ b/Doc/c-api/memory.rst
@@ -84,6 +84,48 @@
 for the I/O buffer escapes completely the Python memory manager.
 
 
+Raw Memory Interface
+====================
+
+The following function sets are wrappers to the system allocator. These
+functions are thread-safe, the :term:`GIL <global interpreter lock>` does not
+need to be held.
+
+The default raw memory block allocator uses the following functions:
+:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
+requesting zero bytes.
+
+.. versionadded:: 3.4
+
+.. c:function:: void* PyMem_RawMalloc(size_t n)
+
+   Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the
+   allocated memory, or *NULL* if the request fails. Requesting zero bytes
+   returns a distinct non-*NULL* pointer if possible, as if
+   ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have
+   been initialized in any way.
+
+
+.. c:function:: void* PyMem_RawRealloc(void *p, size_t n)
+
+   Resizes the memory block pointed to by *p* to *n* bytes. The contents will
+   be unchanged to the minimum of the old and the new sizes. If *p* is *NULL*,
+   the call is equivalent to ``PyMem_RawMalloc(n)``; else if *n* is equal to
+   zero, the memory block is resized but is not freed, and the returned pointer
+   is non-*NULL*. Unless *p* is *NULL*, it must have been returned by a
+   previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`. If
+   the request fails, :c:func:`PyMem_RawRealloc` returns *NULL* and *p* remains
+   a valid pointer to the previous memory area.
+
+
+.. c:function:: void PyMem_RawFree(void *p)
+
+   Frees the memory block pointed to by *p*, which must have been returned by a
+   previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`.
+   Otherwise, or if ``PyMem_Free(p)`` has been called before, undefined
+   behavior occurs. If *p* is *NULL*, no operation is performed.
+
+
 .. _memoryinterface:
 
 Memory Interface
@@ -91,8 +133,16 @@
 
 The following function sets, modeled after the ANSI C standard, but specifying
 behavior when requesting zero bytes, are available for allocating and releasing
-memory from the Python heap:
+memory from the Python heap.
 
+The default memory block allocator uses the following functions:
+:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
+requesting zero bytes.
+
+.. warning::
+
+   The :term:`GIL <global interpreter lock>` must be held when using these
+   functions.
 
 .. c:function:: void* PyMem_Malloc(size_t n)
 
@@ -155,6 +205,125 @@
 :c:func:`PyMem_NEW`, :c:func:`PyMem_RESIZE`, :c:func:`PyMem_DEL`.
 
 
+Customize Memory Allocators
+===========================
+
+.. versionadded:: 3.4
+
+.. c:type:: PyMemAllocator
+
+   Structure used to describe a memory block allocator. The structure has
+   four fields:
+
+   +----------------------------------------------------------+---------------------------------------+
+   | Field                                                    | Meaning                               |
+   +==========================================================+=======================================+
+   | ``void *ctx``                                            | user context passed as first argument |
+   +----------------------------------------------------------+---------------------------------------+
+   | ``void* malloc(void *ctx, size_t size)``                 | allocate a memory block               |
+   +----------------------------------------------------------+---------------------------------------+
+   | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block     |
+   +----------------------------------------------------------+---------------------------------------+
+   | ``void free(void *ctx, void *ptr)``                      | free a memory block                   |
+   +----------------------------------------------------------+---------------------------------------+
+
+.. c:type:: PyMemAllocatorDomain
+
+   Enum used to identify an allocator domain. Domains:
+
+   * :c:data:`PYMEM_DOMAIN_RAW`: functions :c:func:`PyMem_RawMalloc`,
+     :c:func:`PyMem_RawRealloc` and :c:func:`PyMem_RawFree`
+   * :c:data:`PYMEM_DOMAIN_MEM`: functions :c:func:`PyMem_Malloc`,
+     :c:func:`PyMem_Realloc` and :c:func:`PyMem_Free`
+   * :c:data:`PYMEM_DOMAIN_OBJ`: functions :c:func:`PyObject_Malloc`,
+     :c:func:`PyObject_Realloc` and :c:func:`PyObject_Free`
+
+
+.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
+
+   Get the memory block allocator of the specified domain.
+
+
+.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
+
+   Set the memory block allocator of the specified domain.
+
+   The new allocator must return a distinct non-NULL pointer when requesting
+   zero bytes.
+
+   For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be
+   thread-safe: the :term:`GIL <global interpreter lock>` is not held when the
+   allocator is called.
+
+   If the new allocator is not a hook (does not call the previous allocator),
+   the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the
+   debug hooks on top on the new allocator.
+
+
+.. c:function:: void PyMem_SetupDebugHooks(void)
+
+   Setup hooks to detect bugs in the following Python memory allocator
+   functions:
+
+   - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc`,
+     :c:func:`PyMem_RawFree`
+   - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc`, :c:func:`PyMem_Free`
+   - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc`,
+     :c:func:`PyObject_Free`
+
+   Newly allocated memory is filled with the byte ``0xCB``, freed memory is
+   filled with the byte ``0xDB``. Additionnal checks:
+
+   - detect API violations, ex: :c:func:`PyObject_Free` called on a buffer
+     allocated by :c:func:`PyMem_Malloc`
+   - detect write before the start of the buffer (buffer underflow)
+   - detect write after the end of the buffer (buffer overflow)
+
+   The function does nothing if Python is not compiled is debug mode.
+
+
+Customize PyObject Arena Allocator
+==================================
+
+Python has a *pymalloc* allocator for allocations smaller than 512 bytes. This
+allocator is optimized for small objects with a short lifetime. It uses memory
+mappings called "arenas" with a fixed size of 256 KB. It falls back to
+:c:func:`PyMem_Malloc` and :c:func:`PyMem_Realloc` for allocations larger than
+512 bytes.  *pymalloc* is the default allocator used by
+:c:func:`PyObject_Malloc`.
+
+The default arena allocator uses the following functions:
+
+* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows,
+* :c:func:`mmap` and :c:func:`munmap` if available,
+* :c:func:`malloc` and :c:func:`free` otherwise.
+
+.. versionadded:: 3.4
+
+.. c:type:: PyObjectArenaAllocator
+
+   Structure used to describe an arena allocator. The structure has
+   three fields:
+
+   +--------------------------------------------------+---------------------------------------+
+   | Field                                            | Meaning                               |
+   +==================================================+=======================================+
+   | ``void *ctx``                                    | user context passed as first argument |
+   +--------------------------------------------------+---------------------------------------+
+   | ``void* alloc(void *ctx, size_t size)``          | allocate an arena of size bytes       |
+   +--------------------------------------------------+---------------------------------------+
+   | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena                         |
+   +--------------------------------------------------+---------------------------------------+
+
+.. c:function:: PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)
+
+   Get the arena allocator.
+
+.. c:function:: PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)
+
+   Set the arena allocator.
+
+
 .. _memoryexamples:
 
 Examples
diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst
--- a/Doc/whatsnew/3.4.rst
+++ b/Doc/whatsnew/3.4.rst
@@ -112,21 +112,11 @@
 Please read on for a comprehensive list of user-facing changes.
 
 
-.. PEP-sized items next.
+PEP 445: Add new APIs to customize Python memory allocators
+===========================================================
 
-.. _pep-4XX:
-
-.. PEP 4XX: Example PEP
-.. ====================
-
-
-.. (Implemented by Foo Bar.)
-
-.. .. seealso::
-
-   :pep:`4XX` - Example PEP
-      PEP written by Example Author
-
+The :pep:`445` adds new Application Programming Interfaces (API) to customize
+Python memory allocators.
 
 
 
diff --git a/Include/objimpl.h b/Include/objimpl.h
--- a/Include/objimpl.h
+++ b/Include/objimpl.h
@@ -94,9 +94,9 @@
    the object gets initialized via PyObject_{Init, InitVar} after obtaining
    the raw memory.
 */
-PyAPI_FUNC(void *) PyObject_Malloc(size_t);
-PyAPI_FUNC(void *) PyObject_Realloc(void *, size_t);
-PyAPI_FUNC(void) PyObject_Free(void *);
+PyAPI_FUNC(void *) PyObject_Malloc(size_t size);
+PyAPI_FUNC(void *) PyObject_Realloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyObject_Free(void *ptr);
 
 /* This function returns the number of allocated memory blocks, regardless of size */
 PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
@@ -106,41 +106,15 @@
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out);
 #endif /* #ifndef Py_LIMITED_API */
-#ifdef PYMALLOC_DEBUG   /* WITH_PYMALLOC && PYMALLOC_DEBUG */
-PyAPI_FUNC(void *) _PyObject_DebugMalloc(size_t nbytes);
-PyAPI_FUNC(void *) _PyObject_DebugRealloc(void *p, size_t nbytes);
-PyAPI_FUNC(void) _PyObject_DebugFree(void *p);
-PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p);
-PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p);
-PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes);
-PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes);
-PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p);
-PyAPI_FUNC(void) _PyObject_DebugCheckAddressApi(char api, const void *p);
-PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes);
-PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes);
-PyAPI_FUNC(void) _PyMem_DebugFree(void *p);
-#define PyObject_MALLOC         _PyObject_DebugMalloc
-#define PyObject_Malloc         _PyObject_DebugMalloc
-#define PyObject_REALLOC        _PyObject_DebugRealloc
-#define PyObject_Realloc        _PyObject_DebugRealloc
-#define PyObject_FREE           _PyObject_DebugFree
-#define PyObject_Free           _PyObject_DebugFree
+#endif
 
-#else   /* WITH_PYMALLOC && ! PYMALLOC_DEBUG */
+/* Macros */
 #define PyObject_MALLOC         PyObject_Malloc
 #define PyObject_REALLOC        PyObject_Realloc
 #define PyObject_FREE           PyObject_Free
-#endif
+#define PyObject_Del            PyObject_Free
+#define PyObject_DEL            PyObject_Free
 
-#else   /* ! WITH_PYMALLOC */
-#define PyObject_MALLOC         PyMem_MALLOC
-#define PyObject_REALLOC        PyMem_REALLOC
-#define PyObject_FREE           PyMem_FREE
-
-#endif  /* WITH_PYMALLOC */
-
-#define PyObject_Del            PyObject_Free
-#define PyObject_DEL            PyObject_FREE
 
 /*
  * Generic object allocator interface
@@ -224,6 +198,26 @@
    constructor you would start directly with PyObject_Init/InitVar
 */
 
+#ifndef Py_LIMITED_API
+typedef struct {
+    /* user context passed as the first argument to the 2 functions */
+    void *ctx;
+
+    /* allocate an arena of size bytes */
+    void* (*alloc) (void *ctx, size_t size);
+
+    /* free an arena */
+    void (*free) (void *ctx, void *ptr, size_t size);
+} PyObjectArenaAllocator;
+
+/* Get the arena allocator. */
+PyAPI_FUNC(void) PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator);
+
+/* Set the arena allocator. */
+PyAPI_FUNC(void) PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator);
+#endif
+
+
 /*
  * Garbage Collection Support
  * ==========================
diff --git a/Include/pymem.h b/Include/pymem.h
--- a/Include/pymem.h
+++ b/Include/pymem.h
@@ -11,6 +11,11 @@
 extern "C" {
 #endif
 
+PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
+PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
+
+
 /* BEWARE:
 
    Each interface exports both functions and macros.  Extension modules should
@@ -49,21 +54,11 @@
    performed on failure (no exception is set, no warning is printed, etc).
 */
 
-PyAPI_FUNC(void *) PyMem_Malloc(size_t);
-PyAPI_FUNC(void *) PyMem_Realloc(void *, size_t);
-PyAPI_FUNC(void) PyMem_Free(void *);
-
-/* Starting from Python 1.6, the wrappers Py_{Malloc,Realloc,Free} are
-   no longer supported. They used to call PyErr_NoMemory() on failure. */
+PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
+PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_Free(void *ptr);
 
 /* Macros. */
-#ifdef PYMALLOC_DEBUG
-/* Redirect all memory operations to Python's debugging allocator. */
-#define PyMem_MALLOC		_PyMem_DebugMalloc
-#define PyMem_REALLOC		_PyMem_DebugRealloc
-#define PyMem_FREE		_PyMem_DebugFree
-
-#else	/* ! PYMALLOC_DEBUG */
 
 /* PyMem_MALLOC(0) means malloc(1). Some systems would return NULL
    for malloc(0), which would be treated as an error. Some platforms
@@ -71,13 +66,9 @@
    pymalloc. To solve these problems, allocate an extra byte. */
 /* Returns NULL to indicate error if a negative size or size larger than
    Py_ssize_t can represent is supplied.  Helps prevents security holes. */
-#define PyMem_MALLOC(n)		((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \
-				: malloc((n) ? (n) : 1))
-#define PyMem_REALLOC(p, n)	((size_t)(n) > (size_t)PY_SSIZE_T_MAX  ? NULL \
-				: realloc((p), (n) ? (n) : 1))
-#define PyMem_FREE		free
-
-#endif	/* PYMALLOC_DEBUG */
+#define PyMem_MALLOC(n)         PyMem_Malloc(n)
+#define PyMem_REALLOC(p, n)     PyMem_Realloc(p, n)
+#define PyMem_FREE(p)           PyMem_Free(p)
 
 /*
  * Type-oriented memory interface
@@ -115,6 +106,69 @@
 #define PyMem_Del		PyMem_Free
 #define PyMem_DEL		PyMem_FREE
 
+#ifndef Py_LIMITED_API
+typedef enum {
+    /* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */
+    PYMEM_DOMAIN_RAW,
+
+    /* PyMem_Malloc(), PyMem_Realloc() and PyMem_Free() */
+    PYMEM_DOMAIN_MEM,
+
+    /* PyObject_Malloc(), PyObject_Realloc() and PyObject_Free() */
+    PYMEM_DOMAIN_OBJ
+} PyMemAllocatorDomain;
+
+typedef struct {
+    /* user context passed as the first argument to the 3 functions */
+    void *ctx;
+
+    /* allocate a memory block */
+    void* (*malloc) (void *ctx, size_t size);
+
+    /* allocate or resize a memory block */
+    void* (*realloc) (void *ctx, void *ptr, size_t new_size);
+
+    /* release a memory block */
+    void (*free) (void *ctx, void *ptr);
+} PyMemAllocator;
+
+/* Get the memory block allocator of the specified domain. */
+PyAPI_FUNC(void) PyMem_GetAllocator(PyMemAllocatorDomain domain,
+                                    PyMemAllocator *allocator);
+
+/* Set the memory block allocator of the specified domain.
+
+   The new allocator must return a distinct non-NULL pointer when requesting
+   zero bytes.
+
+   For the PYMEM_DOMAIN_RAW domain, the allocator must be thread-safe: the GIL
+   is not held when the allocator is called.
+
+   If the new allocator is not a hook (don't call the previous allocator), the
+   PyMem_SetupDebugHooks() function must be called to reinstall the debug hooks
+   on top on the new allocator. */
+PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain,
+                                    PyMemAllocator *allocator);
+
+/* Setup hooks to detect bugs in the following Python memory allocator
+   functions:
+
+   - PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree()
+   - PyMem_Malloc(), PyMem_Realloc(), PyMem_Free()
+   - PyObject_Malloc(), PyObject_Realloc() and PyObject_Free()
+
+   Newly allocated memory is filled with the byte 0xCB, freed memory is filled
+   with the byte 0xDB. Additionnal checks:
+
+   - detect API violations, ex: PyObject_Free() called on a buffer allocated
+     by PyMem_Malloc()
+   - detect write before the start of the buffer (buffer underflow)
+   - detect write after the end of the buffer (buffer overflow)
+
+   The function does nothing if Python is not compiled is debug mode. */
+PyAPI_FUNC(void) PyMem_SetupDebugHooks(void);
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,13 @@
 Core and Builtins
 -----------------
 
+- Issue #17206: On Windows, increase the stack size from 2 MB to 4.2 MB to fix
+  a stack overflow in the marshal module (fix a crash in test_marshal).
+  Patch written by Jeremy Kloth.
+
+- Issue #3329: Implement the PEP 445: Add new APIs to customize Python memory
+  allocators.
+
 - Issue #18328: Reorder ops in PyThreadState_Delete*() functions. Now the
   tstate is first removed from TLS and then deallocated.
 
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -2511,6 +2511,161 @@
     Py_RETURN_NONE;
 }
 
+static PyObject *
+test_pymem_alloc0(PyObject *self)
+{
+    void *ptr;
+
+    ptr = PyMem_Malloc(0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_Malloc(0) returns NULL");
+        return NULL;
+    }
+    PyMem_Free(ptr);
+
+    ptr = PyObject_Malloc(0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyObject_Malloc(0) returns NULL");
+        return NULL;
+    }
+    PyObject_Free(ptr);
+
+    Py_RETURN_NONE;
+}
+
+typedef struct {
+    PyMemAllocator alloc;
+
+    size_t malloc_size;
+    void *realloc_ptr;
+    size_t realloc_new_size;
+    void *free_ptr;
+} alloc_hook_t;
+
+static void* hook_malloc (void* ctx, size_t size)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->malloc_size = size;
+    return hook->alloc.malloc(hook->alloc.ctx, size);
+}
+
+static void* hook_realloc (void* ctx, void* ptr, size_t new_size)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->realloc_ptr = ptr;
+    hook->realloc_new_size = new_size;
+    return hook->alloc.realloc(hook->alloc.ctx, ptr, new_size);
+}
+
+static void hook_free (void *ctx, void *ptr)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->free_ptr = ptr;
+    hook->alloc.free(hook->alloc.ctx, ptr);
+}
+
+static PyObject *
+test_setallocators(PyMemAllocatorDomain domain)
+{
+    PyObject *res = NULL;
+    const char *error_msg;
+    alloc_hook_t hook;
+    PyMemAllocator alloc;
+    size_t size, size2;
+    void *ptr, *ptr2;
+
+    hook.malloc_size = 0;
+    hook.realloc_ptr = NULL;
+    hook.realloc_new_size = 0;
+    hook.free_ptr = NULL;
+
+    alloc.ctx = &hook;
+    alloc.malloc = &hook_malloc;
+    alloc.realloc = &hook_realloc;
+    alloc.free = &hook_free;
+    PyMem_GetAllocator(domain, &hook.alloc);
+    PyMem_SetAllocator(domain, &alloc);
+
+    size = 42;
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: ptr = PyMem_RawMalloc(size); break;
+    case PYMEM_DOMAIN_MEM: ptr = PyMem_Malloc(size); break;
+    case PYMEM_DOMAIN_OBJ: ptr = PyObject_Malloc(size); break;
+    default: ptr = NULL; break;
+    }
+
+    if (ptr == NULL) {
+        error_msg = "malloc failed";
+        goto fail;
+    }
+
+    if (hook.malloc_size != size) {
+        error_msg = "malloc invalid size";
+        goto fail;
+    }
+
+    size2 = 200;
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: ptr2 = PyMem_RawRealloc(ptr, size2); break;
+    case PYMEM_DOMAIN_MEM: ptr2 = PyMem_Realloc(ptr, size2); break;
+    case PYMEM_DOMAIN_OBJ: ptr2 = PyObject_Realloc(ptr, size2); break;
+    }
+
+    if (ptr2 == NULL) {
+        error_msg = "realloc failed";
+        goto fail;
+    }
+
+    if (hook.realloc_ptr != ptr
+        || hook.realloc_new_size != size2) {
+        error_msg = "realloc invalid parameters";
+        goto fail;
+    }
+
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: PyMem_RawFree(ptr2); break;
+    case PYMEM_DOMAIN_MEM: PyMem_Free(ptr2); break;
+    case PYMEM_DOMAIN_OBJ: PyObject_Free(ptr2); break;
+    }
+
+    if (hook.free_ptr != ptr2) {
+        error_msg = "free invalid pointer";
+        goto fail;
+    }
+
+    Py_INCREF(Py_None);
+    res = Py_None;
+    goto finally;
+
+fail:
+    PyErr_SetString(PyExc_RuntimeError, error_msg);
+
+finally:
+    PyMem_SetAllocator(domain, &hook.alloc);
+    return res;
+}
+
+static PyObject *
+test_pymem_setrawallocators(PyObject *self)
+{
+    return test_setallocators(PYMEM_DOMAIN_RAW);
+}
+
+static PyObject *
+test_pymem_setallocators(PyObject *self)
+{
+    return test_setallocators(PYMEM_DOMAIN_MEM);
+}
+
+static PyObject *
+test_pyobject_setallocators(PyObject *self)
+{
+    return test_setallocators(PYMEM_DOMAIN_OBJ);
+}
+
 static PyMethodDef TestMethods[] = {
     {"raise_exception",         raise_exception,                 METH_VARARGS},
     {"raise_memoryerror",   (PyCFunction)raise_memoryerror,  METH_NOARGS},
@@ -2611,6 +2766,14 @@
     {"pytime_object_to_time_t", test_pytime_object_to_time_t,  METH_VARARGS},
     {"pytime_object_to_timeval", test_pytime_object_to_timeval,  METH_VARARGS},
     {"pytime_object_to_timespec", test_pytime_object_to_timespec,  METH_VARARGS},
+    {"test_pymem",
+     (PyCFunction)test_pymem_alloc0, METH_NOARGS},
+    {"test_pymem_alloc0",
+     (PyCFunction)test_pymem_setrawallocators, METH_NOARGS},
+    {"test_pymem_setallocators",
+     (PyCFunction)test_pymem_setallocators, METH_NOARGS},
+    {"test_pyobject_setallocators",
+     (PyCFunction)test_pyobject_setallocators, METH_NOARGS},
     {NULL, NULL} /* sentinel */
 };
 
diff --git a/Objects/object.c b/Objects/object.c
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -1859,26 +1859,6 @@
 Py_ssize_t (*_Py_abstract_hack)(PyObject *) = PyObject_Size;
 
 
-/* Python's malloc wrappers (see pymem.h) */
-
-void *
-PyMem_Malloc(size_t nbytes)
-{
-    return PyMem_MALLOC(nbytes);
-}
-
-void *
-PyMem_Realloc(void *p, size_t nbytes)
-{
-    return PyMem_REALLOC(p, nbytes);
-}
-
-void
-PyMem_Free(void *p)
-{
-    PyMem_FREE(p);
-}
-
 void
 _PyObject_DebugTypeStats(FILE *out)
 {
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -1,18 +1,326 @@
 #include "Python.h"
 
+/* Python's malloc wrappers (see pymem.h) */
+
+#ifdef PYMALLOC_DEBUG   /* WITH_PYMALLOC && PYMALLOC_DEBUG */
+/* Forward declaration */
+static void* _PyMem_DebugMalloc(void *ctx, size_t size);
+static void _PyMem_DebugFree(void *ctx, void *p);
+static void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size);
+
+static void _PyObject_DebugDumpAddress(const void *p);
+static void _PyMem_DebugCheckAddress(char api_id, const void *p);
+#endif
+
 #ifdef WITH_PYMALLOC
 
-#ifdef HAVE_MMAP
- #include <sys/mman.h>
- #ifdef MAP_ANONYMOUS
-  #define ARENAS_USE_MMAP
- #endif
+#ifdef MS_WINDOWS
+#  include <windows.h>
+#elif defined(HAVE_MMAP)
+#  include <sys/mman.h>
+#  ifdef MAP_ANONYMOUS
+#    define ARENAS_USE_MMAP
+#  endif
 #endif
 
+/* Forward declaration */
+static void* _PyObject_Malloc(void *ctx, size_t size);
+static void _PyObject_Free(void *ctx, void *p);
+static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
+#endif
+
+
+static void *
+_PyMem_RawMalloc(void *ctx, size_t size)
+{
+    /* PyMem_Malloc(0) means malloc(1). Some systems would return NULL
+       for malloc(0), which would be treated as an error. Some platforms would
+       return a pointer with no memory behind it, which would break pymalloc.
+       To solve these problems, allocate an extra byte. */
+    if (size == 0)
+        size = 1;
+    return malloc(size);
+}
+
+static void *
+_PyMem_RawRealloc(void *ctx, void *ptr, size_t size)
+{
+    if (size == 0)
+        size = 1;
+    return realloc(ptr, size);
+}
+
+static void
+_PyMem_RawFree(void *ctx, void *ptr)
+{
+    free(ptr);
+}
+
+
 #ifdef MS_WINDOWS
-#include <windows.h>
+static void *
+_PyObject_ArenaVirtualAlloc(void *ctx, size_t size)
+{
+    return VirtualAlloc(NULL, size,
+                        MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
+}
+
+static void
+_PyObject_ArenaVirtualFree(void *ctx, void *ptr, size_t size)
+{
+    VirtualFree(ptr, 0, MEM_RELEASE);
+}
+
+#elif defined(ARENAS_USE_MMAP)
+static void *
+_PyObject_ArenaMmap(void *ctx, size_t size)
+{
+    void *ptr;
+    ptr = mmap(NULL, size, PROT_READ|PROT_WRITE,
+               MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+    if (ptr == MAP_FAILED)
+        return NULL;
+    assert(ptr != NULL);
+    return ptr;
+}
+
+static void
+_PyObject_ArenaMunmap(void *ctx, void *ptr, size_t size)
+{
+    munmap(ptr, size);
+}
+
+#else
+static void *
+_PyObject_ArenaMalloc(void *ctx, size_t size)
+{
+    return malloc(size);
+}
+
+static void
+_PyObject_ArenaFree(void *ctx, void *ptr, size_t size)
+{
+    free(ptr);
+}
 #endif
 
+
+#define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawRealloc, _PyMem_RawFree
+#ifdef WITH_PYMALLOC
+#define PYOBJECT_FUNCS _PyObject_Malloc, _PyObject_Realloc, _PyObject_Free
+#else
+#define PYOBJECT_FUNCS PYRAW_FUNCS
+#endif
+
+#ifdef PYMALLOC_DEBUG
+typedef struct {
+    /* We tag each block with an API ID in order to tag API violations */
+    char api_id;
+    PyMemAllocator alloc;
+} debug_alloc_api_t;
+static struct {
+    debug_alloc_api_t raw;
+    debug_alloc_api_t mem;
+    debug_alloc_api_t obj;
+} _PyMem_Debug = {
+    {'r', {NULL, PYRAW_FUNCS}},
+    {'m', {NULL, PYRAW_FUNCS}},
+    {'o', {NULL, PYOBJECT_FUNCS}}
+    };
+
+#define PYDEBUG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugRealloc, _PyMem_DebugFree
+#endif
+
+static PyMemAllocator _PyMem_Raw = {
+#ifdef PYMALLOC_DEBUG
+    &_PyMem_Debug.raw, PYDEBUG_FUNCS
+#else
+    NULL, PYRAW_FUNCS
+#endif
+    };
+
+static PyMemAllocator _PyMem = {
+#ifdef PYMALLOC_DEBUG
+    &_PyMem_Debug.mem, PYDEBUG_FUNCS
+#else
+    NULL, PYRAW_FUNCS
+#endif
+    };
+
+static PyMemAllocator _PyObject = {
+#ifdef PYMALLOC_DEBUG
+    &_PyMem_Debug.obj, PYDEBUG_FUNCS
+#else
+    NULL, PYOBJECT_FUNCS
+#endif
+    };
+
+#undef PYRAW_FUNCS
+#undef PYOBJECT_FUNCS
+#undef PYDEBUG_FUNCS
+
+static PyObjectArenaAllocator _PyObject_Arena = {NULL,
+#ifdef MS_WINDOWS
+    _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree
+#elif defined(ARENAS_USE_MMAP)
+    _PyObject_ArenaMmap, _PyObject_ArenaMunmap
+#else
+    _PyObject_ArenaMalloc, _PyObject_ArenaFree
+#endif
+    };
+
+void
+PyMem_SetupDebugHooks(void)
+{
+#ifdef PYMALLOC_DEBUG
+    PyMemAllocator alloc;
+
+    alloc.malloc = _PyMem_DebugMalloc;
+    alloc.realloc = _PyMem_DebugRealloc;
+    alloc.free = _PyMem_DebugFree;
+
+    if (_PyMem_Raw.malloc != _PyMem_DebugMalloc) {
+        alloc.ctx = &_PyMem_Debug.raw;
+        PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &_PyMem_Debug.raw.alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc);
+    }
+
+    if (_PyMem.malloc != _PyMem_DebugMalloc) {
+        alloc.ctx = &_PyMem_Debug.mem;
+        PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc);
+    }
+
+    if (_PyObject.malloc != _PyMem_DebugMalloc) {
+        alloc.ctx = &_PyMem_Debug.obj;
+        PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc);
+    }
+#endif
+}
+
+void
+PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
+{
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: *allocator = _PyMem_Raw; break;
+    case PYMEM_DOMAIN_MEM: *allocator = _PyMem; break;
+    case PYMEM_DOMAIN_OBJ: *allocator = _PyObject; break;
+    default:
+        /* unknown domain */
+        allocator->ctx = NULL;
+        allocator->malloc = NULL;
+        allocator->realloc = NULL;
+        allocator->free = NULL;
+    }
+}
+
+void
+PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
+{
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: _PyMem_Raw = *allocator; break;
+    case PYMEM_DOMAIN_MEM: _PyMem = *allocator; break;
+    case PYMEM_DOMAIN_OBJ: _PyObject = *allocator; break;
+    /* ignore unknown domain */
+    }
+
+}
+
+void
+PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)
+{
+    *allocator = _PyObject_Arena;
+}
+
+void
+PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)
+{
+    _PyObject_Arena = *allocator;
+}
+
+void *
+PyMem_RawMalloc(size_t size)
+{
+    /*
+     * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.
+     * Most python internals blindly use a signed Py_ssize_t to track
+     * things without checking for overflows or negatives.
+     * As size_t is unsigned, checking for size < 0 is not required.
+     */
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+
+    return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);
+}
+
+void*
+PyMem_RawRealloc(void *ptr, size_t new_size)
+{
+    /* see PyMem_RawMalloc() */
+    if (new_size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size);
+}
+
+void PyMem_RawFree(void *ptr)
+{
+    _PyMem_Raw.free(_PyMem_Raw.ctx, ptr);
+}
+
+void *
+PyMem_Malloc(size_t size)
+{
+    /* see PyMem_RawMalloc() */
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    return _PyMem.malloc(_PyMem.ctx, size);
+}
+
+void *
+PyMem_Realloc(void *ptr, size_t new_size)
+{
+    /* see PyMem_RawMalloc() */
+    if (new_size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    return _PyMem.realloc(_PyMem.ctx, ptr, new_size);
+}
+
+void
+PyMem_Free(void *ptr)
+{
+    _PyMem.free(_PyMem.ctx, ptr);
+}
+
+void *
+PyObject_Malloc(size_t size)
+{
+    /* see PyMem_RawMalloc() */
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    return _PyObject.malloc(_PyObject.ctx, size);
+}
+
+void *
+PyObject_Realloc(void *ptr, size_t new_size)
+{
+    /* see PyMem_RawMalloc() */
+    if (new_size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    return _PyObject.realloc(_PyObject.ctx, ptr, new_size);
+}
+
+void
+PyObject_Free(void *ptr)
+{
+    _PyObject.free(_PyObject.ctx, ptr);
+}
+
+
+#ifdef WITH_PYMALLOC
+
 #ifdef WITH_VALGRIND
 #include <valgrind/valgrind.h>
 
@@ -549,7 +857,6 @@
     struct arena_object* arenaobj;
     uint excess;        /* number of bytes above pool alignment */
     void *address;
-    int err;
 
 #ifdef PYMALLOC_DEBUG
     if (Py_GETENV("PYTHONMALLOCSTATS"))
@@ -571,7 +878,7 @@
             return NULL;                /* overflow */
 #endif
         nbytes = numarenas * sizeof(*arenas);
-        arenaobj = (struct arena_object *)realloc(arenas, nbytes);
+        arenaobj = (struct arena_object *)PyMem_Realloc(arenas, nbytes);
         if (arenaobj == NULL)
             return NULL;
         arenas = arenaobj;
@@ -602,19 +909,8 @@
     arenaobj = unused_arena_objects;
     unused_arena_objects = arenaobj->nextarena;
     assert(arenaobj->address == 0);
-#ifdef MS_WINDOWS
-    address = (void*)VirtualAlloc(NULL, ARENA_SIZE,
-                                 MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
-    err = (address == NULL);
-#elif defined(ARENAS_USE_MMAP)
-    address = mmap(NULL, ARENA_SIZE, PROT_READ|PROT_WRITE,
-                                   MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
-    err = (address == MAP_FAILED);
-#else
-    address = malloc(ARENA_SIZE);
-    err = (address == 0);
-#endif
-    if (err) {
+    address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE);
+    if (address == NULL) {
         /* The allocation failed: return NULL after putting the
          * arenaobj back.
          */
@@ -777,9 +1073,8 @@
  * Unless the optimizer reorders everything, being too smart...
  */
 
-#undef PyObject_Malloc
-void *
-PyObject_Malloc(size_t nbytes)
+static void *
+_PyObject_Malloc(void *ctx, size_t nbytes)
 {
     block *bp;
     poolp pool;
@@ -796,17 +1091,6 @@
 #endif
 
     /*
-     * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.
-     * Most python internals blindly use a signed Py_ssize_t to track
-     * things without checking for overflows or negatives.
-     * As size_t is unsigned, checking for nbytes < 0 is not required.
-     */
-    if (nbytes > PY_SSIZE_T_MAX) {
-        _Py_AllocatedBlocks--;
-        return NULL;
-    }
-
-    /*
      * This implicitly redirects malloc(0).
      */
     if ((nbytes - 1) < SMALL_REQUEST_THRESHOLD) {
@@ -978,10 +1262,8 @@
      * last chance to serve the request) or when the max memory limit
      * has been reached.
      */
-    if (nbytes == 0)
-        nbytes = 1;
     {
-        void *result = malloc(nbytes);
+        void *result = PyMem_Malloc(nbytes);
         if (!result)
             _Py_AllocatedBlocks--;
         return result;
@@ -990,9 +1272,8 @@
 
 /* free */
 
-#undef PyObject_Free
-void
-PyObject_Free(void *p)
+static void
+_PyObject_Free(void *ctx, void *p)
 {
     poolp pool;
     block *lastfree;
@@ -1101,13 +1382,8 @@
                 unused_arena_objects = ao;
 
                 /* Free the entire arena. */
-#ifdef MS_WINDOWS
-                VirtualFree((void *)ao->address, 0, MEM_RELEASE);
-#elif defined(ARENAS_USE_MMAP)
-                munmap((void *)ao->address, ARENA_SIZE);
-#else
-                free((void *)ao->address);
-#endif
+                _PyObject_Arena.free(_PyObject_Arena.ctx,
+                                     (void *)ao->address, ARENA_SIZE);
                 ao->address = 0;                        /* mark unassociated */
                 --narenas_currently_allocated;
 
@@ -1216,7 +1492,7 @@
 redirect:
 #endif
     /* We didn't allocate this address. */
-    free(p);
+    PyMem_Free(p);
 }
 
 /* realloc.  If p is NULL, this acts like malloc(nbytes).  Else if nbytes==0,
@@ -1224,9 +1500,8 @@
  * return a non-NULL result.
  */
 
-#undef PyObject_Realloc
-void *
-PyObject_Realloc(void *p, size_t nbytes)
+static void *
+_PyObject_Realloc(void *ctx, void *p, size_t nbytes)
 {
     void *bp;
     poolp pool;
@@ -1236,16 +1511,7 @@
 #endif
 
     if (p == NULL)
-        return PyObject_Malloc(nbytes);
-
-    /*
-     * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.
-     * Most python internals blindly use a signed Py_ssize_t to track
-     * things without checking for overflows or negatives.
-     * As size_t is unsigned, checking for nbytes < 0 is not required.
-     */
-    if (nbytes > PY_SSIZE_T_MAX)
-        return NULL;
+        return _PyObject_Malloc(ctx, nbytes);
 
 #ifdef WITH_VALGRIND
     /* Treat running_on_valgrind == -1 the same as 0 */
@@ -1273,10 +1539,10 @@
             }
             size = nbytes;
         }
-        bp = PyObject_Malloc(nbytes);
+        bp = _PyObject_Malloc(ctx, nbytes);
         if (bp != NULL) {
             memcpy(bp, p, size);
-            PyObject_Free(p);
+            _PyObject_Free(ctx, p);
         }
         return bp;
     }
@@ -1294,14 +1560,14 @@
      * at p.  Instead we punt:  let C continue to manage this block.
      */
     if (nbytes)
-        return realloc(p, nbytes);
+        return PyMem_Realloc(p, nbytes);
     /* C doesn't define the result of realloc(p, 0) (it may or may not
      * return NULL then), but Python's docs promise that nbytes==0 never
      * returns NULL.  We don't pass 0 to realloc(), to avoid that endcase
      * to begin with.  Even then, we can't be sure that realloc() won't
      * return NULL.
      */
-    bp = realloc(p, 1);
+    bp = PyMem_Realloc(p, 1);
     return bp ? bp : p;
 }
 
@@ -1311,24 +1577,6 @@
 /* pymalloc not enabled:  Redirect the entry points to malloc.  These will
  * only be used by extensions that are compiled with pymalloc enabled. */
 
-void *
-PyObject_Malloc(size_t n)
-{
-    return PyMem_MALLOC(n);
-}
-
-void *
-PyObject_Realloc(void *p, size_t n)
-{
-    return PyMem_REALLOC(p, n);
-}
-
-void
-PyObject_Free(void *p)
-{
-    PyMem_FREE(p);
-}
-
 Py_ssize_t
 _Py_GetAllocatedBlocks(void)
 {
@@ -1354,10 +1602,6 @@
 #define DEADBYTE       0xDB    /* dead (newly freed) memory */
 #define FORBIDDENBYTE  0xFB    /* untouchable bytes at each end of a block */
 
-/* We tag each block with an API ID in order to tag API violations */
-#define _PYMALLOC_MEM_ID 'm'   /* the PyMem_Malloc() API */
-#define _PYMALLOC_OBJ_ID 'o'   /* The PyObject_Malloc() API */
-
 static size_t serialno = 0;     /* incremented on each debug {m,re}alloc */
 
 /* serialno is always incremented via calling this routine.  The point is
@@ -1440,58 +1684,18 @@
 p[2*S+n: 2*S+n+S]
     Copies of FORBIDDENBYTE.  Used to catch over- writes and reads.
 p[2*S+n+S: 2*S+n+2*S]
-    A serial number, incremented by 1 on each call to _PyObject_DebugMalloc
-    and _PyObject_DebugRealloc.
+    A serial number, incremented by 1 on each call to _PyMem_DebugMalloc
+    and _PyMem_DebugRealloc.
     This is a big-endian size_t.
     If "bad memory" is detected later, the serial number gives an
     excellent way to set a breakpoint on the next run, to capture the
     instant at which this block was passed out.
 */
 
-/* debug replacements for the PyMem_* memory API */
-void *
-_PyMem_DebugMalloc(size_t nbytes)
+static void *
+_PyMem_DebugMalloc(void *ctx, size_t nbytes)
 {
-    return _PyObject_DebugMallocApi(_PYMALLOC_MEM_ID, nbytes);
-}
-void *
-_PyMem_DebugRealloc(void *p, size_t nbytes)
-{
-    return _PyObject_DebugReallocApi(_PYMALLOC_MEM_ID, p, nbytes);
-}
-void
-_PyMem_DebugFree(void *p)
-{
-    _PyObject_DebugFreeApi(_PYMALLOC_MEM_ID, p);
-}
-
-/* debug replacements for the PyObject_* memory API */
-void *
-_PyObject_DebugMalloc(size_t nbytes)
-{
-    return _PyObject_DebugMallocApi(_PYMALLOC_OBJ_ID, nbytes);
-}
-void *
-_PyObject_DebugRealloc(void *p, size_t nbytes)
-{
-    return _PyObject_DebugReallocApi(_PYMALLOC_OBJ_ID, p, nbytes);
-}
-void
-_PyObject_DebugFree(void *p)
-{
-    _PyObject_DebugFreeApi(_PYMALLOC_OBJ_ID, p);
-}
-void
-_PyObject_DebugCheckAddress(const void *p)
-{
-    _PyObject_DebugCheckAddressApi(_PYMALLOC_OBJ_ID, p);
-}
-
-
-/* generic debug memory api, with an "id" to identify the API in use */
-void *
-_PyObject_DebugMallocApi(char id, size_t nbytes)
-{
+    debug_alloc_api_t *api = (debug_alloc_api_t *)ctx;
     uchar *p;           /* base address of malloc'ed block */
     uchar *tail;        /* p + 2*SST + nbytes == pointer to tail pad bytes */
     size_t total;       /* nbytes + 4*SST */
@@ -1502,14 +1706,14 @@
         /* overflow:  can't represent total as a size_t */
         return NULL;
 
-    p = (uchar *)PyObject_Malloc(total);
+    p = (uchar *)api->alloc.malloc(api->alloc.ctx, total);
     if (p == NULL)
         return NULL;
 
     /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
     write_size_t(p, nbytes);
-    p[SST] = (uchar)id;
-    memset(p + SST + 1 , FORBIDDENBYTE, SST-1);
+    p[SST] = (uchar)api->api_id;
+    memset(p + SST + 1, FORBIDDENBYTE, SST-1);
 
     if (nbytes > 0)
         memset(p + 2*SST, CLEANBYTE, nbytes);
@@ -1527,25 +1731,27 @@
    Then fills the original bytes with DEADBYTE.
    Then calls the underlying free.
 */
-void
-_PyObject_DebugFreeApi(char api, void *p)
+static void
+_PyMem_DebugFree(void *ctx, void *p)
 {
+    debug_alloc_api_t *api = (debug_alloc_api_t *)ctx;
     uchar *q = (uchar *)p - 2*SST;  /* address returned from malloc */
     size_t nbytes;
 
     if (p == NULL)
         return;
-    _PyObject_DebugCheckAddressApi(api, p);
+    _PyMem_DebugCheckAddress(api->api_id, p);
     nbytes = read_size_t(q);
     nbytes += 4*SST;
     if (nbytes > 0)
         memset(q, DEADBYTE, nbytes);
-    PyObject_Free(q);
+    api->alloc.free(api->alloc.ctx, q);
 }
 
-void *
-_PyObject_DebugReallocApi(char api, void *p, size_t nbytes)
+static void *
+_PyMem_DebugRealloc(void *ctx, void *p, size_t nbytes)
 {
+    debug_alloc_api_t *api = (debug_alloc_api_t *)ctx;
     uchar *q = (uchar *)p;
     uchar *tail;
     size_t total;       /* nbytes + 4*SST */
@@ -1553,9 +1759,9 @@
     int i;
 
     if (p == NULL)
-        return _PyObject_DebugMallocApi(api, nbytes);
+        return _PyMem_DebugMalloc(ctx, nbytes);
 
-    _PyObject_DebugCheckAddressApi(api, p);
+    _PyMem_DebugCheckAddress(api->api_id, p);
     bumpserialno();
     original_nbytes = read_size_t(q - 2*SST);
     total = nbytes + 4*SST;
@@ -1572,12 +1778,12 @@
      * case we didn't get the chance to mark the old memory with DEADBYTE,
      * but we live with that.
      */
-    q = (uchar *)PyObject_Realloc(q - 2*SST, total);
+    q = (uchar *)api->alloc.realloc(api->alloc.ctx, q - 2*SST, total);
     if (q == NULL)
         return NULL;
 
     write_size_t(q, nbytes);
-    assert(q[SST] == (uchar)api);
+    assert(q[SST] == (uchar)api->api_id);
     for (i = 1; i < SST; ++i)
         assert(q[SST + i] == FORBIDDENBYTE);
     q += 2*SST;
@@ -1599,8 +1805,8 @@
  * and call Py_FatalError to kill the program.
  * The API id, is also checked.
  */
- void
-_PyObject_DebugCheckAddressApi(char api, const void *p)
+static void
+_PyMem_DebugCheckAddress(char api, const void *p)
 {
     const uchar *q = (const uchar *)p;
     char msgbuf[64];
@@ -1652,7 +1858,7 @@
 }
 
 /* Display info to stderr about the memory block at p. */
-void
+static void
 _PyObject_DebugDumpAddress(const void *p)
 {
     const uchar *q = (const uchar *)p;
diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj
--- a/PCbuild/python.vcxproj
+++ b/PCbuild/python.vcxproj
@@ -243,7 +243,7 @@
     <Link>
       <OutputFile>$(OutDir)python_d.exe</OutputFile>
       <SubSystem>Console</SubSystem>
-      <StackReserveSize>2100000</StackReserveSize>
+      <StackReserveSize>4194304</StackReserveSize>
       <BaseAddress>0x1d000000</BaseAddress>
     </Link>
   </ItemDefinitionGroup>

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 09:28:16 2013
From: python-checkins at python.org (ronald.oussoren)
Date: Sun,  7 Jul 2013 09:28:16 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Cleanup_of_doc?=
 =?utf-8?q?umentation_change_from_=2317860?=
Message-ID: <3bp1c01hmbz7Ljk@mail.python.org>

http://hg.python.org/cpython/rev/f4747e1ce2b1
changeset:   84476:f4747e1ce2b1
branch:      3.3
parent:      84467:b8028f74bac4
user:        Ronald Oussoren <ronaldoussoren at mac.com>
date:        Sun Jul 07 09:26:45 2013 +0200
summary:
  Cleanup of documentation change from #17860

Reformulated the textual change, and applied it to the docstring as well.

files:
  Doc/library/subprocess.rst |  11 +++++++----
  Lib/subprocess.py          |   3 +++
  2 files changed, 10 insertions(+), 4 deletions(-)


diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -290,11 +290,14 @@
    .. index::
       single: universal newlines; subprocess module
 
-   If *universal_newlines* is ``True``, the file objects *stdin*, *stdout* and
-   *stderr* will be opened as text streams in :term:`universal newlines` mode
+   If *universal_newlines* is ``False`` the file objects *stdin*, *stdout* and
+   *stderr* will be opened as binary streams, and no line ending conversion is
+   done.
+
+   If *universal_newlines* is ``True``, these file objects
+   will be opened as text streams in :term:`universal newlines` mode
    using the encoding returned by :func:`locale.getpreferredencoding(False)
-   <locale.getpreferredencoding>`, otherwise these streams will be opened
-   as binary streams.  For *stdin*, line ending characters
+   <locale.getpreferredencoding>`.  For *stdin*, line ending characters
    ``'\n'`` in the input will be converted to the default line separator
    :data:`os.linesep`.  For *stdout* and *stderr*, all line endings in the
    output will be converted to ``'\n'``.  For more information see the
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -104,6 +104,9 @@
 If env is not None, it defines the environment variables for the new
 process.
 
+If universal_newlines is false, the file objects stdin, stdout and stderr
+are opened as binary files, and no line ending conversion is done.
+
 If universal_newlines is true, the file objects stdout and stderr are
 opened as a text files, but lines may be terminated by any of '\n',
 the Unix end-of-line convention, '\r', the old Macintosh convention or

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 09:28:17 2013
From: python-checkins at python.org (ronald.oussoren)
Date: Sun,  7 Jul 2013 09:28:17 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_=283=2E3-=3Edefault=29_Cleanup_of_documentation_change_f?=
 =?utf-8?q?rom_=2317860?=
Message-ID: <3bp1c13v4Hz7Lkn@mail.python.org>

http://hg.python.org/cpython/rev/83810f67d16b
changeset:   84477:83810f67d16b
parent:      84475:cc0c8c1c9cca
parent:      84476:f4747e1ce2b1
user:        Ronald Oussoren <ronaldoussoren at mac.com>
date:        Sun Jul 07 09:28:01 2013 +0200
summary:
  (3.3->default) Cleanup of documentation change from #17860

Reformulated the textual change, and applied it to the docstring as well.

files:
  Doc/library/subprocess.rst |  11 +++++++----
  Lib/subprocess.py          |   3 +++
  2 files changed, 10 insertions(+), 4 deletions(-)


diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -300,11 +300,14 @@
    .. index::
       single: universal newlines; subprocess module
 
-   If *universal_newlines* is ``True``, the file objects *stdin*, *stdout* and
-   *stderr* will be opened as text streams in :term:`universal newlines` mode
+   If *universal_newlines* is ``False`` the file objects *stdin*, *stdout* and
+   *stderr* will be opened as binary streams, and no line ending conversion is
+   done.
+
+   If *universal_newlines* is ``True``, these file objects
+   will be opened as text streams in :term:`universal newlines` mode
    using the encoding returned by :func:`locale.getpreferredencoding(False)
-   <locale.getpreferredencoding>`, otherwise these streams will be opened
-   as binary streams.  For *stdin*, line ending characters
+   <locale.getpreferredencoding>`.  For *stdin*, line ending characters
    ``'\n'`` in the input will be converted to the default line separator
    :data:`os.linesep`.  For *stdout* and *stderr*, all line endings in the
    output will be converted to ``'\n'``.  For more information see the
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -104,6 +104,9 @@
 If env is not None, it defines the environment variables for the new
 process.
 
+If universal_newlines is false, the file objects stdin, stdout and stderr
+are opened as binary files, and no line ending conversion is done.
+
 If universal_newlines is true, the file objects stdout and stderr are
 opened as a text files, but lines may be terminated by any of '\n',
 the Unix end-of-line convention, '\r', the old Macintosh convention or

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 09:50:23 2013
From: python-checkins at python.org (ronald.oussoren)
Date: Sun,  7 Jul 2013 09:50:23 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4Mzc3?=
 =?utf-8?q?=3A_Code_cleanup_in_Python_Launcher?=
Message-ID: <3bp25W2N3nzQT2@mail.python.org>

http://hg.python.org/cpython/rev/5d41ebc79738
changeset:   84478:5d41ebc79738
branch:      2.7
parent:      84457:7ec9255d4189
user:        Ronald Oussoren <ronaldoussoren at mac.com>
date:        Sun Jul 07 09:49:23 2013 +0200
summary:
  Issue #18377: Code cleanup in Python Launcher

This changeset fixes a number of compiler warnings in the Python Launcher
binary for OSX. It also cleans up whitespace usage in those sources.

files:
  Mac/PythonLauncher/FileSettings.h                |   5 -
  Mac/PythonLauncher/FileSettings.m                |  46 ++---
  Mac/PythonLauncher/MyAppDelegate.m               |   6 +-
  Mac/PythonLauncher/MyDocument.m                  |  20 +-
  Mac/PythonLauncher/PreferencesWindowController.m |  23 +-
  Mac/PythonLauncher/doscript.h                    |   2 +-
  Mac/PythonLauncher/doscript.m                    |  78 +++++-----
  Mac/PythonLauncher/main.m                        |   4 +-
  8 files changed, 83 insertions(+), 101 deletions(-)


diff --git a/Mac/PythonLauncher/FileSettings.h b/Mac/PythonLauncher/FileSettings.h
--- a/Mac/PythonLauncher/FileSettings.h
+++ b/Mac/PythonLauncher/FileSettings.h
@@ -45,18 +45,13 @@
 + (id)getFactorySettingsForFileType: (NSString *)filetype;
 + (id)newSettingsForFileType: (NSString *)filetype;
 
-//- (id)init;
 - (id)initForFileType: (NSString *)filetype;
 - (id)initForFSDefaultFileType: (NSString *)filetype;
 - (id)initForDefaultFileType: (NSString *)filetype;
-//- (id)initWithFileSettings: (FileSettings *)source;
 
 - (void)updateFromSource: (id <FileSettingsSource>)source;
 - (NSString *)commandLineForScript: (NSString *)script;
 
-//- (void)applyFactorySettingsForFileType: (NSString *)filetype;
-//- (void)saveDefaults;
-//- (void)applyUserDefaults: (NSString *)filetype;
 - (void)applyValuesFromDict: (NSDictionary *)dict;
 - (void)reset;
 - (NSArray *) interpreters;
diff --git a/Mac/PythonLauncher/FileSettings.m b/Mac/PythonLauncher/FileSettings.m
--- a/Mac/PythonLauncher/FileSettings.m
+++ b/Mac/PythonLauncher/FileSettings.m
@@ -14,7 +14,7 @@
 {
     static FileSettings *fsdefault_py, *fsdefault_pyw, *fsdefault_pyc;
     FileSettings **curdefault;
-    
+
     if ([filetype isEqualToString: @"Python Script"]) {
         curdefault = &fsdefault_py;
     } else if ([filetype isEqualToString: @"Python GUI Script"]) {
@@ -36,7 +36,7 @@
 {
     static FileSettings *default_py, *default_pyw, *default_pyc;
     FileSettings **curdefault;
-    
+
     if ([filetype isEqualToString: @"Python Script"]) {
         curdefault = &default_py;
     } else if ([filetype isEqualToString: @"Python GUI Script"]) {
@@ -57,7 +57,7 @@
 + (id)newSettingsForFileType: (NSString *)filetype
 {
     FileSettings *cur;
-    
+
     cur = [FileSettings new];
     [cur initForFileType: filetype];
     return [cur retain];
@@ -67,7 +67,7 @@
 {
     self = [super init];
     if (!self) return self;
-    
+
     interpreter = [source->interpreter retain];
     honourhashbang = source->honourhashbang;
     debug = source->debug;
@@ -81,36 +81,30 @@
     with_terminal = source->with_terminal;
     prefskey = source->prefskey;
     if (prefskey) [prefskey retain];
-    
+
     return self;
 }
 
 - (id)initForFileType: (NSString *)filetype
 {
     FileSettings *defaults;
-    
+
     defaults = [FileSettings getDefaultsForFileType: filetype];
     self = [self initWithFileSettings: defaults];
     origsource = [defaults retain];
     return self;
 }
 
-//- (id)init
-//{
-//    self = [self initForFileType: @"Python Script"];
-//    return self;
-//}
-
 - (id)initForFSDefaultFileType: (NSString *)filetype
 {
     int i;
     NSString *filename;
     NSDictionary *dict;
     static NSDictionary *factorySettings;
-    
+
     self = [super init];
     if (!self) return self;
-    
+
     if (factorySettings == NULL) {
         NSBundle *bdl = [NSBundle mainBundle];
         NSString *path = [ bdl pathForResource: @"factorySettings"
@@ -149,18 +143,18 @@
 {
     NSUserDefaults *defaults;
     NSDictionary *dict;
-    
+
     defaults = [NSUserDefaults standardUserDefaults];
     dict = [defaults dictionaryForKey: filetype];
     if (!dict)
         return;
     [self applyValuesFromDict: dict];
 }
-    
+
 - (id)initForDefaultFileType: (NSString *)filetype
 {
     FileSettings *fsdefaults;
-    
+
     fsdefaults = [FileSettings getFactorySettingsForFileType: filetype];
     self = [self initWithFileSettings: fsdefaults];
     if (!self) return self;
@@ -220,7 +214,7 @@
 - (void)applyValuesFromDict: (NSDictionary *)dict
 {
     id value;
-    
+
     value = [dict objectForKey: @"interpreter"];
     if (value) interpreter = [value retain];
     value = [dict objectForKey: @"honourhashbang"];
@@ -247,12 +241,12 @@
 
 - (NSString*)_replaceSingleQuotes: (NSString*)string
 {
-	/* Replace all single-quotes by '"'"', that way shellquoting will
-	 * be correct when the result value is delimited  using single quotes.
-	 */
-	NSArray* components = [string componentsSeparatedByString:@"'"];
+    /* Replace all single-quotes by '"'"', that way shellquoting will
+     * be correct when the result value is delimited  using single quotes.
+     */
+    NSArray* components = [string componentsSeparatedByString:@"'"];
 
-	return [components componentsJoinedByString:@"'\"'\"'"];
+    return [components componentsJoinedByString:@"'\"'\"'"];
 }
 
 - (NSString *)commandLineForScript: (NSString *)script
@@ -265,7 +259,7 @@
 
     script_dir = [script substringToIndex:
 	    [script length]-[[script lastPathComponent] length]];
-    
+
     if (honourhashbang &&
        (fp=fopen([script fileSystemRepresentation], "r")) &&
        fgets(hashbangbuf, sizeof(hashbangbuf), fp) &&
@@ -278,7 +272,7 @@
     }
     if (!cur_interp)
         cur_interp = interpreter;
-        
+
     return [NSString stringWithFormat:
         @"cd '%@' && '%@'%s%s%s%s%s%s %@ '%@' %@ %s",
     	[self _replaceSingleQuotes:script_dir],
@@ -297,7 +291,7 @@
 
 - (NSArray *) interpreters { return interpreters;};
 
-// FileSettingsSource protocol 
+// FileSettingsSource protocol
 - (NSString *) interpreter { return interpreter;};
 - (BOOL) honourhashbang { return honourhashbang; };
 - (BOOL) debug { return debug;};
diff --git a/Mac/PythonLauncher/MyAppDelegate.m b/Mac/PythonLauncher/MyAppDelegate.m
--- a/Mac/PythonLauncher/MyAppDelegate.m
+++ b/Mac/PythonLauncher/MyAppDelegate.m
@@ -33,7 +33,7 @@
 
 - (BOOL)shouldShowUI
 {
-    // if this call comes before applicationDidFinishLaunching: we 
+    // if this call comes before applicationDidFinishLaunching: we
     // should terminate immedeately after starting the script.
     if (!initial_action_done)
         should_terminate = YES;
@@ -62,7 +62,7 @@
     static NSString *extensions[] = { @"py", @"pyw", @"pyc", NULL};
     NSString **ext_p;
     int i;
-    
+
     if ([[NSUserDefaults standardUserDefaults] boolForKey: @"SkipFileBindingTest"])
         return;
     ourUrl = [NSURL fileURLWithPath: [[NSBundle mainBundle] bundlePath]];
@@ -92,5 +92,5 @@
         }
     }
 }
-        
+
 @end
diff --git a/Mac/PythonLauncher/MyDocument.m b/Mac/PythonLauncher/MyDocument.m
--- a/Mac/PythonLauncher/MyDocument.m
+++ b/Mac/PythonLauncher/MyDocument.m
@@ -16,7 +16,7 @@
 {
     self = [super init];
     if (self) {
-    
+
         // Add your subclass-specific initialization here.
         // If an error occurs here, send a [self dealloc] message and return nil.
         script = [@"<no script>.py" retain];
@@ -37,20 +37,17 @@
 {
     NSApplication *app = [NSApplication sharedApplication];
     [super close];
-    if ([[app delegate] shouldTerminate])
+    if ([(MyAppDelegate*)[app delegate] shouldTerminate])
         [app terminate: self];
 }
 
 - (void)load_defaults
 {
-//    if (settings) [settings release];
     settings = [FileSettings newSettingsForFileType: filetype];
 }
 
 - (void)update_display
 {
-//    [[self window] setTitle: script];
-    
     [interpreter setStringValue: [settings interpreter]];
     [honourhashbang setState: [settings honourhashbang]];
     [debug setState: [settings debug]];
@@ -62,7 +59,7 @@
     [others setStringValue: [settings others]];
     [scriptargs setStringValue: [settings scriptargs]];
     [with_terminal setState: [settings with_terminal]];
-    
+
     [commandline setStringValue: [settings commandLineForScript: script]];
 }
 
@@ -75,7 +72,7 @@
 {
     const char *cmdline;
     int sts;
-    
+
      cmdline = [[settings commandLineForScript: script] UTF8String];
    if ([settings with_terminal]) {
         sts = doscript(cmdline);
@@ -107,14 +104,13 @@
 {
     // Insert code here to read your document from the given data.  You can also choose to override -loadFileWrapperRepresentation:ofType: or -readFromFile:ofType: instead.
     BOOL show_ui;
-    
-    // ask the app delegate whether we should show the UI or not. 
-    show_ui = [[[NSApplication sharedApplication] delegate] shouldShowUI];
+
+    // ask the app delegate whether we should show the UI or not.
+    show_ui = [(MyAppDelegate*)[[NSApplication sharedApplication] delegate] shouldShowUI];
     [script release];
     script = [fileName retain];
     [filetype release];
     filetype = [type retain];
-//    if (settings) [settings release];
     settings = [FileSettings newSettingsForFileType: filetype];
     if (show_ui) {
         [self update_display];
@@ -152,7 +148,7 @@
     [self update_display];
 }
 
-// FileSettingsSource protocol 
+// FileSettingsSource protocol
 - (NSString *) interpreter { return [interpreter stringValue];};
 - (BOOL) honourhashbang { return [honourhashbang state];};
 - (BOOL) debug { return [debug state];};
diff --git a/Mac/PythonLauncher/PreferencesWindowController.m b/Mac/PythonLauncher/PreferencesWindowController.m
--- a/Mac/PythonLauncher/PreferencesWindowController.m
+++ b/Mac/PythonLauncher/PreferencesWindowController.m
@@ -5,7 +5,7 @@
 + getPreferencesWindow
 {
     static PreferencesWindowController *_singleton;
-    
+
     if (!_singleton)
         _singleton = [[PreferencesWindowController alloc] init];
     [_singleton showWindow: _singleton];
@@ -21,15 +21,13 @@
 - (void)load_defaults
 {
     NSString *title = [filetype titleOfSelectedItem];
-    
+
     settings = [FileSettings getDefaultsForFileType: title];
 }
 
 - (void)update_display
 {
-//    [[self window] setTitle: script];
-    
-	[interpreter reloadData];
+    [interpreter reloadData];
     [interpreter setStringValue: [settings interpreter]];
     [honourhashbang setState: [settings honourhashbang]];
     [debug setState: [settings debug]];
@@ -41,7 +39,6 @@
     [others setStringValue: [settings others]];
     [with_terminal setState: [settings with_terminal]];
     // Not scriptargs, it isn't for preferences
-    
     [commandline setStringValue: [settings commandLineForScript: @"<your script here>"]];
 }
 
@@ -75,7 +72,7 @@
     [self update_display];
 }
 
-// FileSettingsSource protocol 
+// FileSettingsSource protocol
 - (NSString *) interpreter { return [interpreter stringValue];};
 - (BOOL) honourhashbang { return [honourhashbang state]; };
 - (BOOL) debug { return [debug state];};
@@ -98,23 +95,23 @@
 // NSComboBoxDataSource protocol
 - (unsigned int)comboBox:(NSComboBox *)aComboBox indexOfItemWithStringValue:(NSString *)aString
 {
-	NSArray *interp_list = [settings interpreters];
+    NSArray *interp_list = [settings interpreters];
     unsigned int rv = [interp_list indexOfObjectIdenticalTo: aString];
-	return rv;
+    return rv;
 }
 
 - (id)comboBox:(NSComboBox *)aComboBox objectValueForItemAtIndex:(int)index
 {
-	NSArray *interp_list = [settings interpreters];
+    NSArray *interp_list = [settings interpreters];
     id rv = [interp_list objectAtIndex: index];
-	return rv;
+    return rv;
 }
 
 - (int)numberOfItemsInComboBox:(NSComboBox *)aComboBox
 {
-	NSArray *interp_list = [settings interpreters];
+    NSArray *interp_list = [settings interpreters];
     int rv = [interp_list count];
-	return rv;
+    return rv;
 }
 
 
diff --git a/Mac/PythonLauncher/doscript.h b/Mac/PythonLauncher/doscript.h
--- a/Mac/PythonLauncher/doscript.h
+++ b/Mac/PythonLauncher/doscript.h
@@ -9,4 +9,4 @@
 
 #include <Carbon/Carbon.h>
 
-extern int doscript(const char *command);
\ No newline at end of file
+extern int doscript(const char *command);
diff --git a/Mac/PythonLauncher/doscript.m b/Mac/PythonLauncher/doscript.m
--- a/Mac/PythonLauncher/doscript.m
+++ b/Mac/PythonLauncher/doscript.m
@@ -11,49 +11,49 @@
 #import <ApplicationServices/ApplicationServices.h>
 #import "doscript.h"
 
-extern int 
+extern int
 doscript(const char *command)
 {
-	char *bundleID = "com.apple.Terminal";
-	AppleEvent evt, res;
-	AEDesc desc;
-	OSStatus err;
+    char *bundleID = "com.apple.Terminal";
+    AppleEvent evt, res;
+    AEDesc desc;
+    OSStatus err;
 
-	[[NSWorkspace sharedWorkspace] launchApplication:@"/Applications/Utilities/Terminal.app/"];
+    [[NSWorkspace sharedWorkspace] launchApplication:@"/Applications/Utilities/Terminal.app/"];
 
-	// Build event
-	err = AEBuildAppleEvent(kAECoreSuite, kAEDoScript,
-	                         typeApplicationBundleID,
-	                         bundleID, strlen(bundleID),
-	                         kAutoGenerateReturnID,
-	                         kAnyTransactionID,
-	                         &evt, NULL,
-	                         "'----':utf8(@)", strlen(command),
-	                         command);
-	if (err) {
-		NSLog(@"AEBuildAppleEvent failed: %d\n", err);
-		return err;
-	}
+    // Build event
+    err = AEBuildAppleEvent(kAECoreSuite, kAEDoScript,
+                             typeApplicationBundleID,
+                             bundleID, strlen(bundleID),
+                             kAutoGenerateReturnID,
+                             kAnyTransactionID,
+                             &evt, NULL,
+                             "'----':utf8(@)", strlen(command),
+                             command);
+    if (err) {
+        NSLog(@"AEBuildAppleEvent failed: %ld\n", (long)err);
+        return err;
+    }
 
-	// Send event and check for any Apple Event Manager errors
-	err = AESendMessage(&evt, &res, kAEWaitReply, kAEDefaultTimeout);
-	AEDisposeDesc(&evt);
-	if (err) {
-		NSLog(@"AESendMessage failed: %d\n", err);
-		return err;
-	}
-	// Check for any application errors
-	err = AEGetParamDesc(&res, keyErrorNumber, typeSInt32, &desc);
-	AEDisposeDesc(&res);
-	if (!err) {
-		AEGetDescData(&desc, &err, sizeof(err));
-		NSLog(@"Terminal returned an error: %d", err);
-		AEDisposeDesc(&desc);
-	} else if (err == errAEDescNotFound) {
-		err = noErr;
-	} else {
-		NSLog(@"AEGetPArmDesc returned an error: %d", err);
-	}
+    // Send event and check for any Apple Event Manager errors
+    err = AESendMessage(&evt, &res, kAEWaitReply, kAEDefaultTimeout);
+    AEDisposeDesc(&evt);
+    if (err) {
+        NSLog(@"AESendMessage failed: %ld\n", (long)err);
+        return err;
+    }
+    // Check for any application errors
+    err = AEGetParamDesc(&res, keyErrorNumber, typeSInt32, &desc);
+    AEDisposeDesc(&res);
+    if (!err) {
+        AEGetDescData(&desc, &err, sizeof(err));
+        NSLog(@"Terminal returned an error: %ld", (long)err);
+        AEDisposeDesc(&desc);
+    } else if (err == errAEDescNotFound) {
+        err = noErr;
+    } else {
+        NSLog(@"AEGetPArmDesc returned an error: %ld", (long)err);
+    }
 
-	return err;
+    return err;
 }
diff --git a/Mac/PythonLauncher/main.m b/Mac/PythonLauncher/main.m
--- a/Mac/PythonLauncher/main.m
+++ b/Mac/PythonLauncher/main.m
@@ -11,7 +11,7 @@
 
 int main(int argc, const char *argv[])
 {
-	char *home = getenv("HOME");
-	if (home) chdir(home);
+    char *home = getenv("HOME");
+    if (home) chdir(home);
     return NSApplicationMain(argc, argv);
 }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 09:54:24 2013
From: python-checkins at python.org (ronald.oussoren)
Date: Sun,  7 Jul 2013 09:54:24 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4Mzc3?=
 =?utf-8?q?=3A_Code_cleanup_in_Python_Launcher?=
Message-ID: <3bp2B8137Lz7LjX@mail.python.org>

http://hg.python.org/cpython/rev/738cba2bf849
changeset:   84479:738cba2bf849
branch:      3.3
parent:      84476:f4747e1ce2b1
user:        Ronald Oussoren <ronaldoussoren at mac.com>
date:        Sun Jul 07 09:53:08 2013 +0200
summary:
  Issue #18377: Code cleanup in Python Launcher

This changeset fixes a number of compiler warnings in the Python Launcher
binary for OSX. It also cleans up whitespace usage in those sources.

files:
  Mac/PythonLauncher/FileSettings.h                |   5 -
  Mac/PythonLauncher/FileSettings.m                |  46 ++---
  Mac/PythonLauncher/MyAppDelegate.m               |   6 +-
  Mac/PythonLauncher/MyDocument.m                  |  20 +-
  Mac/PythonLauncher/PreferencesWindowController.m |  11 +-
  Mac/PythonLauncher/doscript.h                    |   2 +-
  Mac/PythonLauncher/doscript.m                    |  78 +++++-----
  Mac/PythonLauncher/main.m                        |   4 +-
  8 files changed, 77 insertions(+), 95 deletions(-)


diff --git a/Mac/PythonLauncher/FileSettings.h b/Mac/PythonLauncher/FileSettings.h
--- a/Mac/PythonLauncher/FileSettings.h
+++ b/Mac/PythonLauncher/FileSettings.h
@@ -45,18 +45,13 @@
 + (id)getFactorySettingsForFileType: (NSString *)filetype;
 + (id)newSettingsForFileType: (NSString *)filetype;
 
-//- (id)init;
 - (id)initForFileType: (NSString *)filetype;
 - (id)initForFSDefaultFileType: (NSString *)filetype;
 - (id)initForDefaultFileType: (NSString *)filetype;
-//- (id)initWithFileSettings: (FileSettings *)source;
 
 - (void)updateFromSource: (id <FileSettingsSource>)source;
 - (NSString *)commandLineForScript: (NSString *)script;
 
-//- (void)applyFactorySettingsForFileType: (NSString *)filetype;
-//- (void)saveDefaults;
-//- (void)applyUserDefaults: (NSString *)filetype;
 - (void)applyValuesFromDict: (NSDictionary *)dict;
 - (void)reset;
 - (NSArray *) interpreters;
diff --git a/Mac/PythonLauncher/FileSettings.m b/Mac/PythonLauncher/FileSettings.m
--- a/Mac/PythonLauncher/FileSettings.m
+++ b/Mac/PythonLauncher/FileSettings.m
@@ -14,7 +14,7 @@
 {
     static FileSettings *fsdefault_py, *fsdefault_pyw, *fsdefault_pyc;
     FileSettings **curdefault;
-    
+
     if ([filetype isEqualToString: @"Python Script"]) {
         curdefault = &fsdefault_py;
     } else if ([filetype isEqualToString: @"Python GUI Script"]) {
@@ -36,7 +36,7 @@
 {
     static FileSettings *default_py, *default_pyw, *default_pyc;
     FileSettings **curdefault;
-    
+
     if ([filetype isEqualToString: @"Python Script"]) {
         curdefault = &default_py;
     } else if ([filetype isEqualToString: @"Python GUI Script"]) {
@@ -57,7 +57,7 @@
 + (id)newSettingsForFileType: (NSString *)filetype
 {
     FileSettings *cur;
-    
+
     cur = [FileSettings new];
     [cur initForFileType: filetype];
     return [cur retain];
@@ -67,7 +67,7 @@
 {
     self = [super init];
     if (!self) return self;
-    
+
     interpreter = [source->interpreter retain];
     honourhashbang = source->honourhashbang;
     debug = source->debug;
@@ -81,36 +81,30 @@
     with_terminal = source->with_terminal;
     prefskey = source->prefskey;
     if (prefskey) [prefskey retain];
-    
+
     return self;
 }
 
 - (id)initForFileType: (NSString *)filetype
 {
     FileSettings *defaults;
-    
+
     defaults = [FileSettings getDefaultsForFileType: filetype];
     self = [self initWithFileSettings: defaults];
     origsource = [defaults retain];
     return self;
 }
 
-//- (id)init
-//{
-//    self = [self initForFileType: @"Python Script"];
-//    return self;
-//}
-
 - (id)initForFSDefaultFileType: (NSString *)filetype
 {
     int i;
     NSString *filename;
     NSDictionary *dict;
     static NSDictionary *factorySettings;
-    
+
     self = [super init];
     if (!self) return self;
-    
+
     if (factorySettings == NULL) {
         NSBundle *bdl = [NSBundle mainBundle];
         NSString *path = [ bdl pathForResource: @"factorySettings"
@@ -149,18 +143,18 @@
 {
     NSUserDefaults *defaults;
     NSDictionary *dict;
-    
+
     defaults = [NSUserDefaults standardUserDefaults];
     dict = [defaults dictionaryForKey: filetype];
     if (!dict)
         return;
     [self applyValuesFromDict: dict];
 }
-    
+
 - (id)initForDefaultFileType: (NSString *)filetype
 {
     FileSettings *fsdefaults;
-    
+
     fsdefaults = [FileSettings getFactorySettingsForFileType: filetype];
     self = [self initWithFileSettings: fsdefaults];
     if (!self) return self;
@@ -220,7 +214,7 @@
 - (void)applyValuesFromDict: (NSDictionary *)dict
 {
     id value;
-    
+
     value = [dict objectForKey: @"interpreter"];
     if (value) interpreter = [value retain];
     value = [dict objectForKey: @"honourhashbang"];
@@ -247,12 +241,12 @@
 
 - (NSString*)_replaceSingleQuotes: (NSString*)string
 {
-	/* Replace all single-quotes by '"'"', that way shellquoting will
-	 * be correct when the result value is delimited  using single quotes.
-	 */
-	NSArray* components = [string componentsSeparatedByString:@"'"];
+    /* Replace all single-quotes by '"'"', that way shellquoting will
+     * be correct when the result value is delimited  using single quotes.
+     */
+    NSArray* components = [string componentsSeparatedByString:@"'"];
 
-	return [components componentsJoinedByString:@"'\"'\"'"];
+    return [components componentsJoinedByString:@"'\"'\"'"];
 }
 
 - (NSString *)commandLineForScript: (NSString *)script
@@ -265,7 +259,7 @@
 
     script_dir = [script substringToIndex:
 	    [script length]-[[script lastPathComponent] length]];
-    
+
     if (honourhashbang &&
        (fp=fopen([script fileSystemRepresentation], "r")) &&
        fgets(hashbangbuf, sizeof(hashbangbuf), fp) &&
@@ -278,7 +272,7 @@
     }
     if (!cur_interp)
         cur_interp = interpreter;
-        
+
     return [NSString stringWithFormat:
         @"cd '%@' && '%@'%s%s%s%s%s%s %@ '%@' %@ %s",
     	[self _replaceSingleQuotes:script_dir],
@@ -297,7 +291,7 @@
 
 - (NSArray *) interpreters { return interpreters;};
 
-// FileSettingsSource protocol 
+// FileSettingsSource protocol
 - (NSString *) interpreter { return interpreter;};
 - (BOOL) honourhashbang { return honourhashbang; };
 - (BOOL) debug { return debug;};
diff --git a/Mac/PythonLauncher/MyAppDelegate.m b/Mac/PythonLauncher/MyAppDelegate.m
--- a/Mac/PythonLauncher/MyAppDelegate.m
+++ b/Mac/PythonLauncher/MyAppDelegate.m
@@ -33,7 +33,7 @@
 
 - (BOOL)shouldShowUI
 {
-    // if this call comes before applicationDidFinishLaunching: we 
+    // if this call comes before applicationDidFinishLaunching: we
     // should terminate immedeately after starting the script.
     if (!initial_action_done)
         should_terminate = YES;
@@ -62,7 +62,7 @@
     static NSString *extensions[] = { @"py", @"pyw", @"pyc", NULL};
     NSString **ext_p;
     int i;
-    
+
     if ([[NSUserDefaults standardUserDefaults] boolForKey: @"SkipFileBindingTest"])
         return;
     ourUrl = [NSURL fileURLWithPath: [[NSBundle mainBundle] bundlePath]];
@@ -92,5 +92,5 @@
         }
     }
 }
-        
+
 @end
diff --git a/Mac/PythonLauncher/MyDocument.m b/Mac/PythonLauncher/MyDocument.m
--- a/Mac/PythonLauncher/MyDocument.m
+++ b/Mac/PythonLauncher/MyDocument.m
@@ -16,7 +16,7 @@
 {
     self = [super init];
     if (self) {
-    
+
         // Add your subclass-specific initialization here.
         // If an error occurs here, send a [self dealloc] message and return nil.
         script = [@"<no script>.py" retain];
@@ -37,20 +37,17 @@
 {
     NSApplication *app = [NSApplication sharedApplication];
     [super close];
-    if ([[app delegate] shouldTerminate])
+    if ([(MyAppDelegate*)[app delegate] shouldTerminate])
         [app terminate: self];
 }
 
 - (void)load_defaults
 {
-//    if (settings) [settings release];
     settings = [FileSettings newSettingsForFileType: filetype];
 }
 
 - (void)update_display
 {
-//    [[self window] setTitle: script];
-    
     [interpreter setStringValue: [settings interpreter]];
     [honourhashbang setState: [settings honourhashbang]];
     [debug setState: [settings debug]];
@@ -62,7 +59,7 @@
     [others setStringValue: [settings others]];
     [scriptargs setStringValue: [settings scriptargs]];
     [with_terminal setState: [settings with_terminal]];
-    
+
     [commandline setStringValue: [settings commandLineForScript: script]];
 }
 
@@ -75,7 +72,7 @@
 {
     const char *cmdline;
     int sts;
-    
+
      cmdline = [[settings commandLineForScript: script] UTF8String];
    if ([settings with_terminal]) {
         sts = doscript(cmdline);
@@ -107,14 +104,13 @@
 {
     // Insert code here to read your document from the given data.  You can also choose to override -loadFileWrapperRepresentation:ofType: or -readFromFile:ofType: instead.
     BOOL show_ui;
-    
-    // ask the app delegate whether we should show the UI or not. 
-    show_ui = [[[NSApplication sharedApplication] delegate] shouldShowUI];
+
+    // ask the app delegate whether we should show the UI or not.
+    show_ui = [(MyAppDelegate*)[[NSApplication sharedApplication] delegate] shouldShowUI];
     [script release];
     script = [fileName retain];
     [filetype release];
     filetype = [type retain];
-//    if (settings) [settings release];
     settings = [FileSettings newSettingsForFileType: filetype];
     if (show_ui) {
         [self update_display];
@@ -152,7 +148,7 @@
     [self update_display];
 }
 
-// FileSettingsSource protocol 
+// FileSettingsSource protocol
 - (NSString *) interpreter { return [interpreter stringValue];};
 - (BOOL) honourhashbang { return [honourhashbang state];};
 - (BOOL) debug { return [debug state];};
diff --git a/Mac/PythonLauncher/PreferencesWindowController.m b/Mac/PythonLauncher/PreferencesWindowController.m
--- a/Mac/PythonLauncher/PreferencesWindowController.m
+++ b/Mac/PythonLauncher/PreferencesWindowController.m
@@ -5,7 +5,7 @@
 + getPreferencesWindow
 {
     static PreferencesWindowController *_singleton;
-    
+
     if (!_singleton)
         _singleton = [[PreferencesWindowController alloc] init];
     [_singleton showWindow: _singleton];
@@ -21,15 +21,13 @@
 - (void)load_defaults
 {
     NSString *title = [filetype titleOfSelectedItem];
-    
+
     settings = [FileSettings getDefaultsForFileType: title];
 }
 
 - (void)update_display
 {
-//    [[self window] setTitle: script];
-    
-	[interpreter reloadData];
+    [interpreter reloadData];
     [interpreter setStringValue: [settings interpreter]];
     [honourhashbang setState: [settings honourhashbang]];
     [debug setState: [settings debug]];
@@ -41,7 +39,6 @@
     [others setStringValue: [settings others]];
     [with_terminal setState: [settings with_terminal]];
     // Not scriptargs, it isn't for preferences
-    
     [commandline setStringValue: [settings commandLineForScript: @"<your script here>"]];
 }
 
@@ -75,7 +72,7 @@
     [self update_display];
 }
 
-// FileSettingsSource protocol 
+// FileSettingsSource protocol
 - (NSString *) interpreter { return [interpreter stringValue];};
 - (BOOL) honourhashbang { return [honourhashbang state]; };
 - (BOOL) debug { return [debug state];};
diff --git a/Mac/PythonLauncher/doscript.h b/Mac/PythonLauncher/doscript.h
--- a/Mac/PythonLauncher/doscript.h
+++ b/Mac/PythonLauncher/doscript.h
@@ -9,4 +9,4 @@
 
 #include <Carbon/Carbon.h>
 
-extern int doscript(const char *command);
\ No newline at end of file
+extern int doscript(const char *command);
diff --git a/Mac/PythonLauncher/doscript.m b/Mac/PythonLauncher/doscript.m
--- a/Mac/PythonLauncher/doscript.m
+++ b/Mac/PythonLauncher/doscript.m
@@ -11,49 +11,49 @@
 #import <ApplicationServices/ApplicationServices.h>
 #import "doscript.h"
 
-extern int 
+extern int
 doscript(const char *command)
 {
-	char *bundleID = "com.apple.Terminal";
-	AppleEvent evt, res;
-	AEDesc desc;
-	OSStatus err;
+    char *bundleID = "com.apple.Terminal";
+    AppleEvent evt, res;
+    AEDesc desc;
+    OSStatus err;
 
-	[[NSWorkspace sharedWorkspace] launchApplication:@"/Applications/Utilities/Terminal.app/"];
+    [[NSWorkspace sharedWorkspace] launchApplication:@"/Applications/Utilities/Terminal.app/"];
 
-	// Build event
-	err = AEBuildAppleEvent(kAECoreSuite, kAEDoScript,
-	                         typeApplicationBundleID,
-	                         bundleID, strlen(bundleID),
-	                         kAutoGenerateReturnID,
-	                         kAnyTransactionID,
-	                         &evt, NULL,
-	                         "'----':utf8(@)", strlen(command),
-	                         command);
-	if (err) {
-		NSLog(@"AEBuildAppleEvent failed: %d\n", err);
-		return err;
-	}
+    // Build event
+    err = AEBuildAppleEvent(kAECoreSuite, kAEDoScript,
+                             typeApplicationBundleID,
+                             bundleID, strlen(bundleID),
+                             kAutoGenerateReturnID,
+                             kAnyTransactionID,
+                             &evt, NULL,
+                             "'----':utf8(@)", strlen(command),
+                             command);
+    if (err) {
+        NSLog(@"AEBuildAppleEvent failed: %ld\n", (long)err);
+        return err;
+    }
 
-	// Send event and check for any Apple Event Manager errors
-	err = AESendMessage(&evt, &res, kAEWaitReply, kAEDefaultTimeout);
-	AEDisposeDesc(&evt);
-	if (err) {
-		NSLog(@"AESendMessage failed: %d\n", err);
-		return err;
-	}
-	// Check for any application errors
-	err = AEGetParamDesc(&res, keyErrorNumber, typeSInt32, &desc);
-	AEDisposeDesc(&res);
-	if (!err) {
-		AEGetDescData(&desc, &err, sizeof(err));
-		NSLog(@"Terminal returned an error: %d", err);
-		AEDisposeDesc(&desc);
-	} else if (err == errAEDescNotFound) {
-		err = noErr;
-	} else {
-		NSLog(@"AEGetPArmDesc returned an error: %d", err);
-	}
+    // Send event and check for any Apple Event Manager errors
+    err = AESendMessage(&evt, &res, kAEWaitReply, kAEDefaultTimeout);
+    AEDisposeDesc(&evt);
+    if (err) {
+        NSLog(@"AESendMessage failed: %ld\n", (long)err);
+        return err;
+    }
+    // Check for any application errors
+    err = AEGetParamDesc(&res, keyErrorNumber, typeSInt32, &desc);
+    AEDisposeDesc(&res);
+    if (!err) {
+        AEGetDescData(&desc, &err, sizeof(err));
+        NSLog(@"Terminal returned an error: %ld", (long)err);
+        AEDisposeDesc(&desc);
+    } else if (err == errAEDescNotFound) {
+        err = noErr;
+    } else {
+        NSLog(@"AEGetPArmDesc returned an error: %ld", (long)err);
+    }
 
-	return err;
+    return err;
 }
diff --git a/Mac/PythonLauncher/main.m b/Mac/PythonLauncher/main.m
--- a/Mac/PythonLauncher/main.m
+++ b/Mac/PythonLauncher/main.m
@@ -11,7 +11,7 @@
 
 int main(int argc, const char *argv[])
 {
-	char *home = getenv("HOME");
-	if (home) chdir(home);
+    char *home = getenv("HOME");
+    if (home) chdir(home);
     return NSApplicationMain(argc, argv);
 }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 09:54:25 2013
From: python-checkins at python.org (ronald.oussoren)
Date: Sun,  7 Jul 2013 09:54:25 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_=283=2E3-=3Edefault=29_Issue_=2318377=3A_Code_cleanup_in?=
 =?utf-8?q?_Python_Launcher?=
Message-ID: <3bp2B94TFbz7Ljk@mail.python.org>

http://hg.python.org/cpython/rev/d7a59e6f48df
changeset:   84480:d7a59e6f48df
parent:      84477:83810f67d16b
parent:      84479:738cba2bf849
user:        Ronald Oussoren <ronaldoussoren at mac.com>
date:        Sun Jul 07 09:54:08 2013 +0200
summary:
  (3.3->default) Issue #18377: Code cleanup in Python Launcher

This changeset fixes a number of compiler warnings in the Python Launcher
binary for OSX. It also cleans up whitespace usage in those sources.

files:
  Mac/PythonLauncher/FileSettings.h                |   5 -
  Mac/PythonLauncher/FileSettings.m                |  46 ++---
  Mac/PythonLauncher/MyAppDelegate.m               |   6 +-
  Mac/PythonLauncher/MyDocument.m                  |  20 +-
  Mac/PythonLauncher/PreferencesWindowController.m |  11 +-
  Mac/PythonLauncher/doscript.h                    |   2 +-
  Mac/PythonLauncher/doscript.m                    |  78 +++++-----
  Mac/PythonLauncher/main.m                        |   4 +-
  8 files changed, 77 insertions(+), 95 deletions(-)


diff --git a/Mac/PythonLauncher/FileSettings.h b/Mac/PythonLauncher/FileSettings.h
--- a/Mac/PythonLauncher/FileSettings.h
+++ b/Mac/PythonLauncher/FileSettings.h
@@ -45,18 +45,13 @@
 + (id)getFactorySettingsForFileType: (NSString *)filetype;
 + (id)newSettingsForFileType: (NSString *)filetype;
 
-//- (id)init;
 - (id)initForFileType: (NSString *)filetype;
 - (id)initForFSDefaultFileType: (NSString *)filetype;
 - (id)initForDefaultFileType: (NSString *)filetype;
-//- (id)initWithFileSettings: (FileSettings *)source;
 
 - (void)updateFromSource: (id <FileSettingsSource>)source;
 - (NSString *)commandLineForScript: (NSString *)script;
 
-//- (void)applyFactorySettingsForFileType: (NSString *)filetype;
-//- (void)saveDefaults;
-//- (void)applyUserDefaults: (NSString *)filetype;
 - (void)applyValuesFromDict: (NSDictionary *)dict;
 - (void)reset;
 - (NSArray *) interpreters;
diff --git a/Mac/PythonLauncher/FileSettings.m b/Mac/PythonLauncher/FileSettings.m
--- a/Mac/PythonLauncher/FileSettings.m
+++ b/Mac/PythonLauncher/FileSettings.m
@@ -14,7 +14,7 @@
 {
     static FileSettings *fsdefault_py, *fsdefault_pyw, *fsdefault_pyc;
     FileSettings **curdefault;
-    
+
     if ([filetype isEqualToString: @"Python Script"]) {
         curdefault = &fsdefault_py;
     } else if ([filetype isEqualToString: @"Python GUI Script"]) {
@@ -36,7 +36,7 @@
 {
     static FileSettings *default_py, *default_pyw, *default_pyc;
     FileSettings **curdefault;
-    
+
     if ([filetype isEqualToString: @"Python Script"]) {
         curdefault = &default_py;
     } else if ([filetype isEqualToString: @"Python GUI Script"]) {
@@ -57,7 +57,7 @@
 + (id)newSettingsForFileType: (NSString *)filetype
 {
     FileSettings *cur;
-    
+
     cur = [FileSettings new];
     [cur initForFileType: filetype];
     return [cur retain];
@@ -67,7 +67,7 @@
 {
     self = [super init];
     if (!self) return self;
-    
+
     interpreter = [source->interpreter retain];
     honourhashbang = source->honourhashbang;
     debug = source->debug;
@@ -81,36 +81,30 @@
     with_terminal = source->with_terminal;
     prefskey = source->prefskey;
     if (prefskey) [prefskey retain];
-    
+
     return self;
 }
 
 - (id)initForFileType: (NSString *)filetype
 {
     FileSettings *defaults;
-    
+
     defaults = [FileSettings getDefaultsForFileType: filetype];
     self = [self initWithFileSettings: defaults];
     origsource = [defaults retain];
     return self;
 }
 
-//- (id)init
-//{
-//    self = [self initForFileType: @"Python Script"];
-//    return self;
-//}
-
 - (id)initForFSDefaultFileType: (NSString *)filetype
 {
     int i;
     NSString *filename;
     NSDictionary *dict;
     static NSDictionary *factorySettings;
-    
+
     self = [super init];
     if (!self) return self;
-    
+
     if (factorySettings == NULL) {
         NSBundle *bdl = [NSBundle mainBundle];
         NSString *path = [ bdl pathForResource: @"factorySettings"
@@ -149,18 +143,18 @@
 {
     NSUserDefaults *defaults;
     NSDictionary *dict;
-    
+
     defaults = [NSUserDefaults standardUserDefaults];
     dict = [defaults dictionaryForKey: filetype];
     if (!dict)
         return;
     [self applyValuesFromDict: dict];
 }
-    
+
 - (id)initForDefaultFileType: (NSString *)filetype
 {
     FileSettings *fsdefaults;
-    
+
     fsdefaults = [FileSettings getFactorySettingsForFileType: filetype];
     self = [self initWithFileSettings: fsdefaults];
     if (!self) return self;
@@ -220,7 +214,7 @@
 - (void)applyValuesFromDict: (NSDictionary *)dict
 {
     id value;
-    
+
     value = [dict objectForKey: @"interpreter"];
     if (value) interpreter = [value retain];
     value = [dict objectForKey: @"honourhashbang"];
@@ -247,12 +241,12 @@
 
 - (NSString*)_replaceSingleQuotes: (NSString*)string
 {
-	/* Replace all single-quotes by '"'"', that way shellquoting will
-	 * be correct when the result value is delimited  using single quotes.
-	 */
-	NSArray* components = [string componentsSeparatedByString:@"'"];
+    /* Replace all single-quotes by '"'"', that way shellquoting will
+     * be correct when the result value is delimited  using single quotes.
+     */
+    NSArray* components = [string componentsSeparatedByString:@"'"];
 
-	return [components componentsJoinedByString:@"'\"'\"'"];
+    return [components componentsJoinedByString:@"'\"'\"'"];
 }
 
 - (NSString *)commandLineForScript: (NSString *)script
@@ -265,7 +259,7 @@
 
     script_dir = [script substringToIndex:
 	    [script length]-[[script lastPathComponent] length]];
-    
+
     if (honourhashbang &&
        (fp=fopen([script fileSystemRepresentation], "r")) &&
        fgets(hashbangbuf, sizeof(hashbangbuf), fp) &&
@@ -278,7 +272,7 @@
     }
     if (!cur_interp)
         cur_interp = interpreter;
-        
+
     return [NSString stringWithFormat:
         @"cd '%@' && '%@'%s%s%s%s%s%s %@ '%@' %@ %s",
     	[self _replaceSingleQuotes:script_dir],
@@ -297,7 +291,7 @@
 
 - (NSArray *) interpreters { return interpreters;};
 
-// FileSettingsSource protocol 
+// FileSettingsSource protocol
 - (NSString *) interpreter { return interpreter;};
 - (BOOL) honourhashbang { return honourhashbang; };
 - (BOOL) debug { return debug;};
diff --git a/Mac/PythonLauncher/MyAppDelegate.m b/Mac/PythonLauncher/MyAppDelegate.m
--- a/Mac/PythonLauncher/MyAppDelegate.m
+++ b/Mac/PythonLauncher/MyAppDelegate.m
@@ -33,7 +33,7 @@
 
 - (BOOL)shouldShowUI
 {
-    // if this call comes before applicationDidFinishLaunching: we 
+    // if this call comes before applicationDidFinishLaunching: we
     // should terminate immedeately after starting the script.
     if (!initial_action_done)
         should_terminate = YES;
@@ -62,7 +62,7 @@
     static NSString *extensions[] = { @"py", @"pyw", @"pyc", NULL};
     NSString **ext_p;
     int i;
-    
+
     if ([[NSUserDefaults standardUserDefaults] boolForKey: @"SkipFileBindingTest"])
         return;
     ourUrl = [NSURL fileURLWithPath: [[NSBundle mainBundle] bundlePath]];
@@ -92,5 +92,5 @@
         }
     }
 }
-        
+
 @end
diff --git a/Mac/PythonLauncher/MyDocument.m b/Mac/PythonLauncher/MyDocument.m
--- a/Mac/PythonLauncher/MyDocument.m
+++ b/Mac/PythonLauncher/MyDocument.m
@@ -16,7 +16,7 @@
 {
     self = [super init];
     if (self) {
-    
+
         // Add your subclass-specific initialization here.
         // If an error occurs here, send a [self dealloc] message and return nil.
         script = [@"<no script>.py" retain];
@@ -37,20 +37,17 @@
 {
     NSApplication *app = [NSApplication sharedApplication];
     [super close];
-    if ([[app delegate] shouldTerminate])
+    if ([(MyAppDelegate*)[app delegate] shouldTerminate])
         [app terminate: self];
 }
 
 - (void)load_defaults
 {
-//    if (settings) [settings release];
     settings = [FileSettings newSettingsForFileType: filetype];
 }
 
 - (void)update_display
 {
-//    [[self window] setTitle: script];
-    
     [interpreter setStringValue: [settings interpreter]];
     [honourhashbang setState: [settings honourhashbang]];
     [debug setState: [settings debug]];
@@ -62,7 +59,7 @@
     [others setStringValue: [settings others]];
     [scriptargs setStringValue: [settings scriptargs]];
     [with_terminal setState: [settings with_terminal]];
-    
+
     [commandline setStringValue: [settings commandLineForScript: script]];
 }
 
@@ -75,7 +72,7 @@
 {
     const char *cmdline;
     int sts;
-    
+
      cmdline = [[settings commandLineForScript: script] UTF8String];
    if ([settings with_terminal]) {
         sts = doscript(cmdline);
@@ -107,14 +104,13 @@
 {
     // Insert code here to read your document from the given data.  You can also choose to override -loadFileWrapperRepresentation:ofType: or -readFromFile:ofType: instead.
     BOOL show_ui;
-    
-    // ask the app delegate whether we should show the UI or not. 
-    show_ui = [[[NSApplication sharedApplication] delegate] shouldShowUI];
+
+    // ask the app delegate whether we should show the UI or not.
+    show_ui = [(MyAppDelegate*)[[NSApplication sharedApplication] delegate] shouldShowUI];
     [script release];
     script = [fileName retain];
     [filetype release];
     filetype = [type retain];
-//    if (settings) [settings release];
     settings = [FileSettings newSettingsForFileType: filetype];
     if (show_ui) {
         [self update_display];
@@ -152,7 +148,7 @@
     [self update_display];
 }
 
-// FileSettingsSource protocol 
+// FileSettingsSource protocol
 - (NSString *) interpreter { return [interpreter stringValue];};
 - (BOOL) honourhashbang { return [honourhashbang state];};
 - (BOOL) debug { return [debug state];};
diff --git a/Mac/PythonLauncher/PreferencesWindowController.m b/Mac/PythonLauncher/PreferencesWindowController.m
--- a/Mac/PythonLauncher/PreferencesWindowController.m
+++ b/Mac/PythonLauncher/PreferencesWindowController.m
@@ -5,7 +5,7 @@
 + getPreferencesWindow
 {
     static PreferencesWindowController *_singleton;
-    
+
     if (!_singleton)
         _singleton = [[PreferencesWindowController alloc] init];
     [_singleton showWindow: _singleton];
@@ -21,15 +21,13 @@
 - (void)load_defaults
 {
     NSString *title = [filetype titleOfSelectedItem];
-    
+
     settings = [FileSettings getDefaultsForFileType: title];
 }
 
 - (void)update_display
 {
-//    [[self window] setTitle: script];
-    
-	[interpreter reloadData];
+    [interpreter reloadData];
     [interpreter setStringValue: [settings interpreter]];
     [honourhashbang setState: [settings honourhashbang]];
     [debug setState: [settings debug]];
@@ -41,7 +39,6 @@
     [others setStringValue: [settings others]];
     [with_terminal setState: [settings with_terminal]];
     // Not scriptargs, it isn't for preferences
-    
     [commandline setStringValue: [settings commandLineForScript: @"<your script here>"]];
 }
 
@@ -75,7 +72,7 @@
     [self update_display];
 }
 
-// FileSettingsSource protocol 
+// FileSettingsSource protocol
 - (NSString *) interpreter { return [interpreter stringValue];};
 - (BOOL) honourhashbang { return [honourhashbang state]; };
 - (BOOL) debug { return [debug state];};
diff --git a/Mac/PythonLauncher/doscript.h b/Mac/PythonLauncher/doscript.h
--- a/Mac/PythonLauncher/doscript.h
+++ b/Mac/PythonLauncher/doscript.h
@@ -9,4 +9,4 @@
 
 #include <Carbon/Carbon.h>
 
-extern int doscript(const char *command);
\ No newline at end of file
+extern int doscript(const char *command);
diff --git a/Mac/PythonLauncher/doscript.m b/Mac/PythonLauncher/doscript.m
--- a/Mac/PythonLauncher/doscript.m
+++ b/Mac/PythonLauncher/doscript.m
@@ -11,49 +11,49 @@
 #import <ApplicationServices/ApplicationServices.h>
 #import "doscript.h"
 
-extern int 
+extern int
 doscript(const char *command)
 {
-	char *bundleID = "com.apple.Terminal";
-	AppleEvent evt, res;
-	AEDesc desc;
-	OSStatus err;
+    char *bundleID = "com.apple.Terminal";
+    AppleEvent evt, res;
+    AEDesc desc;
+    OSStatus err;
 
-	[[NSWorkspace sharedWorkspace] launchApplication:@"/Applications/Utilities/Terminal.app/"];
+    [[NSWorkspace sharedWorkspace] launchApplication:@"/Applications/Utilities/Terminal.app/"];
 
-	// Build event
-	err = AEBuildAppleEvent(kAECoreSuite, kAEDoScript,
-	                         typeApplicationBundleID,
-	                         bundleID, strlen(bundleID),
-	                         kAutoGenerateReturnID,
-	                         kAnyTransactionID,
-	                         &evt, NULL,
-	                         "'----':utf8(@)", strlen(command),
-	                         command);
-	if (err) {
-		NSLog(@"AEBuildAppleEvent failed: %d\n", err);
-		return err;
-	}
+    // Build event
+    err = AEBuildAppleEvent(kAECoreSuite, kAEDoScript,
+                             typeApplicationBundleID,
+                             bundleID, strlen(bundleID),
+                             kAutoGenerateReturnID,
+                             kAnyTransactionID,
+                             &evt, NULL,
+                             "'----':utf8(@)", strlen(command),
+                             command);
+    if (err) {
+        NSLog(@"AEBuildAppleEvent failed: %ld\n", (long)err);
+        return err;
+    }
 
-	// Send event and check for any Apple Event Manager errors
-	err = AESendMessage(&evt, &res, kAEWaitReply, kAEDefaultTimeout);
-	AEDisposeDesc(&evt);
-	if (err) {
-		NSLog(@"AESendMessage failed: %d\n", err);
-		return err;
-	}
-	// Check for any application errors
-	err = AEGetParamDesc(&res, keyErrorNumber, typeSInt32, &desc);
-	AEDisposeDesc(&res);
-	if (!err) {
-		AEGetDescData(&desc, &err, sizeof(err));
-		NSLog(@"Terminal returned an error: %d", err);
-		AEDisposeDesc(&desc);
-	} else if (err == errAEDescNotFound) {
-		err = noErr;
-	} else {
-		NSLog(@"AEGetPArmDesc returned an error: %d", err);
-	}
+    // Send event and check for any Apple Event Manager errors
+    err = AESendMessage(&evt, &res, kAEWaitReply, kAEDefaultTimeout);
+    AEDisposeDesc(&evt);
+    if (err) {
+        NSLog(@"AESendMessage failed: %ld\n", (long)err);
+        return err;
+    }
+    // Check for any application errors
+    err = AEGetParamDesc(&res, keyErrorNumber, typeSInt32, &desc);
+    AEDisposeDesc(&res);
+    if (!err) {
+        AEGetDescData(&desc, &err, sizeof(err));
+        NSLog(@"Terminal returned an error: %ld", (long)err);
+        AEDisposeDesc(&desc);
+    } else if (err == errAEDescNotFound) {
+        err = noErr;
+    } else {
+        NSLog(@"AEGetPArmDesc returned an error: %ld", (long)err);
+    }
 
-	return err;
+    return err;
 }
diff --git a/Mac/PythonLauncher/main.m b/Mac/PythonLauncher/main.m
--- a/Mac/PythonLauncher/main.m
+++ b/Mac/PythonLauncher/main.m
@@ -11,7 +11,7 @@
 
 int main(int argc, const char *argv[])
 {
-	char *home = getenv("HOME");
-	if (home) chdir(home);
+    char *home = getenv("HOME");
+    if (home) chdir(home);
     return NSApplicationMain(argc, argv);
 }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 11:11:37 2013
From: python-checkins at python.org (ezio.melotti)
Date: Sun,  7 Jul 2013 11:11:37 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_=2318020=3A_improve_html?=
 =?utf-8?q?=2Eescape_speed_by_an_order_of_magnitude=2E__Patch_by_Matt?=
Message-ID: <3bp3vF07Nsz7Lk7@mail.python.org>

http://hg.python.org/cpython/rev/db5f2b74e369
changeset:   84481:db5f2b74e369
user:        Ezio Melotti <ezio.melotti at gmail.com>
date:        Sun Jul 07 11:11:24 2013 +0200
summary:
  #18020: improve html.escape speed by an order of magnitude.  Patch by Matt Bryant.

files:
  Lib/html/__init__.py |  13 ++++++-------
  Misc/ACKS            |   1 +
  Misc/NEWS            |   3 +++
  3 files changed, 10 insertions(+), 7 deletions(-)


diff --git a/Lib/html/__init__.py b/Lib/html/__init__.py
--- a/Lib/html/__init__.py
+++ b/Lib/html/__init__.py
@@ -2,11 +2,6 @@
 General functions for HTML manipulation.
 """
 
-
-_escape_map = {ord('&'): '&amp;', ord('<'): '&lt;', ord('>'): '&gt;'}
-_escape_map_full = {ord('&'): '&amp;', ord('<'): '&lt;', ord('>'): '&gt;',
-                    ord('"'): '&quot;', ord('\''): '&#x27;'}
-
 # NB: this is a candidate for a bytes/string polymorphic interface
 
 def escape(s, quote=True):
@@ -16,6 +11,10 @@
     characters, both double quote (") and single quote (') characters are also
     translated.
     """
+    s = s.replace("&", "&amp;") # Must be done first!
+    s = s.replace("<", "&lt;")
+    s = s.replace(">", "&gt;")
     if quote:
-        return s.translate(_escape_map_full)
-    return s.translate(_escape_map)
+        s = s.replace('"', "&quot;")
+        s = s.replace('\'', "&#x27;")
+    return s
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -172,6 +172,7 @@
 Francisco Mart?n Brugu?
 Ian Bruntlett
 Floris Bruynooghe
+Matt Bryant
 Stan Bubrouski
 Erik de Bueger
 Jan-Hein B?hrman
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -142,6 +142,9 @@
 Library
 -------
 
+- Issue #18020: improve html.escape speed by an order of magnitude.
+  Patch by Matt Bryant.
+
 - Issue #18347: ElementTree's html serializer now preserves the case of
   closing tags.
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 12:47:12 2013
From: python-checkins at python.org (florent.xicluna)
Date: Sun,  7 Jul 2013 12:47:12 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4MDEz?=
 =?utf-8?q?=3A_Fix_cgi=2EFieldStorage_to_parse_the_W3C_sample_form=2E?=
Message-ID: <3bp61X2mRrzSqb@mail.python.org>

http://hg.python.org/cpython/rev/406ce103c170
changeset:   84482:406ce103c170
branch:      3.3
parent:      84479:738cba2bf849
user:        Florent Xicluna <florent.xicluna at gmail.com>
date:        Sun Jul 07 12:44:28 2013 +0200
summary:
  Issue #18013: Fix cgi.FieldStorage to parse the W3C sample form.

files:
  Lib/cgi.py           |   2 +-
  Lib/test/test_cgi.py |  46 ++++++++++++++++++++++++++++++++
  Misc/NEWS            |   2 +
  3 files changed, 49 insertions(+), 1 deletions(-)


diff --git a/Lib/cgi.py b/Lib/cgi.py
--- a/Lib/cgi.py
+++ b/Lib/cgi.py
@@ -699,7 +699,7 @@
                          self.encoding, self.errors)
             self.bytes_read += part.bytes_read
             self.list.append(part)
-            if self.bytes_read >= self.length:
+            if part.done or self.bytes_read >= self.length > 0:
                 break
         self.skip_lines()
 
diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py
--- a/Lib/test/test_cgi.py
+++ b/Lib/test/test_cgi.py
@@ -279,6 +279,27 @@
         check('x' * (maxline - 1) + '\r')
         check('x' * (maxline - 1) + '\r' + 'y' * (maxline - 1))
 
+    def test_fieldstorage_multipart_w3c(self):
+        # Test basic FieldStorage multipart parsing (W3C sample)
+        env = {
+            'REQUEST_METHOD': 'POST',
+            'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY_W3),
+            'CONTENT_LENGTH': str(len(POSTDATA_W3))}
+        fp = BytesIO(POSTDATA_W3.encode('latin-1'))
+        fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1")
+        self.assertEqual(len(fs.list), 2)
+        self.assertEqual(fs.list[0].name, 'submit-name')
+        self.assertEqual(fs.list[0].value, 'Larry')
+        self.assertEqual(fs.list[1].name, 'files')
+        files = fs.list[1].value
+        self.assertEqual(len(files), 2)
+        expect = [{'name': None, 'filename': 'file1.txt', 'value': b'... contents of file1.txt ...'},
+                  {'name': None, 'filename': 'file2.gif', 'value': b'...contents of file2.gif...'}]
+        for x in range(len(files)):
+            for k, exp in expect[x].items():
+                got = getattr(files[x], k)
+                self.assertEqual(got, exp)
+
     _qs_result = {
         'key1': 'value1',
         'key2': ['value2x', 'value2y'],
@@ -428,6 +449,31 @@
 -----------------------------721837373350705526688164684
 """
 
+# http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4
+BOUNDARY_W3 = "AaB03x"
+POSTDATA_W3 = """--AaB03x
+Content-Disposition: form-data; name="submit-name"
+
+Larry
+--AaB03x
+Content-Disposition: form-data; name="files"
+Content-Type: multipart/mixed; boundary=BbC04y
+
+--BbC04y
+Content-Disposition: file; filename="file1.txt"
+Content-Type: text/plain
+
+... contents of file1.txt ...
+--BbC04y
+Content-Disposition: file; filename="file2.gif"
+Content-Type: image/gif
+Content-Transfer-Encoding: binary
+
+...contents of file2.gif...
+--BbC04y--
+--AaB03x--
+"""
+
 
 def test_main():
     run_unittest(CgiTests)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -41,6 +41,8 @@
 Library
 -------
 
+- Issue #18013: Fix cgi.FieldStorage to parse the W3C sample form.
+
 - Issue #18347: ElementTree's html serializer now preserves the case of
   closing tags.
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 12:47:13 2013
From: python-checkins at python.org (florent.xicluna)
Date: Sun,  7 Jul 2013 12:47:13 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Merge_=2318013=3A_Fix_cgi=2EFieldStorage_to_parse_the_W3?=
 =?utf-8?q?C_sample_form=2E?=
Message-ID: <3bp61Y57Ldz7LjN@mail.python.org>

http://hg.python.org/cpython/rev/2ab2a2bfea49
changeset:   84483:2ab2a2bfea49
parent:      84481:db5f2b74e369
parent:      84482:406ce103c170
user:        Florent Xicluna <florent.xicluna at gmail.com>
date:        Sun Jul 07 12:46:28 2013 +0200
summary:
  Merge #18013: Fix cgi.FieldStorage to parse the W3C sample form.

files:
  Lib/cgi.py           |   2 +-
  Lib/test/test_cgi.py |  46 ++++++++++++++++++++++++++++++++
  Misc/NEWS            |   2 +
  3 files changed, 49 insertions(+), 1 deletions(-)


diff --git a/Lib/cgi.py b/Lib/cgi.py
--- a/Lib/cgi.py
+++ b/Lib/cgi.py
@@ -699,7 +699,7 @@
                          self.encoding, self.errors)
             self.bytes_read += part.bytes_read
             self.list.append(part)
-            if self.bytes_read >= self.length:
+            if part.done or self.bytes_read >= self.length > 0:
                 break
         self.skip_lines()
 
diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py
--- a/Lib/test/test_cgi.py
+++ b/Lib/test/test_cgi.py
@@ -279,6 +279,27 @@
         check('x' * (maxline - 1) + '\r')
         check('x' * (maxline - 1) + '\r' + 'y' * (maxline - 1))
 
+    def test_fieldstorage_multipart_w3c(self):
+        # Test basic FieldStorage multipart parsing (W3C sample)
+        env = {
+            'REQUEST_METHOD': 'POST',
+            'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY_W3),
+            'CONTENT_LENGTH': str(len(POSTDATA_W3))}
+        fp = BytesIO(POSTDATA_W3.encode('latin-1'))
+        fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1")
+        self.assertEqual(len(fs.list), 2)
+        self.assertEqual(fs.list[0].name, 'submit-name')
+        self.assertEqual(fs.list[0].value, 'Larry')
+        self.assertEqual(fs.list[1].name, 'files')
+        files = fs.list[1].value
+        self.assertEqual(len(files), 2)
+        expect = [{'name': None, 'filename': 'file1.txt', 'value': b'... contents of file1.txt ...'},
+                  {'name': None, 'filename': 'file2.gif', 'value': b'...contents of file2.gif...'}]
+        for x in range(len(files)):
+            for k, exp in expect[x].items():
+                got = getattr(files[x], k)
+                self.assertEqual(got, exp)
+
     _qs_result = {
         'key1': 'value1',
         'key2': ['value2x', 'value2y'],
@@ -428,6 +449,31 @@
 -----------------------------721837373350705526688164684
 """
 
+# http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4
+BOUNDARY_W3 = "AaB03x"
+POSTDATA_W3 = """--AaB03x
+Content-Disposition: form-data; name="submit-name"
+
+Larry
+--AaB03x
+Content-Disposition: form-data; name="files"
+Content-Type: multipart/mixed; boundary=BbC04y
+
+--BbC04y
+Content-Disposition: file; filename="file1.txt"
+Content-Type: text/plain
+
+... contents of file1.txt ...
+--BbC04y
+Content-Disposition: file; filename="file2.gif"
+Content-Type: image/gif
+Content-Transfer-Encoding: binary
+
+...contents of file2.gif...
+--BbC04y--
+--AaB03x--
+"""
+
 
 def test_main():
     run_unittest(CgiTests)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -142,6 +142,8 @@
 Library
 -------
 
+- Issue #18013: Fix cgi.FieldStorage to parse the W3C sample form.
+
 - Issue #18020: improve html.escape speed by an order of magnitude.
   Patch by Matt Bryant.
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 13:16:16 2013
From: python-checkins at python.org (ezio.melotti)
Date: Sun,  7 Jul 2013 13:16:16 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE3MTk4OiBGaXgg?=
 =?utf-8?q?a_NameError_in_the_dbm_module=2E__Patch_by_Valentina_Mukhamedzh?=
 =?utf-8?q?anova=2E?=
Message-ID: <3bp6g423Qbz7Lk7@mail.python.org>

http://hg.python.org/cpython/rev/65fce1dad331
changeset:   84484:65fce1dad331
branch:      3.3
parent:      84482:406ce103c170
user:        Ezio Melotti <ezio.melotti at gmail.com>
date:        Sun Jul 07 13:15:08 2013 +0200
summary:
  #17198: Fix a NameError in the dbm module.  Patch by Valentina Mukhamedzhanova.

files:
  Lib/dbm/__init__.py  |   5 +++++
  Lib/test/test_dbm.py |  18 ++++++++++++++++--
  Misc/NEWS            |   3 +++
  3 files changed, 24 insertions(+), 2 deletions(-)


diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py
--- a/Lib/dbm/__init__.py
+++ b/Lib/dbm/__init__.py
@@ -44,6 +44,11 @@
 
 error = (error, IOError)
 
+try:
+    from dbm import ndbm
+except ImportError:
+    ndbm = None
+
 
 def open(file, flag='r', mode=0o666):
     """Open or create database at path given by *file*.
diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py
--- a/Lib/test/test_dbm.py
+++ b/Lib/test/test_dbm.py
@@ -9,6 +9,11 @@
 # Skip tests if dbm module doesn't exist.
 dbm = test.support.import_module('dbm')
 
+try:
+    from dbm import ndbm
+except ImportError:
+    ndbm = None
+
 _fname = test.support.TESTFN
 
 #
@@ -130,7 +135,7 @@
             delete_files()
             f = module.open(_fname, 'c')
             f.close()
-            self.assertEqual(name, dbm.whichdb(_fname))
+            self.assertEqual(name, self.dbm.whichdb(_fname))
             # Now add a key
             f = module.open(_fname, 'w')
             f[b"1"] = b"1"
@@ -139,7 +144,15 @@
             # and read it
             self.assertTrue(f[b"1"] == b"1")
             f.close()
-            self.assertEqual(name, dbm.whichdb(_fname))
+            self.assertEqual(name, self.dbm.whichdb(_fname))
+
+    @unittest.skipUnless(ndbm, reason='Test requires ndbm')
+    def test_whichdb_ndbm(self):
+        # Issue 17198: check that ndbm which is referenced in whichdb is defined
+        db_file = '{}_ndbm.db'.format(_fname)
+        with open(db_file, 'w'):
+            self.addCleanup(test.support.unlink, db_file)
+        self.assertIsNone(self.dbm.whichdb(db_file[:-3]))
 
     def tearDown(self):
         delete_files()
@@ -149,6 +162,7 @@
         self.filename = test.support.TESTFN
         self.d = dbm.open(self.filename, 'c')
         self.d.close()
+        self.dbm = test.support.import_fresh_module('dbm')
 
     def test_keys(self):
         self.d = dbm.open(self.filename, 'c')
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -41,6 +41,9 @@
 Library
 -------
 
+- Issue #17198: Fix a NameError in the dbm module.  Patch by Valentina
+  Mukhamedzhanova.
+
 - Issue #18013: Fix cgi.FieldStorage to parse the W3C sample form.
 
 - Issue #18347: ElementTree's html serializer now preserves the case of

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 13:16:17 2013
From: python-checkins at python.org (ezio.melotti)
Date: Sun,  7 Jul 2013 13:16:17 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?b?KTogIzE3MTk4OiBtZXJnZSB3aXRoIDMuMy4=?=
Message-ID: <3bp6g54Klpz7Ljp@mail.python.org>

http://hg.python.org/cpython/rev/e91e9b9ba180
changeset:   84485:e91e9b9ba180
parent:      84483:2ab2a2bfea49
parent:      84484:65fce1dad331
user:        Ezio Melotti <ezio.melotti at gmail.com>
date:        Sun Jul 07 13:16:05 2013 +0200
summary:
  #17198: merge with 3.3.

files:
  Lib/dbm/__init__.py  |   5 +++++
  Lib/test/test_dbm.py |  18 ++++++++++++++++--
  Misc/NEWS            |   3 +++
  3 files changed, 24 insertions(+), 2 deletions(-)


diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py
--- a/Lib/dbm/__init__.py
+++ b/Lib/dbm/__init__.py
@@ -44,6 +44,11 @@
 
 error = (error, OSError)
 
+try:
+    from dbm import ndbm
+except ImportError:
+    ndbm = None
+
 
 def open(file, flag='r', mode=0o666):
     """Open or create database at path given by *file*.
diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py
--- a/Lib/test/test_dbm.py
+++ b/Lib/test/test_dbm.py
@@ -9,6 +9,11 @@
 # Skip tests if dbm module doesn't exist.
 dbm = test.support.import_module('dbm')
 
+try:
+    from dbm import ndbm
+except ImportError:
+    ndbm = None
+
 _fname = test.support.TESTFN
 
 #
@@ -130,7 +135,7 @@
             delete_files()
             f = module.open(_fname, 'c')
             f.close()
-            self.assertEqual(name, dbm.whichdb(_fname))
+            self.assertEqual(name, self.dbm.whichdb(_fname))
             # Now add a key
             f = module.open(_fname, 'w')
             f[b"1"] = b"1"
@@ -139,7 +144,15 @@
             # and read it
             self.assertTrue(f[b"1"] == b"1")
             f.close()
-            self.assertEqual(name, dbm.whichdb(_fname))
+            self.assertEqual(name, self.dbm.whichdb(_fname))
+
+    @unittest.skipUnless(ndbm, reason='Test requires ndbm')
+    def test_whichdb_ndbm(self):
+        # Issue 17198: check that ndbm which is referenced in whichdb is defined
+        db_file = '{}_ndbm.db'.format(_fname)
+        with open(db_file, 'w'):
+            self.addCleanup(test.support.unlink, db_file)
+        self.assertIsNone(self.dbm.whichdb(db_file[:-3]))
 
     def tearDown(self):
         delete_files()
@@ -149,6 +162,7 @@
         self.filename = test.support.TESTFN
         self.d = dbm.open(self.filename, 'c')
         self.d.close()
+        self.dbm = test.support.import_fresh_module('dbm')
 
     def test_keys(self):
         self.d = dbm.open(self.filename, 'c')
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -142,6 +142,9 @@
 Library
 -------
 
+- Issue #17198: Fix a NameError in the dbm module.  Patch by Valentina
+  Mukhamedzhanova.
+
 - Issue #18013: Fix cgi.FieldStorage to parse the W3C sample form.
 
 - Issue #18020: improve html.escape speed by an order of magnitude.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 13:37:32 2013
From: python-checkins at python.org (ezio.melotti)
Date: Sun,  7 Jul 2013 13:37:32 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_=2318106=3A_refactor_tests?=
 =?utf-8?q?_to_use_subtests_and_proper_assert_methods=2E__Patch_by?=
Message-ID: <3bp77c2lbdz7LjS@mail.python.org>

http://hg.python.org/cpython/rev/a5010de76eda
changeset:   84486:a5010de76eda
user:        Ezio Melotti <ezio.melotti at gmail.com>
date:        Sun Jul 07 13:37:20 2013 +0200
summary:
  #18106: refactor tests to use subtests and proper assert methods.  Patch by Vajrasky Kok.

files:
  Lib/test/test_collections.py |  77 +++++++++++++----------
  1 files changed, 42 insertions(+), 35 deletions(-)


diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py
--- a/Lib/test/test_collections.py
+++ b/Lib/test/test_collections.py
@@ -918,23 +918,26 @@
         words = Counter('which witch had which witches wrist watch'.split())
         update_test = Counter()
         update_test.update(words)
-        for i, dup in enumerate([
-                    words.copy(),
-                    copy.copy(words),
-                    copy.deepcopy(words),
-                    pickle.loads(pickle.dumps(words, 0)),
-                    pickle.loads(pickle.dumps(words, 1)),
-                    pickle.loads(pickle.dumps(words, 2)),
-                    pickle.loads(pickle.dumps(words, -1)),
-                    eval(repr(words)),
-                    update_test,
-                    Counter(words),
-                    ]):
-            msg = (i, dup, words)
-            self.assertTrue(dup is not words)
-            self.assertEqual(dup, words)
-            self.assertEqual(len(dup), len(words))
-            self.assertEqual(type(dup), type(words))
+        for label, dup in [
+                    ('words.copy()', words.copy()),
+                    ('copy.copy(words)', copy.copy(words)),
+                    ('copy.deepcopy(words)', copy.deepcopy(words)),
+                    ('pickle.loads(pickle.dumps(words, 0))',
+                        pickle.loads(pickle.dumps(words, 0))),
+                    ('pickle.loads(pickle.dumps(words, 1))',
+                        pickle.loads(pickle.dumps(words, 1))),
+                    ('pickle.loads(pickle.dumps(words, 2))',
+                        pickle.loads(pickle.dumps(words, 2))),
+                    ('pickle.loads(pickle.dumps(words, -1))',
+                        pickle.loads(pickle.dumps(words, -1))),
+                    ('eval(repr(words))', eval(repr(words))),
+                    ('update_test', update_test),
+                    ('Counter(words)', Counter(words)),
+                    ]:
+            with self.subTest(label=label):
+                msg = "\ncopy: %s\nwords: %s" % (dup, words)
+                self.assertIsNot(dup, words, msg)
+                self.assertEqual(dup, words)
 
     def test_copy_subclass(self):
         class MyCounter(Counter):
@@ -1213,24 +1216,28 @@
         od = OrderedDict(pairs)
         update_test = OrderedDict()
         update_test.update(od)
-        for i, dup in enumerate([
-                    od.copy(),
-                    copy.copy(od),
-                    copy.deepcopy(od),
-                    pickle.loads(pickle.dumps(od, 0)),
-                    pickle.loads(pickle.dumps(od, 1)),
-                    pickle.loads(pickle.dumps(od, 2)),
-                    pickle.loads(pickle.dumps(od, 3)),
-                    pickle.loads(pickle.dumps(od, -1)),
-                    eval(repr(od)),
-                    update_test,
-                    OrderedDict(od),
-                    ]):
-            self.assertTrue(dup is not od)
-            self.assertEqual(dup, od)
-            self.assertEqual(list(dup.items()), list(od.items()))
-            self.assertEqual(len(dup), len(od))
-            self.assertEqual(type(dup), type(od))
+        for label, dup in [
+                    ('od.copy()', od.copy()),
+                    ('copy.copy(od)', copy.copy(od)),
+                    ('copy.deepcopy(od)', copy.deepcopy(od)),
+                    ('pickle.loads(pickle.dumps(od, 0))',
+                        pickle.loads(pickle.dumps(od, 0))),
+                    ('pickle.loads(pickle.dumps(od, 1))',
+                        pickle.loads(pickle.dumps(od, 1))),
+                    ('pickle.loads(pickle.dumps(od, 2))',
+                        pickle.loads(pickle.dumps(od, 2))),
+                    ('pickle.loads(pickle.dumps(od, 3))',
+                        pickle.loads(pickle.dumps(od, 3))),
+                    ('pickle.loads(pickle.dumps(od, -1))',
+                        pickle.loads(pickle.dumps(od, -1))),
+                    ('eval(repr(od))', eval(repr(od))),
+                    ('update_test', update_test),
+                    ('OrderedDict(od)', OrderedDict(od)),
+                    ]:
+            with self.subTest(label=label):
+                msg = "\ncopy: %s\nod: %s" % (dup, od)
+                self.assertIsNot(dup, od, msg)
+                self.assertEqual(dup, od)
 
     def test_yaml_linkage(self):
         # Verify that __reduce__ is setup in a way that supports PyYAML's dump() feature.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 13:43:55 2013
From: python-checkins at python.org (raymond.hettinger)
Date: Sun,  7 Jul 2013 13:43:55 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Use_macros_for_marking_and?=
 =?utf-8?q?_checking_endpoints_in_the_doubly-linked_list_of?=
Message-ID: <3bp7Gz2KYWz7Ljn@mail.python.org>

http://hg.python.org/cpython/rev/ae9ee46bd471
changeset:   84487:ae9ee46bd471
user:        Raymond Hettinger <python at rcn.com>
date:        Sun Jul 07 01:43:42 2013 -1000
summary:
  Use macros for marking and checking endpoints in the doubly-linked list of blocks.

* Add comment explaining the endpoint checks
* Only do the checks in a debug build
* Simplify newblock() to only require a length argument
  and leave the link updates to the calling code.
* Also add comment for the freelisting logic.

files:
  Modules/_collectionsmodule.c |  128 ++++++++++++++--------
  1 files changed, 81 insertions(+), 47 deletions(-)


diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -18,16 +18,14 @@
 #define CENTER ((BLOCKLEN - 1) / 2)
 
 /* A `dequeobject` is composed of a doubly-linked list of `block` nodes.
- * This list is not circular (the leftmost block has leftlink==NULL,
- * and the rightmost block has rightlink==NULL).  A deque d's first
- * element is at d.leftblock[leftindex] and its last element is at
- * d.rightblock[rightindex]; note that, unlike as for Python slice
- * indices, these indices are inclusive on both ends.  By being inclusive
- * on both ends, algorithms for left and right operations become
- * symmetrical which simplifies the design.
+ * The list of blocks is never empty, so d.leftblock and d.rightblock
+ * are never equal to NULL.  The list is not circular.
  *
- * The list of blocks is never empty, so d.leftblock and d.rightblock
- * are never equal to NULL.
+ * A deque d's first element is at d.leftblock[leftindex]
+ * and its last element is at d.rightblock[rightindex].
+ * Unlike Python slice indices, these indices are inclusive
+ * on both ends.  This makes the algorithms for left and
+ * right operations more symmetrical and simplifies the design.
  *
  * The indices, d.leftindex and d.rightindex are always in the range
  *     0 <= index < BLOCKLEN.
@@ -52,12 +50,38 @@
     struct BLOCK *rightlink;
 } block;
 
+/* For debug builds, add error checking to track the endpoints
+ * in the chain of links.  The goal is to make sure that link
+ * assignments only take place at endpoints so that links already
+ * in use do not get overwritten.
+ *
+ * CHECK_END should happen before each assignment to a block's link field.
+ * MARK_END should happen whenever a link field becomes a new endpoint.
+ * This happens when new blocks are added or whenever an existing
+ * block is freed leaving another existing block as the new endpoint.
+ */
+
+#if Py_DEBUG
+#define MARK_END(link)  link = NULL;
+#define CHECK_END(link) assert(link == NULL);
+#define CHECK_NOT_END(link) assert(link != NULL);
+#else
+#define MARK_END(link)
+#define CHECK_END(link)
+#define CHECK_NOT_END(link)
+#endif
+
+/* A simple freelisting scheme is used to minimize calls to the memory
+   allocator.  It accomodates common use cases where new blocks are being
+   added at about the same rate as old blocks are being freed.
+ */
+
 #define MAXFREEBLOCKS 10
 static Py_ssize_t numfreeblocks = 0;
 static block *freeblocks[MAXFREEBLOCKS];
 
 static block *
-newblock(block *leftlink, block *rightlink, Py_ssize_t len) {
+newblock(Py_ssize_t len) {
     block *b;
     /* To prevent len from overflowing PY_SSIZE_T_MAX, we refuse to
      * allocate new blocks if the current len is nearing overflow. */
@@ -68,17 +92,14 @@
     }
     if (numfreeblocks) {
         numfreeblocks--;
-        b = freeblocks[numfreeblocks];
-    } else {
-        b = PyMem_Malloc(sizeof(block));
-        if (b == NULL) {
-            PyErr_NoMemory();
-            return NULL;
-        }
+        return freeblocks[numfreeblocks];
     }
-    b->leftlink = leftlink;
-    b->rightlink = rightlink;
-    return b;
+    b = PyMem_Malloc(sizeof(block));
+    if (b != NULL) {
+        return b;
+    }
+    PyErr_NoMemory();
+    return NULL;
 }
 
 static void
@@ -132,11 +153,13 @@
     if (deque == NULL)
         return NULL;
 
-    b = newblock(NULL, NULL, 0);
+    b = newblock(0);
     if (b == NULL) {
         Py_DECREF(deque);
         return NULL;
     }
+    MARK_END(b->leftlink);
+    MARK_END(b->rightlink);
 
     assert(BLOCKLEN >= 2);
     deque->leftblock = b;
@@ -177,7 +200,8 @@
             prevblock = deque->rightblock->leftlink;
             assert(deque->leftblock != deque->rightblock);
             freeblock(deque->rightblock);
-            prevblock->rightlink = NULL;
+            CHECK_NOT_END(prevblock);
+            MARK_END(prevblock->rightlink);
             deque->rightblock = prevblock;
             deque->rightindex = BLOCKLEN - 1;
         }
@@ -214,8 +238,8 @@
             assert(deque->leftblock != deque->rightblock);
             prevblock = deque->leftblock->rightlink;
             freeblock(deque->leftblock);
-            assert(prevblock != NULL);
-            prevblock->leftlink = NULL;
+            CHECK_NOT_END(prevblock);
+            MARK_END(prevblock->leftlink);
             deque->leftblock = prevblock;
             deque->leftindex = 0;
         }
@@ -230,12 +254,14 @@
 {
     deque->state++;
     if (deque->rightindex == BLOCKLEN-1) {
-        block *b = newblock(deque->rightblock, NULL, Py_SIZE(deque));
+        block *b = newblock(Py_SIZE(deque));
         if (b == NULL)
             return NULL;
-        assert(deque->rightblock->rightlink == NULL);
+        b->leftlink = deque->rightblock;
+        CHECK_END(deque->rightblock->rightlink);
         deque->rightblock->rightlink = b;
         deque->rightblock = b;
+        MARK_END(b->rightlink);
         deque->rightindex = -1;
     }
     Py_INCREF(item);
@@ -253,12 +279,14 @@
 {
     deque->state++;
     if (deque->leftindex == 0) {
-        block *b = newblock(NULL, deque->leftblock, Py_SIZE(deque));
+        block *b = newblock(Py_SIZE(deque));
         if (b == NULL)
             return NULL;
-        assert(deque->leftblock->leftlink == NULL);
+        b->rightlink = deque->leftblock;
+        CHECK_END(deque->leftblock->leftlink);
         deque->leftblock->leftlink = b;
         deque->leftblock = b;
+        MARK_END(b->leftlink);
         deque->leftindex = BLOCKLEN;
     }
     Py_INCREF(item);
@@ -314,16 +342,17 @@
     while ((item = PyIter_Next(it)) != NULL) {
         deque->state++;
         if (deque->rightindex == BLOCKLEN-1) {
-            block *b = newblock(deque->rightblock, NULL,
-                                Py_SIZE(deque));
+            block *b = newblock(Py_SIZE(deque));
             if (b == NULL) {
                 Py_DECREF(item);
                 Py_DECREF(it);
                 return NULL;
             }
-            assert(deque->rightblock->rightlink == NULL);
+            b->leftlink = deque->rightblock;
+            CHECK_END(deque->rightblock->rightlink);
             deque->rightblock->rightlink = b;
             deque->rightblock = b;
+            MARK_END(b->rightlink);
             deque->rightindex = -1;
         }
         Py_SIZE(deque)++;
@@ -366,16 +395,17 @@
     while ((item = PyIter_Next(it)) != NULL) {
         deque->state++;
         if (deque->leftindex == 0) {
-            block *b = newblock(NULL, deque->leftblock,
-                                Py_SIZE(deque));
+            block *b = newblock(Py_SIZE(deque));
             if (b == NULL) {
                 Py_DECREF(item);
                 Py_DECREF(it);
                 return NULL;
             }
-            assert(deque->leftblock->leftlink == NULL);
+            b->rightlink = deque->leftblock;
+            CHECK_END(deque->leftblock->leftlink);
             deque->leftblock->leftlink = b;
             deque->leftblock = b;
+            MARK_END(b->leftlink);
             deque->leftindex = BLOCKLEN;
         }
         Py_SIZE(deque)++;
@@ -430,14 +460,16 @@
     deque->state++;
     while (n > 0) {
         if (leftindex == 0) {
-            block *b = newblock(NULL, leftblock, len);
+            block *b = newblock(len);
             if (b == NULL) {
                 rv = -1;
                 goto done;
             }
-            assert(leftblock->leftlink == NULL);
+            b->rightlink = leftblock;
+            CHECK_END(leftblock->leftlink);
             leftblock->leftlink = b;
             leftblock = b;
+            MARK_END(b->leftlink);
             leftindex = BLOCKLEN;
         }
         assert(leftindex > 0);
@@ -462,24 +494,26 @@
 
         if (rightindex == -1) {
             block *prevblock = rightblock->leftlink;
-            assert(rightblock != NULL);
             assert(leftblock != rightblock);
             freeblock(rightblock);
-            prevblock->rightlink = NULL;
+            CHECK_NOT_END(prevblock);
+            MARK_END(prevblock->rightlink);
             rightblock = prevblock;
             rightindex = BLOCKLEN - 1;
         }
     }
     while (n < 0) {
         if (rightindex == BLOCKLEN - 1) {
-            block *b = newblock(rightblock, NULL, len);
+            block *b = newblock(len);
             if (b == NULL) {
                 rv = -1;
                 goto done;
             }
-            assert(rightblock->rightlink == NULL);
+            b->leftlink = rightblock;
+            CHECK_END(rightblock->rightlink);
             rightblock->rightlink = b;
             rightblock = b;
+            MARK_END(b->rightlink);
             rightindex = -1;
         }
         assert (rightindex < BLOCKLEN - 1);
@@ -506,8 +540,8 @@
             block *nextblock = leftblock->rightlink;
             assert(leftblock != rightblock);
             freeblock(leftblock);
-            assert(nextblock != NULL);
-            nextblock->leftlink = NULL;
+            CHECK_NOT_END(nextblock);
+            MARK_END(nextblock->leftlink);
             leftblock = nextblock;
             leftindex = 0;
         }
@@ -550,8 +584,8 @@
     for (i=0 ; i<n ; i++) {
         /* Validate that pointers haven't met in the middle */
         assert(leftblock != rightblock || leftindex < rightindex);
-        assert(leftblock != NULL);
-        assert(rightblock != NULL);
+        CHECK_NOT_END(leftblock);
+        CHECK_NOT_END(rightblock);
 
         /* Swap */
         tmp = leftblock->data[leftindex];
@@ -591,7 +625,7 @@
     int cmp;
 
     for (i=0 ; i<n ; i++) {
-        assert(b != NULL);
+        CHECK_NOT_END(b);
         item = b->data[index];
         cmp = PyObject_RichCompareBool(item, v, Py_EQ);
         if (cmp > 0)
@@ -1199,7 +1233,7 @@
     it->index++;
     it->counter--;
     if (it->index == BLOCKLEN && it->counter > 0) {
-        assert (it->b->rightlink != NULL);
+        CHECK_NOT_END(it->b->rightlink);
         it->b = it->b->rightlink;
         it->index = 0;
     }
@@ -1341,7 +1375,7 @@
     it->index--;
     it->counter--;
     if (it->index == -1 && it->counter > 0) {
-        assert (it->b->leftlink != NULL);
+        CHECK_NOT_END(it->b->leftlink);
         it->b = it->b->leftlink;
         it->index = BLOCKLEN - 1;
     }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 14:07:31 2013
From: python-checkins at python.org (raymond.hettinger)
Date: Sun,  7 Jul 2013 14:07:31 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Fix_=23ifdef?=
Message-ID: <3bp7pC5pTjzS1v@mail.python.org>

http://hg.python.org/cpython/rev/744dd749e25b
changeset:   84488:744dd749e25b
user:        Raymond Hettinger <python at rcn.com>
date:        Sun Jul 07 02:07:23 2013 -1000
summary:
  Fix #ifdef

files:
  Modules/_collectionsmodule.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -61,7 +61,7 @@
  * block is freed leaving another existing block as the new endpoint.
  */
 
-#if Py_DEBUG
+#ifdef Py_DEBUG
 #define MARK_END(link)  link = NULL;
 #define CHECK_END(link) assert(link == NULL);
 #define CHECK_NOT_END(link) assert(link != NULL);

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 15:01:52 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 15:01:52 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_446=3A_add_a_link_to_=22G?=
 =?utf-8?q?hosts_of_Unix_past=2C_part_2=3A_Conflated_designs=22_article?=
Message-ID: <3bp90w6KQ6zSCB@mail.python.org>

http://hg.python.org/peps/rev/a7faa4aa505e
changeset:   4983:a7faa4aa505e
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 15:00:53 2013 +0200
summary:
  PEP 446: add a link to "Ghosts of Unix past, part 2: Conflated designs" article (LWN)

files:
  pep-0446.txt |  3 +++
  1 files changed, 3 insertions(+), 0 deletions(-)


diff --git a/pep-0446.txt b/pep-0446.txt
--- a/pep-0446.txt
+++ b/pep-0446.txt
@@ -221,6 +221,9 @@
 * `Secure File Descriptor Handling
   <http://udrepper.livejournal.com/20407.html>`_ (Ulrich Drepper,
   2008)
+* `Ghosts of Unix past, part 2: Conflated designs
+  <http://lwn.net/Articles/412131/>`_ (Neil Brown, 2010) explains the
+  history of ``O_CLOEXEC`` and ``O_NONBLOCK`` flags
 
 
 Copyright

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Sun Jul  7 15:06:11 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 15:06:11 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_446=3A_add_new_os=2Eget/s?=
 =?utf-8?q?et=5Fblocking=28=29_functions?=
Message-ID: <3bp95v32tDzQNc@mail.python.org>

http://hg.python.org/peps/rev/ce61588d244c
changeset:   4984:ce61588d244c
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 15:05:58 2013 +0200
summary:
  PEP 446: add new os.get/set_blocking() functions

files:
  pep-0446.txt |  13 +++++++++++--
  1 files changed, 11 insertions(+), 2 deletions(-)


diff --git a/pep-0446.txt b/pep-0446.txt
--- a/pep-0446.txt
+++ b/pep-0446.txt
@@ -127,21 +127,30 @@
 setting close-on-exec and blocking flags at the creation of the file
 descriptor or socket, the flags are set using additional system calls.
 
+
 New Functions
 -------------
 
 Add new functions the get and set the close-on-exec flag of a file
-descriptor:
+descriptor, available on all platforms:
 
 * ``os.get_cloexec(fd:int) -> bool``
 * ``os.set_cloexec(fd:int, cloexec: bool)``
 
+Add new functions the get and set the blocking flag of a file
+descriptor, only available on UNIX:
+
+* ``os.get_blocking(fd:int) -> bool``
+* ``os.set_blocking(fd:int, blocking: bool)``
+
 
 Other Changes
 -------------
 
 The ``subprocess.Popen`` class must clear the close-on-exec flag of file
-descriptors of the ``pass_fds`` parameter.
+descriptors of the ``pass_fds`` parameter. The flag is cleared in the
+child process before executing the program, the change does not change
+the flag in the parent process.
 
 The close-on-exec flag must also be set on private file descriptors and
 sockets in the Python standard library. For example, on UNIX,

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Sun Jul  7 15:10:57 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 15:10:57 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_446=3A_typo?=
Message-ID: <3bp9CP6DWrz7Ljm@mail.python.org>

http://hg.python.org/peps/rev/e91b098137c5
changeset:   4985:e91b098137c5
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 15:10:39 2013 +0200
summary:
  PEP 446: typo

files:
  pep-0446.txt |  4 ++--
  1 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/pep-0446.txt b/pep-0446.txt
--- a/pep-0446.txt
+++ b/pep-0446.txt
@@ -174,8 +174,8 @@
 backward compatibility issue), and is much simpler.
 
 
-Add blocking parameter for file descriptors and Windows overlapped I/O
-----------------------------------------------------------------------
+Add blocking parameter for file descriptors and use Windows overlapped I/O
+--------------------------------------------------------------------------
 
 Windows supports non-blocking operations on files using an extension of
 the Windows API called "Overlapped I/O". Using this extension requires

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Sun Jul  7 16:26:03 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 16:26:03 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318203=3A_Fix_Py?=
 =?utf-8?q?=5FFinalize=28=29=3A_destroy_the_GIL_after_the_last_call_to?=
Message-ID: <3bpBt36gbRzRCK@mail.python.org>

http://hg.python.org/cpython/rev/213d6d7f5979
changeset:   84489:213d6d7f5979
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 15:50:49 2013 +0200
summary:
  Issue #18203: Fix Py_Finalize(): destroy the GIL after the last call to
PyMem_Malloc() or PyObject_Malloc().

For example, PyCFunction_Fini() calls PyObject_GC_Del() which calls
PyObject_FREE().

files:
  Python/pythonrun.c |  18 +++++++++---------
  1 files changed, 9 insertions(+), 9 deletions(-)


diff --git a/Python/pythonrun.c b/Python/pythonrun.c
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -606,11 +606,6 @@
 
     _PyExc_Fini();
 
-    /* Cleanup auto-thread-state */
-#ifdef WITH_THREAD
-    _PyGILState_Fini();
-#endif /* WITH_THREAD */
-
     /* Sundry finalizers */
     PyMethod_Fini();
     PyFrame_Fini();
@@ -629,10 +624,6 @@
     /* Cleanup Unicode implementation */
     _PyUnicode_Fini();
 
-    /* Delete current thread. After this, many C API calls become crashy. */
-    PyThreadState_Swap(NULL);
-    PyInterpreterState_Delete(interp);
-
     /* reset file system default encoding */
     if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) {
         free((char*)Py_FileSystemDefaultEncoding);
@@ -647,6 +638,15 @@
 
     PyGrammar_RemoveAccelerators(&_PyParser_Grammar);
 
+    /* Cleanup auto-thread-state */
+#ifdef WITH_THREAD
+    _PyGILState_Fini();
+#endif /* WITH_THREAD */
+
+    /* Delete current thread. After this, many C API calls become crashy. */
+    PyThreadState_Swap(NULL);
+    PyInterpreterState_Delete(interp);
+
 #ifdef Py_TRACE_REFS
     /* Display addresses (& refcnts) of all objects still alive.
      * An address can be used to find the repr of the object, printed

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 16:26:05 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 16:26:05 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318203=3A_Replace_?=
 =?utf-8?q?malloc=28=29_with_PyMem=5FRawMalloc=28=29_at_Python_initializat?=
 =?utf-8?q?ion?=
Message-ID: <3bpBt54R67zT0x@mail.python.org>

http://hg.python.org/cpython/rev/18bb92b0c458
changeset:   84490:18bb92b0c458
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 16:25:15 2013 +0200
summary:
  Issue #18203: Replace malloc() with PyMem_RawMalloc() at Python initialization

* Replace malloc() with PyMem_RawMalloc()
* Replace PyMem_Malloc() with PyMem_RawMalloc() where the GIL is not held.
* _Py_char2wchar() now returns a buffer allocated by PyMem_RawMalloc(), instead
  of PyMem_Malloc()

files:
  Modules/getpath.c       |  20 ++++++--------
  Modules/main.c          |  14 +++++-----
  Modules/python.c        |  23 ++++++++++------
  Objects/unicodeobject.c |   6 ++--
  PC/getpathp.c           |  39 +++++++++++++---------------
  Python/fileutils.c      |  22 ++++++++--------
  Python/pystate.c        |  20 +++++++-------
  Python/thread.c         |   8 ++--
  8 files changed, 76 insertions(+), 76 deletions(-)


diff --git a/Modules/getpath.c b/Modules/getpath.c
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -343,7 +343,7 @@
         if (vpath != NULL) {
             wcscpy(prefix, argv0_path);
             joinpath(prefix, vpath);
-            PyMem_Free(vpath);
+            PyMem_RawFree(vpath);
             joinpath(prefix, L"Lib");
             joinpath(prefix, LANDMARK);
             if (ismodule(prefix))
@@ -554,8 +554,7 @@
     }
     else
         progpath[0] = '\0';
-    if (path_buffer != NULL)
-        PyMem_Free(path_buffer);
+    PyMem_RawFree(path_buffer);
     if (progpath[0] != SEP && progpath[0] != '\0')
         absolutize(progpath);
     wcsncpy(argv0_path, progpath, MAXPATHLEN);
@@ -597,7 +596,7 @@
             /* Use the location of the library as the progpath */
             wcsncpy(argv0_path, wbuf, MAXPATHLEN);
         }
-        PyMem_Free(wbuf);
+        PyMem_RawFree(wbuf);
     }
 #endif
 
@@ -808,11 +807,10 @@
     else
         wcsncpy(exec_prefix, _exec_prefix, MAXPATHLEN);
 
-    PyMem_Free(_pythonpath);
-    PyMem_Free(_prefix);
-    PyMem_Free(_exec_prefix);
-    if (rtpypath != NULL)
-        PyMem_Free(rtpypath);
+    PyMem_RawFree(_pythonpath);
+    PyMem_RawFree(_prefix);
+    PyMem_RawFree(_exec_prefix);
+    PyMem_RawFree(rtpypath);
 }
 
 
@@ -822,7 +820,7 @@
 {
     if (module_search_path != NULL) {
         if (module_search_path_malloced)
-            PyMem_Free(module_search_path);
+            PyMem_RawFree(module_search_path);
         module_search_path = NULL;
         module_search_path_malloced = 0;
     }
@@ -831,7 +829,7 @@
         wchar_t *prog = Py_GetProgramName();
         wcsncpy(progpath, prog, MAXPATHLEN);
         exec_prefix[0] = prefix[0] = L'\0';
-        module_search_path = PyMem_Malloc((wcslen(path) + 1) * sizeof(wchar_t));
+        module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t));
         module_search_path_malloced = 1;
         if (module_search_path != NULL)
             wcscpy(module_search_path, path);
diff --git a/Modules/main.c b/Modules/main.c
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -391,7 +391,7 @@
                command to interpret. */
 
             len = wcslen(_PyOS_optarg) + 1 + 1;
-            command = (wchar_t *)malloc(sizeof(wchar_t) * len);
+            command = (wchar_t *)PyMem_RawMalloc(sizeof(wchar_t) * len);
             if (command == NULL)
                 Py_FatalError(
                    "not enough memory to copy -c argument");
@@ -520,7 +520,7 @@
         *wp != L'\0') {
         wchar_t *buf, *warning;
 
-        buf = (wchar_t *)malloc((wcslen(wp) + 1) * sizeof(wchar_t));
+        buf = (wchar_t *)PyMem_RawMalloc((wcslen(wp) + 1) * sizeof(wchar_t));
         if (buf == NULL)
             Py_FatalError(
                "not enough memory to copy PYTHONWARNINGS");
@@ -530,7 +530,7 @@
              warning = wcstok(NULL, L",")) {
             PySys_AddWarnOption(warning);
         }
-        free(buf);
+        PyMem_RawFree(buf);
     }
 #else
     if ((p = Py_GETENV("PYTHONWARNINGS")) && *p != '\0') {
@@ -539,7 +539,7 @@
 
         /* settle for strtok here as there's no one standard
            C89 wcstok */
-        buf = (char *)malloc(strlen(p) + 1);
+        buf = (char *)PyMem_RawMalloc(strlen(p) + 1);
         if (buf == NULL)
             Py_FatalError(
                "not enough memory to copy PYTHONWARNINGS");
@@ -563,7 +563,7 @@
         }
         setlocale(LC_ALL, oldloc);
         free(oldloc);
-        free(buf);
+        PyMem_RawFree(buf);
     }
 #endif
 
@@ -633,7 +633,7 @@
         wchar_t* buffer;
         size_t len = strlen(p) + 1;
 
-        buffer = malloc(len * sizeof(wchar_t));
+        buffer = PyMem_RawMalloc(len * sizeof(wchar_t));
         if (buffer == NULL) {
             Py_FatalError(
                "not enough memory to copy PYTHONEXECUTABLE");
@@ -707,7 +707,7 @@
 
     if (command) {
         sts = run_command(command, &cf);
-        free(command);
+        PyMem_RawFree(command);
     } else if (module) {
         sts = (RunModule(module, 1) != 0);
     }
diff --git a/Modules/python.c b/Modules/python.c
--- a/Modules/python.c
+++ b/Modules/python.c
@@ -18,11 +18,19 @@
 int
 main(int argc, char **argv)
 {
-    wchar_t **argv_copy = (wchar_t **)PyMem_Malloc(sizeof(wchar_t*)*(argc+1));
+    wchar_t **argv_copy;
     /* We need a second copies, as Python might modify the first one. */
-    wchar_t **argv_copy2 = (wchar_t **)PyMem_Malloc(sizeof(wchar_t*)*(argc+1));
+    wchar_t **argv_copy2;
     int i, res;
     char *oldloc;
+
+    argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
+    argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
+    if (!argv_copy || !argv_copy2) {
+        fprintf(stderr, "out of memory\n");
+        return 1;
+    }
+
     /* 754 requires that FP exceptions run in "no stop" mode by default,
      * and until C vendors implement C99's ways to control FP exceptions,
      * Python requires non-stop mode.  Alas, some platforms enable FP
@@ -34,10 +42,7 @@
     m = fpgetmask();
     fpsetmask(m & ~FP_X_OFL);
 #endif
-    if (!argv_copy || !argv_copy2) {
-        fprintf(stderr, "out of memory\n");
-        return 1;
-    }
+
     oldloc = strdup(setlocale(LC_ALL, NULL));
     setlocale(LC_ALL, "");
     for (i = 0; i < argc; i++) {
@@ -57,10 +62,10 @@
     free(oldloc);
     res = Py_Main(argc, argv_copy);
     for (i = 0; i < argc; i++) {
-        PyMem_Free(argv_copy2[i]);
+        PyMem_RawFree(argv_copy2[i]);
     }
-    PyMem_Free(argv_copy);
-    PyMem_Free(argv_copy2);
+    PyMem_RawFree(argv_copy);
+    PyMem_RawFree(argv_copy2);
     return res;
 }
 #endif
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -3316,7 +3316,7 @@
         wstr = _Py_char2wchar(errmsg, &errlen);
         if (wstr != NULL) {
             reason = PyUnicode_FromWideChar(wstr, errlen);
-            PyMem_Free(wstr);
+            PyMem_RawFree(wstr);
         } else
             errmsg = NULL;
     }
@@ -3535,7 +3535,7 @@
         }
 
         unicode = PyUnicode_FromWideChar(wstr, wlen);
-        PyMem_Free(wstr);
+        PyMem_RawFree(wstr);
     }
     else {
         /* strict mode */
@@ -3583,7 +3583,7 @@
         wstr = _Py_char2wchar(errmsg, &errlen);
         if (wstr != NULL) {
             reason = PyUnicode_FromWideChar(wstr, errlen);
-            PyMem_Free(wstr);
+            PyMem_RawFree(wstr);
         } else
             errmsg = NULL;
     }
diff --git a/PC/getpathp.c b/PC/getpathp.c
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -245,9 +245,9 @@
     /* Tried to use sysget("winver") but here is too early :-( */
     versionLen = strlen(PyWin_DLLVersionString);
     /* Space for all the chars, plus one \0 */
-    keyBuf = keyBufPtr = malloc(sizeof(keyPrefix) +
-                                sizeof(WCHAR)*(versionLen-1) +
-                                sizeof(keySuffix));
+    keyBuf = keyBufPtr = PyMem_RawMalloc(sizeof(keyPrefix) +
+                                         sizeof(WCHAR)*(versionLen-1) +
+                                         sizeof(keySuffix));
     if (keyBuf==NULL) goto done;
 
     memcpy(keyBufPtr, keyPrefix, sizeof(keyPrefix)-sizeof(WCHAR));
@@ -271,7 +271,7 @@
     /* Allocate a temp array of char buffers, so we only need to loop
        reading the registry once
     */
-    ppPaths = malloc( sizeof(WCHAR *) * numKeys );
+    ppPaths = PyMem_RawMalloc( sizeof(WCHAR *) * numKeys );
     if (ppPaths==NULL) goto done;
     memset(ppPaths, 0, sizeof(WCHAR *) * numKeys);
     /* Loop over all subkeys, allocating a temp sub-buffer. */
@@ -293,7 +293,7 @@
         /* Find the value of the buffer size, malloc, then read it */
         RegQueryValueExW(subKey, NULL, 0, NULL, NULL, &reqdSize);
         if (reqdSize) {
-            ppPaths[index] = malloc(reqdSize);
+            ppPaths[index] = PyMem_RawMalloc(reqdSize);
             if (ppPaths[index]) {
                 RegQueryValueExW(subKey, NULL, 0, NULL,
                                 (LPBYTE)ppPaths[index],
@@ -308,7 +308,7 @@
     if (dataSize == 0) goto done;
 
     /* original datasize from RegQueryInfo doesn't include the \0 */
-    dataBuf = malloc((dataSize+1) * sizeof(WCHAR));
+    dataBuf = PyMem_RawMalloc((dataSize+1) * sizeof(WCHAR));
     if (dataBuf) {
         WCHAR *szCur = dataBuf;
         DWORD reqdSize = dataSize;
@@ -346,14 +346,13 @@
 done:
     /* Loop freeing my temp buffers */
     if (ppPaths) {
-        for(index=0;index<numKeys;index++)
-            if (ppPaths[index]) free(ppPaths[index]);
-        free(ppPaths);
+        for(index=0; index<numKeys; index++)
+            PyMem_RawFree(ppPaths[index]);
+        PyMem_RawFree(ppPaths);
     }
     if (newKey)
         RegCloseKey(newKey);
-    if (keyBuf)
-        free(keyBuf);
+    PyMem_RawFree(keyBuf);
     return retval;
 }
 #endif /* Py_ENABLE_SHARED */
@@ -616,7 +615,7 @@
     if (envpath != NULL)
         bufsz += wcslen(envpath) + 1;
 
-    module_search_path = buf = malloc(bufsz*sizeof(wchar_t));
+    module_search_path = buf = PyMem_RawMalloc(bufsz*sizeof(wchar_t));
     if (buf == NULL) {
         /* We can't exit, so print a warning and limp along */
         fprintf(stderr, "Can't malloc dynamic PYTHONPATH.\n");
@@ -629,10 +628,8 @@
             module_search_path = PYTHONPATH;
         }
 #ifdef MS_WINDOWS
-        if (machinepath)
-            free(machinepath);
-        if (userpath)
-            free(userpath);
+        PyMem_RawFree(machinepath);
+        PyMem_RawFree(userpath);
 #endif /* MS_WINDOWS */
         return;
     }
@@ -652,13 +649,13 @@
         wcscpy(buf, userpath);
         buf = wcschr(buf, L'\0');
         *buf++ = DELIM;
-        free(userpath);
+        PyMem_RawFree(userpath);
     }
     if (machinepath) {
         wcscpy(buf, machinepath);
         buf = wcschr(buf, L'\0');
         *buf++ = DELIM;
-        free(machinepath);
+        PyMem_RawFree(machinepath);
     }
     if (pythonhome == NULL) {
         if (!skipdefault) {
@@ -745,7 +742,7 @@
 Py_SetPath(const wchar_t *path)
 {
     if (module_search_path != NULL) {
-        free(module_search_path);
+        PyMem_RawFree(module_search_path);
         module_search_path = NULL;
     }
     if (path != NULL) {
@@ -753,10 +750,10 @@
         wchar_t *prog = Py_GetProgramName();
         wcsncpy(progpath, prog, MAXPATHLEN);
         prefix[0] = L'\0';
-        module_search_path = malloc((wcslen(path) + 1) * sizeof(wchar_t));
+        module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t));
         if (module_search_path != NULL)
             wcscpy(module_search_path, path);
-	}
+    }
 }
 
 wchar_t *
diff --git a/Python/fileutils.c b/Python/fileutils.c
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -229,7 +229,7 @@
    Use _Py_wchar2char() to encode the character string back to a byte string.
 
    Return a pointer to a newly allocated wide character string (use
-   PyMem_Free() to free the memory) and write the number of written wide
+   PyMem_RawFree() to free the memory) and write the number of written wide
    characters excluding the null character into *size if size is not NULL, or
    NULL on error (decoding or memory allocation error). If size is not NULL,
    *size is set to (size_t)-1 on memory error and (size_t)-2 on decoding
@@ -283,7 +283,7 @@
     argsize = mbstowcs(NULL, arg, 0);
 #endif
     if (argsize != (size_t)-1) {
-        res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t));
+        res = (wchar_t *)PyMem_RawMalloc((argsize+1)*sizeof(wchar_t));
         if (!res)
             goto oom;
         count = mbstowcs(res, arg, argsize+1);
@@ -300,7 +300,7 @@
                 return res;
             }
         }
-        PyMem_Free(res);
+        PyMem_RawFree(res);
     }
     /* Conversion failed. Fall back to escaping with surrogateescape. */
 #ifdef HAVE_MBRTOWC
@@ -309,7 +309,7 @@
     /* Overallocate; as multi-byte characters are in the argument, the
        actual output could use less memory. */
     argsize = strlen(arg) + 1;
-    res = (wchar_t*)PyMem_Malloc(argsize*sizeof(wchar_t));
+    res = (wchar_t*)PyMem_RawMalloc(argsize*sizeof(wchar_t));
     if (!res)
         goto oom;
     in = (unsigned char*)arg;
@@ -325,7 +325,7 @@
                since we provide everything that we have -
                unless there is a bug in the C library, or I
                misunderstood how mbrtowc works. */
-            PyMem_Free(res);
+            PyMem_RawFree(res);
             if (size != NULL)
                 *size = (size_t)-2;
             return NULL;
@@ -648,12 +648,12 @@
         return -1;
     }
     if (bufsiz <= r1) {
-        PyMem_Free(wbuf);
+        PyMem_RawFree(wbuf);
         errno = EINVAL;
         return -1;
     }
     wcsncpy(buf, wbuf, bufsiz);
-    PyMem_Free(wbuf);
+    PyMem_RawFree(wbuf);
     return (int)r1;
 }
 #endif
@@ -689,12 +689,12 @@
         return NULL;
     }
     if (resolved_path_size <= r) {
-        PyMem_Free(wresolved_path);
+        PyMem_RawFree(wresolved_path);
         errno = EINVAL;
         return NULL;
     }
     wcsncpy(resolved_path, wresolved_path, resolved_path_size);
-    PyMem_Free(wresolved_path);
+    PyMem_RawFree(wresolved_path);
     return resolved_path;
 }
 #endif
@@ -720,11 +720,11 @@
     if (wname == NULL)
         return NULL;
     if (size <= len) {
-        PyMem_Free(wname);
+        PyMem_RawFree(wname);
         return NULL;
     }
     wcsncpy(buf, wname, size);
-    PyMem_Free(wname);
+    PyMem_RawFree(wname);
     return buf;
 #endif
 }
diff --git a/Python/pystate.c b/Python/pystate.c
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -6,11 +6,11 @@
 /* --------------------------------------------------------------------------
 CAUTION
 
-Always use malloc() and free() directly in this file.  A number of these
-functions are advertised as safe to call when the GIL isn't held, and in
-a debug build Python redirects (e.g.) PyMem_NEW (etc) to Python's debugging
-obmalloc functions.  Those aren't thread-safe (they rely on the GIL to avoid
-the expense of doing their own locking).
+Always use PyMem_RawMalloc() and PyMem_RawFree() directly in this file.  A
+number of these functions are advertised as safe to call when the GIL isn't
+held, and in a debug build Python redirects (e.g.) PyMem_NEW (etc) to Python's
+debugging obmalloc functions.  Those aren't thread-safe (they rely on the GIL
+to avoid the expense of doing their own locking).
 -------------------------------------------------------------------------- */
 
 #ifdef HAVE_DLOPEN
@@ -60,7 +60,7 @@
 PyInterpreterState_New(void)
 {
     PyInterpreterState *interp = (PyInterpreterState *)
-                                 malloc(sizeof(PyInterpreterState));
+                                 PyMem_RawMalloc(sizeof(PyInterpreterState));
 
     if (interp != NULL) {
         HEAD_INIT();
@@ -148,7 +148,7 @@
         Py_FatalError("PyInterpreterState_Delete: remaining threads");
     *p = interp->next;
     HEAD_UNLOCK();
-    free(interp);
+    PyMem_RawFree(interp);
 #ifdef WITH_THREAD
     if (interp_head == NULL && head_mutex != NULL) {
         PyThread_free_lock(head_mutex);
@@ -168,7 +168,7 @@
 static PyThreadState *
 new_threadstate(PyInterpreterState *interp, int init)
 {
-    PyThreadState *tstate = (PyThreadState *)malloc(sizeof(PyThreadState));
+    PyThreadState *tstate = (PyThreadState *)PyMem_RawMalloc(sizeof(PyThreadState));
 
     if (_PyThreadState_GetFrame == NULL)
         _PyThreadState_GetFrame = threadstate_getframe;
@@ -365,7 +365,7 @@
     if (tstate->next)
         tstate->next->prev = tstate->prev;
     HEAD_UNLOCK();
-    free(tstate);
+    PyMem_RawFree(tstate);
 }
 
 
@@ -432,7 +432,7 @@
     for (p = garbage; p; p = next) {
         next = p->next;
         PyThreadState_Clear(p);
-        free(p);
+        PyMem_RawFree(p);
     }
 }
 
diff --git a/Python/thread.c b/Python/thread.c
--- a/Python/thread.c
+++ b/Python/thread.c
@@ -231,7 +231,7 @@
         assert(p == NULL);
         goto Done;
     }
-    p = (struct key *)malloc(sizeof(struct key));
+    p = (struct key *)PyMem_RawMalloc(sizeof(struct key));
     if (p != NULL) {
         p->id = id;
         p->key = key;
@@ -270,7 +270,7 @@
     while ((p = *q) != NULL) {
         if (p->key == key) {
             *q = p->next;
-            free((void *)p);
+            PyMem_RawFree((void *)p);
             /* NB This does *not* free p->value! */
         }
         else
@@ -324,7 +324,7 @@
     while ((p = *q) != NULL) {
         if (p->key == key && p->id == id) {
             *q = p->next;
-            free((void *)p);
+            PyMem_RawFree((void *)p);
             /* NB This does *not* free p->value! */
             break;
         }
@@ -357,7 +357,7 @@
     while ((p = *q) != NULL) {
         if (p->id != id) {
             *q = p->next;
-            free((void *)p);
+            PyMem_RawFree((void *)p);
             /* NB This does *not* free p->value! */
         }
         else

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 16:26:07 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 16:26:07 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318203=3A_Replace_?=
 =?utf-8?q?malloc=28=29_with_PyMem=5FMalloc=28=29_in_Python_modules?=
Message-ID: <3bpBt716S4zSnC@mail.python.org>

http://hg.python.org/cpython/rev/41ef797e6639
changeset:   84491:41ef797e6639
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 16:21:41 2013 +0200
summary:
  Issue #18203: Replace malloc() with PyMem_Malloc() in Python modules

Replace malloc() with PyMem_Malloc() when the GIL is held, or with
PyMem_RawMalloc() otherwise.

files:
  Modules/_curses_panel.c |   6 ++--
  Modules/_cursesmodule.c |   4 +-
  Modules/_lsprof.c       |  18 +++++++-------
  Modules/_ssl.c          |   4 +-
  Modules/audioop.c       |  10 +++----
  Modules/posixmodule.c   |  35 ++++++++++++++--------------
  Modules/pyexpat.c       |  18 +++++++-------
  Modules/readline.c      |   4 +-
  Modules/zlibmodule.c    |   4 +-
  PC/winreg.c             |   4 +-
  10 files changed, 52 insertions(+), 55 deletions(-)


diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c
--- a/Modules/_curses_panel.c
+++ b/Modules/_curses_panel.c
@@ -117,7 +117,7 @@
 {
     list_of_panels *new;
 
-    if ((new = (list_of_panels *)malloc(sizeof(list_of_panels))) == NULL) {
+    if ((new = (list_of_panels *)PyMem_Malloc(sizeof(list_of_panels))) == NULL) {
         PyErr_NoMemory();
         return -1;
     }
@@ -136,7 +136,7 @@
     temp = lop;
     if (temp->po == po) {
         lop = temp->next;
-        free(temp);
+        PyMem_Free(temp);
         return;
     }
     while (temp->next == NULL || temp->next->po != po) {
@@ -148,7 +148,7 @@
         temp = temp->next;
     }
     n = temp->next->next;
-    free(temp->next);
+    PyMem_Free(temp->next);
     temp->next = n;
     return;
 }
diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c
--- a/Modules/_cursesmodule.c
+++ b/Modules/_cursesmodule.c
@@ -3406,7 +3406,7 @@
                 continue;
             if (strncmp(key_n,"KEY_F(",6)==0) {
                 char *p1, *p2;
-                key_n2 = malloc(strlen(key_n)+1);
+                key_n2 = PyMem_Malloc(strlen(key_n)+1);
                 if (!key_n2) {
                     PyErr_NoMemory();
                     break;
@@ -3425,7 +3425,7 @@
                 key_n2 = key_n;
             SetDictInt(key_n2,key);
             if (key_n2 != key_n)
-                free(key_n2);
+                PyMem_Free(key_n2);
         }
 #endif
         SetDictInt("KEY_MIN", KEY_MIN);
diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c
--- a/Modules/_lsprof.c
+++ b/Modules/_lsprof.c
@@ -223,7 +223,7 @@
 newProfilerEntry(ProfilerObject *pObj, void *key, PyObject *userObj)
 {
     ProfilerEntry *self;
-    self = (ProfilerEntry*) malloc(sizeof(ProfilerEntry));
+    self = (ProfilerEntry*) PyMem_Malloc(sizeof(ProfilerEntry));
     if (self == NULL) {
         pObj->flags |= POF_NOMEMORY;
         return NULL;
@@ -231,7 +231,7 @@
     userObj = normalizeUserObj(userObj);
     if (userObj == NULL) {
         PyErr_Clear();
-        free(self);
+        PyMem_Free(self);
         pObj->flags |= POF_NOMEMORY;
         return NULL;
     }
@@ -264,7 +264,7 @@
 newSubEntry(ProfilerObject *pObj,  ProfilerEntry *caller, ProfilerEntry* entry)
 {
     ProfilerSubEntry *self;
-    self = (ProfilerSubEntry*) malloc(sizeof(ProfilerSubEntry));
+    self = (ProfilerSubEntry*) PyMem_Malloc(sizeof(ProfilerSubEntry));
     if (self == NULL) {
         pObj->flags |= POF_NOMEMORY;
         return NULL;
@@ -282,7 +282,7 @@
 static int freeSubEntry(rotating_node_t *header, void *arg)
 {
     ProfilerSubEntry *subentry = (ProfilerSubEntry*) header;
-    free(subentry);
+    PyMem_Free(subentry);
     return 0;
 }
 
@@ -291,7 +291,7 @@
     ProfilerEntry *entry = (ProfilerEntry*) header;
     RotatingTree_Enum(entry->calls, freeSubEntry, NULL);
     Py_DECREF(entry->userObj);
-    free(entry);
+    PyMem_Free(entry);
     return 0;
 }
 
@@ -301,13 +301,13 @@
     pObj->profilerEntries = EMPTY_ROTATING_TREE;
     /* release the memory hold by the ProfilerContexts */
     if (pObj->currentProfilerContext) {
-        free(pObj->currentProfilerContext);
+        PyMem_Free(pObj->currentProfilerContext);
         pObj->currentProfilerContext = NULL;
     }
     while (pObj->freelistProfilerContext) {
         ProfilerContext *c = pObj->freelistProfilerContext;
         pObj->freelistProfilerContext = c->previous;
-        free(c);
+        PyMem_Free(c);
     }
     pObj->freelistProfilerContext = NULL;
 }
@@ -393,7 +393,7 @@
     else {
         /* free list exhausted, allocate a new one */
         pContext = (ProfilerContext*)
-            malloc(sizeof(ProfilerContext));
+            PyMem_Malloc(sizeof(ProfilerContext));
         if (pContext == NULL) {
             pObj->flags |= POF_NOMEMORY;
             goto restorePyerr;
@@ -712,7 +712,7 @@
         else
             pObj->currentProfilerContext = pContext->previous;
         if (pContext)
-            free(pContext);
+            PyMem_Free(pContext);
     }
 
 }
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -3118,7 +3118,7 @@
     if (_ssl_locks == NULL) {
         _ssl_locks_count = CRYPTO_num_locks();
         _ssl_locks = (PyThread_type_lock *)
-            malloc(sizeof(PyThread_type_lock) * _ssl_locks_count);
+            PyMem_Malloc(sizeof(PyThread_type_lock) * _ssl_locks_count);
         if (_ssl_locks == NULL)
             return 0;
         memset(_ssl_locks, 0,
@@ -3130,7 +3130,7 @@
                 for (j = 0;  j < i;  j++) {
                     PyThread_free_lock(_ssl_locks[j]);
                 }
-                free(_ssl_locks);
+                PyMem_Free(_ssl_locks);
                 return 0;
             }
         }
diff --git a/Modules/audioop.c b/Modules/audioop.c
--- a/Modules/audioop.c
+++ b/Modules/audioop.c
@@ -1137,8 +1137,8 @@
                         "not enough memory for output buffer");
         return 0;
     }
-    prev_i = (int *) malloc(nchannels * sizeof(int));
-    cur_i = (int *) malloc(nchannels * sizeof(int));
+    prev_i = (int *) PyMem_Malloc(nchannels * sizeof(int));
+    cur_i = (int *) PyMem_Malloc(nchannels * sizeof(int));
     if (prev_i == NULL || cur_i == NULL) {
         (void) PyErr_NoMemory();
         goto exit;
@@ -1257,10 +1257,8 @@
         }
     }
   exit:
-    if (prev_i != NULL)
-        free(prev_i);
-    if (cur_i != NULL)
-        free(cur_i);
+    PyMem_Free(prev_i);
+    PyMem_Free(cur_i);
     return rv;
 }
 
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -1298,14 +1298,14 @@
     if (!result)
         return FALSE;
     if (result > MAX_PATH+1) {
-        new_path = malloc(result * sizeof(wchar_t));
+        new_path = PyMem_RawMalloc(result * sizeof(wchar_t));
         if (!new_path) {
             SetLastError(ERROR_OUTOFMEMORY);
             return FALSE;
         }
         result = GetCurrentDirectoryW(result, new_path);
         if (!result) {
-            free(new_path);
+            PyMem_RawFree(new_path);
             return FALSE;
         }
     }
@@ -1316,7 +1316,7 @@
     env[1] = new_path[0];
     result = SetEnvironmentVariableW(env, new_path);
     if (new_path != _new_path)
-        free(new_path);
+        PyMem_RawFree(new_path);
     return result;
 }
 #endif
@@ -1497,7 +1497,7 @@
     if(!buf_size)
         return FALSE;
 
-    buf = (wchar_t *)malloc((buf_size+1)*sizeof(wchar_t));
+    buf = (wchar_t *)PyMem_Malloc((buf_size+1)*sizeof(wchar_t));
     if (!buf) {
         SetLastError(ERROR_OUTOFMEMORY);
         return FALSE;
@@ -1507,12 +1507,12 @@
                        buf, buf_size, VOLUME_NAME_DOS);
 
     if(!result_length) {
-        free(buf);
+        PyMem_Free(buf);
         return FALSE;
     }
 
     if(!CloseHandle(hdl)) {
-        free(buf);
+        PyMem_Free(buf);
         return FALSE;
     }
 
@@ -1603,7 +1603,7 @@
                     return -1;
 
                 code = win32_xstat_impl_w(target_path, result, FALSE);
-                free(target_path);
+                PyMem_Free(target_path);
                 return code;
             }
         } else
@@ -1699,7 +1699,7 @@
                     return -1;
 
                 code = win32_xstat_impl_w(target_path, result, FALSE);
-                free(target_path);
+                PyMem_Free(target_path);
                 return code;
             }
         } else
@@ -3089,7 +3089,7 @@
            terminating \0. If the buffer is too small, len includes
            the space needed for the terminator. */
         if (len >= sizeof wbuf/ sizeof wbuf[0]) {
-            wbuf2 = malloc(len * sizeof(wchar_t));
+            wbuf2 = PyMem_RawMalloc(len * sizeof(wchar_t));
             if (wbuf2)
                 len = GetCurrentDirectoryW(len, wbuf2);
         }
@@ -3100,12 +3100,12 @@
         }
         if (!len) {
             if (wbuf2 != wbuf)
-                free(wbuf2);
+                PyMem_RawFree(wbuf2);
             return PyErr_SetFromWindowsErr(0);
         }
         resobj = PyUnicode_FromWideChar(wbuf2, len);
         if (wbuf2 != wbuf)
-            free(wbuf2);
+            PyMem_RawFree(wbuf2);
         return resobj;
     }
 
@@ -3289,7 +3289,7 @@
             len = wcslen(path->wide);
         }
         /* The +5 is so we can append "\\*.*\0" */
-        wnamebuf = malloc((len + 5) * sizeof(wchar_t));
+        wnamebuf = PyMem_Malloc((len + 5) * sizeof(wchar_t));
         if (!wnamebuf) {
             PyErr_NoMemory();
             goto exit;
@@ -3410,8 +3410,7 @@
             }
         }
     }
-    if (wnamebuf)
-        free(wnamebuf);
+    PyMem_Free(wnamebuf);
 
     return list;
 }  /* end of _listdir_windows_no_opendir */
@@ -3575,7 +3574,7 @@
                                   Py_ARRAY_LENGTH(woutbuf),
                                   woutbuf, &wtemp);
         if (result > Py_ARRAY_LENGTH(woutbuf)) {
-            woutbufp = malloc(result * sizeof(wchar_t));
+            woutbufp = PyMem_Malloc(result * sizeof(wchar_t));
             if (!woutbufp)
                 return PyErr_NoMemory();
             result = GetFullPathNameW(wpath, result, woutbufp, &wtemp);
@@ -3585,7 +3584,7 @@
         else
             v = win32_error_object("GetFullPathNameW", po);
         if (woutbufp != woutbuf)
-            free(woutbufp);
+            PyMem_Free(woutbufp);
         return v;
     }
     /* Drop the argument parsing error as narrow strings
@@ -3655,7 +3654,7 @@
     if(!buf_size)
         return win32_error_object("GetFinalPathNameByHandle", po);
 
-    target_path = (wchar_t *)malloc((buf_size+1)*sizeof(wchar_t));
+    target_path = (wchar_t *)PyMem_Malloc((buf_size+1)*sizeof(wchar_t));
     if(!target_path)
         return PyErr_NoMemory();
 
@@ -3669,7 +3668,7 @@
 
     target_path[result_length] = 0;
     result = PyUnicode_FromWideChar(target_path, result_length);
-    free(target_path);
+    PyMem_Free(target_path);
     return result;
 
 } /* end of posix__getfinalpathname */
diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c
--- a/Modules/pyexpat.c
+++ b/Modules/pyexpat.c
@@ -997,7 +997,7 @@
     PyObject_GC_Track(new_parser);
 
     if (self->buffer != NULL) {
-        new_parser->buffer = malloc(new_parser->buffer_size);
+        new_parser->buffer = PyMem_Malloc(new_parser->buffer_size);
         if (new_parser->buffer == NULL) {
             Py_DECREF(new_parser);
             return PyErr_NoMemory();
@@ -1014,7 +1014,7 @@
     for (i = 0; handler_info[i].name != NULL; i++)
         /* do nothing */;
 
-    new_parser->handlers = malloc(sizeof(PyObject *) * i);
+    new_parser->handlers = PyMem_Malloc(sizeof(PyObject *) * i);
     if (!new_parser->handlers) {
         Py_DECREF(new_parser);
         return PyErr_NoMemory();
@@ -1206,7 +1206,7 @@
     for (i = 0; handler_info[i].name != NULL; i++)
         /* do nothing */;
 
-    self->handlers = malloc(sizeof(PyObject *) * i);
+    self->handlers = PyMem_Malloc(sizeof(PyObject *) * i);
     if (!self->handlers) {
         Py_DECREF(self);
         return PyErr_NoMemory();
@@ -1233,11 +1233,11 @@
             self->handlers[i] = NULL;
             Py_XDECREF(temp);
         }
-        free(self->handlers);
+        PyMem_Free(self->handlers);
         self->handlers = NULL;
     }
     if (self->buffer != NULL) {
-        free(self->buffer);
+        PyMem_Free(self->buffer);
         self->buffer = NULL;
     }
     Py_XDECREF(self->intern);
@@ -1437,7 +1437,7 @@
             return -1;
         if (b) {
             if (self->buffer == NULL) {
-                self->buffer = malloc(self->buffer_size);
+                self->buffer = PyMem_Malloc(self->buffer_size);
                 if (self->buffer == NULL) {
                     PyErr_NoMemory();
                     return -1;
@@ -1448,7 +1448,7 @@
         else if (self->buffer != NULL) {
             if (flush_character_buffer(self) < 0)
                 return -1;
-            free(self->buffer);
+            PyMem_Free(self->buffer);
             self->buffer = NULL;
         }
         return 0;
@@ -1508,9 +1508,9 @@
           flush_character_buffer(self);
         }
         /* free existing buffer */
-        free(self->buffer);
+        PyMem_Free(self->buffer);
       }
-      self->buffer = malloc(new_buffer_size);
+      self->buffer = PyMem_Malloc(new_buffer_size);
       if (self->buffer == NULL) {
         PyErr_NoMemory();
         return -1;
diff --git a/Modules/readline.c b/Modules/readline.c
--- a/Modules/readline.c
+++ b/Modules/readline.c
@@ -84,12 +84,12 @@
         return NULL;
     /* Make a copy -- rl_parse_and_bind() modifies its argument */
     /* Bernard Herzog */
-    copy = malloc(1 + strlen(s));
+    copy = PyMem_Malloc(1 + strlen(s));
     if (copy == NULL)
         return PyErr_NoMemory();
     strcpy(copy, s);
     rl_parse_and_bind(copy);
-    free(copy); /* Free the copy */
+    PyMem_Free(copy); /* Free the copy */
     Py_RETURN_NONE;
 }
 
diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c
--- a/Modules/zlibmodule.c
+++ b/Modules/zlibmodule.c
@@ -165,7 +165,7 @@
 
     zst.avail_out = length + length/1000 + 12 + 1;
 
-    output = (Byte*)malloc(zst.avail_out);
+    output = (Byte*)PyMem_Malloc(zst.avail_out);
     if (output == NULL) {
         PyErr_SetString(PyExc_MemoryError,
                         "Can't allocate memory to compress data");
@@ -218,7 +218,7 @@
 
  error:
     PyBuffer_Release(&pinput);
-    free(output);
+    PyMem_Free(output);
 
     return ReturnVal;
 }
diff --git a/PC/winreg.c b/PC/winreg.c
--- a/PC/winreg.c
+++ b/PC/winreg.c
@@ -938,7 +938,7 @@
                 wchar_t *data = (wchar_t *)retDataBuf;
                 int len = retDataSize / 2;
                 int s = countStrings(data, len);
-                wchar_t **str = (wchar_t **)malloc(sizeof(wchar_t *)*s);
+                wchar_t **str = (wchar_t **)PyMem_Malloc(sizeof(wchar_t *)*s);
                 if (str == NULL)
                     return PyErr_NoMemory();
 
@@ -959,7 +959,7 @@
                                    index,
                                    PyUnicode_FromWideChar(str[index], len));
                 }
-                free(str);
+                PyMem_Free(str);
 
                 break;
             }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 16:36:08 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 16:36:08 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Fix_a_compiler_warning_in_?=
 =?utf-8?q?posix=5Fsendfile=28=29_on_FreeBSD=3A?=
Message-ID: <3bpC5h0GFqzQ1R@mail.python.org>

http://hg.python.org/cpython/rev/adafc38852c4
changeset:   84492:adafc38852c4
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 16:32:36 2013 +0200
summary:
  Fix a compiler warning in posix_sendfile() on FreeBSD:

Modules/posixmodule.c: In function 'posix_sendfile':
Modules/posixmodule.c:7700: warning: ISO C90 forbids mixed declarations and code

files:
  Modules/posixmodule.c |  5 +++--
  1 files changed, 3 insertions(+), 2 deletions(-)


diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -7695,12 +7695,13 @@
     off_t sbytes;
     struct sf_hdtr sf;
     int flags = 0;
-    sf.headers = NULL;
-    sf.trailers = NULL;
     static char *keywords[] = {"out", "in",
                                 "offset", "count",
                                 "headers", "trailers", "flags", NULL};
 
+    sf.headers = NULL;
+    sf.trailers = NULL;
+
 #ifdef __APPLE__
     if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iiO&O&|OOi:sendfile",
         keywords, &out, &in, _parse_off_t, &offset, _parse_off_t, &sbytes,

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 16:36:09 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 16:36:09 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318203=3A_Fix_deco?=
 =?utf-8?q?de=5Fascii=5Fsurrogateescape=28=29=2C_use_PyMem=5FRawMalloc=28?=
 =?utf-8?q?=29_as?=
Message-ID: <3bpC5j2W7zzSJT@mail.python.org>

http://hg.python.org/cpython/rev/fc01f9497da7
changeset:   84493:fc01f9497da7
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 16:35:54 2013 +0200
summary:
  Issue #18203: Fix decode_ascii_surrogateescape(), use PyMem_RawMalloc() as _Py_char2wchar()

files:
  Python/fileutils.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Python/fileutils.c b/Python/fileutils.c
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -201,7 +201,7 @@
     unsigned char *in;
     wchar_t *out;
 
-    res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t));
+    res = PyMem_RawMalloc((strlen(arg)+1)*sizeof(wchar_t));
     if (!res)
         return NULL;
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 16:50:41 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 16:50:41 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318227=3A_Use_PyMe?=
 =?utf-8?q?m=5FRawAlloc=28=29_in_bz2=2C_lzma_and_zlib_modules?=
Message-ID: <3bpCQT4y8VzSCB@mail.python.org>

http://hg.python.org/cpython/rev/a876d9d2e4fc
changeset:   84494:a876d9d2e4fc
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 16:50:27 2013 +0200
summary:
  Issue #18227: Use PyMem_RawAlloc() in bz2, lzma and zlib modules

files:
  Modules/_bz2module.c  |  21 ++++++++++++++++++
  Modules/_lzmamodule.c |  28 ++++++++++++++++++++++++
  Modules/zlibmodule.c  |  36 ++++++++++++++++++++++++------
  3 files changed, 77 insertions(+), 8 deletions(-)


diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c
--- a/Modules/_bz2module.c
+++ b/Modules/_bz2module.c
@@ -248,6 +248,24 @@
     return result;
 }
 
+static void*
+BZ2_Malloc(void* ctx, int items, int size)
+{
+    if (items < 0 || size < 0)
+        return NULL;
+    if ((size_t)items > (size_t)PY_SSIZE_T_MAX / (size_t)size)
+        return NULL;
+    /* PyMem_Malloc() cannot be used: compress() and decompress()
+       release the GIL */
+    return PyMem_RawMalloc(items * size);
+}
+
+static void
+BZ2_Free(void* ctx, void *ptr)
+{
+    return PyMem_RawFree(ptr);
+}
+
 static int
 BZ2Compressor_init(BZ2Compressor *self, PyObject *args, PyObject *kwargs)
 {
@@ -270,6 +288,9 @@
     }
 #endif
 
+    self->bzs.opaque = NULL;
+    self->bzs.bzalloc = BZ2_Malloc;
+    self->bzs.bzfree = BZ2_Free;
     bzerror = BZ2_bzCompressInit(&self->bzs, compresslevel, 0, 0);
     if (catch_bz2_error(bzerror))
         goto error;
diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c
--- a/Modules/_lzmamodule.c
+++ b/Modules/_lzmamodule.c
@@ -51,6 +51,7 @@
 
 typedef struct {
     PyObject_HEAD
+    lzma_allocator alloc;
     lzma_stream lzs;
     int flushed;
 #ifdef WITH_THREAD
@@ -60,6 +61,7 @@
 
 typedef struct {
     PyObject_HEAD
+    lzma_allocator alloc;
     lzma_stream lzs;
     int check;
     char eof;
@@ -117,6 +119,22 @@
     }
 }
 
+static void*
+PyLzma_Malloc(void *opaque, size_t items, size_t size)
+{
+    if (items > (size_t)PY_SSIZE_T_MAX / size)
+        return NULL;
+    /* PyMem_Malloc() cannot be used:
+       the GIL is not held when lzma_code() is called */
+    return PyMem_RawMalloc(items * size);
+}
+
+static void
+PyLzma_Free(void *opaque, void *ptr)
+{
+    return PyMem_RawFree(ptr);
+}
+
 #if BUFSIZ < 8192
 #define INITIAL_BUFFER_SIZE 8192
 #else
@@ -656,6 +674,11 @@
         if (!uint32_converter(preset_obj, &preset))
             return -1;
 
+    self->alloc.opaque = NULL;
+    self->alloc.alloc = PyLzma_Malloc;
+    self->alloc.free = PyLzma_Free;
+    self->lzs.allocator = &self->alloc;
+
 #ifdef WITH_THREAD
     self->lock = PyThread_allocate_lock();
     if (self->lock == NULL) {
@@ -922,6 +945,11 @@
         return -1;
     }
 
+    self->alloc.opaque = NULL;
+    self->alloc.alloc = PyLzma_Malloc;
+    self->alloc.free = PyLzma_Free;
+    self->lzs.allocator = &self->alloc;
+
 #ifdef WITH_THREAD
     self->lock = PyThread_allocate_lock();
     if (self->lock == NULL) {
diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c
--- a/Modules/zlibmodule.c
+++ b/Modules/zlibmodule.c
@@ -136,6 +136,22 @@
     return self;
 }
 
+static void*
+PyZlib_Malloc(voidpf ctx, uInt items, uInt size)
+{
+    if (items > (size_t)PY_SSIZE_T_MAX / size)
+        return NULL;
+    /* PyMem_Malloc() cannot be used: the GIL is not held when
+       inflate() and deflate() are called */
+    return PyMem_RawMalloc(items * size);
+}
+
+static void
+PyZlib_Free(voidpf ctx, void *ptr)
+{
+    return PyMem_RawFree(ptr);
+}
+
 PyDoc_STRVAR(compress__doc__,
 "compress(string[, level]) -- Returned compressed string.\n"
 "\n"
@@ -175,8 +191,9 @@
     /* Past the point of no return.  From here on out, we need to make sure
        we clean up mallocs & INCREFs. */
 
-    zst.zalloc = (alloc_func)NULL;
-    zst.zfree = (free_func)Z_NULL;
+    zst.opaque = NULL;
+    zst.zalloc = PyZlib_Malloc;
+    zst.zfree = PyZlib_Free;
     zst.next_out = (Byte *)output;
     zst.next_in = (Byte *)input;
     zst.avail_in = length;
@@ -262,8 +279,9 @@
     if (!(result_str = PyBytes_FromStringAndSize(NULL, r_strlen)))
         goto error;
 
-    zst.zalloc = (alloc_func)NULL;
-    zst.zfree = (free_func)Z_NULL;
+    zst.opaque = NULL;
+    zst.zalloc = PyZlib_Malloc;
+    zst.zfree = PyZlib_Free;
     zst.next_out = (Byte *)PyBytes_AS_STRING(result_str);
     zst.next_in = (Byte *)input;
     err = inflateInit2(&zst, wsize);
@@ -356,8 +374,9 @@
     self = newcompobject(&Comptype);
     if (self==NULL)
         goto error;
-    self->zst.zalloc = (alloc_func)NULL;
-    self->zst.zfree = (free_func)Z_NULL;
+    self->zst.opaque = NULL;
+    self->zst.zalloc = PyZlib_Malloc;
+    self->zst.zfree = PyZlib_Free;
     self->zst.next_in = NULL;
     self->zst.avail_in = 0;
     err = deflateInit2(&self->zst, level, method, wbits, memLevel, strategy);
@@ -420,8 +439,9 @@
     self = newcompobject(&Decomptype);
     if (self == NULL)
         return(NULL);
-    self->zst.zalloc = (alloc_func)NULL;
-    self->zst.zfree = (free_func)Z_NULL;
+    self->zst.opaque = NULL;
+    self->zst.zalloc = PyZlib_Malloc;
+    self->zst.zfree = PyZlib_Free;
     self->zst.next_in = NULL;
     self->zst.avail_in = 0;
     if (zdict != NULL) {

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 17:26:33 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 17:26:33 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318203=3A_Replace_?=
 =?utf-8?q?malloc=28=29_with_PyMem=5FMalloc=28=29_in_=5Fssl_for_the_passwo?=
 =?utf-8?q?rd?=
Message-ID: <3bpDCs1w59zRk0@mail.python.org>

http://hg.python.org/cpython/rev/638d43665356
changeset:   84495:638d43665356
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 17:07:52 2013 +0200
summary:
  Issue #18203: Replace malloc() with PyMem_Malloc() in _ssl for the password

files:
  Modules/_ssl.c |  8 ++++----
  1 files changed, 4 insertions(+), 4 deletions(-)


diff --git a/Modules/_ssl.c b/Modules/_ssl.c
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -2070,8 +2070,8 @@
         goto error;
     }
 
-    free(pw_info->password);
-    pw_info->password = malloc(size);
+    PyMem_Free(pw_info->password);
+    pw_info->password = PyMem_Malloc(size);
     if (!pw_info->password) {
         PyErr_SetString(PyExc_MemoryError,
                         "unable to allocate password buffer");
@@ -2215,13 +2215,13 @@
     }
     SSL_CTX_set_default_passwd_cb(self->ctx, orig_passwd_cb);
     SSL_CTX_set_default_passwd_cb_userdata(self->ctx, orig_passwd_userdata);
-    free(pw_info.password);
+    PyMem_Free(pw_info.password);
     Py_RETURN_NONE;
 
 error:
     SSL_CTX_set_default_passwd_cb(self->ctx, orig_passwd_cb);
     SSL_CTX_set_default_passwd_cb_userdata(self->ctx, orig_passwd_userdata);
-    free(pw_info.password);
+    PyMem_Free(pw_info.password);
     Py_XDECREF(keyfile_bytes);
     Py_XDECREF(certfile_bytes);
     return NULL;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 17:26:34 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 17:26:34 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318227=3A_=22Free?=
 =?utf-8?q?=22_function_of_bz2=2C_lzma_and_zlib_modules_has_no_return_valu?=
 =?utf-8?q?e?=
Message-ID: <3bpDCt4DHhzS4x@mail.python.org>

http://hg.python.org/cpython/rev/12f26c356611
changeset:   84496:12f26c356611
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 17:10:34 2013 +0200
summary:
  Issue #18227: "Free" function of bz2, lzma and zlib modules has no return value (void)

files:
  Modules/_bz2module.c  |  2 +-
  Modules/_lzmamodule.c |  2 +-
  Modules/zlibmodule.c  |  2 +-
  3 files changed, 3 insertions(+), 3 deletions(-)


diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c
--- a/Modules/_bz2module.c
+++ b/Modules/_bz2module.c
@@ -263,7 +263,7 @@
 static void
 BZ2_Free(void* ctx, void *ptr)
 {
-    return PyMem_RawFree(ptr);
+    PyMem_RawFree(ptr);
 }
 
 static int
diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c
--- a/Modules/_lzmamodule.c
+++ b/Modules/_lzmamodule.c
@@ -132,7 +132,7 @@
 static void
 PyLzma_Free(void *opaque, void *ptr)
 {
-    return PyMem_RawFree(ptr);
+    PyMem_RawFree(ptr);
 }
 
 #if BUFSIZ < 8192
diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c
--- a/Modules/zlibmodule.c
+++ b/Modules/zlibmodule.c
@@ -149,7 +149,7 @@
 static void
 PyZlib_Free(voidpf ctx, void *ptr)
 {
-    return PyMem_RawFree(ptr);
+    PyMem_RawFree(ptr);
 }
 
 PyDoc_STRVAR(compress__doc__,

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 17:26:35 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 17:26:35 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318203=3A_Replace_?=
 =?utf-8?q?malloc=28=29_with_PyMem=5FRawMalloc=28=29_to_allocate_thread_lo?=
 =?utf-8?q?cks?=
Message-ID: <3bpDCv6RVpzSbH@mail.python.org>

http://hg.python.org/cpython/rev/9af1905f20af
changeset:   84497:9af1905f20af
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 17:17:59 2013 +0200
summary:
  Issue #18203: Replace malloc() with PyMem_RawMalloc() to allocate thread locks

files:
  Python/thread_nt.h      |   8 ++++----
  Python/thread_pthread.h |  12 ++++++------
  2 files changed, 10 insertions(+), 10 deletions(-)


diff --git a/Python/thread_nt.h b/Python/thread_nt.h
--- a/Python/thread_nt.h
+++ b/Python/thread_nt.h
@@ -34,7 +34,7 @@
 PNRMUTEX
 AllocNonRecursiveMutex()
 {
-    PNRMUTEX m = (PNRMUTEX)malloc(sizeof(NRMUTEX));
+    PNRMUTEX m = (PNRMUTEX)PyMem_RawMalloc(sizeof(NRMUTEX));
     if (!m)
         return NULL;
     if (PyCOND_INIT(&m->cv))
@@ -46,7 +46,7 @@
     m->locked = 0;
     return m;
 fail:
-    free(m);
+    PyMem_RawFree(m);
     return NULL;
 }
 
@@ -56,7 +56,7 @@
     if (mutex) {
         PyCOND_FINI(&mutex->cv);
         PyMUTEX_FINI(&mutex->cs);
-        free(mutex);
+        PyMem_RawFree(mutex);
     }
 }
 
@@ -107,7 +107,7 @@
     result = PyCOND_SIGNAL(&mutex->cv);
     result &= PyMUTEX_UNLOCK(&mutex->cs);
     return result;
-}    
+}
 
 #else /* if ! _PY_USE_CV_LOCKS */
 
diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h
--- a/Python/thread_pthread.h
+++ b/Python/thread_pthread.h
@@ -282,14 +282,14 @@
     if (!initialized)
         PyThread_init_thread();
 
-    lock = (sem_t *)malloc(sizeof(sem_t));
+    lock = (sem_t *)PyMem_RawMalloc(sizeof(sem_t));
 
     if (lock) {
         status = sem_init(lock,0,1);
         CHECK_STATUS("sem_init");
 
         if (error) {
-            free((void *)lock);
+            PyMem_RawFree((void *)lock);
             lock = NULL;
         }
     }
@@ -313,7 +313,7 @@
     status = sem_destroy(thelock);
     CHECK_STATUS("sem_destroy");
 
-    free((void *)thelock);
+    PyMem_RawFree((void *)thelock);
 }
 
 /*
@@ -410,7 +410,7 @@
     if (!initialized)
         PyThread_init_thread();
 
-    lock = (pthread_lock *) malloc(sizeof(pthread_lock));
+    lock = (pthread_lock *) PyMem_RawMalloc(sizeof(pthread_lock));
     if (lock) {
         memset((void *)lock, '\0', sizeof(pthread_lock));
         lock->locked = 0;
@@ -430,7 +430,7 @@
         CHECK_STATUS("pthread_cond_init");
 
         if (error) {
-            free((void *)lock);
+            PyMem_RawFree((void *)lock);
             lock = 0;
         }
     }
@@ -457,7 +457,7 @@
     status = pthread_mutex_destroy( &thelock->mut );
     CHECK_STATUS("pthread_mutex_destroy");
 
-    free((void *)thelock);
+    PyMem_RawFree((void *)thelock);
 }
 
 PyLockStatus

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 17:26:37 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 17:26:37 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318203=3A_Replace_?=
 =?utf-8?q?malloc=28=29_with_PyMem=5FMalloc=28=29_to_allocate_arena_object?=
 =?utf-8?q?s?=
Message-ID: <3bpDCx1Pn3zS4x@mail.python.org>

http://hg.python.org/cpython/rev/fb7d346b45fa
changeset:   84498:fb7d346b45fa
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 17:18:53 2013 +0200
summary:
  Issue #18203: Replace malloc() with PyMem_Malloc() to allocate arena objects

files:
  Python/pyarena.c |  12 ++++++------
  1 files changed, 6 insertions(+), 6 deletions(-)


diff --git a/Python/pyarena.c b/Python/pyarena.c
--- a/Python/pyarena.c
+++ b/Python/pyarena.c
@@ -77,7 +77,7 @@
 {
     /* Allocate header and block as one unit.
        ab_mem points just past header. */
-    block *b = (block *)malloc(sizeof(block) + size);
+    block *b = (block *)PyMem_Malloc(sizeof(block) + size);
     if (!b)
         return NULL;
     b->ab_size = size;
@@ -92,7 +92,7 @@
 block_free(block *b) {
     while (b) {
         block *next = b->ab_next;
-        free(b);
+        PyMem_Free(b);
         b = next;
     }
 }
@@ -127,20 +127,20 @@
 PyArena *
 PyArena_New()
 {
-    PyArena* arena = (PyArena *)malloc(sizeof(PyArena));
+    PyArena* arena = (PyArena *)PyMem_Malloc(sizeof(PyArena));
     if (!arena)
         return (PyArena*)PyErr_NoMemory();
 
     arena->a_head = block_new(DEFAULT_BLOCK_SIZE);
     arena->a_cur = arena->a_head;
     if (!arena->a_head) {
-        free((void *)arena);
+        PyMem_Free((void *)arena);
         return (PyArena*)PyErr_NoMemory();
     }
     arena->a_objects = PyList_New(0);
     if (!arena->a_objects) {
         block_free(arena->a_head);
-        free((void *)arena);
+        PyMem_Free((void *)arena);
         return (PyArena*)PyErr_NoMemory();
     }
 #if defined(Py_DEBUG)
@@ -173,7 +173,7 @@
     */
 
     Py_DECREF(arena->a_objects);
-    free(arena);
+    PyMem_Free(arena);
 }
 
 void *

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 17:26:38 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 17:26:38 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318203=3A__Replace?=
 =?utf-8?q?_malloc=28=29_with_PyMem=5FMalloc=28=29_in?=
Message-ID: <3bpDCy3h4kzSr1@mail.python.org>

http://hg.python.org/cpython/rev/10db0c67fc72
changeset:   84499:10db0c67fc72
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 17:22:41 2013 +0200
summary:
  Issue #18203:  Replace malloc() with PyMem_Malloc() in _PySequence_BytesToCharpArray()

files:
  Objects/abstract.c |  19 +++++++++++--------
  1 files changed, 11 insertions(+), 8 deletions(-)


diff --git a/Objects/abstract.c b/Objects/abstract.c
--- a/Objects/abstract.c
+++ b/Objects/abstract.c
@@ -1238,7 +1238,7 @@
   to be an int or have an __int__ method. Steals integral's
   reference. error_format will be used to create the TypeError if integral
   isn't actually an Integral instance. error_format should be a format string
-  that can accept a char* naming integral's type. 
+  that can accept a char* naming integral's type.
 */
 static PyObject *
 convert_integral_to_int(PyObject *integral, const char *error_format)
@@ -1257,7 +1257,7 @@
     }
     PyErr_Format(PyExc_TypeError, error_format, Py_TYPE(integral)->tp_name);
     Py_DECREF(integral);
-    return NULL;    
+    return NULL;
 }
 
 
@@ -2721,8 +2721,8 @@
  * NULL terminated string pointers with a NULL char* terminating the array.
  * (ie: an argv or env list)
  *
- * Memory allocated for the returned list is allocated using malloc() and MUST
- * be freed by the caller using a free() loop or _Py_FreeCharPArray().
+ * Memory allocated for the returned list is allocated using PyMem_Malloc()
+ * and MUST be freed by _Py_FreeCharPArray().
  */
 char *const *
 _PySequence_BytesToCharpArray(PyObject* self)
@@ -2730,6 +2730,7 @@
     char **array;
     Py_ssize_t i, argc;
     PyObject *item = NULL;
+    Py_ssize_t size;
 
     argc = PySequence_Size(self);
     if (argc == -1)
@@ -2742,7 +2743,7 @@
         return NULL;
     }
 
-    array = malloc((argc + 1) * sizeof(char *));
+    array = PyMem_Malloc((argc + 1) * sizeof(char *));
     if (array == NULL) {
         PyErr_NoMemory();
         return NULL;
@@ -2761,11 +2762,13 @@
             array[i] = NULL;
             goto fail;
         }
-        array[i] = strdup(data);
+        size = PyBytes_GET_SIZE(item) + 1;
+        array[i] = PyMem_Malloc(size);
         if (!array[i]) {
             PyErr_NoMemory();
             goto fail;
         }
+        memcpy(array[i], data, size);
         Py_DECREF(item);
     }
     array[argc] = NULL;
@@ -2785,7 +2788,7 @@
 {
     Py_ssize_t i;
     for (i = 0; array[i] != NULL; ++i) {
-        free(array[i]);
+        PyMem_Free(array[i]);
     }
-    free((void*)array);
+    PyMem_Free((void*)array);
 }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 17:35:20 2013
From: python-checkins at python.org (christian.heimes)
Date: Sun,  7 Jul 2013 17:35:20 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318227=3A_pyexpat_?=
 =?utf-8?q?now_uses_a_static_XML=5FMemory=5FHandling=5FSuite=2E_cElementTr?=
 =?utf-8?q?ee?=
Message-ID: <3bpDQ03GzTzR4L@mail.python.org>

http://hg.python.org/cpython/rev/7f17c67b5bf6
changeset:   84500:7f17c67b5bf6
user:        Christian Heimes <christian at cheimes.de>
date:        Sun Jul 07 17:35:11 2013 +0200
summary:
  Issue #18227: pyexpat now uses a static XML_Memory_Handling_Suite. cElementTree uses the same approach since at least Python 2.6

files:
  Modules/pyexpat.c |  12 ++++++------
  1 files changed, 6 insertions(+), 6 deletions(-)


diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c
--- a/Modules/pyexpat.c
+++ b/Modules/pyexpat.c
@@ -10,6 +10,9 @@
 
 #define FIX_TRACE
 
+static XML_Memory_Handling_Suite ExpatMemoryHandler = {
+    PyObject_Malloc, PyObject_Realloc, PyObject_Free};
+
 enum HandlerTypes {
     StartElement,
     EndElement,
@@ -1177,12 +1180,9 @@
     self->in_callback = 0;
     self->ns_prefixes = 0;
     self->handlers = NULL;
-    if (namespace_separator != NULL) {
-        self->itself = XML_ParserCreateNS(encoding, *namespace_separator);
-    }
-    else {
-        self->itself = XML_ParserCreate(encoding);
-    }
+    /* namespace_separator is either NULL or contains one char + \0 */
+    self->itself = XML_ParserCreate_MM(encoding, &ExpatMemoryHandler,
+                                       namespace_separator);
 #if ((XML_MAJOR_VERSION >= 2) && (XML_MINOR_VERSION >= 1)) || defined(XML_HAS_SET_HASH_SALT)
     /* This feature was added upstream in libexpat 2.1.0.  Our expat copy
      * has a backport of this feature where we also define XML_HAS_SET_HASH_SALT

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 22:58:06 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 22:58:06 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318203=3A_Fix_=5FP?=
 =?utf-8?q?y=5FDecodeUTF8=5Fsurrogateescape=28=29=2C_use_PyMem=5FRawMalloc?=
 =?utf-8?b?KCkgYXM=?=
Message-ID: <3bpMZQ44RXzSCh@mail.python.org>

http://hg.python.org/cpython/rev/31a635303e55
changeset:   84501:31a635303e55
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 22:57:45 2013 +0200
summary:
  Issue #18203: Fix _Py_DecodeUTF8_surrogateescape(), use PyMem_RawMalloc() as _Py_char2wchar()

files:
  Objects/unicodeobject.c |  4 ++--
  1 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4806,7 +4806,7 @@
    used to decode the command line arguments on Mac OS X.
 
    Return a pointer to a newly allocated wide character string (use
-   PyMem_Free() to free the memory), or NULL on memory allocation error. */
+   PyMem_RawFree() to free the memory), or NULL on memory allocation error. */
 
 wchar_t*
 _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size)
@@ -4819,7 +4819,7 @@
        character count */
     if (PY_SSIZE_T_MAX / sizeof(wchar_t) < (size + 1))
         return NULL;
-    unicode = PyMem_Malloc((size + 1) * sizeof(wchar_t));
+    unicode = PyMem_RawMalloc((size + 1) * sizeof(wchar_t));
     if (!unicode)
         return NULL;
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Sun Jul  7 23:32:58 2013
From: python-checkins at python.org (victor.stinner)
Date: Sun,  7 Jul 2013 23:32:58 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318203=3A_Add_=5FP?=
 =?utf-8?b?eU1lbV9SYXdTdHJkdXAoKSBhbmQgX1B5TWVtX1N0cmR1cCgp?=
Message-ID: <3bpNLf4PQnzPrk@mail.python.org>

http://hg.python.org/cpython/rev/65f2c92ed079
changeset:   84502:65f2c92ed079
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sun Jul 07 23:30:24 2013 +0200
summary:
  Issue #18203: Add _PyMem_RawStrdup() and _PyMem_Strdup()

Replace strdup() with _PyMem_RawStrdup() or _PyMem_Strdup(), depending if the
GIL is held or not.

files:
  Include/pymem.h         |   3 +++
  Modules/_cursesmodule.c |   8 ++++----
  Modules/_pickle.c       |  12 ++++++------
  Modules/faulthandler.c  |  11 ++++++-----
  Modules/main.c          |   4 ++--
  Modules/python.c        |   6 +++---
  Objects/obmalloc.c      |  28 ++++++++++++++++++++++++++++
  Python/pythonrun.c      |  21 ++++++++++++---------
  8 files changed, 64 insertions(+), 29 deletions(-)


diff --git a/Include/pymem.h b/Include/pymem.h
--- a/Include/pymem.h
+++ b/Include/pymem.h
@@ -58,6 +58,9 @@
 PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
 PyAPI_FUNC(void) PyMem_Free(void *ptr);
 
+PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str);
+PyAPI_FUNC(char *) _PyMem_Strdup(const char *str);
+
 /* Macros. */
 
 /* PyMem_MALLOC(0) means malloc(1). Some systems would return NULL
diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c
--- a/Modules/_cursesmodule.c
+++ b/Modules/_cursesmodule.c
@@ -529,7 +529,7 @@
     wo = PyObject_NEW(PyCursesWindowObject, &PyCursesWindow_Type);
     if (wo == NULL) return NULL;
     wo->win = win;
-    wo->encoding = strdup(encoding);
+    wo->encoding = _PyMem_Strdup(encoding);
     if (wo->encoding == NULL) {
         Py_DECREF(wo);
         PyErr_NoMemory();
@@ -543,7 +543,7 @@
 {
     if (wo->win != stdscr) delwin(wo->win);
     if (wo->encoding != NULL)
-        free(wo->encoding);
+        PyMem_Free(wo->encoding);
     PyObject_DEL(wo);
 }
 
@@ -1938,13 +1938,13 @@
     ascii = PyUnicode_AsASCIIString(value);
     if (ascii == NULL)
         return -1;
-    encoding = strdup(PyBytes_AS_STRING(ascii));
+    encoding = _PyMem_Strdup(PyBytes_AS_STRING(ascii));
     Py_DECREF(ascii);
     if (encoding == NULL) {
         PyErr_NoMemory();
         return -1;
     }
-    free(self->encoding);
+    PyMem_Free(self->encoding);
     self->encoding = encoding;
     return 0;
 }
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -1213,8 +1213,8 @@
     if (errors == NULL)
         errors = "strict";
 
-    self->encoding = strdup(encoding);
-    self->errors = strdup(errors);
+    self->encoding = _PyMem_Strdup(encoding);
+    self->errors = _PyMem_Strdup(errors);
     if (self->encoding == NULL || self->errors == NULL) {
         PyErr_NoMemory();
         return -1;
@@ -5590,8 +5590,8 @@
     _Unpickler_MemoCleanup(self);
     PyMem_Free(self->marks);
     PyMem_Free(self->input_line);
-    free(self->encoding);
-    free(self->errors);
+    PyMem_Free(self->encoding);
+    PyMem_Free(self->errors);
 
     Py_TYPE(self)->tp_free((PyObject *)self);
 }
@@ -5627,9 +5627,9 @@
     self->marks = NULL;
     PyMem_Free(self->input_line);
     self->input_line = NULL;
-    free(self->encoding);
+    PyMem_Free(self->encoding);
     self->encoding = NULL;
-    free(self->errors);
+    PyMem_Free(self->errors);
     self->errors = NULL;
 
     return 0;
diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c
--- a/Modules/faulthandler.c
+++ b/Modules/faulthandler.c
@@ -475,7 +475,7 @@
 
     Py_CLEAR(thread.file);
     if (thread.header) {
-        free(thread.header);
+        PyMem_Free(thread.header);
         thread.header = NULL;
     }
 }
@@ -504,7 +504,7 @@
                       "Timeout (%lu:%02lu:%02lu)!\n",
                       hour, min, sec);
 
-    return strdup(buffer);
+    return _PyMem_Strdup(buffer);
 }
 
 static PyObject*
@@ -570,7 +570,7 @@
     if (PyThread_start_new_thread(faulthandler_thread, NULL) == -1) {
         PyThread_release_lock(thread.running);
         Py_CLEAR(thread.file);
-        free(header);
+        PyMem_Free(header);
         thread.header = NULL;
         PyErr_SetString(PyExc_RuntimeError,
                         "unable to start watchdog thread");
@@ -729,9 +729,10 @@
         return NULL;
 
     if (user_signals == NULL) {
-        user_signals = calloc(NSIG, sizeof(user_signal_t));
+        user_signals = PyMem_Malloc(NSIG * sizeof(user_signal_t));
         if (user_signals == NULL)
             return PyErr_NoMemory();
+        memset(user_signals, 0, NSIG * sizeof(user_signal_t));
     }
     user = &user_signals[signum];
 
@@ -1136,7 +1137,7 @@
     if (user_signals != NULL) {
         for (signum=0; signum < NSIG; signum++)
             faulthandler_unregister(&user_signals[signum], signum);
-        free(user_signals);
+        PyMem_Free(user_signals);
         user_signals = NULL;
     }
 #endif
diff --git a/Modules/main.c b/Modules/main.c
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -544,7 +544,7 @@
             Py_FatalError(
                "not enough memory to copy PYTHONWARNINGS");
         strcpy(buf, p);
-        oldloc = strdup(setlocale(LC_ALL, NULL));
+        oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL));
         setlocale(LC_ALL, "");
         for (p = strtok(buf, ","); p != NULL; p = strtok(NULL, ",")) {
 #ifdef __APPLE__
@@ -562,7 +562,7 @@
             Py_DECREF(unicode);
         }
         setlocale(LC_ALL, oldloc);
-        free(oldloc);
+        PyMem_RawFree(oldloc);
         PyMem_RawFree(buf);
     }
 #endif
diff --git a/Modules/python.c b/Modules/python.c
--- a/Modules/python.c
+++ b/Modules/python.c
@@ -43,12 +43,12 @@
     fpsetmask(m & ~FP_X_OFL);
 #endif
 
-    oldloc = strdup(setlocale(LC_ALL, NULL));
+    oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL));
     setlocale(LC_ALL, "");
     for (i = 0; i < argc; i++) {
         argv_copy[i] = _Py_char2wchar(argv[i], NULL);
         if (!argv_copy[i]) {
-            free(oldloc);
+            PyMem_RawFree(oldloc);
             fprintf(stderr, "Fatal Python error: "
                             "unable to decode the command line argument #%i\n",
                             i + 1);
@@ -59,7 +59,7 @@
     argv_copy2[argc] = argv_copy[argc] = NULL;
 
     setlocale(LC_ALL, oldloc);
-    free(oldloc);
+    PyMem_RawFree(oldloc);
     res = Py_Main(argc, argv_copy);
     for (i = 0; i < argc; i++) {
         PyMem_RawFree(argv_copy2[i]);
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -294,6 +294,34 @@
     _PyMem.free(_PyMem.ctx, ptr);
 }
 
+char *
+_PyMem_RawStrdup(const char *str)
+{
+    size_t size;
+    char *copy;
+
+    size = strlen(str) + 1;
+    copy = PyMem_RawMalloc(size);
+    if (copy == NULL)
+        return NULL;
+    memcpy(copy, str, size);
+    return copy;
+}
+
+char *
+_PyMem_Strdup(const char *str)
+{
+    size_t size;
+    char *copy;
+
+    size = strlen(str) + 1;
+    copy = PyMem_Malloc(size);
+    if (copy == NULL)
+        return NULL;
+    memcpy(copy, str, size);
+    return copy;
+}
+
 void *
 PyObject_Malloc(size_t size)
 {
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -174,7 +174,7 @@
     name_utf8 = _PyUnicode_AsString(name);
     if (name_utf8 == NULL)
         goto error;
-    name_str = strdup(name_utf8);
+    name_str = _PyMem_RawStrdup(name_utf8);
     Py_DECREF(name);
     if (name_str == NULL) {
         PyErr_NoMemory();
@@ -626,7 +626,7 @@
 
     /* reset file system default encoding */
     if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) {
-        free((char*)Py_FileSystemDefaultEncoding);
+        PyMem_RawFree((char*)Py_FileSystemDefaultEncoding);
         Py_FileSystemDefaultEncoding = NULL;
     }
 
@@ -1081,7 +1081,11 @@
     encoding = Py_GETENV("PYTHONIOENCODING");
     errors = NULL;
     if (encoding) {
-        encoding = strdup(encoding);
+        encoding = _PyMem_Strdup(encoding);
+        if (encoding == NULL) {
+            PyErr_NoMemory();
+            goto error;
+        }
         errors = strchr(encoding, ':');
         if (errors) {
             *errors = '\0';
@@ -1140,10 +1144,10 @@
        when import.c tries to write to stderr in verbose mode. */
     encoding_attr = PyObject_GetAttrString(std, "encoding");
     if (encoding_attr != NULL) {
-        const char * encoding;
-        encoding = _PyUnicode_AsString(encoding_attr);
-        if (encoding != NULL) {
-            PyObject *codec_info = _PyCodec_Lookup(encoding);
+        const char * std_encoding;
+        std_encoding = _PyUnicode_AsString(encoding_attr);
+        if (std_encoding != NULL) {
+            PyObject *codec_info = _PyCodec_Lookup(std_encoding);
             Py_XDECREF(codec_info);
         }
         Py_DECREF(encoding_attr);
@@ -1160,8 +1164,7 @@
         status = -1;
     }
 
-    if (encoding)
-        free(encoding);
+    PyMem_Free(encoding);
     Py_XDECREF(bimod);
     Py_XDECREF(iomod);
     return status;

-- 
Repository URL: http://hg.python.org/cpython

From solipsis at pitrou.net  Mon Jul  8 05:48:29 2013
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Mon, 08 Jul 2013 05:48:29 +0200
Subject: [Python-checkins] Daily reference leaks (65f2c92ed079): sum=-855
Message-ID: <E1Uw2RC-0003yy-01@ap.vmr.nerim.net>

results for 65f2c92ed079 on branch "default"
--------------------------------------------

test_urllib2net leaked [-467, 0, 0] references, sum=-467
test_urllib2net leaked [-391, 2, 1] memory blocks, sum=-388


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflog6P1LxH', '-x']

From python-checkins at python.org  Mon Jul  8 17:50:13 2013
From: python-checkins at python.org (ezio.melotti)
Date: Mon,  8 Jul 2013 17:50:13 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Add_a_couple_o?=
 =?utf-8?q?f_tests_for_str=2Ecenter_with_non-ASCII_chars=2E?=
Message-ID: <3bprhj0hNCz7LjP@mail.python.org>

http://hg.python.org/cpython/rev/d249203f423c
changeset:   84503:d249203f423c
branch:      3.3
parent:      84484:65fce1dad331
user:        Ezio Melotti <ezio.melotti at gmail.com>
date:        Mon Jul 08 17:48:29 2013 +0200
summary:
  Add a couple of tests for str.center with non-ASCII chars.

files:
  Lib/test/test_unicode.py |  9 +++++++++
  1 files changed, 9 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -652,6 +652,15 @@
         self.assertEqual('?'.swapcase(), 'SS')
         self.assertEqual('\u1fd2'.swapcase(), '\u0399\u0308\u0300')
 
+    def test_center(self):
+        string_tests.CommonTest.test_center(self)
+        self.assertEqual('x'.center(2, '\U0010FFFF'),
+                         'x\U0010FFFF')
+        self.assertEqual('x'.center(3, '\U0010FFFF'),
+                         '\U0010FFFFx\U0010FFFF')
+        self.assertEqual('x'.center(4, '\U0010FFFF'),
+                         '\U0010FFFFx\U0010FFFF\U0010FFFF')
+
     def test_contains(self):
         # Testing Unicode contains method
         self.assertIn('a', 'abdb')

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 17:50:14 2013
From: python-checkins at python.org (ezio.melotti)
Date: Mon,  8 Jul 2013 17:50:14 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Merge_str=2Ecenter_tests_from_3=2E3=2E?=
Message-ID: <3bprhk2yTcz7LjP@mail.python.org>

http://hg.python.org/cpython/rev/4a0391d61e4c
changeset:   84504:4a0391d61e4c
parent:      84502:65f2c92ed079
parent:      84503:d249203f423c
user:        Ezio Melotti <ezio.melotti at gmail.com>
date:        Mon Jul 08 17:49:59 2013 +0200
summary:
  Merge str.center tests from 3.3.

files:
  Lib/test/test_unicode.py |  9 +++++++++
  1 files changed, 9 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -663,6 +663,15 @@
         self.assertEqual('?'.swapcase(), 'SS')
         self.assertEqual('\u1fd2'.swapcase(), '\u0399\u0308\u0300')
 
+    def test_center(self):
+        string_tests.CommonTest.test_center(self)
+        self.assertEqual('x'.center(2, '\U0010FFFF'),
+                         'x\U0010FFFF')
+        self.assertEqual('x'.center(3, '\U0010FFFF'),
+                         '\U0010FFFFx\U0010FFFF')
+        self.assertEqual('x'.center(4, '\U0010FFFF'),
+                         '\U0010FFFFx\U0010FFFF\U0010FFFF')
+
     def test_contains(self):
         # Testing Unicode contains method
         self.assertIn('a', 'abdb')

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 17:53:45 2013
From: python-checkins at python.org (ezio.melotti)
Date: Mon,  8 Jul 2013 17:53:45 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE4NDAzOiBmaXgg?=
 =?utf-8?q?an_off-by-one_typo_noticed_by_Xue_Fuqiao=2E?=
Message-ID: <3bprmn0Crqz7LjP@mail.python.org>

http://hg.python.org/cpython/rev/6f16fa5223cc
changeset:   84505:6f16fa5223cc
branch:      3.3
parent:      84503:d249203f423c
user:        Ezio Melotti <ezio.melotti at gmail.com>
date:        Mon Jul 08 17:52:54 2013 +0200
summary:
  #18403: fix an off-by-one typo noticed by Xue Fuqiao.

files:
  Doc/tutorial/introduction.rst |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst
--- a/Doc/tutorial/introduction.rst
+++ b/Doc/tutorial/introduction.rst
@@ -262,7 +262,7 @@
 
    >>> word[0:2]  # characters from position 0 (included) to 2 (excluded)
    'Py'
-   >>> word[2:5]  # characters from position 2 (included) to 4 (excluded)
+   >>> word[2:5]  # characters from position 2 (included) to 5 (excluded)
    'tho'
 
 Note how the start is always included, and the end always excluded.  This

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 17:53:46 2013
From: python-checkins at python.org (ezio.melotti)
Date: Mon,  8 Jul 2013 17:53:46 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?b?KTogIzE4NDAzOiBtZXJnZSB3aXRoIDMuMy4=?=
Message-ID: <3bprmp2bYqz7Ljs@mail.python.org>

http://hg.python.org/cpython/rev/d41adb657bd4
changeset:   84506:d41adb657bd4
parent:      84504:4a0391d61e4c
parent:      84505:6f16fa5223cc
user:        Ezio Melotti <ezio.melotti at gmail.com>
date:        Mon Jul 08 17:53:32 2013 +0200
summary:
  #18403: merge with 3.3.

files:
  Doc/tutorial/introduction.rst |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst
--- a/Doc/tutorial/introduction.rst
+++ b/Doc/tutorial/introduction.rst
@@ -262,7 +262,7 @@
 
    >>> word[0:2]  # characters from position 0 (included) to 2 (excluded)
    'Py'
-   >>> word[2:5]  # characters from position 2 (included) to 4 (excluded)
+   >>> word[2:5]  # characters from position 2 (included) to 5 (excluded)
    'tho'
 
 Note how the start is always included, and the end always excluded.  This

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 22:36:31 2013
From: python-checkins at python.org (victor.stinner)
Date: Mon,  8 Jul 2013 22:36:31 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_gcmodule=2Ec=3A_strip_trai?=
	=?utf-8?q?ling_spaces?=
Message-ID: <3bpz336CGdzMwG@mail.python.org>

http://hg.python.org/cpython/rev/781be7ad1074
changeset:   84507:781be7ad1074
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Jul 08 22:15:05 2013 +0200
summary:
  gcmodule.c: strip trailing spaces

files:
  Modules/gcmodule.c |  14 +++++++-------
  1 files changed, 7 insertions(+), 7 deletions(-)


diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -118,7 +118,7 @@
 
 /*
    NOTE: about untracking of mutable objects.
-   
+
    Certain types of container cannot participate in a reference cycle, and
    so do not need to be tracked by the garbage collector. Untracking these
    objects reduces the cost of garbage collections. However, determining
@@ -136,10 +136,10 @@
    not survive until garbage collection. It is therefore not worthwhile
    to untrack eligible tuples at creation time.
 
-   Instead, all tuples except the empty tuple are tracked when created. 
-   During garbage collection it is determined whether any surviving tuples 
-   can be untracked. A tuple can be untracked if all of its contents are 
-   already not tracked. Tuples are examined for untracking in all garbage 
+   Instead, all tuples except the empty tuple are tracked when created.
+   During garbage collection it is determined whether any surviving tuples
+   can be untracked. A tuple can be untracked if all of its contents are
+   already not tracked. Tuples are examined for untracking in all garbage
    collection cycles. It may take more than one cycle to untrack a tuple.
 
    Dictionaries containing only immutable objects also do not need to be
@@ -152,8 +152,8 @@
    The module provides the python function is_tracked(obj), which returns
    the CURRENT tracking status of the object. Subsequent garbage
    collections may change the tracking status of the object.
-   
-   Untracking of certain containers was introduced in issue #4688, and 
+
+   Untracking of certain containers was introduced in issue #4688, and
    the algorithm was refined in response to issue #14775.
 */
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 22:36:33 2013
From: python-checkins at python.org (victor.stinner)
Date: Mon,  8 Jul 2013 22:36:33 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_PyObject?=
 =?utf-8?q?=5FGC=5FNewVar=28=29_now_raises_SystemError_exception_if_nitems?=
Message-ID: <3bpz351MF9z7LjM@mail.python.org>

http://hg.python.org/cpython/rev/111c2a070f28
changeset:   84508:111c2a070f28
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Jul 08 22:17:52 2013 +0200
summary:
  Issue #18408: PyObject_GC_NewVar() now raises SystemError exception if nitems
is negative

files:
  Modules/gcmodule.c |  11 +++++++++--
  1 files changed, 9 insertions(+), 2 deletions(-)


diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -1689,8 +1689,15 @@
 PyVarObject *
 _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems)
 {
-    const size_t size = _PyObject_VAR_SIZE(tp, nitems);
-    PyVarObject *op = (PyVarObject *) _PyObject_GC_Malloc(size);
+    size_t size;
+    PyVarObject *op;
+
+    if (nitems < 0) {
+        PyErr_BadInternalCall();
+        return NULL;
+    }
+    size = _PyObject_VAR_SIZE(tp, nitems);
+    op = (PyVarObject *) _PyObject_GC_Malloc(size);
     if (op != NULL)
         op = PyObject_INIT_VAR(op, tp, nitems);
     return op;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 22:36:34 2013
From: python-checkins at python.org (victor.stinner)
Date: Mon,  8 Jul 2013 22:36:34 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_PyDi?=
 =?utf-8?q?ct=5FNew=28=29_to_handle_correctly_new=5Fkeys=5Fobject=28=29_fa?=
 =?utf-8?q?ilure?=
Message-ID: <3bpz363dDvz7LjX@mail.python.org>

http://hg.python.org/cpython/rev/ba766323b53a
changeset:   84509:ba766323b53a
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Jul 08 22:19:20 2013 +0200
summary:
  Issue #18408: Fix PyDict_New() to handle correctly new_keys_object() failure
(MemoryError).

files:
  Objects/dictobject.c |  6 +++++-
  1 files changed, 5 insertions(+), 1 deletions(-)


diff --git a/Objects/dictobject.c b/Objects/dictobject.c
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -389,6 +389,7 @@
 new_dict(PyDictKeysObject *keys, PyObject **values)
 {
     PyDictObject *mp;
+    assert(keys != NULL);
     if (numfree) {
         mp = free_list[--numfree];
         assert (mp != NULL);
@@ -431,7 +432,10 @@
 PyObject *
 PyDict_New(void)
 {
-    return new_dict(new_keys_object(PyDict_MINSIZE_COMBINED), NULL);
+    PyDictKeysObject *keys = new_keys_object(PyDict_MINSIZE_COMBINED);
+    if (keys == NULL)
+        return NULL;
+    return new_dict(keys, NULL);
 }
 
 /*

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 22:36:35 2013
From: python-checkins at python.org (victor.stinner)
Date: Mon,  8 Jul 2013 22:36:35 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_list?=
 =?utf-8?q?=2Epop=28=29_to_handle_list=5Fresize=28=29_failure_=28MemoryErr?=
 =?utf-8?q?or=29=2E?=
Message-ID: <3bpz375nQNz7LkW@mail.python.org>

http://hg.python.org/cpython/rev/68887e177dd4
changeset:   84510:68887e177dd4
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Jul 08 22:20:44 2013 +0200
summary:
  Issue #18408: Fix list.pop() to handle list_resize() failure (MemoryError).

files:
  Objects/listobject.c |  6 ++++--
  1 files changed, 4 insertions(+), 2 deletions(-)


diff --git a/Objects/listobject.c b/Objects/listobject.c
--- a/Objects/listobject.c
+++ b/Objects/listobject.c
@@ -925,8 +925,10 @@
     v = self->ob_item[i];
     if (i == Py_SIZE(self) - 1) {
         status = list_resize(self, Py_SIZE(self) - 1);
-        assert(status >= 0);
-        return v; /* and v now owns the reference the list had */
+        if (status >= 0)
+            return v; /* and v now owns the reference the list had */
+        else
+            return NULL;
     }
     Py_INCREF(v);
     status = list_ass_slice(self, i, i+1, (PyObject *)NULL);

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 22:36:37 2013
From: python-checkins at python.org (victor.stinner)
Date: Mon,  8 Jul 2013 22:36:37 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_mars?=
 =?utf-8?q?hal_reader_for_Unicode_strings=3A_handle?=
Message-ID: <3bpz390pgbz7LkZ@mail.python.org>

http://hg.python.org/cpython/rev/697d722d97f9
changeset:   84511:697d722d97f9
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Jul 08 22:23:32 2013 +0200
summary:
  Issue #18408: Fix marshal reader for Unicode strings: handle
PyUnicode_DecodeUTF8() failure (ex: MemoryError).

files:
  Python/marshal.c |  4 ++++
  1 files changed, 4 insertions(+), 0 deletions(-)


diff --git a/Python/marshal.c b/Python/marshal.c
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -998,6 +998,10 @@
         else {
             v = PyUnicode_New(0, 0);
         }
+        if (v == NULL) {
+            retval = NULL;
+            break;
+        }
         if (type == TYPE_INTERNED)
             PyUnicode_InternInPlace(&v);
         retval = v;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 22:36:38 2013
From: python-checkins at python.org (victor.stinner)
Date: Mon,  8 Jul 2013 22:36:38 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_PyTy?=
 =?utf-8?q?pe=5FReady=28=29_and_type=2E=5F=5Fbases=5F=5F_setter_to_handle?=
Message-ID: <3bpz3B36DFz7Lkf@mail.python.org>

http://hg.python.org/cpython/rev/de1473f4503b
changeset:   84512:de1473f4503b
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Jul 08 22:25:48 2013 +0200
summary:
  Issue #18408: Fix PyType_Ready() and type.__bases__ setter to handle
PyWeakref_NewRef() failure (ex: MemoryError).

files:
  Objects/typeobject.c |  2 ++
  1 files changed, 2 insertions(+), 0 deletions(-)


diff --git a/Objects/typeobject.c b/Objects/typeobject.c
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -4329,6 +4329,8 @@
     }
     assert(PyList_Check(list));
     newobj = PyWeakref_NewRef((PyObject *)type, NULL);
+    if (newobj == NULL)
+        return -1;
     i = PyList_GET_SIZE(list);
     while (--i >= 0) {
         ref = PyList_GET_ITEM(list, i);

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 22:36:39 2013
From: python-checkins at python.org (victor.stinner)
Date: Mon,  8 Jul 2013 22:36:39 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_call?=
 =?utf-8?q?=5Ffunction=28=29_of_ceval=2Ec_to_handle_PyTuple=5FNew=28=29_fa?=
 =?utf-8?q?ilure?=
Message-ID: <3bpz3C5NYyz7Ljx@mail.python.org>

http://hg.python.org/cpython/rev/f4311870e329
changeset:   84513:f4311870e329
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Jul 08 22:27:42 2013 +0200
summary:
  Issue #18408: Fix call_function() of ceval.c to handle PyTuple_New() failure
(in load_args()), ex: MemoryError.

files:
  Python/ceval.c |  13 +++++++++----
  1 files changed, 9 insertions(+), 4 deletions(-)


diff --git a/Python/ceval.c b/Python/ceval.c
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -4171,10 +4171,15 @@
         else {
             PyObject *callargs;
             callargs = load_args(pp_stack, na);
-            READ_TIMESTAMP(*pintr0);
-            C_TRACE(x, PyCFunction_Call(func,callargs,NULL));
-            READ_TIMESTAMP(*pintr1);
-            Py_XDECREF(callargs);
+            if (callargs != NULL) {
+                READ_TIMESTAMP(*pintr0);
+                C_TRACE(x, PyCFunction_Call(func,callargs,NULL));
+                READ_TIMESTAMP(*pintr1);
+                Py_XDECREF(callargs);
+            }
+            else {
+                x = NULL;
+            }
         }
     } else {
         if (PyMethod_Check(func) && PyMethod_GET_SELF(func) != NULL) {

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 22:36:41 2013
From: python-checkins at python.org (victor.stinner)
Date: Mon,  8 Jul 2013 22:36:41 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_fix_indentation?=
Message-ID: <3bpz3F0S2vz7Ljx@mail.python.org>

http://hg.python.org/cpython/rev/890496c5a4d6
changeset:   84514:890496c5a4d6
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Jul 08 22:28:27 2013 +0200
summary:
  fix indentation

files:
  Modules/cjkcodecs/multibytecodec.c |  26 +++++++++---------
  1 files changed, 13 insertions(+), 13 deletions(-)


diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c
--- a/Modules/cjkcodecs/multibytecodec.c
+++ b/Modules/cjkcodecs/multibytecodec.c
@@ -1275,19 +1275,19 @@
             if (PyBytes_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) {
                 PyErr_NoMemory();
                 goto errorexit;
-        }
-                    rsize = PyBytes_GET_SIZE(cres) + self->pendingsize;
-                    ctr = PyBytes_FromStringAndSize(NULL, rsize);
-                    if (ctr == NULL)
-                            goto errorexit;
-                    ctrdata = PyBytes_AS_STRING(ctr);
-                    memcpy(ctrdata, self->pending, self->pendingsize);
-                    memcpy(ctrdata + self->pendingsize,
-                            PyBytes_AS_STRING(cres),
-                            PyBytes_GET_SIZE(cres));
-                    Py_DECREF(cres);
-                    cres = ctr;
-                    self->pendingsize = 0;
+            }
+            rsize = PyBytes_GET_SIZE(cres) + self->pendingsize;
+            ctr = PyBytes_FromStringAndSize(NULL, rsize);
+            if (ctr == NULL)
+                goto errorexit;
+            ctrdata = PyBytes_AS_STRING(ctr);
+            memcpy(ctrdata, self->pending, self->pendingsize);
+            memcpy(ctrdata + self->pendingsize,
+                    PyBytes_AS_STRING(cres),
+                    PyBytes_GET_SIZE(cres));
+            Py_DECREF(cres);
+            cres = ctr;
+            self->pendingsize = 0;
         }
 
         rsize = PyBytes_GET_SIZE(cres);

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 22:36:42 2013
From: python-checkins at python.org (victor.stinner)
Date: Mon,  8 Jul 2013 22:36:42 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_=5FP?=
 =?utf-8?q?yUnicodeWriter=5FFinish=28=29=3A_clear_writer-=3Ebuffer=2C?=
Message-ID: <3bpz3G2ykVz7Ljc@mail.python.org>

http://hg.python.org/cpython/rev/df8b40593a08
changeset:   84515:df8b40593a08
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Jul 08 22:29:55 2013 +0200
summary:
  Issue #18408: Fix _PyUnicodeWriter_Finish(): clear writer->buffer,
so _PyUnicodeWriter_Dealloc() can be called on the writer after finish.

files:
  Objects/unicodeobject.c |  7 +++++--
  1 files changed, 5 insertions(+), 2 deletions(-)


diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -13157,6 +13157,7 @@
 PyObject *
 _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer)
 {
+    PyObject *str;
     if (writer->pos == 0) {
         Py_XDECREF(writer->buffer);
         _Py_RETURN_UNICODE_EMPTY();
@@ -13174,8 +13175,10 @@
         }
         writer->buffer = newbuffer;
     }
-    assert(_PyUnicode_CheckConsistency(writer->buffer, 1));
-    return unicode_result_ready(writer->buffer);
+    str = writer->buffer;
+    writer->buffer = NULL;
+    assert(_PyUnicode_CheckConsistency(str, 1));
+    return unicode_result_ready(str);
 }
 
 void

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Mon Jul  8 23:33:54 2013
From: python-checkins at python.org (ned.deily)
Date: Mon,  8 Jul 2013 23:33:54 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Avoid_spurious_non-fatal_i?=
 =?utf-8?q?nstall_errors_for_OS_X_frameworks=3A?=
Message-ID: <3bq0KG2wngzQkJ@mail.python.org>

http://hg.python.org/cpython/rev/c505ece63c80
changeset:   84516:c505ece63c80
user:        Ned Deily <nad at acm.org>
date:        Mon Jul 08 14:33:03 2013 -0700
summary:
  Avoid spurious non-fatal install errors for OS X frameworks:
for a framework install, the python shared library is installed in
the frameworkinstallstructure target, not in altbininstall.

files:
  Makefile.pre.in |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -945,7 +945,7 @@
 		fi; \
 		(cd $(DESTDIR)$(BINDIR); $(LN) python$(LDVERSION)$(EXE) python$(VERSION)$(EXE)); \
 	fi
-	if test -f $(LDLIBRARY); then \
+	if test -f $(LDLIBRARY) && test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \
 		if test -n "$(DLLLIBRARY)" ; then \
 			$(INSTALL_SHARED) $(DLLLIBRARY) $(DESTDIR)$(BINDIR); \
 		else \

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  9 00:53:47 2013
From: python-checkins at python.org (victor.stinner)
Date: Tue,  9 Jul 2013 00:53:47 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_Conv?=
 =?utf-8?q?Param=28=29_of_the_ctypes_module_to_handle_paramfunc_failure?=
Message-ID: <3bq25R4zdhz7LkP@mail.python.org>

http://hg.python.org/cpython/rev/ba79f6a86300
changeset:   84517:ba79f6a86300
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Tue Jul 09 00:27:12 2013 +0200
summary:
  Issue #18408: Fix ConvParam() of the ctypes module to handle paramfunc failure
(MemoryError).

files:
  Modules/_ctypes/callproc.c |  2 ++
  1 files changed, 2 insertions(+), 0 deletions(-)


diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
--- a/Modules/_ctypes/callproc.c
+++ b/Modules/_ctypes/callproc.c
@@ -620,6 +620,8 @@
         assert(dict->paramfunc);
         /* If it has an stgdict, it is a CDataObject */
         carg = dict->paramfunc((CDataObject *)obj);
+        if (carg == NULL)
+            return -1;
         pa->ffi_type = carg->pffi_type;
         memcpy(&pa->value, &carg->value, sizeof(pa->value));
         pa->keep = (PyObject *)carg;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  9 00:53:49 2013
From: python-checkins at python.org (victor.stinner)
Date: Tue,  9 Jul 2013 00:53:49 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_zlib?=
 =?utf-8?q?=2Ecompressobj=28=29_to_handle_PyThread=5Fallocate=5Flock=28=29?=
 =?utf-8?q?_failure?=
Message-ID: <3bq25T05jPz7LkV@mail.python.org>

http://hg.python.org/cpython/rev/2ef2edfd1a4c
changeset:   84518:2ef2edfd1a4c
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Tue Jul 09 00:29:03 2013 +0200
summary:
  Issue #18408: Fix zlib.compressobj() to handle PyThread_allocate_lock() failure
(MemoryError).

files:
  Modules/zlibmodule.c |  4 ++++
  1 files changed, 4 insertions(+), 0 deletions(-)


diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c
--- a/Modules/zlibmodule.c
+++ b/Modules/zlibmodule.c
@@ -132,6 +132,10 @@
     }
 #ifdef WITH_THREAD
     self->lock = PyThread_allocate_lock();
+    if (self->lock == NULL) {
+        PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock");
+        return NULL;
+    }
 #endif
     return self;
 }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  9 00:53:50 2013
From: python-checkins at python.org (victor.stinner)
Date: Tue,  9 Jul 2013 00:53:50 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_PyCo?=
 =?utf-8?q?de=5FOptimize=28=29=3A_raise_a_MemoryError_on_memory_allocation?=
Message-ID: <3bq25V29Fjz7Lk7@mail.python.org>

http://hg.python.org/cpython/rev/a45407fa4a5b
changeset:   84519:a45407fa4a5b
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Tue Jul 09 00:32:04 2013 +0200
summary:
  Issue #18408: Fix PyCode_Optimize(): raise a MemoryError on memory allocation
failure.

files:
  Python/peephole.c |  8 ++++++--
  1 files changed, 6 insertions(+), 2 deletions(-)


diff --git a/Python/peephole.c b/Python/peephole.c
--- a/Python/peephole.c
+++ b/Python/peephole.c
@@ -381,8 +381,10 @@
 
     /* Make a modifiable copy of the code string */
     codestr = (unsigned char *)PyMem_Malloc(codelen);
-    if (codestr == NULL)
+    if (codestr == NULL) {
+        PyErr_NoMemory();
         goto exitError;
+    }
     codestr = (unsigned char *)memcpy(codestr,
                                       PyBytes_AS_STRING(code), codelen);
 
@@ -396,8 +398,10 @@
 
     /* Mapping to new jump targets after NOPs are removed */
     addrmap = (int *)PyMem_Malloc(codelen * sizeof(int));
-    if (addrmap == NULL)
+    if (addrmap == NULL) {
+        PyErr_NoMemory();
         goto exitError;
+    }
 
     blocks = markblocks(codestr, codelen);
     if (blocks == NULL)

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  9 00:53:51 2013
From: python-checkins at python.org (victor.stinner)
Date: Tue,  9 Jul 2013 00:53:51 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_usag?=
 =?utf-8?q?e_of_=5FPyBytes=5FResize=28=29?=
Message-ID: <3bq25W5TRHz7LkB@mail.python.org>

http://hg.python.org/cpython/rev/ed0c9d77e179
changeset:   84520:ed0c9d77e179
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Tue Jul 09 00:35:22 2013 +0200
summary:
  Issue #18408: Fix usage of _PyBytes_Resize()

_PyBytes_Resize(&v, new_size) sets v to NULL on error, so v cannot be used
anymore. Replace "Py_DECREF(v); v = NULL;" with "Py_CLEAR(v);".

files:
  Modules/binascii.c   |  23 ++++++++---------------
  Modules/zlibmodule.c |  24 ++++++++----------------
  2 files changed, 16 insertions(+), 31 deletions(-)


diff --git a/Modules/binascii.c b/Modules/binascii.c
--- a/Modules/binascii.c
+++ b/Modules/binascii.c
@@ -361,8 +361,7 @@
     if (_PyBytes_Resize(&rv,
                        (ascii_data -
                         (unsigned char *)PyBytes_AS_STRING(rv))) < 0) {
-        Py_DECREF(rv);
-        rv = NULL;
+        Py_CLEAR(rv);
     }
     PyBuffer_Release(&pbin);
     return rv;
@@ -491,8 +490,7 @@
     */
     if (bin_len > 0) {
         if (_PyBytes_Resize(&rv, bin_len) < 0) {
-            Py_DECREF(rv);
-            rv = NULL;
+            Py_CLEAR(rv);
         }
     }
     else {
@@ -563,8 +561,7 @@
     if (_PyBytes_Resize(&rv,
                        (ascii_data -
                         (unsigned char *)PyBytes_AS_STRING(rv))) < 0) {
-        Py_DECREF(rv);
-        rv = NULL;
+        Py_CLEAR(rv);
     }
     PyBuffer_Release(&pbuf);
     return rv;
@@ -642,8 +639,7 @@
     if (_PyBytes_Resize(&rv,
                        (bin_data -
                         (unsigned char *)PyBytes_AS_STRING(rv))) < 0) {
-        Py_DECREF(rv);
-        rv = NULL;
+        Py_CLEAR(rv);
     }
     if (rv) {
         PyObject *rrv = Py_BuildValue("Oi", rv, done);
@@ -713,8 +709,7 @@
     if (_PyBytes_Resize(&rv,
                        (out_data -
                         (unsigned char *)PyBytes_AS_STRING(rv))) < 0) {
-        Py_DECREF(rv);
-        rv = NULL;
+        Py_CLEAR(rv);
     }
     PyBuffer_Release(&pbuf);
     return rv;
@@ -770,8 +765,7 @@
     if (_PyBytes_Resize(&rv,
                        (ascii_data -
                         (unsigned char *)PyBytes_AS_STRING(rv))) < 0) {
-        Py_DECREF(rv);
-        rv = NULL;
+        Py_CLEAR(rv);
     }
     PyBuffer_Release(&pbin);
     return rv;
@@ -834,7 +828,7 @@
              if ( --out_len_left < 0 ) { \
                       if ( out_len > PY_SSIZE_T_MAX / 2) return PyErr_NoMemory(); \
                       if (_PyBytes_Resize(&rv, 2*out_len) < 0) \
-                        { Py_DECREF(rv); PyBuffer_Release(&pin); return NULL; } \
+                        { Py_XDECREF(rv); PyBuffer_Release(&pin); return NULL; } \
                       out_data = (unsigned char *)PyBytes_AS_STRING(rv) \
                                                              + out_len; \
                       out_len_left = out_len-1; \
@@ -887,8 +881,7 @@
     if (_PyBytes_Resize(&rv,
                        (out_data -
                         (unsigned char *)PyBytes_AS_STRING(rv))) < 0) {
-        Py_DECREF(rv);
-        rv = NULL;
+        Py_CLEAR(rv);
     }
     PyBuffer_Release(&pin);
     return rv;
diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c
--- a/Modules/zlibmodule.c
+++ b/Modules/zlibmodule.c
@@ -549,8 +549,7 @@
        so extend the output buffer and try again */
     while (err == Z_OK && self->zst.avail_out == 0) {
         if (_PyBytes_Resize(&RetVal, length << 1) < 0) {
-            Py_DECREF(RetVal);
-            RetVal = NULL;
+            Py_CLEAR(RetVal);
             goto error;
         }
         self->zst.next_out =
@@ -574,8 +573,7 @@
         goto error;
     }
     if (_PyBytes_Resize(&RetVal, self->zst.total_out - start_total_out) < 0) {
-        Py_DECREF(RetVal);
-        RetVal = NULL;
+        Py_CLEAR(RetVal);
     }
 
  error:
@@ -722,8 +720,7 @@
             length = max_length;
 
         if (_PyBytes_Resize(&RetVal, length) < 0) {
-            Py_DECREF(RetVal);
-            RetVal = NULL;
+            Py_CLEAR(RetVal);
             goto error;
         }
         self->zst.next_out =
@@ -757,8 +754,7 @@
     }
 
     if (_PyBytes_Resize(&RetVal, self->zst.total_out - start_total_out) < 0) {
-        Py_DECREF(RetVal);
-        RetVal = NULL;
+        Py_CLEAR(RetVal);
     }
 
  error:
@@ -811,8 +807,7 @@
        so extend the output buffer and try again */
     while (err == Z_OK && self->zst.avail_out == 0) {
         if (_PyBytes_Resize(&RetVal, length << 1) < 0) {
-            Py_DECREF(RetVal);
-            RetVal = NULL;
+            Py_CLEAR(RetVal);
             goto error;
         }
         self->zst.next_out =
@@ -851,8 +846,7 @@
     }
 
     if (_PyBytes_Resize(&RetVal, self->zst.total_out - start_total_out) < 0) {
-        Py_DECREF(RetVal);
-        RetVal = NULL;
+        Py_CLEAR(RetVal);
     }
 
  error:
@@ -1012,8 +1006,7 @@
        so extend the output buffer and try again */
     while ((err == Z_OK || err == Z_BUF_ERROR) && self->zst.avail_out == 0) {
         if (_PyBytes_Resize(&retval, length << 1) < 0) {
-            Py_DECREF(retval);
-            retval = NULL;
+            Py_CLEAR(retval);
             goto error;
         }
         self->zst.next_out = (Byte *)PyBytes_AS_STRING(retval) + length;
@@ -1045,8 +1038,7 @@
     }
 
     if (_PyBytes_Resize(&retval, self->zst.total_out - start_total_out) < 0) {
-        Py_DECREF(retval);
-        retval = NULL;
+        Py_CLEAR(retval);
     }
 
 error:

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  9 00:53:53 2013
From: python-checkins at python.org (victor.stinner)
Date: Tue,  9 Jul 2013 00:53:53 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_=5FPyUni?=
 =?utf-8?q?codeWriter=5FFinish=28=29_now_clears_its_buffer_attribute_in_al?=
 =?utf-8?q?l?=
Message-ID: <3bq25Y0cSQz7Ljk@mail.python.org>

http://hg.python.org/cpython/rev/d605c6b8095c
changeset:   84521:d605c6b8095c
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Tue Jul 09 00:37:24 2013 +0200
summary:
  Issue #18408: _PyUnicodeWriter_Finish() now clears its buffer attribute in all
cases, so _PyUnicodeWriter_Dealloc() can be called after finish.

files:
  Objects/unicodeobject.c |  9 ++++++---
  1 files changed, 6 insertions(+), 3 deletions(-)


diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -13159,18 +13159,21 @@
 {
     PyObject *str;
     if (writer->pos == 0) {
-        Py_XDECREF(writer->buffer);
+        Py_CLEAR(writer->buffer);
         _Py_RETURN_UNICODE_EMPTY();
     }
     if (writer->readonly) {
-        assert(PyUnicode_GET_LENGTH(writer->buffer) == writer->pos);
-        return writer->buffer;
+        str = writer->buffer;
+        writer->buffer = NULL;
+        assert(PyUnicode_GET_LENGTH(str) == writer->pos);
+        return str;
     }
     if (PyUnicode_GET_LENGTH(writer->buffer) != writer->pos) {
         PyObject *newbuffer;
         newbuffer = resize_compact(writer->buffer, writer->pos);
         if (newbuffer == NULL) {
             Py_DECREF(writer->buffer);
+            writer->buffer = NULL;
             return NULL;
         }
         writer->buffer = newbuffer;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  9 00:53:54 2013
From: python-checkins at python.org (victor.stinner)
Date: Tue,  9 Jul 2013 00:53:54 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_=5FP?=
	=?utf-8?q?yMem=5FDebugRealloc=28=29?=
Message-ID: <3bq25Z2sKvz7Ljp@mail.python.org>

http://hg.python.org/cpython/rev/549d8d3297f2
changeset:   84522:549d8d3297f2
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Tue Jul 09 00:44:43 2013 +0200
summary:
  Issue #18408: Fix _PyMem_DebugRealloc()

Don't mark old extra memory dead before calling realloc(). realloc() can fail
and realloc() must not touch the original buffer on failure.

So mark old extra memory dead only on success if the new buffer did not move
(has the same address).

files:
  Objects/obmalloc.c |  14 ++++++++------
  1 files changed, 8 insertions(+), 6 deletions(-)


diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -1780,7 +1780,7 @@
 _PyMem_DebugRealloc(void *ctx, void *p, size_t nbytes)
 {
     debug_alloc_api_t *api = (debug_alloc_api_t *)ctx;
-    uchar *q = (uchar *)p;
+    uchar *q = (uchar *)p, *oldq;
     uchar *tail;
     size_t total;       /* nbytes + 4*SST */
     size_t original_nbytes;
@@ -1797,24 +1797,26 @@
         /* overflow:  can't represent total as a size_t */
         return NULL;
 
-    if (nbytes < original_nbytes) {
-        /* shrinking:  mark old extra memory dead */
-        memset(q + nbytes, DEADBYTE, original_nbytes - nbytes + 2*SST);
-    }
-
     /* Resize and add decorations. We may get a new pointer here, in which
      * case we didn't get the chance to mark the old memory with DEADBYTE,
      * but we live with that.
      */
+    oldq = q;
     q = (uchar *)api->alloc.realloc(api->alloc.ctx, q - 2*SST, total);
     if (q == NULL)
         return NULL;
 
+    if (q == oldq && nbytes < original_nbytes) {
+        /* shrinking:  mark old extra memory dead */
+        memset(q + nbytes, DEADBYTE, original_nbytes - nbytes);
+    }
+
     write_size_t(q, nbytes);
     assert(q[SST] == (uchar)api->api_id);
     for (i = 1; i < SST; ++i)
         assert(q[SST + i] == FORBIDDENBYTE);
     q += 2*SST;
+
     tail = q + nbytes;
     memset(tail, FORBIDDENBYTE, SST);
     write_size_t(tail + SST, serialno);

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  9 00:53:55 2013
From: python-checkins at python.org (victor.stinner)
Date: Tue,  9 Jul 2013 00:53:55 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_sele?=
 =?utf-8?q?ct=2Eselect=28=29_to_handle_PyList=5FNew=28=29_failure_=28Memor?=
 =?utf-8?q?yError=29?=
Message-ID: <3bq25b51znz7Ljh@mail.python.org>

http://hg.python.org/cpython/rev/c91e7f707562
changeset:   84523:c91e7f707562
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Tue Jul 09 00:49:03 2013 +0200
summary:
  Issue #18408: Fix select.select() to handle PyList_New() failure (MemoryError)
in set2list()

files:
  Modules/selectmodule.c |  6 +++---
  1 files changed, 3 insertions(+), 3 deletions(-)


diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c
--- a/Modules/selectmodule.c
+++ b/Modules/selectmodule.c
@@ -299,9 +299,9 @@
         else
             ret = PyTuple_Pack(3, ifdlist, ofdlist, efdlist);
 
-        Py_DECREF(ifdlist);
-        Py_DECREF(ofdlist);
-        Py_DECREF(efdlist);
+        Py_XDECREF(ifdlist);
+        Py_XDECREF(ofdlist);
+        Py_XDECREF(efdlist);
     }
 
   finally:

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  9 04:29:44 2013
From: python-checkins at python.org (richard.jones)
Date: Tue,  9 Jul 2013 04:29:44 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_remove_setuptools_note=3B_set?=
 =?utf-8?q?uptools_will_now_be_installed?=
Message-ID: <3bq6tc5TkJz7Ljk@mail.python.org>

http://hg.python.org/peps/rev/11458f6dc9f7
changeset:   4986:11458f6dc9f7
parent:      4924:0a051a14592b
user:        Richard Jones <richard at mechanicalcat.net>
date:        Tue Jul 09 11:20:06 2013 +1000
summary:
  remove setuptools note; setuptools will now be installed
mention bootstrapping target and command-line options
mention python 2.6+ bootstrap possibility

files:
  pep-0439.txt |  102 ++++++++++++++++++--------------------
  1 files changed, 49 insertions(+), 53 deletions(-)


diff --git a/pep-0439.txt b/pep-0439.txt
--- a/pep-0439.txt
+++ b/pep-0439.txt
@@ -54,9 +54,12 @@
 Proposal
 ========
 
-This proposal affects three components of packaging: `the pip bootstrap`_,
-`setuptools`_ and, thanks to easier package installation, `modifications to
-publishing packages`_.
+This proposal affects two components of packaging: `the pip bootstrap`_ and,
+thanks to easier package installation, `modifications to publishing
+packages`_.
+
+The core of this proposal is that the user experience of using pip should not
+require the user to install pip.
 
 
 The pip bootstrap
@@ -65,9 +68,11 @@
 The Python installation includes an executable called "pip3" (see PEP 394 for
 naming rationale etc.) that attempts to import pip machinery.  If it can then
 the pip command proceeds as normal.  If it cannot it will bootstrap pip by
-downloading the pip implementation wheel file.  Once installed, the pip
-command proceeds as normal. Once the bootstrap process is complete the "pip3"
-command is no longer the bootstrap but rather the full pip command.
+downloading the pip implementation and setuptools wheel files. Hereafter the
+installation of the "pip implementation" will imply installation of
+setuptools.  Once installed, the pip command proceeds as normal. Once the
+bootstrap process is complete the "pip3" command is no longer the bootstrap
+but rather the full pip command.
 
 A boostrap is used in the place of a the full pip code so that we don't have
 to bundle pip and also pip is upgradeable outside of the regular Python
@@ -89,15 +94,16 @@
 2. The user will invoke a pip command, typically "pip3 install
    <package>", for example "pip3 install Django".
 3. The boostrap script will attempt to import the pip implementation.
-   If this succeeds, the pip command is processed normally.
+   If this succeeds, the pip command is processed normally. Stop.
 4. On failing to import the pip implementation the bootstrap notifies
-   the user that it is "upgrading pip" and contacts PyPI to obtain the
-   latest download wheel file (see PEP 427.)
-5. Upon downloading the file it is installed using the distlib
-   installation machinery for wheel packages.  Upon completing the
-   installation the user is notified that "pip3 has been upgraded."
-   TODO how is it verified?
-6. The pip tool may now import the pip implementation and continues to
+   the user that it needs to "install pip". It will ask the user whether it
+   should install pip as a system-wide site-packages or as a user-only
+   package. This choice will also be present as a command-line option to pip
+   so non-interactive use is possible.
+5. The bootstrap will and contact PyPI to obtain the latest download wheel
+   file (see PEP 427.)
+6. Upon downloading the file it is installed using "python setup.py install".
+7. The pip tool may now import the pip implementation and continues to
    process the requested user command normally.
 
 Users may be running in an environment which cannot access the public
@@ -109,7 +115,7 @@
    additional locations to discover packages and attempting to
    download the package from those locations.
 2. If the package is not found there then we attempt to donwload it
-   using the standard "https://pypi.python.org/pypi/simple/pip" index.
+   using the standard "https://pypi.python.org/simple/pip/" index.
 3. If that also fails, for any reason, we indicate to the user the
    operation we were attempting, the reason for failure (if we know
    it) and display further instructions for downloading and installing
@@ -118,52 +124,33 @@
 Some users may have no Internet access suitable for fetching the pip
 implementation file. Manual installation of the pip implementation will be
 supported through the manual download of the wheel file and "pip3 install
-<downloaded wheel file>". This installation - since it uses only the bootstrap
-code - will not perform standard pip installation steps of saving the file to
-a cache directory or updating any local database of installed files.
+<downloaded wheel file>".
 
-The download of the pip implementation install file should be performed
-securely.  The transport from pypi.python.org will be done over HTTPS but the CA
-certificate check will most likely not be performed, and therefore the download
-would still be vulnerable to active MITM attacks.  To mitigate this
-risk we will use the embedded signature support in the wheel format to validate
-the downloaded file.
+The download of the pip implementation install file will be performed
+securely.  The transport from pypi.python.org will be done over HTTPS with the
+CA certificate check performed (see PEP XXXX).
 
 Beyond those arguments controlling index location and download
 options, the "pip3" boostrap command may support further standard pip
 options for verbosity, quietness and logging.
 
+The "pip3" command will support two new command-line options that are used
+in the boostrapping, and otherwise ignored. They control where the pip
+implementation is installed:
+
+--bootstrap
+  Install to the user's packages directory. The name of this option is chosen
+  to promote it as the preferred installation option.
+
+--bootstrap-to-system
+  Install to the system site-packages directory.
+
+These command-line options will also need to be implemented, but otherwise
+ignored, in the pip implementation.
+
 The "--no-install" option to the "pip3" command will not affect the
 bootstrapping process.
 
-setuptools
-----------
-
-The deprecation of requiring setuptools for installation is an existing goal of
-the packaging comminity (TODO ref needed). Currently pip depends upon setuptools
-functionality, and it is installed by the current pip boostrap. This PEP does
-not propose installing setuptools during the new bootstrap.
-
-It is intended that before Python 3.4 is shipped the functionlity required by
-pip will be present in Python's standard library as the distlib module, and that
-pip would be modified to use that functionality when present. TODO PEP reference
-for distlib
-
-Many existing "setup.py" files require setuptools to be installed (because one
-of the first things they do is import setuptools). It is intended that pip's
-behaviour will be either:
-
-1. If setuptools is not present it can only install from wheel files and
-   sdists with 2.0+ metadata, or
-2. If setuptools is present it can also install from sdists with legacy
-   metadata and eggs
-
-By default, installing setuptools when necessary should be automatic so that
-users are not inconvenienced, but advanced users should be able to ask that it
-instead be treated as an error if no wheel is available to satisfy an
-installation request or dependency (so they don't inadvertently install
-setuptools on their production systems if they don't want to).
-
 
 Modifications to publishing packages
 ------------------------------------
@@ -189,7 +176,16 @@
 ==============
 
 The changes to pip required by this PEP are being tracked in that project's
-issue tracker [2]_
+issue tracker [2]_. Most notably, the addition of --bootstrap and --bootstrap-
+to-system to the pip command-line.
+
+The required code for this implementation is the "pip3" command described
+above. The additional pypublish can be developed outside of the scope of this
+PEP's work.
+
+Finally, it would be desirable that "pip3" be ported to Python 2.6+ to allow
+the single command to replace all existing pip/setuptools/distribute and
+possibly virtualenv bootstrap scripts.
 
 
 Risks

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Tue Jul  9 04:29:46 2013
From: python-checkins at python.org (richard.jones)
Date: Tue,  9 Jul 2013 04:29:46 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_mention_having_pip_default_to?=
 =?utf-8?q?_--user_when_itself_installed_in_=7E/=2Elocal?=
Message-ID: <3bq6tf0PM5z7Lk8@mail.python.org>

http://hg.python.org/peps/rev/08a91307085f
changeset:   4987:08a91307085f
user:        Richard Jones <richard at mechanicalcat.net>
date:        Tue Jul 09 12:23:12 2013 +1000
summary:
  mention having pip default to --user when itself installed in ~/.local
remove consideration of support for unnecessary installation options

files:
  pep-0439.txt |  22 ++++++++--------------
  1 files changed, 8 insertions(+), 14 deletions(-)


diff --git a/pep-0439.txt b/pep-0439.txt
--- a/pep-0439.txt
+++ b/pep-0439.txt
@@ -109,22 +109,13 @@
 Users may be running in an environment which cannot access the public
 Internet and are relying solely on a local package repository.  They
 would use the "-i" (Base URL of Python Package Index) argument to the
-"pip3 install" command.  This use case will be handled by:
-
-1. Recognising the command-line arguments that specify alternative or
-   additional locations to discover packages and attempting to
-   download the package from those locations.
-2. If the package is not found there then we attempt to donwload it
-   using the standard "https://pypi.python.org/simple/pip/" index.
-3. If that also fails, for any reason, we indicate to the user the
-   operation we were attempting, the reason for failure (if we know
-   it) and display further instructions for downloading and installing
-   the file manually.
+"pip3 install" command. This simply overrides the default index URL pointing
+to PyPI.
 
 Some users may have no Internet access suitable for fetching the pip
-implementation file. Manual installation of the pip implementation will be
-supported through the manual download of the wheel file and "pip3 install
-<downloaded wheel file>".
+implementation file. These users can manually download and install the
+setuptools and pip tar files. Adding specific support for this use-case is
+unnecessary.
 
 The download of the pip implementation install file will be performed
 securely.  The transport from pypi.python.org will be done over HTTPS with the
@@ -148,6 +139,9 @@
 These command-line options will also need to be implemented, but otherwise
 ignored, in the pip implementation.
 
+Consideration should be given to defaulting pip to install packages to the
+user's packages directory if pip is installed in that location.
+
 The "--no-install" option to the "pip3" command will not affect the
 bootstrapping process.
 

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Tue Jul  9 04:29:48 2013
From: python-checkins at python.org (richard.jones)
Date: Tue,  9 Jul 2013 04:29:48 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps_=28merge_default_-=3E_default=29?=
	=?utf-8?q?=3A_merge?=
Message-ID: <3bq6th2sx6z7Lkv@mail.python.org>

http://hg.python.org/peps/rev/43d441b2b2ea
changeset:   4988:43d441b2b2ea
parent:      4987:08a91307085f
parent:      4985:e91b098137c5
user:        Richard Jones <richard at mechanicalcat.net>
date:        Tue Jul 09 12:29:13 2013 +1000
summary:
  merge

files:
  pep-0008.txt                |    18 +-
  pep-0315.txt                |    23 +-
  pep-0426.txt                |  1999 ++++++++++++++--------
  pep-0426/pymeta-schema.json |   249 ++
  pep-0435.txt                |     6 +-
  pep-0440.txt                |   325 ++-
  pep-0442.txt                |     7 +-
  pep-0443.txt                |    75 +-
  pep-0445.txt                |   773 ++++++++
  pep-0446.txt                |   242 ++
  10 files changed, 2759 insertions(+), 958 deletions(-)


diff --git a/pep-0008.txt b/pep-0008.txt
--- a/pep-0008.txt
+++ b/pep-0008.txt
@@ -158,9 +158,21 @@
 line continuation inside parentheses, brackets and braces.  Long lines
 can be broken over multiple lines by wrapping expressions in
 parentheses. These should be used in preference to using a backslash
-for line continuation.  Make sure to indent the continued line
-appropriately.  The preferred place to break around a binary operator
-is *after* the operator, not before it.  Some examples::
+for line continuation.
+
+Backslashes may still be appropriate at times.  For example, long,
+multiple ``with``-statements cannot use implicit continuation, so
+backslashes are acceptable::
+
+    with open('/path/to/some/file/you/want/to/read') as file_1, \
+            open('/path/to/some/file/being/written', 'w') as file_2:
+        file_2.write(file_1.read())
+
+Another such case is with ``assert`` statements.
+
+Make sure to indent the continued line appropriately.  The preferred
+place to break around a binary operator is *after* the operator, not
+before it.  Some examples::
 
     class Rectangle(Blob):
 
diff --git a/pep-0315.txt b/pep-0315.txt
--- a/pep-0315.txt
+++ b/pep-0315.txt
@@ -4,7 +4,7 @@
 Last-Modified: $Date$
 Author: Raymond Hettinger <python at rcn.com>
         W Isaac Carroll <icarroll at pobox.com>
-Status: Deferred
+Status: Rejected
 Type: Standards Track
 Content-Type: text/plain
 Created: 25-Apr-2003
@@ -21,19 +21,32 @@
 
 Notice
 
-    Deferred; see
+    Rejected; see
+    http://mail.python.org/pipermail/python-ideas/2013-June/021610.html
+
+    This PEP has been deferred since 2006; see
     http://mail.python.org/pipermail/python-dev/2006-February/060718.html
 
     Subsequent efforts to revive the PEP in April 2009 did not
     meet with success because no syntax emerged that could
-    compete with a while-True and an inner if-break.
+    compete with the following form:
 
-    A syntax was found for a basic do-while loop but it found
-    had little support because the condition was at the top:
+        while True:
+            <setup code>
+            if not <condition>:
+                break
+            <loop body>
+
+    A syntax alternative to the one proposed in the PEP was found for
+    a basic do-while loop but it gained little support because the
+    condition was at the top:
 
         do ... while <cond>:
             <loop body>
 
+    Users of the language are advised to use the while-True form with
+    an inner if-break when a do-while loop would have been appropriate.
+
 
 Motivation
 
diff --git a/pep-0426.txt b/pep-0426.txt
--- a/pep-0426.txt
+++ b/pep-0426.txt
@@ -12,7 +12,8 @@
 Content-Type: text/x-rst
 Requires: 440
 Created: 30 Aug 2012
-Post-History: 14 Nov 2012, 5 Feb 2013, 7 Feb 2013, 9 Feb 2013, 27-May-2013
+Post-History: 14 Nov 2012, 5 Feb 2013, 7 Feb 2013, 9 Feb 2013,
+              27 May 2013, 20 Jun 2013, 23 Jun 2013
 Replaces: 345
 
 
@@ -21,8 +22,7 @@
 
 This PEP describes a mechanism for publishing and exchanging metadata
 related to Python distributions. It includes specifics of the field names,
-and their semantics and
-usage.
+and their semantics and usage.
 
 This document specifies version 2.0 of the metadata format.
 Version 1.0 is specified in PEP 241.
@@ -42,7 +42,9 @@
 
    "I" in this doc refers to Nick Coghlan. Daniel and Donald either wrote or
    contributed to earlier versions, and have been providing feedback as this
-   initial draft of the JSON-based rewrite has taken shape.
+   JSON-based rewrite has taken shape. Daniel and Donald have also been
+   vetting the proposal as we go to ensure it is practical to implement for
+   both clients and index servers.
 
    Metadata 2.0 represents a major upgrade to the Python packaging ecosystem,
    and attempts to incorporate experience gained over the 15 years(!) since
@@ -63,8 +65,7 @@
    * a new PEP to define v2.0 of the sdist format
    * an updated wheel PEP (v1.1) to add pymeta.json
    * an updated installation database PEP both for pymeta.json and to add
-     a linking scheme to better support runtime selection of dependencies,
-     as well as recording which extras are currently available
+     a linking scheme to better support runtime selection of dependencies
    * a new static config PEP to standardise metadata generation and
      creation of sdists
    * PEP 439, covering a bootstrapping mechanism for ``pip``
@@ -83,138 +84,241 @@
    "rationale" section at the end of the document, as it would otherwise be
    an irrelevant distraction for future readers.
 
-
-Definitions
-===========
+Purpose
+=======
+
+The purpose of this PEP is to define a common metadata interchange format
+for communication between software publication tools and software integration
+tools in the Python ecosystem. One key aim is to support full dependency
+analysis in that ecosystem without requiring the execution of arbitrary
+Python code by those doing the analysis. Another aim is to encourage good
+software distribution practices by default, while continuing to support the
+current practices of almost all existing users of the Python Package Index
+(both publishers and integrators).
+
+The design draws on the Python community's 15 years of experience with
+distutils based software distribution, and incorporates ideas and concepts
+from other distribution systems, including Python's setuptools, pip and
+other projects, Ruby's gems, Perl's CPAN, Node.js's npm, PHP's composer
+and Linux packaging systems such as RPM and APT.
+
+
+Development, Distribution and Deployment of Python Software
+===========================================================
+
+The metadata design in this PEP is based on a particular conceptual model
+of the software development and distribution process. This model consists of
+the following phases:
+
+* Software development: this phase involves working with a source checkout
+  for a particular application to add features and fix bugs. It is
+  expected that developers in this phase will need to be able to build the
+  software, run the software's automated test suite, run project specific
+  utility scripts and publish the software.
+
+* Software publication: this phase involves taking the developed software
+  and making it available for use by software integrators. This includes
+  creating the descriptive metadata defined in this PEP, as well making the
+  software available (typically by uploading it to an index server).
+
+* Software integration: this phase involves taking published software
+  components and combining them into a coherent, integrated system. This
+  may be done directly using Python specific cross-platform tools, or it may
+  be handled through conversion to development language neutral platform
+  specific packaging systems.
+
+* Software deployment: this phase involves taking integrated software
+  components and deploying them on to the target system where the software
+  will actually execute.
+
+The publication and integration phases are collectively referred to as
+the distribution phase, and the individual software components distributed
+in that phase are referred to as "distributions".
+
+The exact details of these phases will vary greatly for particular use cases.
+Deploying a web application to a public Platform-as-a-Service provider,
+publishing a new release of a web framework or scientific library,
+creating an integrated Linux distribution or upgrading a custom application
+running in a secure enclave are all situations this metadata design should
+be able to handle.
+
+The complexity of the metadata described in this PEP thus arises directly
+from the actual complexities associated with software development,
+distribution and deployment in a wide range of scenarios.
+
+
+Supporting definitions
+----------------------
 
 The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
 "SHOULD", "SHOULD NOT", "RECOMMENDED",  "MAY", and "OPTIONAL" in this
 document are to be interpreted as described in RFC 2119.
 
-"Distributions" are deployable software components published through an index
-server or otherwise made available for installation.
-
-"Versions" are uniquely identified snapshots of a distribution.
-
-"Distribution archives" are the packaged files which are used to publish
-and distribute the software.
-
-"Source archives" require build tools to be available on the target
-system.
+"Projects" are software components that are made available for integration.
+Projects include Python libraries, frameworks, scripts, plugins,
+applications, collections of data or other resources, and various
+combinations thereof. Public Python projects are typically registered on
+the `Python Package Index`_.
+
+"Releases" are uniquely identified snapshots of a project.
+
+"Distributions" are the packaged files which are used to publish
+and distribute a release.
+
+"Source archive" and "VCS checkout" both refer to the raw source code for
+a release, prior to creation of an sdist or binary archive.
+
+An "sdist" is a publication format providing the distribution metadata and
+and any source files that are essential to creating a binary archive for
+the distribution. Creating a binary archive from an sdist requires that
+the appropriate build tools be available on the system.
 
 "Binary archives" only require that prebuilt files be moved to the correct
 location on the target system. As Python is a dynamically bound
-cross-platform language, many "binary" archives will contain only pure
-Python source code.
+cross-platform language, many so-called "binary" archives will contain only
+pure Python source code.
+
+"Contributors" are individuals and organizations that work together to
+develop a software component.
+
+"Publishers" are individuals and organizations that make software components
+available for integration (typically by uploading distributions to an
+index server)
+
+"Integrators" are individuals and organizations that incorporate published
+distributions as components of an application or larger system.
 
 "Build tools" are automated tools intended to run on development systems,
 producing source and binary distribution archives. Build tools may also be
-invoked by installation tools in order to install software distributed as
-source archives rather than prebuilt binary archives.
+invoked by integration tools in order to build software distributed as
+sdists rather than prebuilt binary archives.
 
 "Index servers" are active distribution registries which publish version and
 dependency metadata and place constraints on the permitted metadata.
 
+"Public index servers" are index servers which allow distribution uploads
+from untrusted third parties. The `Python Package Index`_ is a public index
+server.
+
 "Publication tools" are automated tools intended to run on development
 systems and upload source and binary distribution archives to index servers.
 
-"Installation tools" are automated tools intended to run on production
-systems, consuming source and binary distribution archives from an index
-server or other designated location and deploying them to the target system.
+"Integration tools" are automated tools that consume the metadata and
+distribution archives published by an index server or other designated
+source, and make use of them in some fashion, such as installing them or
+converting them to a platform specific packaging format.
+
+"Installation tools" are integration tools specifically intended to run on
+deployment targets, consuming source and binary distribution archives from
+an index server or other designated location and deploying them to the target
+system.
 
 "Automated tools" is a collective term covering build tools, index servers,
-publication tools, installation tools and any other software that produces
+publication tools, integration tools and any other software that produces
 or consumes distribution version and dependency metadata.
 
-"Projects" refers to the developers that manage the creation of a particular
-distribution.
-
 "Legacy metadata" refers to earlier versions of this metadata specification,
 along with the supporting metadata file formats defined by the
 ``setuptools`` project.
 
-
-Development and distribution activities
-=======================================
-
-Making effective use of a common metadata format requires a common
-understanding of the most complex development and distribution model
-the format is intended to support. The metadata format described in this
-PEP is based on the following activities:
-
-* Development: during development, a user is operating from a
-  source checkout (or equivalent) for the current project. Dependencies must
-  be available in order to build, test and create a source archive of the
-  distribution.
-
-  .. note::
-     As a generated file, the full distribution metadata often won't be
-     available in a raw source checkout or tarball. In such cases, the
-     relevant distribution metadata is generally obtained from another
-     location, such as the last published release, or by generating it
-     based on a command given in a standard input file. This spec
-     deliberately avoids handling that scenario, instead falling back on
-     the existing ``setup.py`` functionality.
-
-* Build: the build step is the process of turning a source archive into a
-  binary archive. Dependencies must be available in order to build and
-  create a binary archive of the distribution (including any documentation
-  that is installed on target systems).
-
-* Deployment: the deployment phase consists of two subphases:
-
-  * Installation: the installation phase involves getting the distribution
-    and all of its runtime dependencies onto the target system. In this
-    phase, the distribution may already be on the system (when upgrading or
-    reinstalling) or else it may be a completely new installation.
-
-  * Usage: the usage phase, also referred to as "runtime", is normal usage
-    of the distribution after it has been installed on the target system.
-
-The metadata format described in this PEP is designed to enable the
-following:
-
-* It should be practical to have separate development systems, build systems
-  and deployment systems.
-* It should be practical to install dependencies needed specifically to
-  build source archives only on development systems.
-* It should be practical to install dependencies needed specifically to
-  build the software only on development and build systems, as well as
-  optionally on deployment systems if installation from source archives
-  is needed.
-* It should be practical to install dependencies needed to run the
-  distribution only on development and deployment systems.
-* It should be practical to install the dependencies needed to run a
-  distribution's test suite only on development systems, as well as
-  optionally on deployment systems.
-* It should be practical for repackagers to separate out the build
-  dependencies needed to build the application itself from those required
-  to build its documentation (as the documentation often doesn't need to
-  be rebuilt when porting an application to a different platform).
-
-.. note::
-
-    This "most complex supported scenario" is almost *exactly* what has to
-    happen to get an upstream Python package into a Linux distribution, and
-    is why the current crop of automatic Python metadata -> Linux distro
-    metadata converters have some serious issues, at least from the point of
-    view of complying with distro packaging policies: the information
-    they need to comply with those policies isn't available from the
-    upstream projects, and all current formats for publishing it are
-    distro specific. This means either upstreams have to maintain metadata
-    for multiple distributions (which rarely happens) or else repackagers
-    have to do a lot of work manually in order to separate out these
-    dependencies in a  way that complies with those policies.
-
-    One thing this PEP aims to do is define a metadata format that at least
-    has the *potential* to provide the info repackagers need, thus allowing
-    upstream Python projects and Linux distro repackagers to collaborate more
-    effectively (and, ideally, make it possible to reliably automate
-    the process of converting upstream Python distributions into policy
-    compliant distro packages).
-
-    Some items in this section (and the contents of this note) will likely
-    end up moving down to the "Rationale for changes from PEP 345" section.
+"Entry points" are a scheme for identifying Python callables or other
+objects as strings consisting of a Python module name and a module
+attribute name, separated by a colon. For example: ``"test.regrtest:main"``.
+
+"Distros" is used as the preferred term for Linux distributions, to help
+avoid confusion with the Python-specific meaning of the term.
+
+
+Integration and deployment of distributions
+-------------------------------------------
+
+The primary purpose of the distribution metadata is to support integration
+and deployment of distributions as part of larger applications and systems.
+
+Integration and deployment can in turn be broken down into further substeps.
+
+* Build: the build step is the process of turning a VCS checkout, source
+  archive or sdist into a binary archive. Dependencies must be available
+  in order to build and create a binary archive of the distribution
+  (including any documentation that is installed on target systems).
+
+* Installation: the installation step involves getting the distribution
+  and all of its runtime dependencies onto the target system. In this
+  step, the distribution may already be on the system (when upgrading or
+  reinstalling) or else it may be a completely new installation.
+
+* Runtime: this is normal usage of a distribution after it has been
+  installed on the target system.
+
+These three steps may all occur directly on the target system. Alternatively
+the build step may be separated out by using binary archives provided by the
+publisher of the distribution, or by creating the binary archives on a
+separate system prior to deployment.
+
+The published metadata for distributions SHOULD allow integrators, with the
+aid of build and integration tools, to:
+
+* obtain the original source code that was used to create a distribution
+* identify and retrieve the dependencies (if any) required to use a
+  distribution
+* identify and retrieve the dependencies (if any) required to build a
+  distribution from source
+* identify and retrieve the dependencies (if any) required to run a
+  distribution's test suite
+* find resources on using and contributing to the project
+* access sufficiently rich metadata to support contacting distribution
+  publishers through appropriate channels, as well as finding distributions
+  that are relevant to particular problems
+
+
+Development and publication of distributions
+--------------------------------------------
+
+The secondary purpose of the distribution metadata is to support effective
+collaboration amongst software contributors and publishers during the
+development phase.
+
+The published metadata for distributions SHOULD allow contributors
+and publishers, with the aid of build and publication tools, to:
+
+* perform all the same activities needed to effectively integrate and
+  deploy the distribution
+* identify and retrieve the additional dependencies needed to develop and
+  publish the distribution
+* specify the dependencies (if any) required to use the distribution
+* specify the dependencies (if any) required to build the distribution
+  from source
+* specify the dependencies (if any) required to run the distribution's
+  test suite
+* specify the additional dependencies (if any) required to develop and
+  publish the distribution
+
+
+Standard build system
+---------------------
+
+Both development and integration of distributions relies on the ability to
+build extension modules and perform other operations in a distribution
+independent manner.
+
+The current iteration of the metadata relies on the
+``distutils``/``setuptools`` commands system to support these necessary
+development and integration activities:
+
+* ``python setup.py dist_info``: generate distribution metadata in place
+  given a source archive or VCS checkout
+* ``python setup.py sdist``: create an sdist from a source archive
+  or VCS checkout
+* ``python setup.py build_ext --inplace``: build extension modules in place
+  given an sdist, source archive or VCS checkout
+* ``python setup.py test``: run the distribution's test suite in place
+  given an sdist, source archive or VCS checkout
+* ``python setup.py bdist_wheel``: create a binary archive from an sdist,
+  source archive or VCS checkout
+
+Future iterations of the metadata and associated PEPs may aim to replace
+these ``distutils``/``setuptools`` dependent commands with build system
+independent entry points.
 
 
 Metadata format
@@ -247,14 +351,22 @@
 Automated tools MAY automatically derive valid values from other
 information sources (such as a version control system).
 
+Automated tools, especially public index servers, MAY impose additional
+length restrictions on metadata beyond those enumerated in this PEP. Such
+limits SHOULD be imposed where necessary to protect the integrity of a
+service, based on the available resources and the service provider's
+judgment of reasonable metadata capacity requirements.
+
 
 Metadata files
 --------------
 
 The information defined in this PEP is serialised to ``pymeta.json``
-files for some use cases. As indicated by the extension, these
-are JSON-encoded files. Each file consists of a single serialised mapping,
-with fields as described in this PEP.
+files for some use cases. These are files containing UTF-8 encoded JSON
+metadata.
+
+Each metadata file consists of a single serialised mapping, with fields as
+described in this PEP.
 
 There are three standard locations for these metadata files:
 
@@ -270,21 +382,16 @@
    These locations are to be confirmed, since they depend on the definition
    of sdist 2.0 and the revised installation database standard. There will
    also be a wheel 1.1 format update after this PEP is approved that
-   mandates 2.0+ metadata.
+   mandates provision of 2.0+ metadata.
 
 Other tools involved in Python distribution may also use this format.
 
-It is expected that these metadata files will be generated by build tools
-based on other input formats (such as ``setup.py``) rather than being
-edited by hand.
-
-.. note::
-
-    It may be appropriate to add a "./setup.py dist_info" command to
-    setuptools to allow just the sdist metadata files to be generated
-    without having to build the full sdist archive. This would be
-    similar to the existing "./setup.py egg_info" command in setuptools,
-    which would continue to emit the legacy metadata format.
+As JSON files are generally awkward to edit by hand, it is RECOMMENDED
+that these metadata files be generated by build tools based on other
+input formats (such as ``setup.py``) rather than being used directly as
+a data input format. Generating the metadata as part of the publication
+process also helps to deal with version specific fields (including the
+source URL and the version field itself).
 
 For backwards compatibility with older installation tools, metadata 2.0
 files MAY be distributed alongside legacy metadata.
@@ -292,6 +399,10 @@
 Index servers MAY allow distributions to be uploaded and installation tools
 MAY allow distributions to be installed with only legacy metadata.
 
+Automated tools MAY attempt to automatically translate legacy metadata to
+the format described in this PEP. Advice for doing so effectively is given
+in Appendix A.
+
 
 Essential dependency resolution metadata
 ----------------------------------------
@@ -304,13 +415,18 @@
 fields:
 
 * ``metadata_version``
+* ``generator``
 * ``name``
 * ``version``
-* ``build_label``
-* ``version_url``
+* ``source_label``
+* ``source_url``
 * ``extras``
-* ``requires``
-* ``may_require``
+* ``meta_requires``
+* ``meta_may_require``
+* ``run_requires``
+* ``run_may_require``
+* ``test_requires``
+* ``test_may_require``
 * ``build_requires``
 * ``build_may_require``
 * ``dev_requires``
@@ -320,23 +436,34 @@
 * ``supports_environments``
 
 When serialised to a file, the name used for this metadata set SHOULD
-be ``pymeta-minimal.json``.
-
-Abbreviated metadata
---------------------
-
-Some metadata fields have the potential to contain a lot of information
-that will rarely be referenced, greatly increasing storage requirements
-without providing significant benefits.
-
-The abbreviated metadata for a distribution consists of all fields
-*except* the following:
-
-* ``description``
-* ``contributors``
-
-When serialised to a file, the name used for this metadata set SHOULD
-be ``pymeta-short.json``.
+be ``pymeta-dependencies.json``.
+
+
+Included documents
+------------------
+
+Rather than being incorporated directly into the structured metadata, some
+supporting documents are included alongside the metadata file in the
+``dist-info`` metadata directory.
+
+To accommodate the variety of existing naming conventions for these files,
+they are explicitly identified in the ``document_names`` field, rather
+than expecting index servers and other automated tools to identify them
+automatically.
+
+
+Metadata validation
+-------------------
+
+A `jsonschema <https://pypi.python.org/pypi/jsonschema>`__ description of
+the distribution metadata is `available
+<http://hg.python.org/peps/file/default/pep-0426/pymeta-schema.json>`__.
+
+This schema does NOT currently handle validation of some of the more complex
+string fields (instead treating them as opaque strings).
+
+Except where otherwise noted, all URL fields in the metadata MUST comply
+with RFC 3986.
 
 
 Core metadata
@@ -376,6 +503,17 @@
     "metadata_version": "2.0"
 
 
+Generator
+---------
+
+Name (and optional version) of the program that generated the file,
+if any.  A manually produced file would omit this field.
+
+Example::
+
+    "generator": "setuptools (0.8)"
+
+
 Name
 ----
 
@@ -391,7 +529,7 @@
 * hyphens (``-``)
 * periods (``.``)
 
-Distributions named MUST start and end with an ASCII letter or digit.
+Distribution names MUST start and end with an ASCII letter or digit.
 
 Automated tools MUST reject non-compliant names.
 
@@ -399,14 +537,14 @@
 consider hyphens and underscores to be equivalent.
 
 Index servers MAY consider "confusable" characters (as defined by the
-Unicode Consortium in `TR39: Unicode Security Mechanisms <TR39>`__) to be
+Unicode Consortium in `TR39: Unicode Security Mechanisms <TR39>`_) to be
 equivalent.
 
 Index servers that permit arbitrary distribution name registrations from
 untrusted sources SHOULD consider confusable characters to be equivalent
 when registering new distributions (and hence reject them as duplicates).
 
-Installation tools MUST NOT silently accept a confusable alternate
+Integration tools MUST NOT silently accept a confusable alternate
 spelling as matching a requested distribution name.
 
 At time of writing, the characters in the ASCII subset designated as
@@ -421,45 +559,6 @@
 
     "name": "ComfyChair"
 
-.. note::
-
-    Debian doesn't actually permit underscores in names, but that seems
-    unduly restrictive for this spec given the common practice of using
-    valid Python identifiers as Python distribution names. A Debian side
-    policy of converting underscores to hyphens seems easy enough to
-    implement (and the requirement to consider hyphens and underscores as
-    equivalent ensures that doing so won't introduce any conflicts).
-
-    We're deliberately *not* following Python 3 down the path of arbitrary
-    unicode identifiers at this time. The security implications of doing so
-    are substantially worse in the software distribution use case (it opens
-    up far more interesting attack vectors than mere code obfuscation), the
-    existing tooling really only works properly if you abide by the stated
-    restrictions and changing it would require a *lot* of work for all
-    the automated tools in the chain.
-
-    PyPI has recently been updated to reject non-compliant names for newly
-    registered projects, but existing non-compliant names are still
-    tolerated when using legacy metadata formats. Affected distributions
-    will need to change their names (typically be replacing spaces with
-    hyphens) before they can migrate to the new metadata formats.
-
-    Donald Stufft ran an analysis, and the new restrictions impact less
-    than 230 projects out of the ~31k already on PyPI. This isn't that
-    surprising given the fact that many existing tools could already
-    exhibit odd behaviour when attempting to deal with non-compliant
-    names, implicitly discouraging the use of more exotic names.
-
-    Of those projects, ~200 have the only non-compliant character as an
-    internal space (e.g. "Twisted Web"). These will be automatically
-    migrated by replacing the spaces with hyphens (e.g. "Twisted-Web"),
-    which is what you have to actually type to install these distributions
-    with ``setuptools`` (which powers both ``easy_install`` and ``pip``).
-
-    The remaining ~30 will be investigated manually and decided upon on a
-    case by case basis how to migrate them to the new naming rules (in
-    consultation with the maintainers of those projects where possible).
-
 
 Version
 -------
@@ -469,11 +568,33 @@
 variety of flexible version specification mechanisms (see PEP 440 for
 details).
 
+Version identifiers MUST comply with the format defined in PEP 440.
+
+Version identifiers MUST be unique within each project.
+
 Example::
 
     "version": "1.0a2"
 
 
+Summary
+-------
+
+A short summary of what the distribution does.
+
+This field SHOULD contain fewer than 512 characters and MUST contain fewer
+than 2048.
+
+This field SHOULD NOT contain any line breaks.
+
+A more complete description SHOULD be included as a separate file in the
+sdist for the distribution. See `Document names`_ for details.
+
+Example::
+
+    "summary": "A module that is more fiendish than soft cushions."
+
+
 Source code metadata
 ====================
 
@@ -489,9 +610,13 @@
 ------------
 
 A constrained identifying text string, as defined in PEP 440. Source labels
-cannot be used in ordered version comparisons, but may be used to select
-an exact version (see PEP 440 for details).
-
+cannot be used in version specifiers - they are included for information
+purposes only.
+
+Source labels MUST meet the character restrictions defined in PEP 440.
+
+Source labels MUST be unique within each project and MUST NOT match any
+defined version for the project.
 
 Examples::
 
@@ -508,19 +633,23 @@
 ----------
 
 A string containing a full URL where the source for this specific version of
-the distribution can be downloaded.  (This means that the URL can't be
-something like ``"https://github.com/pypa/pip/archive/master.zip"``, but
-instead must be ``"https://github.com/pypa/pip/archive/1.3.1.zip"``.)
-
-Some appropriate targets for a source URL are a source tarball, an sdist
-archive or a direct reference to a tag or specific commit in an online
-version control system.
-
-All source URL references SHOULD either specify a secure transport
-mechanism (such as ``https``) or else include an expected hash value in the
-URL for verification purposes. If an insecure transport is specified without
-any hash information (or with hash information that the tool doesn't
-understand), automated tools SHOULD at least emit a warning and MAY
+the distribution can be downloaded.
+
+Source URLs MUST be unique within each project. This means that the URL
+can't be something like ``"https://github.com/pypa/pip/archive/master.zip"``,
+but instead must be ``"https://github.com/pypa/pip/archive/1.3.1.zip"``.
+
+The source URL MUST reference either a source archive or a tag or specific
+commit in an online version control system that permits creation of a
+suitable VCS checkout. It is intended primarily for integrators that
+wish to recreate the distribution from the original source form.
+
+All source URL references SHOULD specify a secure transport
+mechanism (such as ``https``), include an expected hash value in the
+URL for verification purposes, or both. If an insecure transport is specified
+without any hash information, with hash information that the tool doesn't
+understand, or with a selected hash algorithm that the tool considers too
+weak to trust, automated tools SHOULD at least emit a warning and MAY
 refuse to rely on the URL.
 
 It is RECOMMENDED that only hashes which are unconditionally provided by
@@ -530,7 +659,7 @@
 ``'sha512'``.
 
 For source archive references, an expected hash value may be specified by
-including a ``<hash-algorithm>=<expected-hash>`` as part of the URL
+including a ``<hash-algorithm>=<expected-hash>`` entry as part of the URL
 fragment.
 
 For version control references, the ``VCS+protocol`` scheme SHOULD be
@@ -546,29 +675,6 @@
     "source_url": "http://github.com/pypa/pip/archive/1.3.1.zip#sha1=da9234ee9982d4bbb3c72346a6de940a148ea686"
     "source_url": "git+https://github.com/pypa/pip.git at 1.3.1"
 
-.. note::
-
-    This was called "Download-URL" in previous versions of the metadata. It
-    has been renamed, since there are plenty of other download locations and
-    this URL is meant to be a way to get the original source for development
-    purposes (or to generate an SRPM or other platform specific equivalent).
-
-    For extra fun and games, it appears that unlike "svn+ssh://",
-    neither "git+ssh://" nor "hg+ssh://" natively support direct linking to a
-    particular tag (hg does support direct links to bookmarks through the URL
-    fragment, but that doesn't help for git and doesn't appear to be what I
-    want anyway).
-
-    However pip does have a `defined convention
-    <http://www.pip-installer.org/en/latest/logic.html#vcs-support>`__ for
-    this kind of link, which effectively splits a "URL" into "<repo>@<tag>".
-
-    The PEP simply adopts pip's existing solution to this problem.
-
-    This field is separate from the project URLs, as it's expected to
-    change for each version, while the project URLs are expected to
-    be fairly stable.
-
 
 Additional descriptive metadata
 ===============================
@@ -580,74 +686,29 @@
 a distribution does not provide them, including failing cleanly when an
 operation depending on one of these fields is requested.
 
-Summary
+
+License
 -------
 
-A one-line summary of what the distribution does.
-
-Publication tools SHOULD emit a warning if this field is not provided. Index
-servers MAY require that this field be present before allowing a
-distribution to be uploaded.
+A short string summarising the license used for this distribution.
+
+Note that distributions that provide this field should still specify any
+applicable license Trove classifiers in the `Classifiers`_ field. Even
+when an appropriate Trove classifier is available, the license summary can
+be a good way to specify a particular version of that license, or to
+indicate any variations or exception to the license.
+
+This field SHOULD contain fewer than 512 characters and MUST contain fewer
+than 2048.
+
+This field SHOULD NOT contain any line breaks.
+
+The full license text SHOULD be included as a separate file in the source
+archive for the distribution. See `Document names`_ for details.
 
 Example::
 
-    "summary": "A module that is more fiendish than soft cushions."
-
-.. note::
-
-   This used to be mandatory, and it's still highly recommended, but really,
-   nothing should break even when it's missing.
-
-
-Description
------------
-
-The distribution metadata should include a longer description of the
-distribution that may run to several paragraphs. Software that deals
-with metadata should not assume any maximum size for the description.
-
-The distribution description can be written using reStructuredText
-markup [1]_.  For programs that work with the metadata, supporting
-markup is optional; programs may also display the contents of the
-field as plain text without any special formatting.  This means that
-authors should be conservative in the markup they use.
-
-Example::
-
-    "description": "The ComfyChair module replaces SoftCushions.\\n\\nUse until lunchtime, but pause for a cup of coffee at eleven."
-
-.. note::
-
-   The difficulty of editing this field in a raw JSON file is one of the
-   main reasons this metadata interchange format is NOT recommended for
-   use as an input format for build tools.
-
-
-Description Format
-------------------
-
-A field indicating the intended format of the text in the description field.
-This allows index servers to render the description field correctly and
-provide feedback on rendering errors, rather than having to guess the
-intended format.
-
-If this field is omitted, or contains an unrecognised value, the default
-rendering format MUST be plain text.
-
-The following format names SHOULD be used for the specified markup formats:
-
-* ``txt``: Plain text (default handling if field is omitted)
-* ``rst``: reStructured Text
-* ``md``: Markdown (exact syntax variant will be implementation dependent)
-* ``adoc``: AsciiDoc
-* ``html``: HTML
-
-Automated tools MAY render one or more of the listed formats as plain
-text and MAY accept other markup formats beyond those listed.
-
-Example::
-
-    "description_format": "rst"
+    "license": "GPL version 3, excluding DRM provisions"
 
 
 Keywords
@@ -661,40 +722,6 @@
     "keywords": ["comfy", "chair", "cushions", "too silly", "monty python"]
 
 
-License
--------
-
-A string indicating the license covering the distribution where the license
-is not a simple selection from the "License" Trove classifiers. See
-Classifiers" below.  This field may also be used to specify a
-particular version of a license which is named via the ``Classifier``
-field, or to indicate a variation or exception to such a license.
-
-Example::
-
-    "license": "GPL version 3, excluding DRM provisions"
-
-
-License URL
------------
-
-A specific URL referencing the full licence text for this version of the
-distribution.
-
-Example::
-
-    "license_url": "https://github.com/pypa/pip/blob/1.3.1/LICENSE.txt"
-
-.. note::
-
-   Like Version URL, this is handled separately from the project URLs
-   as it is important that it remain accurate for this *specific*
-   version of the distribution, even if the project later switches to a
-   different license.
-
-   The project URLs field is intended for more stable references.
-
-
 Classifiers
 -----------
 
@@ -704,11 +731,60 @@
 Example::
 
     "classifiers": [
-        "Development Status :: 4 - Beta",
-        "Environment :: Console (Text Based)"
+      "Development Status :: 4 - Beta",
+      "Environment :: Console (Text Based)",
+      "License :: OSI Approved :: GNU General Public License v3 (GPLv3)"
     ]
 
 
+Document names
+--------------
+
+Filenames for supporting documents included in the distribution's
+``dist-info`` metadata directory.
+
+The following supporting documents can be named:
+
+* ``description``: a file containing a long description of the distribution
+* ``license``: a file with the full text of the distribution's license
+* ``changelog``: a file describing changes made to the distribution
+
+Supporting documents MUST be included directly in the ``dist-info``
+directory. Directory separators are NOT permitted in document names.
+
+The markup format (if any) for the file is indicated by the file extension.
+This allows index servers and other automated tools to render included
+text documents correctly and provide feedback on rendering errors, rather
+than having to guess the intended format.
+
+If the filename has no extension, or the extension is not recognised, the
+default rendering format MUST be plain text.
+
+The following markup renderers SHOULD be used for the specified file
+extensions:
+
+* Plain text: ``.txt``, no extension, unknown extension
+* reStructured Text: ``.rst``
+* Markdown: ``.md``
+* AsciiDoc: ``.adoc``, ``.asc``, ``.asciidoc``
+* HTML: ``.html``, ``.htm``
+
+Automated tools MAY render one or more of the specified formats as plain
+text and MAY render other markup formats beyond those listed.
+
+Automated tools SHOULD NOT make any assumptions regarding the maximum length
+of supporting document content, except as necessary to protect the
+integrity of a service.
+
+Example::
+
+    "document_names": {
+        "description": "README.rst",
+        "license": "LICENSE.rst",
+        "changelog": "NEWS"
+    }
+
+
 Contributor metadata
 ====================
 
@@ -726,6 +802,12 @@
 
 If no specific role is stated, the default is ``contributor``.
 
+Email addresses must be in the form ``local-part at domain`` where the
+local-part may be up to 64 characters long and the entire email address
+contains no more than 254 characters. The formal specification of the
+format is in RFC 5322 (sections 3.2.3 and 3.4.1) and RFC 5321, with a more
+readable form given in the informational RFC 3696 and the associated errata.
+
 The defined contributor roles are as follows:
 
 * ``author``: the original creator of a distribution
@@ -734,15 +816,6 @@
 * ``contributor``: any other individuals or organizations involved in the
   creation of the distribution
 
-.. note::
-
-   The contributor role field is included primarily to replace the
-   Author, Author-Email, Maintainer, Maintainer-Email fields from metadata
-   1.2 in a way that allows those distinctions to be fully represented for
-   lossless translation, while allowing future distributions to pretty
-   much ignore everything other than the contact/contributor distinction
-   if they so choose.
-
 Contact and contributor metadata is optional. Automated tools MUST operate
 correctly if a distribution does not provide it, including failing cleanly
 when an operation depending on one of these fields is requested.
@@ -789,12 +862,12 @@
 Example::
 
     "contributors": [
-        {"name": "John C."},
-        {"name": "Erik I."},
-        {"name": "Terry G."},
-        {"name": "Mike P."},
-        {"name": "Graeme C."},
-        {"name": "Terry J."}
+      {"name": "John C."},
+      {"name": "Erik I."},
+      {"name": "Terry G."},
+      {"name": "Mike P."},
+      {"name": "Graeme C."},
+      {"name": "Terry J."}
     ]
 
 
@@ -815,34 +888,45 @@
 Example::
 
     "project_urls": {
-        "Documentation": "https://distlib.readthedocs.org"
-        "Home": "https://bitbucket.org/pypa/distlib"
-        "Repository": "https://bitbucket.org/pypa/distlib/src"
-        "Tracker": "https://bitbucket.org/pypa/distlib/issues"
+      "Documentation": "https://distlib.readthedocs.org"
+      "Home": "https://bitbucket.org/pypa/distlib"
+      "Repository": "https://bitbucket.org/pypa/distlib/src"
+      "Tracker": "https://bitbucket.org/pypa/distlib/issues"
     }
 
 
-Dependency metadata
-===================
+Semantic dependencies
+=====================
 
 Dependency metadata allows distributions to make use of functionality
 provided by other distributions, without needing to bundle copies of those
 distributions.
 
+Semantic dependencies allow publishers to indicate not only which other
+distributions are needed, but also *why* they're needed. This additional
+information allows integrators to install just the dependencies they need
+for specific activities, making it easier to minimise installation
+footprints in constrained environments (regardless of the reasons for
+those constraints).
+
+Distributions may declare five differents kinds of dependency:
+
+* "Meta" dependencies: subdistributions that are grouped together into a
+  single larger metadistribution for ease of reference and installation.
+* Runtime dependencies: other distributions that are needed to actually use
+  this distribution (but are not considered subdistributions).
+* Test dependencies: other distributions that are needed to run the
+  automated test suite for this distribution (but are not needed just to
+  use it).
+* Build dependencies: other distributions that are needed to build this
+  distribution.
+* Development dependencies: other distributions that are needed when
+  working on this distribution (but do not fit into one of the other
+  dependency categories).
+
 Dependency management is heavily dependent on the version identification
 and specification scheme defined in PEP 440.
 
-.. note::
-
-    This substantially changes the old two-phase setup vs runtime dependency
-    model in metadata 1.2 (which was in turn derived from the setuptools
-    dependency parameters). The translation is that ``dev_requires`` and
-    ``build_requires`` both map to ``Setup-Requires-Dist``
-    in 1.2, while ``requires`` and ``distributes`` map to ``Requires-Dist``.
-    To go the other  way, ``Setup-Requires-Dist`` maps to ``build_requires``
-    and ``Requires-Dist`` maps to ``distributes`` (for exact comparisons)
-    and ``requires`` (for all other version specifiers).
-
 All of these fields are optional. Automated tools MUST operate correctly if
 a distribution does not provide them, by assuming that a missing field
 indicates "Not applicable for this distribution".
@@ -854,10 +938,11 @@
 Individual dependencies are typically defined as strings containing a
 distribution name (as found in the ``name`` field). The dependency name
 may be followed by an extras specifier (enclosed in square
-brackets) and by a version specification (within parentheses).
+brackets) and by a version specifier or direct reference (within
+parentheses).
 
 See `Extras (optional dependencies)`_ for details on extras and PEP 440
-for details on version specifiers.
+for details on version specifiers and direct references.
 
 The distribution names should correspond to names as found on the `Python
 Package Index`_; while these names are often the same as the module names
@@ -903,13 +988,6 @@
 conditional dependencies. This may happen, for example, if an extra itself
 only needs some of its dependencies in specific environments.
 
-.. note::
-
-   Technically, you could store the conditional and unconditional
-   dependencies in a single list and switch based on the entry type
-   (string or mapping), but the ``*requires`` vs ``*may-require`` two
-   list design seems easier to understand and work with.
-
 
 Mapping dependencies to development and distribution activities
 ---------------------------------------------------------------
@@ -918,26 +996,34 @@
 and development activities identified above, and govern which dependencies
 should be installed for the specified activities:
 
-* Deployment dependencies:
-
-    * ``distributes``
-    * ``requires``
-    * ``may_require``
-    * Request the ``test`` extra to also install
-
+* Implied runtime dependencies:
+
+    * ``meta_requires``
+    * ``meta_may_require``
+    * ``run_requires``
+    * ``run_may_require``
+
+* Implied build dependencies:
+
+    * ``build_requires``
+    * ``build_may_require``
+    * If running the distribution's test suite as part of the build process,
+      request the ``:meta:``, ``:run:`` and ``:test:`` extras to also
+      install:
+
+      * ``meta_requires``
+      * ``meta_may_require``
+      * ``run_requires``
+      * ``run_may_require``
       * ``test_requires``
       * ``test_may_require``
 
-* Build dependencies:
-
-    * ``build_requires``
-    * ``build_may_require``
-
-* Development dependencies:
-
-    * ``distributes``
-    * ``requires``
-    * ``may_require``
+* Implied development and publication dependencies:
+
+    * ``meta_requires``
+    * ``meta_may_require``
+    * ``run_requires``
+    * ``run_may_require``
     * ``build_requires``
     * ``build_may_require``
     * ``test_requires``
@@ -945,139 +1031,146 @@
     * ``dev_requires``
     * ``dev_may_require``
 
-
-To ease compatibility with existing two phase setup/deployment toolchains,
-installation tools MAY treat ``dev_requires`` and ``dev_may_require`` as
-additions to ``build_requires`` and ``build_may_require`` rather than
-as separate fields.
-
-Installation tools SHOULD allow users to request at least the following
-operations for a named distribution:
-
-* Install the distribution and any deployment dependencies.
-* Install just the build dependencies without installing the distribution
-* Install just the development dependencies without installing
-  the distribution
-* Install just the development dependencies without installing
-  the distribution or any dependencies listed in ``distributes``
-
-The notation described in `Extras (optional dependencies)`_ SHOULD be used to
-request additional optional dependencies when installing deployment
-or build dependencies.
+The notation described in `Extras (optional dependencies)`_ SHOULD be used
+to determine exactly what gets installed for various operations.
 
 Installation tools SHOULD report an error if dependencies cannot be found,
 MUST at least emit a warning, and MAY allow the user to force the
 installation to proceed regardless.
 
-.. note::
-
-    As an example of mapping this to Linux distro packages, assume an
-    example project without any extras defined is split into 2 RPMs
-    in a SPEC file: example and example-devel
-
-    The ``distributes``, ``requires`` and applicable ``may_require``
-    dependencies would be mapped to the Requires dependencies for the
-    "example" RPM (a mapping from environment markers to SPEC file
-    conditions would also allow those to be handled correctly)
-
-    The ``build_requires`` and ``build_may_require`` dependencies would be
-    mapped to the BuildRequires dependencies for the "example" RPM.
-
-    All defined dependencies relevant to Linux, including those in
-    ``dev_requires`` and ``test_requires``, would become Requires
-    dependencies for the "example-devel" RPM.
-
-    If a project defines any extras, those would be mapped to additional
-    virtual RPMs with appropriate BuildRequires and Requires entries based
-    on the details of the dependency specifications.
-
-    A documentation toolchain dependency like Sphinx would either go in
-    ``build_requires`` (for example, if man pages were included in the
-    built distribution) or in ``dev_requires`` (for example, if the
-    documentation is published solely through ReadTheDocs or the
-    project website). This would be enough to allow an automated converter
-    to map it to an appropriate dependency in the spec file.
-
-
-Distributes
------------
-
-A list of subdistributions that can easily be installed and used together
-by depending on this metadistribution.
-
-Automated tools MUST allow strict version matching and source reference
-clauses in this field and MUST NOT allow more permissive version specifiers.
-
-Example::
-
-    "distributes": ["ComfyUpholstery (== 1.0a2)",
-                    "ComfySeatCushion (== 1.0a2)"]
-
-
-Requires
---------
-
-A list of other distributions needed when this distribution is deployed.
-
-Automated tools MAY disallow strict version matching clauses and source
-references in this field and SHOULD at least emit a warning for such clauses.
-
-Example::
-
-    "requires": ["SciPy", "PasteDeploy", "zope.interface (>3.5.0)"]
+See Appendix B for an overview of mapping these dependencies to an RPM
+spec file.
 
 
 Extras
 ------
 
 A list of optional sets of dependencies that may be used to define
-conditional dependencies in ``"may_require"`` and similar fields. See
-`Extras (optional dependencies)`_ for details.
-
-The extra name``"test"`` is reserved for requesting the dependencies
-specified in ``test_requires`` and ``test_may_require`` and is NOT
-permitted in this field.
+conditional dependencies in ``"may_distribute"``, ``"run_may_require"`` and
+similar fields. See `Extras (optional dependencies)`_ for details.
+
+The names of extras MUST abide by the same restrictions as those for
+distribution names.
 
 Example::
 
     "extras": ["warmup"]
 
 
-May require
------------
-
-A list of other distributions that may be needed when this distribution
-is deployed, based on the extras requested and the target deployment
+Meta requires
+-------------
+
+An abbreviation of "metadistribution requires". This is a list of
+subdistributions that can easily be installed and used together by
+depending on this metadistribution.
+
+In this field, automated tools:
+
+* MUST allow strict version matching
+* MUST NOT allow more permissive version specifiers.
+* MAY allow direct references
+
+Public index servers SHOULD NOT allow the use of direct references in
+uploaded distributions. Direct references are intended primarily as a
+tool for software integrators rather than publishers.
+
+Distributions that rely on direct references to platform specific binary
+archives SHOULD define appropriate constraints in their
+``supports_environments`` field.
+
+Example::
+
+    "meta_requires": ["ComfyUpholstery (== 1.0a2)",
+                      "ComfySeatCushion (== 1.0a2)"]
+
+
+Meta may require
+----------------
+
+An abbreviation of "metadistribution may require". This is a list of
+subdistributions that can easily be installed and used together by
+depending on this metadistribution, but are not required in all
+circumstances.
+
+Any extras referenced from this field MUST be named in the `Extras`_ field.
+
+In this field, automated tools:
+
+* MUST allow strict version matching
+* MUST NOT allow more permissive version specifiers.
+* MAY allow direct references
+
+Public index servers SHOULD NOT allow the use of direct references in
+uploaded distributions. Direct references are intended primarily as a
+tool for software integrators rather than publishers.
+
+Distributions that rely on direct references to platform specific binary
+archives SHOULD defined appropriate constraints in their
+``supports_environments`` field.
+
+Example::
+
+    "meta_may_require": [
+      {
+        "dependencies": ["CupOfTeaAtEleven (== 1.0a2)"],
+        "environment": "'linux' in sys.platform"
+      }
+    ]
+
+
+Run requires
+------------
+
+A list of other distributions needed to actually run this distribution.
+
+Automated tools MUST NOT allow strict version matching clauses or direct
+references in this field - if permitted at all, such clauses should appear
+in ``meta_requires`` instead.
+
+Example::
+
+    "run_requires": ["SciPy", "PasteDeploy", "zope.interface (>3.5.0)"]
+
+
+Run may require
+---------------
+
+A list of other distributions that may be needed to actually run this
+distribution, based on the extras requested and the target deployment
 environment.
 
 Any extras referenced from this field MUST be named in the `Extras`_ field.
 
-Automated tools MAY disallow strict version matching clauses and source
-references in this field and SHOULD at least emit a warning for such clauses.
+Automated tools MUST NOT allow strict version matching clauses or direct
+references in this field - if permitted at all, such clauses should appear
+in ``meta_may_require`` instead.
 
 Example::
 
-        "may_require": [
-          {
-            "dependencies": ["pywin32 (>1.0)"],
-            "environment": "sys.platform == 'win32'"
-          },
-          {
-            "dependencies": ["SoftCushions"],
-            "extra": "warmup"
-          }
-        ]
+    "run_may_require": [
+      {
+        "dependencies": ["pywin32 (>1.0)"],
+        "environment": "sys.platform == 'win32'"
+      },
+      {
+        "dependencies": ["SoftCushions"],
+        "extra": "warmup"
+      }
+    ]
+
 
 Test requires
 -------------
 
 A list of other distributions needed in order to run the automated tests
-for this distribution, either during development or when running the
-``test_installed_dist`` metabuild when deployed.
-
-Automated tools MAY disallow strict version matching clauses and source
+for this distribution..
+
+Automated tools MAY disallow strict version matching clauses and direct
 references in this field and SHOULD at least emit a warning for such clauses.
 
+Public index servers SHOULD NOT allow strict version matching clauses or
+direct references in this field.
+
 Example::
 
     "test_requires": ["unittest2"]
@@ -1087,41 +1180,45 @@
 ----------------
 
 A list of other distributions that may be needed in order to run the
-automated tests for this distribution, either during development or when
-running the ``test_installed_dist`` metabuild when deployed, based on the
-extras requested and the target deployment environment.
+automated tests for this distribution.
 
 Any extras referenced from this field MUST be named in the `Extras`_ field.
 
-Automated tools MAY disallow strict version matching clauses and source
+Automated tools MAY disallow strict version matching clauses and direct
 references in this field and SHOULD at least emit a warning for such clauses.
 
+Public index servers SHOULD NOT allow strict version matching clauses or
+direct references in this field.
+
 Example::
 
-        "test_may_require": [
-          {
-            "dependencies": ["pywin32 (>1.0)"],
-            "environment": "sys.platform == 'win32'"
-          },
-          {
-            "dependencies": ["CompressPadding"],
-            "extra": "warmup"
-          }
-        ]
+    "test_may_require": [
+      {
+        "dependencies": ["pywin32 (>1.0)"],
+        "environment": "sys.platform == 'win32'"
+      },
+      {
+        "dependencies": ["CompressPadding"],
+        "extra": "warmup"
+      }
+    ]
 
 
 Build requires
 --------------
 
 A list of other distributions needed when this distribution is being built
-(creating a binary archive from a source archive).
+(creating a binary archive from an sdist, source archive or VCS checkout).
 
 Note that while these are build dependencies for the distribution being
 built, the installation is a *deployment* scenario for the dependencies.
 
-Automated tools MAY disallow strict version matching clauses and source
+Automated tools MAY disallow strict version matching clauses and direct
 references in this field and SHOULD at least emit a warning for such clauses.
 
+Public index servers SHOULD NOT allow strict version matching clauses or
+direct references in this field.
+
 Example::
 
     "build_requires": ["setuptools (>= 0.7)"]
@@ -1131,8 +1228,8 @@
 -----------------
 
 A list of other distributions that may be needed when this distribution
-is built (creating a binary archive from a source archive), based on the
-features requested and the build environment.
+is built (creating a binary archive from an sdist, source archive or
+VCS checkout), based on the features requested and the build environment.
 
 Note that while these are build dependencies for the distribution being
 built, the installation is a *deployment* scenario for the dependencies.
@@ -1142,21 +1239,24 @@
 Automated tools MAY assume that all extras are implicitly requested when
 installing build dependencies.
 
-Automated tools MAY disallow strict version matching clauses and source
+Automated tools MAY disallow strict version matching clauses and direct
 references in this field and SHOULD at least emit a warning for such clauses.
 
+Public index servers SHOULD NOT allow strict version matching clauses or
+direct references in this field.
+
 Example::
 
-        "build_may_require": [
-          {
-            "dependencies": ["pywin32 (>1.0)"],
-            "environment": "sys.platform == 'win32'"
-          },
-          {
-            "dependencies": ["cython"],
-            "extra": "c-accelerators"
-          }
-        ]
+    "build_may_require": [
+      {
+        "dependencies": ["pywin32 (>1.0)"],
+        "environment": "sys.platform == 'win32'"
+      },
+      {
+        "dependencies": ["cython"],
+        "extra": "c-accelerators"
+      }
+    ]
 
 
 Dev requires
@@ -1168,17 +1268,16 @@
 
 Additional dependencies that may be listed in this field include:
 
-* tools needed to create a source archive
+* tools needed to create an sdist from a source archive or VCS checkout
 * tools needed to generate project documentation that is published online
   rather than distributed along with the rest of the software
-* additional test dependencies for tests which are not executed when the
-  test is invoked through the ``test_installed_dist`` metabuild hook (for
-  example, tests that require a local database server and web server and
-  may not work when fully installed on a production system)
-
-Automated tools MAY disallow strict version matching clauses and source
+
+Automated tools MAY disallow strict version matching clauses and direct
 references in this field and SHOULD at least emit a warning for such clauses.
 
+Public index servers SHOULD NOT allow strict version matching clauses or
+direct references in this field.
+
 Example::
 
     "dev_requires": ["hgtools", "sphinx (>= 1.0)"]
@@ -1199,17 +1298,20 @@
 Automated tools MAY assume that all extras are implicitly requested when
 installing development dependencies.
 
-Automated tools MAY disallow strict version matching clauses and source
+Automated tools MAY disallow strict version matching clauses and direct
 references in this field and SHOULD at least emit a warning for such clauses.
 
+Public index servers SHOULD NOT allow strict version matching clauses or
+direct references in this field.
+
 Example::
 
-        "dev_may_require": [
-          {
-            "dependencies": ["pywin32 (>1.0)"],
-            "environment": "sys.platform == 'win32'"
-          }
-        ]
+    "dev_may_require": [
+      {
+        "dependencies": ["pywin32 (>1.0)"],
+        "environment": "sys.platform == 'win32'"
+      }
+    ]
 
 
 Provides
@@ -1231,6 +1333,19 @@
 project is able to include a ``"provides": ["distribute"]`` entry to
 satisfy any projects that require the now obsolete distribution's name.
 
+To avoid malicious hijacking of names, when interpreting metadata retrieved
+from a public index server, automated tools MUST prefer the distribution
+named in a version specifier over other distributions using that
+distribution's name in a ``"provides"`` entry. Index servers MAY drop such
+entries from the metadata they republish, but SHOULD NOT refuse to publish
+such distributions.
+
+However, to appropriately handle project forks and mergers, automated tools
+MUST accept ``"provides"`` entries that name other distributions when the
+entry is retrieved from a local installation database or when there is a
+corresponding ``"obsoleted_by"`` entry in the metadata for the named
+distribution.
+
 A distribution may also provide a "virtual" project name, which does
 not correspond to any separately distributed project:  such a name
 might be used to indicate an abstract capability which could be supplied
@@ -1291,63 +1406,66 @@
 Individual entries are environment markers, as described in
 `Environment markers`_.
 
-Installation tools SHOULD report an error if supported platforms are
+Installation tools SHOULD report an error if supported environments are
 specified by the distribution and the current platform fails to match
 any of them, MUST at least emit a warning, and MAY allow the user to
 force the installation to proceed regardless.
 
-Examples::
-
+The two main uses of this field are to declare which versions of Python
+and which underlying operating systems are supported.
+
+Examples indicating supported Python versions::
+
+   # Supports Python 2.6+
+   "supports_environments": ["python_version >= '2.6'"]
+
+   # Supports Python 2.6+ (for 2.x) or 3.3+ (for 3.x)
+   "supports_environments": ["python_version >= '3.3'",
+                             "'3.0' > python_version >= '2.6'"]
+
+Examples indicating supported operating systems::
+
+   # Windows only
    "supports_environments": ["sys_platform == 'win32'"]
+
+   # Anything except Windows
    "supports_environments": ["sys_platform != 'win32'"]
+
+   # Linux or BSD only
    "supports_environments": ["'linux' in sys_platform",
                              "'bsd' in sys_platform"]
 
-
-.. note::
-
-   This field replaces the old Platform, Requires-Platform and
-   Requires-Python fields and has been redesigned with environment
-   marker based semantics that should make it possible to reliably flag,
-   for example, Unix specific or Windows specific distributions, as well
-   as Python 2 only and Python 3 only distributions.
-
-
-Metabuild system
-================
-
-The ``metabuild_hooks`` field is used to define various operations that
-may be invoked on a distribution in a platform independent manner.
-
-The metabuild system currently defines three operations as part of the
-deployment of a distribution:
+Example where the supported Python version varies by platform::
+
+   # The standard library's os module has long supported atomic renaming
+   # on POSIX systems, but only gained atomic renaming on Windows in Python
+   # 3.3. A distribution that needs atomic renaming support for reliable
+   # operation might declare the following supported environments.
+   "supports_environments": ["python_version >= '2.6' and sys_platform != 'win32'",
+                             "python_version >= '3.3' and sys_platform == 'win32'"]
+
+
+Install hooks
+=============
+
+The ``install_hooks`` field is used to define operations to be
+invoked on the distribution  in the following situations:
 
 * Installing to a deployment system
 * Uninstalling from a deployment system
-* Running the distribution's test suite on a deployment system (hence the
-  ``test`` runtime extra)
-
-Distributions may define handles for each of these operations as an
-"entry point", a reference to a Python callable, with the module name
-separated from the reference within the module by a colon (``:``).
-
-Example metabuild hooks::
-
-    "metabuild_hooks": {
-        "postinstall": "myproject.build_hooks:postinstall",
-        "preuininstall": "myproject.build_hooks:preuninstall",
-        "test_installed_dist": "some_test_harness:metabuild_hook"
+
+Distributions may define handlers for each of these operations as an
+"entry point", which is a reference to a Python callable, with the module
+name separated from the reference within the module by a colon (``:``).
+
+Example install hooks::
+
+    "install_hooks": {
+      "postinstall": "ComfyChair.install_hooks:postinstall",
+      "preuininstall": "ComfyChair.install_hooks:preuninstall"
     }
 
-Build and installation tools MAY offer additional operations beyond the
-core metabuild operations. These operations SHOULD be composed from the
-defined metabuild operations where appropriate.
-
-Build and installation tools SHOULD support the legacy ``setup.py`` based
-commands for metabuild operations not yet defined as metabuild hooks.
-
-The metabuild hooks are gathered together into a single top level
-``metabuild_hooks`` field. The individual hooks are:
+The currently defined install hooks are:
 
 * ``postinstall``: run after the distribution has been installed to a
   target deployment system (or after it has been upgraded). If the hook is
@@ -1357,18 +1475,15 @@
   deployment system (or before it is upgraded). If the hook is not defined,
   it indicates no distribution specific actions are needed prior to
   uninstallation.
-* ``test_installed_dist``: test an installed distribution is working. If the
-  hook is not defined, it indicates the distribution does not support
-  execution of the test suite after deployment.
-
-The expected signatures of these hooks are as follows::
+
+The required signatures of these hooks are as follows::
 
     def postinstall(current_meta, previous_meta=None):
         """Run following installation or upgrade of the distribution
 
         *current_meta* is the distribution metadata for the version now
         installed on the current system
-        *previous_meta* is either missing or ``None`` (indicating a fresh
+        *previous_meta* is either omitted or ``None`` (indicating a fresh
         install) or else the distribution metadata for the version that
         was previously installed (indicating an upgrade or downgrade).
         """
@@ -1378,30 +1493,65 @@
 
         *current_meta* is the distribution metadata for the version now
         installed on the current system
-        *next_meta* is either missing or ``None`` (indicating complete
+        *next_meta* is either omitted or ``None`` (indicating complete
         uninstallation) or else the distribution metadata for the version
         that is about to be installed (indicating an upgrade or downgrade).
         """
 
-    def test_installed_dist(current_meta):
-        """Check an installed distribution is working correctly
-
-        Note that this check should always be non-destructive as it may be
-        invoked automatically by some tools.
-
-        Requires that the distribution's test dependencies be installed
-        (indicated by the ``test`` runtime extra).
-
-        Returns ``True`` if the check passes, ``False`` otherwise.
-        """
-
-Metabuild hooks MUST be called with at least abbreviated metadata, and MAY
-be called with full metadata.
-
-Where necessary, metabuild hooks check for the presence or absence of
-optional dependencies defined as extras using the same techniques used
-during normal operation of the distribution (for example, checking for
-import failures for optional dependencies).
+When install hooks are defined, it is assumed that they MUST be executed
+to obtain a properly working installation of the distribution, and to
+properly remove the distribution from a system.
+
+Install hooks SHOULD NOT be used to provide functionality that is
+expected to be provided by installation tools (such as rewriting of
+shebang lines and generation of executable wrappers for Windows).
+
+Installation tools MUST ensure the distribution is fully installed, and
+available through the import system and installation database when invoking
+install hooks.
+
+Installation tools MUST call install hooks with full metadata, rather than
+only the essential dependency resolution metadata.
+
+The given parameter names are considered part of the hook signature.
+Installation tools MUST call install hooks solely with keyword arguments.
+Install hook implementations MUST use the given parameter names.
+
+Installation tools SHOULD invoke install hooks automatically after
+installing a distribution from a binary archive.
+
+When installing from an sdist, source archive or VCS checkout, installation
+tools SHOULD create a binary archive using ``setup.py bdist_wheel`` and
+then install binary archive normally (including invocation of any install
+hooks). Installation tools SHOULD NOT invoke ``setup.py install`` directly.
+
+Installation tools SHOULD treat an exception thrown by a postinstall hook
+as a failure of the installation and revert any other changes made to the
+system.
+
+Installation tools SHOULD treat an exception thrown by a preuninstall hook
+as an indication the removal of the distribution should be aborted.
+
+Installation tools MUST NOT silently ignore install hooks, as failing
+to call these hooks may result in a misconfigured installation that fails
+unexpectedly at runtime. Installation tools MAY refuse to install
+distributions that define install hooks, or require that users
+explicitly opt in to permitting the execution of such hooks.
+
+Install hook implementations MUST NOT make any assumptions regarding the
+current working directory when they are invoked, and MUST NOT make
+persistent alterations to the working directory or any other process global
+state (other than potentially importing additional modules, or other
+expected side effects of running the distribution).
+
+Install hooks have access to the full metadata for the release being
+installed, that of the previous/next release (as appropriate), as well as
+to all the normal runtime information (such as available imports). Hook
+implementations can use this information to perform additional platform
+specific installation steps. To check for the presence or absence of
+"extras", hook implementations should use the same runtime checks that
+would be used during normal operation (such as checking for the availability
+of the relevant dependencies).
 
 
 Metadata Extensions
@@ -1413,14 +1563,20 @@
 in JSON::
 
     "extensions" : {
-        "chili" : { "type" : "Poblano", "heat" : "Mild" },
-        "languages" : [ "French", "Italian", "Hebrew" ]
+      "chili" : { "type" : "Poblano", "heat" : "Mild" },
+      "languages" : [ "French", "Italian", "Hebrew" ]
     }
 
-To avoid name conflicts, it is recommended that distribution names be used
+To avoid name conflicts, it is RECOMMENDED that distribution names be used
 to identify metadata extensions. This practice will also make it easier to
 find authoritative documentation for metadata extensions.
 
+Metadata extensions allow development tools to record information in the
+metadata that may be useful during later phases of distribution. For
+example, a build tool could include default build options in a metadata
+extension when creating an sdist, and use those when creating the wheel
+files later.
+
 
 Extras (optional dependencies)
 ==============================
@@ -1440,7 +1596,7 @@
 
     "name": "ComfyChair",
     "extras": ["warmup", "c-accelerators"]
-    "may_require": [
+    "run_may_require": [
       {
         "dependencies": ["SoftCushions"],
         "extra": "warmup"
@@ -1457,15 +1613,32 @@
 relevant extra names inside square brackets after the distribution name when
 specifying the dependency.
 
-Extra specifications MUST support the following additional syntax:
-
-* Multiple features can be requested by separating them with a comma within
+Extra specifications MUST allow the following additional syntax:
+
+* Multiple extras can be requested by separating them with a comma within
   the brackets.
-* All explicitly defined extras may be requested with the ``*`` wildcard
-  character. Note that this does NOT request the implicitly defined
-  ``test`` extra - that must always be requested explicitly when it is
-  desired.
-* Extras may be explicitly excluded by prefixing their name with a hyphen.
+* The following special extras request processing of the corresponding
+  lists of dependencies:
+
+  * ``:meta:``: ``meta_requires`` and ``meta_may_require``
+  * ``:run:``: ``run_requires`` and ``run_may_require``
+  * ``:test:``: ``test_requires`` and ``test_may_require``
+  * ``:build:``: ``build_requires`` and ``build_may_require``
+  * ``:dev:``: ``dev_requires`` and ``dev_may_require``
+  * ``:*:``: process *all* dependency lists
+
+* The ``*`` character as an extra is a wild card that enables all of the
+  entries defined in the distribution's ``extras`` field.
+* Extras may be explicitly excluded by prefixing their name with a ``-``
+  character (this is useful in conjunction with ``*`` to exclude only
+  particular extras that are definitely not wanted, while enabling all
+  others).
+
+* The ``-`` character as an extra specification indicates that the
+  distribution itself should NOT be installed, and also disables the
+  normally implied processing of ``:meta:`` and ``:run:`` dependencies
+  (those may still be requested explicitly using the appropriate extra
+  specifications).
 
 Command line based installation tools SHOULD support this same syntax to
 allow extras to be requested explicitly.
@@ -1473,15 +1646,31 @@
 The full set of dependency requirements is then based on the top level
 dependencies, along with those of any requested extras.
 
-Example::
-
-    "requires": ["ComfyChair[warmup]"]
+Dependency examples::
+
+    "run_requires": ["ComfyChair[warmup]"]
         -> requires ``ComfyChair`` and ``SoftCushions`` at run time
 
-    "requires": ["ComfyChair[*]"]
+    "run_requires": ["ComfyChair[*]"]
         -> requires ``ComfyChair`` and ``SoftCushions`` at run time, but
-           will also pick up any new optional dependencies other than those
-           needed solely to run the tests
+           will also pick up any new extras defined in later versions
+
+Command line examples::
+
+    pip install ComfyChair
+        -> installs ComfyChair with applicable :meta: and :run: dependencies
+
+    pip install ComfyChair[*]
+        -> as above, but also installs all extra dependencies
+
+    pip install ComfyChair[-,:build:,*]
+        -> installs just the build dependencies with all extras
+
+    pip install ComfyChair[-,:build:,:run:,:meta:,:test:,*]
+        -> as above, but also installs dependencies needed to run the tests
+
+    pip install ComfyChair[-,:*:,*]
+        -> installs the full set of development dependencies
 
 
 Environment markers
@@ -1504,7 +1693,7 @@
 requires PyWin32 both at runtime and buildtime when using Windows::
 
     "name": "ComfyChair",
-    "may_require": [
+    "run_may_require": [
       {
         "dependencies": ["pywin32 (>1.0)"],
         "environment": "sys.platform == 'win32'"
@@ -1539,6 +1728,12 @@
 * ``platform_version``: ``platform.version()``
 * ``platform_machine``: ``platform.machine()``
 * ``platform_python_implementation``: ``platform.python_implementation()``
+* ``implementation_name````: ``sys.implementation.name``
+* ``implementation_version````: see definition below
+
+If a particular value is not available (such as the ``sys.implementation``
+subattributes in versions of Python prior to 3.3), the corresponding marker
+variable MUST be considered equivalent to the empty string.
 
 Note that all subexpressions are restricted to strings or one of the
 marker variable names (which refer to string values), meaning that it is
@@ -1548,17 +1743,24 @@
 Chaining of comparison operations is permitted using the normal Python
 semantics of an implied ``and``.
 
-The ``python_full_version`` marker variable is derived from
-``sys.version_info()`` in accordance with the following algorithm::
-
-    def format_full_version():
-        info = sys.version_info
+The ``python_full_version`` and ``implementation_version`` marker variables
+are derived from ``sys.version_info()`` and ``sys.implementation.version``
+respectively, in accordance with the following algorithm::
+
+    def format_full_version(info):
         version = '{0.major}.{0.minor}.{0.micro}'.format(info)
         kind = info.releaselevel
         if kind != 'final':
             version += kind[0] + str(info.serial)
         return version
 
+    python_full_version = format_full_version(sys.version_info)
+    implementation_version = format_full_version(sys.implementation.version)
+
+``python_full_version`` will typically correspond to the leading segment
+of ``sys.version()``.
+
+
 Updating the metadata specification
 ===================================
 
@@ -1570,8 +1772,68 @@
 defined in a new PEP.
 
 
-Summary of differences from \PEP 345
-====================================
+Appendix A: Conversion notes for legacy metadata
+================================================
+
+The reference implementations for converting from legacy metadata to
+metadata 2.0 are:
+
+* the `wheel project <https://bitbucket.org/dholth/wheel/overview>`__, which
+  adds the ``bdist_wheel`` command to ``setuptools``
+* the `Warehouse project <https://github.com/dstufft/warehouse>`__, which
+  will eventually be migrated to the Python Packaging Authority as the next
+  generation Python Package Index implementation
+* the `distlib project <https://bitbucket.org/pypa/distlib/>`__ which is
+  derived from the core packaging infrastructure created for the
+  ``distutils2`` project and
+
+While it is expected that there may be some edge cases where manual
+intervention is needed for clean conversion, the specification has been
+designed to allow fully automated conversion of almost all projects on
+PyPI.
+
+Metadata conversion (especially on the part of the index server) is a
+necessary step to allow installation and analysis tools to start
+benefiting from the new metadata format, without having to wait for
+developers to upgrade to newer build systems.
+
+
+Appendix B: Mapping dependency declarations to an RPM SPEC file
+===============================================================
+
+
+As an example of mapping this PEP to Linux distro packages, assume an
+example project without any extras defined is split into 2 RPMs
+in a SPEC file: ``example`` and ``example-devel``.
+
+The ``meta_requires``, ``run_requires`` and applicable
+``meta_may_require`` ``run_may_require`` dependencies would be mapped
+to the Requires dependencies for the "example" RPM (a mapping from
+environment markers relevant to Linux to SPEC file conditions would
+also allow those to be handled correctly)
+
+The ``build_requires`` and ``build_may_require`` dependencies would be
+mapped to the BuildRequires dependencies for the "example" RPM.
+
+All defined dependencies relevant to Linux, including those in
+``dev_requires``, ``test_requires``, ``dev_may_require``, and
+``test_may_require`` would become Requires dependencies for the
+"example-devel" RPM.
+
+If the project did define any extras, those would likely be mapped to
+additional virtual RPMs with appropriate BuildRequires and Requires
+entries based on the details of the dependency specifications.
+
+A documentation toolchain dependency like Sphinx would either go in
+``build_requires`` (for example, if man pages were included in the
+built distribution) or in ``dev_requires`` (for example, if the
+documentation is published solely through ReadTheDocs or the
+project website). This would be enough to allow an automated converter
+to map it to an appropriate dependency in the spec file.
+
+
+Appendix C: Summary of differences from \PEP 345
+=================================================
 
 * Metadata-Version is now 2.0, with semantics specified for handling
   version changes
@@ -1592,21 +1854,21 @@
 
 * Changed the version scheme to be based on PEP 440 rather than PEP 386
 
-* Added the build label mechanism as described in PEP 440
-
-* Support for different development, build, test and deployment dependencies
+* Added the source label mechanism as described in PEP 440
+
+* Support for different kinds of dependencies
 
 * The "Extras" optional dependency mechanism
 
 * A well-defined metadata extension mechanism
 
-* Metabuild hook system
+* Install hook system
 
 * Clarify and simplify various aspects of environment markers:
 
   * allow use of parentheses for grouping in the pseudo-grammar
   * consistently use underscores instead of periods in the variable names
-  * clarify that chained comparisons are not permitted
+  * allow ordered string comparisons and chained comparisons
 
 * More flexible system for defining contact points and contributors
 
@@ -1616,9 +1878,11 @@
 
 * Updated obsolescence mechanism
 
-* Added "License URL" field
-
-* Explicit declaration of description markup format
+* Identification of supporting documents in the ``dist-info`` directory:
+
+  * Allows markup formats to be indicated through file extensions
+  * Standardises the common practice of taking the description from README
+  * Also supports inclusion of license files and changelogs
 
 * With all due respect to Charles Schulz and Peanuts, many of the examples
   have been updated to be more `thematically appropriate`_ for Python ;)
@@ -1667,7 +1931,7 @@
 subfields.
 
 The old serialisation format also wasn't amenable to easy conversion to
-standard Python data structures for use in the new metabuild hook APIs, or
+standard Python data structures for use in the new install hook APIs, or
 in future extensions to the importer APIs to allow them to provide
 information for inclusion in the installation database.
 
@@ -1691,33 +1955,47 @@
 See PEP 440 for the rationale behind the addition of this field.
 
 
-Development, build and deployment dependencies
-----------------------------------------------
-
-The separation of the ``requires``, ``build_requires`` and ``dev_requires``
-fields allows a distribution to indicate whether a dependency is needed
-specifically to develop, build or deploy the distribution.
-
-As distribution metadata improves, this should allow much greater control
-over where particular dependencies end up being installed .
+Support for different kinds of dependencies
+-------------------------------------------
+
+The separation of the five different kinds of dependency allows a
+distribution to indicate whether a dependency is needed specifically to
+develop, build, test or use the distribution.
+
+To allow for metadistributions like PyObjC, while still actively
+discouraging overly strict dependency specifications, the separate
+``meta`` dependency fields are used to separate out those dependencies
+where exact version specifications are appropriate.
+
+The advantage of having these distinctions supported in the upstream Python
+specific metadata is that even if a project doesn't care about these
+distinction themselves, they may be more amenable to patches from
+downstream redistributors that separate the fields appropriately. Over time,
+this should allow much greater control over where and when particular
+dependencies end up being installed.
+
+The names for the dependency fields have been deliberately chosen to avoid
+conflicting with the existing terminology in setuptools and previous
+versions of the metadata standard. Specifically, the names ``requires``,
+``install_requires`` and ``setup_requires`` are not used, which will
+hopefully reduce confustion when converting legacy metadata to the new
+standard.
 
 
 Support for optional dependencies for distributions
 ---------------------------------------------------
 
 The new extras system allows distributions to declare optional
-features, and to use the ``may_require`` and ``build_may_require`` fields
-to indicate when particular dependencies are needed only to support those
-features. It is derived from the equivalent system that is already in
-widespread use as part of ``setuptools`` and allows that aspect of the
-legacy ``setuptools`` metadata to be accurately represented in the new
-metadata format.
-
-The ``test`` extra is implicitly defined for all distributions, as it
-ties in with the new metabuild hook offering a standard way to request
-execution of a distribution's test suite. Identifying test suite
-dependencies is already one of the most popular uses of the extras system
-in ``setuptools``.
+behaviour, and to use the ``*may_require`` fields to indicate when
+particular dependencies are needed only to support that behaviour. It is
+derived from the equivalent system that is already in widespread use as
+part of ``setuptools`` and allows that aspect of the legacy ``setuptools``
+metadata to be accurately represented in the new metadata format.
+
+The additions to the extras syntax relative to setuptools are defined to
+make it easier to express the various possible combinations of dependencies,
+in particular those associated with build systems (with optional support
+for running the test suite) and development systems.
 
 
 Support for metadata extensions
@@ -1734,21 +2012,39 @@
 particular extensions to be provided as optional features.
 
 
-Support for metabuild hooks
+Support for install hooks
 ---------------------------
 
-The new metabuild system is designed to allow the wheel format to fully
-replace direct installation on deployment targets, by allowing projects like
-Twisted to still execute code following installation from a wheel file.
-
-Falling back to invoking ``setup.py`` directly rather than using a
-metabuild hook will remain an option when relying on version 1.x metadata,
-and is also used as the interim solution for installation from source
-archives.
-
-The ``test_installed_dist`` metabuild hook is included in order to integrate
-with build systems that can automatically invoke test suites, and as
-a complement to the ability to explicitly specify test dependencies.
+The new install hook system is designed to allow the wheel format to fully
+replace direct installation on deployment targets, by allowing projects to
+explicitly define code that should be executed following installation from
+a wheel file.
+
+This may be something relatively simple, like the `two line
+refresh <https://twistedmatrix.com/documents/current/core/howto/plugin.html#auto3>`__
+of the Twisted plugin caches that the Twisted developers recommend for
+any project that provides Twisted plugins, to more complex platform
+dependent behaviour, potentially in conjunction with appropriate
+metadata extensions and ``supports_environments`` entries.
+
+For example, upstream declaration of external dependencies for various
+Linux distributions in a distribution neutral format may be supported by
+defining an appropriate metadata extension that is read by a postinstall
+hook and converted into an appropriate invocation of the system package
+manager. Other operations (such as registering COM DLLs on Windows,
+registering services for automatic startup on any platform, or altering
+firewall settings) may need to be undertaken with elevated privileges,
+meaning they cannot be deferred to implicit execution on first use of the
+distribution.
+
+The install hook and metadata extension systems allow support for such
+activities to be pursued independently by the individual platform
+communities, while still interoperating with the cross-platform Python
+tools.
+
+Legacy packages that expect to able to run code on target systems using
+``setup.py install`` will no longer work correctly. Such packages will
+already break when pip 1.4+ is configured to use a wheel cache directory.
 
 
 Changes to environment markers
@@ -1805,8 +2101,9 @@
 has been used to replace several older fields with poorly defined semantics.
 
 For the moment, the old ``Requires-External`` field has been removed
-entirely. Possible replacements may be explored through the metadata
-extension mechanism.
+entirely. The combination of explicit support for post install hooks and the
+metadata extension mechanism will hopefully prove to be a more useful
+replacement.
 
 
 Updated obsolescence mechanism
@@ -1824,22 +2121,55 @@
 is not widely supported, and so removing it does not present any significant
 barrier to tools and projects adopting the new metadata format.
 
-Explicit markup for description
--------------------------------
-
-Currently, PyPI attempts to detect the markup format by rendering it as
-reStructuredText, and if that fails, treating it as plain text. Allowing
-the intended format to be stated explicitly will allow this guessing to be
-removed, and more informative error reports to be provided to users when
-a rendering error occurs.
-
-This is especially necessary since PyPI applies additional restrictions to
+
+Included text documents
+-----------------------
+
+Currently, PyPI attempts to determine the description's markup format by
+rendering it as reStructuredText, and if that fails, treating it as plain
+text.
+
+Furthermore, many projects simply read their long description in from an
+existing README file in ``setup.py``. The popularity of this practice is
+only expected to increase, as many online version control systems
+(including both GitHub and BitBucket) automatically display such files
+on the landing page for the project.
+
+Standardising on the inclusion of the long description as a separate
+file in the ``dist-info`` directory allows this to be simplified:
+
+* An existing file can just be copied into the ``dist-info`` directory as
+  part of creating the sdist
+* The expected markup format can be determined by inspecting the file
+  extension of the specified path
+
+Allowing the intended format to be stated explicitly in the path allows
+the format guessing to be removed and more informative error reports to be
+provided to users when a rendering error occurs.
+
+This is especially helpful since PyPI applies additional restrictions to
 the rendering process for security reasons, thus a description that renders
 correctly on a developer's system may still fail to render on the server.
 
-
-Deferred features
-=================
+The document naming system used to achieve this then makes it relatively
+straightforward to allow declaration of alternative markup formats like
+HTML, Markdown and AsciiDoc through the use of appropriate file
+extensions, as well as to define similar included documents for the
+project's license and changelog.
+
+Grouping the included document names into a single top level field gives
+automated tools the option of treating them as arbitrary documents without
+worrying about their contents.
+
+Requiring that the included documents be added to the ``dist-info`` metadata
+directory means that the complete metadata for the distribution can be
+extracted from an sdist or binary archive simply by extracting that
+directory, without needing to check for references to other files in the
+sdist.
+
+
+Appendix D: Deferred features
+=============================
 
 Several potentially useful features have been deliberately deferred in
 order to better prioritise our efforts in migrating to the new metadata
@@ -1847,15 +2177,26 @@
 new metadata, but which can be readily added in metadata 2.1 without
 breaking any use cases already supported by metadata 2.0.
 
-Once the ``pypi``, ``setuptools``, ``pip`` and ``distlib`` projects
-support creation and consumption of metadata 2.0, then we may revisit
-the creation of metadata 2.1 with these additional features.
-
-.. note::
-
-   Given the nature of this PEP as an interoperability specification,
-   this section will probably be removed before the PEP is accepted.
-   However, it's useful to have it here while discussion is ongoing.
+Once the ``pypi``, ``setuptools``, ``pip``, ``wheel`` and ``distlib``
+projects support creation and consumption of metadata 2.0, then we may
+revisit the creation of metadata 2.1 with some or all of these additional
+features.
+
+
+MIME type registration
+----------------------
+
+At some point after acceptance of the PEP, I will likely submit the
+following MIME type registration requests to IANA:
+
+* Full metadata: ``application/vnd.python.pymeta+json``
+* Abbreviated metadata: ``application/vnd.python.pymeta-short+json``
+* Essential dependency resolution metadata:
+  ``application/vnd.python.pymeta-dependencies+json``
+
+It's even possible we may be able to just register the ``vnd.python``
+namespace under the banner of the PSF rather than having to register
+the individual subformats.
 
 
 String methods in environment markers
@@ -1870,61 +2211,82 @@
 than a little strange.
 
 
-Module listing
---------------
-
-A top level ``"module"`` key, referencing a list of strings, with each
-giving the fully qualified name of a public package or module provided
-by the distribution.
-
-A flat list would be used in order to correctly accommodate namespace
-packages (where a distribution may provide subpackages or submodules without
-explicitly providing the parent namespace package).
-
-Example::
-
-    "modules": [
-        "comfy.chair"
-    ]
+Module and file listings
+------------------------
+
+Derived metadata giving the modules and files included in built
+distributions may be useful at some point in the future. (At least RPM
+provides this, and I believe the APT equivalent does as well)
 
 Explicitly providing a list of public module names will likely help
 with enabling features in RPM like "Requires: python(requests)", as well
 as providing richer static metadata for analysis from PyPI.
 
-However, this is just extra info that doesn't impact installing from wheels,
-so it is a good candidate for postponing to metadata 2.1.
-
-
-Additional metabuild hooks
---------------------------
-
-The following draft metabuild operations have been deferred for now:
+However, this is just extra info that doesn't impact reliably installing
+from wheels, so it is a good candidate for postponing to metadata 2.1
+(at the earliest).
+
+
+Additional install hooks
+------------------------
+
+In addition to the postinstall and preuninstall hooks described in the PEP,
+other distribution systems (like RPM) include the notion of preinstall
+and postuninstall hooks. These hooks would run with the runtime dependencies
+installed, but without the distribution itself. These have been deliberately
+omitted, as they're well suited to being explored further as metadata
+extensions.
+
+Similarly, the idea of "optional" postinstall and preuninstall hooks can
+be pursued as a metadata extension.
+
+By contrast, the mandatory postinstall and preuninstall hooks have been
+included directly in the PEP, specifically to ensure installation tools
+don't silently ignore them. This ensures users will either be able to
+install such distributions, or else receive an explicit error at installation
+time.
+
+
+Metabuild system
+----------------
+
+This version of the metadata specification continues to use ``setup.py``
+and the distutils command syntax to invoke build and test related
+operations on a source archive or VCS checkout.
+
+It may be desirable to replace these in the future with tool independent
+entry points that support:
 
 * Generating the metadata file on a development system
-* Generating a source archive on a development system
+* Generating an sdist on a development system
 * Generating a binary archive on a build system
+* Running the test suite on a built (but not installed) distribution
 
 Metadata 2.0 deliberately focuses on wheel based installation, leaving
-tarball and sdist based installation to use the existing ``setup.py``
-based ``distutils`` command interface.
-
-In the meantime, the above three operations will continue to be handled
-through the ``distutils``/``setuptools`` command system:
+sdist, source archive, and VCS checkout based installation to use the
+existing ``setup.py`` based ``distutils`` command interface.
+
+In the meantime, the above operations will be handled through the
+``distutils``/``setuptools`` command system:
 
 * ``python setup.py dist_info``
 * ``python setup.py sdist``
+* ``python setup.py build_ext --inplace``
+* ``python setup.py test``
 * ``python setup.py bdist_wheel``
 
-The following additional metabuild hooks may be added in metadata 2.1 to
+The following metabuild hooks may be defined in metadata 2.1 to
 cover these operations without relying on ``setup.py``:
 
-* ``make_dist_info``: generate the source archive's dist_info directory
-* ``make_sdist``: construct a source archive
-* ``build_wheel``: construct a binary wheel archive from an sdist source
-  archive
-
-Tentative signatures have been designed for those hooks, but they will
-not be pursued further until 2.1::
+* ``make_dist_info``: generate the sdist's dist_info directory
+* ``make_sdist``: create the contents of an sdist
+* ``build_dist``: create the contents of a binary wheel archive from an
+  unpacked sdist
+* ``test_built_dist``: run the test suite for a built distribution
+
+Tentative signatures have been designed for those hooks, but in order to
+better focus initial development efforts on the integration and installation
+use cases, they will not be pursued further until metadata 2.1::
 
     def make_dist_info(source_dir, info_dir):
         """Generate the contents of dist_info for an sdist archive
@@ -1949,11 +2311,11 @@
         Returns the distribution metadata as a dictionary.
         """
 
-    def build_wheel(sdist_dir, contents_dir, info_dir, compatibility=None):
-        """Generate the contents of a wheel archive
-
-        *source_dir* points to an unpacked source archive
-        *contents_dir* is the destination where the wheel contents should be
+    def build_dist(sdist_dir, built_dir, info_dir, compatibility=None):
+        """Generate the contents of a binary wheel archive
+
+        *sdist_dir* points to an unpacked sdist
+        *built_dir* is the destination where the wheel contents should be
         written (note that archiving the contents is the responsibility of
         the metabuild tool rather than the hook function)
         *info_dir* is the destination where the wheel metadata files should
@@ -1965,36 +2327,93 @@
         Returns the actual compatibility tag for the build
         """
 
-As with the existing metabuild hooks, checking for extras would be done
+    def test_built_dist(sdist_dir, built_dir, info_dir):
+        """Check a built (but not installed) distribution works as expected
+
+        *sdist_dir* points to an unpacked sdist
+        *built_dir* points to a platform appropriate unpacked wheel archive
+        (which may be missing the wheel metadata directory)
+        *info_dir* points to the appropriate wheel metadata directory
+
+        Requires that the distribution's test dependencies be installed
+        (indicated by the ``:test:`` extra).
+
+        Returns ``True`` if the check passes, ``False`` otherwise.
+        """
+
+As with the existing install hooks, checking for extras would be done
 using the same import based checks as are used for runtime extras. That
 way it doesn't matter if the additional dependencies were requested
 explicitly or just happen to be available on the system.
 
-
-Rejected Features
-=================
+There are still a number of open questions with this design, such as whether
+a single build hook is sufficient to cover both "build for testing" and
+"prep for deployment", as well as various complexities like support for
+cross-compilation of binaries, specification of target platforms and
+Python versions when creating wheel files, etc.
+
+Opting to retain the status quo for now allows us to make progress on
+improved metadata publication and binary installation support, rather than
+having to delay that awaiting the creation of a viable metabuild framework.
+
+
+Appendix E: Rejected features
+=============================
 
 The following features have been explicitly considered and rejected as
 introducing too much additional complexity for too small a gain in
 expressiveness.
 
-.. note::
-
-   Given the nature of this PEP as an interoperability specification,
-   this section will probably be removed before the PEP is accepted.
-   However, it's useful to have it here while discussion is ongoing.
-
-
-Detached metadata
------------------
-
-Rather than allowing some large items (such as the description field) to
-be distributed separately, this PEP instead defines two metadata subsets
-that should support more reasonable caching and API designs (for example,
-only the essential dependency resolution metadata would be distributed
-through TUF, and it is entirely possible the updated sdist, wheel and
-installation database specs will use the abbreviated metadata, leaving
-the full metadata as the province of index servers).
+
+Disallowing underscores in distribution names
+---------------------------------------------
+
+Debian doesn't actually permit underscores in names, but that seems
+unduly restrictive for this spec given the common practice of using
+valid Python identifiers as Python distribution names. A Debian side
+policy of converting underscores to hyphens seems easy enough to
+implement (and the requirement to consider hyphens and underscores as
+equivalent ensures that doing so won't introduce any conflicts).
+
+
+Allowing the use of Unicode in distribution names
+-------------------------------------------------
+
+This PEP deliberately avoids following Python 3 down the path of arbitrary
+Unicode identifiers, as the security implications of doing so are
+substantially worse in the software distribution use case (it opens
+up far more interesting attack vectors than mere code obfuscation).
+
+In addition, the existing tools really only work properly if you restrict
+names to ASCII and changing that would require a *lot* of work for all
+the automated tools in the chain.
+
+It may be reasonable to revisit this question at some point in the (distant)
+future, but setting up a more reliable software distribution system is
+challenging enough without adding more general Unicode identifier support
+into the mix.
+
+
+Single list for conditional and unconditional dependencies
+----------------------------------------------------------
+
+It's technically possible to store the conditional and unconditional
+dependencies of each kind in a single list and switch the handling based on
+the entry type (string or mapping).
+
+However, the current ``*requires`` vs ``*may-require`` two list design seems
+easier to understand and work with, since it's only the conditional
+dependencies that need to be checked against the requested extras list and
+the target installation environment.
+
+
+Depending on source labels
+--------------------------
+
+There is no mechanism to express a dependency on a source label - they
+are included in the metadata for internal project reference only. Instead,
+dependencies must be expressed in terms of either public versions or else
+direct URL references.
 
 
 Alternative dependencies
@@ -2019,7 +2438,7 @@
 database driver" metadata extension where a project depends on SQL Alchemy,
 and then declares in the extension which database drivers are checked for
 compatibility by the upstream project (similar to the advisory
-``supports-platform`` field in the main metadata).
+``supports_environments`` field in the main metadata).
 
 We're also getting better support for "virtual provides" in this version of
 the metadata standard, so this may end up being an installer and index
@@ -2047,9 +2466,67 @@
 
 Under the revised metadata design, conditional "provides" based on runtime
 features or the environment would go in a separate "may_provide" field.
-However, I'm not convinced there's a great use case for that, so the idea
+However, it isn't clear there's any use case for doing that, so the idea
 is rejected unless someone can present a compelling use case (and even then
-the idea wouldn't be reconsidered until metadata 2.1 at the earliest).
+the idea won't be reconsidered until metadata 2.1 at the earliest).
+
+
+A hook to run tests against installed distributions
+---------------------------------------------------
+
+Earlier drafts of this PEP defined a hook for running automated
+tests against an *installed* distribution. This isn't actually what you
+generally want - you want the ability to test a *built* distribution,
+potentially relying on files which won't be included in the binary archives.
+
+RPM's "check" step also runs between the build step and the install step,
+rather than after the install step.
+
+Accordingly, the ``test_installed_dist`` hook has been removed, and the
+``test_built_dist`` metabuild hook has been tentatively defined. However,
+along with the rest of the metabuild hooks, further consideration has been
+deferred until metadata 2.1 at the earliest.
+
+
+Extensible signatures for the install hooks
+-------------------------------------------
+
+The install hooks have been deliberately designed to NOT accept arbitary
+keyword arguments that the hook implementation is then expected to ignore.
+
+The argument in favour of that API design technique is to allow the addition
+of new optional arguments in the future, without requiring the definition
+of a new install hook, or migration to version 3.0 of the metadata
+specification. It is a technique very commonly seen in function wrappers
+which merely pass arguments along to the inner function rather than
+processing them directly.
+
+However, the install hooks are already designed to have access to the full
+metadata for the distribution (including all metadata extensions and
+the previous/next version when appropriate), as well as to the full target
+deployment environment.
+
+This means there are two candidates for additional information that
+could be passed as arbitrary keyword arguments:
+
+* installer dependent settings
+* user provided installation options
+
+The first of those runs explicitly counter to one of the core goals of the
+metadata 2.0 specification: decoupling the software developer's choice of
+development and publication tools from the software integrator's choice of
+integration and deployment tools.
+
+The second is a complex problem that has a readily available workaround in
+the form of operating system level environment variables (this is also
+one way to interoperate with platform specific installation tools).
+
+Alternatively, installer developers may either implicitly inject an
+additional metadata extension when invoking the install hook, or else
+define an alternate hook signature as a distinct metadata extension to be
+provided by the distribution. Either of these approaches makes the
+reliance on installer-dependent behaviour suitably explicit in either
+the install hook implementation or the distribution metadata.
 
 
 References
diff --git a/pep-0426/pymeta-schema.json b/pep-0426/pymeta-schema.json
new file mode 100644
--- /dev/null
+++ b/pep-0426/pymeta-schema.json
@@ -0,0 +1,249 @@
+{
+  "id": "http://www.python.org/dev/peps/pep-0426/",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Metadata for Python Software Packages 2.0",
+  "type": "object",
+  "properties": {
+    "metadata_version": {
+      "description": "Version of the file format",
+      "type": "string",
+      "pattern": "^(\\d+(\\.\\d+)*)$"
+    },
+    "generator": {
+      "description": "Name and version of the program that produced this file.",
+      "type": "string",
+      "pattern": "^[0-9A-Za-z]([0-9A-Za-z_.-]*[0-9A-Za-z])( \\((\\d+(\\.\\d+)*)((a|b|c|rc)(\\d+))?(\\.(post)(\\d+))?(\\.(dev)(\\d+))\\))?$"
+    },
+    "name": {
+      "description": "The name of the distribution.",
+      "type": "string",
+      "pattern": "^[0-9A-Za-z]([0-9A-Za-z_.-]*[0-9A-Za-z])?$"
+    },
+    "version": {
+      "description": "The distribution's public version identifier",
+      "type": "string",
+      "pattern": "^(\\d+(\\.\\d+)*)((a|b|c|rc)(\\d+))?(\\.(post)(\\d+))?(\\.(dev)(\\d+))?$"
+    },
+    "source_label": {
+      "description": "A constrained identifying text string",
+      "type": "string",
+      "pattern": "^[0-9a-z_.-+]+$"
+    },
+    "source_url": {
+      "description": "A string containing a full URL where the source for this specific version of the distribution can be downloaded.",
+      "type": "string",
+      "format": "uri"
+    },
+    "summary": {
+      "description": "A one-line summary of what the distribution does.",
+      "type": "string"
+    },
+    "document_names": {
+      "description": "Names of supporting metadata documents",
+      "type": "object",
+      "properties": {
+        "description": {
+          "type": "string",
+          "$ref": "#/definitions/document_name"
+        },
+        "changelog": {
+          "type": "string",
+          "$ref": "#/definitions/document_name"
+        },
+        "license": {
+          "type": "string",
+          "$ref": "#/definitions/document_name"
+        }
+      },
+      "additionalProperties": false
+    },
+    "keywords": {
+      "description": "A list of additional keywords to be used to assist searching for the distribution in a larger catalog.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "license": {
+      "description": "A string indicating the license covering the distribution.",
+      "type": "string"
+    },
+    "classifiers": {
+      "description": "A list of strings, with each giving a single classification value for the distribution.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "contacts": {
+      "description": "A list of contributor entries giving the recommended contact points for getting more information about the project.",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "$ref": "#/definitions/contact"
+      }
+    },
+    "contributors": {
+      "description": "A list of contributor entries for other contributors not already listed as current project points of contact.",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "$ref": "#/definitions/contact"
+      }
+    },
+    "project_urls": {
+      "description": "A mapping of arbitrary text labels to additional URLs relevant to the project.",
+      "type": "object"
+    },
+    "extras": {
+      "description": "A list of optional sets of dependencies that may be used to define conditional dependencies in \"may_require\" and similar fields.",
+      "type": "array",
+      "items": {
+        "type": "string",
+        "$ref": "#/definitions/extra_name"
+      }
+    },
+    "distributes": {
+      "description": "A list of subdistributions made available through this metadistribution.",
+      "type": "array",
+      "$ref": "#/definitions/dependencies"
+    },
+    "may_distribute": {
+      "description": "A list of subdistributions that may be made available through this metadistribution, based on the extras requested and the target deployment environment.",
+      "$ref": "#/definitions/conditional_dependencies"
+    },
+    "run_requires": {
+      "description": "A list of other distributions needed when to run this distribution.",
+      "type": "array",
+      "$ref": "#/definitions/dependencies"
+    },
+    "run_may_require": {
+      "description": "A list of other distributions that may be needed when this distribution is deployed, based on the extras requested and the target deployment environment.",
+      "$ref": "#/definitions/conditional_dependencies"
+    },
+    "test_requires": {
+      "description": "A list of other distributions needed when this distribution is tested.",
+      "type": "array",
+      "$ref": "#/definitions/dependencies"
+    },
+    "test_may_require": {
+      "description": "A list of other distributions that may be needed when this distribution is tested, based on the extras requested and the target deployment environment.",
+      "type": "array",
+      "$ref": "#/definitions/conditional_dependencies"
+    },
+    "build_requires": {
+      "description": "A list of other distributions needed when this distribution is built.",
+      "type": "array",
+      "$ref": "#/definitions/dependencies"
+    },
+    "build_may_require": {
+      "description": "A list of other distributions that may be needed when this distribution is built, based on the extras requested and the target deployment environment.",
+      "type": "array",
+      "$ref": "#/definitions/conditional_dependencies"
+    },
+    "dev_requires": {
+      "description": "A list of other distributions needed when this distribution is developed.",
+      "type": "array",
+      "$ref": "#/definitions/dependencies"
+    },
+    "dev_may_require": {
+      "description": "A list of other distributions that may be needed when this distribution is developed, based on the extras requested and the target deployment environment.",
+      "type": "array",
+      "$ref": "#/definitions/conditional_dependencies"
+    },
+    "provides": {
+      "description": "A list of strings naming additional dependency requirements that are satisfied by installing this distribution. These strings must be of the form Name or Name (Version), as for the requires field.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "obsoleted_by": {
+      "description": "A string that indicates that this project is no longer being developed. The named project provides a substitute or replacement.",
+      "type": "string",
+      "$ref": "#/definitions/version_specifier"
+    },
+    "supports_environments": {
+      "description": "A list of strings specifying the environments that the distribution explicitly supports.",
+      "type": "array",
+      "items": {
+        "type": "string",
+        "$ref": "#/definitions/environment_marker"
+      }
+    },
+    "metabuild_hooks": {
+      "description": "The metabuild_hooks field is used to define various operations that may be invoked on a distribution in a platform independent manner.",
+      "type": "object"
+    },
+    "extensions": {
+      "description": "Extensions to the metadata may be present in a mapping under the 'extensions' key.",
+      "type": "object"
+    }
+  },
+
+  "required": ["metadata_version", "name", "version"],
+  "additionalProperties": false,
+
+  "definitions": {
+    "contact": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "email": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        },
+        "role": {
+          "type": "string"
+        }
+      },
+      "required": ["name"],
+      "additionalProperties": false
+    },
+    "dependencies": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "$ref": "#/definitions/version_specifier"
+      }
+    },
+    "conditional_dependencies": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "extra": {
+            "type": "string",
+            "$ref": "#/definitions/extra_name"
+          },
+          "environment": {
+            "type": "string",
+            "$ref": "#/definitions/environment_marker"
+          },
+          "dependencies": {
+            "type": "array",
+            "$ref": "#/definitions/dependencies"
+          }
+        },
+        "required": ["dependencies"],
+        "additionalProperties": false
+      }
+    },
+    "version_specifier": {
+        "type": "string"
+    },
+    "extra_name": {
+        "type": "string"
+    },
+    "environment_marker": {
+        "type": "string"
+    },
+    "document_name": {
+        "type": "string"
+    }
+  }
+}
diff --git a/pep-0435.txt b/pep-0435.txt
--- a/pep-0435.txt
+++ b/pep-0435.txt
@@ -5,7 +5,7 @@
 Author: Barry Warsaw <barry at python.org>,
         Eli Bendersky <eliben at gmail.com>,
         Ethan Furman <ethan at stoneleaf.us>
-Status: Accepted
+Status: Final
 Type: Standards Track
 Content-Type: text/x-rst
 Created: 2013-02-23
@@ -467,6 +467,10 @@
     ...   cat = 3
     ...   dog = 4
 
+The reason for defaulting to ``1`` as the starting number and not ``0`` is
+that ``0`` is ``False`` in a boolean sense, but enum members all evaluate
+to ``True``.
+
 
 Proposed variations
 ===================
diff --git a/pep-0440.txt b/pep-0440.txt
--- a/pep-0440.txt
+++ b/pep-0440.txt
@@ -9,7 +9,7 @@
 Type: Standards Track
 Content-Type: text/x-rst
 Created: 18 Mar 2013
-Post-History: 30 Mar 2013, 27-May-2013
+Post-History: 30 Mar 2013, 27 May 2013, 20 Jun 2013
 Replaces: 386
 
 
@@ -27,7 +27,7 @@
    This PEP was broken out of the metadata 2.0 specification in PEP 426.
 
    Unlike PEP 426, the notes that remain in this document are intended as
-   part of the final specification.
+   part of the final specification (except for this one).
 
 
 Definitions
@@ -40,7 +40,7 @@
 The following terms are to be interpreted as described in PEP 426:
 
 * "Distributions"
-* "Versions"
+* "Releases"
 * "Build tools"
 * "Index servers"
 * "Publication tools"
@@ -52,9 +52,13 @@
 Version scheme
 ==============
 
-Distribution versions are identified by both a public version identifier,
-which supports all defined version comparison operations, and a build
-label, which supports only strict equality comparisons.
+Distributions are identified by a public version identifier which
+supports all defined version comparison operations
+
+Distributions may also define a source label, which is not used by
+automated tools. Source labels are useful when a project internal
+versioning scheme requires translation to create a compliant public
+version identifier.
 
 The version scheme is used both to describe the distribution version
 provided by a particular distribution archive, as well as to place
@@ -84,7 +88,7 @@
 * Post-release segment: ``.postN``
 * Development release segment: ``.devN``
 
-Any given version will be a "release", "pre-release", "post-release" or
+Any given release will be a "final release", "pre-release", "post-release" or
 "developmental release" as defined in the following sections.
 
 .. note::
@@ -105,28 +109,37 @@
 Source labels are text strings with minimal defined semantics.
 
 To ensure source labels can be readily incorporated as part of file names
-and URLs, they MUST be comprised of only ASCII alphanumerics, plus signs,
-periods and hyphens.
+and URLs, and to avoid formatting inconsistences in hexadecimal hash
+representations they MUST be limited to the following set of permitted
+characters:
 
-In addition, source labels MUST be unique within a given distribution.
+* Lowercase ASCII letters (``[a-z]``)
+* ASCII digits (``[0-9]``)
+* underscores (``_``)
+* hyphens (``-``)
+* periods (``.``)
+* plus signs (``+``)
 
-As with distribution names, all comparisons of source labels MUST be case
-insensitive.
+Source labels MUST start and end with an ASCII letter or digit.
 
+Source labels MUST be unique within each project and MUST NOT match any
+defined version for the project.
 
-Releases
---------
 
-A version identifier that consists solely of a release segment is termed
-a "release".
+Final releases
+--------------
 
-The release segment consists of one or more non-negative integer values,
-separated by dots::
+A version identifier that consists solely of a release segment is
+termed a "final release".
+
+The release segment consists of one or more non-negative integer
+values, separated by dots::
 
     N[.N]+
 
-Releases within a project will typically be numbered in a consistently
-increasing fashion.
+Final releases within a project MUST be numbered in a consistently
+increasing fashion, otherwise automated tools will not be able to upgrade
+them correctly.
 
 Comparison and ordering of release segments considers the numeric value
 of each component of the release segment in turn. When comparing release
@@ -157,8 +170,8 @@
     2.0
     2.0.1
 
-A release series is any set of release numbers that start with a common
-prefix. For example, ``3.3.1``, ``3.3.5`` and ``3.3.9.45`` are all
+A release series is any set of final release numbers that start with a
+common prefix. For example, ``3.3.1``, ``3.3.5`` and ``3.3.9.45`` are all
 part of the ``3.3`` release series.
 
 .. note::
@@ -206,8 +219,8 @@
 Post-releases
 -------------
 
-Some projects use post-releases to address minor errors in a release that
-do not affect the distributed software (for example, correcting an error
+Some projects use post-releases to address minor errors in a final release
+that do not affect the distributed software (for example, correcting an error
 in the release notes).
 
 If used as part of a project's development cycle, these post-releases are
@@ -371,7 +384,7 @@
    .devN, aN, bN, cN, rcN, <no suffix>, .postN
 
 Note that `rc` will always sort after `c` (regardless of the numeric
-component) although they are semantically equivalent. Tools are free to
+component) although they are semantically equivalent. Tools MAY
 reject this case as ambiguous and remain in compliance with the PEP.
 
 Within an alpha (``1.0a1``), beta (``1.0b1``), or release candidate
@@ -506,6 +519,22 @@
 version comparison semantics.
 
 
+Olson database versioning
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``pytz`` project inherits its versioning scheme from the corresponding
+Olson timezone database versioning scheme: the year followed by a lowercase
+character indicating the version of the database within that year.
+
+This can be translated to a compliant 3-part version identifier as
+``0.<year>.<serial>``, where the serial starts at zero (for the '<year>a'
+release) and is incremented with each subsequent database update within the
+year.
+
+As with other translated version identifiers, the corresponding Olson
+database version would be recorded in the source label field.
+
+
 Version specifiers
 ==================
 
@@ -521,7 +550,6 @@
 * ``~=``: `Compatible release`_ clause
 * ``==``: `Version matching`_ clause
 * ``!=``: `Version exclusion`_ clause
-* ``is``: `Source reference`_ clause
 * ``<=``, ``>=``: `Inclusive ordered comparison`_ clause
 * ``<``, ``>``: `Exclusive ordered comparison`_ clause
 
@@ -605,6 +633,11 @@
 release segment to ensure the release segments are compared with the same
 length.
 
+Whether or not strict version matching is appropriate depends on the specific
+use case for the version specifier. Automated tools SHOULD at least issue
+warnings and MAY reject them entirely when strict version matches are used
+inappropriately.
+
 Prefix matching may be requested instead of strict comparison, by appending
 a trailing ``.*`` to the version identifier in the version matching clause.
 This means that additional trailing segments will be ignored when
@@ -645,75 +678,6 @@
     != 1.1.*      # Same prefix, so 1.1.post1 does not match clause
 
 
-Source reference
-----------------
-
-A source reference includes the source reference operator ``is`` and
-a source label or a source URL.
-
-Installation tools MAY also permit direct references to a platform
-appropriate binary archive in a source reference clause.
-
-Publication tools and public index servers SHOULD NOT permit direct
-references to a platform appropriate binary archive in a source
-reference clause.
-
-Source label matching works solely on strict equality comparisons: the
-candidate source label must be exactly the same as the source label in the
-version clause for the clause to match the candidate distribution.
-
-For example, a source reference could be used to depend directly on a
-version control hash based identifier rather than the translated public
-version::
-
-    exact-dependency (is 1.3.7+build.11.e0f985a)
-
-A source URL is distinguished from a source label by the presence of
-``:`` and ``/`` characters in the source reference. As these characters
-are not permitted in source labels, they indicate that the reference uses
-a source URL.
-
-Some appropriate targets for a source URL are a source tarball, an sdist
-archive or a direct reference to a tag or specific commit in an online
-version control system. The exact URLs and
-targets supported will be installation tool specific.
-
-For example, a local source archive may be referenced directly::
-
-    pip (is file:///localbuilds/pip-1.3.1.zip)
-
-All source URL references SHOULD either specify a local file URL, a secure
-transport mechanism (such as ``https``) or else include an expected hash
-value in the URL for verification purposes. If an insecure network
-transport is specified without any hash information (or with hash
-information that the tool doesn't understand), automated tools SHOULD
-at least emit a warning and MAY refuse to rely on the URL.
-
-It is RECOMMENDED that only hashes which are unconditionally provided by
-the latest version of the standard library's ``hashlib`` module be used
-for source archive hashes. At time of writing, that list consists of
-``'md5'``, ``'sha1'``, ``'sha224'``, ``'sha256'``, ``'sha384'``, and
-``'sha512'``.
-
-For source archive references, an expected hash value may be
-specified by including a ``<hash-algorithm>=<expected-hash>`` as part of
-the URL fragment.
-
-For version control references, the ``VCS+protocol`` scheme SHOULD be
-used to identify both the version control system and the secure transport.
-
-To support version control systems that do not support including commit or
-tag references directly in the URL, that information may be appended to the
-end of the URL using the ``@<tag>`` notation.
-
-The use of ``is`` when defining dependencies for published distributions
-is strongly discouraged as it greatly complicates the deployment of
-security fixes. The source label matching operator is intended primarily
-for use when defining dependencies for repeatable *deployments of
-applications* while using a shared distribution index, as well as to
-reference dependencies which are not published through an index server.
-
-
 Inclusive ordered comparison
 ----------------------------
 
@@ -752,62 +716,108 @@
 ------------------------
 
 Pre-releases of any kind, including developmental releases, are implicitly
-excluded from all version specifiers, *unless* a pre-release or developmental
-release is explicitly mentioned in one of the clauses. For example, these
-specifiers implicitly exclude all pre-releases and development
-releases of later versions::
-
-    2.2
-    >= 1.0
-
-While these specifiers would include at least some of them::
-
-    2.2.dev0
-    2.2, != 2.3b2
-    >= 1.0a1
-    >= 1.0c1
-    >= 1.0, != 1.0b2
-    >= 1.0, < 2.0.dev123
+excluded from all version specifiers, *unless* they are already present
+on the system, explicitly requested by the user, or if the only available
+version that satisfies the version specifier is a pre-release.
 
 By default, dependency resolution tools SHOULD:
 
 * accept already installed pre-releases for all version specifiers
-* accept remotely available pre-releases for version specifiers which
-  include at least one version clauses that references a pre-release
+* accept remotely available pre-releases for version specifiers where
+  there is no final or post release that satisfies the version specifier
 * exclude all other pre-releases from consideration
 
+Dependency resolution tools MAY issue a warning if a pre-release is needed
+to satisfy a version specifier.
+
 Dependency resolution tools SHOULD also allow users to request the
 following alternative behaviours:
 
 * accepting pre-releases for all version specifiers
 * excluding pre-releases for all version specifiers (reporting an error or
-  warning if a pre-release is already installed locally)
+  warning if a pre-release is already installed locally, or if a
+  pre-release is the only way to satisfy a particular specifier)
 
 Dependency resolution tools MAY also allow the above behaviour to be
 controlled on a per-distribution basis.
 
-Post-releases and purely numeric releases receive no special treatment in
-version specifiers - they are always included unless explicitly excluded.
+Post-releases and final releases receive no special treatment in version
+specifiers - they are always included unless explicitly excluded.
 
 
 Examples
 --------
 
-* ``3.1``: version 3.1 or later, but not
-   version 4.0 or later. Excludes pre-releases and developmental releases.
-* ``3.1.2``: version 3.1.2 or later, but not
-   version 3.2.0 or later. Excludes pre-releases and developmental releases.
-* ``3.1a1``: version 3.1a1 or later, but not
-   version 4.0 or later. Allows pre-releases like 3.2a4 and developmental
-   releases like 3.2.dev1.
+* ``3.1``: version 3.1 or later, but not version 4.0 or later.
+* ``3.1.2``: version 3.1.2 or later, but not version 3.2.0 or later.
+* ``3.1a1``: version 3.1a1 or later, but not version 4.0 or later.
 * ``== 3.1``: specifically version 3.1 (or 3.1.0), excludes all pre-releases,
   post releases, developmental releases and any 3.1.x maintenance releases.
-* ``== 3.1.*``: any version that starts with 3.1, excluding pre-releases and
-  developmental releases. Equivalent to the ``3.1.0`` compatible release
-  clause.
+* ``== 3.1.*``: any version that starts with 3.1. Equivalent to the
+  ``3.1.0`` compatible release clause.
 * ``3.1.0, != 3.1.3``: version 3.1.0 or later, but not version 3.1.3 and
-  not version 3.2.0 or later. Excludes pre-releases and developmental
-  releases.
+  not version 3.2.0 or later.
+
+
+Direct references
+=================
+
+Some automated tools may permit the use of a direct reference as an
+alternative to a normal version specifier. A direct reference consists of
+the word ``from`` and an explicit URL.
+
+Whether or not direct references are appropriate depends on the specific
+use case for the version specifier. Automated tools SHOULD at least issue
+warnings and MAY reject them entirely when direct references are used
+inappropriately.
+
+Public index servers SHOULD NOT allow the use of direct references in
+uploaded distributions. Direct references are intended as a tool for
+software integrators rather than publishers.
+
+Depending on the use case, some appropriate targets for a direct URL
+reference may be a valid ``source_url`` entry (see PEP 426), an sdist, or
+a wheel binary archive. The exact URLs and targets supported will be tool
+dependent.
+
+For example, a local source archive may be referenced directly::
+
+    pip (from file:///localbuilds/pip-1.3.1.zip)
+
+Alternatively, a prebuilt archive may also be referenced::
+
+    pip (from file:///localbuilds/pip-1.3.1-py33-none-any.whl)
+
+All direct references that do not refer to a local file URL SHOULD
+specify a secure transport mechanism (such as ``https``), include an
+expected hash value in the URL for verification purposes, or both. If an
+insecure transport is specified without any hash information, with hash
+information that the tool doesn't understand, or with a selected hash
+algorithm that the tool considers too weak to trust, automated tools
+SHOULD at least emit a warning and MAY refuse to rely on the URL.
+
+It is RECOMMENDED that only hashes which are unconditionally provided by
+the latest version of the standard library's ``hashlib`` module be used
+for source archive hashes. At time of writing, that list consists of
+``'md5'``, ``'sha1'``, ``'sha224'``, ``'sha256'``, ``'sha384'``, and
+``'sha512'``.
+
+For source archive and wheel references, an expected hash value may be
+specified by including a ``<hash-algorithm>=<expected-hash>`` entry as
+part of the URL fragment.
+
+Version control references, the ``VCS+protocol`` scheme SHOULD be
+used to identify both the version control system and the secure transport.
+
+To support version control systems that do not support including commit or
+tag references directly in the URL, that information may be appended to the
+end of the URL using the ``@<tag>`` notation.
+
+Remote URL examples::
+
+    pip (from https://github.com/pypa/pip/archive/1.3.1.zip)
+    pip (from http://github.com/pypa/pip/archive/1.3.1.zip#sha1=da9234ee9982d4bbb3c72346a6de940a148ea686)
+    pip (from git+https://github.com/pypa/pip.git at 1.3.1)
 
 
 Updating the versioning specification
@@ -825,28 +835,30 @@
 
 * Moved the description of version specifiers into the versioning PEP
 
-* added the "source label" concept to better handle projects that wish to
+* Added the "source label" concept to better handle projects that wish to
   use a non-compliant versioning scheme internally, especially those based
   on DVCS hashes
-  
-* added the "compatible release" clause
 
-* added the "source reference" clause
+* Added the "direct reference" concept as a standard notation for direct
+  references to resources (rather than each tool needing to invents its own)
 
-* added the trailing wildcard syntax for prefix based version matching
+* Added the "compatible release" clause
+
+* Added the trailing wildcard syntax for prefix based version matching
   and exclusion
 
-* changed the top level sort position of the ``.devN`` suffix
+* Changed the top level sort position of the ``.devN`` suffix
 
-* allowed single value version numbers
+* Allowed single value version numbers
 
-* explicit exclusion of leading or trailing whitespace
+* Explicit exclusion of leading or trailing whitespace
 
-* explicit criterion for the exclusion of date based versions
+* Explicit criterion for the exclusion of date based versions
 
-* implicitly exclude pre-releases unless explicitly requested
+* Implicitly exclude pre-releases unless they're already present or
+  needed to satisfy a dependency
 
-* treat post releases the same way as unqualified releases
+* Treat post releases the same way as unqualified releases
 
 * Discuss ordering and dependencies across metadata versions
 
@@ -995,11 +1007,12 @@
 specifiers for no adequately justified reason.
 
 The updated interpretation is intended to make it difficult to accidentally
-accept a pre-release version as satisfying a dependency, while allowing
-pre-release versions to be explicitly requested when needed.
+accept a pre-release version as satisfying a dependency, while still
+allowing pre-release versions to be retrieved automatically when that's the
+only way to satisfy a dependency.
 
 The "some forward compatibility assumed" default version constraint is
-taken directly from the Ruby community's "pessimistic version constraint"
+derived from the Ruby community's "pessimistic version constraint"
 operator [2]_ to allow projects to take a cautious approach to forward
 compatibility promises, while still easily setting a minimum required
 version for their dependencies. It is made the default behaviour rather
@@ -1022,16 +1035,26 @@
 
 The trailing wildcard syntax to request prefix based version matching was
 added to make it possible to sensibly define both compatible release clauses
-and the desired pre-release handling semantics for ``<`` and ``>`` ordered
-comparison clauses.
+and the desired pre- and post-release handling semantics for ``<`` and ``>``
+ordered comparison clauses.
 
-Source references are added for two purposes. In conjunction with source
-labels, they allow hash based references to exact versions that aren't
-compliant with the fully ordered public version scheme, such as those
-generated from version control. In combination with source URLs, they
-also allow the new metadata standard to natively support an existing
-feature of ``pip``, which allows arbitrary URLs like
-``file:///localbuilds/exampledist-1.0-py33-none-any.whl``.
+
+Adding direct references
+------------------------
+
+Direct references are added as an "escape clause" to handle messy real
+world situations that don't map neatly to the standard distribution model.
+This includes dependencies on unpublished software for internal use, as well
+as handling the more complex compatibility issues that may arise when
+wrapping third party libraries as C extensions (this is of especial concern
+to the scientific community).
+
+Index servers are deliberately given a lot of freedom to disallow direct
+references, since they're intended primarily as a tool for integrators
+rather than publishers. PyPI in particular is currently going through the
+process of *eliminating* dependencies on external references, as unreliable
+external services have the effect of slowing down installation operations,
+as well as reducing PyPI's own apparent reliability.
 
 
 References
diff --git a/pep-0442.txt b/pep-0442.txt
--- a/pep-0442.txt
+++ b/pep-0442.txt
@@ -4,13 +4,13 @@
 Last-Modified: $Date$
 Author: Antoine Pitrou <solipsis at pitrou.net>
 BDFL-Delegate: Benjamin Peterson <benjamin at python.org>
-Status: Draft
+Status: Accepted
 Type: Standards Track
 Content-Type: text/x-rst
 Created: 2013-05-18
 Python-Version: 3.4
 Post-History: 2013-05-18
-Resolution: TBD
+Resolution: http://mail.python.org/pipermail/python-dev/2013-June/126746.html
 
 
 Abstract
@@ -201,8 +201,7 @@
 --------------
 
 Following this scheme, an object's finalizer is always called exactly
-once.  The only exception is if an object is resurrected: the finalizer
-will be called again when the object becomes unreachable again.
+once, even if it was resurrected afterwards.
 
 For CI objects, the order in which finalizers are called (step 2 above)
 is undefined.
diff --git a/pep-0443.txt b/pep-0443.txt
--- a/pep-0443.txt
+++ b/pep-0443.txt
@@ -4,7 +4,7 @@
 Last-Modified: $Date$
 Author: ?ukasz Langa <lukasz at langa.pl>
 Discussions-To: Python-Dev <python-dev at python.org>
-Status: Accepted
+Status: Final
 Type: Standards Track
 Content-Type: text/x-rst
 Created: 22-May-2013
@@ -193,48 +193,37 @@
 importantly, it introduces support for Abstract Base Classes (ABC).
 
 When a generic function implementation is registered for an ABC, the
-dispatch algorithm switches to a mode of MRO calculation for the
-provided argument which includes the relevant ABCs. The algorithm is as
-follows::
+dispatch algorithm switches to an extended form of C3 linearization,
+which includes the relevant ABCs in the MRO of the provided argument.
+The algorithm inserts ABCs where their functionality is introduced, i.e.
+``issubclass(cls, abc)`` returns ``True`` for the class itself but
+returns ``False`` for all its direct base classes. Implicit ABCs for
+a given class (either registered or inferred from the presence of
+a special method like ``__len__()``) are inserted directly after the
+last ABC explicitly listed in the MRO of said class.
 
-  def _compose_mro(cls, haystack):
-      """Calculates the MRO for a given class `cls`, including relevant
-      abstract base classes from `haystack`."""
-      bases = set(cls.__mro__)
-      mro = list(cls.__mro__)
-      for regcls in haystack:
-          if regcls in bases or not issubclass(cls, regcls):
-              continue   # either present in the __mro__ or unrelated
-          for index, base in enumerate(mro):
-              if not issubclass(base, regcls):
-                  break
-          if base in bases and not issubclass(regcls, base):
-              # Conflict resolution: put classes present in __mro__
-              # and their subclasses first.
-              index += 1
-          mro.insert(index, regcls)
-      return mro
-
-In its most basic form, it returns the MRO for the given type::
+In its most basic form, this linearization returns the MRO for the given
+type::
 
   >>> _compose_mro(dict, [])
   [<class 'dict'>, <class 'object'>]
 
-When the haystack consists of ABCs that the specified type is a subclass
-of, they are inserted in a predictable order::
+When the second argument contains ABCs that the specified type is
+a subclass of, they are inserted in a predictable order::
 
   >>> _compose_mro(dict, [Sized, MutableMapping, str,
   ...                     Sequence, Iterable])
   [<class 'dict'>, <class 'collections.abc.MutableMapping'>,
-   <class 'collections.abc.Iterable'>, <class 'collections.abc.Sized'>,
+   <class 'collections.abc.Mapping'>, <class 'collections.abc.Sized'>,
+   <class 'collections.abc.Iterable'>, <class 'collections.abc.Container'>,
    <class 'object'>]
 
 While this mode of operation is significantly slower, all dispatch
 decisions are cached. The cache is invalidated on registering new
 implementations on the generic function or when user code calls
-``register()`` on an ABC to register a new virtual subclass. In the
-latter case, it is possible to create a situation with ambiguous
-dispatch, for instance::
+``register()`` on an ABC to implicitly subclass it. In the latter case,
+it is possible to create a situation with ambiguous dispatch, for
+instance::
 
   >>> from collections import Iterable, Container
   >>> class P:
@@ -261,20 +250,38 @@
   RuntimeError: Ambiguous dispatch: <class 'collections.abc.Container'>
   or <class 'collections.abc.Iterable'>
 
-Note that this exception would not be raised if ``Iterable`` and
-``Container`` had been provided as base classes during class definition.
-In this case dispatch happens in the MRO order::
+Note that this exception would not be raised if one or more ABCs had
+been provided explicitly as base classes during class definition. In
+this case dispatch happens in the MRO order::
 
   >>> class Ten(Iterable, Container):
   ...     def __iter__(self):
   ...         for i in range(10):
   ...             yield i
   ...     def __contains__(self, value):
-  ...       return value in range(10)
+  ...         return value in range(10)
   ...
   >>> g(Ten())
   'iterable'
 
+A similar conflict arises when subclassing an ABC is inferred from the
+presence of a special method like ``__len__()`` or ``__contains__()``::
+
+  >>> class Q:
+  ...   def __contains__(self, value):
+  ...     return False
+  ...
+  >>> issubclass(Q, Container)
+  True
+  >>> Iterable.register(Q)
+  >>> g(Q())
+  Traceback (most recent call last):
+  ...
+  RuntimeError: Ambiguous dispatch: <class 'collections.abc.Container'>
+  or <class 'collections.abc.Iterable'>
+
+An early version of the PEP contained a custom approach that was simpler
+but created a number of edge cases with surprising results [#why-c3]_.
 
 Usage Patterns
 ==============
@@ -378,6 +385,8 @@
    a particular annotation style".
    (http://www.python.org/dev/peps/pep-0008)
 
+.. [#why-c3] http://bugs.python.org/issue18244
+
 .. [#pep-3124] http://www.python.org/dev/peps/pep-3124/
 
 .. [#peak-rules] http://peak.telecommunity.com/DevCenter/PEAK_2dRules
diff --git a/pep-0445.txt b/pep-0445.txt
new file mode 100644
--- /dev/null
+++ b/pep-0445.txt
@@ -0,0 +1,773 @@
+PEP: 445
+Title: Add new APIs to customize Python memory allocators
+Version: $Revision$
+Last-Modified: $Date$
+Author: Victor Stinner <victor.stinner at gmail.com>
+BDFL-Delegate: Antoine Pitrou <solipsis at pitrou.net>
+Status: Accepted
+Type: Standards Track
+Content-Type: text/x-rst
+Created: 15-june-2013
+Python-Version: 3.4
+Resolution: http://mail.python.org/pipermail/python-dev/2013-July/127222.html
+
+Abstract
+========
+
+This PEP proposes new Application Programming Interfaces (API) to customize
+Python memory allocators.  The only implementation required to conform to
+this PEP is CPython, but other implementations may choose to be compatible,
+or to re-use a similar scheme.
+
+
+Rationale
+=========
+
+Use cases:
+
+* Applications embedding Python which want to isolate Python memory from
+  the memory of the application, or want to use a different memory
+  allocator optimized for its Python usage
+* Python running on embedded devices with low memory and slow CPU.
+  A custom memory allocator can be used for efficiency and/or to get
+  access all the memory of the device.
+* Debug tools for memory allocators:
+
+  - track the memory usage (find memory leaks)
+  - get the location of a memory allocation: Python filename and line
+    number, and the size of a memory block
+  - detect buffer underflow, buffer overflow and misuse of Python
+    allocator APIs (see `Redesign Debug Checks on Memory Block
+    Allocators as Hooks`_)
+  - force memory allocations to fail to test handling of the
+    ``MemoryError`` exception
+
+
+Proposal
+========
+
+New Functions and Structures
+----------------------------
+
+* Add a new GIL-free (no need to hold the GIL) memory allocator:
+
+  - ``void* PyMem_RawMalloc(size_t size)``
+  - ``void* PyMem_RawRealloc(void *ptr, size_t new_size)``
+  - ``void PyMem_RawFree(void *ptr)``
+  - The newly allocated memory will not have been initialized in any
+    way.
+  - Requesting zero bytes returns a distinct non-*NULL* pointer if
+    possible, as if ``PyMem_Malloc(1)`` had been called instead.
+
+* Add a new ``PyMemAllocator`` structure::
+
+    typedef struct {
+        /* user context passed as the first argument to the 3 functions */
+        void *ctx;
+
+        /* allocate a memory block */
+        void* (*malloc) (void *ctx, size_t size);
+
+        /* allocate or resize a memory block */
+        void* (*realloc) (void *ctx, void *ptr, size_t new_size);
+
+        /* release a memory block */
+        void (*free) (void *ctx, void *ptr);
+    } PyMemAllocator;
+
+* Add a new ``PyMemAllocatorDomain`` enum to choose the Python
+  allocator domain. Domains:
+
+  - ``PYMEM_DOMAIN_RAW``: ``PyMem_RawMalloc()``, ``PyMem_RawRealloc()``
+    and ``PyMem_RawFree()``
+
+  - ``PYMEM_DOMAIN_MEM``: ``PyMem_Malloc()``, ``PyMem_Realloc()`` and
+    ``PyMem_Free()``
+
+  - ``PYMEM_DOMAIN_OBJ``: ``PyObject_Malloc()``, ``PyObject_Realloc()``
+    and ``PyObject_Free()``
+
+* Add new functions to get and set memory block allocators:
+
+  - ``void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)``
+  - ``void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)``
+  - The new allocator must return a distinct non-*NULL* pointer when
+    requesting zero bytes
+  - For the ``PYMEM_DOMAIN_RAW`` domain, the allocator must be
+    thread-safe: the GIL is not held when the allocator is called.
+
+* Add a new ``PyObjectArenaAllocator`` structure::
+
+    typedef struct {
+        /* user context passed as the first argument to the 2 functions */
+        void *ctx;
+
+        /* allocate an arena */
+        void* (*alloc) (void *ctx, size_t size);
+
+        /* release an arena */
+        void (*free) (void *ctx, void *ptr, size_t size);
+    } PyObjectArenaAllocator;
+
+* Add new functions to get and set the arena allocator used by
+  *pymalloc*:
+
+  - ``void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)``
+  - ``void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)``
+
+* Add a new function to reinstall the debug checks on memory allocators when
+  a memory allocator is replaced with ``PyMem_SetAllocator()``:
+
+  - ``void PyMem_SetupDebugHooks(void)``
+  - Install the debug hooks on all memory block allocators. The function can be
+    called more than once, hooks are only installed once.
+  - The function does nothing is Python is not compiled in debug mode.
+
+* Memory block allocators always return *NULL* if *size* is greater than
+  ``PY_SSIZE_T_MAX``. The check is done before calling the inner
+  function.
+
+.. note::
+    The *pymalloc* allocator is optimized for objects smaller than 512 bytes
+    with a short lifetime. It uses memory mappings with a fixed size of 256
+    KB called "arenas".
+
+Here is how the allocators are set up by default:
+
+* ``PYMEM_DOMAIN_RAW``, ``PYMEM_DOMAIN_MEM``: ``malloc()``,
+  ``realloc()`` and ``free()``; call ``malloc(1)`` when requesting zero
+  bytes
+* ``PYMEM_DOMAIN_OBJ``: *pymalloc* allocator which falls back on
+  ``PyMem_Malloc()`` for allocations larger than 512 bytes
+* *pymalloc* arena allocator: ``VirtualAlloc()`` and ``VirtualFree()`` on
+  Windows, ``mmap()`` and ``munmap()`` when available, or ``malloc()``
+  and ``free()``
+
+
+Redesign Debug Checks on Memory Block Allocators as Hooks
+---------------------------------------------------------
+
+Since Python 2.3, Python implements different checks on memory
+allocators in debug mode:
+
+* Newly allocated memory is filled with the byte ``0xCB``, freed memory
+  is filled with the byte ``0xDB``.
+* Detect API violations, ex: ``PyObject_Free()`` called on a memory
+  block allocated by ``PyMem_Malloc()``
+* Detect write before the start of the buffer (buffer underflow)
+* Detect write after the end of the buffer (buffer overflow)
+
+In Python 3.3, the checks are installed by replacing ``PyMem_Malloc()``,
+``PyMem_Realloc()``, ``PyMem_Free()``, ``PyObject_Malloc()``,
+``PyObject_Realloc()`` and ``PyObject_Free()`` using macros. The new
+allocator allocates a larger buffer and writes a pattern to detect buffer
+underflow, buffer overflow and use after free (by filling the buffer with
+the byte ``0xDB``). It uses the original ``PyObject_Malloc()``
+function to allocate memory. So ``PyMem_Malloc()`` and
+``PyMem_Realloc()`` indirectly call``PyObject_Malloc()`` and
+``PyObject_Realloc()``.
+
+This PEP redesigns the debug checks as hooks on the existing allocators
+in debug mode. Examples of call traces without the hooks:
+
+* ``PyMem_RawMalloc()`` => ``_PyMem_RawMalloc()`` => ``malloc()``
+* ``PyMem_Realloc()`` => ``_PyMem_RawRealloc()`` => ``realloc()``
+* ``PyObject_Free()`` => ``_PyObject_Free()``
+
+Call traces when the hooks are installed (debug mode):
+
+* ``PyMem_RawMalloc()`` => ``_PyMem_DebugMalloc()``
+  => ``_PyMem_RawMalloc()`` => ``malloc()``
+* ``PyMem_Realloc()`` => ``_PyMem_DebugRealloc()``
+  => ``_PyMem_RawRealloc()`` => ``realloc()``
+* ``PyObject_Free()`` => ``_PyMem_DebugFree()``
+  => ``_PyObject_Free()``
+
+As a result, ``PyMem_Malloc()`` and ``PyMem_Realloc()`` now call
+``malloc()`` and ``realloc()`` in both release mode and debug mode,
+instead of calling ``PyObject_Malloc()`` and ``PyObject_Realloc()`` in
+debug mode.
+
+When at least one memory allocator is replaced with
+``PyMem_SetAllocator()``, the ``PyMem_SetupDebugHooks()`` function must
+be called to reinstall the debug hooks on top on the new allocator.
+
+
+Don't call malloc() directly anymore
+------------------------------------
+
+``PyObject_Malloc()`` falls back on ``PyMem_Malloc()`` instead of
+``malloc()`` if size is greater or equal than 512 bytes, and
+``PyObject_Realloc()`` falls back on ``PyMem_Realloc()`` instead of
+``realloc()``
+
+Direct calls to ``malloc()`` are replaced with ``PyMem_Malloc()``, or
+``PyMem_RawMalloc()`` if the GIL is not held.
+
+External libraries like zlib or OpenSSL can be configured to allocate memory
+using ``PyMem_Malloc()`` or ``PyMem_RawMalloc()``. If the allocator of a
+library can only be replaced globally (rather than on an object-by-object
+basis), it shouldn't be replaced when Python is embedded in an application.
+
+For the "track memory usage" use case, it is important to track memory
+allocated in external libraries to have accurate reports, because these
+allocations can be large (e.g. they can raise a ``MemoryError`` exception)
+and would otherwise be missed in memory usage reports.
+
+
+Examples
+========
+
+Use case 1: Replace Memory Allocators, keep pymalloc
+----------------------------------------------------
+
+Dummy example wasting 2 bytes per memory block,
+and 10 bytes per *pymalloc* arena::
+
+    #include <stdlib.h>
+
+    size_t alloc_padding = 2;
+    size_t arena_padding = 10;
+
+    void* my_malloc(void *ctx, size_t size)
+    {
+        int padding = *(int *)ctx;
+        return malloc(size + padding);
+    }
+
+    void* my_realloc(void *ctx, void *ptr, size_t new_size)
+    {
+        int padding = *(int *)ctx;
+        return realloc(ptr, new_size + padding);
+    }
+
+    void my_free(void *ctx, void *ptr)
+    {
+        free(ptr);
+    }
+
+    void* my_alloc_arena(void *ctx, size_t size)
+    {
+        int padding = *(int *)ctx;
+        return malloc(size + padding);
+    }
+
+    void my_free_arena(void *ctx, void *ptr, size_t size)
+    {
+        free(ptr);
+    }
+
+    void setup_custom_allocator(void)
+    {
+        PyMemAllocator alloc;
+        PyObjectArenaAllocator arena;
+
+        alloc.ctx = &alloc_padding;
+        alloc.malloc = my_malloc;
+        alloc.realloc = my_realloc;
+        alloc.free = my_free;
+
+        PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc);
+        /* leave PYMEM_DOMAIN_OBJ unchanged, use pymalloc */
+
+        arena.ctx = &arena_padding;
+        arena.alloc = my_alloc_arena;
+        arena.free = my_free_arena;
+        PyObject_SetArenaAllocator(&arena);
+
+        PyMem_SetupDebugHooks();
+    }
+
+
+Use case 2: Replace Memory Allocators, override pymalloc
+--------------------------------------------------------
+
+If you have a dedicated allocator optimized for allocations of objects
+smaller than 512 bytes with a short lifetime, pymalloc can be overriden
+(replace ``PyObject_Malloc()``).
+
+Dummy example wasting 2 bytes per memory block::
+
+    #include <stdlib.h>
+
+    size_t padding = 2;
+
+    void* my_malloc(void *ctx, size_t size)
+    {
+        int padding = *(int *)ctx;
+        return malloc(size + padding);
+    }
+
+    void* my_realloc(void *ctx, void *ptr, size_t new_size)
+    {
+        int padding = *(int *)ctx;
+        return realloc(ptr, new_size + padding);
+    }
+
+    void my_free(void *ctx, void *ptr)
+    {
+        free(ptr);
+    }
+
+    void setup_custom_allocator(void)
+    {
+        PyMemAllocator alloc;
+        alloc.ctx = &padding;
+        alloc.malloc = my_malloc;
+        alloc.realloc = my_realloc;
+        alloc.free = my_free;
+
+        PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc);
+
+        PyMem_SetupDebugHooks();
+    }
+
+The *pymalloc* arena does not need to be replaced, because it is no more
+used by the new allocator.
+
+
+Use case 3: Setup Hooks On Memory Block Allocators
+--------------------------------------------------
+
+Example to setup hooks on all memory block allocators::
+
+    struct {
+        PyMemAllocator raw;
+        PyMemAllocator mem;
+        PyMemAllocator obj;
+        /* ... */
+    } hook;
+
+    static void* hook_malloc(void *ctx, size_t size)
+    {
+        PyMemAllocator *alloc = (PyMemAllocator *)ctx;
+        void *ptr;
+        /* ... */
+        ptr = alloc->malloc(alloc->ctx, size);
+        /* ... */
+        return ptr;
+    }
+
+    static void* hook_realloc(void *ctx, void *ptr, size_t new_size)
+    {
+        PyMemAllocator *alloc = (PyMemAllocator *)ctx;
+        void *ptr2;
+        /* ... */
+        ptr2 = alloc->realloc(alloc->ctx, ptr, new_size);
+        /* ... */
+        return ptr2;
+    }
+
+    static void hook_free(void *ctx, void *ptr)
+    {
+        PyMemAllocator *alloc = (PyMemAllocator *)ctx;
+        /* ... */
+        alloc->free(alloc->ctx, ptr);
+        /* ... */
+    }
+
+    void setup_hooks(void)
+    {
+        PyMemAllocator alloc;
+        static int installed = 0;
+
+        if (installed)
+            return;
+        installed = 1;
+
+        alloc.malloc = hook_malloc;
+        alloc.realloc = hook_realloc;
+        alloc.free = hook_free;
+        PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &hook.raw);
+        PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &hook.mem);
+        PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &hook.obj);
+
+        alloc.ctx = &hook.raw;
+        PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc);
+
+        alloc.ctx = &hook.mem;
+        PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc);
+
+        alloc.ctx = &hook.obj;
+        PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc);
+    }
+
+.. note::
+   ``PyMem_SetupDebugHooks()`` does not need to be called because
+   memory allocator are not replaced: the debug checks on memory
+   block allocators are installed automatically at startup.
+
+
+Performances
+============
+
+The implementation of this PEP (issue #3329) has no visible overhead on
+the Python benchmark suite.
+
+Results of the `Python benchmarks suite
+<http://hg.python.org/benchmarks>`_ (-b 2n3): some tests are 1.04x
+faster, some tests are 1.04 slower. Results of pybench microbenchmark:
+"+0.1%" slower globally (diff between -4.9% and +5.6%).
+
+The full output of benchmarks is attached to the issue #3329.
+
+
+Rejected Alternatives
+=====================
+
+More specific functions to get/set memory allocators
+----------------------------------------------------
+
+It was originally proposed a larger set of C API functions, with one pair
+of functions for each allocator domain:
+
+* ``void PyMem_GetRawAllocator(PyMemAllocator *allocator)``
+* ``void PyMem_GetAllocator(PyMemAllocator *allocator)``
+* ``void PyObject_GetAllocator(PyMemAllocator *allocator)``
+* ``void PyMem_SetRawAllocator(PyMemAllocator *allocator)``
+* ``void PyMem_SetAllocator(PyMemAllocator *allocator)``
+* ``void PyObject_SetAllocator(PyMemAllocator *allocator)``
+
+This alternative was rejected because it is not possible to write
+generic code with more specific functions: code must be duplicated for
+each memory allocator domain.
+
+
+Make PyMem_Malloc() reuse PyMem_RawMalloc() by default
+------------------------------------------------------
+
+If ``PyMem_Malloc()`` called ``PyMem_RawMalloc()`` by default,
+calling ``PyMem_SetAllocator(PYMEM_DOMAIN_RAW, alloc)`` would also
+patch ``PyMem_Malloc()`` indirectly.
+
+This alternative was rejected because ``PyMem_SetAllocator()`` would
+have a different behaviour depending on the domain. Always having the
+same behaviour is less error-prone.
+
+
+Add a new PYDEBUGMALLOC environment variable
+--------------------------------------------
+
+It was proposed to add a new ``PYDEBUGMALLOC`` environment variable to
+enable debug checks on memory block allocators. It would have had the same
+effect as calling the ``PyMem_SetupDebugHooks()``, without the need
+to write any C code.  Another advantage is to allow to enable debug checks
+even in release mode: debug checks would always be compiled in, but only
+enabled when the environment variable is present and non-empty.
+
+This alternative was rejected because a new environment variable would
+make Python initialization even more complex. `PEP 432
+<http://www.python.org/dev/peps/pep-0432/>`_ tries to simplify the
+CPython startup sequence.
+
+
+Use macros to get customizable allocators
+-----------------------------------------
+
+To have no overhead in the default configuration, customizable
+allocators would be an optional feature enabled by a configuration
+option or by macros.
+
+This alternative was rejected because the use of macros implies having
+to recompile extensions modules to use the new allocator and allocator
+hooks. Not having to recompile Python nor extension modules makes debug
+hooks easier to use in practice.
+
+
+Pass the C filename and line number
+-----------------------------------
+
+Define allocator functions as macros using ``__FILE__`` and ``__LINE__``
+to get the C filename and line number of a memory allocation.
+
+Example of ``PyMem_Malloc`` macro with the modified
+``PyMemAllocator`` structure::
+
+    typedef struct {
+        /* user context passed as the first argument
+           to the 3 functions */
+        void *ctx;
+
+        /* allocate a memory block */
+        void* (*malloc) (void *ctx, const char *filename, int lineno,
+                         size_t size);
+
+        /* allocate or resize a memory block */
+        void* (*realloc) (void *ctx, const char *filename, int lineno,
+                          void *ptr, size_t new_size);
+
+        /* release a memory block */
+        void (*free) (void *ctx, const char *filename, int lineno,
+                      void *ptr);
+    } PyMemAllocator;
+
+    void* _PyMem_MallocTrace(const char *filename, int lineno,
+                             size_t size);
+
+    /* the function is still needed for the Python stable ABI */
+    void* PyMem_Malloc(size_t size);
+
+    #define PyMem_Malloc(size) \
+            _PyMem_MallocTrace(__FILE__, __LINE__, size)
+
+The GC allocator functions would also have to be patched. For example,
+``_PyObject_GC_Malloc()`` is used in many C functions and so objects of
+different types would have the same allocation location.
+
+This alternative was rejected because passing a filename and a line
+number to each allocator makes the API more complex: pass 3 new
+arguments (ctx, filename, lineno) to each allocator function, instead of
+just a context argument (ctx). Having to also modify GC allocator
+functions adds too much complexity for a little gain.
+
+
+GIL-free PyMem_Malloc()
+-----------------------
+
+In Python 3.3, when Python is compiled in debug mode, ``PyMem_Malloc()``
+indirectly calls ``PyObject_Malloc()`` which requires the GIL to be
+held (it isn't thread-safe).  That's why ``PyMem_Malloc()`` must be called
+with the GIL held.
+
+This PEP changes ``PyMem_Malloc()``: it now always calls ``malloc()``
+rather than ``PyObject_Malloc()``.  The "GIL must be held" restriction
+could therefore be removed from ``PyMem_Malloc()``.
+
+This alternative was rejected because allowing to call
+``PyMem_Malloc()`` without holding the GIL can break applications
+which setup their own allocators or allocator hooks.  Holding the GIL is
+convenient to develop a custom allocator: no need to care about other
+threads.  It is also convenient for a debug allocator hook: Python
+objects can be safely inspected, and the C API may be used for reporting.
+
+Moreover, calling ``PyGILState_Ensure()`` in a memory allocator has
+unexpected behaviour, especially at Python startup and when creating of a
+new Python thread state.  It is better to free custom allocators of
+the responsibility of acquiring the GIL.
+
+
+Don't add PyMem_RawMalloc()
+---------------------------
+
+Replace ``malloc()`` with ``PyMem_Malloc()``, but only if the GIL is
+held.  Otherwise, keep ``malloc()`` unchanged.
+
+The ``PyMem_Malloc()`` is used without the GIL held in some Python
+functions.  For example, the ``main()`` and ``Py_Main()`` functions of
+Python call ``PyMem_Malloc()`` whereas the GIL do not exist yet. In this
+case, ``PyMem_Malloc()`` would be replaced with ``malloc()`` (or
+``PyMem_RawMalloc()``).
+
+This alternative was rejected because ``PyMem_RawMalloc()`` is required
+for accurate reports of the memory usage. When a debug hook is used to
+track the memory usage, the memory allocated by direct calls to
+``malloc()`` cannot be tracked. ``PyMem_RawMalloc()`` can be hooked and
+so all the memory allocated by Python can be tracked, including
+memory allocated without holding the GIL.
+
+
+Use existing debug tools to analyze memory use
+----------------------------------------------
+
+There are many existing debug tools to analyze memory use. Some
+examples: `Valgrind <http://valgrind.org/>`_, `Purify
+<http://ibm.com/software/awdtools/purify/>`_, `Clang AddressSanitizer
+<http://code.google.com/p/address-sanitizer/>`_, `failmalloc
+<http://www.nongnu.org/failmalloc/>`_, etc.
+
+The problem is to retrieve the Python object related to a memory pointer
+to read its type and/or its content. Another issue is to retrieve the
+source of the memory allocation: the C backtrace is usually useless
+(same reasoning than macros using ``__FILE__`` and ``__LINE__``, see
+`Pass the C filename and line number`_), the Python filename and line
+number (or even the Python traceback) is more useful.
+
+This alternative was rejected because classic tools are unable to
+introspect Python internals to collect such information. Being able to
+setup a hook on allocators called with the GIL held allows to collect a
+lot of useful data from Python internals.
+
+
+Add a msize() function
+----------------------
+
+Add another function to ``PyMemAllocator`` and
+``PyObjectArenaAllocator`` structures::
+
+    size_t msize(void *ptr);
+
+This function returns the size of a memory block or a memory mapping.
+Return (size_t)-1 if the function is not implemented or if the pointer
+is unknown (ex: NULL pointer).
+
+On Windows, this function can be implemented using ``_msize()`` and
+``VirtualQuery()``.
+
+The function can be used to implement a hook tracking the memory usage.
+The ``free()`` method of an allocator only gets the address of a memory
+block, whereas the size of the memory block is required to update the
+memory usage.
+
+The additional ``msize()`` function was rejected because only few
+platforms implement it. For example, Linux with the GNU libc does not
+provide a function to get the size of a memory block. ``msize()`` is not
+currently used in the Python source code. The function would only be
+used to track memory use, and make the API more complex. A debug hook
+can implement the function internally, there is no need to add it to
+``PyMemAllocator`` and ``PyObjectArenaAllocator`` structures.
+
+
+No context argument
+-------------------
+
+Simplify the signature of allocator functions, remove the context
+argument:
+
+* ``void* malloc(size_t size)``
+* ``void* realloc(void *ptr, size_t new_size)``
+* ``void free(void *ptr)``
+
+It is likely for an allocator hook to be reused for
+``PyMem_SetAllocator()`` and ``PyObject_SetAllocator()``, or even
+``PyMem_SetRawAllocator()``, but the hook must call a different function
+depending on the allocator. The context is a convenient way to reuse the
+same custom allocator or hook for different Python allocators.
+
+In C++, the context can be used to pass *this*.
+
+
+External Libraries
+==================
+
+Examples of API used to customize memory allocators.
+
+Libraries used by Python:
+
+* OpenSSL: `CRYPTO_set_mem_functions()
+  <http://git.openssl.org/gitweb/?p=openssl.git;a=blob;f=crypto/mem.c;h=f7984fa958eb1edd6c61f6667f3f2b29753be662;hb=HEAD#l124>`_
+  to set memory management functions globally
+* expat: `parserCreate()
+  <http://hg.python.org/cpython/file/cc27d50bd91a/Modules/expat/xmlparse.c#l724>`_
+  has a per-instance memory handler
+* zlib: `zlib 1.2.8 Manual <http://www.zlib.net/manual.html#Usage>`_,
+  pass an opaque pointer
+* bz2: `bzip2 and libbzip2, version 1.0.5
+  <http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html>`_,
+  pass an opaque pointer
+* lzma: `LZMA SDK - How to Use
+  <http://www.asawicki.info/news_1368_lzma_sdk_-_how_to_use.html>`_,
+  pass an opaque pointer
+* lipmpdec: no opaque pointer (classic malloc API)
+
+Other libraries:
+
+* glib: `g_mem_set_vtable()
+  <http://developer.gnome.org/glib/unstable/glib-Memory-Allocation.html#g-mem-set-vtable>`_
+* libxml2:
+  `xmlGcMemSetup() <http://xmlsoft.org/html/libxml-xmlmemory.html>`_,
+  global
+* Oracle's OCI: `Oracle Call Interface Programmer's Guide,
+  Release 2 (9.2)
+  <http://docs.oracle.com/cd/B10501_01/appdev.920/a96584/oci15re4.htm>`_,
+  pass an opaque pointer
+
+The new *ctx* parameter of this PEP was inspired by the API of zlib and
+Oracle's OCI libraries.
+
+See also the `GNU libc: Memory Allocation Hooks
+<http://www.gnu.org/software/libc/manual/html_node/Hooks-for-Malloc.html>`_
+which uses a different approach to hook memory allocators.
+
+
+Memory Allocators
+=================
+
+The C standard library provides the well known ``malloc()`` function.
+Its implementation depends on the platform and of the C library. The GNU
+C library uses a modified ptmalloc2, based on "Doug Lea's Malloc"
+(dlmalloc). FreeBSD uses `jemalloc
+<http://www.canonware.com/jemalloc/>`_. Google provides *tcmalloc* which
+is part of `gperftools <http://code.google.com/p/gperftools/>`_.
+
+``malloc()`` uses two kinds of memory: heap and memory mappings. Memory
+mappings are usually used for large allocations (ex: larger than 256
+KB), whereas the heap is used for small allocations.
+
+On UNIX, the heap is handled by ``brk()`` and ``sbrk()`` system calls,
+and it is contiguous.  On Windows, the heap is handled by
+``HeapAlloc()`` and can be discontiguous. Memory mappings are handled by
+``mmap()`` on UNIX and ``VirtualAlloc()`` on Windows, they can be
+discontiguous.
+
+Releasing a memory mapping gives back immediatly the memory to the
+system. On UNIX, the heap memory is only given back to the system if the
+released block is located at the end of the heap. Otherwise, the memory
+will only be given back to the system when all the memory located after
+the released memory is also released.
+
+To allocate memory on the heap, an allocator tries to reuse free space.
+If there is no contiguous space big enough, the heap must be enlarged,
+even if there is more free space than required size.  This issue is
+called the "memory fragmentation": the memory usage seen by the system
+is higher than real usage. On Windows, ``HeapAlloc()`` creates
+a new memory mapping with ``VirtualAlloc()`` if there is not enough free
+contiguous memory.
+
+CPython has a *pymalloc* allocator for allocations smaller than 512
+bytes. This allocator is optimized for small objects with a short
+lifetime. It uses memory mappings called "arenas" with a fixed size of
+256 KB.
+
+Other allocators:
+
+* Windows provides a `Low-fragmentation Heap
+  <http://msdn.microsoft.com/en-us/library/windows/desktop/aa366750%28v=vs.85%29.aspx>`_.
+
+* The Linux kernel uses `slab allocation
+  <http://en.wikipedia.org/wiki/Slab_allocation>`_.
+
+* The glib library has a `Memory Slice API
+  <https://developer.gnome.org/glib/unstable/glib-Memory-Slices.html>`_:
+  efficient way to allocate groups of equal-sized chunks of memory
+
+This PEP allows to choose exactly which memory allocator is used for your
+application depending on its usage of the memory (number of allocations,
+size of allocations, lifetime of objects, etc.).
+
+
+Links
+=====
+
+CPython issues related to memory allocation:
+
+* `Issue #3329: Add new APIs to customize memory allocators
+  <http://bugs.python.org/issue3329>`_
+* `Issue #13483: Use VirtualAlloc to allocate memory arenas
+  <http://bugs.python.org/issue13483>`_
+* `Issue #16742: PyOS_Readline drops GIL and calls PyOS_StdioReadline,
+  which isn't thread safe <http://bugs.python.org/issue16742>`_
+* `Issue #18203: Replace calls to malloc() with PyMem_Malloc() or
+  PyMem_RawMalloc() <http://bugs.python.org/issue18203>`_
+* `Issue #18227: Use Python memory allocators in external libraries like
+  zlib or OpenSSL <http://bugs.python.org/issue18227>`_
+
+Projects analyzing the memory usage of Python applications:
+
+* `pytracemalloc
+  <https://pypi.python.org/pypi/pytracemalloc>`_
+* `Meliae: Python Memory Usage Analyzer
+  <https://pypi.python.org/pypi/meliae>`_
+* `Guppy-PE: umbrella package combining Heapy and GSL
+  <http://guppy-pe.sourceforge.net/>`_
+* `PySizer (developed for Python 2.4)
+  <http://pysizer.8325.org/>`_
+
+
+Copyright
+=========
+
+This document has been placed into the public domain.
+
diff --git a/pep-0446.txt b/pep-0446.txt
new file mode 100644
--- /dev/null
+++ b/pep-0446.txt
@@ -0,0 +1,242 @@
+PEP: 446
+Title: Add new parameters to configure the inheritance of files and for non-blocking sockets
+Version: $Revision$
+Last-Modified: $Date$
+Author: Victor Stinner <victor.stinner at gmail.com>
+Status: Draft
+Type: Standards Track
+Content-Type: text/x-rst
+Created: 3-July-2013
+Python-Version: 3.4
+
+
+Abstract
+========
+
+This PEP proposes new portable parameters and functions to configure the
+inheritance of file descriptors and the non-blocking flag of sockets.
+
+
+Rationale
+=========
+
+Inheritance of file descriptors
+-------------------------------
+
+The inheritance of file descriptors in child processes can be configured
+on each file descriptor using a *close-on-exec* flag. By default, the
+close-on-exec flag is not set.
+
+On Windows, the close-on-exec flag is ``HANDLE_FLAG_INHERIT``. File
+descriptors are not inherited if the ``bInheritHandles`` parameter of
+the ``CreateProcess()`` function is ``FALSE``, even if the
+``HANDLE_FLAG_INHERIT`` flag is set. If ``bInheritHandles`` is ``TRUE``,
+only file descriptors with ``HANDLE_FLAG_INHERIT`` flag set are
+inherited, others are not.
+
+On UNIX, the close-on-exec flag is ``O_CLOEXEC``. File descriptors with
+the ``O_CLOEXEC`` flag set are closed at the execution of a new program
+(ex: when calling ``execv()``).
+
+The ``O_CLOEXEC`` flag has no effect on ``fork()``, all file descriptors
+are inherited by the child process. Futhermore, most properties file
+descriptors are shared between the parent and the child processes,
+except file attributes which are duplicated (``O_CLOEXEC`` is the only
+file attribute).  Setting ``O_CLOEXEC`` flag of a file descriptor in the
+child process does not change the ``O_CLOEXEC`` flag of the file
+descriptor in the parent process.
+
+
+Issues of the inheritance of file descriptors
+---------------------------------------------
+
+Inheritance of file descriptors causes issues. For example, closing a
+file descriptor in the parent process does not release the resource
+(file, socket, ...), because the file descriptor is still open in the
+child process.
+
+Leaking file descriptors is also a major security vulnerability. An
+untrusted child process can read sensitive data like passwords and take
+control of the parent process though leaked file descriptors. It is for
+example a known vulnerability to escape from a chroot.
+
+
+Non-blocking sockets
+--------------------
+
+To handle multiple network clients in a single thread, a multiplexing
+function like ``select()`` can be used. For best performances, sockets
+must be configured as non-blocking. Operations like ``send()`` and
+``recv()`` return an ``EAGAIN`` or ``EWOULDBLOCK`` error if the
+operation would block.
+
+By default, newly created sockets are blocking. Setting the non-blocking
+mode requires additional system calls.
+
+On UNIX, the blocking flag is ``O_NONBLOCK``: a pipe and a socket are
+non-blocking if the ``O_NONBLOCK`` flag is set.
+
+
+Setting flags at the creation of the file descriptor
+----------------------------------------------------
+
+Windows and recent versions of other operating systems like Linux
+support setting the close-on-exec flag directly at the creation of file
+descriptors, and close-on-exec and blocking flags at the creation of
+sockets.
+
+Setting these flags at the creation is atomic and avoids additional
+system calls.
+
+
+Proposal
+========
+
+New cloexec And blocking Parameters
+-----------------------------------
+
+Add a new optional *cloexec* on functions creating file descriptors:
+
+* ``io.FileIO``
+* ``io.open()``
+* ``open()``
+* ``os.dup()``
+* ``os.dup2()``
+* ``os.fdopen()``
+* ``os.open()``
+* ``os.openpty()``
+* ``os.pipe()``
+* ``select.devpoll()``
+* ``select.epoll()``
+* ``select.kqueue()``
+
+Add new optional *cloexec* and *blocking* parameters to functions
+creating sockets:
+
+* ``asyncore.dispatcher.create_socket()``
+* ``socket.socket()``
+* ``socket.socket.accept()``
+* ``socket.socket.dup()``
+* ``socket.socket.fromfd``
+* ``socket.socketpair()``
+
+The default value of *cloexec* is ``False`` and the default value of
+*blocking* is ``True``.
+
+The atomicity is not guaranteed. If the platform does not support
+setting close-on-exec and blocking flags at the creation of the file
+descriptor or socket, the flags are set using additional system calls.
+
+
+New Functions
+-------------
+
+Add new functions the get and set the close-on-exec flag of a file
+descriptor, available on all platforms:
+
+* ``os.get_cloexec(fd:int) -> bool``
+* ``os.set_cloexec(fd:int, cloexec: bool)``
+
+Add new functions the get and set the blocking flag of a file
+descriptor, only available on UNIX:
+
+* ``os.get_blocking(fd:int) -> bool``
+* ``os.set_blocking(fd:int, blocking: bool)``
+
+
+Other Changes
+-------------
+
+The ``subprocess.Popen`` class must clear the close-on-exec flag of file
+descriptors of the ``pass_fds`` parameter. The flag is cleared in the
+child process before executing the program, the change does not change
+the flag in the parent process.
+
+The close-on-exec flag must also be set on private file descriptors and
+sockets in the Python standard library. For example, on UNIX,
+os.urandom() opens ``/dev/urandom`` to read some random bytes and the
+file descriptor is closed at function exit. The file descriptor is not
+expected to be inherited by child processes.
+
+
+Rejected Alternatives
+=====================
+
+PEP 433
+-------
+
+The PEP 433 entitled "Easier suppression of file descriptor inheritance"
+is a previous attempt proposing various other alternatives, but no
+consensus could be reached.
+
+This PEP has a well defined behaviour (the default value of the new
+*cloexec* parameter is not configurable), is more conservative (no
+backward compatibility issue), and is much simpler.
+
+
+Add blocking parameter for file descriptors and use Windows overlapped I/O
+--------------------------------------------------------------------------
+
+Windows supports non-blocking operations on files using an extension of
+the Windows API called "Overlapped I/O". Using this extension requires
+to modify the Python standard library and applications to pass a
+``OVERLAPPED`` structure and an event loop to wait for the completion of
+operations.
+
+This PEP only tries to expose portable flags on file descriptors and
+sockets. Supporting overlapped I/O requires an abstraction providing a
+high-level and portable API for asynchronous operations on files and
+sockets. Overlapped I/O are out of the scope of this PEP.
+
+UNIX supports non-blocking files, moreover recent versions of operating
+systems support setting the non-blocking flag at the creation of a file
+descriptor. It would be possible to add a new optional *blocking*
+parameter to Python functions creating file descriptors. On Windows,
+creating a file descriptor with ``blocking=False``  would raise a
+``NotImplementedError``. This behaviour is not acceptable for the ``os``
+module which is designed as a thin wrapper on the C functions of the
+operating system. If a platform does not support a function, the
+function should not be available on the platform. For example,
+the ``os.fork()`` function is not available on Windows.
+
+For all these reasons, this alternative was rejected. The PEP 3156
+proposes an abstraction for asynchronous I/O supporting non-blocking
+files on Windows.
+
+
+Links
+=====
+
+Python issues:
+
+* `#10115: Support accept4() for atomic setting of flags at socket
+  creation <http://bugs.python.org/issue10115>`_
+* `#12105: open() does not able to set flags, such as O_CLOEXEC
+  <http://bugs.python.org/issue12105>`_
+* `#12107: TCP listening sockets created without FD_CLOEXEC flag
+  <http://bugs.python.org/issue12107>`_
+* `#16850: Add "e" mode to open(): close-and-exec
+  (O_CLOEXEC) / O_NOINHERIT <http://bugs.python.org/issue16850>`_
+* `#16860: Use O_CLOEXEC in the tempfile module
+  <http://bugs.python.org/issue16860>`_
+* `#16946: subprocess: _close_open_fd_range_safe() does not set
+  close-on-exec flag on Linux < 2.6.23 if O_CLOEXEC is defined
+  <http://bugs.python.org/issue16946>`_
+* `#17070: Use the new cloexec to improve security and avoid bugs
+  <http://bugs.python.org/issue17070>`_
+
+Other links:
+
+* `Secure File Descriptor Handling
+  <http://udrepper.livejournal.com/20407.html>`_ (Ulrich Drepper,
+  2008)
+* `Ghosts of Unix past, part 2: Conflated designs
+  <http://lwn.net/Articles/412131/>`_ (Neil Brown, 2010) explains the
+  history of ``O_CLOEXEC`` and ``O_NONBLOCK`` flags
+
+
+Copyright
+=========
+
+This document has been placed into the public domain.
+

-- 
Repository URL: http://hg.python.org/peps

From solipsis at pitrou.net  Tue Jul  9 05:47:47 2013
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Tue, 09 Jul 2013 05:47:47 +0200
Subject: [Python-checkins] Daily reference leaks (c91e7f707562): sum=0
Message-ID: <E1UwOu3-00084H-VB@ap.vmr.nerim.net>

results for c91e7f707562 on branch "default"
--------------------------------------------



Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogZuuOHW', '-x']

From python-checkins at python.org  Tue Jul  9 09:13:31 2013
From: python-checkins at python.org (raymond.hettinger)
Date: Tue,  9 Jul 2013 09:13:31 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Add_a_spacing_saving_heuri?=
 =?utf-8?q?stic_to_deque=27s_extend_methods?=
Message-ID: <3bqFB31F10z7Lkw@mail.python.org>

http://hg.python.org/cpython/rev/904399fae081
changeset:   84524:904399fae081
user:        Raymond Hettinger <python at rcn.com>
date:        Tue Jul 09 00:13:21 2013 -0700
summary:
  Add a spacing saving heuristic to deque's extend methods

files:
  Lib/test/test_deque.py       |   4 ++--
  Modules/_collectionsmodule.c |  16 ++++++++++++++++
  2 files changed, 18 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py
--- a/Lib/test/test_deque.py
+++ b/Lib/test/test_deque.py
@@ -543,8 +543,8 @@
         check = self.check_sizeof
         check(deque(), basesize + blocksize)
         check(deque('a'), basesize + blocksize)
-        check(deque('a' * (BLOCKLEN // 2)), basesize + blocksize)
-        check(deque('a' * (BLOCKLEN // 2 + 1)), basesize + 2 * blocksize)
+        check(deque('a' * (BLOCKLEN - 1)), basesize + blocksize)
+        check(deque('a' * BLOCKLEN), basesize + 2 * blocksize)
         check(deque('a' * (42 * BLOCKLEN)), basesize + 43 * blocksize)
 
 class TestVariousIteratorArgs(unittest.TestCase):
diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -332,6 +332,14 @@
         return result;
     }
 
+    /* Space saving heuristic.  Start filling from the left */
+    if (Py_SIZE(deque) == 0) {
+        assert(deque->leftblock == deque->rightblock);
+        assert(deque->leftindex == deque->rightindex+1);
+        deque->leftindex = 1;
+        deque->rightindex = 0;
+    }
+
     it = PyObject_GetIter(iterable);
     if (it == NULL)
         return NULL;
@@ -385,6 +393,14 @@
         return result;
     }
 
+    /* Space saving heuristic.  Start filling from the right */
+    if (Py_SIZE(deque) == 0) {
+        assert(deque->leftblock == deque->rightblock);
+        assert(deque->leftindex == deque->rightindex+1);
+        deque->leftindex = BLOCKLEN - 1;
+        deque->rightindex = BLOCKLEN - 2;
+    }
+
     it = PyObject_GetIter(iterable);
     if (it == NULL)
         return NULL;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  9 14:30:45 2013
From: python-checkins at python.org (christian.heimes)
Date: Tue,  9 Jul 2013 14:30:45 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogdXNlICQoTE4pIG1h?=
 =?utf-8?q?kefile_variable_instead_of_ln?=
Message-ID: <3bqND56qwWz7Ljk@mail.python.org>

http://hg.python.org/cpython/rev/09f86b4ac1a0
changeset:   84525:09f86b4ac1a0
branch:      3.3
parent:      84505:6f16fa5223cc
user:        Christian Heimes <christian at cheimes.de>
date:        Tue Jul 09 14:30:04 2013 +0200
summary:
  use $(LN) makefile variable instead of ln

files:
  Makefile.pre.in |  12 ++++++------
  1 files changed, 6 insertions(+), 6 deletions(-)


diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1285,12 +1285,12 @@
 # Install a number of symlinks to keep software that expects a normal unix
 # install (which includes python-config) happy.
 frameworkinstallmaclib:
-	ln -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(LDVERSION).a"
-	ln -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(LDVERSION).dylib"
-	ln -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(VERSION).a"
-	ln -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(VERSION).dylib"
-	ln -fs "../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/libpython$(LDVERSION).dylib"
-	ln -fs "../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/libpython$(VERSION).dylib"
+	$(LN) -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(LDVERSION).a"
+	$(LN) -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(LDVERSION).dylib"
+	$(LN) -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(VERSION).a"
+	$(LN) -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(VERSION).dylib"
+	$(LN) -fs "../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/libpython$(LDVERSION).dylib"
+	$(LN) -fs "../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/libpython$(VERSION).dylib"
 
 # This installs the IDE, the Launcher and other apps into /Applications
 frameworkinstallapps:

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  9 14:30:47 2013
From: python-checkins at python.org (christian.heimes)
Date: Tue,  9 Jul 2013 14:30:47 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_use_=24=28LN=29_makefile_variable_instead_of_ln?=
Message-ID: <3bqND72qQFz7LkZ@mail.python.org>

http://hg.python.org/cpython/rev/3f3cbfd52f94
changeset:   84526:3f3cbfd52f94
parent:      84524:904399fae081
parent:      84525:09f86b4ac1a0
user:        Christian Heimes <christian at cheimes.de>
date:        Tue Jul 09 14:30:22 2013 +0200
summary:
  use $(LN) makefile variable instead of ln

files:
  Makefile.pre.in |  12 ++++++------
  1 files changed, 6 insertions(+), 6 deletions(-)


diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1287,12 +1287,12 @@
 # Install a number of symlinks to keep software that expects a normal unix
 # install (which includes python-config) happy.
 frameworkinstallmaclib:
-	ln -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(LDVERSION).a"
-	ln -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(LDVERSION).dylib"
-	ln -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(VERSION).a"
-	ln -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(VERSION).dylib"
-	ln -fs "../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/libpython$(LDVERSION).dylib"
-	ln -fs "../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/libpython$(VERSION).dylib"
+	$(LN) -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(LDVERSION).a"
+	$(LN) -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(LDVERSION).dylib"
+	$(LN) -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(VERSION).a"
+	$(LN) -fs "../../../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/python$(VERSION)/config-$(LDVERSION)/libpython$(VERSION).dylib"
+	$(LN) -fs "../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/libpython$(LDVERSION).dylib"
+	$(LN) -fs "../$(PYTHONFRAMEWORK)" "$(DESTDIR)$(prefix)/lib/libpython$(VERSION).dylib"
 
 # This installs the IDE, the Launcher and other apps into /Applications
 frameworkinstallapps:

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  9 15:27:25 2013
From: python-checkins at python.org (barry.warsaw)
Date: Tue,  9 Jul 2013 15:27:25 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_Clarify_the_wording=2E?=
Message-ID: <3bqPTT00nFz7LkF@mail.python.org>

http://hg.python.org/peps/rev/c6a5738d5eb3
changeset:   4989:c6a5738d5eb3
user:        Barry Warsaw <barry at python.org>
date:        Tue Jul 09 09:27:16 2013 -0400
summary:
  Clarify the wording.

files:
  pep-0008.txt |  3 ++-
  1 files changed, 2 insertions(+), 1 deletions(-)


diff --git a/pep-0008.txt b/pep-0008.txt
--- a/pep-0008.txt
+++ b/pep-0008.txt
@@ -101,7 +101,8 @@
       var_three, var_four)
 
 The closing brace/bracket/parenthesis on multi-line constructs may
-either line up under the last item of the list, as in::
+either line up under the first non-whitespace character of the last
+line of list, as in::
 
     my_list = [
         1, 2, 3,

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Tue Jul  9 19:17:29 2013
From: python-checkins at python.org (charles-francois.natali)
Date: Tue,  9 Jul 2013 19:17:29 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4MzA4?=
 =?utf-8?q?=3A_don=27t_take_the_scope_ID_into_account_when_comparing_IPv6?=
Message-ID: <3bqVZx73TBz7Lk3@mail.python.org>

http://hg.python.org/cpython/rev/330c7aa2922b
changeset:   84527:330c7aa2922b
branch:      3.3
parent:      84525:09f86b4ac1a0
user:        Charles-Fran?ois Natali <cf.natali at gmail.com>
date:        Tue Jul 09 19:15:43 2013 +0200
summary:
  Issue #18308: don't take the scope ID into account when comparing IPv6
addresses.

files:
  Lib/test/test_socket.py |  6 +++++-
  1 files changed, 5 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -3225,7 +3225,11 @@
 class SendrecvmsgUDP6TestBase(SendrecvmsgDgramFlagsBase,
                               SendrecvmsgConnectionlessBase,
                               ThreadedSocketTestMixin, UDP6TestBase):
-    pass
+
+    def checkRecvmsgAddress(self, addr1, addr2):
+        # Called to compare the received address with the address of
+        # the peer, ignoring scope ID
+        self.assertEqual(addr1[:-1], addr2[:-1])
 
 @requireAttrs(socket.socket, "sendmsg")
 @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.')

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Tue Jul  9 19:17:31 2013
From: python-checkins at python.org (charles-francois.natali)
Date: Tue,  9 Jul 2013 19:17:31 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2318308=3A_don=27t_take_the_scope_ID_into_account?=
 =?utf-8?q?_when_comparing_IPv6?=
Message-ID: <3bqVZz3BBsz7LkX@mail.python.org>

http://hg.python.org/cpython/rev/b44749cee660
changeset:   84528:b44749cee660
parent:      84526:3f3cbfd52f94
parent:      84527:330c7aa2922b
user:        Charles-Fran?ois Natali <cf.natali at gmail.com>
date:        Tue Jul 09 19:16:32 2013 +0200
summary:
  Issue #18308: don't take the scope ID into account when comparing IPv6
addresses.

files:
  Lib/test/test_socket.py |  6 +++++-
  1 files changed, 5 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -3312,7 +3312,11 @@
 class SendrecvmsgUDP6TestBase(SendrecvmsgDgramFlagsBase,
                               SendrecvmsgConnectionlessBase,
                               ThreadedSocketTestMixin, UDP6TestBase):
-    pass
+
+    def checkRecvmsgAddress(self, addr1, addr2):
+        # Called to compare the received address with the address of
+        # the peer, ignoring scope ID
+        self.assertEqual(addr1[:-1], addr2[:-1])
 
 @requireAttrs(socket.socket, "sendmsg")
 @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.')

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Wed Jul 10 05:02:05 2013
From: python-checkins at python.org (richard.jones)
Date: Wed, 10 Jul 2013 05:02:05 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_clarify_CA_cert_issue=3B_inst?=
 =?utf-8?q?all_virtualenv_as_well=3B_mention_Python_2=2E7?=
Message-ID: <3bqlYT3PDBz7Ljd@mail.python.org>

http://hg.python.org/peps/rev/bd9b2df92af5
changeset:   4990:bd9b2df92af5
user:        Richard Jones <richard at mechanicalcat.net>
date:        Wed Jul 10 13:01:57 2013 +1000
summary:
  clarify CA cert issue; install virtualenv as well; mention Python 2.7 possibility

files:
  pep-0439.txt |  34 ++++++++++++++++++++++++----------
  1 files changed, 24 insertions(+), 10 deletions(-)


diff --git a/pep-0439.txt b/pep-0439.txt
--- a/pep-0439.txt
+++ b/pep-0439.txt
@@ -45,6 +45,12 @@
 considerably reduced.  It is hoped that this will therefore increase
 the likelihood that Python projects will reuse third party software.
 
+The Python community also has an issue of complexity around the current
+bootstrap procedure for pip, setuptools and virtualenv. They all have
+their own bootstrap download file with slightly different usages and
+even refer to each other in some cases. Having a single bootstrap which
+is common amongst them all, with a simple usage, would be far preferable.
+
 It is also hoped that this is reduces the number of proposals to
 include more and more software in the Python standard library, and
 therefore that more popular Python software is more easily upgradeable
@@ -54,6 +60,9 @@
 Proposal
 ========
 
+The bootstrap will install the pip implementation, setuptools and virtualenv
+by downloading their installation files from PyPI.
+
 This proposal affects two components of packaging: `the pip bootstrap`_ and,
 thanks to easier package installation, `modifications to publishing
 packages`_.
@@ -68,11 +77,11 @@
 The Python installation includes an executable called "pip3" (see PEP 394 for
 naming rationale etc.) that attempts to import pip machinery.  If it can then
 the pip command proceeds as normal.  If it cannot it will bootstrap pip by
-downloading the pip implementation and setuptools wheel files. Hereafter the
-installation of the "pip implementation" will imply installation of
-setuptools.  Once installed, the pip command proceeds as normal. Once the
-bootstrap process is complete the "pip3" command is no longer the bootstrap
-but rather the full pip command.
+downloading the pip implementation, setuptools and virtualenv wheel files.
+Hereafter the installation of the "pip implementation" will imply installation
+of setuptools and virtualenv.  Once installed, the pip command proceeds as
+normal. Once the bootstrap process is complete the "pip3" command is no longer
+the bootstrap but rather the full pip command.
 
 A boostrap is used in the place of a the full pip code so that we don't have
 to bundle pip and also pip is upgradeable outside of the regular Python
@@ -114,12 +123,13 @@
 
 Some users may have no Internet access suitable for fetching the pip
 implementation file. These users can manually download and install the
-setuptools and pip tar files. Adding specific support for this use-case is
-unnecessary.
+setuptools, virtualenv and pip tar files. Adding specific support for this
+use-case is unnecessary.
 
 The download of the pip implementation install file will be performed
 securely.  The transport from pypi.python.org will be done over HTTPS with the
-CA certificate check performed (see PEP XXXX).
+CA certificate check performed. This facility will be present in Python 3.4+
+using Operating System certificates (see PEP XXXX).
 
 Beyond those arguments controlling index location and download
 options, the "pip3" boostrap command may support further standard pip
@@ -173,13 +183,17 @@
 issue tracker [2]_. Most notably, the addition of --bootstrap and --bootstrap-
 to-system to the pip command-line.
 
+It would be preferable that the pip, setuptools and virtualenv projects
+distribute a wheel format download.
+
 The required code for this implementation is the "pip3" command described
 above. The additional pypublish can be developed outside of the scope of this
 PEP's work.
 
 Finally, it would be desirable that "pip3" be ported to Python 2.6+ to allow
-the single command to replace all existing pip/setuptools/distribute and
-possibly virtualenv bootstrap scripts.
+the single command to replace all existing pip, setuptools and virtualenv
+bootstrap scripts. Having that bootstrap included in a future Python 2.7
+release would also be highly desirable.
 
 
 Risks

-- 
Repository URL: http://hg.python.org/peps

From solipsis at pitrou.net  Wed Jul 10 05:46:46 2013
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Wed, 10 Jul 2013 05:46:46 +0200
Subject: [Python-checkins] Daily reference leaks (b44749cee660): sum=0
Message-ID: <E1UwlMc-0003tp-NN@ap.vmr.nerim.net>

results for b44749cee660 on branch "default"
--------------------------------------------



Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflog2WC8Iu', '-x']

From python-checkins at python.org  Wed Jul 10 07:07:51 2013
From: python-checkins at python.org (richard.jones)
Date: Wed, 10 Jul 2013 07:07:51 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?peps=3A_remove_virtualenv_from_the_pi?=
	=?utf-8?q?p3_install?=
Message-ID: <3bqpLb5Q3dz7LjX@mail.python.org>

http://hg.python.org/peps/rev/c40ed3544492
changeset:   4991:c40ed3544492
user:        Richard Jones <richard at mechanicalcat.net>
date:        Wed Jul 10 15:07:43 2013 +1000
summary:
  remove virtualenv from the pip3 install

files:
  pep-0439.txt |  30 +++++++++++++++---------------
  1 files changed, 15 insertions(+), 15 deletions(-)


diff --git a/pep-0439.txt b/pep-0439.txt
--- a/pep-0439.txt
+++ b/pep-0439.txt
@@ -46,7 +46,7 @@
 the likelihood that Python projects will reuse third party software.
 
 The Python community also has an issue of complexity around the current
-bootstrap procedure for pip, setuptools and virtualenv. They all have
+bootstrap procedure for pip and setuptools. They all have
 their own bootstrap download file with slightly different usages and
 even refer to each other in some cases. Having a single bootstrap which
 is common amongst them all, with a simple usage, would be far preferable.
@@ -60,8 +60,8 @@
 Proposal
 ========
 
-The bootstrap will install the pip implementation, setuptools and virtualenv
-by downloading their installation files from PyPI.
+The bootstrap will install the pip implementation, setuptools by downloading
+their installation files from PyPI.
 
 This proposal affects two components of packaging: `the pip bootstrap`_ and,
 thanks to easier package installation, `modifications to publishing
@@ -77,11 +77,11 @@
 The Python installation includes an executable called "pip3" (see PEP 394 for
 naming rationale etc.) that attempts to import pip machinery.  If it can then
 the pip command proceeds as normal.  If it cannot it will bootstrap pip by
-downloading the pip implementation, setuptools and virtualenv wheel files.
-Hereafter the installation of the "pip implementation" will imply installation
-of setuptools and virtualenv.  Once installed, the pip command proceeds as
-normal. Once the bootstrap process is complete the "pip3" command is no longer
-the bootstrap but rather the full pip command.
+downloading the pip implementation and setuptools wheel files. Hereafter the
+installation of the "pip implementation" will imply installation of setuptools
+and virtualenv.  Once installed, the pip command proceeds as normal. Once the
+bootstrap process is complete the "pip3" command is no longer the bootstrap
+but rather the full pip command.
 
 A boostrap is used in the place of a the full pip code so that we don't have
 to bundle pip and also pip is upgradeable outside of the regular Python
@@ -123,8 +123,8 @@
 
 Some users may have no Internet access suitable for fetching the pip
 implementation file. These users can manually download and install the
-setuptools, virtualenv and pip tar files. Adding specific support for this
-use-case is unnecessary.
+setuptools and pip tar files. Adding specific support for this use-case is
+unnecessary.
 
 The download of the pip implementation install file will be performed
 securely.  The transport from pypi.python.org will be done over HTTPS with the
@@ -183,17 +183,17 @@
 issue tracker [2]_. Most notably, the addition of --bootstrap and --bootstrap-
 to-system to the pip command-line.
 
-It would be preferable that the pip, setuptools and virtualenv projects
-distribute a wheel format download.
+It would be preferable that the pip and setuptools projects distribute a wheel
+format download.
 
 The required code for this implementation is the "pip3" command described
 above. The additional pypublish can be developed outside of the scope of this
 PEP's work.
 
 Finally, it would be desirable that "pip3" be ported to Python 2.6+ to allow
-the single command to replace all existing pip, setuptools and virtualenv
-bootstrap scripts. Having that bootstrap included in a future Python 2.7
-release would also be highly desirable.
+the single command to replace existing pip, setuptools and virtualenv (which
+would be added to the bootstrap) bootstrap scripts. Having that bootstrap
+included in a future Python 2.7 release would also be highly desirable.
 
 
 Risks

-- 
Repository URL: http://hg.python.org/peps

From python-checkins at python.org  Wed Jul 10 14:04:13 2013
From: python-checkins at python.org (victor.stinner)
Date: Wed, 10 Jul 2013 14:04:13 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_call?=
 =?utf-8?q?=5Fexc=5Ftrace=28=29=3A_if_the_traceback_is_NULL=2C_use_None_wh?=
 =?utf-8?q?en?=
Message-ID: <3bqzb14Ntzz7LnW@mail.python.org>

http://hg.python.org/cpython/rev/4f730c045f5f
changeset:   84529:4f730c045f5f
user:        Victor Stinner <vstinner at wyplay.com>
date:        Wed Jul 10 13:57:55 2013 +0200
summary:
  Issue #18408: Fix call_exc_trace(): if the traceback is NULL, use None when
building the tuple (type, value, traceback) passed to the callback.

PyTuple_Pack() does crash if an argument is NULL.

files:
  Python/ceval.c |  11 ++++++++---
  1 files changed, 8 insertions(+), 3 deletions(-)


diff --git a/Python/ceval.c b/Python/ceval.c
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -3817,7 +3817,7 @@
 static void
 call_exc_trace(Py_tracefunc func, PyObject *self, PyFrameObject *f)
 {
-    PyObject *type, *value, *traceback, *arg;
+    PyObject *type, *value, *traceback, *orig_traceback, *arg;
     int err;
     PyErr_Fetch(&type, &value, &traceback);
     if (value == NULL) {
@@ -3825,6 +3825,11 @@
         Py_INCREF(value);
     }
     PyErr_NormalizeException(&type, &value, &traceback);
+    orig_traceback = traceback;
+    if (traceback == NULL) {
+        Py_INCREF(Py_None);
+        traceback = Py_None;
+    }
     arg = PyTuple_Pack(3, type, value, traceback);
     if (arg == NULL) {
         PyErr_Restore(type, value, traceback);
@@ -3833,11 +3838,11 @@
     err = call_trace(func, self, f, PyTrace_EXCEPTION, arg);
     Py_DECREF(arg);
     if (err == 0)
-        PyErr_Restore(type, value, traceback);
+        PyErr_Restore(type, value, orig_traceback);
     else {
         Py_XDECREF(type);
         Py_XDECREF(value);
-        Py_XDECREF(traceback);
+        Py_XDECREF(orig_traceback);
     }
 }
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Wed Jul 10 17:59:21 2013
From: python-checkins at python.org (r.david.murray)
Date: Wed, 10 Jul 2013 17:59:21 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE4Mzk5OiBmaXgg?=
	=?utf-8?q?comment_typo=2E?=
Message-ID: <3br4pK1cGNz7Lkt@mail.python.org>

http://hg.python.org/cpython/rev/c02656962b9c
changeset:   84530:c02656962b9c
branch:      3.3
parent:      84527:330c7aa2922b
user:        R David Murray <rdmurray at bitdance.com>
date:        Wed Jul 10 10:57:39 2013 -0400
summary:
  #18399: fix comment typo.

Patch by Andrew Rowe.

files:
  Modules/python.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Modules/python.c b/Modules/python.c
--- a/Modules/python.c
+++ b/Modules/python.c
@@ -19,7 +19,7 @@
 main(int argc, char **argv)
 {
     wchar_t **argv_copy = (wchar_t **)PyMem_Malloc(sizeof(wchar_t*)*(argc+1));
-    /* We need a second copies, as Python might modify the first one. */
+    /* We need a second copy, as Python might modify the first one. */
     wchar_t **argv_copy2 = (wchar_t **)PyMem_Malloc(sizeof(wchar_t*)*(argc+1));
     int i, res;
     char *oldloc;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Wed Jul 10 17:59:22 2013
From: python-checkins at python.org (r.david.murray)
Date: Wed, 10 Jul 2013 17:59:22 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Merge=3A_=2318399=3A_fix_comment_typo=2E?=
Message-ID: <3br4pL3q8Hz7LkH@mail.python.org>

http://hg.python.org/cpython/rev/b583fd54c8d6
changeset:   84531:b583fd54c8d6
parent:      84529:4f730c045f5f
parent:      84530:c02656962b9c
user:        R David Murray <rdmurray at bitdance.com>
date:        Wed Jul 10 11:57:39 2013 -0400
summary:
  Merge: #18399: fix comment typo.

files:
  Modules/python.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Modules/python.c b/Modules/python.c
--- a/Modules/python.c
+++ b/Modules/python.c
@@ -19,7 +19,7 @@
 main(int argc, char **argv)
 {
     wchar_t **argv_copy;
-    /* We need a second copies, as Python might modify the first one. */
+    /* We need a second copy, as Python might modify the first one. */
     wchar_t **argv_copy2;
     int i, res;
     char *oldloc;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Wed Jul 10 22:23:47 2013
From: python-checkins at python.org (r.david.murray)
Date: Wed, 10 Jul 2013 22:23:47 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE4NDI0OiBQRVA4?=
 =?utf-8?q?ify_the_tense_of_the_sum_docstring=2E?=
Message-ID: <3brBgR5zTDz7Ljk@mail.python.org>

http://hg.python.org/cpython/rev/4b3b87719e2c
changeset:   84532:4b3b87719e2c
branch:      3.3
parent:      84530:c02656962b9c
user:        R David Murray <rdmurray at bitdance.com>
date:        Wed Jul 10 16:22:14 2013 -0400
summary:
  #18424: PEP8ify the tense of the sum docstring.

files:
  Python/bltinmodule.c |  4 ++--
  1 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -2097,9 +2097,9 @@
 PyDoc_STRVAR(sum_doc,
 "sum(iterable[, start]) -> value\n\
 \n\
-Returns the sum of an iterable of numbers (NOT strings) plus the value\n\
+Return the sum of an iterable of numbers (NOT strings) plus the value\n\
 of parameter 'start' (which defaults to 0).  When the iterable is\n\
-empty, returns start.");
+empty, return start.");
 
 
 static PyObject *

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Wed Jul 10 22:23:49 2013
From: python-checkins at python.org (r.david.murray)
Date: Wed, 10 Jul 2013 22:23:49 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Merge=3A_=2318424=3A_PEP8ify_the_tense_of_the_sum_docstr?=
 =?utf-8?q?ing=2E?=
Message-ID: <3brBgT187Xz7Lk5@mail.python.org>

http://hg.python.org/cpython/rev/38b42ffdf86b
changeset:   84533:38b42ffdf86b
parent:      84531:b583fd54c8d6
parent:      84532:4b3b87719e2c
user:        R David Murray <rdmurray at bitdance.com>
date:        Wed Jul 10 16:22:59 2013 -0400
summary:
  Merge: #18424: PEP8ify the tense of the sum docstring.

files:
  Python/bltinmodule.c |  4 ++--
  1 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -2113,9 +2113,9 @@
 PyDoc_STRVAR(sum_doc,
 "sum(iterable[, start]) -> value\n\
 \n\
-Returns the sum of an iterable of numbers (NOT strings) plus the value\n\
+Return the sum of an iterable of numbers (NOT strings) plus the value\n\
 of parameter 'start' (which defaults to 0).  When the iterable is\n\
-empty, returns start.");
+empty, return start.");
 
 
 static PyObject *

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Wed Jul 10 22:23:50 2013
From: python-checkins at python.org (r.david.murray)
Date: Wed, 10 Jul 2013 22:23:50 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogIzE4NDI0OiBQRVA4?=
 =?utf-8?q?ify_the_tense_of_the_sum_docstring=2E?=
Message-ID: <3brBgV3M98z7LlQ@mail.python.org>

http://hg.python.org/cpython/rev/c5f5b5e89a94
changeset:   84534:c5f5b5e89a94
branch:      2.7
parent:      84478:5d41ebc79738
user:        R David Murray <rdmurray at bitdance.com>
date:        Wed Jul 10 16:23:15 2013 -0400
summary:
  #18424: PEP8ify the tense of the sum docstring.

files:
  Python/bltinmodule.c |  4 ++--
  1 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -2434,9 +2434,9 @@
 PyDoc_STRVAR(sum_doc,
 "sum(sequence[, start]) -> value\n\
 \n\
-Returns the sum of a sequence of numbers (NOT strings) plus the value\n\
+Return the sum of a sequence of numbers (NOT strings) plus the value\n\
 of parameter 'start' (which defaults to 0).  When the sequence is\n\
-empty, returns start.");
+empty, return start.");
 
 
 static PyObject *

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Wed Jul 10 23:10:44 2013
From: python-checkins at python.org (r.david.murray)
Date: Wed, 10 Jul 2013 23:10:44 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_=2318116=3A_getpass_no_lon?=
 =?utf-8?q?ger_always_falls_back_to_stdin=2E?=
Message-ID: <3brCjc3TyXz7Ljg@mail.python.org>

http://hg.python.org/cpython/rev/70f55dc9d43f
changeset:   84535:70f55dc9d43f
parent:      84533:38b42ffdf86b
user:        R David Murray <rdmurray at bitdance.com>
date:        Wed Jul 10 17:02:24 2013 -0400
summary:
  #18116: getpass no longer always falls back to stdin.

Also fixes a resource warning that occurred when the fallback is taken.

Patch by Serhiy Storchaka.

(We couldn't figure out how to write tests for this.)

files:
  Lib/getpass.py           |  94 +++++++++++++++------------
  Lib/test/test_getpass.py |  20 +++--
  Misc/NEWS                |   4 +
  3 files changed, 68 insertions(+), 50 deletions(-)


diff --git a/Lib/getpass.py b/Lib/getpass.py
--- a/Lib/getpass.py
+++ b/Lib/getpass.py
@@ -15,7 +15,11 @@
 #          Guido van Rossum (Windows support and cleanup)
 #          Gregory P. Smith (tty support & GetPassWarning)
 
-import os, sys, warnings
+import contextlib
+import io
+import os
+import sys
+import warnings
 
 __all__ = ["getpass","getuser","GetPassWarning"]
 
@@ -38,53 +42,57 @@
 
     Always restores terminal settings before returning.
     """
-    fd = None
-    tty = None
     passwd = None
-    try:
-        # Always try reading and writing directly on the tty first.
-        fd = os.open('/dev/tty', os.O_RDWR|os.O_NOCTTY)
-        tty = os.fdopen(fd, 'w+', 1)
-        input = tty
-        if not stream:
-            stream = tty
-    except OSError as e:
-        # If that fails, see if stdin can be controlled.
+    with contextlib.ExitStack() as stack:
         try:
-            fd = sys.stdin.fileno()
-        except (AttributeError, ValueError):
-            passwd = fallback_getpass(prompt, stream)
-        input = sys.stdin
-        if not stream:
-            stream = sys.stderr
+            # Always try reading and writing directly on the tty first.
+            fd = os.open('/dev/tty', os.O_RDWR|os.O_NOCTTY)
+            tty = io.FileIO(fd, 'w+')
+            stack.enter_context(tty)
+            input = io.TextIOWrapper(tty)
+            stack.enter_context(input)
+            if not stream:
+                stream = input
+        except OSError as e:
+            # If that fails, see if stdin can be controlled.
+            stack.close()
+            try:
+                fd = sys.stdin.fileno()
+            except (AttributeError, ValueError):
+                fd = None
+                passwd = fallback_getpass(prompt, stream)
+            input = sys.stdin
+            if not stream:
+                stream = sys.stderr
 
-    if fd is not None:
-        passwd = None
-        try:
-            old = termios.tcgetattr(fd)     # a copy to save
-            new = old[:]
-            new[3] &= ~termios.ECHO  # 3 == 'lflags'
-            tcsetattr_flags = termios.TCSAFLUSH
-            if hasattr(termios, 'TCSASOFT'):
-                tcsetattr_flags |= termios.TCSASOFT
+        if fd is not None:
             try:
-                termios.tcsetattr(fd, tcsetattr_flags, new)
-                passwd = _raw_input(prompt, stream, input=input)
-            finally:
-                termios.tcsetattr(fd, tcsetattr_flags, old)
-                stream.flush()  # issue7208
-        except termios.error:
-            if passwd is not None:
-                # _raw_input succeeded.  The final tcsetattr failed.  Reraise
-                # instead of leaving the terminal in an unknown state.
-                raise
-            # We can't control the tty or stdin.  Give up and use normal IO.
-            # fallback_getpass() raises an appropriate warning.
-            del input, tty  # clean up unused file objects before blocking
-            passwd = fallback_getpass(prompt, stream)
+                old = termios.tcgetattr(fd)     # a copy to save
+                new = old[:]
+                new[3] &= ~termios.ECHO  # 3 == 'lflags'
+                tcsetattr_flags = termios.TCSAFLUSH
+                if hasattr(termios, 'TCSASOFT'):
+                    tcsetattr_flags |= termios.TCSASOFT
+                try:
+                    termios.tcsetattr(fd, tcsetattr_flags, new)
+                    passwd = _raw_input(prompt, stream, input=input)
+                finally:
+                    termios.tcsetattr(fd, tcsetattr_flags, old)
+                    stream.flush()  # issue7208
+            except termios.error:
+                if passwd is not None:
+                    # _raw_input succeeded.  The final tcsetattr failed.  Reraise
+                    # instead of leaving the terminal in an unknown state.
+                    raise
+                # We can't control the tty or stdin.  Give up and use normal IO.
+                # fallback_getpass() raises an appropriate warning.
+                if stream is not input:
+                    # clean up unused file objects before blocking
+                    stack.close()
+                passwd = fallback_getpass(prompt, stream)
 
-    stream.write('\n')
-    return passwd
+        stream.write('\n')
+        return passwd
 
 
 def win_getpass(prompt='Password: ', stream=None):
diff --git a/Lib/test/test_getpass.py b/Lib/test/test_getpass.py
--- a/Lib/test/test_getpass.py
+++ b/Lib/test/test_getpass.py
@@ -1,7 +1,7 @@
 import getpass
 import os
 import unittest
-from io import StringIO
+from io import BytesIO, StringIO
 from unittest import mock
 from test import support
 
@@ -88,7 +88,8 @@
 
     def test_uses_tty_directly(self):
         with mock.patch('os.open') as open, \
-                mock.patch('os.fdopen'):
+                mock.patch('io.FileIO') as fileio, \
+                mock.patch('io.TextIOWrapper') as textio:
             # By setting open's return value to None the implementation will
             # skip code we don't care about in this test.  We can mock this out
             # fully if an alternate implementation works differently.
@@ -96,10 +97,13 @@
             getpass.unix_getpass()
             open.assert_called_once_with('/dev/tty',
                                          os.O_RDWR | os.O_NOCTTY)
+            fileio.assert_called_once_with(open.return_value, 'w+')
+            textio.assert_called_once_with(fileio.return_value)
 
     def test_resets_termios(self):
         with mock.patch('os.open') as open, \
-                mock.patch('os.fdopen'), \
+                mock.patch('io.FileIO'), \
+                mock.patch('io.TextIOWrapper'), \
                 mock.patch('termios.tcgetattr') as tcgetattr, \
                 mock.patch('termios.tcsetattr') as tcsetattr:
             open.return_value = 3
@@ -110,21 +114,23 @@
 
     def test_falls_back_to_fallback_if_termios_raises(self):
         with mock.patch('os.open') as open, \
-                mock.patch('os.fdopen') as fdopen, \
+                mock.patch('io.FileIO') as fileio, \
+                mock.patch('io.TextIOWrapper') as textio, \
                 mock.patch('termios.tcgetattr'), \
                 mock.patch('termios.tcsetattr') as tcsetattr, \
                 mock.patch('getpass.fallback_getpass') as fallback:
             open.return_value = 3
-            fdopen.return_value = StringIO()
+            fileio.return_value = BytesIO()
             tcsetattr.side_effect = termios.error
             getpass.unix_getpass()
             fallback.assert_called_once_with('Password: ',
-                                             fdopen.return_value)
+                                             textio.return_value)
 
     def test_flushes_stream_after_input(self):
         # issue 7208
         with mock.patch('os.open') as open, \
-                mock.patch('os.fdopen'), \
+                mock.patch('io.FileIO'), \
+                mock.patch('io.TextIOWrapper'), \
                 mock.patch('termios.tcgetattr'), \
                 mock.patch('termios.tcsetattr'):
             open.return_value = 3
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -142,6 +142,10 @@
 Library
 -------
 
+- Issue #18116: getpass was always getting an error when testing /dev/tty,
+  and thus was always falling back to stdin.  It also leaked an open file
+  when it did so.  Both of these issues are now fixed.
+
 - Issue #17198: Fix a NameError in the dbm module.  Patch by Valentina
   Mukhamedzhanova.
 

-- 
Repository URL: http://hg.python.org/cpython

From solipsis at pitrou.net  Thu Jul 11 05:47:28 2013
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Thu, 11 Jul 2013 05:47:28 +0200
Subject: [Python-checkins] Daily reference leaks (70f55dc9d43f): sum=-1
Message-ID: <E1Ux7qq-0007MQ-Ga@ap.vmr.nerim.net>

results for 70f55dc9d43f on branch "default"
--------------------------------------------

test_support leaked [0, 0, -1] references, sum=-1


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogUSE_86', '-x']

From python-checkins at python.org  Thu Jul 11 11:24:13 2013
From: python-checkins at python.org (christian.heimes)
Date: Thu, 11 Jul 2013 11:24:13 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4NDI2?=
 =?utf-8?q?=3A_Fix_NULL_pointer_dereference_in_C_extension_import_when?=
Message-ID: <3brWzx3w0lz7Lls@mail.python.org>

http://hg.python.org/cpython/rev/4343dfaca8e2
changeset:   84536:4343dfaca8e2
branch:      3.3
parent:      84532:4b3b87719e2c
user:        Christian Heimes <christian at cheimes.de>
date:        Thu Jul 11 11:22:21 2013 +0200
summary:
  Issue #18426: Fix NULL pointer dereference in C extension import when
PyModule_GetDef() returns an error.

files:
  Misc/NEWS         |  3 +++
  Python/importdl.c |  2 ++
  2 files changed, 5 insertions(+), 0 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,9 @@
 Core and Builtins
 -----------------
 
+- Issue #18426: Fix NULL pointer dereference in C extension import when
+  PyModule_GetDef() returns an error.
+
 - Issue #18328: Reorder ops in PyThreadState_Delete*() functions. Now the
   tstate is first removed from TLS and then deallocated.
 
diff --git a/Python/importdl.c b/Python/importdl.c
--- a/Python/importdl.c
+++ b/Python/importdl.c
@@ -97,6 +97,8 @@
 
     /* Remember pointer to module init function. */
     def = PyModule_GetDef(m);
+    if (def == NULL)
+        goto error;
     def->m_base.m_init = p;
 
     /* Remember the filename as the __file__ attribute */

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 11:24:14 2013
From: python-checkins at python.org (christian.heimes)
Date: Thu, 11 Jul 2013 11:24:14 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2318426=3A_Fix_NULL_pointer_dereference_in_C_exte?=
 =?utf-8?q?nsion_import_when?=
Message-ID: <3brWzy6DVZz7Lln@mail.python.org>

http://hg.python.org/cpython/rev/9fb3656b178a
changeset:   84537:9fb3656b178a
parent:      84535:70f55dc9d43f
parent:      84536:4343dfaca8e2
user:        Christian Heimes <christian at cheimes.de>
date:        Thu Jul 11 11:23:34 2013 +0200
summary:
  Issue #18426: Fix NULL pointer dereference in C extension import when
PyModule_GetDef() returns an error.

files:
  Misc/NEWS         |  3 +++
  Python/importdl.c |  2 ++
  2 files changed, 5 insertions(+), 0 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #18426: Fix NULL pointer dereference in C extension import when
+  PyModule_GetDef() returns an error.
+
 - Issue #17206: On Windows, increase the stack size from 2 MB to 4.2 MB to fix
   a stack overflow in the marshal module (fix a crash in test_marshal).
   Patch written by Jeremy Kloth.
diff --git a/Python/importdl.c b/Python/importdl.c
--- a/Python/importdl.c
+++ b/Python/importdl.c
@@ -97,6 +97,8 @@
 
     /* Remember pointer to module init function. */
     def = PyModule_GetDef(m);
+    if (def == NULL)
+        goto error;
     def->m_base.m_init = p;
 
     /* Remember the filename as the __file__ attribute */

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 13:03:47 2013
From: python-checkins at python.org (christian.heimes)
Date: Thu, 11 Jul 2013 13:03:47 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4NDI2?=
 =?utf-8?q?=3A_improve_exception_message=2E_Courtesy_of_Amaury?=
Message-ID: <3brZBq6R43zR9k@mail.python.org>

http://hg.python.org/cpython/rev/fce581643cb6
changeset:   84538:fce581643cb6
branch:      3.3
parent:      84536:4343dfaca8e2
user:        Christian Heimes <christian at cheimes.de>
date:        Thu Jul 11 13:02:30 2013 +0200
summary:
  Issue #18426: improve exception message. Courtesy of Amaury

files:
  Python/importdl.c |  6 +++++-
  1 files changed, 5 insertions(+), 1 deletions(-)


diff --git a/Python/importdl.c b/Python/importdl.c
--- a/Python/importdl.c
+++ b/Python/importdl.c
@@ -97,8 +97,12 @@
 
     /* Remember pointer to module init function. */
     def = PyModule_GetDef(m);
-    if (def == NULL)
+    if (def == NULL) {
+        PyErr_Format(PyExc_SystemError,
+                     "initialization of %s did not return an extension "
+                     "module", shortname);
         goto error;
+    }
     def->m_base.m_init = p;
 
     /* Remember the filename as the __file__ attribute */

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 13:03:49 2013
From: python-checkins at python.org (christian.heimes)
Date: Thu, 11 Jul 2013 13:03:49 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2318426=3A_improve_exception_message=2E_Courtesy_?=
 =?utf-8?q?of_Amaury?=
Message-ID: <3brZBs1czvzRJm@mail.python.org>

http://hg.python.org/cpython/rev/7a50d3c0aa61
changeset:   84539:7a50d3c0aa61
parent:      84537:9fb3656b178a
parent:      84538:fce581643cb6
user:        Christian Heimes <christian at cheimes.de>
date:        Thu Jul 11 13:02:37 2013 +0200
summary:
  Issue #18426: improve exception message. Courtesy of Amaury

files:
  Python/importdl.c |  6 +++++-
  1 files changed, 5 insertions(+), 1 deletions(-)


diff --git a/Python/importdl.c b/Python/importdl.c
--- a/Python/importdl.c
+++ b/Python/importdl.c
@@ -97,8 +97,12 @@
 
     /* Remember pointer to module init function. */
     def = PyModule_GetDef(m);
-    if (def == NULL)
+    if (def == NULL) {
+        PyErr_Format(PyExc_SystemError,
+                     "initialization of %s did not return an extension "
+                     "module", shortname);
         goto error;
+    }
     def->m_base.m_init = p;
 
     /* Remember the filename as the __file__ attribute */

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 13:35:29 2013
From: python-checkins at python.org (ronald.oussoren)
Date: Thu, 11 Jul 2013 13:35:29 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4NDI3?=
 =?utf-8?q?=3A_str=2Ereplace_could_crash_the_interpreter_with_huge_strings?=
 =?utf-8?q?=2E?=
Message-ID: <3brZvP33hvzRPr@mail.python.org>

http://hg.python.org/cpython/rev/2921f6c2009e
changeset:   84540:2921f6c2009e
branch:      2.7
parent:      84534:c5f5b5e89a94
user:        Ronald Oussoren <ronaldoussoren at mac.com>
date:        Thu Jul 11 13:33:55 2013 +0200
summary:
  Issue #18427: str.replace could crash the interpreter with huge strings.

This fixes two places where 'int'  was used to represent
the size of strings, instead of 'Py_ssize_t'.

(The issue is not present in the corresponding code in the 3.x branches)

Fixes #18427

files:
  Misc/NEWS              |  4 +++-
  Objects/stringobject.c |  6 +++---
  2 files changed, 6 insertions(+), 4 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -24,6 +24,8 @@
 Library
 -------
 
+- Issue #18427: str.replace could crash the interpreter with huge strings.
+
 - Issue #18347: ElementTree's html serializer now preserves the case of
   closing tags.
 
@@ -88,7 +90,7 @@
 
 - Issue #7136: In the Idle File menu, "New Window" is renamed "New File".
   Patch by Tal Einat, Roget Serwy, and Todd Rovito.
-  
+
 - Issue #8515: Set __file__ when run file in IDLE.
   Initial patch by Bruce Frederiksen.
 
diff --git a/Objects/stringobject.c b/Objects/stringobject.c
--- a/Objects/stringobject.c
+++ b/Objects/stringobject.c
@@ -882,9 +882,9 @@
             size -= chunk_size;
         }
 #ifdef __VMS
-        if (size) fwrite(data, (int)size, 1, fp);
+        if (size) fwrite(data, (size_t)size, 1, fp);
 #else
-        fwrite(data, 1, (int)size, fp);
+        fwrite(data, 1, (size_t)size, fp);
 #endif
         Py_END_ALLOW_THREADS
         return 0;
@@ -2332,7 +2332,7 @@
 }
 
 Py_LOCAL_INLINE(Py_ssize_t)
-countchar(const char *target, int target_len, char c, Py_ssize_t maxcount)
+countchar(const char *target, Py_ssize_t target_len, char c, Py_ssize_t maxcount)
 {
     Py_ssize_t count=0;
     const char *start=target;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 17:33:27 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 17:33:27 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4MzM2?=
 =?utf-8?q?=2E_Fix_a_link_to_StreamReader=27s_read=28=29_method=2E?=
Message-ID: <3brh9z2vfSz7LjY@mail.python.org>

http://hg.python.org/cpython/rev/7e186bb1642c
changeset:   84541:7e186bb1642c
branch:      2.7
parent:      84451:328781ae35d2
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 18:25:19 2013 +0300
summary:
  Issue #18336. Fix a link to StreamReader's read() method.

files:
  Doc/library/codecs.rst |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -653,7 +653,7 @@
       Read one line from the input stream and return the decoded data.
 
       *size*, if given, is passed as size argument to the stream's
-      :meth:`readline` method.
+      :meth:`read` method.
 
       If *keepends* is false line-endings will be stripped from the lines
       returned.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 17:33:28 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 17:33:28 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4MzM2?=
 =?utf-8?q?=2E_Fix_a_link_to_StreamReader=27s_read=28=29_method=2E?=
Message-ID: <3brhB05GpNz7LjY@mail.python.org>

http://hg.python.org/cpython/rev/8dd67c20cab7
changeset:   84542:8dd67c20cab7
branch:      3.3
parent:      84538:fce581643cb6
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 18:26:13 2013 +0300
summary:
  Issue #18336. Fix a link to StreamReader's read() method.

files:
  Doc/library/codecs.rst |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -694,7 +694,7 @@
       Read one line from the input stream and return the decoded data.
 
       *size*, if given, is passed as size argument to the stream's
-      :meth:`readline` method.
+      :meth:`read` method.
 
       If *keepends* is false line-endings will be stripped from the lines
       returned.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 17:33:30 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 17:33:30 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2318336=2E_Fix_a_link_to_StreamReader=27s_read=28?=
 =?utf-8?b?KSBtZXRob2Qu?=
Message-ID: <3brhB20pqrz7LmB@mail.python.org>

http://hg.python.org/cpython/rev/a53ac166fa58
changeset:   84543:a53ac166fa58
parent:      84539:7a50d3c0aa61
parent:      84542:8dd67c20cab7
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 18:27:20 2013 +0300
summary:
  Issue #18336. Fix a link to StreamReader's read() method.

files:
  Doc/library/codecs.rst |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -694,7 +694,7 @@
       Read one line from the input stream and return the decoded data.
 
       *size*, if given, is passed as size argument to the stream's
-      :meth:`readline` method.
+      :meth:`read` method.
 
       If *keepends* is false line-endings will be stripped from the lines
       returned.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 17:33:31 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 17:33:31 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMi43IC0+IDIuNyk6?=
	=?utf-8?q?_Merge_heads?=
Message-ID: <3brhB368cFz7LkL@mail.python.org>

http://hg.python.org/cpython/rev/7d75fe8b3210
changeset:   84544:7d75fe8b3210
branch:      2.7
parent:      84541:7e186bb1642c
parent:      84540:2921f6c2009e
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 18:28:35 2013 +0300
summary:
  Merge heads

files:
  Mac/PythonLauncher/FileSettings.h                |   5 -
  Mac/PythonLauncher/FileSettings.m                |  46 ++---
  Mac/PythonLauncher/MyAppDelegate.m               |   6 +-
  Mac/PythonLauncher/MyDocument.m                  |  22 +-
  Mac/PythonLauncher/PreferencesWindowController.m |  23 +-
  Mac/PythonLauncher/doscript.h                    |   2 +-
  Mac/PythonLauncher/doscript.m                    |  78 +++++-----
  Mac/PythonLauncher/main.m                        |   4 +-
  Misc/NEWS                                        |  10 +-
  Objects/stringobject.c                           |   6 +-
  Python/bltinmodule.c                             |   4 +-
  11 files changed, 98 insertions(+), 108 deletions(-)


diff --git a/Mac/PythonLauncher/FileSettings.h b/Mac/PythonLauncher/FileSettings.h
--- a/Mac/PythonLauncher/FileSettings.h
+++ b/Mac/PythonLauncher/FileSettings.h
@@ -45,18 +45,13 @@
 + (id)getFactorySettingsForFileType: (NSString *)filetype;
 + (id)newSettingsForFileType: (NSString *)filetype;
 
-//- (id)init;
 - (id)initForFileType: (NSString *)filetype;
 - (id)initForFSDefaultFileType: (NSString *)filetype;
 - (id)initForDefaultFileType: (NSString *)filetype;
-//- (id)initWithFileSettings: (FileSettings *)source;
 
 - (void)updateFromSource: (id <FileSettingsSource>)source;
 - (NSString *)commandLineForScript: (NSString *)script;
 
-//- (void)applyFactorySettingsForFileType: (NSString *)filetype;
-//- (void)saveDefaults;
-//- (void)applyUserDefaults: (NSString *)filetype;
 - (void)applyValuesFromDict: (NSDictionary *)dict;
 - (void)reset;
 - (NSArray *) interpreters;
diff --git a/Mac/PythonLauncher/FileSettings.m b/Mac/PythonLauncher/FileSettings.m
--- a/Mac/PythonLauncher/FileSettings.m
+++ b/Mac/PythonLauncher/FileSettings.m
@@ -14,7 +14,7 @@
 {
     static FileSettings *fsdefault_py, *fsdefault_pyw, *fsdefault_pyc;
     FileSettings **curdefault;
-    
+
     if ([filetype isEqualToString: @"Python Script"]) {
         curdefault = &fsdefault_py;
     } else if ([filetype isEqualToString: @"Python GUI Script"]) {
@@ -36,7 +36,7 @@
 {
     static FileSettings *default_py, *default_pyw, *default_pyc;
     FileSettings **curdefault;
-    
+
     if ([filetype isEqualToString: @"Python Script"]) {
         curdefault = &default_py;
     } else if ([filetype isEqualToString: @"Python GUI Script"]) {
@@ -57,7 +57,7 @@
 + (id)newSettingsForFileType: (NSString *)filetype
 {
     FileSettings *cur;
-    
+
     cur = [FileSettings new];
     [cur initForFileType: filetype];
     return [cur retain];
@@ -67,7 +67,7 @@
 {
     self = [super init];
     if (!self) return self;
-    
+
     interpreter = [source->interpreter retain];
     honourhashbang = source->honourhashbang;
     debug = source->debug;
@@ -81,36 +81,30 @@
     with_terminal = source->with_terminal;
     prefskey = source->prefskey;
     if (prefskey) [prefskey retain];
-    
+
     return self;
 }
 
 - (id)initForFileType: (NSString *)filetype
 {
     FileSettings *defaults;
-    
+
     defaults = [FileSettings getDefaultsForFileType: filetype];
     self = [self initWithFileSettings: defaults];
     origsource = [defaults retain];
     return self;
 }
 
-//- (id)init
-//{
-//    self = [self initForFileType: @"Python Script"];
-//    return self;
-//}
-
 - (id)initForFSDefaultFileType: (NSString *)filetype
 {
     int i;
     NSString *filename;
     NSDictionary *dict;
     static NSDictionary *factorySettings;
-    
+
     self = [super init];
     if (!self) return self;
-    
+
     if (factorySettings == NULL) {
         NSBundle *bdl = [NSBundle mainBundle];
         NSString *path = [ bdl pathForResource: @"factorySettings"
@@ -149,18 +143,18 @@
 {
     NSUserDefaults *defaults;
     NSDictionary *dict;
-    
+
     defaults = [NSUserDefaults standardUserDefaults];
     dict = [defaults dictionaryForKey: filetype];
     if (!dict)
         return;
     [self applyValuesFromDict: dict];
 }
-    
+
 - (id)initForDefaultFileType: (NSString *)filetype
 {
     FileSettings *fsdefaults;
-    
+
     fsdefaults = [FileSettings getFactorySettingsForFileType: filetype];
     self = [self initWithFileSettings: fsdefaults];
     if (!self) return self;
@@ -220,7 +214,7 @@
 - (void)applyValuesFromDict: (NSDictionary *)dict
 {
     id value;
-    
+
     value = [dict objectForKey: @"interpreter"];
     if (value) interpreter = [value retain];
     value = [dict objectForKey: @"honourhashbang"];
@@ -247,12 +241,12 @@
 
 - (NSString*)_replaceSingleQuotes: (NSString*)string
 {
-	/* Replace all single-quotes by '"'"', that way shellquoting will
-	 * be correct when the result value is delimited  using single quotes.
-	 */
-	NSArray* components = [string componentsSeparatedByString:@"'"];
+    /* Replace all single-quotes by '"'"', that way shellquoting will
+     * be correct when the result value is delimited  using single quotes.
+     */
+    NSArray* components = [string componentsSeparatedByString:@"'"];
 
-	return [components componentsJoinedByString:@"'\"'\"'"];
+    return [components componentsJoinedByString:@"'\"'\"'"];
 }
 
 - (NSString *)commandLineForScript: (NSString *)script
@@ -265,7 +259,7 @@
 
     script_dir = [script substringToIndex:
 	    [script length]-[[script lastPathComponent] length]];
-    
+
     if (honourhashbang &&
        (fp=fopen([script fileSystemRepresentation], "r")) &&
        fgets(hashbangbuf, sizeof(hashbangbuf), fp) &&
@@ -278,7 +272,7 @@
     }
     if (!cur_interp)
         cur_interp = interpreter;
-        
+
     return [NSString stringWithFormat:
         @"cd '%@' && '%@'%s%s%s%s%s%s %@ '%@' %@ %s",
     	[self _replaceSingleQuotes:script_dir],
@@ -297,7 +291,7 @@
 
 - (NSArray *) interpreters { return interpreters;};
 
-// FileSettingsSource protocol 
+// FileSettingsSource protocol
 - (NSString *) interpreter { return interpreter;};
 - (BOOL) honourhashbang { return honourhashbang; };
 - (BOOL) debug { return debug;};
diff --git a/Mac/PythonLauncher/MyAppDelegate.m b/Mac/PythonLauncher/MyAppDelegate.m
--- a/Mac/PythonLauncher/MyAppDelegate.m
+++ b/Mac/PythonLauncher/MyAppDelegate.m
@@ -33,7 +33,7 @@
 
 - (BOOL)shouldShowUI
 {
-    // if this call comes before applicationDidFinishLaunching: we 
+    // if this call comes before applicationDidFinishLaunching: we
     // should terminate immedeately after starting the script.
     if (!initial_action_done)
         should_terminate = YES;
@@ -62,7 +62,7 @@
     static NSString *extensions[] = { @"py", @"pyw", @"pyc", NULL};
     NSString **ext_p;
     int i;
-    
+
     if ([[NSUserDefaults standardUserDefaults] boolForKey: @"SkipFileBindingTest"])
         return;
     ourUrl = [NSURL fileURLWithPath: [[NSBundle mainBundle] bundlePath]];
@@ -92,5 +92,5 @@
         }
     }
 }
-        
+
 @end
diff --git a/Mac/PythonLauncher/MyDocument.m b/Mac/PythonLauncher/MyDocument.m
--- a/Mac/PythonLauncher/MyDocument.m
+++ b/Mac/PythonLauncher/MyDocument.m
@@ -16,7 +16,7 @@
 {
     self = [super init];
     if (self) {
-    
+
         // Add your subclass-specific initialization here.
         // If an error occurs here, send a [self dealloc] message and return nil.
         script = [@"<no script>.py" retain];
@@ -37,20 +37,17 @@
 {
     NSApplication *app = [NSApplication sharedApplication];
     [super close];
-    if ([[app delegate] shouldTerminate])
+    if ([(MyAppDelegate*)[app delegate] shouldTerminate])
         [app terminate: self];
 }
 
 - (void)load_defaults
 {
-//    if (settings) [settings release];
     settings = [FileSettings newSettingsForFileType: filetype];
 }
 
 - (void)update_display
 {
-//    [[self window] setTitle: script];
-    
     [interpreter setStringValue: [settings interpreter]];
     [honourhashbang setState: [settings honourhashbang]];
     [debug setState: [settings debug]];
@@ -62,7 +59,7 @@
     [others setStringValue: [settings others]];
     [scriptargs setStringValue: [settings scriptargs]];
     [with_terminal setState: [settings with_terminal]];
-    
+
     [commandline setStringValue: [settings commandLineForScript: script]];
 }
 
@@ -75,8 +72,8 @@
 {
     const char *cmdline;
     int sts;
-    
-     cmdline = [[settings commandLineForScript: script] cString];
+
+     cmdline = [[settings commandLineForScript: script] UTF8String];
    if ([settings with_terminal]) {
         sts = doscript(cmdline);
     } else {
@@ -107,14 +104,13 @@
 {
     // Insert code here to read your document from the given data.  You can also choose to override -loadFileWrapperRepresentation:ofType: or -readFromFile:ofType: instead.
     BOOL show_ui;
-    
-    // ask the app delegate whether we should show the UI or not. 
-    show_ui = [[[NSApplication sharedApplication] delegate] shouldShowUI];
+
+    // ask the app delegate whether we should show the UI or not.
+    show_ui = [(MyAppDelegate*)[[NSApplication sharedApplication] delegate] shouldShowUI];
     [script release];
     script = [fileName retain];
     [filetype release];
     filetype = [type retain];
-//    if (settings) [settings release];
     settings = [FileSettings newSettingsForFileType: filetype];
     if (show_ui) {
         [self update_display];
@@ -152,7 +148,7 @@
     [self update_display];
 }
 
-// FileSettingsSource protocol 
+// FileSettingsSource protocol
 - (NSString *) interpreter { return [interpreter stringValue];};
 - (BOOL) honourhashbang { return [honourhashbang state];};
 - (BOOL) debug { return [debug state];};
diff --git a/Mac/PythonLauncher/PreferencesWindowController.m b/Mac/PythonLauncher/PreferencesWindowController.m
--- a/Mac/PythonLauncher/PreferencesWindowController.m
+++ b/Mac/PythonLauncher/PreferencesWindowController.m
@@ -5,7 +5,7 @@
 + getPreferencesWindow
 {
     static PreferencesWindowController *_singleton;
-    
+
     if (!_singleton)
         _singleton = [[PreferencesWindowController alloc] init];
     [_singleton showWindow: _singleton];
@@ -21,15 +21,13 @@
 - (void)load_defaults
 {
     NSString *title = [filetype titleOfSelectedItem];
-    
+
     settings = [FileSettings getDefaultsForFileType: title];
 }
 
 - (void)update_display
 {
-//    [[self window] setTitle: script];
-    
-	[interpreter reloadData];
+    [interpreter reloadData];
     [interpreter setStringValue: [settings interpreter]];
     [honourhashbang setState: [settings honourhashbang]];
     [debug setState: [settings debug]];
@@ -41,7 +39,6 @@
     [others setStringValue: [settings others]];
     [with_terminal setState: [settings with_terminal]];
     // Not scriptargs, it isn't for preferences
-    
     [commandline setStringValue: [settings commandLineForScript: @"<your script here>"]];
 }
 
@@ -75,7 +72,7 @@
     [self update_display];
 }
 
-// FileSettingsSource protocol 
+// FileSettingsSource protocol
 - (NSString *) interpreter { return [interpreter stringValue];};
 - (BOOL) honourhashbang { return [honourhashbang state]; };
 - (BOOL) debug { return [debug state];};
@@ -98,23 +95,23 @@
 // NSComboBoxDataSource protocol
 - (unsigned int)comboBox:(NSComboBox *)aComboBox indexOfItemWithStringValue:(NSString *)aString
 {
-	NSArray *interp_list = [settings interpreters];
+    NSArray *interp_list = [settings interpreters];
     unsigned int rv = [interp_list indexOfObjectIdenticalTo: aString];
-	return rv;
+    return rv;
 }
 
 - (id)comboBox:(NSComboBox *)aComboBox objectValueForItemAtIndex:(int)index
 {
-	NSArray *interp_list = [settings interpreters];
+    NSArray *interp_list = [settings interpreters];
     id rv = [interp_list objectAtIndex: index];
-	return rv;
+    return rv;
 }
 
 - (int)numberOfItemsInComboBox:(NSComboBox *)aComboBox
 {
-	NSArray *interp_list = [settings interpreters];
+    NSArray *interp_list = [settings interpreters];
     int rv = [interp_list count];
-	return rv;
+    return rv;
 }
 
 
diff --git a/Mac/PythonLauncher/doscript.h b/Mac/PythonLauncher/doscript.h
--- a/Mac/PythonLauncher/doscript.h
+++ b/Mac/PythonLauncher/doscript.h
@@ -9,4 +9,4 @@
 
 #include <Carbon/Carbon.h>
 
-extern int doscript(const char *command);
\ No newline at end of file
+extern int doscript(const char *command);
diff --git a/Mac/PythonLauncher/doscript.m b/Mac/PythonLauncher/doscript.m
--- a/Mac/PythonLauncher/doscript.m
+++ b/Mac/PythonLauncher/doscript.m
@@ -11,49 +11,49 @@
 #import <ApplicationServices/ApplicationServices.h>
 #import "doscript.h"
 
-extern int 
+extern int
 doscript(const char *command)
 {
-	char *bundleID = "com.apple.Terminal";
-	AppleEvent evt, res;
-	AEDesc desc;
-	OSStatus err;
+    char *bundleID = "com.apple.Terminal";
+    AppleEvent evt, res;
+    AEDesc desc;
+    OSStatus err;
 
-	[[NSWorkspace sharedWorkspace] launchApplication:@"/Applications/Utilities/Terminal.app/"];
+    [[NSWorkspace sharedWorkspace] launchApplication:@"/Applications/Utilities/Terminal.app/"];
 
-	// Build event
-	err = AEBuildAppleEvent(kAECoreSuite, kAEDoScript,
-	                         typeApplicationBundleID,
-	                         bundleID, strlen(bundleID),
-	                         kAutoGenerateReturnID,
-	                         kAnyTransactionID,
-	                         &evt, NULL,
-	                         "'----':utf8(@)", strlen(command),
-	                         command);
-	if (err) {
-		NSLog(@"AEBuildAppleEvent failed: %d\n", err);
-		return err;
-	}
+    // Build event
+    err = AEBuildAppleEvent(kAECoreSuite, kAEDoScript,
+                             typeApplicationBundleID,
+                             bundleID, strlen(bundleID),
+                             kAutoGenerateReturnID,
+                             kAnyTransactionID,
+                             &evt, NULL,
+                             "'----':utf8(@)", strlen(command),
+                             command);
+    if (err) {
+        NSLog(@"AEBuildAppleEvent failed: %ld\n", (long)err);
+        return err;
+    }
 
-	// Send event and check for any Apple Event Manager errors
-	err = AESendMessage(&evt, &res, kAEWaitReply, kAEDefaultTimeout);
-	AEDisposeDesc(&evt);
-	if (err) {
-		NSLog(@"AESendMessage failed: %d\n", err);
-		return err;
-	}
-	// Check for any application errors
-	err = AEGetParamDesc(&res, keyErrorNumber, typeSInt32, &desc);
-	AEDisposeDesc(&res);
-	if (!err) {
-		AEGetDescData(&desc, &err, sizeof(err));
-		NSLog(@"Terminal returned an error: %d", err);
-		AEDisposeDesc(&desc);
-	} else if (err == errAEDescNotFound) {
-		err = noErr;
-	} else {
-		NSLog(@"AEGetPArmDesc returned an error: %d", err);
-	}
+    // Send event and check for any Apple Event Manager errors
+    err = AESendMessage(&evt, &res, kAEWaitReply, kAEDefaultTimeout);
+    AEDisposeDesc(&evt);
+    if (err) {
+        NSLog(@"AESendMessage failed: %ld\n", (long)err);
+        return err;
+    }
+    // Check for any application errors
+    err = AEGetParamDesc(&res, keyErrorNumber, typeSInt32, &desc);
+    AEDisposeDesc(&res);
+    if (!err) {
+        AEGetDescData(&desc, &err, sizeof(err));
+        NSLog(@"Terminal returned an error: %ld", (long)err);
+        AEDisposeDesc(&desc);
+    } else if (err == errAEDescNotFound) {
+        err = noErr;
+    } else {
+        NSLog(@"AEGetPArmDesc returned an error: %ld", (long)err);
+    }
 
-	return err;
+    return err;
 }
diff --git a/Mac/PythonLauncher/main.m b/Mac/PythonLauncher/main.m
--- a/Mac/PythonLauncher/main.m
+++ b/Mac/PythonLauncher/main.m
@@ -11,7 +11,7 @@
 
 int main(int argc, const char *argv[])
 {
-	char *home = getenv("HOME");
-	if (home) chdir(home);
+    char *home = getenv("HOME");
+    if (home) chdir(home);
     return NSApplicationMain(argc, argv);
 }
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -24,6 +24,8 @@
 Library
 -------
 
+- Issue #18427: str.replace could crash the interpreter with huge strings.
+
 - Issue #18347: ElementTree's html serializer now preserves the case of
   closing tags.
 
@@ -67,6 +69,12 @@
 - Issue #18113: Fixed a refcount leak in the curses.panel module's
   set_userptr() method.  Reported by Atsuo Ishimoto.
 
+Tools/Demos
+-----------
+
+- Issue #12990: The "Python Launcher" on OSX could not launch python scripts
+  that have paths that include wide characters.
+
 Build
 -----
 
@@ -82,7 +90,7 @@
 
 - Issue #7136: In the Idle File menu, "New Window" is renamed "New File".
   Patch by Tal Einat, Roget Serwy, and Todd Rovito.
-  
+
 - Issue #8515: Set __file__ when run file in IDLE.
   Initial patch by Bruce Frederiksen.
 
diff --git a/Objects/stringobject.c b/Objects/stringobject.c
--- a/Objects/stringobject.c
+++ b/Objects/stringobject.c
@@ -882,9 +882,9 @@
             size -= chunk_size;
         }
 #ifdef __VMS
-        if (size) fwrite(data, (int)size, 1, fp);
+        if (size) fwrite(data, (size_t)size, 1, fp);
 #else
-        fwrite(data, 1, (int)size, fp);
+        fwrite(data, 1, (size_t)size, fp);
 #endif
         Py_END_ALLOW_THREADS
         return 0;
@@ -2332,7 +2332,7 @@
 }
 
 Py_LOCAL_INLINE(Py_ssize_t)
-countchar(const char *target, int target_len, char c, Py_ssize_t maxcount)
+countchar(const char *target, Py_ssize_t target_len, char c, Py_ssize_t maxcount)
 {
     Py_ssize_t count=0;
     const char *start=target;
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -2434,9 +2434,9 @@
 PyDoc_STRVAR(sum_doc,
 "sum(sequence[, start]) -> value\n\
 \n\
-Returns the sum of a sequence of numbers (NOT strings) plus the value\n\
+Return the sum of a sequence of numbers (NOT strings) plus the value\n\
 of parameter 'start' (which defaults to 0).  When the sequence is\n\
-empty, returns start.");
+empty, return start.");
 
 
 static PyObject *

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 18:21:55 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 18:21:55 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Fix_reference_?=
 =?utf-8?q?leaks_introduced_by_the_patch_for_issue_=235308=2E?=
Message-ID: <3brjFv1vxTz7Lk3@mail.python.org>

http://hg.python.org/cpython/rev/1cf2c42af815
changeset:   84545:1cf2c42af815
branch:      2.7
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 19:14:07 2013 +0300
summary:
  Fix reference leaks introduced by the patch for issue #5308.

files:
  Python/marshal.c |  20 +++++++++++---------
  1 files changed, 11 insertions(+), 9 deletions(-)


diff --git a/Python/marshal.c b/Python/marshal.c
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -88,7 +88,7 @@
 }
 
 static void
-w_string(char *s, Py_ssize_t n, WFILE *p)
+w_string(const char *s, Py_ssize_t n, WFILE *p)
 {
     if (p->fp != NULL) {
         fwrite(s, 1, n, p->fp);
@@ -141,6 +141,13 @@
 # define W_SIZE  w_long
 #endif
 
+static void
+w_pstring(const char *s, Py_ssize_t n, WFILE *p)
+{
+        W_SIZE(n, p);
+        w_string(s, n, p);
+}
+
 /* We assume that Python longs are stored internally in base some power of
    2**15; for the sake of portability we'll always read and write them in base
    exactly 2**15. */
@@ -338,9 +345,7 @@
         else {
             w_byte(TYPE_STRING, p);
         }
-        n = PyString_GET_SIZE(v);
-        W_SIZE(n, p);
-        w_string(PyString_AS_STRING(v), n, p);
+        w_pstring(PyBytes_AS_STRING(v), PyString_GET_SIZE(v), p);
     }
 #ifdef Py_USING_UNICODE
     else if (PyUnicode_CheckExact(v)) {
@@ -352,9 +357,7 @@
             return;
         }
         w_byte(TYPE_UNICODE, p);
-        n = PyString_GET_SIZE(utf8);
-        W_SIZE(n, p);
-        w_string(PyString_AS_STRING(utf8), n, p);
+        w_pstring(PyString_AS_STRING(utf8), PyString_GET_SIZE(utf8), p);
         Py_DECREF(utf8);
     }
 #endif
@@ -441,8 +444,7 @@
         PyBufferProcs *pb = v->ob_type->tp_as_buffer;
         w_byte(TYPE_STRING, p);
         n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
-        W_SIZE(n, p);
-        w_string(s, n, p);
+        w_pstring(s, n, p);
     }
     else {
         w_byte(TYPE_UNKNOWN, p);

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 18:21:56 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 18:21:56 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E3=29=3A_Fix_reference_?=
 =?utf-8?q?leaks_introduced_by_the_patch_for_issue_=235308=2E?=
Message-ID: <3brjFw4Clbz7Llb@mail.python.org>

http://hg.python.org/cpython/rev/8b99f2224c3a
changeset:   84546:8b99f2224c3a
branch:      3.3
parent:      84542:8dd67c20cab7
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 19:14:26 2013 +0300
summary:
  Fix reference leaks introduced by the patch for issue #5308.

files:
  Python/marshal.c |  23 +++++++++++------------
  1 files changed, 11 insertions(+), 12 deletions(-)


diff --git a/Python/marshal.c b/Python/marshal.c
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -95,7 +95,7 @@
 }
 
 static void
-w_string(char *s, Py_ssize_t n, WFILE *p)
+w_string(const char *s, Py_ssize_t n, WFILE *p)
 {
     if (p->fp != NULL) {
         fwrite(s, 1, n, p->fp);
@@ -139,6 +139,13 @@
 # define W_SIZE  w_long
 #endif
 
+static void
+w_pstring(const char *s, Py_ssize_t n, WFILE *p)
+{
+        W_SIZE(n, p);
+        w_string(s, n, p);
+}
+
 /* We assume that Python longs are stored internally in base some power of
    2**15; for the sake of portability we'll always read and write them in base
    exactly 2**15. */
@@ -313,9 +320,7 @@
     }
     else if (PyBytes_CheckExact(v)) {
         w_byte(TYPE_STRING, p);
-        n = PyBytes_GET_SIZE(v);
-        W_SIZE(n, p);
-        w_string(PyBytes_AS_STRING(v), n, p);
+        w_pstring(PyBytes_AS_STRING(v), PyBytes_GET_SIZE(v), p);
     }
     else if (PyUnicode_CheckExact(v)) {
         PyObject *utf8;
@@ -326,9 +331,7 @@
             return;
         }
         w_byte(TYPE_UNICODE, p);
-        n = PyBytes_GET_SIZE(utf8);
-        W_SIZE(n, p);
-        w_string(PyBytes_AS_STRING(utf8), n, p);
+        w_pstring(PyBytes_AS_STRING(utf8), PyBytes_GET_SIZE(utf8), p);
         Py_DECREF(utf8);
     }
     else if (PyTuple_CheckExact(v)) {
@@ -411,7 +414,6 @@
     }
     else if (PyObject_CheckBuffer(v)) {
         /* Write unknown buffer-style objects as a string */
-        char *s;
         Py_buffer view;
         if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) != 0) {
             w_byte(TYPE_UNKNOWN, p);
@@ -420,10 +422,7 @@
             return;
         }
         w_byte(TYPE_STRING, p);
-        n = view.len;
-        s = view.buf;
-        W_SIZE(n, p);
-        w_string(s, n, p);
+        w_pstring(view.buf, view.len, p);
         PyBuffer_Release(&view);
     }
     else {

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 18:21:58 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 18:21:58 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Fix_reference_leaks_introduced_by_the_patch_for_issue_?=
 =?utf-8?b?IzUzMDgu?=
Message-ID: <3brjFy00Ysz7LlQ@mail.python.org>

http://hg.python.org/cpython/rev/19ed630d8d75
changeset:   84547:19ed630d8d75
parent:      84543:a53ac166fa58
parent:      84546:8b99f2224c3a
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 19:19:47 2013 +0300
summary:
  Fix reference leaks introduced by the patch for issue #5308.

files:
  Python/marshal.c |  23 +++++++++++------------
  1 files changed, 11 insertions(+), 12 deletions(-)


diff --git a/Python/marshal.c b/Python/marshal.c
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -97,7 +97,7 @@
 }
 
 static void
-w_string(char *s, Py_ssize_t n, WFILE *p)
+w_string(const char *s, Py_ssize_t n, WFILE *p)
 {
     if (p->fp != NULL) {
         fwrite(s, 1, n, p->fp);
@@ -141,6 +141,13 @@
 # define W_SIZE  w_long
 #endif
 
+static void
+w_pstring(const char *s, Py_ssize_t n, WFILE *p)
+{
+        W_SIZE(n, p);
+        w_string(s, n, p);
+}
+
 /* We assume that Python longs are stored internally in base some power of
    2**15; for the sake of portability we'll always read and write them in base
    exactly 2**15. */
@@ -384,9 +391,7 @@
     }
     else if (PyBytes_CheckExact(v)) {
         W_TYPE(TYPE_STRING, p);
-        n = PyBytes_GET_SIZE(v);
-        W_SIZE(n, p);
-        w_string(PyBytes_AS_STRING(v), n, p);
+        w_pstring(PyBytes_AS_STRING(v), PyBytes_GET_SIZE(v), p);
     }
     else if (PyUnicode_CheckExact(v)) {
         PyObject *utf8;
@@ -400,9 +405,7 @@
             W_TYPE(TYPE_INTERNED, p);
         else
             W_TYPE(TYPE_UNICODE, p);
-        n = PyBytes_GET_SIZE(utf8);
-        W_SIZE(n, p);
-        w_string(PyBytes_AS_STRING(utf8), n, p);
+        w_pstring(PyBytes_AS_STRING(utf8), PyBytes_GET_SIZE(utf8), p);
         Py_DECREF(utf8);
     }
     else if (PyTuple_CheckExact(v)) {
@@ -485,7 +488,6 @@
     }
     else if (PyObject_CheckBuffer(v)) {
         /* Write unknown buffer-style objects as a string */
-        char *s;
         Py_buffer view;
         if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) != 0) {
             w_byte(TYPE_UNKNOWN, p);
@@ -494,10 +496,7 @@
             return;
         }
         W_TYPE(TYPE_STRING, p);
-        n = view.len;
-        s = view.buf;
-        W_SIZE(n, p);
-        w_string(s, n, p);
+        w_pstring(view.buf, view.len, p);
         PyBuffer_Release(&view);
     }
     else {

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 18:30:10 2013
From: python-checkins at python.org (r.david.murray)
Date: Thu, 11 Jul 2013 18:30:10 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE3OTg3OiBwcm9w?=
 =?utf-8?q?erly_document_support=2Ecaptured=5Fxxx=2E?=
Message-ID: <3brjRQ4qQpz7LkL@mail.python.org>

http://hg.python.org/cpython/rev/af2416c2e27c
changeset:   84548:af2416c2e27c
branch:      3.3
parent:      84546:8b99f2224c3a
user:        R David Murray <rdmurray at bitdance.com>
date:        Thu Jul 11 12:28:40 2013 -0400
summary:
  #17987: properly document support.captured_xxx.

Patch by Dmi Baranov.

files:
  Doc/library/test.rst     |  26 +++++++++++++++++++-------
  Lib/test/support.py      |  19 +++++++++++++++++--
  Lib/test/test_support.py |  17 ++++++++++-------
  Misc/ACKS                |   1 +
  4 files changed, 47 insertions(+), 16 deletions(-)


diff --git a/Doc/library/test.rst b/Doc/library/test.rst
--- a/Doc/library/test.rst
+++ b/Doc/library/test.rst
@@ -362,17 +362,29 @@
       New optional arguments *filters* and *quiet*.
 
 
-.. function:: captured_stdout()
+.. function:: captured_stdin()
+              captured_stdout()
+              captured_stderr()
 
-   A context manager that runs the :keyword:`with` statement body using a
-   :class:`io.StringIO` object as sys.stdout.  That object can be retrieved
-   using the ``as`` clause of the :keyword:`with` statement.
+   A context managers that temporarily replaces the named stream with
+   :class:`io.StringIO` object.
 
-   Example use::
+   Example use with output streams::
 
-      with captured_stdout() as s:
+      with captured_stdout() as stdout, captured_stderr() as stderr:
           print("hello")
-      assert s.getvalue() == "hello\n"
+          print("error", file=sys.stderr)
+      assert stdout.getvalue() == "hello\n"
+      assert stderr.getvalue() == "error\n"
+
+   Example use with input stream::
+
+      with captured_stdin() as stdin:
+          stdin.write('hello\n')
+          stdin.seek(0)
+          # call test code that consumes from sys.stdin
+          captured = input()
+      self.assertEqual(captured, "hello")
 
 
 .. function:: temp_cwd(name='tempcwd', quiet=False, path=None)
diff --git a/Lib/test/support.py b/Lib/test/support.py
--- a/Lib/test/support.py
+++ b/Lib/test/support.py
@@ -1184,16 +1184,31 @@
 def captured_stdout():
     """Capture the output of sys.stdout:
 
-       with captured_stdout() as s:
+       with captured_stdout() as stdout:
            print("hello")
-       self.assertEqual(s.getvalue(), "hello")
+       self.assertEqual(stdout.getvalue(), "hello\n")
     """
     return captured_output("stdout")
 
 def captured_stderr():
+    """Capture the output of sys.stderr:
+
+       with captured_stderr() as stderr:
+           print("hello", file=sys.stderr)
+       self.assertEqual(stderr.getvalue(), "hello\n")
+    """
     return captured_output("stderr")
 
 def captured_stdin():
+    """Capture the input to sys.stdin:
+
+       with captured_stdin() as stdin:
+           stdin.write('hello\n')
+           stdin.seek(0)
+           # call test code that consumes from sys.stdin
+           captured = input()
+       self.assertEqual(captured, "hello")
+    """
     return captured_output("stdin")
 
 
diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
--- a/Lib/test/test_support.py
+++ b/Lib/test/test_support.py
@@ -130,19 +130,22 @@
         self.assertNotIn("bar", sys.path)
 
     def test_captured_stdout(self):
-        with support.captured_stdout() as s:
+        with support.captured_stdout() as stdout:
             print("hello")
-        self.assertEqual(s.getvalue(), "hello\n")
+        self.assertEqual(stdout.getvalue(), "hello\n")
 
     def test_captured_stderr(self):
-        with support.captured_stderr() as s:
+        with support.captured_stderr() as stderr:
             print("hello", file=sys.stderr)
-        self.assertEqual(s.getvalue(), "hello\n")
+        self.assertEqual(stderr.getvalue(), "hello\n")
 
     def test_captured_stdin(self):
-        with support.captured_stdin() as s:
-            print("hello", file=sys.stdin)
-        self.assertEqual(s.getvalue(), "hello\n")
+        with support.captured_stdin() as stdin:
+            stdin.write('hello\n')
+            stdin.seek(0)
+            # call test code that consumes from sys.stdin
+            captured = input()
+        self.assertEqual(captured, "hello")
 
     def test_gc_collect(self):
         support.gc_collect()
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -65,6 +65,7 @@
 Jeff Balogh
 Manuel Balsera
 Matt Bandy
+Dmi Baranov
 Michael J. Barber
 Daniel Barclay
 Nicolas Bareil

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 18:30:12 2013
From: python-checkins at python.org (r.david.murray)
Date: Thu, 11 Jul 2013 18:30:12 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Merge_=2317987=3A_properly_document_support=2Ecaptured?=
 =?utf-8?b?X3h4eC4=?=
Message-ID: <3brjRS1Hccz7LlQ@mail.python.org>

http://hg.python.org/cpython/rev/d0f7f1996001
changeset:   84549:d0f7f1996001
parent:      84547:19ed630d8d75
parent:      84548:af2416c2e27c
user:        R David Murray <rdmurray at bitdance.com>
date:        Thu Jul 11 12:29:31 2013 -0400
summary:
  Merge #17987: properly document support.captured_xxx.

files:
  Doc/library/test.rst     |  26 +++++++++++++++++++-------
  Lib/test/support.py      |  19 +++++++++++++++++--
  Lib/test/test_support.py |  17 ++++++++++-------
  Misc/ACKS                |   1 +
  4 files changed, 47 insertions(+), 16 deletions(-)


diff --git a/Doc/library/test.rst b/Doc/library/test.rst
--- a/Doc/library/test.rst
+++ b/Doc/library/test.rst
@@ -362,17 +362,29 @@
       New optional arguments *filters* and *quiet*.
 
 
-.. function:: captured_stdout()
+.. function:: captured_stdin()
+              captured_stdout()
+              captured_stderr()
 
-   A context manager that runs the :keyword:`with` statement body using a
-   :class:`io.StringIO` object as sys.stdout.  That object can be retrieved
-   using the ``as`` clause of the :keyword:`with` statement.
+   A context managers that temporarily replaces the named stream with
+   :class:`io.StringIO` object.
 
-   Example use::
+   Example use with output streams::
 
-      with captured_stdout() as s:
+      with captured_stdout() as stdout, captured_stderr() as stderr:
           print("hello")
-      assert s.getvalue() == "hello\n"
+          print("error", file=sys.stderr)
+      assert stdout.getvalue() == "hello\n"
+      assert stderr.getvalue() == "error\n"
+
+   Example use with input stream::
+
+      with captured_stdin() as stdin:
+          stdin.write('hello\n')
+          stdin.seek(0)
+          # call test code that consumes from sys.stdin
+          captured = input()
+      self.assertEqual(captured, "hello")
 
 
 .. function:: temp_cwd(name='tempcwd', quiet=False, path=None)
diff --git a/Lib/test/support.py b/Lib/test/support.py
--- a/Lib/test/support.py
+++ b/Lib/test/support.py
@@ -1186,16 +1186,31 @@
 def captured_stdout():
     """Capture the output of sys.stdout:
 
-       with captured_stdout() as s:
+       with captured_stdout() as stdout:
            print("hello")
-       self.assertEqual(s.getvalue(), "hello")
+       self.assertEqual(stdout.getvalue(), "hello\n")
     """
     return captured_output("stdout")
 
 def captured_stderr():
+    """Capture the output of sys.stderr:
+
+       with captured_stderr() as stderr:
+           print("hello", file=sys.stderr)
+       self.assertEqual(stderr.getvalue(), "hello\n")
+    """
     return captured_output("stderr")
 
 def captured_stdin():
+    """Capture the input to sys.stdin:
+
+       with captured_stdin() as stdin:
+           stdin.write('hello\n')
+           stdin.seek(0)
+           # call test code that consumes from sys.stdin
+           captured = input()
+       self.assertEqual(captured, "hello")
+    """
     return captured_output("stdin")
 
 
diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
--- a/Lib/test/test_support.py
+++ b/Lib/test/test_support.py
@@ -130,19 +130,22 @@
         self.assertNotIn("bar", sys.path)
 
     def test_captured_stdout(self):
-        with support.captured_stdout() as s:
+        with support.captured_stdout() as stdout:
             print("hello")
-        self.assertEqual(s.getvalue(), "hello\n")
+        self.assertEqual(stdout.getvalue(), "hello\n")
 
     def test_captured_stderr(self):
-        with support.captured_stderr() as s:
+        with support.captured_stderr() as stderr:
             print("hello", file=sys.stderr)
-        self.assertEqual(s.getvalue(), "hello\n")
+        self.assertEqual(stderr.getvalue(), "hello\n")
 
     def test_captured_stdin(self):
-        with support.captured_stdin() as s:
-            print("hello", file=sys.stdin)
-        self.assertEqual(s.getvalue(), "hello\n")
+        with support.captured_stdin() as stdin:
+            stdin.write('hello\n')
+            stdin.seek(0)
+            # call test code that consumes from sys.stdin
+            captured = input()
+        self.assertEqual(captured, "hello")
 
     def test_gc_collect(self):
         support.gc_collect()
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -66,6 +66,7 @@
 Jeff Balogh
 Manuel Balsera
 Matt Bandy
+Dmi Baranov
 Michael J. Barber
 Daniel Barclay
 Nicolas Bareil

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 19:01:57 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 19:01:57 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318338=3A_=60pytho?=
 =?utf-8?q?n_--version=60_now_prints_version_string_to_stdout=2C_and?=
Message-ID: <3brk854vr8z7LjQ@mail.python.org>

http://hg.python.org/cpython/rev/e6384b8b2325
changeset:   84550:e6384b8b2325
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 20:01:17 2013 +0300
summary:
  Issue #18338: `python --version` now prints version string to stdout, and
not to stderr.  Patch by Berker Peksag and Michael Dickens.

files:
  Lib/test/test_cmd_line.py |  6 ++++--
  Misc/NEWS                 |  3 +++
  Modules/main.c            |  2 +-
  3 files changed, 8 insertions(+), 3 deletions(-)


diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py
--- a/Lib/test/test_cmd_line.py
+++ b/Lib/test/test_cmd_line.py
@@ -41,8 +41,10 @@
 
     def test_version(self):
         version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii")
-        rc, out, err = assert_python_ok('-V')
-        self.assertTrue(err.startswith(version))
+        for switch in '-V', '--version':
+            rc, out, err = assert_python_ok(switch)
+            self.assertFalse(err.startswith(version))
+            self.assertTrue(out.startswith(version))
 
     def test_verbose(self):
         # -v causes imports to write to stderr.  If the write to
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #18338: `python --version` now prints version string to stdout, and
+  not to stderr.  Patch by Berker Peksag and Michael Dickens.
+
 - Issue #18426: Fix NULL pointer dereference in C extension import when
   PyModule_GetDef() returns an error.
 
diff --git a/Modules/main.c b/Modules/main.c
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -500,7 +500,7 @@
         return usage(0, argv[0]);
 
     if (version) {
-        fprintf(stderr, "Python %s\n", PY_VERSION);
+        printf("Python %s\n", PY_VERSION);
         return 0;
     }
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 19:38:38 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 19:38:38 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE4MTAx?=
 =?utf-8?q?=3A_Tcl=2Esplit=28=29_now_process_Unicode_strings_nested_in_a_t?=
 =?utf-8?q?uple_as_it?=
Message-ID: <3brkyQ3tQxzSwQ@mail.python.org>

http://hg.python.org/cpython/rev/f53cdd4e2689
changeset:   84551:f53cdd4e2689
branch:      2.7
parent:      84545:1cf2c42af815
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 20:32:48 2013 +0300
summary:
  Issue #18101: Tcl.split() now process Unicode strings nested in a tuple as it
do with byte strings.

Added tests for Tcl.split() and tcl.splitline().

files:
  Lib/test/test_tcl.py |  60 ++++++++++++++++++++++++++++++++
  Misc/NEWS            |   3 +
  Modules/_tkinter.c   |  27 ++++++++++++++
  3 files changed, 90 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py
--- a/Lib/test/test_tcl.py
+++ b/Lib/test/test_tcl.py
@@ -184,6 +184,66 @@
                 self.assertEqual(passValue(f), f)
         self.assertEqual(passValue((1, '2', (3.4,))), (1, '2', (3.4,)))
 
+    def test_splitlist(self):
+        splitlist = self.interp.tk.splitlist
+        call = self.interp.tk.call
+        self.assertRaises(TypeError, splitlist)
+        self.assertRaises(TypeError, splitlist, 'a', 'b')
+        self.assertRaises(TypeError, splitlist, 2)
+        testcases = [
+            ('2', ('2',)),
+            ('', ()),
+            ('{}', ('',)),
+            ('""', ('',)),
+            ('a\n b\t\r c\n ', ('a', 'b', 'c')),
+            (u'a\n b\t\r c\n ', ('a', 'b', 'c')),
+            ('a \xe2\x82\xac', ('a', '\xe2\x82\xac')),
+            (u'a \u20ac', ('a', '\xe2\x82\xac')),
+            ('a {b c}', ('a', 'b c')),
+            (r'a b\ c', ('a', 'b c')),
+            (('a', 'b c'), ('a', 'b c')),
+            ('a 2', ('a', '2')),
+            (('a', 2), ('a', 2)),
+            ('a 3.4', ('a', '3.4')),
+            (('a', 3.4), ('a', 3.4)),
+            ((), ()),
+            (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))),
+        ]
+        for arg, res in testcases:
+            self.assertEqual(splitlist(arg), res)
+        self.assertRaises(TclError, splitlist, '{')
+
+    def test_split(self):
+        split = self.interp.tk.split
+        call = self.interp.tk.call
+        self.assertRaises(TypeError, split)
+        self.assertRaises(TypeError, split, 'a', 'b')
+        self.assertRaises(TypeError, split, 2)
+        testcases = [
+            ('2', '2'),
+            ('', ''),
+            ('{}', ''),
+            ('""', ''),
+            ('{', '{'),
+            ('a\n b\t\r c\n ', ('a', 'b', 'c')),
+            (u'a\n b\t\r c\n ', ('a', 'b', 'c')),
+            ('a \xe2\x82\xac', ('a', '\xe2\x82\xac')),
+            (u'a \u20ac', ('a', '\xe2\x82\xac')),
+            ('a {b c}', ('a', ('b', 'c'))),
+            (r'a b\ c', ('a', ('b', 'c'))),
+            (('a', 'b c'), ('a', ('b', 'c'))),
+            (('a', u'b c'), ('a', ('b', 'c'))),
+            ('a 2', ('a', '2')),
+            (('a', 2), ('a', 2)),
+            ('a 3.4', ('a', '3.4')),
+            (('a', 3.4), ('a', 3.4)),
+            (('a', (2, 3.4)), ('a', (2, 3.4))),
+            ((), ()),
+            (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))),
+        ]
+        for arg, res in testcases:
+            self.assertEqual(split(arg), res)
+
 
 def test_main():
     test_support.run_unittest(TclTest, TkinterTest)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -24,6 +24,9 @@
 Library
 -------
 
+- Issue #18101: Tcl.split() now process Unicode strings nested in a tuple as it
+  do with byte strings.
+
 - Issue #18427: str.replace could crash the interpreter with huge strings.
 
 - Issue #18347: ElementTree's html serializer now preserves the case of
diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c
--- a/Modules/_tkinter.c
+++ b/Modules/_tkinter.c
@@ -547,6 +547,33 @@
             return Split(PyString_AsString(arg));
         /* Fall through, returning arg. */
     }
+    else if (PyUnicode_Check(arg)) {
+        int argc;
+        char **argv;
+        char *list;
+        PyObject *s = PyUnicode_AsUTF8String(arg);
+
+        if (s == NULL) {
+            Py_INCREF(arg);
+            return arg;
+        }
+        list = PyString_AsString(s);
+
+        if (list == NULL ||
+            Tcl_SplitList((Tcl_Interp *)NULL, list, &argc, &argv) != TCL_OK) {
+            Py_DECREF(s);
+            Py_INCREF(arg);
+            return arg;
+        }
+        Tcl_Free(FREECAST argv);
+        if (argc > 1) {
+            PyObject *v = Split(list);
+            Py_DECREF(s);
+            return v;
+        }
+        Py_DECREF(s);
+        /* Fall through, returning arg. */
+    }
     Py_INCREF(arg);
     return arg;
 }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 19:38:40 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 19:38:40 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4MTAx?=
 =?utf-8?q?=3A_Tcl=2Esplit=28=29_now_process_strings_nested_in_a_tuple_as_?=
 =?utf-8?q?it?=
Message-ID: <3brkyS0QW9zSyb@mail.python.org>

http://hg.python.org/cpython/rev/9486c07929a1
changeset:   84552:9486c07929a1
branch:      3.3
parent:      84548:af2416c2e27c
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 20:34:47 2013 +0300
summary:
  Issue #18101: Tcl.split() now process strings nested in a tuple as it
do with byte strings.

Added tests for Tcl.split() and Tcl.splitline().

files:
  Lib/test/test_tcl.py |  60 ++++++++++++++++++++++++++++++++
  Misc/NEWS            |   3 +
  Modules/_tkinter.c   |  15 ++++++++
  3 files changed, 78 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py
--- a/Lib/test/test_tcl.py
+++ b/Lib/test/test_tcl.py
@@ -175,6 +175,66 @@
                 self.assertEqual(passValue(f), f)
         self.assertEqual(passValue((1, '2', (3.4,))), (1, '2', (3.4,)))
 
+    def test_splitlist(self):
+        splitlist = self.interp.tk.splitlist
+        call = self.interp.tk.call
+        self.assertRaises(TypeError, splitlist)
+        self.assertRaises(TypeError, splitlist, 'a', 'b')
+        self.assertRaises(TypeError, splitlist, 2)
+        testcases = [
+            ('2', ('2',)),
+            ('', ()),
+            ('{}', ('',)),
+            ('""', ('',)),
+            ('a\n b\t\r c\n ', ('a', 'b', 'c')),
+            (b'a\n b\t\r c\n ', ('a', 'b', 'c')),
+            ('a \u20ac', ('a', '\u20ac')),
+            (b'a \xe2\x82\xac', ('a', '\u20ac')),
+            ('a {b c}', ('a', 'b c')),
+            (r'a b\ c', ('a', 'b c')),
+            (('a', 'b c'), ('a', 'b c')),
+            ('a 2', ('a', '2')),
+            (('a', 2), ('a', 2)),
+            ('a 3.4', ('a', '3.4')),
+            (('a', 3.4), ('a', 3.4)),
+            ((), ()),
+            (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))),
+        ]
+        for arg, res in testcases:
+            self.assertEqual(splitlist(arg), res, msg=arg)
+        self.assertRaises(TclError, splitlist, '{')
+
+    def test_split(self):
+        split = self.interp.tk.split
+        call = self.interp.tk.call
+        self.assertRaises(TypeError, split)
+        self.assertRaises(TypeError, split, 'a', 'b')
+        self.assertRaises(TypeError, split, 2)
+        testcases = [
+            ('2', '2'),
+            ('', ''),
+            ('{}', ''),
+            ('""', ''),
+            ('{', '{'),
+            ('a\n b\t\r c\n ', ('a', 'b', 'c')),
+            (b'a\n b\t\r c\n ', ('a', 'b', 'c')),
+            ('a \u20ac', ('a', '\u20ac')),
+            (b'a \xe2\x82\xac', ('a', '\u20ac')),
+            ('a {b c}', ('a', ('b', 'c'))),
+            (r'a b\ c', ('a', ('b', 'c'))),
+            (('a', b'b c'), ('a', ('b', 'c'))),
+            (('a', 'b c'), ('a', ('b', 'c'))),
+            ('a 2', ('a', '2')),
+            (('a', 2), ('a', 2)),
+            ('a 3.4', ('a', '3.4')),
+            (('a', 3.4), ('a', 3.4)),
+            (('a', (2, 3.4)), ('a', (2, 3.4))),
+            ((), ()),
+            (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))),
+        ]
+        for arg, res in testcases:
+            self.assertEqual(split(arg), res, msg=arg)
+
 
 def test_main():
     support.run_unittest(TclTest, TkinterTest)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -44,6 +44,9 @@
 Library
 -------
 
+- Issue #18101: Tcl.split() now process strings nested in a tuple as it
+  do with byte strings.
+
 - Issue #17198: Fix a NameError in the dbm module.  Patch by Valentina
   Mukhamedzhanova.
 
diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c
--- a/Modules/_tkinter.c
+++ b/Modules/_tkinter.c
@@ -519,6 +519,21 @@
             return result;
         /* Fall through, returning arg. */
     }
+    else if (PyUnicode_Check(arg)) {
+        int argc;
+        char **argv;
+        char *list = PyUnicode_AsUTF8(arg);
+
+        if (list == NULL ||
+            Tcl_SplitList((Tcl_Interp *)NULL, list, &argc, &argv) != TCL_OK) {
+            Py_INCREF(arg);
+            return arg;
+        }
+        Tcl_Free(FREECAST argv);
+        if (argc > 1)
+            return Split(list);
+        /* Fall through, returning arg. */
+    }
     else if (PyBytes_Check(arg)) {
         int argc;
         char **argv;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 19:38:41 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 19:38:41 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2318101=3A_Tcl=2Esplit=28=29_now_process_strings_?=
 =?utf-8?q?nested_in_a_tuple_as_it?=
Message-ID: <3brkyT3xzcz7Ljc@mail.python.org>

http://hg.python.org/cpython/rev/16c48d553ddb
changeset:   84553:16c48d553ddb
parent:      84550:e6384b8b2325
parent:      84552:9486c07929a1
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 20:36:00 2013 +0300
summary:
  Issue #18101: Tcl.split() now process strings nested in a tuple as it
do with byte strings.

Added tests for Tcl.split() and Tcl.splitline().

files:
  Lib/test/test_tcl.py |  60 ++++++++++++++++++++++++++++++++
  Misc/NEWS            |   3 +
  Modules/_tkinter.c   |  15 ++++++++
  3 files changed, 78 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py
--- a/Lib/test/test_tcl.py
+++ b/Lib/test/test_tcl.py
@@ -175,6 +175,66 @@
                 self.assertEqual(passValue(f), f)
         self.assertEqual(passValue((1, '2', (3.4,))), (1, '2', (3.4,)))
 
+    def test_splitlist(self):
+        splitlist = self.interp.tk.splitlist
+        call = self.interp.tk.call
+        self.assertRaises(TypeError, splitlist)
+        self.assertRaises(TypeError, splitlist, 'a', 'b')
+        self.assertRaises(TypeError, splitlist, 2)
+        testcases = [
+            ('2', ('2',)),
+            ('', ()),
+            ('{}', ('',)),
+            ('""', ('',)),
+            ('a\n b\t\r c\n ', ('a', 'b', 'c')),
+            (b'a\n b\t\r c\n ', ('a', 'b', 'c')),
+            ('a \u20ac', ('a', '\u20ac')),
+            (b'a \xe2\x82\xac', ('a', '\u20ac')),
+            ('a {b c}', ('a', 'b c')),
+            (r'a b\ c', ('a', 'b c')),
+            (('a', 'b c'), ('a', 'b c')),
+            ('a 2', ('a', '2')),
+            (('a', 2), ('a', 2)),
+            ('a 3.4', ('a', '3.4')),
+            (('a', 3.4), ('a', 3.4)),
+            ((), ()),
+            (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))),
+        ]
+        for arg, res in testcases:
+            self.assertEqual(splitlist(arg), res, msg=arg)
+        self.assertRaises(TclError, splitlist, '{')
+
+    def test_split(self):
+        split = self.interp.tk.split
+        call = self.interp.tk.call
+        self.assertRaises(TypeError, split)
+        self.assertRaises(TypeError, split, 'a', 'b')
+        self.assertRaises(TypeError, split, 2)
+        testcases = [
+            ('2', '2'),
+            ('', ''),
+            ('{}', ''),
+            ('""', ''),
+            ('{', '{'),
+            ('a\n b\t\r c\n ', ('a', 'b', 'c')),
+            (b'a\n b\t\r c\n ', ('a', 'b', 'c')),
+            ('a \u20ac', ('a', '\u20ac')),
+            (b'a \xe2\x82\xac', ('a', '\u20ac')),
+            ('a {b c}', ('a', ('b', 'c'))),
+            (r'a b\ c', ('a', ('b', 'c'))),
+            (('a', b'b c'), ('a', ('b', 'c'))),
+            (('a', 'b c'), ('a', ('b', 'c'))),
+            ('a 2', ('a', '2')),
+            (('a', 2), ('a', 2)),
+            ('a 3.4', ('a', '3.4')),
+            (('a', 3.4), ('a', 3.4)),
+            (('a', (2, 3.4)), ('a', (2, 3.4))),
+            ((), ()),
+            (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))),
+        ]
+        for arg, res in testcases:
+            self.assertEqual(split(arg), res, msg=arg)
+
 
 def test_main():
     support.run_unittest(TclTest, TkinterTest)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -148,6 +148,9 @@
 Library
 -------
 
+- Issue #18101: Tcl.split() now process strings nested in a tuple as it
+  do with byte strings.
+
 - Issue #18116: getpass was always getting an error when testing /dev/tty,
   and thus was always falling back to stdin.  It also leaked an open file
   when it did so.  Both of these issues are now fixed.
diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c
--- a/Modules/_tkinter.c
+++ b/Modules/_tkinter.c
@@ -423,6 +423,21 @@
             return result;
         /* Fall through, returning arg. */
     }
+    else if (PyUnicode_Check(arg)) {
+        int argc;
+        char **argv;
+        char *list = PyUnicode_AsUTF8(arg);
+
+        if (list == NULL ||
+            Tcl_SplitList((Tcl_Interp *)NULL, list, &argc, &argv) != TCL_OK) {
+            Py_INCREF(arg);
+            return arg;
+        }
+        Tcl_Free(FREECAST argv);
+        if (argc > 1)
+            return Split(list);
+        /* Fall through, returning arg. */
+    }
     else if (PyBytes_Check(arg)) {
         int argc;
         char **argv;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 21:01:31 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 21:01:31 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4MDg1?=
 =?utf-8?q?=3A_Add_missed_const_modifier_for_some_entries_in_refcounts=2Ed?=
 =?utf-8?b?YXQu?=
Message-ID: <3brmp33kCzzSDD@mail.python.org>

http://hg.python.org/cpython/rev/ffe24e3e7a2a
changeset:   84554:ffe24e3e7a2a
branch:      3.3
parent:      84552:9486c07929a1
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 21:57:34 2013 +0300
summary:
  Issue #18085: Add missed const modifier for some entries in refcounts.dat.

files:
  Doc/data/refcounts.dat |  154 ++++++++++++++--------------
  1 files changed, 77 insertions(+), 77 deletions(-)


diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat
--- a/Doc/data/refcounts.dat
+++ b/Doc/data/refcounts.dat
@@ -210,7 +210,7 @@
 
 PyDict_DelItemString:int:::
 PyDict_DelItemString:PyObject*:p:0:
-PyDict_DelItemString:char*:key::
+PyDict_DelItemString:const char*:key::
 
 PyDict_GetItem:PyObject*::0:0
 PyDict_GetItem:PyObject*:p:0:
@@ -218,7 +218,7 @@
 
 PyDict_GetItemString:PyObject*::0:
 PyDict_GetItemString:PyObject*:p:0:
-PyDict_GetItemString:char*:key::
+PyDict_GetItemString:const char*:key::
 
 PyDict_Items:PyObject*::+1:
 PyDict_Items:PyObject*:p:0:
@@ -244,7 +244,7 @@
 
 PyDict_SetItemString:int:::
 PyDict_SetItemString:PyObject*:p:0:
-PyDict_SetItemString:char*:key::
+PyDict_SetItemString:const char*:key::
 PyDict_SetItemString:PyObject*:val:+1:
 
 PyDict_Size:int:::
@@ -277,13 +277,13 @@
 PyErr_GivenExceptionMatches:PyObject*:exc:0:
 
 PyErr_NewException:PyObject*::+1:
-PyErr_NewException:char*:name::
+PyErr_NewException:const char*:name::
 PyErr_NewException:PyObject*:base:0:
 PyErr_NewException:PyObject*:dict:0:
 
 PyErr_NewExceptionWithDoc:PyObject*::+1:
-PyErr_NewExceptionWithDoc:char*:name::
-PyErr_NewExceptionWithDoc:char*:doc::
+PyErr_NewExceptionWithDoc:const char*:name::
+PyErr_NewExceptionWithDoc:const char*:doc::
 PyErr_NewExceptionWithDoc:PyObject*:base:0:
 PyErr_NewExceptionWithDoc:PyObject*:dict:0:
 
@@ -310,21 +310,21 @@
 PyErr_SetExcFromWindowsErrWithFilename:PyObject*::null:
 PyErr_SetExcFromWindowsErrWithFilename:PyObject*:type:0:
 PyErr_SetExcFromWindowsErrWithFilename:int:ierr::
-PyErr_SetExcFromWindowsErrWithFilename:char*:filename::
+PyErr_SetExcFromWindowsErrWithFilename:const char*:filename::
 
 PyErr_SetFromErrno:PyObject*::null:
 PyErr_SetFromErrno:PyObject*:type:0:
 
 PyErr_SetFromErrnoWithFilename:PyObject*::null:
 PyErr_SetFromErrnoWithFilename:PyObject*:type:0:
-PyErr_SetFromErrnoWithFilename:char*:filename::
+PyErr_SetFromErrnoWithFilename:const char*:filename::
 
 PyErr_SetFromWindowsErr:PyObject*::null:
 PyErr_SetFromWindowsErr:int:ierr::
 
 PyErr_SetFromWindowsErrWithFilename:PyObject*::null:
 PyErr_SetFromWindowsErrWithFilename:int:ierr::
-PyErr_SetFromWindowsErrWithFilename:char*:filename::
+PyErr_SetFromWindowsErrWithFilename:const char*:filename::
 
 PyErr_SetInterrupt:void:::
 
@@ -337,11 +337,11 @@
 
 PyErr_SetString:void:::
 PyErr_SetString:PyObject*:type:+1:
-PyErr_SetString:char*:message::
+PyErr_SetString:const char*:message::
 
 PyErr_Format:PyObject*::null:
 PyErr_Format:PyObject*:exception:+1:
-PyErr_Format:char*:format::
+PyErr_Format:const char*:format::
 PyErr_Format::...::
 
 PyErr_WarnEx:int:::
@@ -386,22 +386,22 @@
 
 PyFile_FromFile:PyObject*::+1:
 PyFile_FromFile:FILE*:fp::
-PyFile_FromFile:char*:name::
-PyFile_FromFile:char*:mode::
+PyFile_FromFile:const char*:name::
+PyFile_FromFile:const char*:mode::
 PyFile_FromFile:int(*:close)::
 
 PyFile_FromFileEx:PyObject*::+1:
 PyFile_FromFileEx:FILE*:fp::
-PyFile_FromFileEx:char*:name::
-PyFile_FromFileEx:char*:mode::
+PyFile_FromFileEx:const char*:name::
+PyFile_FromFileEx:const char*:mode::
 PyFile_FromFileEx:int(*:close)::
 PyFile_FromFileEx:int:buffering::
-PyFile_FromFileEx:char*:encoding::
-PyFile_FromFileEx:char*:newline::
+PyFile_FromFileEx:const char*:encoding::
+PyFile_FromFileEx:const char*:newline::
 
 PyFile_FromString:PyObject*::+1:
-PyFile_FromString:char*:name::
-PyFile_FromString:char*:mode::
+PyFile_FromString:const char*:name::
+PyFile_FromString:const char*:mode::
 
 PyFile_GetLine:PyObject*::+1:
 PyFile_GetLine:PyObject*:p::
@@ -482,23 +482,23 @@
 PyGen_New:PyFrameObject*:frame:0:
 
 Py_InitModule:PyObject*::0:
-Py_InitModule:char*:name::
+Py_InitModule:const char*:name::
 Py_InitModule:PyMethodDef[]:methods::
 
 Py_InitModule3:PyObject*::0:
-Py_InitModule3:char*:name::
+Py_InitModule3:const char*:name::
 Py_InitModule3:PyMethodDef[]:methods::
-Py_InitModule3:char*:doc::
+Py_InitModule3:const char*:doc::
 
 Py_InitModule4:PyObject*::0:
-Py_InitModule4:char*:name::
+Py_InitModule4:const char*:name::
 Py_InitModule4:PyMethodDef[]:methods::
-Py_InitModule4:char*:doc::
+Py_InitModule4:const char*:doc::
 Py_InitModule4:PyObject*:self::
 Py_InitModule4:int:apiver::usually provided by Py_InitModule or Py_InitModule3
 
 PyImport_AddModule:PyObject*::0:reference borrowed from sys.modules
-PyImport_AddModule:char*:name::
+PyImport_AddModule:const char*:name::
 
 PyImport_Cleanup:void:::
 
@@ -522,16 +522,16 @@
 PyImport_ImportFrozenModule:char*:::
 
 PyImport_ImportModule:PyObject*::+1:
-PyImport_ImportModule:char*:name::
+PyImport_ImportModule:const char*:name::
 
 PyImport_ImportModuleEx:PyObject*::+1:
-PyImport_ImportModuleEx:char*:name::
+PyImport_ImportModuleEx:const char*:name::
 PyImport_ImportModuleEx:PyObject*:globals:0:???
 PyImport_ImportModuleEx:PyObject*:locals:0:???
 PyImport_ImportModuleEx:PyObject*:fromlist:0:???
 
 PyImport_ImportModuleLevel:PyObject*::+1:
-PyImport_ImportModuleLevel:char*:name::
+PyImport_ImportModuleLevel:const char*:name::
 PyImport_ImportModuleLevel:PyObject*:globals:0:???
 PyImport_ImportModuleLevel:PyObject*:locals:0:???
 PyImport_ImportModuleLevel:PyObject*:fromlist:0:???
@@ -692,7 +692,7 @@
 
 PyMapping_DelItemString:int:::
 PyMapping_DelItemString:PyObject*:o:0:
-PyMapping_DelItemString:char*:key::
+PyMapping_DelItemString:const char*:key::
 
 PyMapping_GetItemString:PyObject*::+1:
 PyMapping_GetItemString:PyObject*:o:0:
@@ -762,10 +762,10 @@
 PyModule_GetDict:PyObject*::0:
 PyModule_GetDict::PyObject* module:0:
 
-PyModule_GetFilename:char*:::
+PyModule_GetFilename:const char*:::
 PyModule_GetFilename:PyObject*:module:0:
 
-PyModule_GetName:char*:::
+PyModule_GetName:const char*:::
 PyModule_GetName:PyObject*:module:0:
 
 PyModule_New:PyObject*::+1:
@@ -949,7 +949,7 @@
 
 PyObject_DelAttrString:int:::
 PyObject_DelAttrString:PyObject*:o:0:
-PyObject_DelAttrString:char*:attr_name::
+PyObject_DelAttrString:const char*:attr_name::
 
 PyObject_DelItem:int:::
 PyObject_DelItem:PyObject*:o:0:
@@ -964,7 +964,7 @@
 
 PyObject_GetAttrString:PyObject*::+1:
 PyObject_GetAttrString:PyObject*:o:0:
-PyObject_GetAttrString:char*:attr_name::
+PyObject_GetAttrString:const char*:attr_name::
 
 PyObject_GetItem:PyObject*::+1:
 PyObject_GetItem:PyObject*:o:0:
@@ -979,7 +979,7 @@
 
 PyObject_HasAttrString:int:::
 PyObject_HasAttrString:PyObject*:o:0:
-PyObject_HasAttrString:char*:attr_name:0:
+PyObject_HasAttrString:const char*:attr_name:0:
 
 PyObject_Hash:int:::
 PyObject_Hash:PyObject*:o:0:
@@ -1029,7 +1029,7 @@
 
 PyObject_SetAttrString:int:::
 PyObject_SetAttrString:PyObject*:o:0:
-PyObject_SetAttrString:char*:attr_name::
+PyObject_SetAttrString:const char*:attr_name::
 PyObject_SetAttrString:PyObject*:v:+1:
 
 PyObject_SetItem:int:::
@@ -1048,27 +1048,27 @@
 
 PyParser_SimpleParseFile:struct _node*:::
 PyParser_SimpleParseFile:FILE*:fp::
-PyParser_SimpleParseFile:char*:filename::
+PyParser_SimpleParseFile:const char*:filename::
 PyParser_SimpleParseFile:int:start::
 
 PyParser_SimpleParseString:struct _node*:::
-PyParser_SimpleParseString:char*:str::
+PyParser_SimpleParseString:const char*:str::
 PyParser_SimpleParseString:int:start::
 
 PyRun_AnyFile:int:::
 PyRun_AnyFile:FILE*:fp::
-PyRun_AnyFile:char*:filename::
+PyRun_AnyFile:const char*:filename::
 
 PyRun_File:PyObject*::+1:??? -- same as eval_code2()
 PyRun_File:FILE*:fp::
-PyRun_File:char*:filename::
+PyRun_File:const char*:filename::
 PyRun_File:int:start::
 PyRun_File:PyObject*:globals:0:
 PyRun_File:PyObject*:locals:0:
 
 PyRun_FileEx:PyObject*::+1:??? -- same as eval_code2()
 PyRun_FileEx:FILE*:fp::
-PyRun_FileEx:char*:filename::
+PyRun_FileEx:const char*:filename::
 PyRun_FileEx:int:start::
 PyRun_FileEx:PyObject*:globals:0:
 PyRun_FileEx:PyObject*:locals:0:
@@ -1076,7 +1076,7 @@
 
 PyRun_FileFlags:PyObject*::+1:??? -- same as eval_code2()
 PyRun_FileFlags:FILE*:fp::
-PyRun_FileFlags:char*:filename::
+PyRun_FileFlags:const char*:filename::
 PyRun_FileFlags:int:start::
 PyRun_FileFlags:PyObject*:globals:0:
 PyRun_FileFlags:PyObject*:locals:0:
@@ -1084,7 +1084,7 @@
 
 PyRun_FileExFlags:PyObject*::+1:??? -- same as eval_code2()
 PyRun_FileExFlags:FILE*:fp::
-PyRun_FileExFlags:char*:filename::
+PyRun_FileExFlags:const char*:filename::
 PyRun_FileExFlags:int:start::
 PyRun_FileExFlags:PyObject*:globals:0:
 PyRun_FileExFlags:PyObject*:locals:0:
@@ -1093,27 +1093,27 @@
 
 PyRun_InteractiveLoop:int:::
 PyRun_InteractiveLoop:FILE*:fp::
-PyRun_InteractiveLoop:char*:filename::
+PyRun_InteractiveLoop:const char*:filename::
 
 PyRun_InteractiveOne:int:::
 PyRun_InteractiveOne:FILE*:fp::
-PyRun_InteractiveOne:char*:filename::
+PyRun_InteractiveOne:const char*:filename::
 
 PyRun_SimpleFile:int:::
 PyRun_SimpleFile:FILE*:fp::
-PyRun_SimpleFile:char*:filename::
+PyRun_SimpleFile:const char*:filename::
 
 PyRun_SimpleString:int:::
-PyRun_SimpleString:char*:command::
+PyRun_SimpleString:const char*:command::
 
 PyRun_String:PyObject*::+1:??? -- same as eval_code2()
-PyRun_String:char*:str::
+PyRun_String:const char*:str::
 PyRun_String:int:start::
 PyRun_String:PyObject*:globals:0:
 PyRun_String:PyObject*:locals:0:
 
 PyRun_StringFlags:PyObject*::+1:??? -- same as eval_code2()
-PyRun_StringFlags:char*:str::
+PyRun_StringFlags:const char*:str::
 PyRun_StringFlags:int:start::
 PyRun_StringFlags:PyObject*:globals:0:
 PyRun_StringFlags:PyObject*:locals:0:
@@ -1229,7 +1229,7 @@
 PySlice_New:PyObject*:stop:0:
 PySlice_New:PyObject*:step:0:
 
-PyString_AS_STRING:char*:::
+PyString_AS_STRING:const char*:::
 PyString_AS_STRING:PyObject*:string:0:
 
 PyString_AsDecodedObject:PyObject*::+1:
@@ -1242,7 +1242,7 @@
 PyString_AsEncodedObject:const char*:encoding::
 PyString_AsEncodedObject:const char*:errors::
 
-PyString_AsString:char*:::
+PyString_AsString:const char*:::
 PyString_AsString:PyObject*:string:0:
 
 PyString_AsStringAndSize:int:::
@@ -1310,13 +1310,13 @@
 PyString_AsEncodedString:const char*:errors::
 
 PySys_AddWarnOption:void:::
-PySys_AddWarnOption:char*:s::
+PySys_AddWarnOption:const char*:s::
 
 PySys_AddXOption:void:::
 PySys_AddXOption:const wchar_t*:s::
 
 PySys_GetObject:PyObject*::0:
-PySys_GetObject:char*:name::
+PySys_GetObject:const char*:name::
 
 PySys_GetXOptions:PyObject*::0:
 
@@ -1325,16 +1325,16 @@
 PySys_SetArgv:char**:argv::
 
 PySys_SetObject:int:::
-PySys_SetObject:char*:name::
+PySys_SetObject:const char*:name::
 PySys_SetObject:PyObject*:v:+1:
 
 PySys_ResetWarnOptions:void:::
 
 PySys_WriteStdout:void:::
-PySys_WriteStdout:char*:format::
+PySys_WriteStdout:const char*:format::
 
 PySys_WriteStderr:void:::
-PySys_WriteStderr:char*:format::
+PySys_WriteStderr:const char*:format::
 
 PyThreadState_Clear:void:::
 PyThreadState_Clear:PyThreadState*:tstate::
@@ -1714,16 +1714,16 @@
 Py_AtExit:void (*)():func::
 
 Py_BuildValue:PyObject*::+1:
-Py_BuildValue:char*:format::
+Py_BuildValue:const char*:format::
 
 Py_CompileString:PyObject*::+1:
-Py_CompileString:char*:str::
-Py_CompileString:char*:filename::
+Py_CompileString:const char*:str::
+Py_CompileString:const char*:filename::
 Py_CompileString:int:start::
 
 Py_CompileStringFlags:PyObject*::+1:
-Py_CompileStringFlags:char*:str::
-Py_CompileStringFlags:char*:filename::
+Py_CompileStringFlags:const char*:str::
+Py_CompileStringFlags:const char*:filename::
 Py_CompileStringFlags:int:start::
 Py_CompileStringFlags:PyCompilerFlags*:flags::
 
@@ -1737,33 +1737,33 @@
 Py_Exit:int:status::
 
 Py_FatalError:void:::
-Py_FatalError:char*:message::
+Py_FatalError:const char*:message::
 
 Py_FdIsInteractive:int:::
 Py_FdIsInteractive:FILE*:fp::
-Py_FdIsInteractive:char*:filename::
+Py_FdIsInteractive:const char*:filename::
 
 Py_Finalize:void:::
 
-Py_GetBuildInfoconst:char*:::
+Py_GetBuildInfoconst:const char*:::
 
-Py_GetCompilerconst:char*:::
+Py_GetCompilerconst:const char*:::
 
-Py_GetCopyrightconst:char*:::
+Py_GetCopyrightconst:const char*:::
 
-Py_GetExecPrefix:char*:::
+Py_GetExecPrefix:const char*:::
 
-Py_GetPath:char*:::
+Py_GetPath:const char*:::
 
-Py_GetPlatformconst:char*:::
+Py_GetPlatformconst:const char*:::
 
-Py_GetPrefix:char*:::
+Py_GetPrefix:const char*:::
 
-Py_GetProgramFullPath:char*:::
+Py_GetProgramFullPath:const char*:::
 
-Py_GetProgramName:char*:::
+Py_GetProgramName:const char*:::
 
-Py_GetVersionconst:char*:::
+Py_GetVersionconst:const char*:::
 
 Py_INCREF:void:::
 Py_INCREF:PyObject*:o:+1:
@@ -1775,7 +1775,7 @@
 Py_NewInterpreter:PyThreadState*:::
 
 Py_SetProgramName:void:::
-Py_SetProgramName:char*:name::
+Py_SetProgramName:const char*:name::
 
 Py_XDECREF:void:::
 Py_XDECREF:PyObject*:o:-1:if o is not NULL
@@ -1784,14 +1784,14 @@
 Py_XINCREF:PyObject*:o:+1:if o is not NULL
 
 _PyImport_FindExtension:PyObject*::0:??? see PyImport_AddModule
-_PyImport_FindExtension:char*:::
-_PyImport_FindExtension:char*:::
+_PyImport_FindExtension:const char*:::
+_PyImport_FindExtension:const char*:::
 
 _PyImport_Fini:void:::
 
 _PyImport_FixupExtension:PyObject*:::???
-_PyImport_FixupExtension:char*:::
-_PyImport_FixupExtension:char*:::
+_PyImport_FixupExtension:const char*:::
+_PyImport_FixupExtension:const char*:::
 
 _PyImport_Init:void:::
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 21:01:33 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 21:01:33 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2318085=3A_Add_missed_const_modifier_for_some_ent?=
 =?utf-8?q?ries_in_refcounts=2Edat=2E?=
Message-ID: <3brmp503ZFzSxC@mail.python.org>

http://hg.python.org/cpython/rev/6587fd3d89ae
changeset:   84555:6587fd3d89ae
parent:      84553:16c48d553ddb
parent:      84554:ffe24e3e7a2a
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 22:00:57 2013 +0300
summary:
  Issue #18085: Add missed const modifier for some entries in refcounts.dat.

files:
  Doc/data/refcounts.dat |  152 ++++++++++++++--------------
  1 files changed, 76 insertions(+), 76 deletions(-)


diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat
--- a/Doc/data/refcounts.dat
+++ b/Doc/data/refcounts.dat
@@ -210,7 +210,7 @@
 
 PyDict_DelItemString:int:::
 PyDict_DelItemString:PyObject*:p:0:
-PyDict_DelItemString:char*:key::
+PyDict_DelItemString:const char*:key::
 
 PyDict_GetItem:PyObject*::0:0
 PyDict_GetItem:PyObject*:p:0:
@@ -249,7 +249,7 @@
 
 PyDict_SetItemString:int:::
 PyDict_SetItemString:PyObject*:p:0:
-PyDict_SetItemString:char*:key::
+PyDict_SetItemString:const char*:key::
 PyDict_SetItemString:PyObject*:val:+1:
 
 PyDict_Size:int:::
@@ -282,13 +282,13 @@
 PyErr_GivenExceptionMatches:PyObject*:exc:0:
 
 PyErr_NewException:PyObject*::+1:
-PyErr_NewException:char*:name::
+PyErr_NewException:const char*:name::
 PyErr_NewException:PyObject*:base:0:
 PyErr_NewException:PyObject*:dict:0:
 
 PyErr_NewExceptionWithDoc:PyObject*::+1:
-PyErr_NewExceptionWithDoc:char*:name::
-PyErr_NewExceptionWithDoc:char*:doc::
+PyErr_NewExceptionWithDoc:const char*:name::
+PyErr_NewExceptionWithDoc:const char*:doc::
 PyErr_NewExceptionWithDoc:PyObject*:base:0:
 PyErr_NewExceptionWithDoc:PyObject*:dict:0:
 
@@ -315,21 +315,21 @@
 PyErr_SetExcFromWindowsErrWithFilename:PyObject*::null:
 PyErr_SetExcFromWindowsErrWithFilename:PyObject*:type:0:
 PyErr_SetExcFromWindowsErrWithFilename:int:ierr::
-PyErr_SetExcFromWindowsErrWithFilename:char*:filename::
+PyErr_SetExcFromWindowsErrWithFilename:const char*:filename::
 
 PyErr_SetFromErrno:PyObject*::null:
 PyErr_SetFromErrno:PyObject*:type:0:
 
 PyErr_SetFromErrnoWithFilename:PyObject*::null:
 PyErr_SetFromErrnoWithFilename:PyObject*:type:0:
-PyErr_SetFromErrnoWithFilename:char*:filename::
+PyErr_SetFromErrnoWithFilename:const char*:filename::
 
 PyErr_SetFromWindowsErr:PyObject*::null:
 PyErr_SetFromWindowsErr:int:ierr::
 
 PyErr_SetFromWindowsErrWithFilename:PyObject*::null:
 PyErr_SetFromWindowsErrWithFilename:int:ierr::
-PyErr_SetFromWindowsErrWithFilename:char*:filename::
+PyErr_SetFromWindowsErrWithFilename:const char*:filename::
 
 PyErr_SetInterrupt:void:::
 
@@ -342,11 +342,11 @@
 
 PyErr_SetString:void:::
 PyErr_SetString:PyObject*:type:+1:
-PyErr_SetString:char*:message::
+PyErr_SetString:const char*:message::
 
 PyErr_Format:PyObject*::null:
 PyErr_Format:PyObject*:exception:+1:
-PyErr_Format:char*:format::
+PyErr_Format:const char*:format::
 PyErr_Format::...::
 
 PyErr_WarnEx:int:::
@@ -391,22 +391,22 @@
 
 PyFile_FromFile:PyObject*::+1:
 PyFile_FromFile:FILE*:fp::
-PyFile_FromFile:char*:name::
-PyFile_FromFile:char*:mode::
+PyFile_FromFile:const char*:name::
+PyFile_FromFile:const char*:mode::
 PyFile_FromFile:int(*:close)::
 
 PyFile_FromFileEx:PyObject*::+1:
 PyFile_FromFileEx:FILE*:fp::
-PyFile_FromFileEx:char*:name::
-PyFile_FromFileEx:char*:mode::
+PyFile_FromFileEx:const char*:name::
+PyFile_FromFileEx:const char*:mode::
 PyFile_FromFileEx:int(*:close)::
 PyFile_FromFileEx:int:buffering::
-PyFile_FromFileEx:char*:encoding::
-PyFile_FromFileEx:char*:newline::
+PyFile_FromFileEx:const char*:encoding::
+PyFile_FromFileEx:const char*:newline::
 
 PyFile_FromString:PyObject*::+1:
-PyFile_FromString:char*:name::
-PyFile_FromString:char*:mode::
+PyFile_FromString:const char*:name::
+PyFile_FromString:const char*:mode::
 
 PyFile_GetLine:PyObject*::+1:
 PyFile_GetLine:PyObject*:p::
@@ -487,23 +487,23 @@
 PyGen_New:PyFrameObject*:frame:0:
 
 Py_InitModule:PyObject*::0:
-Py_InitModule:char*:name::
+Py_InitModule:const char*:name::
 Py_InitModule:PyMethodDef[]:methods::
 
 Py_InitModule3:PyObject*::0:
-Py_InitModule3:char*:name::
+Py_InitModule3:const char*:name::
 Py_InitModule3:PyMethodDef[]:methods::
-Py_InitModule3:char*:doc::
+Py_InitModule3:const char*:doc::
 
 Py_InitModule4:PyObject*::0:
-Py_InitModule4:char*:name::
+Py_InitModule4:const char*:name::
 Py_InitModule4:PyMethodDef[]:methods::
-Py_InitModule4:char*:doc::
+Py_InitModule4:const char*:doc::
 Py_InitModule4:PyObject*:self::
 Py_InitModule4:int:apiver::usually provided by Py_InitModule or Py_InitModule3
 
 PyImport_AddModule:PyObject*::0:reference borrowed from sys.modules
-PyImport_AddModule:char*:name::
+PyImport_AddModule:const char*:name::
 
 PyImport_Cleanup:void:::
 
@@ -527,16 +527,16 @@
 PyImport_ImportFrozenModule:char*:::
 
 PyImport_ImportModule:PyObject*::+1:
-PyImport_ImportModule:char*:name::
+PyImport_ImportModule:const char*:name::
 
 PyImport_ImportModuleEx:PyObject*::+1:
-PyImport_ImportModuleEx:char*:name::
+PyImport_ImportModuleEx:const char*:name::
 PyImport_ImportModuleEx:PyObject*:globals:0:???
 PyImport_ImportModuleEx:PyObject*:locals:0:???
 PyImport_ImportModuleEx:PyObject*:fromlist:0:???
 
 PyImport_ImportModuleLevel:PyObject*::+1:
-PyImport_ImportModuleLevel:char*:name::
+PyImport_ImportModuleLevel:const char*:name::
 PyImport_ImportModuleLevel:PyObject*:globals:0:???
 PyImport_ImportModuleLevel:PyObject*:locals:0:???
 PyImport_ImportModuleLevel:PyObject*:fromlist:0:???
@@ -697,7 +697,7 @@
 
 PyMapping_DelItemString:int:::
 PyMapping_DelItemString:PyObject*:o:0:
-PyMapping_DelItemString:char*:key::
+PyMapping_DelItemString:const char*:key::
 
 PyMapping_GetItemString:PyObject*::+1:
 PyMapping_GetItemString:PyObject*:o:0:
@@ -767,10 +767,10 @@
 PyModule_GetDict:PyObject*::0:
 PyModule_GetDict::PyObject* module:0:
 
-PyModule_GetFilename:char*:::
+PyModule_GetFilename:const char*:::
 PyModule_GetFilename:PyObject*:module:0:
 
-PyModule_GetName:char*:::
+PyModule_GetName:const char*:::
 PyModule_GetName:PyObject*:module:0:
 
 PyModule_New:PyObject*::+1:
@@ -954,7 +954,7 @@
 
 PyObject_DelAttrString:int:::
 PyObject_DelAttrString:PyObject*:o:0:
-PyObject_DelAttrString:char*:attr_name::
+PyObject_DelAttrString:const char*:attr_name::
 
 PyObject_DelItem:int:::
 PyObject_DelItem:PyObject*:o:0:
@@ -969,7 +969,7 @@
 
 PyObject_GetAttrString:PyObject*::+1:
 PyObject_GetAttrString:PyObject*:o:0:
-PyObject_GetAttrString:char*:attr_name::
+PyObject_GetAttrString:const char*:attr_name::
 
 PyObject_GetItem:PyObject*::+1:
 PyObject_GetItem:PyObject*:o:0:
@@ -984,7 +984,7 @@
 
 PyObject_HasAttrString:int:::
 PyObject_HasAttrString:PyObject*:o:0:
-PyObject_HasAttrString:char*:attr_name:0:
+PyObject_HasAttrString:const char*:attr_name:0:
 
 PyObject_Hash:int:::
 PyObject_Hash:PyObject*:o:0:
@@ -1034,7 +1034,7 @@
 
 PyObject_SetAttrString:int:::
 PyObject_SetAttrString:PyObject*:o:0:
-PyObject_SetAttrString:char*:attr_name::
+PyObject_SetAttrString:const char*:attr_name::
 PyObject_SetAttrString:PyObject*:v:+1:
 
 PyObject_SetItem:int:::
@@ -1053,27 +1053,27 @@
 
 PyParser_SimpleParseFile:struct _node*:::
 PyParser_SimpleParseFile:FILE*:fp::
-PyParser_SimpleParseFile:char*:filename::
+PyParser_SimpleParseFile:const char*:filename::
 PyParser_SimpleParseFile:int:start::
 
 PyParser_SimpleParseString:struct _node*:::
-PyParser_SimpleParseString:char*:str::
+PyParser_SimpleParseString:const char*:str::
 PyParser_SimpleParseString:int:start::
 
 PyRun_AnyFile:int:::
 PyRun_AnyFile:FILE*:fp::
-PyRun_AnyFile:char*:filename::
+PyRun_AnyFile:const char*:filename::
 
 PyRun_File:PyObject*::+1:??? -- same as eval_code2()
 PyRun_File:FILE*:fp::
-PyRun_File:char*:filename::
+PyRun_File:const char*:filename::
 PyRun_File:int:start::
 PyRun_File:PyObject*:globals:0:
 PyRun_File:PyObject*:locals:0:
 
 PyRun_FileEx:PyObject*::+1:??? -- same as eval_code2()
 PyRun_FileEx:FILE*:fp::
-PyRun_FileEx:char*:filename::
+PyRun_FileEx:const char*:filename::
 PyRun_FileEx:int:start::
 PyRun_FileEx:PyObject*:globals:0:
 PyRun_FileEx:PyObject*:locals:0:
@@ -1081,7 +1081,7 @@
 
 PyRun_FileFlags:PyObject*::+1:??? -- same as eval_code2()
 PyRun_FileFlags:FILE*:fp::
-PyRun_FileFlags:char*:filename::
+PyRun_FileFlags:const char*:filename::
 PyRun_FileFlags:int:start::
 PyRun_FileFlags:PyObject*:globals:0:
 PyRun_FileFlags:PyObject*:locals:0:
@@ -1089,7 +1089,7 @@
 
 PyRun_FileExFlags:PyObject*::+1:??? -- same as eval_code2()
 PyRun_FileExFlags:FILE*:fp::
-PyRun_FileExFlags:char*:filename::
+PyRun_FileExFlags:const char*:filename::
 PyRun_FileExFlags:int:start::
 PyRun_FileExFlags:PyObject*:globals:0:
 PyRun_FileExFlags:PyObject*:locals:0:
@@ -1098,27 +1098,27 @@
 
 PyRun_InteractiveLoop:int:::
 PyRun_InteractiveLoop:FILE*:fp::
-PyRun_InteractiveLoop:char*:filename::
+PyRun_InteractiveLoop:const char*:filename::
 
 PyRun_InteractiveOne:int:::
 PyRun_InteractiveOne:FILE*:fp::
-PyRun_InteractiveOne:char*:filename::
+PyRun_InteractiveOne:const char*:filename::
 
 PyRun_SimpleFile:int:::
 PyRun_SimpleFile:FILE*:fp::
-PyRun_SimpleFile:char*:filename::
+PyRun_SimpleFile:const char*:filename::
 
 PyRun_SimpleString:int:::
-PyRun_SimpleString:char*:command::
+PyRun_SimpleString:const char*:command::
 
 PyRun_String:PyObject*::+1:??? -- same as eval_code2()
-PyRun_String:char*:str::
+PyRun_String:const char*:str::
 PyRun_String:int:start::
 PyRun_String:PyObject*:globals:0:
 PyRun_String:PyObject*:locals:0:
 
 PyRun_StringFlags:PyObject*::+1:??? -- same as eval_code2()
-PyRun_StringFlags:char*:str::
+PyRun_StringFlags:const char*:str::
 PyRun_StringFlags:int:start::
 PyRun_StringFlags:PyObject*:globals:0:
 PyRun_StringFlags:PyObject*:locals:0:
@@ -1234,7 +1234,7 @@
 PySlice_New:PyObject*:stop:0:
 PySlice_New:PyObject*:step:0:
 
-PyString_AS_STRING:char*:::
+PyString_AS_STRING:const char*:::
 PyString_AS_STRING:PyObject*:string:0:
 
 PyString_AsDecodedObject:PyObject*::+1:
@@ -1247,7 +1247,7 @@
 PyString_AsEncodedObject:const char*:encoding::
 PyString_AsEncodedObject:const char*:errors::
 
-PyString_AsString:char*:::
+PyString_AsString:const char*:::
 PyString_AsString:PyObject*:string:0:
 
 PyString_AsStringAndSize:int:::
@@ -1315,13 +1315,13 @@
 PyString_AsEncodedString:const char*:errors::
 
 PySys_AddWarnOption:void:::
-PySys_AddWarnOption:char*:s::
+PySys_AddWarnOption:const char*:s::
 
 PySys_AddXOption:void:::
 PySys_AddXOption:const wchar_t*:s::
 
 PySys_GetObject:PyObject*::0:
-PySys_GetObject:char*:name::
+PySys_GetObject:const char*:name::
 
 PySys_GetXOptions:PyObject*::0:
 
@@ -1330,16 +1330,16 @@
 PySys_SetArgv:char**:argv::
 
 PySys_SetObject:int:::
-PySys_SetObject:char*:name::
+PySys_SetObject:const char*:name::
 PySys_SetObject:PyObject*:v:+1:
 
 PySys_ResetWarnOptions:void:::
 
 PySys_WriteStdout:void:::
-PySys_WriteStdout:char*:format::
+PySys_WriteStdout:const char*:format::
 
 PySys_WriteStderr:void:::
-PySys_WriteStderr:char*:format::
+PySys_WriteStderr:const char*:format::
 
 PyThreadState_Clear:void:::
 PyThreadState_Clear:PyThreadState*:tstate::
@@ -1719,16 +1719,16 @@
 Py_AtExit:void (*)():func::
 
 Py_BuildValue:PyObject*::+1:
-Py_BuildValue:char*:format::
+Py_BuildValue:const char*:format::
 
 Py_CompileString:PyObject*::+1:
-Py_CompileString:char*:str::
-Py_CompileString:char*:filename::
+Py_CompileString:const char*:str::
+Py_CompileString:const char*:filename::
 Py_CompileString:int:start::
 
 Py_CompileStringFlags:PyObject*::+1:
-Py_CompileStringFlags:char*:str::
-Py_CompileStringFlags:char*:filename::
+Py_CompileStringFlags:const char*:str::
+Py_CompileStringFlags:const char*:filename::
 Py_CompileStringFlags:int:start::
 Py_CompileStringFlags:PyCompilerFlags*:flags::
 
@@ -1742,33 +1742,33 @@
 Py_Exit:int:status::
 
 Py_FatalError:void:::
-Py_FatalError:char*:message::
+Py_FatalError:const char*:message::
 
 Py_FdIsInteractive:int:::
 Py_FdIsInteractive:FILE*:fp::
-Py_FdIsInteractive:char*:filename::
+Py_FdIsInteractive:const char*:filename::
 
 Py_Finalize:void:::
 
-Py_GetBuildInfoconst:char*:::
+Py_GetBuildInfoconst:const char*:::
 
-Py_GetCompilerconst:char*:::
+Py_GetCompilerconst:const char*:::
 
-Py_GetCopyrightconst:char*:::
+Py_GetCopyrightconst:const char*:::
 
-Py_GetExecPrefix:char*:::
+Py_GetExecPrefix:const char*:::
 
-Py_GetPath:char*:::
+Py_GetPath:const char*:::
 
-Py_GetPlatformconst:char*:::
+Py_GetPlatformconst:const char*:::
 
-Py_GetPrefix:char*:::
+Py_GetPrefix:const char*:::
 
-Py_GetProgramFullPath:char*:::
+Py_GetProgramFullPath:const char*:::
 
-Py_GetProgramName:char*:::
+Py_GetProgramName:const char*:::
 
-Py_GetVersionconst:char*:::
+Py_GetVersionconst:const char*:::
 
 Py_INCREF:void:::
 Py_INCREF:PyObject*:o:+1:
@@ -1780,7 +1780,7 @@
 Py_NewInterpreter:PyThreadState*:::
 
 Py_SetProgramName:void:::
-Py_SetProgramName:char*:name::
+Py_SetProgramName:const char*:name::
 
 Py_XDECREF:void:::
 Py_XDECREF:PyObject*:o:-1:if o is not NULL
@@ -1789,14 +1789,14 @@
 Py_XINCREF:PyObject*:o:+1:if o is not NULL
 
 _PyImport_FindExtension:PyObject*::0:??? see PyImport_AddModule
-_PyImport_FindExtension:char*:::
-_PyImport_FindExtension:char*:::
+_PyImport_FindExtension:const char*:::
+_PyImport_FindExtension:const char*:::
 
 _PyImport_Fini:void:::
 
 _PyImport_FixupExtension:PyObject*:::???
-_PyImport_FixupExtension:char*:::
-_PyImport_FixupExtension:char*:::
+_PyImport_FixupExtension:const char*:::
+_PyImport_FixupExtension:const char*:::
 
 _PyImport_Init:void:::
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 21:31:14 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 21:31:14 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE3ODcy?=
 =?utf-8?q?=3A_Fix_a_segfault_in_marshal=2Eload=28=29_when_input_stream_re?=
 =?utf-8?q?turns?=
Message-ID: <3brnSL6HHPz7LjP@mail.python.org>

http://hg.python.org/cpython/rev/fc7bab8a8618
changeset:   84556:fc7bab8a8618
branch:      3.3
parent:      84554:ffe24e3e7a2a
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 22:20:47 2013 +0300
summary:
  Issue #17872: Fix a segfault in marshal.load() when input stream returns
more bytes than requested.

files:
  Lib/test/test_marshal.py |  12 +++++++++
  Misc/NEWS                |   3 ++
  Python/marshal.c         |  35 ++++++++++++++++------------
  3 files changed, 35 insertions(+), 15 deletions(-)


diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -2,6 +2,7 @@
 
 from test import support
 import array
+import io
 import marshal
 import sys
 import unittest
@@ -279,6 +280,17 @@
         unicode_string = 'T'
         self.assertRaises(TypeError, marshal.loads, unicode_string)
 
+    def test_bad_reader(self):
+        class BadReader(io.BytesIO):
+            def read(self, n=-1):
+                b = super().read(n)
+                if n is not None and n > 4:
+                    b += b' ' * 10**6
+                return b
+        for value in (1.0, 1j, b'0123456789', '0123456789'):
+            self.assertRaises(ValueError, marshal.load,
+                              BadReader(marshal.dumps(value)))
+
 LARGE_SIZE = 2**31
 pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
 
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,9 @@
 Core and Builtins
 -----------------
 
+- Issue #17872: Fix a segfault in marshal.load() when input stream returns
+  more bytes than requested.
+
 - Issue #18426: Fix NULL pointer dereference in C extension import when
   PyModule_GetDef() returns an error.
 
diff --git a/Python/marshal.c b/Python/marshal.c
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -490,8 +490,17 @@
             else {
                 read = PyBytes_GET_SIZE(data);
                 if (read > 0) {
-                    ptr = PyBytes_AS_STRING(data);
-                    memcpy(s, ptr, read);
+                    if (read > n) {
+                        PyErr_Format(PyExc_ValueError,
+                                    "read() returned too much data: "
+                                    "%zd bytes requested, %zd returned",
+                                    n, read);
+                        read = -1;
+                    }
+                    else {
+                        ptr = PyBytes_AS_STRING(data);
+                        memcpy(s, ptr, read);
+                    }
                 }
             }
             Py_DECREF(data);
@@ -733,11 +742,13 @@
             double dx;
             retval = NULL;
             n = r_byte(p);
-            if (n == EOF || r_string(buf, n, p) != n) {
+            if (n == EOF) {
                 PyErr_SetString(PyExc_EOFError,
                     "EOF read where object expected");
                 break;
             }
+            if (r_string(buf, n, p) != n)
+                break;
             buf[n] = '\0';
             dx = PyOS_string_to_double(buf, NULL, NULL);
             if (dx == -1.0 && PyErr_Occurred())
@@ -751,8 +762,6 @@
             unsigned char buf[8];
             double x;
             if (r_string((char*)buf, 8, p) != 8) {
-                PyErr_SetString(PyExc_EOFError,
-                    "EOF read where object expected");
                 retval = NULL;
                 break;
             }
@@ -771,21 +780,25 @@
             Py_complex c;
             retval = NULL;
             n = r_byte(p);
-            if (n == EOF || r_string(buf, n, p) != n) {
+            if (n == EOF) {
                 PyErr_SetString(PyExc_EOFError,
                     "EOF read where object expected");
                 break;
             }
+            if (r_string(buf, n, p) != n)
+                break;
             buf[n] = '\0';
             c.real = PyOS_string_to_double(buf, NULL, NULL);
             if (c.real == -1.0 && PyErr_Occurred())
                 break;
             n = r_byte(p);
-            if (n == EOF || r_string(buf, n, p) != n) {
+            if (n == EOF) {
                 PyErr_SetString(PyExc_EOFError,
                     "EOF read where object expected");
                 break;
             }
+            if (r_string(buf, n, p) != n)
+                break;
             buf[n] = '\0';
             c.imag = PyOS_string_to_double(buf, NULL, NULL);
             if (c.imag == -1.0 && PyErr_Occurred())
@@ -799,8 +812,6 @@
             unsigned char buf[8];
             Py_complex c;
             if (r_string((char*)buf, 8, p) != 8) {
-                PyErr_SetString(PyExc_EOFError,
-                    "EOF read where object expected");
                 retval = NULL;
                 break;
             }
@@ -810,8 +821,6 @@
                 break;
             }
             if (r_string((char*)buf, 8, p) != 8) {
-                PyErr_SetString(PyExc_EOFError,
-                    "EOF read where object expected");
                 retval = NULL;
                 break;
             }
@@ -842,8 +851,6 @@
         }
         if (r_string(PyBytes_AS_STRING(v), n, p) != n) {
             Py_DECREF(v);
-            PyErr_SetString(PyExc_EOFError,
-                            "EOF read where object expected");
             retval = NULL;
             break;
         }
@@ -871,8 +878,6 @@
         }
         if (r_string(buffer, n, p) != n) {
             PyMem_DEL(buffer);
-            PyErr_SetString(PyExc_EOFError,
-                "EOF read where object expected");
             retval = NULL;
             break;
         }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 21:31:16 2013
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 11 Jul 2013 21:31:16 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2317872=3A_Fix_a_segfault_in_marshal=2Eload=28=29?=
 =?utf-8?q?_when_input_stream_returns?=
Message-ID: <3brnSN2mQ3z7LjT@mail.python.org>

http://hg.python.org/cpython/rev/5fa793ae36cc
changeset:   84557:5fa793ae36cc
parent:      84555:6587fd3d89ae
parent:      84556:fc7bab8a8618
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Jul 11 22:28:18 2013 +0300
summary:
  Issue #17872: Fix a segfault in marshal.load() when input stream returns
more bytes than requested.

files:
  Lib/test/test_marshal.py |  12 +++++++++
  Misc/NEWS                |   3 ++
  Python/marshal.c         |  35 ++++++++++++++++------------
  3 files changed, 35 insertions(+), 15 deletions(-)


diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -2,6 +2,7 @@
 
 from test import support
 import array
+import io
 import marshal
 import sys
 import unittest
@@ -259,6 +260,17 @@
         unicode_string = 'T'
         self.assertRaises(TypeError, marshal.loads, unicode_string)
 
+    def test_bad_reader(self):
+        class BadReader(io.BytesIO):
+            def read(self, n=-1):
+                b = super().read(n)
+                if n is not None and n > 4:
+                    b += b' ' * 10**6
+                return b
+        for value in (1.0, 1j, b'0123456789', '0123456789'):
+            self.assertRaises(ValueError, marshal.load,
+                              BadReader(marshal.dumps(value)))
+
     def _test_eof(self):
         data = marshal.dumps(("hello", "dolly", None))
         for i in range(len(data)):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #17872: Fix a segfault in marshal.load() when input stream returns
+  more bytes than requested.
+
 - Issue #18338: `python --version` now prints version string to stdout, and
   not to stderr.  Patch by Berker Peksag and Michael Dickens.
 
diff --git a/Python/marshal.c b/Python/marshal.c
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -570,8 +570,17 @@
             else {
                 read = (int)PyBytes_GET_SIZE(data);
                 if (read > 0) {
-                    ptr = PyBytes_AS_STRING(data);
-                    memcpy(s, ptr, read);
+                    if (read > n) {
+                        PyErr_Format(PyExc_ValueError,
+                                    "read() returned too much data: "
+                                    "%zd bytes requested, %zd returned",
+                                    n, read);
+                        read = -1;
+                    }
+                    else {
+                        ptr = PyBytes_AS_STRING(data);
+                        memcpy(s, ptr, read);
+                    }
                 }
             }
             Py_DECREF(data);
@@ -841,11 +850,13 @@
             double dx;
             retval = NULL;
             n = r_byte(p);
-            if (n == EOF || r_string(buf, n, p) != n) {
+            if (n == EOF) {
                 PyErr_SetString(PyExc_EOFError,
                     "EOF read where object expected");
                 break;
             }
+            if (r_string(buf, n, p) != n)
+                break;
             buf[n] = '\0';
             dx = PyOS_string_to_double(buf, NULL, NULL);
             if (dx == -1.0 && PyErr_Occurred())
@@ -860,8 +871,6 @@
             unsigned char buf[8];
             double x;
             if (r_string((char*)buf, 8, p) != 8) {
-                PyErr_SetString(PyExc_EOFError,
-                    "EOF read where object expected");
                 retval = NULL;
                 break;
             }
@@ -881,21 +890,25 @@
             Py_complex c;
             retval = NULL;
             n = r_byte(p);
-            if (n == EOF || r_string(buf, n, p) != n) {
+            if (n == EOF) {
                 PyErr_SetString(PyExc_EOFError,
                     "EOF read where object expected");
                 break;
             }
+            if (r_string(buf, n, p) != n)
+                break;
             buf[n] = '\0';
             c.real = PyOS_string_to_double(buf, NULL, NULL);
             if (c.real == -1.0 && PyErr_Occurred())
                 break;
             n = r_byte(p);
-            if (n == EOF || r_string(buf, n, p) != n) {
+            if (n == EOF) {
                 PyErr_SetString(PyExc_EOFError,
                     "EOF read where object expected");
                 break;
             }
+            if (r_string(buf, n, p) != n)
+                break;
             buf[n] = '\0';
             c.imag = PyOS_string_to_double(buf, NULL, NULL);
             if (c.imag == -1.0 && PyErr_Occurred())
@@ -910,8 +923,6 @@
             unsigned char buf[8];
             Py_complex c;
             if (r_string((char*)buf, 8, p) != 8) {
-                PyErr_SetString(PyExc_EOFError,
-                    "EOF read where object expected");
                 retval = NULL;
                 break;
             }
@@ -921,8 +932,6 @@
                 break;
             }
             if (r_string((char*)buf, 8, p) != 8) {
-                PyErr_SetString(PyExc_EOFError,
-                    "EOF read where object expected");
                 retval = NULL;
                 break;
             }
@@ -954,8 +963,6 @@
         }
         if (r_string(PyBytes_AS_STRING(v), n, p) != n) {
             Py_DECREF(v);
-            PyErr_SetString(PyExc_EOFError,
-                            "EOF read where object expected");
             retval = NULL;
             break;
         }
@@ -986,8 +993,6 @@
             }
             if (r_string(buffer, n, p) != n) {
                 PyMem_DEL(buffer);
-                PyErr_SetString(PyExc_EOFError,
-                    "EOF read where object expected");
                 retval = NULL;
                 break;
             }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 21:59:05 2013
From: python-checkins at python.org (r.david.murray)
Date: Thu, 11 Jul 2013 21:59:05 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogIzE4MDQ0OiBGaXgg?=
 =?utf-8?q?parsing_of_encoded_words_of_the_form_=3D=3Futf8=3Fq=3F=3DXX=2E?=
 =?utf-8?b?Li4/PQ==?=
Message-ID: <3brp4T0rH1z7Ljn@mail.python.org>

http://hg.python.org/cpython/rev/4acb822f4c43
changeset:   84558:4acb822f4c43
branch:      3.3
parent:      84556:fc7bab8a8618
user:        R David Murray <rdmurray at bitdance.com>
date:        Thu Jul 11 15:52:57 2013 -0400
summary:
  #18044: Fix parsing of encoded words of the form =?utf8?q?=XX...?=

The problem was I was only checking for decimal digits after the third '?',
not for *hex* digits :(.

This changeset also fixes a couple of comment typos, deletes an unused
function relating to encoded word parsing, and removed an invalid
'if' test from the folding function that was revealed by the tests
written to validate this issue.

files:
  Lib/email/_header_value_parser.py                |  43 +--------
  Lib/test/test_email/test__encoded_words.py       |   5 +
  Lib/test/test_email/test__header_value_parser.py |   9 ++
  Lib/test/test_email/test_headerregistry.py       |  41 ++++++++-
  Misc/NEWS                                        |   4 +
  5 files changed, 62 insertions(+), 40 deletions(-)


diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -69,6 +69,7 @@
 
 import re
 import urllib   # For urllib.parse.unquote
+from string import hexdigits
 from collections import namedtuple, OrderedDict
 from email import _encoded_words as _ew
 from email import errors
@@ -392,10 +393,6 @@
     token_type = 'unstructured'
 
     def _fold(self, folded):
-        if any(x.token_type=='encoded-word' for x in self):
-            return self._fold_encoded(folded)
-        # Here we can have either a pure ASCII string that may or may not
-        # have surrogateescape encoded bytes, or a unicode string.
         last_ew = None
         for part in self.parts:
             tstr = str(part)
@@ -1389,35 +1386,6 @@
         pos = pos + 1
     return ''.join(vchars), ''.join([fragment[pos:]] + remainder), had_qp
 
-def _decode_ew_run(value):
-    """ Decode a run of RFC2047 encoded words.
-
-        _decode_ew_run(value) -> (text, value, defects)
-
-    Scans the supplied value for a run of tokens that look like they are RFC
-    2047 encoded words, decodes those words into text according to RFC 2047
-    rules (whitespace between encoded words is discarded), and returns the text
-    and the remaining value (including any leading whitespace on the remaining
-    value), as well as a list of any defects encountered while decoding.  The
-    input value may not have any leading whitespace.
-
-    """
-    res = []
-    defects = []
-    last_ws = ''
-    while value:
-        try:
-            tok, ws, value = _wsp_splitter(value, 1)
-        except ValueError:
-            tok, ws, value = value, '', ''
-        if not (tok.startswith('=?') and tok.endswith('?=')):
-            return ''.join(res), last_ws + tok + ws + value, defects
-        text, charset, lang, new_defects = _ew.decode(tok)
-        res.append(text)
-        defects.extend(new_defects)
-        last_ws = ws
-    return ''.join(res), last_ws, defects
-
 def get_fws(value):
     """FWS = 1*WSP
 
@@ -1443,7 +1411,8 @@
         raise errors.HeaderParseError(
             "expected encoded word but found {}".format(value))
     remstr = ''.join(remainder)
-    if remstr[:2].isdigit():
+    if len(remstr) > 1 and remstr[0] in hexdigits and remstr[1] in hexdigits:
+        # The ? after the CTE was followed by an encoded word escape (=XX).
         rest, *remainder = remstr.split('?=', 1)
         tok = tok + '?=' + rest
     if len(tok.split()) > 1:
@@ -1491,8 +1460,8 @@
 
     """
     # XXX: but what about bare CR and LF?  They might signal the start or
-    # end of an encoded word.  YAGNI for now, since out current parsers
-    # will never send us strings with bard CR or LF.
+    # end of an encoded word.  YAGNI for now, since our current parsers
+    # will never send us strings with bare CR or LF.
 
     unstructured = UnstructuredTokenList()
     while value:
@@ -1504,6 +1473,8 @@
             try:
                 token, value = get_encoded_word(value)
             except errors.HeaderParseError:
+                # XXX: Need to figure out how to register defects when
+                # appropriate here.
                 pass
             else:
                 have_ws = True
diff --git a/Lib/test/test_email/test__encoded_words.py b/Lib/test/test_email/test__encoded_words.py
--- a/Lib/test/test_email/test__encoded_words.py
+++ b/Lib/test/test_email/test__encoded_words.py
@@ -122,6 +122,11 @@
                    # XXX Should this be a new Defect instead?
                    defects = [errors.CharsetError])
 
+    def test_q_nonascii(self):
+        self._test('=?utf-8?q?=C3=89ric?=',
+                   '?ric',
+                   charset='utf-8')
+
 
 class TestEncodeQ(TestEmailBase):
 
diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py
--- a/Lib/test/test_email/test__header_value_parser.py
+++ b/Lib/test/test_email/test__header_value_parser.py
@@ -170,6 +170,15 @@
                         [],
                         '')
 
+    def test_get_encoded_word_quopri_utf_escape_follows_cte(self):
+        # Issue 18044
+        self._test_get_x(parser.get_encoded_word,
+                        '=?utf-8?q?=C3=89ric?=',
+                        '?ric',
+                        '?ric',
+                        [],
+                        '')
+
     # get_unstructured
 
     def _get_unst(self, value):
diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py
--- a/Lib/test/test_email/test_headerregistry.py
+++ b/Lib/test/test_email/test_headerregistry.py
@@ -123,12 +123,45 @@
     #    self.assertEqual(h, value)
     #    self.assertDefectsEqual(h.defects, [errors.ObsoleteHeaderDefect])
 
-    def test_RFC2047_value_decoded(self):
-        value = '=?utf-8?q?this_is_a_test?='
-        h = self.make_header('subject', value)
-        self.assertEqual(h, 'this is a test')
 
+ at parameterize
+class TestUnstructuredHeader(TestHeaderBase):
 
+    def string_as_value(self,
+                        source,
+                        decoded,
+                        *args):
+        l = len(args)
+        defects = args[0] if l>0 else []
+        header = 'Subject:' + (' ' if source else '')
+        folded = header + (args[1] if l>1 else source) + '\n'
+        h = self.make_header('Subject', source)
+        self.assertEqual(h, decoded)
+        self.assertDefectsEqual(h.defects, defects)
+        self.assertEqual(h.fold(policy=policy.default), folded)
+
+    string_params = {
+
+        'rfc2047_simple_quopri': (
+            '=?utf-8?q?this_is_a_test?=',
+            'this is a test',
+            [],
+            'this is a test'),
+
+        'rfc2047_gb2312_base64': (
+            '=?gb2312?b?1eLKx9bQzsSy4srUo6E=?=',
+            '\u8fd9\u662f\u4e2d\u6587\u6d4b\u8bd5\uff01',
+            [],
+            '=?utf-8?b?6L+Z5piv5Lit5paH5rWL6K+V77yB?='),
+
+        'rfc2047_simple_nonascii_quopri': (
+            '=?utf-8?q?=C3=89ric?=',
+            '?ric'),
+
+    }
+
+
+ at parameterize
 class TestDateHeader(TestHeaderBase):
 
     datestring = 'Sun, 23 Sep 2001 20:10:55 -0700'
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -47,6 +47,10 @@
 Library
 -------
 
+- Issue #18044: The new email header parser was mis-parsing encoded words where
+  an encoded character immediately followed the '?' that follows the CTE
+  character, resulting in a decoding failure.  They are now decoded correctly.
+
 - Issue #18101: Tcl.split() now process strings nested in a tuple as it
   do with byte strings.
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 21:59:06 2013
From: python-checkins at python.org (r.david.murray)
Date: Thu, 11 Jul 2013 21:59:06 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E3_-=3E_default?=
 =?utf-8?q?=29=3A_Merge=3A_=2318044=3A_Fix_parsing_of_encoded_words_of_the?=
 =?utf-8?b?IGZvcm0gPT91dGY4P3E/PVhYLi4uPz0=?=
Message-ID: <3brp4V4sW1z7Ljn@mail.python.org>

http://hg.python.org/cpython/rev/32c6cfffbddd
changeset:   84559:32c6cfffbddd
parent:      84557:5fa793ae36cc
parent:      84558:4acb822f4c43
user:        R David Murray <rdmurray at bitdance.com>
date:        Thu Jul 11 15:58:07 2013 -0400
summary:
  Merge: #18044: Fix parsing of encoded words of the form =?utf8?q?=XX...?=

files:
  Lib/email/_header_value_parser.py                |  43 +--------
  Lib/test/test_email/test__encoded_words.py       |   5 +
  Lib/test/test_email/test__header_value_parser.py |   9 ++
  Lib/test/test_email/test_headerregistry.py       |  41 ++++++++-
  Misc/NEWS                                        |   4 +
  5 files changed, 62 insertions(+), 40 deletions(-)


diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -69,6 +69,7 @@
 
 import re
 import urllib   # For urllib.parse.unquote
+from string import hexdigits
 from collections import namedtuple, OrderedDict
 from email import _encoded_words as _ew
 from email import errors
@@ -391,10 +392,6 @@
     token_type = 'unstructured'
 
     def _fold(self, folded):
-        if any(x.token_type=='encoded-word' for x in self):
-            return self._fold_encoded(folded)
-        # Here we can have either a pure ASCII string that may or may not
-        # have surrogateescape encoded bytes, or a unicode string.
         last_ew = None
         for part in self.parts:
             tstr = str(part)
@@ -1386,35 +1383,6 @@
         pos = pos + 1
     return ''.join(vchars), ''.join([fragment[pos:]] + remainder), had_qp
 
-def _decode_ew_run(value):
-    """ Decode a run of RFC2047 encoded words.
-
-        _decode_ew_run(value) -> (text, value, defects)
-
-    Scans the supplied value for a run of tokens that look like they are RFC
-    2047 encoded words, decodes those words into text according to RFC 2047
-    rules (whitespace between encoded words is discarded), and returns the text
-    and the remaining value (including any leading whitespace on the remaining
-    value), as well as a list of any defects encountered while decoding.  The
-    input value may not have any leading whitespace.
-
-    """
-    res = []
-    defects = []
-    last_ws = ''
-    while value:
-        try:
-            tok, ws, value = _wsp_splitter(value, 1)
-        except ValueError:
-            tok, ws, value = value, '', ''
-        if not (tok.startswith('=?') and tok.endswith('?=')):
-            return ''.join(res), last_ws + tok + ws + value, defects
-        text, charset, lang, new_defects = _ew.decode(tok)
-        res.append(text)
-        defects.extend(new_defects)
-        last_ws = ws
-    return ''.join(res), last_ws, defects
-
 def get_fws(value):
     """FWS = 1*WSP
 
@@ -1440,7 +1408,8 @@
         raise errors.HeaderParseError(
             "expected encoded word but found {}".format(value))
     remstr = ''.join(remainder)
-    if remstr[:2].isdigit():
+    if len(remstr) > 1 and remstr[0] in hexdigits and remstr[1] in hexdigits:
+        # The ? after the CTE was followed by an encoded word escape (=XX).
         rest, *remainder = remstr.split('?=', 1)
         tok = tok + '?=' + rest
     if len(tok.split()) > 1:
@@ -1488,8 +1457,8 @@
 
     """
     # XXX: but what about bare CR and LF?  They might signal the start or
-    # end of an encoded word.  YAGNI for now, since out current parsers
-    # will never send us strings with bard CR or LF.
+    # end of an encoded word.  YAGNI for now, since our current parsers
+    # will never send us strings with bare CR or LF.
 
     unstructured = UnstructuredTokenList()
     while value:
@@ -1501,6 +1470,8 @@
             try:
                 token, value = get_encoded_word(value)
             except errors.HeaderParseError:
+                # XXX: Need to figure out how to register defects when
+                # appropriate here.
                 pass
             else:
                 have_ws = True
diff --git a/Lib/test/test_email/test__encoded_words.py b/Lib/test/test_email/test__encoded_words.py
--- a/Lib/test/test_email/test__encoded_words.py
+++ b/Lib/test/test_email/test__encoded_words.py
@@ -122,6 +122,11 @@
                    # XXX Should this be a new Defect instead?
                    defects = [errors.CharsetError])
 
+    def test_q_nonascii(self):
+        self._test('=?utf-8?q?=C3=89ric?=',
+                   '?ric',
+                   charset='utf-8')
+
 
 class TestEncodeQ(TestEmailBase):
 
diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py
--- a/Lib/test/test_email/test__header_value_parser.py
+++ b/Lib/test/test_email/test__header_value_parser.py
@@ -170,6 +170,15 @@
                         [],
                         '')
 
+    def test_get_encoded_word_quopri_utf_escape_follows_cte(self):
+        # Issue 18044
+        self._test_get_x(parser.get_encoded_word,
+                        '=?utf-8?q?=C3=89ric?=',
+                        '?ric',
+                        '?ric',
+                        [],
+                        '')
+
     # get_unstructured
 
     def _get_unst(self, value):
diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py
--- a/Lib/test/test_email/test_headerregistry.py
+++ b/Lib/test/test_email/test_headerregistry.py
@@ -123,12 +123,45 @@
     #    self.assertEqual(h, value)
     #    self.assertDefectsEqual(h.defects, [errors.ObsoleteHeaderDefect])
 
-    def test_RFC2047_value_decoded(self):
-        value = '=?utf-8?q?this_is_a_test?='
-        h = self.make_header('subject', value)
-        self.assertEqual(h, 'this is a test')
 
+ at parameterize
+class TestUnstructuredHeader(TestHeaderBase):
 
+    def string_as_value(self,
+                        source,
+                        decoded,
+                        *args):
+        l = len(args)
+        defects = args[0] if l>0 else []
+        header = 'Subject:' + (' ' if source else '')
+        folded = header + (args[1] if l>1 else source) + '\n'
+        h = self.make_header('Subject', source)
+        self.assertEqual(h, decoded)
+        self.assertDefectsEqual(h.defects, defects)
+        self.assertEqual(h.fold(policy=policy.default), folded)
+
+    string_params = {
+
+        'rfc2047_simple_quopri': (
+            '=?utf-8?q?this_is_a_test?=',
+            'this is a test',
+            [],
+            'this is a test'),
+
+        'rfc2047_gb2312_base64': (
+            '=?gb2312?b?1eLKx9bQzsSy4srUo6E=?=',
+            '\u8fd9\u662f\u4e2d\u6587\u6d4b\u8bd5\uff01',
+            [],
+            '=?utf-8?b?6L+Z5piv5Lit5paH5rWL6K+V77yB?='),
+
+        'rfc2047_simple_nonascii_quopri': (
+            '=?utf-8?q?=C3=89ric?=',
+            '?ric'),
+
+    }
+
+
+ at parameterize
 class TestDateHeader(TestHeaderBase):
 
     datestring = 'Sun, 23 Sep 2001 20:10:55 -0700'
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -151,6 +151,10 @@
 Library
 -------
 
+- Issue #18044: The new email header parser was mis-parsing encoded words where
+  an encoded character immediately followed the '?' that follows the CTE
+  character, resulting in a decoding failure.  They are now decoded correctly.
+
 - Issue #18101: Tcl.split() now process strings nested in a tuple as it
   do with byte strings.
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 22:46:47 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu, 11 Jul 2013 22:46:47 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_typeobject=2Ec=3A_remove_t?=
	=?utf-8?q?railing_spaces?=
Message-ID: <3brq7W0WT0zRSp@mail.python.org>

http://hg.python.org/cpython/rev/a2d6274e2fc8
changeset:   84560:a2d6274e2fc8
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 11 22:42:25 2013 +0200
summary:
  typeobject.c: remove trailing spaces

files:
  Objects/typeobject.c |  6 +++---
  1 files changed, 3 insertions(+), 3 deletions(-)


diff --git a/Objects/typeobject.c b/Objects/typeobject.c
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -2411,7 +2411,7 @@
     char *s;
     char *res_start = (char*)res;
     PyType_Slot *slot;
-    
+
     /* Set the type name and qualname */
     s = strrchr(spec->name, '.');
     if (s == NULL)
@@ -2432,7 +2432,7 @@
     type->tp_name = spec->name;
     if (!type->tp_name)
         goto fail;
-    
+
     /* Adjust for empty tuple bases */
     if (!bases) {
         base = &PyBaseObject_Type;
@@ -2516,7 +2516,7 @@
     /* Set type.__module__ */
     s = strrchr(spec->name, '.');
     if (s != NULL)
-        _PyDict_SetItemId(type->tp_dict, &PyId___module__, 
+        _PyDict_SetItemId(type->tp_dict, &PyId___module__,
             PyUnicode_FromStringAndSize(
                 spec->name, (Py_ssize_t)(s - spec->name)));
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 22:46:48 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu, 11 Jul 2013 22:46:48 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_slot=5Ft?=
 =?utf-8?q?p=5Fstr=28=29_must_not_fallback_on_slot=5Ftp=5Frepr=28=29_on_er?=
 =?utf-8?q?ror?=
Message-ID: <3brq7X3v1Dz7LjT@mail.python.org>

http://hg.python.org/cpython/rev/01a46dc00fc8
changeset:   84561:01a46dc00fc8
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 11 22:46:11 2013 +0200
summary:
  Issue #18408: slot_tp_str() must not fallback on slot_tp_repr() on error

type->tp_str must not point to slot_tp_str() if type has no __str__ attribute,
so there is no reason for slot_tp_str() to fallback on slot_tp_str() on lookup
error. Moreover, calling PyErr_Clear() may hide a real bug like MemoryError.

If __str__ attribute is removed, slots must be updated (which is done by
type_setattro()).

files:
  Objects/typeobject.c |  21 ++-------------------
  1 files changed, 2 insertions(+), 19 deletions(-)


diff --git a/Objects/typeobject.c b/Objects/typeobject.c
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -5274,29 +5274,12 @@
     _Py_IDENTIFIER(__str__);
 
     func = lookup_method(self, &PyId___str__);
-    if (func != NULL) {
+    if (func == NULL)
+        return NULL;
         res = PyEval_CallObject(func, NULL);
         Py_DECREF(func);
         return res;
     }
-    else {
-        /* PyObject *ress; */
-        PyErr_Clear();
-        res = slot_tp_repr(self);
-        if (!res)
-            return NULL;
-        /* XXX this is non-sensical. Why should we return
-           a bytes object from __str__. Is this code even
-           used? - mvl */
-        assert(0);
-        return res;
-        /*
-        ress = _PyUnicode_AsDefaultEncodedString(res);
-        Py_DECREF(res);
-        return ress;
-        */
-    }
-}
 
 static Py_hash_t
 slot_tp_hash(PyObject *self)

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 23:12:09 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu, 11 Jul 2013 23:12:09 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_ste=5Fne?=
 =?utf-8?q?w=28=29_initialize_all_attributes_before_handling_error?=
Message-ID: <3brqhn2MrDz7LjQ@mail.python.org>

http://hg.python.org/cpython/rev/39eb1ce5f377
changeset:   84562:39eb1ce5f377
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 11 22:49:00 2013 +0200
summary:
  Issue #18408: ste_new() initialize all attributes before handling error

If an attribute is not initialized, the destructor can crash

files:
  Python/symtable.c |  22 +++++++++-------------
  1 files changed, 9 insertions(+), 13 deletions(-)


diff --git a/Python/symtable.c b/Python/symtable.c
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -37,25 +37,13 @@
     ste->ste_table = st;
     ste->ste_id = k; /* ste owns reference to k */
 
+    Py_INCREF(name);
     ste->ste_name = name;
-    Py_INCREF(name);
 
     ste->ste_symbols = NULL;
     ste->ste_varnames = NULL;
     ste->ste_children = NULL;
 
-    ste->ste_symbols = PyDict_New();
-    if (ste->ste_symbols == NULL)
-        goto fail;
-
-    ste->ste_varnames = PyList_New(0);
-    if (ste->ste_varnames == NULL)
-        goto fail;
-
-    ste->ste_children = PyList_New(0);
-    if (ste->ste_children == NULL)
-        goto fail;
-
     ste->ste_directives = NULL;
 
     ste->ste_type = block;
@@ -79,6 +67,14 @@
     ste->ste_returns_value = 0;
     ste->ste_needs_class_closure = 0;
 
+    ste->ste_symbols = PyDict_New();
+    ste->ste_varnames = PyList_New(0);
+    ste->ste_children = PyList_New(0);
+    if (ste->ste_symbols == NULL
+        || ste->ste_varnames == NULL
+        || ste->ste_children == NULL)
+        goto fail;
+
     if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0)
         goto fail;
 

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 23:12:10 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu, 11 Jul 2013 23:12:10 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_comp?=
 =?utf-8?q?iler=5Fimport=28=29_to_handle_PyUnicode=5FSubstring=28=29_failu?=
 =?utf-8?q?re?=
Message-ID: <3brqhp4YNPz7Ljq@mail.python.org>

http://hg.python.org/cpython/rev/aaa6e8b8a5c9
changeset:   84563:aaa6e8b8a5c9
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 11 22:50:45 2013 +0200
summary:
  Issue #18408: Fix compiler_import() to handle PyUnicode_Substring() failure properly

files:
  Python/compile.c |  5 ++++-
  1 files changed, 4 insertions(+), 1 deletions(-)


diff --git a/Python/compile.c b/Python/compile.c
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -2316,8 +2316,11 @@
             identifier tmp = alias->name;
             Py_ssize_t dot = PyUnicode_FindChar(
                 alias->name, '.', 0, PyUnicode_GET_LENGTH(alias->name), 1);
-            if (dot != -1)
+            if (dot != -1) {
                 tmp = PyUnicode_Substring(alias->name, 0, dot);
+                if (tmp == NULL)
+                    return 0;
+            }
             r = compiler_nameop(c, tmp, Store);
             if (dot != -1) {
                 Py_DECREF(tmp);

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 23:12:11 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu, 11 Jul 2013 23:12:11 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_parsetok?=
 =?utf-8?q?=28=29_must_not_write_into_stderr_on_memory_allocation_error?=
Message-ID: <3brqhq6b33z7Ljr@mail.python.org>

http://hg.python.org/cpython/rev/ff93930a53c0
changeset:   84564:ff93930a53c0
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 11 22:52:19 2013 +0200
summary:
  Issue #18408: parsetok() must not write into stderr on memory allocation error

The caller gets an error code and can raise a classic Python exception.

files:
  Parser/parsetok.c |  1 -
  1 files changed, 0 insertions(+), 1 deletions(-)


diff --git a/Parser/parsetok.c b/Parser/parsetok.c
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -178,7 +178,6 @@
         len = b - a; /* XXX this may compute NULL - NULL */
         str = (char *) PyObject_MALLOC(len + 1);
         if (str == NULL) {
-            fprintf(stderr, "no mem for next token\n");
             err_ret->error = E_NOMEM;
             break;
         }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 23:12:13 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu, 11 Jul 2013 23:12:13 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_=5FP?=
 =?utf-8?q?ickler=5FNew=28=29_and_=5FUnpickler=5FNew=28=29=3A_initialize_a?=
 =?utf-8?q?ll?=
Message-ID: <3brqhs1j27z7LjQ@mail.python.org>

http://hg.python.org/cpython/rev/1a1869baec4c
changeset:   84565:1a1869baec4c
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 11 22:56:25 2013 +0200
summary:
  Issue #18408: Fix _Pickler_New() and _Unpickler_New(): initialize all
attributes before handling errors

_Pickler_New() now calls PyObject_GC_Del() instead of Py_DECREF() on error,
because the pickle object is created using PyObject_GC_New().

Fix a crash in the destructor when an attribute is not initiallized.

files:
  Modules/_pickle.c |  36 +++++++++++++---------------------
  1 files changed, 14 insertions(+), 22 deletions(-)


diff --git a/Modules/_pickle.c b/Modules/_pickle.c
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -774,18 +774,15 @@
     self->fast_nesting = 0;
     self->fix_imports = 0;
     self->fast_memo = NULL;
-
-    self->memo = PyMemoTable_New();
-    if (self->memo == NULL) {
-        Py_DECREF(self);
-        return NULL;
-    }
     self->max_output_len = WRITE_BUF_SIZE;
     self->output_len = 0;
+
+    self->memo = PyMemoTable_New();
     self->output_buffer = PyBytes_FromStringAndSize(NULL,
                                                     self->max_output_len);
-    if (self->output_buffer == NULL) {
-        Py_DECREF(self);
+
+    if (self->memo == NULL || self->output_buffer == NULL) {
+        PyObject_GC_Del(self);
         return NULL;
     }
     return self;
@@ -1136,20 +1133,6 @@
     if (self == NULL)
         return NULL;
 
-    self->stack = (Pdata *)Pdata_New();
-    if (self->stack == NULL) {
-        Py_DECREF(self);
-        return NULL;
-    }
-    memset(&self->buffer, 0, sizeof(Py_buffer));
-
-    self->memo_size = 32;
-    self->memo = _Unpickler_NewMemo(self->memo_size);
-    if (self->memo == NULL) {
-        Py_DECREF(self);
-        return NULL;
-    }
-
     self->arg = NULL;
     self->pers_func = NULL;
     self->input_buffer = NULL;
@@ -1167,6 +1150,15 @@
     self->marks_size = 0;
     self->proto = 0;
     self->fix_imports = 0;
+    memset(&self->buffer, 0, sizeof(Py_buffer));
+    self->memo_size = 32;
+    self->memo = _Unpickler_NewMemo(self->memo_size);
+    self->stack = (Pdata *)Pdata_New();
+
+    if (self->memo == NULL || self->stack == NULL) {
+        Py_DECREF(self);
+        return NULL;
+    }
 
     return self;
 }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 23:12:14 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu, 11 Jul 2013 23:12:14 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_=5Feleme?=
 =?utf-8?q?nttree=2Ec_now_handles_create=5Fextra=28=29_failure?=
Message-ID: <3brqht3x3Hz7Ljv@mail.python.org>

http://hg.python.org/cpython/rev/e11121b9bd09
changeset:   84566:e11121b9bd09
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 11 23:01:36 2013 +0200
summary:
  Issue #18408: _elementtree.c now handles create_extra() failure

files:
  Modules/_elementtree.c |  36 ++++++++++++++++++++----------
  1 files changed, 24 insertions(+), 12 deletions(-)


diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -374,8 +374,10 @@
     /* make sure self->children can hold the given number of extra
        elements.  set an exception and return -1 if allocation failed */
 
-    if (!self->extra)
-        create_extra(self, NULL);
+    if (!self->extra) {
+        if (create_extra(self, NULL) < 0)
+            return -1;
+    }
 
     size = self->extra->length + extra;
 
@@ -1267,8 +1269,10 @@
                           &Element_Type, &element))
         return NULL;
 
-    if (!self->extra)
-        create_extra(self, NULL);
+    if (!self->extra) {
+        if (create_extra(self, NULL) < 0)
+            return NULL;
+    }
 
     if (index < 0) {
         index += self->extra->length;
@@ -1409,8 +1413,10 @@
     if (!PyArg_ParseTuple(args, "OO:set", &key, &value))
         return NULL;
 
-    if (!self->extra)
-        create_extra(self, NULL);
+    if (!self->extra) {
+        if (create_extra(self, NULL) < 0)
+            return NULL;
+    }
 
     attrib = element_get_attrib(self);
     if (!attrib)
@@ -1525,8 +1531,10 @@
         PyObject* recycle = NULL;
         PyObject* seq = NULL;
 
-        if (!self->extra)
-            create_extra(self, NULL);
+        if (!self->extra) {
+            if (create_extra(self, NULL) < 0)
+                return -1;
+        }
 
         if (PySlice_GetIndicesEx(item,
                 self->extra->length,
@@ -1756,8 +1764,10 @@
         res = element_get_tail(self);
     } else if (strcmp(name, "attrib") == 0) {
         PyErr_Clear();
-        if (!self->extra)
-            create_extra(self, NULL);
+        if (!self->extra) {
+            if (create_extra(self, NULL) < 0)
+                return NULL;
+        }
         res = element_get_attrib(self);
     }
 
@@ -1790,8 +1800,10 @@
         self->tail = value;
         Py_INCREF(self->tail);
     } else if (strcmp(name, "attrib") == 0) {
-        if (!self->extra)
-            create_extra(self, NULL);
+        if (!self->extra) {
+            if (create_extra(self, NULL) < 0)
+                return -1;
+        }
         Py_DECREF(self->extra->attrib);
         self->extra->attrib = value;
         Py_INCREF(self->extra->attrib);

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 23:12:15 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu, 11 Jul 2013 23:12:15 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Cleanup_=5Felementtree=2Ec?=
Message-ID: <3brqhv60qJz7Ljv@mail.python.org>

http://hg.python.org/cpython/rev/e1b398d0decb
changeset:   84567:e1b398d0decb
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 11 23:05:03 2013 +0200
summary:
  Cleanup _elementtree.c

files:
  Modules/_elementtree.c |  13 ++++++-------
  1 files changed, 6 insertions(+), 7 deletions(-)


diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -529,7 +529,6 @@
     }
 
     elem = create_new_element(tag, attrib);
-
     Py_DECREF(attrib);
 
     if (element_add_subelement(parent, elem) < 0) {
@@ -1784,10 +1783,10 @@
     char *name = "";
     if (PyUnicode_Check(nameobj))
         name = _PyUnicode_AsString(nameobj);
-
-    if (name == NULL) {
+    if (name == NULL)
         return -1;
-    } else if (strcmp(name, "tag") == 0) {
+
+    if (strcmp(name, "tag") == 0) {
         Py_DECREF(self->tag);
         self->tag = value;
         Py_INCREF(self->tag);
@@ -2135,15 +2134,15 @@
 
     if (star && PyObject_RichCompareBool(tag, star, Py_EQ) == 1)
         tag = Py_None;
-
     Py_XDECREF(star);
+
+    Py_INCREF(tag);
     it->sought_tag = tag;
     it->root_done = 0;
     it->gettext = gettext;
+    Py_INCREF(self);
     it->root_element = self;
 
-    Py_INCREF(self);
-    Py_INCREF(tag);
 
     PyObject_GC_Track(it);
     return (PyObject *)it;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 23:12:17 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu, 11 Jul 2013 23:12:17 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Differen?=
 =?utf-8?q?t_fixes_in_=5Felementtree=2Ec_to_handle_correctly_MemoryError?=
Message-ID: <3brqhx12Hsz7LjP@mail.python.org>

http://hg.python.org/cpython/rev/0877e17fa25e
changeset:   84568:0877e17fa25e
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 11 23:08:39 2013 +0200
summary:
  Issue #18408: Different fixes in _elementtree.c to handle correctly MemoryError

 * create_new_element() initializes all attributes before handling errors,
   to fix a crash in the destructor
* create_new_element() calls PyObject_GC_Del() on error, instead of
  PyObject_Del(), because the object was created by PyObject_GC_New()
* subelement() now handles create_new_element() failure
* element_getattro() now handles element_get_text() failure
* makeuniversal() now handles PyBytes_FromStringAndSize() failure

files:
  Modules/_elementtree.c |  32 +++++++++++++++++------------
  1 files changed, 19 insertions(+), 13 deletions(-)


diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -224,24 +224,24 @@
         return NULL;
     self->extra = NULL;
 
+    Py_INCREF(tag);
+    self->tag = tag;
+
+    Py_INCREF(Py_None);
+    self->text = Py_None;
+
+    Py_INCREF(Py_None);
+    self->tail = Py_None;
+
+    self->weakreflist = NULL;
+
     if (attrib != Py_None && !is_empty_dict(attrib)) {
         if (create_extra(self, attrib) < 0) {
-            PyObject_Del(self);
+            PyObject_GC_Del(self);
             return NULL;
         }
     }
 
-    Py_INCREF(tag);
-    self->tag = tag;
-
-    Py_INCREF(Py_None);
-    self->text = Py_None;
-
-    Py_INCREF(Py_None);
-    self->tail = Py_None;
-
-    self->weakreflist = NULL;
-
     ALLOC(sizeof(ElementObject), "create element");
     PyObject_GC_Track(self);
     return (PyObject*) self;
@@ -530,6 +530,8 @@
 
     elem = create_new_element(tag, attrib);
     Py_DECREF(attrib);
+    if (elem == NULL)
+        return NULL;
 
     if (element_add_subelement(parent, elem) < 0) {
         Py_DECREF(elem);
@@ -1748,7 +1750,7 @@
         return res;
     } else if (strcmp(name, "text") == 0) {
         res = element_get_text(self);
-        Py_INCREF(res);
+        Py_XINCREF(res);
         return res;
     }
 
@@ -2726,6 +2728,10 @@
         if (i != size) {
             /* convert to universal name */
             tag = PyBytes_FromStringAndSize(NULL, size+1);
+            if (tag == NULL) {
+                Py_DECREF(key);
+                return NULL;
+            }
             p = PyBytes_AS_STRING(tag);
             p[0] = '{';
             memcpy(p+1, string, size);

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 23:47:02 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu, 11 Jul 2013 23:47:02 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_parsetok?=
 =?utf-8?q?=28=29_must_not_write_into_stderr_on_memory_allocation_error?=
Message-ID: <3brrT24dqtzRq4@mail.python.org>

http://hg.python.org/cpython/rev/51eddca9dd6f
changeset:   84569:51eddca9dd6f
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 11 23:17:33 2013 +0200
summary:
  Issue #18408: parsetok() must not write into stderr on memory allocation error

The caller gets an error code and can raise a classic Python exception.

files:
  Parser/parsetok.c |  1 -
  1 files changed, 0 insertions(+), 1 deletions(-)


diff --git a/Parser/parsetok.c b/Parser/parsetok.c
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -138,7 +138,6 @@
     int started = 0;
 
     if ((ps = PyParser_New(g, start)) == NULL) {
-        fprintf(stderr, "no mem for new parser\n");
         err_ret->error = E_NOMEM;
         PyTokenizer_Free(tok);
         return NULL;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Thu Jul 11 23:47:03 2013
From: python-checkins at python.org (victor.stinner)
Date: Thu, 11 Jul 2013 23:47:03 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_In_debug?=
 =?utf-8?q?_mode=2C_PyCFunction=5FCall=28=29_now_checks_if_an_exception_wa?=
 =?utf-8?q?s?=
Message-ID: <3brrT36gVRz7Ljc@mail.python.org>

http://hg.python.org/cpython/rev/5e50f1a0c985
changeset:   84570:5e50f1a0c985
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Jul 11 23:44:46 2013 +0200
summary:
  Issue #18408: In debug mode, PyCFunction_Call() now checks if an exception was
raised if the result is NULL to help to find bugs in C mode (get the error
earlier than the SystemError in ceval.c).

files:
  Objects/methodobject.c |  30 +++++++++++++++++++++++-------
  1 files changed, 23 insertions(+), 7 deletions(-)


diff --git a/Objects/methodobject.c b/Objects/methodobject.c
--- a/Objects/methodobject.c
+++ b/Objects/methodobject.c
@@ -79,23 +79,34 @@
 PyObject *
 PyCFunction_Call(PyObject *func, PyObject *arg, PyObject *kw)
 {
+#define CHECK_RESULT(res) assert(res != NULL || PyErr_Occurred())
+
     PyCFunctionObject* f = (PyCFunctionObject*)func;
     PyCFunction meth = PyCFunction_GET_FUNCTION(func);
     PyObject *self = PyCFunction_GET_SELF(func);
+    PyObject *res;
     Py_ssize_t size;
 
     switch (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST)) {
     case METH_VARARGS:
-        if (kw == NULL || PyDict_Size(kw) == 0)
-            return (*meth)(self, arg);
+        if (kw == NULL || PyDict_Size(kw) == 0) {
+            res = (*meth)(self, arg);
+            CHECK_RESULT(res);
+            return res;
+        }
         break;
     case METH_VARARGS | METH_KEYWORDS:
-        return (*(PyCFunctionWithKeywords)meth)(self, arg, kw);
+        res = (*(PyCFunctionWithKeywords)meth)(self, arg, kw);
+        CHECK_RESULT(res);
+        return res;
     case METH_NOARGS:
         if (kw == NULL || PyDict_Size(kw) == 0) {
             size = PyTuple_GET_SIZE(arg);
-            if (size == 0)
-                return (*meth)(self, NULL);
+            if (size == 0) {
+                res = (*meth)(self, NULL);
+                CHECK_RESULT(res);
+                return res;
+            }
             PyErr_Format(PyExc_TypeError,
                 "%.200s() takes no arguments (%zd given)",
                 f->m_ml->ml_name, size);
@@ -105,8 +116,11 @@
     case METH_O:
         if (kw == NULL || PyDict_Size(kw) == 0) {
             size = PyTuple_GET_SIZE(arg);
-            if (size == 1)
-                return (*meth)(self, PyTuple_GET_ITEM(arg, 0));
+            if (size == 1) {
+                res = (*meth)(self, PyTuple_GET_ITEM(arg, 0));
+                CHECK_RESULT(res);
+                return res;
+            }
             PyErr_Format(PyExc_TypeError,
                 "%.200s() takes exactly one argument (%zd given)",
                 f->m_ml->ml_name, size);
@@ -123,6 +137,8 @@
     PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments",
                  f->m_ml->ml_name);
     return NULL;
+
+#undef CHECK_RESULT
 }
 
 /* Methods (the standard built-in methods, that is) */

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 00:59:04 2013
From: python-checkins at python.org (victor.stinner)
Date: Fri, 12 Jul 2013 00:59:04 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_normaliz?=
 =?utf-8?q?estring=28=29_now_raises_MemoryError_on_memory_allocation?=
Message-ID: <3brt484d4xz7Ljb@mail.python.org>

http://hg.python.org/cpython/rev/4975bcd67aa7
changeset:   84571:4975bcd67aa7
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Jul 12 00:02:55 2013 +0200
summary:
  Issue #18408: normalizestring() now raises MemoryError on memory allocation failure

files:
  Python/codecs.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Python/codecs.c b/Python/codecs.c
--- a/Python/codecs.c
+++ b/Python/codecs.c
@@ -65,7 +65,7 @@
 
     p = PyMem_Malloc(len + 1);
     if (p == NULL)
-        return NULL;
+        return PyErr_NoMemory();
     for (i = 0; i < len; i++) {
         register char ch = string[i];
         if (ch == ' ')

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 00:59:05 2013
From: python-checkins at python.org (victor.stinner)
Date: Fri, 12 Jul 2013 00:59:05 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Oh=2C_I_?=
 =?utf-8?q?was_wrong=3A_Pickler=5FNew=28=29_must_call_Py=5FDECREF=28=29_to?=
 =?utf-8?q?_destroy?=
Message-ID: <3brt496pprz7Ljc@mail.python.org>

http://hg.python.org/cpython/rev/1eac89af9acf
changeset:   84572:1eac89af9acf
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Jul 12 00:08:59 2013 +0200
summary:
  Issue #18408: Oh, I was wrong: Pickler_New() must call Py_DECREF() to destroy
the newly created pickler, and not PyObject_GC_Del().

files:
  Modules/_pickle.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Modules/_pickle.c b/Modules/_pickle.c
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -782,7 +782,7 @@
                                                     self->max_output_len);
 
     if (self->memo == NULL || self->output_buffer == NULL) {
-        PyObject_GC_Del(self);
+        Py_DECREF(self);
         return NULL;
     }
     return self;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 00:59:07 2013
From: python-checkins at python.org (victor.stinner)
Date: Fri, 12 Jul 2013 00:59:07 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_ceval=2E?=
 =?utf-8?q?c=3A_in_debug_mode=2C_convert_the_PyErr=5FOccurred=28=29_check_?=
 =?utf-8?q?on?=
Message-ID: <3brt4C1nsKz7LkQ@mail.python.org>

http://hg.python.org/cpython/rev/d9446c2a2fd4
changeset:   84573:d9446c2a2fd4
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Jul 12 00:11:58 2013 +0200
summary:
  Issue #18408: ceval.c: in debug mode, convert the PyErr_Occurred() check on
exception (when getting NULL) to an assertion to detect bugs earlier

files:
  Python/ceval.c |  4 ++++
  1 files changed, 4 insertions(+), 0 deletions(-)


diff --git a/Python/ceval.c b/Python/ceval.c
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -3026,9 +3026,13 @@
         why = WHY_EXCEPTION;
 
         /* Double-check exception status. */
+#ifdef NDEBUG
         if (!PyErr_Occurred())
             PyErr_SetString(PyExc_SystemError,
                             "error return without exception set");
+#else
+        assert(PyErr_Occurred());
+#endif
 
         /* Log traceback info. */
         PyTraceBack_Here(f);

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 00:59:08 2013
From: python-checkins at python.org (victor.stinner)
Date: Fri, 12 Jul 2013 00:59:08 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_errors?=
 =?utf-8?q?=2Ec=3A_in_debug_mode=2C_calling_PyErr=5FBadInternalCall=28=29_?=
 =?utf-8?q?now?=
Message-ID: <3brt4D45wPz7LkQ@mail.python.org>

http://hg.python.org/cpython/rev/2f7c4df5cc46
changeset:   84574:2f7c4df5cc46
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Jul 12 00:37:30 2013 +0200
summary:
  Issue #18408: errors.c: in debug mode, calling PyErr_BadInternalCall() now
fails with an assertion error

files:
  Python/errors.c |  1 +
  1 files changed, 1 insertions(+), 0 deletions(-)


diff --git a/Python/errors.c b/Python/errors.c
--- a/Python/errors.c
+++ b/Python/errors.c
@@ -675,6 +675,7 @@
 void
 PyErr_BadInternalCall(void)
 {
+    assert(0 && "bad argument to internal function");
     PyErr_Format(PyExc_SystemError,
                  "bad argument to internal function");
 }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 00:59:09 2013
From: python-checkins at python.org (victor.stinner)
Date: Fri, 12 Jul 2013 00:59:09 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_pmerge?=
 =?utf-8?q?=28=29_help_of_mro=5Fimplementation=28=29_now_raises_MemoryErro?=
 =?utf-8?q?r_on?=
Message-ID: <3brt4F6ZgWz7LkJ@mail.python.org>

http://hg.python.org/cpython/rev/affb87b1d7ec
changeset:   84575:affb87b1d7ec
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Jul 12 00:42:14 2013 +0200
summary:
  Issue #18408: pmerge() help of mro_implementation() now raises MemoryError on
memory allocation failure

Replace also PyMem_Free() with PyMem_FREE() to be consistent with the rest of
the function.

files:
  Objects/typeobject.c |  6 ++++--
  1 files changed, 4 insertions(+), 2 deletions(-)


diff --git a/Objects/typeobject.c b/Objects/typeobject.c
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1456,8 +1456,10 @@
        that is not included in acc.
     */
     remain = (int *)PyMem_MALLOC(SIZEOF_INT*to_merge_size);
-    if (remain == NULL)
+    if (remain == NULL) {
+        PyErr_NoMemory();
         return -1;
+    }
     for (i = 0; i < to_merge_size; i++)
         remain[i] = 0;
 
@@ -1489,7 +1491,7 @@
         }
         ok = PyList_Append(acc, candidate);
         if (ok < 0) {
-            PyMem_Free(remain);
+            PyMem_FREE(remain);
             return -1;
         }
         for (j = 0; j < to_merge_size; j++) {

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 00:59:11 2013
From: python-checkins at python.org (victor.stinner)
Date: Fri, 12 Jul 2013 00:59:11 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_=5FPyMem?=
 =?utf-8?q?oTable=5FResizeTable=28=29_now_restores_the_old_table_if?=
Message-ID: <3brt4H1c7gz7Lkc@mail.python.org>

http://hg.python.org/cpython/rev/f85fcbbbe8de
changeset:   84576:f85fcbbbe8de
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Jul 12 00:53:26 2013 +0200
summary:
  Issue #18408: _PyMemoTable_ResizeTable() now restores the old table if
allocating a bigger table failed

PyMemoTable destructor does crash if mt_table is NULL.

files:
  Modules/_pickle.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Modules/_pickle.c b/Modules/_pickle.c
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -529,7 +529,7 @@
     oldtable = self->mt_table;
     self->mt_table = PyMem_MALLOC(new_size * sizeof(PyMemoEntry));
     if (self->mt_table == NULL) {
-        PyMem_FREE(oldtable);
+        self->mt_table = oldtable;
         PyErr_NoMemory();
         return -1;
     }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 00:59:12 2013
From: python-checkins at python.org (victor.stinner)
Date: Fri, 12 Jul 2013 00:59:12 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_=5Fpickl?=
 =?utf-8?q?e=2Ec=3A_Add_missing_PyErr=5FNoMemory=28=29_on_memory_allocatio?=
 =?utf-8?q?n?=
Message-ID: <3brt4J3sMVz7LkX@mail.python.org>

http://hg.python.org/cpython/rev/bb5da6e795a1
changeset:   84577:bb5da6e795a1
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Jul 12 00:53:57 2013 +0200
summary:
  Issue #18408: _pickle.c: Add missing PyErr_NoMemory() on memory allocation failures

files:
  Modules/_pickle.c |  11 ++++++++---
  1 files changed, 8 insertions(+), 3 deletions(-)


diff --git a/Modules/_pickle.c b/Modules/_pickle.c
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -436,6 +436,7 @@
     new->mt_table = PyMem_MALLOC(self->mt_allocated * sizeof(PyMemoEntry));
     if (new->mt_table == NULL) {
         PyMem_FREE(new);
+        PyErr_NoMemory();
         return NULL;
     }
     for (i = 0; i < self->mt_allocated; i++) {
@@ -1003,8 +1004,10 @@
                     char **result)
 {
     char *input_line = PyMem_Realloc(self->input_line, len + 1);
-    if (input_line == NULL)
-        return -1;
+    if (input_line == NULL) {
+        PyErr_NoMemory();
+        return -1;
+    }
 
     memcpy(input_line, line, len);
     input_line[len] = '\0';
@@ -1101,8 +1104,10 @@
 _Unpickler_NewMemo(Py_ssize_t new_size)
 {
     PyObject **memo = PyMem_MALLOC(new_size * sizeof(PyObject *));
-    if (memo == NULL)
+    if (memo == NULL) {
+        PyErr_NoMemory();
         return NULL;
+    }
     memset(memo, 0, new_size * sizeof(PyObject *));
     return memo;
 }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 01:42:53 2013
From: python-checkins at python.org (victor.stinner)
Date: Fri, 12 Jul 2013 01:42:53 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_pars?=
 =?utf-8?q?er=2Esequence2st=28=29_and_parser=2Etuple2st=28=29=3A_raise_Mem?=
 =?utf-8?q?oryError?=
Message-ID: <3brv2j47KtzRBr@mail.python.org>

http://hg.python.org/cpython/rev/89c6495d1ff2
changeset:   84578:89c6495d1ff2
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Jul 12 01:33:59 2013 +0200
summary:
  Issue #18408: Fix parser.sequence2st() and parser.tuple2st(): raise MemoryError
on memory allocation failure

Instead of ignoring the memory allocation failure and create invalid objects.

files:
  Modules/parsermodule.c |  19 +++++++++++++++----
  1 files changed, 15 insertions(+), 4 deletions(-)


diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c
--- a/Modules/parsermodule.c
+++ b/Modules/parsermodule.c
@@ -809,8 +809,13 @@
                 return 0;
             }
             strn = (char *)PyObject_MALLOC(len + 1);
-            if (strn != NULL)
-                (void) memcpy(strn, temp_str, len + 1);
+            if (strn == NULL) {
+                Py_DECREF(temp);
+                Py_XDECREF(elem);
+                PyErr_NoMemory();
+                return 0;
+            }
+            (void) memcpy(strn, temp_str, len + 1);
             Py_DECREF(temp);
         }
         else if (!ISNONTERMINAL(type)) {
@@ -906,8 +911,14 @@
                     return NULL;
                 }
                 res->n_str = (char *)PyObject_MALLOC(len + 1);
-                if (res->n_str != NULL && temp != NULL)
-                    (void) memcpy(res->n_str, temp, len + 1);
+                if (res->n_str == NULL) {
+                    Py_DECREF(res);
+                    Py_DECREF(encoding);
+                    Py_DECREF(tuple);
+                    PyErr_NoMemory();
+                    return NULL;
+                }
+                (void) memcpy(res->n_str, temp, len + 1);
                 Py_DECREF(encoding);
                 Py_DECREF(tuple);
             }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 01:42:54 2013
From: python-checkins at python.org (victor.stinner)
Date: Fri, 12 Jul 2013 01:42:54 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_parser_m?=
 =?utf-8?q?odule=3A_fix_error_handling_in_node2tuple=28=29?=
Message-ID: <3brv2k6KWLzS6Y@mail.python.org>

http://hg.python.org/cpython/rev/c80a9705803a
changeset:   84579:c80a9705803a
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Jul 12 01:35:10 2013 +0200
summary:
  Issue #18408: parser module: fix error handling in node2tuple()

Handle PyLong_FromLong() and PyUnicode_FromString() failures

files:
  Modules/parsermodule.c |  82 +++++++++++++++++++----------
  1 files changed, 53 insertions(+), 29 deletions(-)


diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c
--- a/Modules/parsermodule.c
+++ b/Modules/parsermodule.c
@@ -83,54 +83,78 @@
            int lineno,                  /* include line numbers?         */
            int col_offset)              /* include column offsets?       */
 {
+    PyObject *result = NULL, *w;
+
     if (n == NULL) {
         Py_INCREF(Py_None);
-        return (Py_None);
+        return Py_None;
     }
+
     if (ISNONTERMINAL(TYPE(n))) {
         int i;
-        PyObject *v;
-        PyObject *w;
-
-        v = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl));
-        if (v == NULL)
-            return (v);
+
+        result = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl));
+        if (result == NULL)
+            goto error;
+
         w = PyLong_FromLong(TYPE(n));
-        if (w == NULL) {
-            Py_DECREF(v);
-            return ((PyObject*) NULL);
-        }
-        (void) addelem(v, 0, w);
+        if (w == NULL)
+            goto error;
+        (void) addelem(result, 0, w);
+
         for (i = 0; i < NCH(n); i++) {
             w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset);
-            if (w == NULL) {
-                Py_DECREF(v);
-                return ((PyObject*) NULL);
-            }
-            (void) addelem(v, i+1, w);
+            if (w == NULL)
+                goto error;
+            (void) addelem(result, i+1, w);
         }
 
-        if (TYPE(n) == encoding_decl)
-            (void) addelem(v, i+1, PyUnicode_FromString(STR(n)));
-        return (v);
+        if (TYPE(n) == encoding_decl) {
+            w = PyUnicode_FromString(STR(n));
+            if (w == NULL)
+                goto error;
+            (void) addelem(result, i+1, w);
+        }
     }
     else if (ISTERMINAL(TYPE(n))) {
-        PyObject *result = mkseq(2 + lineno + col_offset);
-        if (result != NULL) {
-            (void) addelem(result, 0, PyLong_FromLong(TYPE(n)));
-            (void) addelem(result, 1, PyUnicode_FromString(STR(n)));
-            if (lineno == 1)
-                (void) addelem(result, 2, PyLong_FromLong(n->n_lineno));
-            if (col_offset == 1)
-                (void) addelem(result, 3, PyLong_FromLong(n->n_col_offset));
+        result = mkseq(2 + lineno + col_offset);
+        if (result == NULL)
+            goto error;
+
+        w = PyLong_FromLong(TYPE(n));
+        if (w == NULL)
+            goto error;
+        (void) addelem(result, 0, w);
+
+        w = PyUnicode_FromString(STR(n));
+        if (w == NULL)
+            goto error;
+        (void) addelem(result, 1, w);
+
+        if (lineno == 1) {
+            w = PyLong_FromLong(n->n_lineno);
+            if (w == NULL)
+                goto error;
+            (void) addelem(result, 2, w);
         }
-        return (result);
+
+        if (col_offset == 1) {
+            w = PyLong_FromLong(n->n_col_offset);
+            if (w == NULL)
+                goto error;
+            (void) addelem(result, 3, w);
+        }
     }
     else {
         PyErr_SetString(PyExc_SystemError,
                         "unrecognized parse tree node type");
         return ((PyObject*) NULL);
     }
+    return result;
+
+error:
+    Py_XDECREF(result);
+    return NULL;
 }
 /*
  *  End of material copyrighted by Stichting Mathematisch Centrum.

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 02:07:51 2013
From: python-checkins at python.org (victor.stinner)
Date: Fri, 12 Jul 2013 02:07:51 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_crea?=
 =?utf-8?q?te=5Fextra=28=29_of_=5Felementtree=2Ec=2C_raise_MemoryError_on_?=
 =?utf-8?q?memory?=
Message-ID: <3brvbW3d0YzSlp@mail.python.org>

http://hg.python.org/cpython/rev/60b1d7967ef8
changeset:   84580:60b1d7967ef8
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Jul 12 02:03:34 2013 +0200
summary:
  Issue #18408: Fix create_extra() of _elementtree.c, raise MemoryError on memory
allocation failure

files:
  Modules/_elementtree.c |  4 +++-
  1 files changed, 3 insertions(+), 1 deletions(-)


diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -170,8 +170,10 @@
 create_extra(ElementObject* self, PyObject* attrib)
 {
     self->extra = PyObject_Malloc(sizeof(ElementObjectExtra));
-    if (!self->extra)
+    if (!self->extra) {
+        PyErr_NoMemory();
         return -1;
+    }
 
     if (!attrib)
         attrib = Py_None;

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 02:07:52 2013
From: python-checkins at python.org (victor.stinner)
Date: Fri, 12 Jul 2013 02:07:52 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318408=3A_Fix_cons?=
 =?utf-8?q?tructors_of_=5Felementtree=2Ec?=
Message-ID: <3brvbX5wmvz7LjQ@mail.python.org>

http://hg.python.org/cpython/rev/1e0afd558ba3
changeset:   84581:1e0afd558ba3
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Jul 12 02:05:17 2013 +0200
summary:
  Issue #18408: Fix constructors of _elementtree.c

* Use Py_DECREF() instead of PyObject_GC_Del() to release correctly all
  resources
* Raise MemoryError on memory allocation failure

files:
  Modules/_elementtree.c |  27 +++++++++++++++------------
  1 files changed, 15 insertions(+), 12 deletions(-)


diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -237,15 +237,16 @@
 
     self->weakreflist = NULL;
 
+    ALLOC(sizeof(ElementObject), "create element");
+    PyObject_GC_Track(self);
+
     if (attrib != Py_None && !is_empty_dict(attrib)) {
         if (create_extra(self, attrib) < 0) {
-            PyObject_GC_Del(self);
+            Py_DECREF(self);
             return NULL;
         }
     }
 
-    ALLOC(sizeof(ElementObject), "create element");
-    PyObject_GC_Track(self);
     return (PyObject*) self;
 }
 
@@ -2122,14 +2123,6 @@
     it = PyObject_GC_New(ElementIterObject, &ElementIter_Type);
     if (!it)
         return NULL;
-    if (!(it->parent_stack = PyObject_Malloc(sizeof(ParentLocator)))) {
-        PyObject_GC_Del(it);
-        return NULL;
-    }
-
-    it->parent_stack->parent = NULL;
-    it->parent_stack->child_index = 0;
-    it->parent_stack->next = NULL;
 
     if (PyUnicode_Check(tag))
         star = PyUnicode_FromString("*");
@@ -2147,8 +2140,18 @@
     Py_INCREF(self);
     it->root_element = self;
 
-
     PyObject_GC_Track(it);
+
+    it->parent_stack = PyObject_Malloc(sizeof(ParentLocator));
+    if (it->parent_stack == NULL) {
+        Py_DECREF(it);
+        PyErr_NoMemory();
+        return NULL;
+    }
+    it->parent_stack->parent = NULL;
+    it->parent_stack->child_index = 0;
+    it->parent_stack->next = NULL;
+
     return (PyObject *)it;
 }
 

-- 
Repository URL: http://hg.python.org/cpython

From solipsis at pitrou.net  Fri Jul 12 05:49:04 2013
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Fri, 12 Jul 2013 05:49:04 +0200
Subject: [Python-checkins] Daily reference leaks (1e0afd558ba3): sum=0
Message-ID: <E1UxULw-0002FM-La@ap.vmr.nerim.net>

results for 1e0afd558ba3 on branch "default"
--------------------------------------------



Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogB_3rgI', '-x']

From python-checkins at python.org  Fri Jul 12 16:48:50 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri, 12 Jul 2013 16:48:50 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?devinabox=3A_Issue_=2318362=3A_Allow_?=
 =?utf-8?q?for_build=5Fpython=2Epy_to_be_used_to_build_CPython_in?=
Message-ID: <3bsH824LlHz7Lkg@mail.python.org>

http://hg.python.org/devinabox/rev/fe1c35cbc9a0
changeset:   49:fe1c35cbc9a0
user:        Brett Cannon <brett at python.org>
date:        Fri Jul 12 10:48:45 2013 -0400
summary:
  Issue #18362: Allow for build_python.py to be used to build CPython in
other clones.

files:
  build_cpython.py |  22 +++++++++++++++-------
  1 files changed, 15 insertions(+), 7 deletions(-)


diff --git a/build_cpython.py b/build_cpython.py
--- a/build_cpython.py
+++ b/build_cpython.py
@@ -4,14 +4,15 @@
 On all platforms, return the path to the executable.
 
 """
+from __future__ import print_function
+
 import multiprocessing
 import os
 import subprocess
 import sys
 
 
-def executable():
-    directory = 'cpython'
+def executable(directory):
     cmd = os.path.join(directory, 'python')
     # UNIX
     if not os.path.isfile(cmd):
@@ -28,12 +29,11 @@
     return os.path.abspath(cmd)
 
 
-def main():
+def main(directory):
     if sys.platform == 'win32':
         print("See the devguide's Getting Set Up guide for building under "
               "Windows")
 
-    directory = 'cpython'
     cwd = os.getcwd()
     os.chdir(directory)
     try:
@@ -46,9 +46,17 @@
         subprocess.call(make_cmd)
     finally:
         os.chdir(cwd)
-    return executable()
+    return executable(directory)
 
 if __name__ == '__main__':
-    executable = main()
-    print(executable or 'No executable found')
+    arg_count = len(sys.argv) - 1
+    if arg_count > 1:
+        raise ValueError(
+                '0 or 1 arguments expected, not {}'.format(arg_count))
+    executable = main(sys.argv[1] if arg_count else 'cpython')
+    if not executable:
+        print('CPython executable NOT found')
+    else:
+        print('CPython executable can be found at:')
+        print(' ', executable)
     sys.exit(0 if executable else 1)

-- 
Repository URL: http://hg.python.org/devinabox

From python-checkins at python.org  Fri Jul 12 16:51:02 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri, 12 Jul 2013 16:51:02 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?devinabox=3A_Issue_=2318361=3A_Use_os?=
 =?utf-8?q?=2Ecpu=5Fcount=28=29_when_available=2E?=
Message-ID: <3bsHBZ2GsLz7LkV@mail.python.org>

http://hg.python.org/devinabox/rev/3446dee09265
changeset:   50:3446dee09265
user:        Brett Cannon <brett at python.org>
date:        Fri Jul 12 10:50:57 2013 -0400
summary:
  Issue #18361: Use os.cpu_count() when available.

files:
  build_cpython.py |  8 ++++++--
  1 files changed, 6 insertions(+), 2 deletions(-)


diff --git a/build_cpython.py b/build_cpython.py
--- a/build_cpython.py
+++ b/build_cpython.py
@@ -6,11 +6,15 @@
 """
 from __future__ import print_function
 
-import multiprocessing
 import os
 import subprocess
 import sys
 
+try:
+    from os import cpu_count
+except ImportError:
+    from multiprocessing import cpu_count
+
 
 def executable(directory):
     cmd = os.path.join(directory, 'python')
@@ -42,7 +46,7 @@
         else:
             subprocess.check_call(['./configure', '--prefix=/tmp/cpython',
                                    '--with-pydebug'])
-        make_cmd = ['make', '-s', '-j', str(multiprocessing.cpu_count())]
+        make_cmd = ['make', '-s', '-j', str(cpu_count())]
         subprocess.call(make_cmd)
     finally:
         os.chdir(cwd)

-- 
Repository URL: http://hg.python.org/devinabox

From python-checkins at python.org  Fri Jul 12 16:53:46 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri, 12 Jul 2013 16:53:46 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?devinabox=3A_Issue_=2318363=3A_Use_ab?=
	=?utf-8?q?br_tag_over_acronym=2E?=
Message-ID: <3bsHFk4VDSz7Lkc@mail.python.org>

http://hg.python.org/devinabox/rev/fae104051a63
changeset:   51:fae104051a63
user:        Brett Cannon <brett at python.org>
date:        Fri Jul 12 10:53:41 2013 -0400
summary:
  Issue #18363: Use abbr tag over acronym.

files:
  index.html |  5 +++--
  1 files changed, 3 insertions(+), 2 deletions(-)


diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -30,7 +30,8 @@
     <li><a href="coverage_report/index.html">Test coverage report</a>
         (in regards to
         <a href="devguide/_build/html/coverage.html">increasing test coverage</a>)
-    <li><a href="peps/pep-0000.html"><acronym XXX>PEPs</acronym></a>
+    <li><a href="peps/pep-0000.html">
+        <abbr title="Python Enhancement Proposals">PEPs</abbr></a>
         (only read if necessary)
 </ul>
 
@@ -40,4 +41,4 @@
         (find the CPython binary &amp; build if possible)
     <li><a href="full_coverage.py">full_coverage.py</a>
         (build, run, and/or report for coverage.py; see help message for usage)
-</ul>
\ No newline at end of file
+</ul>

-- 
Repository URL: http://hg.python.org/devinabox

From python-checkins at python.org  Fri Jul 12 17:04:33 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri, 12 Jul 2013 17:04:33 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318415=3A_Normaliz?=
 =?utf-8?q?e_what_type_of_quotes_are_used_with_string?=
Message-ID: <3bsHV90b5bz7Lkf@mail.python.org>

http://hg.python.org/cpython/rev/33d379c2bb79
changeset:   84582:33d379c2bb79
user:        Brett Cannon <brett at python.org>
date:        Fri Jul 12 11:04:23 2013 -0400
summary:
  Issue #18415: Normalize what type of quotes are used with string
constants in importlib._bootstrap. Along the way clean up from string
interpolation to use the repr explicitly.

Initial patch by Madison May.

files:
  Lib/importlib/_bootstrap.py |  60 ++++++++++++------------
  Python/importlib.h          |  12 ++--
  2 files changed, 36 insertions(+), 36 deletions(-)


diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -175,7 +175,7 @@
                         self.count += 1
                         return True
                     if self.has_deadlock():
-                        raise _DeadlockError("deadlock detected by %r" % self)
+                        raise _DeadlockError('deadlock detected by %r' % self)
                     if self.wakeup.acquire(False):
                         self.waiters += 1
                 # Wait for a release() call
@@ -188,7 +188,7 @@
         tid = _thread.get_ident()
         with self.lock:
             if self.owner != tid:
-                raise RuntimeError("cannot release un-acquired lock")
+                raise RuntimeError('cannot release un-acquired lock')
             assert self.count > 0
             self.count -= 1
             if self.count == 0:
@@ -198,7 +198,7 @@
                     self.wakeup.release()
 
     def __repr__(self):
-        return "_ModuleLock({!r}) at {}".format(self.name, id(self))
+        return '_ModuleLock({!r}) at {}'.format(self.name, id(self))
 
 
 class _DummyModuleLock:
@@ -215,11 +215,11 @@
 
     def release(self):
         if self.count == 0:
-            raise RuntimeError("cannot release un-acquired lock")
+            raise RuntimeError('cannot release un-acquired lock')
         self.count -= 1
 
     def __repr__(self):
-        return "_DummyModuleLock({!r}) at {}".format(self.name, id(self))
+        return '_DummyModuleLock({!r}) at {}'.format(self.name, id(self))
 
 
 # The following two functions are for consumption by Python/import.c.
@@ -603,7 +603,7 @@
         if name is None:
             name = self.name
         elif self.name != name:
-            raise ImportError("loader cannot handle %s" % name, name=name)
+            raise ImportError('loader cannot handle %s' % name, name=name)
         return method(self, name, *args, **kwargs)
     _wrap(_check_name_wrapper, method)
     return _check_name_wrapper
@@ -613,7 +613,7 @@
     """Decorator to verify the named module is built-in."""
     def _requires_builtin_wrapper(self, fullname):
         if fullname not in sys.builtin_module_names:
-            raise ImportError("{} is not a built-in module".format(fullname),
+            raise ImportError('{} is not a built-in module'.format(fullname),
                               name=fullname)
         return fxn(self, fullname)
     _wrap(_requires_builtin_wrapper, fxn)
@@ -624,7 +624,7 @@
     """Decorator to verify the named module is frozen."""
     def _requires_frozen_wrapper(self, fullname):
         if not _imp.is_frozen(fullname):
-            raise ImportError("{} is not a frozen module".format(fullname),
+            raise ImportError('{} is not a frozen module'.format(fullname),
                               name=fullname)
         return fxn(self, fullname)
     _wrap(_requires_frozen_wrapper, fxn)
@@ -639,7 +639,7 @@
     # return None.
     loader, portions = self.find_loader(fullname)
     if loader is None and len(portions):
-        msg = "Not importing directory {}: missing __init__"
+        msg = 'Not importing directory {}: missing __init__'
         _warnings.warn(msg.format(portions[0]), ImportWarning)
     return loader
 
@@ -694,7 +694,7 @@
             pass
         else:
             if _r_long(raw_size) != source_size:
-                raise ImportError("bytecode is stale for {!r}".format(name),
+                raise ImportError('bytecode is stale for {!r}'.format(name),
                                   **exc_details)
     return data[12:]
 
@@ -708,7 +708,7 @@
             _imp._fix_co_filename(code, source_path)
         return code
     else:
-        raise ImportError("Non-code object in {!r}".format(bytecode_path),
+        raise ImportError('Non-code object in {!r}'.format(bytecode_path),
                           name=name, path=bytecode_path)
 
 def _code_to_bytecode(code, mtime=0, source_size=0):
@@ -746,7 +746,7 @@
 
     @classmethod
     def module_repr(cls, module):
-        return "<module '{}' (built-in)>".format(module.__name__)
+        return '<module {!r} (built-in)>'.format(module.__name__)
 
     @classmethod
     def find_module(cls, fullname, path=None):
@@ -798,7 +798,7 @@
 
     @classmethod
     def module_repr(cls, m):
-        return "<module '{}' (frozen)>".format(m.__name__)
+        return '<module {!r} (frozen)>'.format(m.__name__)
 
     @classmethod
     def find_module(cls, fullname, path=None):
@@ -842,11 +842,11 @@
     """
 
     REGISTRY_KEY = (
-        "Software\\Python\\PythonCore\\{sys_version}"
-        "\\Modules\\{fullname}")
+        'Software\\Python\\PythonCore\\{sys_version}'
+        '\\Modules\\{fullname}')
     REGISTRY_KEY_DEBUG = (
-        "Software\\Python\\PythonCore\\{sys_version}"
-        "\\Modules\\{fullname}\\Debug")
+        'Software\\Python\\PythonCore\\{sys_version}'
+        '\\Modules\\{fullname}\\Debug')
     DEBUG_BUILD = False  # Changed in _setup()
 
     @classmethod
@@ -866,7 +866,7 @@
                                   sys_version=sys.version[:3])
         try:
             with cls._open_registry(key) as hkey:
-                filepath = _winreg.QueryValue(hkey, "")
+                filepath = _winreg.QueryValue(hkey, '')
         except OSError:
             return None
         return filepath
@@ -973,7 +973,7 @@
         try:
             source_bytes = self.get_data(path)
         except OSError as exc:
-            raise ImportError("source not available through get_data()",
+            raise ImportError('source not available through get_data()',
                               name=fullname) from exc
         return decode_source(source_bytes)
 
@@ -1218,7 +1218,7 @@
         return len(self._recalculate())
 
     def __repr__(self):
-        return "_NamespacePath({!r})".format(self._path)
+        return '_NamespacePath({!r})'.format(self._path)
 
     def __contains__(self, item):
         return item in self._recalculate()
@@ -1233,7 +1233,7 @@
 
     @classmethod
     def module_repr(cls, module):
-        return "<module '{}' (namespace)>".format(module.__name__)
+        return '<module {!r} (namespace)>'.format(module.__name__)
 
     def is_package(self, fullname):
         return True
@@ -1467,13 +1467,13 @@
         def path_hook_for_FileFinder(path):
             """Path hook for importlib.machinery.FileFinder."""
             if not _path_isdir(path):
-                raise ImportError("only directories are supported", path=path)
+                raise ImportError('only directories are supported', path=path)
             return cls(path, *loader_details)
 
         return path_hook_for_FileFinder
 
     def __repr__(self):
-        return "FileFinder({!r})".format(self.path)
+        return 'FileFinder({!r})'.format(self.path)
 
 
 # Import itself ###############################################################
@@ -1520,18 +1520,18 @@
 def _sanity_check(name, package, level):
     """Verify arguments are "sane"."""
     if not isinstance(name, str):
-        raise TypeError("module name must be str, not {}".format(type(name)))
+        raise TypeError('module name must be str, not {}'.format(type(name)))
     if level < 0:
         raise ValueError('level must be >= 0')
     if package:
         if not isinstance(package, str):
-            raise TypeError("__package__ not set to a string")
+            raise TypeError('__package__ not set to a string')
         elif package not in sys.modules:
-            msg = ("Parent module {!r} not loaded, cannot perform relative "
-                   "import")
+            msg = ('Parent module {!r} not loaded, cannot perform relative '
+                   'import')
             raise SystemError(msg.format(package))
     if not name and level == 0:
-        raise ValueError("Empty module name")
+        raise ValueError('Empty module name')
 
 
 _ERR_MSG_PREFIX = 'No module named '
@@ -1614,8 +1614,8 @@
     module = sys.modules[name]
     if module is None:
         _imp.release_lock()
-        message = ("import of {} halted; "
-                    "None in sys.modules".format(name))
+        message = ('import of {} halted; '
+                   'None in sys.modules'.format(name))
         raise ImportError(message, name=name)
     _lock_unlock_module(name)
     return module
diff --git a/Python/importlib.h b/Python/importlib.h
--- a/Python/importlib.h
+++ b/Python/importlib.h
[stripped]

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 17:08:25 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri, 12 Jul 2013 17:08:25 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?devinabox=3A_Issue_=2318388=3A_Link_t?=
 =?utf-8?q?o_the_contributor_agreement=2E?=
Message-ID: <3bsHZd2h8Lz7Lk4@mail.python.org>

http://hg.python.org/devinabox/rev/28294583a9c9
changeset:   52:28294583a9c9
user:        Brett Cannon <brett at python.org>
date:        Fri Jul 12 11:08:19 2013 -0400
summary:
  Issue #18388: Link to the contributor agreement.

files:
  index.html |  5 +----
  1 files changed, 1 insertions(+), 4 deletions(-)


diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -16,16 +16,13 @@
     a {
         font-size: x-large;
     }
-
-    .note {
-        font-size: small;
-    }
 </style>
 
 <h1>Documentation</h1>
 <ul>
     <li><a href="devguide/_build/html/index.html">Devguide</a>
         (read first)
+    <li><a href="http://python.org/psf/contrib/contrib-form/">Python Contributor Agreement Form</a> (cannot commit your contributions until you sign this)
     <li><a href="cpython/Doc/build/html/index.html">Python documentation</a>
     <li><a href="coverage_report/index.html">Test coverage report</a>
         (in regards to

-- 
Repository URL: http://hg.python.org/devinabox

From python-checkins at python.org  Fri Jul 12 17:22:37 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri, 12 Jul 2013 17:22:37 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318342=3A_Use_the_?=
 =?utf-8?q?repr_of_a_module_name_for_=60=60from_=2E=2E=2E_import?=
Message-ID: <3bsHv165cCz7Ll7@mail.python.org>

http://hg.python.org/cpython/rev/c3f9292c8efe
changeset:   84583:c3f9292c8efe
user:        Brett Cannon <brett at python.org>
date:        Fri Jul 12 11:22:26 2013 -0400
summary:
  Issue #18342: Use the repr of a module name for ``from ... import
...`` when an ImportError occurs.

Other cases had already been switched over to using the repr.

Thanks to Tomasz Ma?kowiak for the patch.

files:
  Lib/test/test_import.py |  8 ++++++++
  Misc/ACKS               |  1 +
  Misc/NEWS               |  3 +++
  Python/ceval.c          |  2 +-
  4 files changed, 13 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
--- a/Lib/test/test_import.py
+++ b/Lib/test/test_import.py
@@ -321,6 +321,14 @@
         stdout, stderr = popen.communicate()
         self.assertIn(b"ImportError", stdout)
 
+    def test_from_import_message_for_nonexistent_module(self):
+        with self.assertRaisesRegexp(ImportError, "^No module named 'bogus'"):
+            from bogus import foo
+
+    def test_from_import_message_for_existing_module(self):
+        with self.assertRaisesRegexp(ImportError, "^cannot import name 'bogus'"):
+            from re import bogus
+
 
 @skip_if_dont_write_bytecode
 class FilePermissionTests(unittest.TestCase):
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -786,6 +786,7 @@
 Tim MacKenzie
 Nick Maclaren
 Don MacMillen
+Tomasz Ma?kowiak
 Steve Majewski
 Grzegorz Makarewicz
 David Malcolm
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #18342: Use the repr of a module name when an import fails when using
+  ``from ... import ...``.
+
 - Issue #17872: Fix a segfault in marshal.load() when input stream returns
   more bytes than requested.
 
diff --git a/Python/ceval.c b/Python/ceval.c
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -4602,7 +4602,7 @@
 
     x = PyObject_GetAttr(v, name);
     if (x == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) {
-        PyErr_Format(PyExc_ImportError, "cannot import name %S", name);
+        PyErr_Format(PyExc_ImportError, "cannot import name %R", name);
     }
     return x;
 }

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 17:30:41 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri, 12 Jul 2013 17:30:41 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2317845=3A_Clarify_?=
 =?utf-8?q?the_message_setup=2Epy_prints_upon_successfully?=
Message-ID: <3bsJ4K5YrHz7LkX@mail.python.org>

http://hg.python.org/cpython/rev/e13ff9fdfaf9
changeset:   84584:e13ff9fdfaf9
user:        Brett Cannon <brett at python.org>
date:        Fri Jul 12 11:30:32 2013 -0400
summary:
  Issue #17845: Clarify the message setup.py prints upon successfully
building Python but having some optional module which didn't build.

Patch by Yogesh Chaudhari.

files:
  Misc/NEWS |  2 ++
  setup.py  |  5 +++--
  2 files changed, 5 insertions(+), 2 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -631,6 +631,8 @@
 Build
 -----
 
+- Issue #17845: Clarified the message printed when some module are not built.
+
 - Issue #18256: Compilation fix for recent AIX releases.  Patch by
   David Edelsohn.
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -259,8 +259,9 @@
 
         if missing:
             print()
-            print("Python build finished, but the necessary bits to build "
-                   "these modules were not found:")
+            print("Python build finished successfully!")
+            print("The necessary bits to build these optional modules were not "
+                  "found:")
             print_three_column(missing)
             print("To find the necessary bits, look in setup.py in"
                   " detect_modules() for the module's name.")

-- 
Repository URL: http://hg.python.org/cpython

From python-checkins at python.org  Fri Jul 12 17:53:41 2013
From: python-checkins at python.org (brett.cannon)
Date: Fri, 12 Jul 2013 17:53:41 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?q?devguide=3A_Issue_=2318390=3A_Add_a_l?=
 =?utf-8?q?ittle_comment_on_how_to_do_a_proper_file?=
Message-ID: <3bsJZs1LLDz7Lll@mail.python.org>

http://hg.python.org/devguide/rev/15249b189347
changeset:   624:15249b189347
user:        Brett Cannon <brett at python.org>
date:        Fri Jul 12 11:53:34 2013 -0400
summary:
  Issue #18390: Add a little comment on how to do a proper file
revert after a merge.

files:
  committing.rst |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/committing.rst b/committing.rst
--- a/committing.rst
+++ b/committing.rst
@@ -385,7 +385,7 @@
 
    cd ../3.4
    hg merge 3.3
-   # Fix any conflicts; compile; run the test suite
+   # Fix any conflicts (e.g. ``hg revert -r default Misc/NEWS``); compile; run the test suite
    hg ci -m '#12345: merge with 3.3.'
 
 If you are not using the share extension, you will need to use

-- 
Repository URL: http://hg.python.org/devguide

From python-checkins at python.org  Fri Jul 12 21:58:29 2013
From: python-checkins at python.org (vinay.sajip)
Date: Fri, 12 Jul 2013 21:58:29 +0200 (CEST)
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy4zKTogSXNzdWUgIzE4NDM0?=
 =?utf-8?q?=3A_Updated_example_script_in_venv_docs_to_use_setuptools_rathe?=
 =?utf-8?q?r_than?=
Message-ID: <3bsQ1K1gSqzPX7@mail.python.org>

http://hg.python.org/cpython/rev/1641a03dbe7b
changeset:   84585:1641a03dbe7b
branch:      3.3
parent:      84558:4acb822f4c43
user:        Vinay Sajip <vinay_sajip at yahoo.co.uk>
date:        Fri Jul 12 20:54:25 2013 +0100
summary:
  Issue #18434: Updated example script in venv docs to use setuptools rather than Distribute.

files:
  Doc/library/venv.rst |  49 ++++++++++++++++++-------------
  1 files changed, 28 insertions(+), 21 deletions(-)


diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst
--- a/Doc/library/venv.rst
+++ b/Doc/library/venv.rst
@@ -187,7 +187,7 @@
 --------------------------------------
 
 The following script shows how to extend :class:`EnvBuilder` by implementing a
-subclass which installs Distribute and pip into a created venv::
+subclass which installs setuptools and pip into a created venv::
 
     import os
     import os.path
@@ -198,16 +198,16 @@
     from urllib.request import urlretrieve
     import venv
 
-    class DistributeEnvBuilder(venv.EnvBuilder):
+    class ExtendedEnvBuilder(venv.EnvBuilder):
         """
-        This builder installs Distribute and pip so that you can pip or
+        This builder installs setuptools and pip so that you can pip or
         easy_install other packages into the created environment.
 
-        :param nodist: If True, Distribute is not installed into the created
-                       environment.
+        :param nodist: If True, setuptools and pip are not installed into the
+                       created environment.
         :param nopip: If True, pip is not installed into the created
                       environment.
-